From a17aead2165301f9039098fbf8f73e4e129552e5 Mon Sep 17 00:00:00 2001 From: Eloston Date: Tue, 9 Aug 2016 21:44:21 -0700 Subject: [PATCH] Add syzygy downloading code for Windows Fix zip packaging on Windows Split more code into separate functions --- buildlib/generic.py | 65 +++++++++++++++++++++++---------------------- buildlib/windows.py | 43 +++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/buildlib/generic.py b/buildlib/generic.py index f103d6ad..a2dcaaae 100644 --- a/buildlib/generic.py +++ b/buildlib/generic.py @@ -134,26 +134,32 @@ class GenericPlatform: else: self.logger.warning("Hash algorithm '{}' not available. Skipping...".format(hash_line[0])) + def _download_file(self, url, file_path): + with urllib.request.urlopen(url) as response: + with file_path.open("wb") as f: + shutil.copyfileobj(response, f) + def _download_source_archive(self): ''' Downloads the original Chromium source code in .tar.xz format ''' download_url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-{version}.tar.xz".format(version=self.version) - with urllib.request.urlopen(download_url) as response: - with self.sourcearchive.open("wb") as f: - shutil.copyfileobj(response, f) + self._download_file(download_url, self.sourcearchive) def _download_source_hashes(self): hashes_url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-{version}.tar.xz.hashes".format(version=self.version) - with urllib.request.urlopen(hashes_url) as response: - with self.sourcearchive_hashes.open("wb") as f: - shutil.copyfileobj(response, f) + self._download_file(download_url, self.sourcearchive_hashes) - def _extract_source_archive(self, cleaning_list): - ''' - Extract the archive located on archive_path to the sandbox root - Also modifies cleaning_list to contain paths not removed - ''' + def _download_helper(self, file_path, force_download, check_if_exists, downloader): + if file_path.exists() and not file_path.is_file(): + raise Exception("{} is an existing non-file".format(str(file_path))) + elif force_download or check_if_exists and not file_path.is_file(): + self.logger.info("Downloading {} ...".format(str(file_path))) + downloader() + else: + self.logger.info("{} already exists. Skipping download.".format(str(file_path))) + + def _extract_tar_file(self, tar_path, destination_dir, ignore_files, relative_to): class NoAppendList(list): # Hack to workaround memory issues with large tar files def append(self, obj): pass @@ -172,28 +178,35 @@ class GenericPlatform: # Unexpected exception raise e - with tarfile.open(str(self.sourcearchive)) as tar_file_obj: + with tarfile.open(str(tar_path)) as tar_file_obj: tar_file_obj.members = NoAppendList() for tarinfo in tar_file_obj: try: - relative_path = pathlib.PurePosixPath(tarinfo.name).relative_to("chromium-{}".format(self.version)) - if str(relative_path) in cleaning_list: - cleaning_list.remove(str(relative_path)) + relative_path = pathlib.PurePosixPath(tarinfo.name).relative_to(relative_to) + if str(relative_path) in ignore_files: + ignore_files.remove(str(relative_path)) else: - destination = self.sandbox_root / pathlib.Path(*relative_path.parts) + destination = destination_dir / pathlib.Path(*relative_path.parts) if tarinfo.issym() and not symlink_supported: # In this situation, TarFile.makelink() will try to create a copy of the target. But this fails because TarFile.members is empty # But if symlinks are not supported, it's safe to assume that symlinks aren't needed. The only situation where this happens is on Windows. continue if tarinfo.islnk(): # Derived from TarFile.extract() - relative_target = pathlib.PurePosixPath(tarinfo.linkname).relative_to("chromium-{}".format(self.version)) - tarinfo._link_target = str(self.sandbox_root / pathlib.Path(*relative_target.parts)) + relative_target = pathlib.PurePosixPath(tarinfo.linkname).relative_to(relative_to) + tarinfo._link_target = str(destination_dir / pathlib.Path(*relative_target.parts)) tar_file_obj._extract_member(tarinfo, str(destination)) except Exception as e: self.logger.error("Exception thrown for tar member {}".format(tarinfo.name)) raise e + def _extract_source_archive(self, cleaning_list): + ''' + Extract the archive located on archive_path to the sandbox root + Also modifies cleaning_list to contain paths not removed + ''' + self._extract_tar_file(self.sourcearchive, self.sandbox_root, cleaning_list, "chromium-{}".format(self.version)) + def _get_parsed_domain_regexes(self): if self._domain_regex_cache is None: self._domain_regex_cache = list() @@ -346,22 +359,10 @@ class GenericPlatform: self.sourcearchive = destination_dir / pathlib.Path("chromium-{version}.tar.xz".format(version=self.version)) self.sourcearchive_hashes = destination_dir / pathlib.Path("chromium-{version}.tar.xz.hashes".format(version=self.version)) - if self.sourcearchive.exists() and not self.sourcearchive.is_file(): - raise Exception("sourcearchive is an existing non-file") - elif force_download or check_if_exists and not self.sourcearchive.is_file(): - self.logger.info("Downloading source archive...") - self._download_source_archive() - else: - self.logger.info("Source archive already exists. Skipping download.") + self._download_helper(self.sourcearchive, force_download, check_if_exists, self._download_source_archive) if check_integrity: - if self.sourcearchive_hashes.exists() and not self.sourcearchive_hashes.is_file(): - raise Exception("sourcearchive_hashes is an existing non-file") - elif force_download or check_if_exists and not self.sourcearchive_hashes.is_file(): - self.logger.info("Downloading source archive hashes...") - self._download_source_hashes() - else: - self.logger.info("Source hashes file already exists. Skipping download.") + self._download_helper(self.sourcearchive_hashes, force_download, check_if_exists, self._download_source_hashes) else: if check_integrity and hashes_path is None: raise Exception("Hashes path must be set with archive_path") diff --git a/buildlib/windows.py b/buildlib/windows.py index 261cf408..5b87a6be 100644 --- a/buildlib/windows.py +++ b/buildlib/windows.py @@ -26,6 +26,16 @@ from . import generic class WindowsPlatform(generic.GenericPlatform): PLATFORM_RESOURCES = pathlib.Path("resources", "windows") FILES_CFG = generic.GenericPlatform.SANDBOX_ROOT / pathlib.Path("chrome", "tools", "build", "win", "FILES.cfg") + SYZYGY_COMMIT = "3c00ec0d484aeada6a3d04a14a11bd7353640107" + + def __init__(self, *args, **kwargs): + super(WindowsPlatform, self).__init__(*args, **kwargs) + + self.syzygyarchive = None + + def _download_syzygy(self): + download_url = "https://github.com/Eloston/syzygy/archive/{}.tar.gz".format(self.SYZYGY_COMMIT) + self._download_file(download_url, self.syzygyarchive) def _run_subprocess(self, *args, **kwargs): # On Windows for some reason, subprocess.run(['python']) will use the current interpreter's executable even though it is not in the PATH or cwd @@ -34,6 +44,22 @@ class WindowsPlatform(generic.GenericPlatform): kwargs["shell"] = True return super(WindowsPlatform, self)._run_subprocess(*args, **kwargs) + def setup_chromium_source(self, *args, check_if_exists=True, force_download=False, extract_archive=True, destination_dir=pathlib.Path("."), syzygyarchive_path=None, **kwargs): + super(WindowsPlatform, self).setup_chromium_source(*args, check_if_exists=check_if_exists, force_download=force_download, extract_archive=extract_archive, destination_dir=destination_dir, **kwargs) + + if syzygyarchive_path is None: + self.syzygyarchive = destination_dir / pathlib.Path("syzygy-{}.tar.gz".format(self.SYZYGY_COMMIT)) + + self._download_helper(self.syzygyarchive, force_download, check_if_exists, self._download_syzygy) + else: + self.syzygyarchive = syzygyarchive_path + + if extract_archive: + self.logger.info("Extracting syzygy archive...") + syzygy_dir = self.sandbox_root / pathlib.Path("third_party", "syzygy") + os.makedirs(str(syzygy_dir.resolve())) + self._extract_tar_file(self.syzygyarchive, syzygy_dir, list(), "syzygy-{}".format(self.SYZYGY_COMMIT)) + def apply_patches(self, patch_command=["patch", "-p1"]): self.logger.info("Applying patches via '{}' ...".format(" ".join(patch_command))) self._generate_patches(self.sandbox_patches, self._ran_domain_substitution) @@ -66,24 +92,15 @@ class WindowsPlatform(generic.GenericPlatform): self.logger.info("Creating build output archive {} ...".format(output_filename)) def file_list_generator(): exec_globals = {"__builtins__": None} - with FILES_CFG.open() as cfg_file: + with self.FILES_CFG.open() as cfg_file: exec(cfg_file.read(), exec_globals) for file_spec in exec_globals["FILES"]: if "dev" in file_spec["buildtype"] and "official" in file_spec["buildtype"]: if "arch" in file_spec and not "32bit" in file_spec["arch"]: continue - file_path = self.sandbox_root / self.build_output / pathlib.Path(file_spec["filename"]) - if file_path.exists(): - if file_path.is_file(): - yield (file_spec["filename"], file_path) - elif file_path.is_dir(): - for i in file_path.iterdir(): - yield (str(i.relative_to(self.sandbox_root / self.build_output)), i) - else: - self.logger.warning("Unknown file type for {}".format(file_spec["filename"])) - else: - if not "optional" in file_spec: - self.logger.warning("Missing file {}".format(file_spec["filename"])) + for file_path in (self.sandbox_root / self.build_output).glob(file_spec["filename"]): + if not file_path.suffix.lower() == "pdb": + yield (str(file_path.relative_to(self.sandbox_root / self.build_output)), file_path) with zipfile.ZipFile(output_filename, mode="w", compression=zipfile.ZIP_DEFLATED) as zip_file: for arcname, real_path in file_list_generator(): zip_file.write(str(real_path), arcname)