morgan/helium
1
0
Fork 0
mirror of https://github.com/morgan9e/helium synced 2026-04-14 00:14:20 +09:00
Code Issues Packages Projects Releases Wiki Activity

Add syzygy downloading code for Windows

Browse Source 
Fix zip packaging on Windows
Split more code into separate functions
This commit is contained in:
Eloston
2016-08-09 21:44:21 -07:00
parent 724293b138
commit a17aead216
2 changed files with 63 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
buildlib/generic.py
View File

@@ -134,26 +134,32 @@ class GenericPlatform:
else:
self.logger.warning("Hash algorithm '{}' not available. Skipping...".format(hash_line[0]))
def _download_file(self, url, file_path):
with urllib.request.urlopen(url) as response:
with file_path.open("wb") as f:
shutil.copyfileobj(response, f)
def _download_source_archive(self):
'''
Downloads the original Chromium source code in .tar.xz format
'''
download_url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-{version}.tar.xz".format(version=self.version)
with urllib.request.urlopen(download_url) as response:
with self.sourcearchive.open("wb") as f:
shutil.copyfileobj(response, f)
self._download_file(download_url, self.sourcearchive)
def _download_source_hashes(self):
hashes_url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-{version}.tar.xz.hashes".format(version=self.version)
with urllib.request.urlopen(hashes_url) as response:
with self.sourcearchive_hashes.open("wb") as f:
shutil.copyfileobj(response, f)
self._download_file(download_url, self.sourcearchive_hashes)
def _extract_source_archive(self, cleaning_list):
'''
Extract the archive located on archive_path to the sandbox root
Also modifies cleaning_list to contain paths not removed
'''
def _download_helper(self, file_path, force_download, check_if_exists, downloader):
if file_path.exists() and not file_path.is_file():
raise Exception("{} is an existing non-file".format(str(file_path)))
elif force_download or check_if_exists and not file_path.is_file():
self.logger.info("Downloading {} ...".format(str(file_path)))
downloader()
else:
self.logger.info("{} already exists. Skipping download.".format(str(file_path)))
def _extract_tar_file(self, tar_path, destination_dir, ignore_files, relative_to):
class NoAppendList(list): # Hack to workaround memory issues with large tar files
def append(self, obj):
pass
@@ -172,28 +178,35 @@ class GenericPlatform:
# Unexpected exception
raise e
with tarfile.open(str(self.sourcearchive)) as tar_file_obj:
with tarfile.open(str(tar_path)) as tar_file_obj:
tar_file_obj.members = NoAppendList()
for tarinfo in tar_file_obj:
try:
relative_path = pathlib.PurePosixPath(tarinfo.name).relative_to("chromium-{}".format(self.version))
if str(relative_path) in cleaning_list:
cleaning_list.remove(str(relative_path))
relative_path = pathlib.PurePosixPath(tarinfo.name).relative_to(relative_to)
if str(relative_path) in ignore_files:
ignore_files.remove(str(relative_path))
else:
destination = self.sandbox_root / pathlib.Path(*relative_path.parts)
destination = destination_dir / pathlib.Path(*relative_path.parts)
if tarinfo.issym() and not symlink_supported:
# In this situation, TarFile.makelink() will try to create a copy of the target. But this fails because TarFile.members is empty
# But if symlinks are not supported, it's safe to assume that symlinks aren't needed. The only situation where this happens is on Windows.
continue
if tarinfo.islnk():
# Derived from TarFile.extract()
relative_target = pathlib.PurePosixPath(tarinfo.linkname).relative_to("chromium-{}".format(self.version))
tarinfo._link_target = str(self.sandbox_root / pathlib.Path(*relative_target.parts))
relative_target = pathlib.PurePosixPath(tarinfo.linkname).relative_to(relative_to)
tarinfo._link_target = str(destination_dir / pathlib.Path(*relative_target.parts))
tar_file_obj._extract_member(tarinfo, str(destination))
except Exception as e:
self.logger.error("Exception thrown for tar member {}".format(tarinfo.name))
raise e
def _extract_source_archive(self, cleaning_list):
'''
Extract the archive located on archive_path to the sandbox root
Also modifies cleaning_list to contain paths not removed
'''
self._extract_tar_file(self.sourcearchive, self.sandbox_root, cleaning_list, "chromium-{}".format(self.version))
def _get_parsed_domain_regexes(self):
if self._domain_regex_cache is None:
self._domain_regex_cache = list()
@@ -346,22 +359,10 @@ class GenericPlatform:
self.sourcearchive = destination_dir / pathlib.Path("chromium-{version}.tar.xz".format(version=self.version))
self.sourcearchive_hashes = destination_dir / pathlib.Path("chromium-{version}.tar.xz.hashes".format(version=self.version))
if self.sourcearchive.exists() and not self.sourcearchive.is_file():
raise Exception("sourcearchive is an existing non-file")
elif force_download or check_if_exists and not self.sourcearchive.is_file():
self.logger.info("Downloading source archive...")
self._download_source_archive()
else:
self.logger.info("Source archive already exists. Skipping download.")
self._download_helper(self.sourcearchive, force_download, check_if_exists, self._download_source_archive)
if check_integrity:
if self.sourcearchive_hashes.exists() and not self.sourcearchive_hashes.is_file():
raise Exception("sourcearchive_hashes is an existing non-file")
elif force_download or check_if_exists and not self.sourcearchive_hashes.is_file():
self.logger.info("Downloading source archive hashes...")
self._download_source_hashes()
else:
self.logger.info("Source hashes file already exists. Skipping download.")
self._download_helper(self.sourcearchive_hashes, force_download, check_if_exists, self._download_source_hashes)
else:
if check_integrity and hashes_path is None:
raise Exception("Hashes path must be set with archive_path")

43
buildlib/windows.py
View File

@@ -26,6 +26,16 @@ from . import generic
class WindowsPlatform(generic.GenericPlatform):
PLATFORM_RESOURCES = pathlib.Path("resources", "windows")
FILES_CFG = generic.GenericPlatform.SANDBOX_ROOT / pathlib.Path("chrome", "tools", "build", "win", "FILES.cfg")
SYZYGY_COMMIT = "3c00ec0d484aeada6a3d04a14a11bd7353640107"
def __init__(self, *args, **kwargs):
super(WindowsPlatform, self).__init__(*args, **kwargs)
self.syzygyarchive = None
def _download_syzygy(self):
download_url = "https://github.com/Eloston/syzygy/archive/{}.tar.gz".format(self.SYZYGY_COMMIT)
self._download_file(download_url, self.syzygyarchive)
def _run_subprocess(self, *args, **kwargs):
# On Windows for some reason, subprocess.run(['python']) will use the current interpreter's executable even though it is not in the PATH or cwd
@@ -34,6 +44,22 @@ class WindowsPlatform(generic.GenericPlatform):
kwargs["shell"] = True
return super(WindowsPlatform, self)._run_subprocess(*args, **kwargs)
def setup_chromium_source(self, *args, check_if_exists=True, force_download=False, extract_archive=True, destination_dir=pathlib.Path("."), syzygyarchive_path=None, **kwargs):
super(WindowsPlatform, self).setup_chromium_source(*args, check_if_exists=check_if_exists, force_download=force_download, extract_archive=extract_archive, destination_dir=destination_dir, **kwargs)
if syzygyarchive_path is None:
self.syzygyarchive = destination_dir / pathlib.Path("syzygy-{}.tar.gz".format(self.SYZYGY_COMMIT))
self._download_helper(self.syzygyarchive, force_download, check_if_exists, self._download_syzygy)
else:
self.syzygyarchive = syzygyarchive_path
if extract_archive:
self.logger.info("Extracting syzygy archive...")
syzygy_dir = self.sandbox_root / pathlib.Path("third_party", "syzygy")
os.makedirs(str(syzygy_dir.resolve()))
self._extract_tar_file(self.syzygyarchive, syzygy_dir, list(), "syzygy-{}".format(self.SYZYGY_COMMIT))
def apply_patches(self, patch_command=["patch", "-p1"]):
self.logger.info("Applying patches via '{}' ...".format(" ".join(patch_command)))
self._generate_patches(self.sandbox_patches, self._ran_domain_substitution)
@@ -66,24 +92,15 @@ class WindowsPlatform(generic.GenericPlatform):
self.logger.info("Creating build output archive {} ...".format(output_filename))
def file_list_generator():
exec_globals = {"__builtins__": None}
with FILES_CFG.open() as cfg_file:
with self.FILES_CFG.open() as cfg_file:
exec(cfg_file.read(), exec_globals)
for file_spec in exec_globals["FILES"]:
if "dev" in file_spec["buildtype"] and "official" in file_spec["buildtype"]:
if "arch" in file_spec and not "32bit" in file_spec["arch"]:
continue
file_path = self.sandbox_root / self.build_output / pathlib.Path(file_spec["filename"])
if file_path.exists():
if file_path.is_file():
yield (file_spec["filename"], file_path)
elif file_path.is_dir():
for i in file_path.iterdir():
yield (str(i.relative_to(self.sandbox_root / self.build_output)), i)
else:
self.logger.warning("Unknown file type for {}".format(file_spec["filename"]))
else:
if not "optional" in file_spec:
self.logger.warning("Missing file {}".format(file_spec["filename"]))
for file_path in (self.sandbox_root / self.build_output).glob(file_spec["filename"]):
if not file_path.suffix.lower() == "pdb":
yield (str(file_path.relative_to(self.sandbox_root / self.build_output)), file_path)
with zipfile.ZipFile(output_filename, mode="w", compression=zipfile.ZIP_DEFLATED) as zip_file:
for arcname, real_path in file_list_generator():
zip_file.write(str(real_path), arcname)