morgan/helium
1
0
Fork 0
mirror of https://github.com/morgan9e/helium synced 2026-04-14 00:14:20 +09:00
Code Issues Packages Projects Releases Wiki Activity

buildkit: Refactoring of CLI and related modules

Browse Source 
* Update CLI (Closes #408)
* Update design docs
* Remove binary pruning from extraction code
* Merge patch applying code into buildkit.patches
This commit is contained in:
Eloston
2018-07-16 06:36:20 +00:00
parent cd52583ecf
commit 2444dd4e27
15 changed files with 379 additions and 697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -2,9 +2,6 @@
__pycache__/
*.py[cod]
# Ignore buildspace directory
/buildspace
# Ignore macOS Finder meta
.DS_Store
.tm_properties

521
buildkit/cli.py
View File

@@ -10,54 +10,24 @@ buildkit: A small helper utility for building ungoogled-chromium.
This is the CLI interface. Available commands each have their own help; pass in
-h or --help after a command.
buildkit has optional environment variables. They are as follows:
* BUILDKIT_RESOURCES - Path to the resources/ directory. Defaults to
the one in buildkit's parent directory.
* BUILDKIT_USER_BUNDLE - Path to the user config bundle. Without it, commands
that need a bundle default to buildspace/user_bundle. This value can be
overridden per-command with the --user-bundle option.
"""
import argparse
import os
from pathlib import Path
from . import config
from . import source_retrieval
from . import downloads
from . import domain_substitution
from .common import (
CONFIG_BUNDLES_DIR, BUILDSPACE_DOWNLOADS, BUILDSPACE_TREE,
BUILDSPACE_TREE_PACKAGING, BUILDSPACE_USER_BUNDLE, SEVENZIP_USE_REGISTRY,
BuildkitAbort, ExtractorEnum, get_resources_dir, get_logger)
from . import patches
from .common import SEVENZIP_USE_REGISTRY, BuildkitAbort, ExtractorEnum, get_logger
from .config import ConfigBundle
from .extraction import prune_dir
# Classes
class _CLIError(RuntimeError):
"""Custom exception for printing argument parser errors from callbacks"""
def get_basebundle_verbosely(base_name):
"""
Returns a ConfigBundle from the given base name, otherwise it logs errors and raises
BuildkitAbort"""
try:
return ConfigBundle.from_base_name(base_name)
except NotADirectoryError as exc:
get_logger().error('resources/ or resources/patches directories could not be found.')
raise BuildkitAbort()
except FileNotFoundError:
get_logger().error('The base config bundle "%s" does not exist.', base_name)
raise BuildkitAbort()
except ValueError as exc:
get_logger().error('Base bundle metadata has an issue: %s', exc)
raise BuildkitAbort()
except BaseException:
get_logger().exception('Unexpected exception caught.')
raise BuildkitAbort()
class NewBaseBundleAction(argparse.Action): #pylint: disable=too-few-public-methods
class NewBundleAction(argparse.Action): #pylint: disable=too-few-public-methods
"""argparse.ArgumentParser action handler with more verbose logging"""
def __init__(self, *args, **kwargs):
@@ -70,189 +40,102 @@ class NewBaseBundleAction(argparse.Action): #pylint: disable=too-few-public-meth
def __call__(self, parser, namespace, values, option_string=None):
try:
base_bundle = get_basebundle_verbosely(values)
except BuildkitAbort:
bundle = ConfigBundle(values)
except BaseException:
get_logger().exception('Error loading config bundle')
parser.exit(status=1)
setattr(namespace, self.dest, base_bundle)
setattr(namespace, self.dest, bundle)
# Methods
def _default_user_bundle_path():
"""Returns the default path to the buildspace user bundle."""
return os.getenv('BUILDKIT_USER_BUNDLE', default=BUILDSPACE_USER_BUNDLE)
def setup_bundle_arg(parser):
"""Helper to add an argparse.ArgumentParser argument for a config bundle"""
parser.add_argument(
'-b', '--bundle', metavar='PATH', dest='bundle', required=True, action=NewBundleAction,
help='Path to the bundle. Dependencies must reside next to the bundle.')
def setup_bundle_group(parser):
"""Helper to add arguments for loading a config bundle to argparse.ArgumentParser"""
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument(
'-b', '--base-bundle', metavar='NAME', dest='bundle', default=argparse.SUPPRESS,
action=NewBaseBundleAction,
help=('The base config bundle name to use (located in resources/config_bundles). '
'Mutually exclusive with --user-bundle. '
'Default value is nothing; a user bundle is used by default'))
config_group.add_argument(
'-u', '--user-bundle', metavar='PATH', dest='bundle',
default=_default_user_bundle_path(),
type=lambda x: ConfigBundle(Path(x)),
help=('The path to a user bundle to use. '
'Mutually exclusive with --base-bundle. Use BUILDKIT_USER_BUNDLE '
'to override the default value. Current default: %(default)s'))
def _add_bunnfo(subparsers):
"""Gets info about base bundles."""
def _callback(args):
if vars(args).get('list'):
for bundle_dir in sorted(
(get_resources_dir() / CONFIG_BUNDLES_DIR).iterdir()):
bundle_meta = config.BaseBundleMetaIni(
bundle_dir / config.BASEBUNDLEMETA_INI)
print(bundle_dir.name, '-', bundle_meta.display_name)
elif vars(args).get('bundle'):
for dependency in args.bundle.get_dependencies():
print(dependency)
else:
raise NotImplementedError()
parser = subparsers.add_parser(
'bunnfo', formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help=_add_bunnfo.__doc__, description=_add_bunnfo.__doc__)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'-l', '--list', action='store_true',
help='Lists all base bundles and their display names.')
group.add_argument(
'-d', '--dependencies', dest='bundle',
action=NewBaseBundleAction,
help=('Prints the dependency order of the given base bundle, '
'delimited by newline characters. '
'See DESIGN.md for the definition of dependency order.'))
parser.set_defaults(callback=_callback)
def _add_genbun(subparsers):
"""Generates a user config bundle from a base config bundle."""
def _callback(args):
def _add_downloads(subparsers):
"""Retrieve, check, and unpack downloads"""
def _add_common_args(parser):
setup_bundle_arg(parser)
parser.add_argument(
'-c', '--cache', type=Path, required=True,
help='Path to the directory to cache downloads.')
def _retrieve_callback(args):
downloads.retrieve_downloads(
args.bundle, args.cache, args.show_progress, args.disable_ssl_verification)
try:
args.base_bundle.write(args.user_bundle_path)
except FileExistsError:
get_logger().error('User bundle dir is not empty: %s', args.user_bundle_path)
raise _CLIError()
except ValueError as exc:
get_logger().error('Error with base bundle: %s', exc)
downloads.check_downloads(args.bundle, args.cache)
except downloads.HashMismatchError as exc:
get_logger().error('File checksum does not match: %s', exc)
raise _CLIError()
def _unpack_callback(args):
extractors = {
ExtractorEnum.SEVENZIP: args.sevenz_path,
ExtractorEnum.TAR: args.tar_path,
}
downloads.unpack_downloads(args.bundle, args.cache, args.output, extractors)
# downloads
parser = subparsers.add_parser(
'genbun', formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help=_add_genbun.__doc__, description=_add_genbun.__doc__)
parser.add_argument(
'-u', '--user-bundle', metavar='PATH', dest='user_bundle_path',
type=Path, default=_default_user_bundle_path(),
help=('The output path for the user config bundle. '
'The path must not already exist. '))
parser.add_argument(
'base_bundle', action=NewBaseBundleAction,
help='The base config bundle name to use.')
parser.set_defaults(callback=_callback)
'downloads', help=_add_downloads.__doc__ + '.', description=_add_downloads.__doc__)
subsubparsers = parser.add_subparsers(title='Download actions', dest='action')
subsubparsers.required = True # Workaround for http://bugs.python.org/issue9253#msg186387
def _add_getsrc(subparsers):
"""Downloads, checks, and unpacks the necessary files into the buildspace tree"""
def _callback(args):
try:
extractors = {
ExtractorEnum.SEVENZIP: args.sevenz_path,
ExtractorEnum.TAR: args.tar_path,
}
source_retrieval.retrieve_and_extract(
config_bundle=args.bundle, buildspace_downloads=args.downloads,
buildspace_tree=args.tree, prune_binaries=args.prune_binaries,
show_progress=args.show_progress, extractors=extractors,
disable_ssl_verification=args.disable_ssl_verification)
except FileExistsError as exc:
get_logger().error('Directory is not empty: %s', exc)
raise _CLIError()
except FileNotFoundError as exc:
get_logger().error('Directory or file not found: %s', exc)
raise _CLIError()
except NotADirectoryError as exc:
get_logger().error('Path is not a directory: %s', exc)
raise _CLIError()
except source_retrieval.NotAFileError as exc:
get_logger().error('Archive path is not a regular file: %s', exc)
raise _CLIError()
except source_retrieval.HashMismatchError as exc:
get_logger().error('Archive checksum is invalid: %s', exc)
raise _CLIError()
parser = subparsers.add_parser(
'getsrc', help=_add_getsrc.__doc__ + '.',
description=_add_getsrc.__doc__ + '; ' + (
'these are the Chromium source code and any extra dependencies. '
'By default, binary pruning is performed during extraction. '
'The %s directory must already exist for storing downloads. '
'If the buildspace tree already exists or there is a checksum mismatch, '
'this command will abort. '
'Only files that are missing will be downloaded. '
'If the files are already downloaded, their checksums are '
'confirmed and then they are unpacked.') % BUILDSPACE_DOWNLOADS)
setup_bundle_group(parser)
parser.add_argument(
'-t', '--tree', type=Path, default=BUILDSPACE_TREE,
help='The buildspace tree path. Default: %(default)s')
parser.add_argument(
'-d', '--downloads', type=Path, default=BUILDSPACE_DOWNLOADS,
help=('Path to store archives of Chromium source code and extra deps. '
'Default: %(default)s'))
parser.add_argument(
'--disable-binary-pruning', action='store_false', dest='prune_binaries',
help='Disables binary pruning during extraction.')
parser.add_argument(
# downloads retrieve
retrieve_parser = subsubparsers.add_parser(
'retrieve', help='Retrieve and check download files',
description='Retrieves and checks downloads without unpacking.')
_add_common_args(retrieve_parser)
retrieve_parser.add_argument(
'--hide-progress-bar', action='store_false', dest='show_progress',
help='Hide the download progress.')
parser.add_argument(
retrieve_parser.add_argument(
'--disable-ssl-verification', action='store_true',
help='Disables certification verification for downloads using HTTPS.')
retrieve_parser.set_defaults(callback=_retrieve_callback)
# downloads unpack
unpack_parser = subsubparsers.add_parser(
'unpack', help='Unpack download files',
description='Verifies hashes of and unpacks download files into the specified directory.')
_add_common_args(unpack_parser)
unpack_parser.add_argument(
'--tar-path', default='tar',
help=('(Linux and macOS only) Command or path to the BSD or GNU tar '
'binary for extraction. Default: %(default)s'))
parser.add_argument(
unpack_parser.add_argument(
'--7z-path', dest='sevenz_path', default=SEVENZIP_USE_REGISTRY,
help=('Command or path to 7-Zip\'s "7z" binary. If "_use_registry" is '
'specified, determine the path from the registry. Default: %(default)s'))
parser.add_argument(
'--disable-ssl-verification', action='store_true',
help='Disables certification verification for downloads using HTTPS.')
parser.set_defaults(callback=_callback)
unpack_parser.add_argument(
'output', type=Path, help='The directory to unpack to.')
unpack_parser.set_defaults(callback=_unpack_callback)
def _add_prubin(subparsers):
"""Prunes binaries from the buildspace tree."""
def _add_prune(subparsers):
"""Prunes binaries in the given path."""
def _callback(args):
logger = get_logger()
try:
resolved_tree = args.tree.resolve()
except FileNotFoundError as exc:
logger.error('File or directory does not exist: %s', exc)
if not args.directory.exists():
get_logger().error('Specified directory does not exist: %s', args.directory)
raise _CLIError()
missing_file = False
for tree_node in args.bundle.pruning:
try:
(resolved_tree / tree_node).unlink()
except FileNotFoundError:
missing_file = True
logger.warning('No such file: %s', resolved_tree / tree_node)
if missing_file:
unremovable_files = prune_dir(args.directory, args.bundle.pruning)
if unremovable_files:
get_logger().error('Files could not be pruned: %s', unremovable_files)
raise _CLIError()
parser = subparsers.add_parser(
'prubin', help=_add_prubin.__doc__, description=_add_prubin.__doc__ + (
' This is NOT necessary if the source code was already pruned '
'during the getsrc command.'))
setup_bundle_group(parser)
'prune', help=_add_prune.__doc__, description=_add_prune.__doc__)
setup_bundle_arg(parser)
parser.add_argument(
'-t', '--tree', type=Path, default=BUILDSPACE_TREE,
help='The buildspace tree path to apply binary pruning. Default: %(default)s')
'directory', type=Path, help='The directory to apply binary pruning.')
parser.set_defaults(callback=_callback)
def _add_subdom(subparsers):
"""Substitutes domain names in buildspace tree or patches with blockable strings."""
def _add_domains(subparsers):
"""Operations with domain substitution"""
def _callback(args):
try:
if args.reverting:
domain_substitution.revert_substitution(args.cache, args.tree)
domain_substitution.revert_substitution(args.cache, args.directory)
else:
domain_substitution.apply_substitution(args.bundle, args.tree, args.cache)
domain_substitution.apply_substitution(args.bundle, args.directory, args.cache)
except FileExistsError as exc:
get_logger().error('File or directory already exists: %s', exc)
raise _CLIError()
@@ -265,202 +148,89 @@ def _add_subdom(subparsers):
except KeyError as exc:
get_logger().error('%s', exc)
raise _CLIError()
# domains
parser = subparsers.add_parser(
'subdom', help=_add_subdom.__doc__, description=_add_subdom.__doc__ + (
' By default, it will substitute the domains on both the buildspace tree and '
'the bundle\'s patches.'))
subsubparsers = parser.add_subparsers(title='Available packaging types', dest='packaging')
'domains', help=_add_domains.__doc__, description=_add_domains.__doc__)
parser.set_defaults(callback=_callback)
subsubparsers = parser.add_subparsers(title='', dest='packaging')
subsubparsers.required = True # Workaround for http://bugs.python.org/issue9253#msg186387
parser.add_argument(
'-c', '--cache', type=Path, default='buildspace/domainsubcache.tar.gz',
help=('The path to the domain substitution cache. For applying, this path must not '
'already exist. For reverting, the path must exist and will be removed '
'if successful. Default: %(default)s'))
parser.add_argument(
'-t', '--tree', type=Path, default=BUILDSPACE_TREE,
help=('The buildspace tree path to apply domain substitution. '
'Not applicable when --only is "patches". Default: %(default)s'))
# domains apply
apply_parser = subsubparsers.add_parser(
'apply', help='Apply domain substitution',
description='Applies domain substitution and creates the domain substitution cache.')
setup_bundle_group(apply_parser)
setup_bundle_arg(apply_parser)
apply_parser.add_argument(
'-c', '--cache', type=Path, required=True,
help='The path to the domain substitution cache. The path must not already exist.')
apply_parser.add_argument(
'directory', type=Path,
help='The directory to apply domain substitution')
apply_parser.set_defaults(reverting=False)
reverse_parser = subsubparsers.add_parser(
# domains revert
revert_parser = subsubparsers.add_parser(
'revert', help='Revert domain substitution',
description='Reverts domain substitution based only on the domain substitution cache.')
reverse_parser.set_defaults(reverting=True)
parser.set_defaults(callback=_callback)
revert_parser.add_argument(
'directory', type=Path,
help='The directory to reverse domain substitution')
revert_parser.add_argument(
'-c', '--cache', type=Path, required=True,
help=('The path to the domain substitution cache. '
'The path must exist and will be removed if successful.'))
revert_parser.set_defaults(reverting=True)
def _add_genpkg_archlinux(subparsers):
"""Generates a PKGBUILD for Arch Linux"""
def _callback(args):
from .packaging import archlinux as packaging_archlinux
try:
packaging_archlinux.generate_packaging(
args.bundle, args.output, repo_version=args.repo_commit,
repo_hash=args.repo_hash)
except FileExistsError as exc:
get_logger().error('PKGBUILD already exists: %s', exc)
raise _CLIError()
except FileNotFoundError as exc:
get_logger().error(
'Output path is not an existing directory: %s', exc)
raise _CLIError()
def _add_patches(subparsers):
"""Operations with patches"""
def _export_callback(args):
patches.export_patches(args.bundle, args.output)
def _apply_callback(args):
patches.apply_patches(
patches.patch_paths_by_bundle(args.bundle),
args.directory,
patch_bin_path=args.patch_bin)
# patches
parser = subparsers.add_parser(
'archlinux', help=_add_genpkg_archlinux.__doc__,
description=_add_genpkg_archlinux.__doc__)
parser.add_argument(
'-o', '--output', type=Path, default='buildspace',
help=('The directory to store packaging files. '
'It must exist and not already contain a PKGBUILD file. '
'Default: %(default)s'))
parser.add_argument(
'--repo-commit', action='store_const', const='git', default='bundle',
help=("Use the current git repo's commit hash to specify the "
"ungoogled-chromium repo to download instead of a tag determined "
"by the config bundle's version config file. Requires git to be "
"in PATH and buildkit to be invoked inside of a clone of "
"ungoogled-chromium's git repository."))
parser.add_argument(
'--repo-hash', default='SKIP',
help=('The SHA-256 hash to verify the archive of the ungoogled-chromium '
'repository to download within the PKGBUILD. If it is "compute", '
'the hash is computed by downloading the archive to memory and '
'computing the hash. If it is "SKIP", hash computation is skipped. '
'Default: %(default)s'))
parser.set_defaults(callback=_callback)
'patches', help=_add_patches.__doc__, description=_add_patches.__doc__)
subsubparsers = parser.add_subparsers(title='Patches actions')
subsubparsers.required = True
def _add_genpkg_debian(subparsers):
"""Generate Debian packaging files"""
def _callback(args):
from .packaging import debian as packaging_debian
try:
packaging_debian.generate_packaging(args.bundle, args.flavor, args.output)
except FileExistsError as exc:
get_logger().error('debian directory is not empty: %s', exc)
raise _CLIError()
except FileNotFoundError as exc:
get_logger().error(
'Parent directories do not exist for path: %s', exc)
raise _CLIError()
parser = subparsers.add_parser(
'debian', help=_add_genpkg_debian.__doc__, description=_add_genpkg_debian.__doc__)
parser.add_argument(
'-f', '--flavor', required=True, help='The Debian packaging flavor to use.')
parser.add_argument(
'-o', '--output', type=Path, default='%s/debian' % BUILDSPACE_TREE,
help=('The path to the debian directory to be created. '
'It must not already exist, but the parent directories must exist. '
'Default: %(default)s'))
parser.set_defaults(callback=_callback)
# patches export
export_parser = subsubparsers.add_parser(
'export', help='Export patches in GNU quilt-compatible format',
description='Export a config bundle\'s patches to a quilt-compatible format')
setup_bundle_arg(export_parser)
export_parser.add_argument(
'output', type=Path,
help='The directory to write to. It must either be empty or not exist.')
export_parser.set_defaults(callback=_export_callback)
def _add_genpkg_linux_simple(subparsers):
"""Generate Linux Simple packaging files"""
def _callback(args):
from .packaging import linux_simple as packaging_linux_simple
try:
packaging_linux_simple.generate_packaging(args.bundle, args.output)
except FileExistsError as exc:
get_logger().error('Output directory is not empty: %s', exc)
raise _CLIError()
except FileNotFoundError as exc:
get_logger().error(
'Parent directories do not exist for path: %s', exc)
raise _CLIError()
parser = subparsers.add_parser(
'linux_simple', help=_add_genpkg_linux_simple.__doc__,
description=_add_genpkg_linux_simple.__doc__)
parser.add_argument(
'-o', '--output', type=Path, default=BUILDSPACE_TREE_PACKAGING,
help=('The directory to store packaging files. '
'It must not already exist, but the parent directories must exist. '
'Default: %(default)s'))
parser.set_defaults(callback=_callback)
# patches apply
apply_parser = subsubparsers.add_parser(
'apply', help='Applies a config bundle\'s patches to the specified source tree')
setup_bundle_arg(apply_parser)
apply_parser.add_argument(
'--patch-bin', help='The GNU patch command to use. Omit to find it automatically.')
apply_parser.add_argument('directory', type=Path, help='The source tree to apply patches.')
apply_parser.set_defaults(callback=_apply_callback)
def _add_genpkg_opensuse(subparsers):
"""Generate OpenSUSE packaging files"""
def _callback(args):
from .packaging import opensuse as packaging_opensuse
try:
packaging_opensuse.generate_packaging(args.bundle, args.output)
except FileExistsError as exc:
get_logger().error('Output directory is not empty: %s', exc)
raise _CLIError()
except FileNotFoundError as exc:
get_logger().error(
'Parent directories do not exist for path: %s', exc)
raise _CLIError()
def _add_gnargs(subparsers):
"""Operations with GN arguments"""
def _print_callback(args):
print(str(args.bundle.gn_flags), end='')
# gnargs
parser = subparsers.add_parser(
'opensuse', help=_add_genpkg_opensuse.__doc__,
description=_add_genpkg_opensuse.__doc__)
parser.add_argument(
'-o', '--output', type=Path, default=BUILDSPACE_TREE_PACKAGING,
help=('The directory to store packaging files. '
'It must not already exist, but the parent directories must exist. '
'Default: %(default)s'))
parser.set_defaults(callback=_callback)
'gnargs', help=_add_gnargs.__doc__, description=_add_gnargs.__doc__)
subsubparsers = parser.add_subparsers(title='GN args actions')
def _add_genpkg_windows(subparsers):
"""Generate Microsoft Windows packaging files"""
def _callback(args):
from .packaging import windows as packaging_windows
try:
packaging_windows.generate_packaging(args.bundle, args.output)
except FileExistsError as exc:
get_logger().error('Output directory is not empty: %s', exc)
raise _CLIError()
except FileNotFoundError as exc:
get_logger().error(
'Parent directories do not exist for path: %s', exc)
raise _CLIError()
parser = subparsers.add_parser(
'windows', help=_add_genpkg_windows.__doc__,
description=_add_genpkg_windows.__doc__)
parser.add_argument(
'-o', '--output', type=Path, default=BUILDSPACE_TREE_PACKAGING,
help=('The directory to store packaging files. '
'It must not already exist, but the parent directories must exist. '
'Default: %(default)s'))
parser.set_defaults(callback=_callback)
def _add_genpkg_macos(subparsers):
"""Generate macOS packaging files"""
def _callback(args):
from .packaging import macos as packaging_macos
try:
packaging_macos.generate_packaging(args.bundle, args.output)
except FileExistsError as exc:
get_logger().error('Output directory is not empty: %s', exc)
raise _CLIError()
except FileNotFoundError as exc:
get_logger().error(
'Parent directories do not exist for path: %s', exc)
raise _CLIError()
parser = subparsers.add_parser(
'macos', help=_add_genpkg_macos.__doc__, description=_add_genpkg_macos.__doc__)
parser.add_argument(
'-o', '--output', type=Path, default=BUILDSPACE_TREE_PACKAGING,
help=('The directory to store packaging files. '
'It must not already exist, but the parent directories must exist. '
'Default: %(default)s'))
parser.set_defaults(callback=_callback)
def _add_genpkg(subparsers):
"""Generates a packaging script."""
parser = subparsers.add_parser(
'genpkg', help=_add_genpkg.__doc__,
description=_add_genpkg.__doc__ + ' Specify no arguments to get a list of different types.')
setup_bundle_group(parser)
# Add subcommands to genpkg for handling different packaging types in the same manner as main()
# However, the top-level argparse.ArgumentParser will be passed the callback.
subsubparsers = parser.add_subparsers(title='Available packaging types', dest='packaging')
subsubparsers.required = True # Workaround for http://bugs.python.org/issue9253#msg186387
_add_genpkg_archlinux(subsubparsers)
_add_genpkg_debian(subsubparsers)
_add_genpkg_linux_simple(subsubparsers)
_add_genpkg_opensuse(subsubparsers)
_add_genpkg_windows(subsubparsers)
_add_genpkg_macos(subsubparsers)
# gnargs print
print_parser = subsubparsers.add_parser(
'print', help='Prints GN args in args.gn format',
description='Prints a list of GN args in args.gn format to standard output')
setup_bundle_arg(print_parser)
print_parser.set_defaults(callback=_print_callback)
def main(arg_list=None):
"""CLI entry point"""
@@ -469,12 +239,11 @@ def main(arg_list=None):
subparsers = parser.add_subparsers(title='Available commands', dest='command')
subparsers.required = True # Workaround for http://bugs.python.org/issue9253#msg186387
_add_bunnfo(subparsers)
_add_genbun(subparsers)
_add_getsrc(subparsers)
_add_prubin(subparsers)
_add_subdom(subparsers)
_add_genpkg(subparsers)
_add_downloads(subparsers)
_add_prune(subparsers)
_add_domains(subparsers)
_add_patches(subparsers)
_add_gnargs(subparsers)
args = parser.parse_args(args=arg_list)
try:

127
buildkit/config.py
View File

@@ -11,13 +11,13 @@ Build configuration generation implementation
import abc
import configparser
import collections
import copy
import io
import re
from pathlib import Path
from .common import (
ENCODING, BuildkitError, ExtractorEnum,
get_logger, get_chromium_version, ensure_empty_dir, schema_dictcast, schema_inisections)
ENCODING, BuildkitError, ExtractorEnum, get_logger, get_chromium_version)
from .downloads import HashesURLEnum
from .third_party import schema
@@ -36,6 +36,23 @@ class _ConfigFile(abc.ABC): #pylint: disable=too-few-public-methods
def __init__(self, path):
self._data = self._parse_data(path)
self._init_instance_members()
def __deepcopy__(self, memo):
"""Make a deep copy of the config file"""
new_copy = copy.copy(self)
new_copy._data = self._copy_data() #pylint: disable=protected-access
new_copy._init_instance_members() #pylint: disable=protected-access
return new_copy
def _init_instance_members(self):
"""
Initialize instance-specific members. These values are not preserved on copy.
"""
@abc.abstractmethod
def _copy_data(self):
"""Returns a copy of _data for deep copying"""
@abc.abstractmethod
def _parse_data(self, path):
@@ -57,6 +74,7 @@ class _IniConfigFile(_ConfigFile): #pylint: disable=too-few-public-methods
"""
_schema = None # Derived classes must specify a schema
_ini_vars = dict() # Global INI interpolation values prefixed with underscore
def _parse_data(self, path):
"""
@@ -64,22 +82,35 @@ class _IniConfigFile(_ConfigFile): #pylint: disable=too-few-public-methods
Raises schema.SchemaError if validation fails
"""
new_data = configparser.ConfigParser()
def _section_generator(data):
for section in data:
if section == configparser.DEFAULTSECT:
continue
yield section, dict(filter(
lambda x: x[0] not in self._ini_vars,
data.items(section)))
new_data = configparser.ConfigParser(defaults=self._ini_vars)
with path.open(encoding=ENCODING) as ini_file:
new_data.read_file(ini_file, source=str(path))
if self._schema is None:
raise BuildkitConfigError('No schema defined for %s' % type(self).__name__)
try:
self._schema.validate(new_data)
self._schema.validate(dict(_section_generator(new_data)))
except schema.SchemaError as exc:
get_logger().error(
'INI file for %s failed schema validation: %s', type(self).__name__, path)
raise exc
return new_data
def _copy_data(self):
"""Returns a copy of _data for deep copying"""
new_data = configparser.ConfigParser()
new_data.read_dict(self._data)
return new_data
def rebase(self, other):
new_data = configparser.ConfigParser()
new_data.read_dict(other.data)
new_data.read_dict(other._data) #pylint: disable=protected-access
new_data.read_dict(self._data)
self._data = new_data
@@ -116,6 +147,10 @@ class ListConfigFile(_ConfigFile): #pylint: disable=too-few-public-methods
with path.open(encoding=ENCODING) as list_file:
return list(filter(len, list_file.read().splitlines()))
def _copy_data(self):
"""Returns a copy of _data for deep copying"""
return self._data[:]
def rebase(self, other):
self._data[:0] = other._data #pylint: disable=protected-access
@@ -148,6 +183,10 @@ class MapConfigFile(_ConfigFile):
new_data[key] = value
return new_data
def _copy_data(self):
"""Returns a copy of _data for deep copying"""
return self._data.copy()
def rebase(self, other):
self._data = collections.ChainMap(other._data, self._data) #pylint: disable=protected-access
@@ -182,12 +221,12 @@ class MapConfigFile(_ConfigFile):
class BundleMetaIni(_IniConfigFile):
"""Represents bundlemeta.ini files"""
_schema = schema.Schema(schema_inisections({
'bundle': schema_dictcast({
_schema = schema.Schema({
'bundle': {
'display_name': schema.And(str, len),
schema.Optional('depends'): schema.And(str, len),
})
}))
}
})
@property
def display_name(self):
@@ -213,9 +252,10 @@ class DomainRegexList(ListConfigFile):
# Constants for format:
_PATTERN_REPLACE_DELIM = '#'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def _init_instance_members(self):
"""
Initialize instance-specific members. These values are not preserved on copy.
"""
# Cache of compiled regex pairs
self._compiled_regex = None
@@ -230,7 +270,7 @@ class DomainRegexList(ListConfigFile):
Returns a tuple of compiled regex pairs
"""
if not self._compiled_regex:
self._compiled_regex = tuple(map(self._compile_regex, self))
self._compiled_regex = tuple(map(self._compile_regex, self)) #pylint: disable=attribute-defined-outside-init
return self._compiled_regex
@property
@@ -244,33 +284,31 @@ class DomainRegexList(ListConfigFile):
class DownloadsIni(_IniConfigFile): #pylint: disable=too-few-public-methods
"""Representation of an downloads.ini file"""
_hashes = ('md5', 'sha1', 'sha256', 'sha512', 'hash_url')
_nonempty_keys = ('version', 'url', 'download_filename')
_optional_keys = ('strip_leading_dirs',)
_hashes = ('md5', 'sha1', 'sha256', 'sha512')
_nonempty_keys = ('url', 'download_filename')
_optional_keys = ('version', 'strip_leading_dirs',)
_passthrough_properties = (*_nonempty_keys, *_optional_keys, 'extractor')
_option_vars = {
_ini_vars = {
'_chromium_version': get_chromium_version(),
}
_schema = schema.Schema(schema_inisections({
schema.Optional(schema.And(str, len)): schema_dictcast({
_schema = schema.Schema({
schema.Optional(schema.And(str, len)): {
**{x: schema.And(str, len) for x in _nonempty_keys},
'output_path': (lambda x: str(Path(x).relative_to(''))),
**{schema.Optional(x): schema.And(str, len) for x in _optional_keys},
schema.Optional('extractor'): schema.Or(ExtractorEnum.TAR, ExtractorEnum.SEVENZIP),
schema.Or(*_hashes): schema.And(str, len),
schema.Optional('hash_url'): schema.And(
lambda x: x.count(':') == 2,
lambda x: x.split(':')[0] in iter(HashesURLEnum)),
})
}))
schema.Optional(schema.Or(*_hashes)): schema.And(str, len),
schema.Optional('hash_url'): (
lambda x: x.count('|') == 2 and x.split('|')[0] in iter(HashesURLEnum)),
}
})
class _DownloadsProperties: #pylint: disable=too-few-public-methods
def __init__(self, section_dict, passthrough_properties, hashes, option_vars):
def __init__(self, section_dict, passthrough_properties, hashes):
self._section_dict = section_dict
self._passthrough_properties = passthrough_properties
self._hashes = hashes
self._option_vars = option_vars
def has_hash_url(self):
"""
@@ -284,7 +322,7 @@ class DownloadsIni(_IniConfigFile): #pylint: disable=too-few-public-methods
elif name == 'hashes':
hashes_dict = dict()
for hash_name in self._hashes:
value = self._section_dict.get(hash_name, vars=self._option_vars, fallback=None)
value = self._section_dict.get(hash_name, fallback=None)
if value:
if hash_name == 'hash_url':
value = value.split(':')
@@ -301,9 +339,9 @@ class DownloadsIni(_IniConfigFile): #pylint: disable=too-few-public-methods
"""
return self._DownloadsProperties(
self._data[section], self._passthrough_properties,
self._hashes, self._option_vars)
self._hashes)
class ConfigBundle:
class ConfigBundle: #pylint: disable=too-few-public-methods
"""Config bundle implementation"""
# All files in a config bundle
@@ -330,8 +368,10 @@ class ConfigBundle:
def __init__(self, path, load_depends=True):
"""
Return a new ConfigBundle from a config bundle name.
Return a new ConfigBundle from a config bundle path.
path must be a pathlib.Path or something accepted by the constructor of
pathlib.Path
load_depends indicates if the bundle's dependencies should be loaded.
This is generally only useful for developer utilities, where config
only from a specific bundle is required.
@@ -341,6 +381,8 @@ class ConfigBundle:
Raises BuildConfigError if there is an issue with the base bundle's or its
dependencies'
"""
if not isinstance(path, Path):
path = Path(path)
self.files = dict() # Config file name -> _ConfigFile object
for config_path in path.iterdir():
@@ -348,7 +390,7 @@ class ConfigBundle:
handler = self._FILE_CLASSES[config_path.name]
except KeyError:
raise BuildkitConfigError(
'Unknown file %s for bundle at %s' % config_path.name, config_path)
'Unknown file "%s" for bundle at "%s"' % (config_path.name, path))
self.files[config_path.name] = handler(config_path)
if load_depends:
for dependency in self.bundlemeta.depends:
@@ -372,19 +414,8 @@ class ConfigBundle:
def rebase(self, other):
"""Rebase the current bundle onto other, saving changes into self"""
for name, current_config_file in self.files.items():
if name in other.files:
current_config_file.rebase(other.files[name])
def to_standalone(self, path):
"""
Save the config bundle as a standalone config bundle
Raises FileExistsError if the directory already exists and is not empty.
Raises FileNotFoundError if the parent directories for path do not exist.
Raises ValueError if the config bundle is malformed.
"""
ensure_empty_dir(path)
for name, config_file in self.files.items():
with (path / name).open('w', encoding=ENCODING) as file_obj:
file_obj.write(str(config_file))
for name, other_config_file in other.files.items():
if name in self.files:
self.files[name].rebase(other_config_file)
else:
self.files[name] = copy.deepcopy(other_config_file)

48
buildkit/domain_substitution.py
View File

@@ -5,7 +5,7 @@
# found in the LICENSE file.
"""
Module for substituting domain names in buildspace tree with blockable strings.
Module for substituting domain names in the source tree with blockable strings.
"""
import io
@@ -18,7 +18,7 @@ from pathlib import Path
from .extraction import extract_tar_file
from .common import ENCODING, get_logger
# Encodings to try on buildspace tree files
# Encodings to try on source tree files
TREE_ENCODINGS = (ENCODING, 'ISO-8859-1')
# Constants for domain substitution cache
@@ -71,7 +71,7 @@ def _substitute_path(path, regex_iter):
def _validate_file_index(index_file, resolved_tree, cache_index_files):
"""
Validation of file index and hashes against the buildspace tree.
Validation of file index and hashes against the source tree.
Updates cache_index_files
Returns True if the file index is valid; False otherwise
@@ -110,26 +110,26 @@ def _validate_file_index(index_file, resolved_tree, cache_index_files):
# Public Methods
def apply_substitution(config_bundle, buildspace_tree, domainsub_cache):
def apply_substitution(config_bundle, source_tree, domainsub_cache):
"""
Substitute domains in buildspace_tree with files and substitutions from config_bundle,
Substitute domains in source_tree with files and substitutions from config_bundle,
and save the pre-domain substitution archive to presubdom_archive.
config_bundle is a config.ConfigBundle
buildspace_tree is a pathlib.Path to the buildspace tree.
source_tree is a pathlib.Path to the source tree.
domainsub_cache is a pathlib.Path to the domain substitution cache.
Raises NotADirectoryError if the patches directory is not a directory or does not exist
Raises FileNotFoundError if the buildspace tree or required directory does not exist.
Raises FileNotFoundError if the source tree or required directory does not exist.
Raises FileExistsError if the domain substitution cache already exists.
Raises ValueError if an entry in the domain substitution list contains the file index
hash delimiter.
"""
if not buildspace_tree.exists():
raise FileNotFoundError(buildspace_tree)
if not source_tree.exists():
raise FileNotFoundError(source_tree)
if domainsub_cache.exists():
raise FileExistsError(domainsub_cache)
resolved_tree = buildspace_tree.resolve()
resolved_tree = source_tree.resolve()
regex_pairs = config_bundle.domain_regex.get_pairs()
fileindex_content = io.BytesIO()
with tarfile.open(str(domainsub_cache),
@@ -161,28 +161,28 @@ def apply_substitution(config_bundle, buildspace_tree, domainsub_cache):
fileindex_content.seek(0)
cache_tar.addfile(fileindex_tarinfo, fileindex_content)
def revert_substitution(domainsub_cache, buildspace_tree):
def revert_substitution(domainsub_cache, source_tree):
"""
Revert domain substitution on buildspace_tree using the pre-domain
Revert domain substitution on source_tree using the pre-domain
substitution archive presubdom_archive.
It first checks if the hashes of the substituted files match the hashes
computed during the creation of the domain substitution cache, raising
KeyError if there are any mismatches. Then, it proceeds to
reverting files in the buildspace_tree.
reverting files in the source_tree.
domainsub_cache is removed only if all the files from the domain substitution cache
were relocated to the buildspace tree.
were relocated to the source tree.
domainsub_cache is a pathlib.Path to the domain substitution cache.
buildspace_tree is a pathlib.Path to the buildspace tree.
source_tree is a pathlib.Path to the source tree.
Raises KeyError if:
* There is a hash mismatch while validating the cache
* The cache's file index is corrupt or missing
* The cache is corrupt or is not consistent with the file index
Raises FileNotFoundError if the buildspace tree or domain substitution cache do not exist.
Raises FileNotFoundError if the source tree or domain substitution cache do not exist.
"""
# This implementation trades disk space/wear for performance (unless a ramdisk is used
# for the buildspace tree)
# for the source tree)
# Assumptions made for this process:
# * The correct tar file was provided (so no huge amount of space is wasted)
# * The tar file is well-behaved (e.g. no files extracted outside of destination path)
@@ -190,9 +190,9 @@ def revert_substitution(domainsub_cache, buildspace_tree):
# one or the other)
if not domainsub_cache.exists():
raise FileNotFoundError(domainsub_cache)
if not buildspace_tree.exists():
raise FileNotFoundError(buildspace_tree)
resolved_tree = buildspace_tree.resolve()
if not source_tree.exists():
raise FileNotFoundError(source_tree)
resolved_tree = source_tree.resolve()
cache_index_files = set() # All files in the file index
@@ -200,15 +200,15 @@ def revert_substitution(domainsub_cache, buildspace_tree):
dir=str(resolved_tree)) as tmp_extract_name:
extract_path = Path(tmp_extract_name)
get_logger().debug('Extracting domain substitution cache...')
extract_tar_file(domainsub_cache, extract_path, Path(), set(), None)
extract_tar_file(domainsub_cache, extract_path, Path())
# Validate buildspace tree file hashes match
get_logger().debug('Validating substituted files in buildspace tree...')
# Validate source tree file hashes match
get_logger().debug('Validating substituted files in source tree...')
with (extract_path / _INDEX_LIST).open('rb') as index_file: #pylint: disable=no-member
if not _validate_file_index(index_file, resolved_tree, cache_index_files):
raise KeyError(
'Domain substitution cache file index is corrupt or hashes mismatch '
'the buildspace tree.')
'the source tree.')
# Move original files over substituted ones
get_logger().debug('Moving original files over substituted ones...')

61
buildkit/downloads.py
View File

@@ -5,7 +5,7 @@
# found in the LICENSE file.
"""
Module for the downloading, checking, and unpacking of necessary files into the buildspace tree
Module for the downloading, checking, and unpacking of necessary files into the source tree
"""
import enum
@@ -78,32 +78,35 @@ def _downloads_iter(config_bundle):
"""Iterator for the downloads ordered by output path"""
return sorted(config_bundle.downloads, key=(lambda x: str(Path(x.output_path))))
def _get_hash_pairs(download_properties, downloads_dir):
def _get_hash_pairs(download_properties, cache_dir):
"""Generator of (hash_name, hash_hex) for the given download"""
for entry_type, entry_value in download_properties.hashes.items():
if entry_type == 'hash_url':
hash_processor, hash_filename, _ = entry_value
if hash_processor == 'chromium':
yield from _chromium_hashes_generator(downloads_dir / hash_filename)
yield from _chromium_hashes_generator(cache_dir / hash_filename)
else:
raise ValueError('Unknown hash_url processor: %s' % hash_processor)
else:
yield entry_type, entry_value
def retrieve_downloads(config_bundle, downloads_dir, show_progress, disable_ssl_verification=False):
def retrieve_downloads(config_bundle, cache_dir, show_progress, disable_ssl_verification=False):
"""
Retrieve all downloads into the buildspace tree.
Retrieve downloads into the downloads cache.
config_bundle is the config.ConfigBundle to retrieve downloads for.
downloads_dir is the pathlib.Path directory to store the retrieved downloads.
cache_dir is the pathlib.Path to the downloads cache.
show_progress is a boolean indicating if download progress is printed to the console.
disable_ssl_verification is a boolean indicating if certificate verification
should be disabled for downloads using HTTPS.
Raises FileNotFoundError if the downloads path does not exist.
Raises NotADirectoryError if the downloads path is not a directory.
"""
if not downloads_dir.exists():
raise FileNotFoundError(downloads_dir)
if not downloads_dir.is_dir():
raise NotADirectoryError(downloads_dir)
if not cache_dir.exists():
raise FileNotFoundError(cache_dir)
if not cache_dir.is_dir():
raise NotADirectoryError(cache_dir)
if disable_ssl_verification:
import ssl
# TODO: Remove this or properly implement disabling SSL certificate verification
@@ -114,57 +117,53 @@ def retrieve_downloads(config_bundle, downloads_dir, show_progress, disable_ssl_
download_properties = config_bundle.downloads[download_name]
get_logger().info('Downloading "%s" to "%s" ...', download_name,
download_properties.download_filename)
download_path = downloads_dir / download_properties.download_filename
download_path = cache_dir / download_properties.download_filename
_download_if_needed(download_path, download_properties.url, show_progress)
if download_properties.has_hash_url():
get_logger().info('Downloading hashes for "%s"', download_name)
_, hash_filename, hash_url = download_properties.hashes['hash_url']
_download_if_needed(downloads_dir / hash_filename, hash_url, show_progress)
_download_if_needed(cache_dir / hash_filename, hash_url, show_progress)
finally:
# Try to reduce damage of hack by reverting original HTTPS context ASAP
if disable_ssl_verification:
ssl._create_default_https_context = orig_https_context #pylint: disable=protected-access
def check_downloads(config_bundle, downloads_dir):
def check_downloads(config_bundle, cache_dir):
"""
Check integrity of all downloads.
Check integrity of the downloads cache.
config_bundle is the config.ConfigBundle to unpack downloads for.
downloads_dir is the pathlib.Path directory containing the retrieved downloads
cache_dir is the pathlib.Path to the downloads cache.
Raises source_retrieval.HashMismatchError when the computed and expected hashes do not match.
May raise undetermined exceptions during archive unpacking.
"""
for download_name in _downloads_iter(config_bundle):
get_logger().info('Verifying hashes for "%s" ...', download_name)
download_properties = config_bundle.downloads[download_name]
download_path = downloads_dir / download_properties.download_filename
download_path = cache_dir / download_properties.download_filename
with download_path.open('rb') as file_obj:
archive_data = file_obj.read()
for hash_name, hash_hex in _get_hash_pairs(download_properties, downloads_dir):
for hash_name, hash_hex in _get_hash_pairs(download_properties, cache_dir):
get_logger().debug('Verifying %s hash...', hash_name)
hasher = hashlib.new(hash_name, data=archive_data)
if not hasher.hexdigest().lower() == hash_hex.lower():
raise HashMismatchError(download_path)
def unpack_downloads(config_bundle, downloads_dir, output_dir, prune_binaries=True,
extractors=None):
def unpack_downloads(config_bundle, cache_dir, output_dir, extractors=None):
"""
Unpack all downloads to output_dir. Assumes all downloads are present.
Unpack downloads in the downloads cache to output_dir. Assumes all downloads are retrieved.
config_bundle is the config.ConfigBundle to unpack downloads for.
downloads_dir is the pathlib.Path directory containing the retrieved downloads
cache_dir is the pathlib.Path directory containing the download cache
output_dir is the pathlib.Path directory to unpack the downloads to.
prune_binaries is a boolean indicating if binary pruning should be performed.
extractors is a dictionary of PlatformEnum to a command or path to the
extractor binary. Defaults to 'tar' for tar, and '_use_registry' for 7-Zip.
Raises source_retrieval.HashMismatchError when the computed and expected hashes do not match.
May raise undetermined exceptions during archive unpacking.
"""
for download_name in _downloads_iter(config_bundle):
download_properties = config_bundle.downloads[download_name]
download_path = downloads_dir / download_properties.download_filename
download_path = cache_dir / download_properties.download_filename
get_logger().info('Unpacking "%s" to %s ...', download_name,
download_properties.output_path)
extractor_name = download_properties.extractor or ExtractorEnum.TAR
@@ -180,17 +179,7 @@ def unpack_downloads(config_bundle, downloads_dir, output_dir, prune_binaries=Tr
else:
strip_leading_dirs_path = Path(download_properties.strip_leading_dirs)
if prune_binaries:
unpruned_files = set(config_bundle.pruning)
else:
unpruned_files = set()
extractor_func(
archive_path=download_path, output_dir=output_dir,
unpack_dir=Path(download_properties.output_path), ignore_files=unpruned_files,
unpack_dir=Path(download_properties.output_path),
relative_to=strip_leading_dirs_path, extractors=extractors)
if unpruned_files:
logger = get_logger()
for path in unpruned_files:
logger.warning('File not found during binary pruning: %s', path)

131
buildkit/extraction.py
View File

@@ -23,8 +23,6 @@ DEFAULT_EXTRACTORS = {
ExtractorEnum.TAR: 'tar',
}
# TODO: Combine buildspace_tree and unpack_dir arguments
def _find_7z_by_registry():
"""
Return a string to 7-zip's 7z.exe from the Windows Registry.
@@ -67,30 +65,30 @@ def _process_relative_to(unpack_root, relative_to):
src_path.rename(dest_path)
relative_root.rmdir()
def _prune_tree(unpack_root, ignore_files):
def prune_dir(unpack_root, ignore_files):
"""
Run through the list of pruned files, delete them, and remove them from the set
Delete files under unpack_root listed in ignore_files. Returns an iterable of unremovable files.
unpack_root is a pathlib.Path to the directory to be pruned
ignore_files is an iterable of files to be removed.
"""
deleted_files = set()
unremovable_files = set()
for relative_file in ignore_files:
file_path = unpack_root / relative_file
if not file_path.is_file():
continue
file_path.unlink()
deleted_files.add(Path(relative_file).as_posix())
for deleted_path in deleted_files:
ignore_files.remove(deleted_path)
try:
file_path.unlink()
except FileNotFoundError:
unremovable_files.add(Path(relative_file).as_posix())
return unremovable_files
def _extract_tar_with_7z(binary, archive_path, buildspace_tree, unpack_dir, ignore_files, #pylint: disable=too-many-arguments
relative_to):
def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to):
get_logger().debug('Using 7-zip extractor')
out_dir = buildspace_tree / unpack_dir
if not relative_to is None and (out_dir / relative_to).exists():
if not relative_to is None and (output_dir / relative_to).exists():
get_logger().error(
'Temporary unpacking directory already exists: %s', out_dir / relative_to)
'Temporary unpacking directory already exists: %s', output_dir / relative_to)
raise BuildkitAbort()
cmd1 = (binary, 'x', str(archive_path), '-so')
cmd2 = (binary, 'x', '-si', '-aoa', '-ttar', '-o{}'.format(str(out_dir)))
cmd2 = (binary, 'x', '-si', '-aoa', '-ttar', '-o{}'.format(str(output_dir)))
get_logger().debug('7z command line: %s | %s',
' '.join(cmd1), ' '.join(cmd2))
@@ -105,16 +103,12 @@ def _extract_tar_with_7z(binary, archive_path, buildspace_tree, unpack_dir, igno
raise BuildkitAbort()
if not relative_to is None:
_process_relative_to(out_dir, relative_to)
_process_relative_to(output_dir, relative_to)
_prune_tree(out_dir, ignore_files)
def _extract_tar_with_tar(binary, archive_path, buildspace_tree, unpack_dir, #pylint: disable=too-many-arguments
ignore_files, relative_to):
def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to):
get_logger().debug('Using BSD or GNU tar extractor')
out_dir = buildspace_tree / unpack_dir
out_dir.mkdir(exist_ok=True)
cmd = (binary, '-xf', str(archive_path), '-C', str(out_dir))
output_dir.mkdir(exist_ok=True)
cmd = (binary, '-xf', str(archive_path), '-C', str(output_dir))
get_logger().debug('tar command line: %s', ' '.join(cmd))
result = subprocess.run(cmd)
if result.returncode != 0:
@@ -124,11 +118,9 @@ def _extract_tar_with_tar(binary, archive_path, buildspace_tree, unpack_dir, #py
# for gnu tar, the --transform option could be used. but to keep compatibility with
# bsdtar on macos, we just do this ourselves
if not relative_to is None:
_process_relative_to(out_dir, relative_to)
_process_relative_to(output_dir, relative_to)
_prune_tree(out_dir, ignore_files)
def _extract_tar_with_python(archive_path, buildspace_tree, unpack_dir, ignore_files, relative_to):
def _extract_tar_with_python(archive_path, output_dir, relative_to):
get_logger().debug('Using pure Python tar extractor')
class NoAppendList(list):
"""Hack to workaround memory issues with large tar files"""
@@ -155,44 +147,36 @@ def _extract_tar_with_python(archive_path, buildspace_tree, unpack_dir, ignore_f
for tarinfo in tar_file_obj:
try:
if relative_to is None:
tree_relative_path = unpack_dir / PurePosixPath(tarinfo.name)
destination = output_dir / PurePosixPath(tarinfo.name)
else:
tree_relative_path = unpack_dir / PurePosixPath(tarinfo.name).relative_to(
destination = output_dir / PurePosixPath(tarinfo.name).relative_to(
relative_to)
try:
ignore_files.remove(tree_relative_path.as_posix())
except KeyError:
destination = buildspace_tree / tree_relative_path
if tarinfo.issym() and not symlink_supported:
# In this situation, TarFile.makelink() will try to create a copy of the
# target. But this fails because TarFile.members is empty
# But if symlinks are not supported, it's safe to assume that symlinks
# aren't needed. The only situation where this happens is on Windows.
continue
if tarinfo.islnk():
# Derived from TarFile.extract()
new_target = buildspace_tree / unpack_dir / PurePosixPath(
tarinfo.linkname).relative_to(relative_to)
tarinfo._link_target = new_target.as_posix() # pylint: disable=protected-access
if destination.is_symlink():
destination.unlink()
tar_file_obj._extract_member(tarinfo, str(destination)) # pylint: disable=protected-access
if tarinfo.issym() and not symlink_supported:
# In this situation, TarFile.makelink() will try to create a copy of the
# target. But this fails because TarFile.members is empty
# But if symlinks are not supported, it's safe to assume that symlinks
# aren't needed. The only situation where this happens is on Windows.
continue
if tarinfo.islnk():
# Derived from TarFile.extract()
new_target = output_dir / PurePosixPath(tarinfo.linkname).relative_to(
relative_to)
tarinfo._link_target = new_target.as_posix() # pylint: disable=protected-access
if destination.is_symlink():
destination.unlink()
tar_file_obj._extract_member(tarinfo, str(destination)) # pylint: disable=protected-access
except BaseException:
get_logger().exception('Exception thrown for tar member: %s', tarinfo.name)
raise BuildkitAbort()
def extract_tar_file(archive_path, buildspace_tree, unpack_dir, ignore_files, relative_to, #pylint: disable=too-many-arguments
def extract_tar_file(archive_path, output_dir, relative_to, #pylint: disable=too-many-arguments
extractors=None):
"""
Extract regular or compressed tar archive into the buildspace tree.
Extract regular or compressed tar archive into the output directory.
archive_path is the pathlib.Path to the archive to unpack
buildspace_tree is a pathlib.Path to the buildspace tree.
unpack_dir is a pathlib.Path relative to buildspace_tree to unpack the archive.
It must already exist.
output_dir is a pathlib.Path to the directory to unpack. It must already exist.
ignore_files is a set of paths as strings that should not be extracted from the archive.
Files that have been ignored are removed from the set.
relative_to is a pathlib.Path for directories that should be stripped relative to the
root of the archive, or None if no path components should be stripped.
extractors is a dictionary of PlatformEnum to a command or path to the
@@ -200,7 +184,6 @@ def extract_tar_file(archive_path, buildspace_tree, unpack_dir, ignore_files, re
Raises BuildkitAbort if unexpected issues arise during unpacking.
"""
resolved_tree = buildspace_tree.resolve()
if extractors is None:
extractors = DEFAULT_EXTRACTORS
@@ -211,39 +194,29 @@ def extract_tar_file(archive_path, buildspace_tree, unpack_dir, ignore_files, re
sevenzip_cmd = str(_find_7z_by_registry())
sevenzip_bin = _find_extractor_by_cmd(sevenzip_cmd)
if not sevenzip_bin is None:
_extract_tar_with_7z(
binary=sevenzip_bin, archive_path=archive_path, buildspace_tree=resolved_tree,
unpack_dir=unpack_dir, ignore_files=ignore_files, relative_to=relative_to)
_extract_tar_with_7z(sevenzip_bin, archive_path, output_dir, relative_to)
return
elif current_platform == PlatformEnum.UNIX:
# NOTE: 7-zip isn't an option because it doesn't preserve file permissions
tar_bin = _find_extractor_by_cmd(extractors.get(ExtractorEnum.TAR))
if not tar_bin is None:
_extract_tar_with_tar(
binary=tar_bin, archive_path=archive_path, buildspace_tree=resolved_tree,
unpack_dir=unpack_dir, ignore_files=ignore_files, relative_to=relative_to)
_extract_tar_with_tar(tar_bin, archive_path, output_dir, relative_to)
return
else:
# This is not a normal code path, so make it clear.
raise NotImplementedError(current_platform)
# Fallback to Python-based extractor on all platforms
_extract_tar_with_python(
archive_path=archive_path, buildspace_tree=resolved_tree, unpack_dir=unpack_dir,
ignore_files=ignore_files, relative_to=relative_to)
_extract_tar_with_python(archive_path, output_dir, relative_to)
def extract_with_7z(archive_path, buildspace_tree, unpack_dir, ignore_files, relative_to, #pylint: disable=too-many-arguments
def extract_with_7z(archive_path, output_dir, relative_to, #pylint: disable=too-many-arguments
extractors=None):
"""
Extract archives with 7-zip into the buildspace tree.
Extract archives with 7-zip into the output directory.
Only supports archives with one layer of unpacking, so compressed tar archives don't work.
archive_path is the pathlib.Path to the archive to unpack
buildspace_tree is a pathlib.Path to the buildspace tree.
unpack_dir is a pathlib.Path relative to buildspace_tree to unpack the archive.
It must already exist.
output_dir is a pathlib.Path to the directory to unpack. It must already exist.
ignore_files is a set of paths as strings that should not be extracted from the archive.
Files that have been ignored are removed from the set.
relative_to is a pathlib.Path for directories that should be stripped relative to the
root of the archive.
extractors is a dictionary of PlatformEnum to a command or path to the
@@ -262,14 +235,12 @@ def extract_with_7z(archive_path, buildspace_tree, unpack_dir, ignore_files, rel
raise BuildkitAbort()
sevenzip_cmd = str(_find_7z_by_registry())
sevenzip_bin = _find_extractor_by_cmd(sevenzip_cmd)
resolved_tree = buildspace_tree.resolve()
out_dir = resolved_tree / unpack_dir
if not relative_to is None and (out_dir / relative_to).exists():
if not relative_to is None and (output_dir / relative_to).exists():
get_logger().error(
'Temporary unpacking directory already exists: %s', out_dir / relative_to)
'Temporary unpacking directory already exists: %s', output_dir / relative_to)
raise BuildkitAbort()
cmd = (sevenzip_bin, 'x', str(archive_path), '-aoa', '-o{}'.format(str(out_dir)))
cmd = (sevenzip_bin, 'x', str(archive_path), '-aoa', '-o{}'.format(str(output_dir)))
get_logger().debug('7z command line: %s', ' '.join(cmd))
result = subprocess.run(cmd)
@@ -278,6 +249,4 @@ def extract_with_7z(archive_path, buildspace_tree, unpack_dir, ignore_files, rel
raise BuildkitAbort()
if not relative_to is None:
_process_relative_to(out_dir, relative_to)
_prune_tree(out_dir, ignore_files)
_process_relative_to(output_dir, relative_to)

42
buildkit/patches.py
View File

@@ -7,9 +7,10 @@
"""Utilities for reading and copying patches"""
import shutil
import subprocess
from pathlib import Path
from .common import ENCODING, ensure_empty_dir
from .common import ENCODING, get_logger, ensure_empty_dir
# Default patches/ directory is next to buildkit
_DEFAULT_PATCH_DIR = Path(__file__).absolute().parent.parent / 'patches'
@@ -51,3 +52,42 @@ def export_patches(config_bundle, path, series=Path('series'), patch_dir=_DEFAUL
shutil.copyfile(str(patch_dir / relative_path), str(destination))
with (path / series).open('w', encoding=ENCODING) as file_obj:
file_obj.write(str(config_bundle.patch_order))
def apply_patches(patch_path_iter, tree_path, reverse=False, patch_bin_path=None):
"""
Applies or reverses a list of patches
tree_path is the pathlib.Path of the source tree to patch
patch_path_iter is a list or tuple of pathlib.Path to patch files to apply
reverse is whether the patches should be reversed
patch_bin_path is the pathlib.Path of the patch binary, or None to find it automatically
On Windows, this will look for the binary in third_party/git/usr/bin/patch.exe
On other platforms, this will search the PATH environment variable for "patch"
Raises ValueError if the patch binary could not be found.
"""
patch_paths = list(patch_path_iter)
if patch_bin_path is None:
windows_patch_bin_path = (tree_path /
'third_party' / 'git' / 'usr' / 'bin' / 'patch.exe')
patch_bin_path = Path(shutil.which('patch') or windows_patch_bin_path)
if not patch_bin_path.exists():
raise ValueError('Could not find the patch binary')
if reverse:
patch_paths.reverse()
logger = get_logger()
for patch_path, patch_num in zip(patch_paths, range(1, len(patch_paths) + 1)):
cmd = [
str(patch_bin_path), '-p1', '--ignore-whitespace', '-i', str(patch_path),
'-d', str(tree_path), '--no-backup-if-mismatch']
if reverse:
cmd.append('--reverse')
log_word = 'Reversing'
else:
cmd.append('--forward')
log_word = 'Applying'
logger.info(
'* %s %s (%s/%s)', log_word, patch_path.name, patch_num, len(patch_paths))
logger.debug(' '.join(cmd))
subprocess.run(cmd, check=True)

4
config_bundles/common/downloads.ini
View File

@@ -2,6 +2,6 @@
# NOTE: Substitutions beginning with underscore are provided by buildkit
[chromium]
url = https://commondatastorage.googleapis.com/chromium-browser-official/chromium-%(_chromium_version)s.tar.xz
download_name = chromium-%(_chromium_version)s.tar.xz
hash_url = chromium:chromium-%(_chromium_version)s.tar.xz.hashes:https://commondatastorage.googleapis.com/chromium-browser-official/chromium-%(_chromium_version)s.tar.xz.hashes
download_filename = chromium-%(_chromium_version)s.tar.xz
hash_url = chromium|chromium-%(_chromium_version)s.tar.xz.hashes|https://commondatastorage.googleapis.com/chromium-browser-official/chromium-%(_chromium_version)s.tar.xz.hashes
output_path = ./

2
config_bundles/debian_buster/version.ini
View File

@@ -1,2 +0,0 @@
[version]
release_extra = stretch

2
config_bundles/debian_stretch/version.ini
View File

@@ -1,2 +0,0 @@
[version]
release_extra = stretch

4
config_bundles/macos/downloads.ini
View File

@@ -7,7 +7,7 @@
[google-toolbox-for-mac]
version = 3c3111d3aefe907c8c0f0e933029608d96ceefeb
url = https://github.com/google/google-toolbox-for-mac/archive/%(version)s.tar.gz
download_name = google-toolbox-for-mac-%(version)s.tar.gz
download_filename = google-toolbox-for-mac-%(version)s.tar.gz
strip_leading_dirs = google-toolbox-for-mac-%(version)s
sha512 = 609b91872d123f9c5531954fad2f434a6ccf709cee8ae05f7f584c005ace511d4744a95e29ea057545ed5e882fe5d12385b6d08c88764f00cd64f7f2a0837790
output_path = third_party/google_toolbox_for_mac/src
@@ -16,7 +16,7 @@ output_path = third_party/google_toolbox_for_mac/src
[llvm]
version = 6.0.0
url = http://llvm.org/releases/%(version)s/clang+llvm-%(version)s-x86_64-apple-darwin.tar.xz
download_name = clang+llvm-%(version)s-x86_64-apple-darwin.tar.xz
download_filename = clang+llvm-%(version)s-x86_64-apple-darwin.tar.xz
strip_leading_dirs = clang+llvm-%(version)s-x86_64-apple-darwin
sha512 = 5240c973f929a7f639735821c560505214a6f0f3ea23807ccc9ba3cf4bc4bd86852c99ba78267415672ab3d3563bc2b0a8495cf7119c3949e400c8c17b56f935
output_path = third_party/llvm-build/Release+Asserts

2
config_bundles/ubuntu_bionic/version.ini
View File

@@ -1,2 +0,0 @@
[version]
release_extra = bionic

16
config_bundles/windows/downloads.ini
View File

@@ -7,7 +7,7 @@
#[third_party/syzygy]
#version = bd0e67f571063e18e7200c72e6152a3a7e4c2a6d
#url = https://github.com/Eloston/syzygy/archive/{version}.tar.gz
#download_name = syzygy-{version}.tar.gz
#download_filename = syzygy-{version}.tar.gz
#strip_leading_dirs = syzygy-{version}
# Use a pre-built LLVM toolchain from LLVM for convenience
@@ -23,7 +23,7 @@
[llvm]
version = 6.0.0
url = http://releases.llvm.org/%(version)s/LLVM-%(version)s-win64.exe
download_name = LLVM-%(version)s-win64.exe
download_filename = LLVM-%(version)s-win64.exe
sha512 = d61b51582f3011f00a130b7e858e36732bb0253d3d17a31d1de1eb8032bec2887caeeae303d2b38b04f517474ebe416f2c6670abb1049225919ff120e56e91d2
extractor = 7z
output_path = third_party/llvm-build/Release+Asserts
@@ -32,7 +32,7 @@ output_path = third_party/llvm-build/Release+Asserts
[gperf]
version = 3.0.1
url = https://sourceforge.net/projects/gnuwin32/files/gperf/%(version)s/gperf-%(version)s-bin.zip/download
download_name = gperf-%(version)s-bin.zip
download_filename = gperf-%(version)s-bin.zip
sha512 = 3f2d3418304390ecd729b85f65240a9e4d204b218345f82ea466ca3d7467789f43d0d2129fcffc18eaad3513f49963e79775b10cc223979540fa2e502fe7d4d9
md5 = f67a2271f68894eeaa1984221d5ef5e5
extractor = 7z
@@ -42,7 +42,7 @@ output_path = third_party/gperf
[bison-bin]
version = 2.4.1
url = https://sourceforge.net/projects/gnuwin32/files/bison/%(version)s/bison-%(version)s-bin.zip/download
download_name = bison-%(version)s-bin.zip
download_filename = bison-%(version)s-bin.zip
md5 = 9d3ccf30fc00ba5e18176c33f45aee0e
sha512 = ea8556c2be1497db96c84d627a63f9a9021423041d81210776836776f1783a91f47ac42d15c46510718d44f14653a2e066834fe3f3dbf901c3cdc98288d0b845
extractor = 7z
@@ -50,7 +50,7 @@ output_path = third_party/bison
[bison-dep]
version = 2.4.1
url = https://sourceforge.net/projects/gnuwin32/files/bison/%(version)s/bison-%(version)s-dep.zip/download
download_name = bison-%(version)s-dep.zip
download_filename = bison-%(version)s-dep.zip
md5 = 6558e5f418483b7c859643686008f475
sha512 = f1ca0737cce547c3e6f9b59202a31b12bbc5a5626b63032b05d7abd9d0f55da68b33ff6015c65ca6c15eecd35c6b1461d19a24a880abcbb4448e09f2fabe2209
extractor = 7z
@@ -58,7 +58,7 @@ output_path = third_party/bison
[bison-lib]
version = 2.4.1
url = https://sourceforge.net/projects/gnuwin32/files/bison/%(version)s/bison-%(version)s-lib.zip/download
download_name = bison-%(version)s-lib.zip
download_filename = bison-%(version)s-lib.zip
md5 = c75406456f8d6584746769b1b4b828d6
sha512 = 7400aa529c6ec412a67de1e96ae5cf43f59694fca69106eec9c6d28d04af30f20b5d4d73bdb5b53052ab848c9fb2925db684be1cf45cbbb910292bf6d1dda091
extractor = 7z
@@ -68,7 +68,7 @@ output_path = third_party/bison
[ninja]
version = 1.8.2
url = https://github.com/ninja-build/ninja/releases/download/v%(version)s/ninja-win.zip
download_name = ninja-win-%(version)s.zip
download_filename = ninja-win-%(version)s.zip
sha512 = 9b9ce248240665fcd6404b989f3b3c27ed9682838225e6dc9b67b551774f251e4ff8a207504f941e7c811e7a8be1945e7bcb94472a335ef15e23a0200a32e6d5
extractor = 7z
output_path = third_party/ninja
@@ -77,7 +77,7 @@ output_path = third_party/ninja
[git]
version = 2.16.3
url = https://github.com/git-for-windows/git/releases/download/v%(version)s.windows.1/PortableGit-%(version)s-64-bit.7z.exe
download_name = PortableGit-%(version)s-64-bit.7z.exe
download_filename = PortableGit-%(version)s-64-bit.7z.exe
sha256 = b8f321d4bb9c350a9b5e58e4330d592410ac6b39df60c5c25ca2020c6e6b273e
extractor = 7z
output_path = third_party/git

17
docs/design.md
View File

@@ -12,7 +12,6 @@ ungoogled-chromium consists of the following major components:
* [Patches](#patches)
* [Packaging](#packaging)
* [buildkit](#buildkit)
* [Buildspace](#buildspace)
The following sections describe each component.
@@ -62,11 +61,11 @@ Bundles merge config file types from its dependencies in the following manner (c
* `.map` - Entries (key-value pairs) are collected together. If a key exists in two or more dependencies, the subsequent dependencies in the dependency order have precedence.
* `.ini` - Sections are collected together. If a section exists in two or more dependencies, its keys are resolved in an identical manner as mapping config files.
Bundles vary in specificity; some apply across multiple kinds of systems, and some apply to a specific family. However, no bundle may become more specific than a "public" system variant; since there is no concrete definition, the policy for Linux distribution bundles is used to illustrate:
Bundles vary in specificity; some apply across multiple kinds of systems, and some apply to a specific family. For example:
* Each family of Linux distributions should have their own bundle (e.g. Debian, Fedora)
* Each distribution within that family can have their own bundle ONLY if they cannot be combined (e.g. Debian and Ubuntu)
* Each version for a distribution can have their own bundle ONLY if the versions in question cannot be combined and should be supported simultaneously (e.g. Debian testing and stable, Ubuntu LTS and regular stables)
* Custom Linux systems for personal or limited use **should not** have a bundle.
* Custom Linux systems for personal or limited use **should not** have a bundle (such modifications should take place in the packaging scripts).
Among the multiple bundles and mixins, here are a few noteworthy ones:
* `common` - The bundle used by all other bundles. It contains most, if not all, of the feature-implementing configuration.
@@ -158,7 +157,7 @@ The directories in `resources/packaging` correspond to the packaging type names.
## buildkit
buildkit is a Python 3 library and CLI application for building ungoogled-chromium. Its main purpose is to setup the buildspace tree and any requested building or packaging scripts from the `resources/` directory.
buildkit is a Python 3 library and CLI application for building ungoogled-chromium. It is designed to be used by the packaging process to assist in building and some of packaging.
Use `buildkit-launcher.py` to invoke the buildkit CLI. Pass in `-h` or `--help` for usage details.
@@ -171,13 +170,3 @@ There is currently no API documentation for buildkit. However, all public classe
buildkit should be simple and transparent instead of limited and intelligent when it is reasonable. As an analogy, buildkit should be like git in terms of the scope and behavior of functionality (e.g. subcommands) and as a system in whole.
buildkit should be as configuration- and platform-agnostic as possible. If there is some new functionality that is configuration-dependent or would require extending the configuration system (e.g. adding new config file types), it is preferred for this to be added to packaging scripts (in which scripts shared among packaging types are preferred over those for specific types).
## Buildspace
Buildspace is a directory that contains all intermediate and final files for building. Its default location is in the repository directory as `buildspace/`. The directory structure is as follows:
* `tree` - The Chromium source tree, which also contains build intermediates.
* `downloads` - Directory containing all files download; this is currently the Chromium source code archive and any potential extra dependencies.
* Packaged build artifacts
(The directory may contain additional files if developer utilities are used)

96
packaging/shared/apply_patch_series.py
View File

@@ -1,96 +0,0 @@
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
# Copyright (c) 2018 The ungoogled-chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Applies patches listed in a Quilt series file
"""
import argparse
import shutil
import subprocess
from pathlib import Path
def _read_series_file(series_path):
"""
Reads a Quilt series file and returns the list of pathlib.Paths contained within.
"""
out = []
with series_path.open() as series_f:
for line in series_f.readlines():
stripped = line.strip()
if stripped == '':
continue
out.append(Path(stripped))
return out
def _apply_patches(patch_bin_path, tree_path, series_path, reverse=False):
"""
Applies or reverses a list of patches
patch_bin_path is the pathlib.Path of the patch binary
tree_path is the pathlib.Path of the source tree to patch
series_path is the pathlib.Path of the Quilt series file
reverse is whether the patches should be reversed
"""
patch_paths = _read_series_file(series_path)
patch_count = len(patch_paths)
if reverse:
patch_paths.reverse()
patch_num = 1
for patch_path in patch_paths:
full_patch_path = series_path.parent / patch_path
cmd = [str(patch_bin_path), '-p1', '--ignore-whitespace', '-i', str(full_patch_path),
'-d', str(tree_path), '--no-backup-if-mismatch']
if reverse:
cmd.append('--reverse')
log_word = 'Reversing'
else:
cmd.append('--forward')
log_word = 'Applying'
print('* {} {} ({}/{})'.format(log_word, patch_path.name, patch_num, patch_count))
print(' '.join(cmd))
subprocess.run(cmd, check=True)
patch_num += 1
def main(arg_list=None):
"""CLI entrypoint"""
script_path = Path(__file__).parent.resolve()
packaging_path = script_path.parent
default_tree_path = packaging_path.parent.resolve()
default_series_path = packaging_path / 'patches' / 'series'
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--tree', metavar='PATH', type=Path, default=default_tree_path,
help='The path to the buildspace tree. Default is "%(default)s".')
parser.add_argument('--series', type=Path, default=default_series_path,
help='The path to the series file to apply. Default is "%(default)s".')
parser.add_argument('--reverse', action='store_true',
help='Whether the patches should be reversed')
args = parser.parse_args(args=arg_list)
tree_path = args.tree
series_path = args.series
if not tree_path.is_dir():
raise FileNotFoundError(str(tree_path))
if not series_path.is_file():
raise FileNotFoundError(str(series_path))
windows_patch_bin_path = (packaging_path.parent /
'third_party' / 'git' / 'usr' / 'bin' / 'patch.exe')
patch_bin_path = Path(shutil.which('patch') or windows_patch_bin_path)
if not patch_bin_path.is_file():
raise Exception('Unable to locate patch binary')
_apply_patches(patch_bin_path, tree_path, series_path, reverse=args.reverse)
if __name__ == "__main__":
main()