From 0bde311580a6d68d22f21e3a2ac1e78f4d8f5d39 Mon Sep 17 00:00:00 2001 From: Blaise Date: Tue, 18 Feb 2025 10:15:02 -0600 Subject: [PATCH] Consolidate contingent paths in prune_binaries.py Handles the differences between full tarball, lite tarball, and cloned sources. Removes the skip_unused and sysroot flags from _extraction.py and downloads.py. --- .cirrus.yml | 2 +- devutils/update_lists.py | 2 + domain_substitution.list | 392 ++++++++--------------------------- pruning.list | 95 ++++++--- utils/_extraction.py | 62 +----- utils/clone.py | 67 ------ utils/domain_substitution.py | 2 +- utils/downloads.py | 27 +-- utils/prune_binaries.py | 96 ++++++++- 9 files changed, 262 insertions(+), 483 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index ea919c78..1c5bac53 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -46,7 +46,7 @@ validate_with_source_task: fi unpack_source_script: | if [ ! -d chromium_src ]; then - ./utils/downloads.py unpack --skip-unused -i downloads.ini -c chromium_download_cache chromium_src + ./utils/downloads.py unpack -i downloads.ini -c chromium_download_cache chromium_src fi validate_patches_script: - ./devutils/validate_patches.py -l chromium_src -v diff --git a/devutils/update_lists.py b/devutils/update_lists.py index 4c6cbb7e..9a2270a6 100755 --- a/devutils/update_lists.py +++ b/devutils/update_lists.py @@ -111,6 +111,8 @@ DOMAIN_EXCLUDE_PREFIXES = [ # Exclusions for files covered with other patches/unnecessary 'components/search_engines/prepopulated_engines.json', 'third_party/blink/renderer/core/dom/document.cc', + # Exclusion to allow download of sysroots + 'build/linux/sysroot_scripts/sysroots.json', ] # pylint: enable=line-too-long diff --git a/domain_substitution.list b/domain_substitution.list index 5323caa6..c4eb15a3 100644 --- a/domain_substitution.list +++ b/domain_substitution.list @@ -478,7 +478,6 @@ build/gn_ast/json_gn_editor.py build/install-build-deps.py build/ios/presubmit_support.py build/linux/install-chromeos-fonts.py -build/linux/sysroot_scripts/sysroots.json build/linux/unbundle/remove_bundled_libraries.py build/mac_toolchain.py build/nocompile.gni @@ -515,20 +514,6 @@ cc/trees/layer_tree_host_unittest_animation.cc cc/trees/proxy_main.cc chrome/BUILD.gn chrome/PRESUBMIT.py -chrome/android/chrome_public_apk_tmpl.gni -chrome/android/expectations/lint-baseline.xml -chrome/android/expectations/lint-suppressions.xml -chrome/android/features/tab_ui/java/strings/android_chrome_tab_ui_strings.grd -chrome/android/java/res/values-sw600dp/values.xml -chrome/android/java/res/values/dimens.xml -chrome/android/java/res/values/strings.xml -chrome/android/java/res/values/values.xml -chrome/android/java/src/PRESUBMIT.py -chrome/android/javatests/src/org/chromium/chrome/test/smoke/AndroidManifest_bundle.xml -chrome/android/trichrome.gni -chrome/android/webapk/PRESUBMIT.py -chrome/android/webapk/shell_apk/manifest/javatest_manifest_config_delta.json -chrome/android/webapk/shell_apk/res/values/dimens.xml chrome/app/PRESUBMIT.py chrome/app/app_management_strings.grdp chrome/app/chrome_command_ids.h @@ -4026,6 +4011,89 @@ content/services/auction_worklet/webidl_compat.cc content/shell/browser/shell_browser_main_parts.cc content/shell/browser/shell_platform_delegate_ios.mm content/test/PRESUBMIT.py +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/latest/report_1.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/latest/report_2.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/latest/report_3.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/latest/report_gcp_1.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/latest/report_gcp_2.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/latest/report_gcp_3.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/version_0.1/report_1.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/version_0.1/report_2.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/version_0.1/report_3.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/version_0.1/report_gcp_1.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/version_0.1/report_gcp_2.json +content/test/data/attribution_reporting/aggregatable_debug_report_goldens/version_0.1/report_gcp_3.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_1.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_10.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_2.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_3.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_4.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_5.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_6.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_7.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_8.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_9.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_1.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_2.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_3.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_4.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_5.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_6.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_7.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_8.json +content/test/data/attribution_reporting/aggregatable_report_goldens/latest/report_gcp_9.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_1.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_2.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_3.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_4.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_5.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_6.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_7.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_8.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_9.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_1.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_2.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_3.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_4.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_5.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_6.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_7.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_8.json +content/test/data/attribution_reporting/aggregatable_report_goldens/version_0.1/report_gcp_9.json +content/test/data/dom_serializer/html_doc_has_base_tag.htm +content/test/data/dom_serializer/youtube_1.htm +content/test/data/dom_serializer/youtube_2.htm +content/test/data/fedcm/accounts_endpoint.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_1.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_10.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_11.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_12.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_13.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_14.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_15.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_16.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_17.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_2.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_3.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_4.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_5.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_6.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_7.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_8.json +content/test/data/private_aggregation/aggregatable_report_goldens/latest/report_9.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_1.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_10.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_2.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_3.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_4.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_5.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_6.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_7.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_8.json +content/test/data/private_aggregation/aggregatable_report_goldens/version_0.1/report_9.json +content/test/data/site_isolation/json-prefixed-3.js +content/test/data/site_isolation/json-prefixed-4.js +content/test/data/web_ui_mojo_ts_test.ts content/test/gpu/PRESUBMIT.py content/test/gpu/find_unknown_typ_tags.py content/test/gpu/flake_suppressor/gpu_results_unittest.py @@ -4322,259 +4390,6 @@ infra/config/generated/health-specs/health-specs.json infra/config/migration/PRESUBMIT.py infra/config/scripts/PRESUBMIT.py infra/config/scripts/sync-pyl-files.py -ios/PRESUBMIT.py -ios/build/bots/scripts/PRESUBMIT.py -ios/build/bots/scripts/plugin/PRESUBMIT.py -ios/build/bots/scripts/result_sink_util.py -ios/build/bots/scripts/result_sink_util_test.py -ios/build/bots/scripts/test_result_util.py -ios/build/bots/scripts/test_runner.py -ios/build/bots/scripts/xcode_util.py -ios/chrome/app/application_delegate/url_opener_unittest.mm -ios/chrome/app/startup/chrome_app_startup_parameters_unittest.mm -ios/chrome/app/strings/ios_chromium_strings.grd -ios/chrome/app/strings/ios_google_chrome_strings.grd -ios/chrome/app/strings/ios_strings.grd -ios/chrome/browser/app_launcher/model/app_launcher_abuse_detector_unittest.mm -ios/chrome/browser/app_launcher/model/app_launcher_tab_helper_unittest.mm -ios/chrome/browser/authentication/ui_bundled/account_menu/account_menu_egtest.mm -ios/chrome/browser/authentication/ui_bundled/account_menu/account_menu_mediator_delegate.h -ios/chrome/browser/authentication/ui_bundled/account_menu/account_menu_mutator.h -ios/chrome/browser/authentication/ui_bundled/authentication_flow/authentication_flow_unittest.mm -ios/chrome/browser/authentication/ui_bundled/authentication_flow_unittest.mm -ios/chrome/browser/authentication/ui_bundled/history_sync/history_sync_capabilities_fetcher_unittest.mm -ios/chrome/browser/authentication/ui_bundled/signin/add_account_signin/add_account_signin_manager_unittest.mm -ios/chrome/browser/authentication/ui_bundled/signin_earl_grey_ui_test_util.mm -ios/chrome/browser/autofill/model/automation/automation_action.mm -ios/chrome/browser/autofill/model/form_suggestion_controller_unittest.mm -ios/chrome/browser/autofill/ui_bundled/authentication/authentication_egtest_util.mm -ios/chrome/browser/autofill/ui_bundled/authentication/card_unmask_authentication_egtest.mm -ios/chrome/browser/autofill/ui_bundled/bottom_sheet/virtual_card_enrollment_bottom_sheet_egtest.mm -ios/chrome/browser/autofill/ui_bundled/cells/target_account_item_unittest.mm -ios/chrome/browser/autofill/ui_bundled/manual_fill/form_observer_helper_unittest.mm -ios/chrome/browser/autofill/ui_bundled/manual_fill/manual_fill_password_mediator_unittest.mm -ios/chrome/browser/autofill/ui_bundled/save_card_infobar_egtest.mm -ios/chrome/browser/autofill/ui_bundled/save_profile_egtest.mm -ios/chrome/browser/bookmarks/ui_bundled/bookmark_earl_grey.mm -ios/chrome/browser/bookmarks/ui_bundled/bookmark_earl_grey_ui.h -ios/chrome/browser/bookmarks/ui_bundled/bookmark_mediator_unittest.mm -ios/chrome/browser/bookmarks/ui_bundled/bookmarks_egtest.mm -ios/chrome/browser/bookmarks/ui_bundled/folder_chooser/bookmarks_folder_chooser_egtest.mm -ios/chrome/browser/bring_android_tabs/ui_bundled/tab_list_from_android_mediator_unittest.mm -ios/chrome/browser/commerce/model/shopping_persisted_data_tab_helper_unittest.mm -ios/chrome/browser/context_menu/ui_bundled/context_menu_configuration_provider_unittest.mm -ios/chrome/browser/crash_report/model/breadcrumbs/breadcrumb_manager_tab_helper_unittest.mm -ios/chrome/browser/credential_provider/model/credential_provider_service_unittest.mm -ios/chrome/browser/credential_provider/model/ios_credential_provider_infobar_delegate_unittest.mm -ios/chrome/browser/credential_provider_promo/ui_bundled/resources/CPE_promo_animation_edu_autofill.json -ios/chrome/browser/credential_provider_promo/ui_bundled/resources/CPE_promo_animation_edu_autofill_darkmode.json -ios/chrome/browser/dialogs/ui_bundled/nsurl_protection_space_util_unittest.mm -ios/chrome/browser/download/model/ar_quick_look_tab_helper_unittest.mm -ios/chrome/browser/drive/model/drive_list.h -ios/chrome/browser/drive/model/manage_storage_url_util.mm -ios/chrome/browser/drive_file_picker/coordinator/drive_file_picker_mediator.mm -ios/chrome/browser/drive_file_picker/coordinator/drive_file_picker_mediator_helper.mm -ios/chrome/browser/first_run/ui_bundled/first_run_search_engine_choice_egtest.mm -ios/chrome/browser/flags/about_flags.mm -ios/chrome/browser/history/model/history_tab_helper_unittest.mm -ios/chrome/browser/home_customization/utils/home_customization_constants.mm -ios/chrome/browser/infobars/ui_bundled/modals/autofill_address_profile/infobar_save_address_profile_table_view_controller_unittest.mm -ios/chrome/browser/infobars/ui_bundled/test/test_infobar_password_delegate.mm -ios/chrome/browser/intents/user_activity_browser_agent_unittest.mm -ios/chrome/browser/lens_overlay/coordinator/lens_omnibox_client_unittest.mm -ios/chrome/browser/lens_overlay/coordinator/lens_overlay_mediator_unittest.mm -ios/chrome/browser/lens_overlay/coordinator/lens_result_page_mediator.mm -ios/chrome/browser/lens_overlay/coordinator/lens_result_page_mediator_unittest.mm -ios/chrome/browser/link_to_text/model/link_to_text_java_script_feature_unittest.mm -ios/chrome/browser/link_to_text/ui_bundled/link_to_text_mediator_unittest.mm -ios/chrome/browser/location_bar/ui_bundled/location_bar_coordinator_unittest.mm -ios/chrome/browser/metrics/model/ios_family_link_user_metrics_provider_unittest.mm -ios/chrome/browser/metrics/model/tab_usage_recorder_browser_agent_unittest.mm -ios/chrome/browser/net/model/retryable_url_fetcher_unittest.mm -ios/chrome/browser/ntp/model/browser_policy_new_tab_page_rewriter_unittest.mm -ios/chrome/browser/ntp/ui_bundled/discover_feed_constants.mm -ios/chrome/browser/ntp/ui_bundled/incognito/incognito_view_util.mm -ios/chrome/browser/ntp/ui_bundled/new_tab_page_coordinator_unittest.mm -ios/chrome/browser/ntp/ui_bundled/new_tab_page_mediator.mm -ios/chrome/browser/ntp/ui_bundled/new_tab_page_mediator_unittest.mm -ios/chrome/browser/omaha/model/omaha_service.mm -ios/chrome/browser/omnibox/ui_bundled/omnibox_egtest.mm -ios/chrome/browser/omnibox/ui_bundled/zero_suggest_prefetch_helper_unittest.mm -ios/chrome/browser/optimization_guide/model/optimization_guide_validation_tab_helper_unittest.mm -ios/chrome/browser/overlays/ui_bundled/infobar_banner/passwords/password_infobar_banner_overlay_mediator_unittest.mm -ios/chrome/browser/overlays/ui_bundled/infobar_modal/passwords/password_infobar_modal_overlay_mediator_unittest.mm -ios/chrome/browser/page_info/about_this_site_tab_helper_unittest.mm -ios/chrome/browser/page_info/ui_bundled/page_info_security_egtest.mm -ios/chrome/browser/passwords/model/password_controller_js_unittest.mm -ios/chrome/browser/passwords/model/password_controller_unittest.mm -ios/chrome/browser/plus_addresses/ui/plus_address_bottom_sheet_constants.h -ios/chrome/browser/policy/model/cloud/user_policy_signin_service_unittest.mm -ios/chrome/browser/policy/model/new_tab_page_location_policy_handler_unittest.cc -ios/chrome/browser/policy/model/restrict_accounts_policy_handler_unittest.cc -ios/chrome/browser/prerender/model/preload_controller_unittest.mm -ios/chrome/browser/prerender/model/prerender_service_impl_unittest.mm -ios/chrome/browser/profile/model/profile_ios_impl.h -ios/chrome/browser/reading_list/model/reading_list_browser_agent_unittest.mm -ios/chrome/browser/reading_list/ui_bundled/reading_list_list_item_factory_unittest.mm -ios/chrome/browser/reading_list/ui_bundled/reading_list_mediator_unittest.mm -ios/chrome/browser/reading_list/ui_bundled/reading_list_table_view_item_unittest.mm -ios/chrome/browser/safe_browsing/model/safe_browsing_client_impl_unittest.mm -ios/chrome/browser/safe_browsing/model/tailored_security/tailored_security_tab_helper_unittest.mm -ios/chrome/browser/safety_check/model/ios_chrome_safety_check_manager_unittest.mm -ios/chrome/browser/save_to_photos/ui_bundled/save_to_photos_mediator.mm -ios/chrome/browser/saved_tab_groups/model/ios_tab_group_sync_util_unittest.mm -ios/chrome/browser/saved_tab_groups/model/tab_group_local_update_observer_unittest.mm -ios/chrome/browser/search_engines/model/extension_search_engine_data_updater_unittest.mm -ios/chrome/browser/search_engines/model/resources/search_engine.ts -ios/chrome/browser/search_engines/model/search_engine_js_unittest.mm -ios/chrome/browser/search_engines/model/search_engine_tab_helper.h -ios/chrome/browser/search_engines/model/search_engine_tab_helper.mm -ios/chrome/browser/settings/ui_bundled/autofill/autofill_profile_settings_egtest.mm -ios/chrome/browser/settings/ui_bundled/clear_browsing_data/clear_browsing_data_egtest.mm -ios/chrome/browser/settings/ui_bundled/clear_browsing_data/quick_delete_egtest.mm -ios/chrome/browser/settings/ui_bundled/google_services/manage_sync_settings_command_handler.h -ios/chrome/browser/settings/ui_bundled/password/password_checkup/password_checkup_constants.mm -ios/chrome/browser/settings/ui_bundled/password/password_checkup/password_checkup_mediator_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_checkup/password_checkup_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_details/add_password_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_details/password_details_mediator_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_details/password_details_table_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_exporter_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_issues/password_issues_table_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_manager_egtest.mm -ios/chrome/browser/settings/ui_bundled/password/password_manager_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_settings/password_settings_mediator_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/family_picker_coordinator_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/family_picker_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/family_promo_coordinator_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/password_picker_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/password_sharing_constants.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/password_sharing_egtest.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/password_sharing_first_run_coordinator_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/password_sharing/sharing_status_mediator_unittest.mm -ios/chrome/browser/settings/ui_bundled/password/passwords_mediator_unittest.mm -ios/chrome/browser/settings/ui_bundled/safety_check/safety_check_mediator_unittest.mm -ios/chrome/browser/settings/ui_bundled/search_engine_table_view_controller_unittest.mm -ios/chrome/browser/settings/ui_bundled/supervised_user_settings_app_interface.mm -ios/chrome/browser/settings/ui_bundled/table_cell_catalog_view_controller.mm -ios/chrome/browser/settings/ui_bundled/translate_table_view_controller.mm -ios/chrome/browser/shared/coordinator/scene/scene_controller.mm -ios/chrome/browser/shared/coordinator/scene/scene_controller_unittest.mm -ios/chrome/browser/shared/model/url/chrome_url_constants.cc -ios/chrome/browser/shared/model/web_state_list/order_controller_source_from_web_state_list_unittest.mm -ios/chrome/browser/shared/model/web_state_list/web_state_list_unittest.mm -ios/chrome/browser/shared/ui/table_view/cells/table_view_detail_icon_item_unittest.mm -ios/chrome/browser/shared/ui/table_view/cells/table_view_url_item_unittest.mm -ios/chrome/browser/shared/ui/util/pasteboard_util_unittest.mm -ios/chrome/browser/sharing/ui_bundled/activity_services/activities/copy_activity_unittest.mm -ios/chrome/browser/sharing/ui_bundled/activity_services/activities/find_in_page_activity_unittest.mm -ios/chrome/browser/sharing/ui_bundled/activity_services/activities/print_activity_unittest.mm -ios/chrome/browser/sharing/ui_bundled/activity_services/activities/send_tab_to_self_activity_unittest.mm -ios/chrome/browser/sharing/ui_bundled/activity_services/activity_service_mediator_unittest.mm -ios/chrome/browser/sharing/ui_bundled/activity_services/activity_type_util.mm -ios/chrome/browser/sharing/ui_bundled/qr_generator/qr_generator_coordinator_unittest.mm -ios/chrome/browser/sharing/ui_bundled/qr_generator/qr_generator_util_unittest.mm -ios/chrome/browser/signin/model/account_capabilities_fetcher_ios_unittest.mm -ios/chrome/browser/signin/model/account_profile_mapper_unittest.mm -ios/chrome/browser/signin/model/authentication_service_unittest.mm -ios/chrome/browser/signin/model/chrome_account_manager_service_unittest.mm -ios/chrome/browser/signin/model/fake_system_identity.mm -ios/chrome/browser/signin/model/fake_system_identity_manager.mm -ios/chrome/browser/signin/model/gaia_auth_fetcher_ios_unittest.mm -ios/chrome/browser/signin/model/pattern_account_restriction_unittest.mm -ios/chrome/browser/signin/model/system_identity_manager.h -ios/chrome/browser/signin/model/test_account_info.mm -ios/chrome/browser/snapshots/model/snapshot_generator_unittest.mm -ios/chrome/browser/ssl/model/ios_ssl_error_handler_unittest.mm -ios/chrome/browser/start_surface/ui_bundled/start_surface_scene_agent_unittest.mm -ios/chrome/browser/supervised_user/model/supervised_user_url_filter_tab_helper_unittest.mm -ios/chrome/browser/tab_switcher/ui_bundled/tab_grid/grid/base_grid_mediator_unittest.mm -ios/chrome/browser/tab_switcher/ui_bundled/tab_grid/tab_grid_egtest.mm -ios/chrome/browser/tab_switcher/ui_bundled/tab_grid/tab_groups/tab_group_sync_earl_grey_app_interface.mm -ios/chrome/browser/tab_switcher/ui_bundled/tab_strip/coordinator/tab_strip_mediator_unittest.mm -ios/chrome/browser/toolbar/ui_bundled/adaptive_toolbar_mediator_unittest.mm -ios/chrome/browser/ui/content_suggestions/cells/most_visited_tiles_mediator_unittest.mm -ios/chrome/browser/ui/content_suggestions/ntp_home_egtest.mm -ios/chrome/browser/ui/content_suggestions/parcel_tracking/parcel_tracking_mediator_unittest.mm -ios/chrome/browser/ui/content_suggestions/tab_resumption/tab_resumption_mediator.mm -ios/chrome/browser/ui/popup_menu/overflow_menu/overflow_menu_mediator_unittest.mm -ios/chrome/browser/ui/popup_menu/popup_menu_mediator_unittest.mm -ios/chrome/browser/url_loading/model/image_search_param_generator_unittest.mm -ios/chrome/browser/web/model/chrome_web_client_unittest.mm -ios/chrome/browser/web/model/font_size/font_size_tab_helper_unittest.mm -ios/chrome/browser/web_state_list/model/web_usage_enabler/web_usage_enabler_browser_agent_unittest.mm -ios/chrome/browser/webui/ui_bundled/interstitials/interstitial_ui_util.mm -ios/chrome/common/ns_regular_expression_unittest.mm -ios/chrome/common/string_util_unittest.mm -ios/chrome/common/x_callback_url_unittest.cc -ios/chrome/credential_provider_extension/strings/ios_credential_provider_extension_strings.grd -ios/chrome/credential_provider_extension/ui/passkey_welcome_screen_view_controller_unittest.mm -ios/chrome/test/data/policy/PRESUBMIT.py -ios/chrome/test/data/policy/pref_mapping/ManagedBookmarks.json -ios/chrome/test/data/policy/pref_mapping/NewTabPageLocation.json -ios/chrome/test/data/policy/pref_mapping/PopupsAllowedForUrls.json -ios/chrome/test/data/policy/pref_mapping/PopupsBlockedForUrls.json -ios/chrome/test/data/policy/pref_mapping/RestrictAccountsToPatterns.json -ios/chrome/test/data/policy/pref_mapping/URLAllowlist.json -ios/chrome/test/data/policy/pref_mapping/URLBlocklist.json -ios/chrome/test/earl_grey/chrome_earl_grey_ui.h -ios/chrome/test/earl_grey/device_check_egtest.mm -ios/chrome/test/providers/omaha/test_omaha.cc -ios/chrome/test/xcuitest/device_check_xctest.mm -ios/components/cookie_util/cookie_util_unittest.mm -ios/components/credential_provider_extension/password_spec_fetcher.mm -ios/components/credential_provider_extension/password_util_unittest.mm -ios/components/security_interstitials/https_only_mode/https_only_mode_blocking_page.mm -ios/net/cookies/cookie_cache_unittest.cc -ios/net/cookies/cookie_store_ios_test_util.mm -ios/net/cookies/cookie_store_ios_unittest.mm -ios/net/cookies/system_cookie_store_unittest_template.h -ios/net/crn_http_protocol_handler.mm -ios/net/nsurlrequest_util_unittest.mm -ios/testing/PRESUBMIT.py -ios/testing/data/http_server_files/opensearch.xml -ios/testing/data/http_server_files/window_open.html -ios/testing/earl_grey/base_earl_grey_test_case.mm -ios/web/browsing_data/browsing_data_removing_util_unittest.mm -ios/web/browsing_data/system_cookie_store_util_unittest.mm -ios/web/favicon/favicon_util_unittest.mm -ios/web/js_features/context_menu/context_menu_js_unittest.mm -ios/web/navigation/history_state_operations_inttest.mm -ios/web/navigation/navigation_manager_impl.mm -ios/web/navigation/navigation_manager_util_unittest.mm -ios/web/net/cookies/crw_wk_http_cookie_store_unittest.mm -ios/web/public/init/web_main_parts.h -ios/web/shell/test/earl_grey/shell_earl_grey.h -ios/web/util/wk_security_origin_util_unittest.mm -ios/web/web_state/permissions_inttest.mm -ios/web/web_state/ui/crw_web_controller_unittest.mm -ios/web/web_state/ui/wk_content_rule_list_util.mm -ios/web/web_state/web_state_context_menu_bridge_unittest.mm -ios/web/web_state/web_state_unittest.mm -ios/web_view/PRESUBMIT.py -ios/web_view/internal/autofill/cwv_autofill_controller_unittest.mm -ios/web_view/internal/autofill/cwv_autofill_data_manager_unittest.mm -ios/web_view/internal/autofill/cwv_credit_card_saver_unittest.mm -ios/web_view/internal/cwv_favicon_unittest.mm -ios/web_view/internal/cwv_flags.mm -ios/web_view/internal/cwv_lookalike_url_handler_unittest.mm -ios/web_view/internal/cwv_ssl_error_handler_unittest.mm -ios/web_view/internal/cwv_web_view.mm -ios/web_view/internal/passwords/cwv_credential_provider_extension_utils_unittest.mm -ios/web_view/internal/passwords/cwv_leak_check_credential_unittest.mm -ios/web_view/internal/passwords/cwv_password_unittest.mm -ios/web_view/internal/passwords/cwv_reuse_check_service_unittest.mm -ios/web_view/internal/safe_browsing/cwv_unsafe_url_handler_unittest.mm -ios/web_view/internal/signin/account_capabilities_fetcher_ios_web_view_unittest.mm -ios/web_view/internal/signin/cwv_identity_unittest.mm -ios/web_view/internal/signin/web_view_device_accounts_provider_impl_unittest.mm -ios/web_view/internal/sync/cwv_sync_controller_unittest.mm -ios/web_view/internal/translate/cwv_translation_controller_unittest.mm -ios/web_view/public/cwv_autofill_data_manager.h -ios/web_view/public/cwv_credential_provider_extension_utils.h -ios/web_view/public/cwv_identity.h -ios/web_view/public/cwv_translation_language.h -ios/web_view/public/cwv_web_view.h ipc/ipc_channel.h media/PRESUBMIT.py media/audio/android/audio_track_output_stream.cc @@ -7266,47 +7081,6 @@ third_party/cld_3/src/src/script_span/getonescriptspan.h third_party/cld_3/src/src/script_span/offsetmap.cc third_party/cld_3/src/src/script_span/offsetmap.h third_party/cld_3/src/src/script_span/utf8statetable.h -third_party/closure_compiler/externs/accessibility_features.js -third_party/closure_compiler/externs/accessibility_private.js -third_party/closure_compiler/externs/arc_apps_private.js -third_party/closure_compiler/externs/audio.js -third_party/closure_compiler/externs/bluetooth.js -third_party/closure_compiler/externs/bluetooth_private.js -third_party/closure_compiler/externs/braille_display_private.js -third_party/closure_compiler/externs/chrome.js -third_party/closure_compiler/externs/chrome_extensions.js -third_party/closure_compiler/externs/chromeos_info_private.js -third_party/closure_compiler/externs/clipboard.js -third_party/closure_compiler/externs/command_line_private.js -third_party/closure_compiler/externs/crash_report_private.js -third_party/closure_compiler/externs/events.js -third_party/closure_compiler/externs/extension_types.js -third_party/closure_compiler/externs/file_system_provider.js -third_party/closure_compiler/externs/image_loader_private.js -third_party/closure_compiler/externs/input_method_private.js -third_party/closure_compiler/externs/language_settings_private.js -third_party/closure_compiler/externs/login_state.js -third_party/closure_compiler/externs/metrics_private.js -third_party/closure_compiler/externs/metrics_private_individual_apis.js -third_party/closure_compiler/externs/mojo_private.js -third_party/closure_compiler/externs/networking_private.js -third_party/closure_compiler/externs/pending.js -third_party/closure_compiler/externs/quick_unlock_private.js -third_party/closure_compiler/externs/runtime.js -third_party/closure_compiler/externs/settings_private.js -third_party/closure_compiler/externs/speech_recognition_private.js -third_party/closure_compiler/externs/tabs.js -third_party/closure_compiler/externs/terminal_private.js -third_party/closure_compiler/externs/test.js -third_party/closure_compiler/externs/tts.js -third_party/closure_compiler/externs/ttsEngine.js -third_party/closure_compiler/externs/virtual_keyboard_private.js -third_party/closure_compiler/externs/webview_tag.js -third_party/closure_compiler/externs/windows.js -third_party/closure_compiler/interfaces/bluetooth_interface.js -third_party/closure_compiler/interfaces/networking_private_interface.js -third_party/closure_compiler/interfaces/settings_private_interface.js -third_party/closure_compiler/interfaces/system_display_interface.js third_party/colorama/src/demos/demo09.py third_party/content_analysis_sdk/src/agent/src/scoped_print_handle_win.cc third_party/content_analysis_sdk/src/common/utils_win.cc diff --git a/pruning.list b/pruning.list index 76586a3a..664403da 100644 --- a/pruning.list +++ b/pruning.list @@ -247,6 +247,77 @@ content/browser/indexed_db/fuzzer_corpus/indexed_db_key/string_type content/browser/indexed_db/fuzzer_corpus/indexed_db_key_path/ascii content/browser/indexed_db/fuzzer_corpus/indexed_db_key_path/ascii_utf8_vector content/browser/indexed_db/fuzzer_corpus/indexed_db_key_path/utf8 +content/test/data/android/SelfConsumingObject.dex +content/test/data/background_resource_fetch/SpaceOnly.otf +content/test/data/cache_storage/invalid_bucket_id/0430f1a484a0ea6d8de562488c5fbeec0111d16f/ba31f73b-95e8-44fb-a273-265e7e76f5ab/3767fbd3edc759b5_0 +content/test/data/cache_storage/invalid_bucket_id/0430f1a484a0ea6d8de562488c5fbeec0111d16f/ba31f73b-95e8-44fb-a273-265e7e76f5ab/index +content/test/data/cache_storage/invalid_bucket_id/0430f1a484a0ea6d8de562488c5fbeec0111d16f/ba31f73b-95e8-44fb-a273-265e7e76f5ab/index-dir/the-real-index +content/test/data/cache_storage/padding_v2/0430f1a484a0ea6d8de562488c5fbeec0111d16f/676288ed-4de0-44af-97d6-dbd75b07a8a3/4bfb34f348c2269e_0 +content/test/data/cache_storage/padding_v2/0430f1a484a0ea6d8de562488c5fbeec0111d16f/676288ed-4de0-44af-97d6-dbd75b07a8a3/index +content/test/data/cache_storage/padding_v2/0430f1a484a0ea6d8de562488c5fbeec0111d16f/676288ed-4de0-44af-97d6-dbd75b07a8a3/index-dir/the-real-index +content/test/data/cache_storage/storage_key/0430f1a484a0ea6d8de562488c5fbeec0111d16f/ba31f73b-95e8-44fb-a273-265e7e76f5ab/3767fbd3edc759b5_0 +content/test/data/cache_storage/storage_key/0430f1a484a0ea6d8de562488c5fbeec0111d16f/ba31f73b-95e8-44fb-a273-265e7e76f5ab/index +content/test/data/cache_storage/storage_key/0430f1a484a0ea6d8de562488c5fbeec0111d16f/ba31f73b-95e8-44fb-a273-265e7e76f5ab/index-dir/the-real-index +content/test/data/download-test1.lib +content/test/data/download/empty.bin +content/test/data/download/gzip-content.gz +content/test/data/fuzzer_corpus/signed_exchange_certificate_chain_data/wildcard_example.org.public.pem.cbor +content/test/data/gzip-content.svgz +content/test/data/indexeddb/corrupt_leveldb/file__0.indexeddb.leveldb/000003.log +content/test/data/indexeddb/corrupt_leveldb/file__0.indexeddb.leveldb/MANIFEST-000002 +content/test/data/indexeddb/crbug899446/file__0.indexeddb.leveldb/000003.log +content/test/data/indexeddb/crbug899446/file__0.indexeddb.leveldb/MANIFEST-000001 +content/test/data/indexeddb/crbug899446_noai/file__0.indexeddb.leveldb/000003.log +content/test/data/indexeddb/crbug899446_noai/file__0.indexeddb.leveldb/MANIFEST-000001 +content/test/data/indexeddb/missing_sst/file__0.indexeddb.leveldb/000008.sst +content/test/data/indexeddb/missing_sst/file__0.indexeddb.leveldb/MANIFEST-000007 +content/test/data/indexeddb/schema_version_123456/file__0.indexeddb.leveldb/000003.log +content/test/data/indexeddb/schema_version_123456/file__0.indexeddb.leveldb/MANIFEST-000002 +content/test/data/indexeddb/ssv_version_987654/file__0.indexeddb.leveldb/000003.log +content/test/data/indexeddb/ssv_version_987654/file__0.indexeddb.leveldb/MANIFEST-000002 +content/test/data/indexeddb/v3_migration_test/file__0.indexeddb.blob/1/00/5 +content/test/data/indexeddb/v3_migration_test/file__0.indexeddb.leveldb/000003.log +content/test/data/indexeddb/v3_migration_test/file__0.indexeddb.leveldb/MANIFEST-000001 +content/test/data/interest_group/auction_only.wbn +content/test/data/interest_group/auction_only_both_new_and_old_names.wbn +content/test/data/interest_group/auction_only_new_name.wbn +content/test/data/interest_group/multiply.wasm +content/test/data/shared_dictionary/path/compressed.data +content/test/data/site_isolation/nosniff.png.octet-stream +content/test/data/sxg/127.0.0.1.public.pem.cbor +content/test/data/sxg/google-com.example.org.public.pem.cbor +content/test/data/sxg/google-com.example.org_test.sxg +content/test/data/sxg/test.example.com_invalid_test.sxg +content/test/data/sxg/test.example.org-long-validity.public.pem.cbor +content/test/data/sxg/test.example.org-noext.public.pem.cbor +content/test/data/sxg/test.example.org-validity-too-long.public.pem.cbor +content/test/data/sxg/test.example.org.public.pem.cbor +content/test/data/sxg/test.example.org_bad_content_type.sxg +content/test/data/sxg/test.example.org_cert_validity_too_long.sxg +content/test/data/sxg/test.example.org_csp.sxg +content/test/data/sxg/test.example.org_fr_variant.sxg +content/test/data/sxg/test.example.org_hello.txt.sxg +content/test/data/sxg/test.example.org_long_cert_validity.sxg +content/test/data/sxg/test.example.org_noext_test.sxg +content/test/data/sxg/test.example.org_test.html.gz.sxg +content/test/data/sxg/test.example.org_test.sxg +content/test/data/sxg/test.example.org_test_bad_mice.sxg +content/test/data/sxg/test.example.org_test_bad_mice_small.sxg +content/test/data/sxg/test.example.org_test_download.sxg +content/test/data/sxg/test.example.org_test_invalid_cbor_header.sxg +content/test/data/sxg/test.example.org_test_invalid_content_type.sxg +content/test/data/sxg/test.example.org_test_invalid_magic_string.sxg +content/test/data/sxg/test.example.org_test_missing_nosniff.sxg +content/test/data/sxg/test.example.org_vary_cookie.sxg +content/test/data/web_bundle/broken_bundle_base_b2.wbn +content/test/data/web_bundle/broken_bundle_broken_first_entry_b2.wbn +content/test/data/web_bundle/broken_bundle_broken_script_entry_b2.wbn +content/test/data/web_bundle/cross_origin_b2.wbn +content/test/data/web_bundle/foo_base_url_bundle_b2.wbn +content/test/data/web_bundle/foo_primary_url_bundle_b2.wbn +content/test/data/web_bundle/same_origin_b2.wbn +content/test/data/web_bundle/uuid-in-package.wbn +content/test/data/web_bundle/web_bundle_browsertest_b2.wbn device/fido/response_data_fuzzer_corpus/get_assertion_response_corpus device/fido/response_data_fuzzer_corpus/make_credential_response_corpus device/fido/response_data_fuzzer_corpus/register1 @@ -259,29 +330,6 @@ fuchsia_web/webengine/test/data/bear-vp8a.webm fuchsia_web/webengine/test/data/bear-vp9-opus.webm fuchsia_web/webengine/test/data/bear-vp9.webm headless/test/data/protocol/emulation/resources/video.webm -ios/chrome/app/profile/profile_controller.mm -ios/chrome/browser/crash_report/model/seed_corpuses/process_snapshot_dump_1 -ios/chrome/browser/crash_report/model/seed_corpuses/process_snapshot_dump_2 -ios/chrome/browser/crash_report/model/seed_corpuses/process_snapshot_dump_3 -ios/chrome/browser/crash_report/model/seed_corpuses/process_snapshot_dump_4 -ios/chrome/browser/omnibox/ui_bundled/popup/carousel/omnibox_popup_carousel_control.mm -ios/chrome/browser/omnibox/ui_bundled/popup/omnibox_popup_view_controller.mm -ios/chrome/browser/omnibox/ui_bundled/popup/row/omnibox_popup_row_content_configuration.mm -ios/chrome/browser/sessions/model/session_restoration_web_state_list_observer.mm -ios/chrome/test/data/optimization_guide/additional_file_doesnt_exist.crx3 -ios/chrome/test/data/optimization_guide/additional_file_exists.crx3 -ios/chrome/test/data/optimization_guide/signed_valid_model.crx3 -ios/chrome/test/data/sessions/corrupted.plist -ios/chrome/test/data/sessions/session_m57.plist -ios/chrome/test/data/sessions/session_m58.plist -ios/chrome/test/data/voice/test_sound.m4a -ios/testing/data/http_server_files/bundle.pkpasses -ios/testing/data/http_server_files/generic.pkpass -ios/testing/data/http_server_files/redchair.usdz -ios/testing/data/http_server_files/semi_bundle.pkpasses -ios/testing/data/http_server_files/video_sample.mov -ios/web/navigation/navigation_item_impl_unittest.mm -ios/web/test/data/testpass.pkpass media/formats/mp4/h264_annex_b_fuzz_corpus/chunk1-config-idr.bin media/formats/mp4/h264_annex_b_fuzz_corpus/chunk2-non-idr.bin media/formats/mp4/h264_annex_b_fuzz_corpus/chunk3-non-idr.bin @@ -7016,7 +7064,6 @@ third_party/catapult/third_party/vinn/third_party/parse5/test/data/tree_construc third_party/catapult/third_party/vinn/third_party/parse5/test/data/tree_construction/plain-text-unsafe.dat third_party/catapult/tracing/skp_data/google_homepage.skp third_party/catapult/tracing/skp_data/lthi_cats.skp -third_party/closure_compiler/compiler/compiler.jar third_party/cpuinfo/src/test/dmesg/alldocube-iwork8.log third_party/cpuinfo/src/test/dmesg/galaxy-a3-2016-eu.log third_party/cpuinfo/src/test/dmesg/galaxy-grand-prime-value-edition.log diff --git a/utils/_extraction.py b/utils/_extraction.py index 93a735fa..ab93107a 100644 --- a/utils/_extraction.py +++ b/utils/_extraction.py @@ -14,7 +14,6 @@ import tarfile from pathlib import Path, PurePosixPath from _common import (USE_REGISTRY, PlatformEnum, ExtractorEnum, get_logger, get_running_platform) -from prune_binaries import CONTINGENT_PATHS DEFAULT_EXTRACTORS = { ExtractorEnum.SEVENZIP: USE_REGISTRY, @@ -88,7 +87,7 @@ def _process_relative_to(unpack_root, relative_to): relative_root.rmdir() -def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unused, sysroot): +def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to): get_logger().debug('Using 7-zip extractor') if not relative_to is None and (output_dir / relative_to).exists(): get_logger().error('Temporary unpacking directory already exists: %s', @@ -96,11 +95,6 @@ def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unu raise Exception() cmd1 = (binary, 'x', str(archive_path), '-so') cmd2 = (binary, 'x', '-si', '-aoa', '-ttar', '-o{}'.format(str(output_dir))) - if skip_unused: - for cpath in CONTINGENT_PATHS: - if sysroot and f'{sysroot}-sysroot' in cpath: - continue - cmd2 += ('-x!%s/%s' % (str(relative_to), cpath[:-1]), ) get_logger().debug('7z command line: %s | %s', ' '.join(cmd1), ' '.join(cmd2)) proc1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE) @@ -116,15 +110,10 @@ def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unu _process_relative_to(output_dir, relative_to) -def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to, skip_unused, sysroot): +def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to): get_logger().debug('Using BSD or GNU tar extractor') output_dir.mkdir(exist_ok=True) cmd = (binary, '-xf', str(archive_path), '-C', str(output_dir)) - if skip_unused: - for cpath in CONTINGENT_PATHS: - if sysroot and f'{sysroot}-sysroot' in cpath: - continue - cmd += ('--exclude=%s/%s' % (str(relative_to), cpath[:-1]), ) get_logger().debug('tar command line: %s', ' '.join(cmd)) result = subprocess.run(cmd, check=False) if result.returncode != 0: @@ -136,15 +125,10 @@ def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to, skip_un _process_relative_to(output_dir, relative_to) -def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to, skip_unused, sysroot): +def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to): get_logger().debug('Using WinRAR extractor') output_dir.mkdir(exist_ok=True) cmd = (binary, 'x', '-o+', str(archive_path), str(output_dir)) - if skip_unused: - for cpath in CONTINGENT_PATHS: - if sysroot and f'{sysroot}-sysroot' in cpath: - continue - cmd += ('-x%s%s%s' % (str(relative_to), os.sep, cpath[:-1].replace('/')), ) get_logger().debug('WinRAR command line: %s', ' '.join(cmd)) result = subprocess.run(cmd, check=False) if result.returncode != 0: @@ -154,7 +138,7 @@ def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to, skip _process_relative_to(output_dir, relative_to) -def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, sysroot): +def _extract_tar_with_python(archive_path, output_dir, relative_to): get_logger().debug('Using pure Python tar extractor') class NoAppendList(list): @@ -181,12 +165,6 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, tar_file_obj.members = NoAppendList() for tarinfo in tar_file_obj: try: - if skip_unused and [ - cpath for cpath in CONTINGENT_PATHS - if tarinfo.name.startswith(str(relative_to) + '/' + cpath) - and not (sysroot and f'{sysroot}-sysroot' in cpath) - ]: - continue if relative_to is None: destination = output_dir / PurePosixPath(tarinfo.name) else: @@ -210,7 +188,7 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, raise -def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot, extractors=None): +def extract_tar_file(archive_path, output_dir, relative_to, extractors=None): """ Extract regular or compressed tar archive into the output directory. @@ -233,8 +211,7 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot sevenzip_cmd = str(_find_7z_by_registry()) sevenzip_bin = _find_extractor_by_cmd(sevenzip_cmd) if sevenzip_bin is not None: - _extract_tar_with_7z(sevenzip_bin, archive_path, output_dir, relative_to, skip_unused, - sysroot) + _extract_tar_with_7z(sevenzip_bin, archive_path, output_dir, relative_to) return # Use WinRAR if 7-zip is not found @@ -243,8 +220,7 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot winrar_cmd = str(_find_winrar_by_registry()) winrar_bin = _find_extractor_by_cmd(winrar_cmd) if winrar_bin is not None: - _extract_tar_with_winrar(winrar_bin, archive_path, output_dir, relative_to, skip_unused, - sysroot) + _extract_tar_with_winrar(winrar_bin, archive_path, output_dir, relative_to) return get_logger().warning( 'Neither 7-zip nor WinRAR were found. Falling back to Python extractor...') @@ -252,17 +228,16 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot # NOTE: 7-zip isn't an option because it doesn't preserve file permissions tar_bin = _find_extractor_by_cmd(extractors.get(ExtractorEnum.TAR)) if not tar_bin is None: - _extract_tar_with_tar(tar_bin, archive_path, output_dir, relative_to, skip_unused, - sysroot) + _extract_tar_with_tar(tar_bin, archive_path, output_dir, relative_to) return else: # This is not a normal code path, so make it clear. raise NotImplementedError(current_platform) # Fallback to Python-based extractor on all platforms - _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, sysroot) + _extract_tar_with_python(archive_path, output_dir, relative_to) -def extract_with_7z(archive_path, output_dir, relative_to, skip_unused, sysroot, extractors=None): +def extract_with_7z(archive_path, output_dir, relative_to, extractors=None): """ Extract archives with 7-zip into the output directory. Only supports archives with one layer of unpacking, so compressed tar archives don't work. @@ -292,11 +267,6 @@ def extract_with_7z(archive_path, output_dir, relative_to, skip_unused, sysroot, output_dir / relative_to) raise Exception() cmd = (sevenzip_bin, 'x', str(archive_path), '-aoa', '-o{}'.format(str(output_dir))) - if skip_unused: - for cpath in CONTINGENT_PATHS: - if sysroot and f'{sysroot}-sysroot' in cpath: - continue - cmd += ('-x!%s/%s' % (str(relative_to), cpath[:-1]), ) get_logger().debug('7z command line: %s', ' '.join(cmd)) result = subprocess.run(cmd, check=False) @@ -307,12 +277,7 @@ def extract_with_7z(archive_path, output_dir, relative_to, skip_unused, sysroot, _process_relative_to(output_dir, relative_to) -def extract_with_winrar(archive_path, - output_dir, - relative_to, - skip_unused, - sysroot, - extractors=None): +def extract_with_winrar(archive_path, output_dir, relative_to, extractors=None): """ Extract archives with WinRAR into the output directory. Only supports archives with one layer of unpacking, so compressed tar archives don't work. @@ -340,11 +305,6 @@ def extract_with_winrar(archive_path, output_dir / relative_to) raise Exception() cmd = (winrar_bin, 'x', '-o+', str(archive_path), str(output_dir)) - if skip_unused: - for cpath in CONTINGENT_PATHS: - if sysroot and f'{sysroot}-sysroot' in cpath: - continue - cmd += ('-x%s%s%s' % (str(relative_to), os.sep, cpath[:-1].replace('/', os.sep)), ) get_logger().debug('WinRAR command line: %s', ' '.join(cmd)) result = subprocess.run(cmd, check=False) diff --git a/utils/clone.py b/utils/clone.py index 8d711b18..92082064 100755 --- a/utils/clone.py +++ b/utils/clone.py @@ -18,7 +18,6 @@ from stat import S_IWRITE from subprocess import run from _common import add_common_params, get_chromium_version, get_logger -from prune_binaries import CONTINGENT_PATHS # Config file for gclient # Instances of 'src' replaced with UC_OUT, which will be replaced with the output directory @@ -233,72 +232,6 @@ def clone(args): # pylint: disable=too-many-branches, too-many-locals, too-many- str(args.output / 'tools' / 'gn' / 'bootstrap')) get_logger().info('Removing uneeded files') - # Match removals for the tarball: - # https://source.chromium.org/chromium/chromium/tools/build/+/main:recipes/recipe_modules/chromium/resources/export_tarball.py - remove_dirs = ( - (args.output / 'base' / 'tracing' / 'test' / 'data'), - (args.output / 'chrome' / 'test' / 'data'), - (args.output / 'components' / 'test' / 'data'), - (args.output / 'content' / 'test' / 'data'), - (args.output / 'courgette' / 'testdata'), - (args.output / 'extensions' / 'test' / 'data'), - (args.output / 'media' / 'test' / 'data'), - (args.output / 'native_client' / 'src' / 'trusted' / 'service_runtime' / 'testdata'), - (args.output / 'testing' / 'libfuzzer' / 'fuzzers' / 'wasm_corpus'), - (args.output / 'third_party' / 'blink' / 'perf_tests'), - (args.output / 'third_party' / 'blink' / 'tools'), - (args.output / 'third_party' / 'blink' / 'web_tests'), - (args.output / 'third_party' / 'breakpad' / 'breakpad' / 'src' / 'processor' / 'testdata'), - (args.output / 'third_party' / 'catapult' / 'tracing' / 'test_data'), - (args.output / 'third_party' / 'dawn' / 'test'), - (args.output / 'third_party' / 'expat' / 'src' / 'testdata'), - (args.output / 'third_party' / 'harfbuzz-ng' / 'src' / 'test'), - (args.output / 'third_party' / 'hunspell' / 'tests'), - (args.output / 'third_party' / 'hunspell_dictionaries'), - (args.output / 'third_party' / 'jdk' / 'current'), - (args.output / 'third_party' / 'jdk' / 'extras'), - (args.output / 'third_party' / 'liblouis' / 'src' / 'tests' / 'braille-specs'), - (args.output / 'third_party' / 'llvm' / 'llvm' / 'test'), - (args.output / 'third_party' / 'ots' / 'src' / 'tests' / 'fonts'), - (args.output / 'third_party' / 'rust-src' / 'src' / 'gcc' / 'gcc' / 'testsuite'), - (args.output / 'third_party' / 'rust-src' / 'src' / 'llvm-project' / 'clang' / 'test'), - (args.output / 'third_party' / 'rust-src' / 'src' / 'llvm-project' / 'llvm' / 'test'), - (args.output / 'third_party' / 'screen-ai' / 'linux' / 'resources'), - (args.output / 'third_party' / 'sqlite' / 'src' / 'test'), - (args.output / 'third_party' / 'swiftshader' / 'tests' / 'regres'), - (args.output / 'third_party' / 'test_fonts' / 'test_fonts'), - (args.output / 'third_party' / 'xdg-utils' / 'tests'), - (args.output / 'tools' / 'perf' / 'testdata'), - (args.output / 'v8' / 'test'), - ) - keep_files = ( - (args.output / 'chrome' / 'test' / 'data' / 'webui' / 'i18n_process_css_test.html'), - (args.output / 'chrome' / 'test' / 'data' / 'webui' / 'mojo' / 'foobar.mojom'), - (args.output / 'chrome' / 'test' / 'data' / 'webui' / 'web_ui_test.mojom'), - (args.output / 'v8' / 'test' / 'torque' / 'test-torque.tq'), - ) - keep_suffix = ('.gn', '.gni', '.grd', '.gyp', '.isolate', '.pydeps') - # Include Contingent Paths - for cpath in CONTINGENT_PATHS: - if args.sysroot and f'{args.sysroot}-sysroot' in cpath: - continue - remove_dirs += (args.output / Path(cpath), ) - for remove_dir in remove_dirs: - for path in sorted(remove_dir.rglob('*'), key=lambda l: len(str(l)), reverse=True): - if path.is_file() and path not in keep_files and path.suffix not in keep_suffix: - try: - path.unlink() - # read-only files can't be deleted on Windows - # so remove the flag and try again. - except PermissionError: - path.chmod(S_IWRITE) - path.unlink() - elif path.is_dir() and not any(path.iterdir()): - try: - path.rmdir() - except PermissionError: - path.chmod(S_IWRITE) - path.rmdir() for path in sorted(args.output.rglob('*'), key=lambda l: len(str(l)), reverse=True): if not path.is_symlink() and '.git' not in path.parts: if path.is_file() and (('out' in path.parts and 'node_modules' not in path.parts) diff --git a/utils/domain_substitution.py b/utils/domain_substitution.py index 91d3034f..0aa5f577 100755 --- a/utils/domain_substitution.py +++ b/utils/domain_substitution.py @@ -284,7 +284,7 @@ def revert_substitution(domainsub_cache, source_tree): dir=str(resolved_tree)) as tmp_extract_name: extract_path = Path(tmp_extract_name) get_logger().debug('Extracting domain substitution cache...') - extract_tar_file(domainsub_cache, extract_path, None, False, None) + extract_tar_file(domainsub_cache, extract_path, None) # Validate source tree file hashes match get_logger().debug('Validating substituted files in source tree...') diff --git a/utils/downloads.py b/utils/downloads.py index b3b2bf72..f2b3b55c 100755 --- a/utils/downloads.py +++ b/utils/downloads.py @@ -332,13 +332,7 @@ def check_downloads(download_info, cache_dir, components, chunk_bytes=262144): raise HashMismatchError(download_path) -def unpack_downloads(download_info, - cache_dir, - components, - output_dir, - skip_unused, - sysroot, - extractors=None): +def unpack_downloads(download_info, cache_dir, components, output_dir, extractors=None): """ Unpack downloads in the downloads cache to output_dir. Assumes all downloads are retrieved. @@ -346,8 +340,6 @@ def unpack_downloads(download_info, cache_dir is the pathlib.Path directory containing the download cache components is a list of component names to unpack, if not empty. output_dir is the pathlib.Path directory to unpack the downloads to. - skip_unused is a boolean that determines if unused paths should be extracted. - sysroot is a string containing a sysroot to unpack if any. extractors is a dictionary of PlatformEnum to a command or path to the extractor binary. Defaults to 'tar' for tar, and '_use_registry' for 7-Zip and WinRAR. @@ -377,8 +369,6 @@ def unpack_downloads(download_info, extractor_func(archive_path=download_path, output_dir=output_dir / Path(download_properties.output_path), relative_to=strip_leading_dirs_path, - skip_unused=skip_unused, - sysroot=sysroot, extractors=extractors) @@ -409,6 +399,9 @@ def _retrieve_callback(args): def _unpack_callback(args): + if args.skip_unused or args.sysroot: + get_logger().warning('The --skip-unused and --sysroot flags for downloads.py are' + ' no longer functional and will be removed in the future.') extractors = { ExtractorEnum.SEVENZIP: args.sevenz_path, ExtractorEnum.WINRAR: args.winrar_path, @@ -416,8 +409,7 @@ def _unpack_callback(args): } info = DownloadInfo(args.ini) info.check_sections_exist(args.components) - unpack_downloads(info, args.cache, args.components, args.output, args.skip_unused, args.sysroot, - extractors) + unpack_downloads(info, args.cache, args.components, args.output, extractors) def main(): @@ -479,13 +471,8 @@ def main(): help=('Command or path to WinRAR\'s "winrar" binary. If "_use_registry" is ' 'specified, determine the path from the registry. Default: %(default)s')) unpack_parser.add_argument('output', type=Path, help='The directory to unpack to.') - unpack_parser.add_argument('--skip-unused', - action='store_true', - help='Skip extraction of unused directories (CONTINGENT_PATHS).') - unpack_parser.add_argument('--sysroot', - choices=('amd64', 'i386'), - help=('Extracts the sysroot for the given architecture ' - 'when --skip-unused is set.')) + unpack_parser.add_argument('--skip-unused', action='store_true', help='Deprecated') + unpack_parser.add_argument('--sysroot', choices=('amd64', 'i386'), help='Deprecated') unpack_parser.set_defaults(callback=_unpack_callback) args = parser.parse_args() diff --git a/utils/prune_binaries.py b/utils/prune_binaries.py index 82e1d564..d0e90f0d 100755 --- a/utils/prune_binaries.py +++ b/utils/prune_binaries.py @@ -16,37 +16,97 @@ from pathlib import Path from _common import ENCODING, get_logger, add_common_params # List of paths to prune if they exist, excluded from domain_substitution and pruning lists -# These allow the lists to be compatible between cloned and tarball sources +# These allow the lists to be compatible between cloned, tarball, and lite tarball sources CONTINGENT_PATHS = ( - # Overridable git sources - 'third_party/angle/third_party/VK-GL-CTS/src/', # CIPD sources + 'buildtools/linux64/', + 'third_party/checkstyle/cipd/', 'third_party/dawn/third_party/ninja/', 'third_party/dawn/tools/golang/', 'third_party/devtools-frontend/src/third_party/esbuild/', 'third_party/google-java-format/', + 'third_party/libei/cipd/', + 'third_party/ninja/', 'third_party/openscreen/src/third_party/ninja/', + 'third_party/siso/cipd/', 'third_party/updater/chrome_linux64/', + 'third_party/updater/chrome_linux64_sans_iid/cipd/', 'third_party/updater/chromium_linux64/', + 'third_party/updater/chromium_linux64_sans_iid/cipd/', + 'tools/luci-go/', + 'tools/resultdb/', + 'tools/skia_goldctl/linux/', # GCS sources - 'build/linux/debian_bullseye_amd64-sysroot/', - 'build/linux/debian_bullseye_i386-sysroot/', + 'buildtools/linux64-format/', + 'third_party/blink/renderer/core/css/perftest_data/', 'third_party/js_code_coverage/', 'third_party/openscreen/src/buildtools/linux64/format/', + 'third_party/opus/tests/resources/', + 'third_party/subresource-filter-ruleset/data/', + 'tools/perf/page_sets/maps_perf_test/dataset/', + # Sysroots(GCS), include all arches the clone script is able to obtain + 'build/linux/debian_bullseye_amd64-sysroot/', + 'build/linux/debian_bullseye_arm64-sysroot/', + 'build/linux/debian_bullseye_armhf-sysroot/', + 'build/linux/debian_bullseye_i386-sysroot/', + 'build/linux/debian_bullseye_mips64el-sysroot/', + 'build/linux/debian_bullseye_mipsel-sysroot/', # other 'third_party/depot_tools/external_bin/', + # Match removals for the tarball: + # https://source.chromium.org/chromium/chromium/tools/build/+/main:recipes/recipe_modules/chromium/resources/export_tarball.py + # nonessential + 'third_party/blink/tools/', + 'third_party/blink/web_tests/', + 'third_party/hunspell_dictionaries/', + 'third_party/hunspell/tests/', + 'third_party/jdk/current/', + 'third_party/jdk/extras/', + 'third_party/liblouis/src/tests/braille-specs/', + 'third_party/xdg-utils/tests/', + 'v8/test/', + # test + 'base/tracing/test/data/', + 'chrome/test/data/', + 'components/test/data/', + 'content/test/data/accessibility/', + 'content/test/data/gpu/', + 'content/test/data/media/', + 'courgette/testdata/', + 'extensions/test/data/', + 'media/test/data/', + 'native_client/src/trusted/service_runtime/testdata/', + 'testing/libfuzzer/fuzzers/wasm_corpus/', + 'third_party/blink/perf_tests/', + 'third_party/breakpad/breakpad/src/processor/testdata/', + 'third_party/catapult/tracing/test_data/', + 'third_party/dawn/test/', + 'third_party/expat/src/testdata/', + 'third_party/harfbuzz-ng/src/test/', + 'third_party/llvm/llvm/test/', + 'third_party/ots/src/tests/fonts/', + 'third_party/rust-src/src/gcc/gcc/testsuite/', + 'third_party/rust-src/src/llvm-project/clang/test/', + 'third_party/rust-src/src/llvm-project/llvm/test/', + 'third_party/screen-ai/linux/resources/', + 'third_party/sqlite/src/test/', + 'third_party/swiftshader/tests/regres/', + 'third_party/test_fonts/test_fonts/', + 'tools/perf/testdata/', # lite tarball paths: # https://source.chromium.org/chromium/chromium/tools/build/+/main:recipes/recipes/publish_tarball.py 'android_webview/', 'buildtools/reclient/', + 'chrome/android/', 'chromecast/', + 'ios/', 'native_client/', 'native_client_sdk/', 'third_party/android_platform/', + 'third_party/angle/third_party/VK-GL-CTS/', 'third_party/apache-linux/', - 'third_party/blink/manual_tests/', - 'third_party/blink/perf_tests/', 'third_party/catapult/third_party/vinn/third_party/v8/', + 'third_party/closure_compiler/', 'third_party/instrumented_libs/', 'third_party/llvm/', 'third_party/llvm-build/', @@ -55,8 +115,20 @@ CONTINGENT_PATHS = ( 'third_party/rust-src/', 'third_party/rust-toolchain/', 'third_party/webgl/', + 'third_party/blink/manual_tests/', + 'third_party/blink/perf_tests/', ) +# Files that should be excluded when pruning contingent paths. +KEEP_FILES = ( + 'chrome/test/data/webui/i18n_process_css_test.html', + 'chrome/test/data/webui/mojo/foobar.mojom', + 'v8/test/torque/test-torque.tq', +) + +# File suffixes that should be excluded when pruning contingent paths. +KEEP_SUFFIXES = ('.gn', '.gni', '.grd', '.grdp', '.isolate', '.pydeps') + def prune_files(unpack_root, prune_list): """ @@ -80,13 +152,17 @@ def prune_files(unpack_root, prune_list): return unremovable_files -def _prune_path(path): +def _prune_path(path, unpack_root=None): """ - Delete all files and directories in path. + Delete files and directories in path. path is a pathlib.Path to the directory to be pruned + unpack_root is a pathlib.Path to the source tree """ for node in sorted(path.rglob('*'), key=lambda l: len(str(l)), reverse=True): + if unpack_root is not None and (node.suffix in KEEP_SUFFIXES + or str(node.relative_to(unpack_root)) in KEEP_FILES): + continue if node.is_file() or node.is_symlink(): try: node.unlink() @@ -120,7 +196,7 @@ def prune_dirs(unpack_root, keep_contingent_paths, sysroot): get_logger().info('%s: %s', 'Exempt', cpath) continue get_logger().info('%s: %s', 'Exists' if Path(cpath).exists() else 'Absent', cpath) - _prune_path(unpack_root / cpath) + _prune_path(unpack_root / cpath, unpack_root) def _callback(args):