From c95f8dd784965019b1bc163d607794f80d39b7d3 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 31 May 2024 13:48:14 +0200 Subject: [PATCH 1/6] mkosi: Drop kernel command line to enable log context The log context is already enabled by virtue of the log level being set to debug, so no need to enable it explicitly. --- mkosi.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/mkosi.conf b/mkosi.conf index 45102ac7af..1f9e987e0b 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -36,8 +36,6 @@ KernelCommandLine=systemd.crash_shell enforcing=0 systemd.early_core_pattern=/core systemd.firstboot=no - systemd.setenv=SYSTEMD_ENABLE_LOG_CONTEXT=yes - SYSTEMD_ENABLE_LOG_CONTEXT=yes raid=noautodetect oops=panic panic=-1 From 5d2a40ab11fb7661c9361c438d4447bb2109f1fe Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 31 May 2024 17:26:01 +0200 Subject: [PATCH 2/6] mkosi: Add note about kernel command line limit --- mkosi.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkosi.conf b/mkosi.conf index 1f9e987e0b..300b86bf97 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -20,6 +20,8 @@ Environment=ASAN_OPTIONS=verify_asan_link_order=false @SELinuxRelabel=no BuildSourcesEphemeral=yes +# Adding more kernel command line arguments is likely to hit the kernel command line limit (512 bytes) in +# various scenarios. Consider adding support for a credential instead if possible and using that. KernelCommandLine=systemd.crash_shell systemd.log_level=debug,console:info systemd.log_ratelimit_kmsg=0 From 8919f86f573f5256283298415027b0a9052507e5 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 16 May 2024 17:18:38 +0200 Subject: [PATCH 3/6] mkosi: Sanitizer improvements - Let's set the environment on the kernel command line so it applies to initrd and main system. - Let's add the necessary wrappers that are also added in test-functions. Unlike test-functions we don't use gcc/clang to get the library path as that requires installing gcc/clang in the initrd. - Let's drop the hack to get journald writing to the console and have it write to kmsg instead. We'll get the output either way. - Stop removing libstdc++ and sanitizer libraries from Arch Linux initrds and other images as it's required by the sanitizer libraries. - Add a workaround for specifying extra meson options for opensuse - Add a leak sanitizer suppression file as a workaround for a false positive leak in verify_selinuxmnt() in libselinux. We do a soname match because the stacktrace can't be properly symbolized on Debian. --- .github/workflows/mkosi.yml | 1 + mkosi.conf | 10 +- mkosi.conf.d/20-sanitizers.conf | 19 +++ mkosi.images/exitrd/mkosi.conf.d/10-arch.conf | 5 - .../minimal-base/mkosi.conf.d/10-arch.conf | 5 - mkosi.images/system/initrd/mkosi.conf | 5 + .../system/leak-sanitizer-suppressions | 1 + mkosi.images/system/mkosi.conf | 8 ++ .../10-opensuse/mkosi.build.chroot | 4 +- .../system/iscsi-init.service.d/asan.conf | 7 + mkosi.images/system/mkosi.postinst.chroot | 42 ------ mkosi.images/system/mkosi.sanitizers.chroot | 130 ++++++++++++++++++ 12 files changed, 177 insertions(+), 60 deletions(-) create mode 100644 mkosi.conf.d/20-sanitizers.conf create mode 100644 mkosi.images/system/initrd/mkosi.conf create mode 100644 mkosi.images/system/leak-sanitizer-suppressions create mode 100644 mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf create mode 100755 mkosi.images/system/mkosi.sanitizers.chroot diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml index 32dab1a7f3..583f287de2 100644 --- a/.github/workflows/mkosi.yml +++ b/.github/workflows/mkosi.yml @@ -119,6 +119,7 @@ jobs: [Host] ToolsTree=default ToolsTreeDistribution=fedora + QemuMem=4G # We build with debuginfo so there's no point in mounting the sources into the machine. RuntimeBuildSources=no EOF diff --git a/mkosi.conf b/mkosi.conf index 300b86bf97..1c552a269e 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -10,13 +10,9 @@ MinimumVersion=23~devel @CacheDirectory=build/mkosi.cache [Content] -# Prevent ASAN warnings when building the image and ship the real ASAN options prefixed with MKOSI_. -Environment=ASAN_OPTIONS=verify_asan_link_order=false - MKOSI_ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 - MKOSI_UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 - # The kernel versions in CentOS Stream 9 and Ubuntu 22.04 don't support orphan_file, but later - # versions of mkfs.ext4 enabled it by default, so we disable it explicitly. - SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file" +# The kernel versions in CentOS Stream 9 and Ubuntu 22.04 don't support orphan_file, but later +# versions of mkfs.ext4 enabled it by default, so we disable it explicitly. +Environment=SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file" @SELinuxRelabel=no BuildSourcesEphemeral=yes diff --git a/mkosi.conf.d/20-sanitizers.conf b/mkosi.conf.d/20-sanitizers.conf new file mode 100644 index 0000000000..235b233e1a --- /dev/null +++ b/mkosi.conf.d/20-sanitizers.conf @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +[Match] +Environment=SANITIZERS + +[Content] +# Set verify_asan_link_order=0 to prevent ASAN warnings when building the image and make sure the real ASAN +# options are set when booting the image. +# Set intercept_tls_get_addr=0 to work around leak sanitizer segmentation fault in test-dlopen-so on CentOS +# Stream 9. +# TODO: Drop intercept_tls_get_addr=0 when we remove CentOS Stream 9 builds. +Environment=ASAN_OPTIONS=verify_asan_link_order=0:intercept_tls_get_addr=0 +KernelCommandLine= + ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 + systemd.setenv=ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 + UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 + systemd.setenv=UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 + LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions + systemd.setenv=LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions diff --git a/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf b/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf index 25d20887ff..c8b1904f6f 100644 --- a/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf +++ b/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf @@ -15,11 +15,6 @@ RemoveFiles= /usr/lib/libgomp.so* /usr/lib/libgphobos.so* /usr/lib/libobjc.so* - /usr/lib/libasan.so* - /usr/lib/libtsan.so* - /usr/lib/liblsan.so* - /usr/lib/libubsan.so* - /usr/lib/libstdc++.so* /usr/lib/libgdruntime.so* # Remove all files that are only required for development. diff --git a/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf b/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf index 30e8fda59e..9b033975d6 100644 --- a/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf +++ b/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf @@ -17,11 +17,6 @@ RemoveFiles= /usr/lib/libgomp.so* /usr/lib/libgphobos.so* /usr/lib/libobjc.so* - /usr/lib/libasan.so* - /usr/lib/libtsan.so* - /usr/lib/liblsan.so* - /usr/lib/libubsan.so* - /usr/lib/libstdc++.so* /usr/lib/libgdruntime.so* # Remove all files that are only required for development. diff --git a/mkosi.images/system/initrd/mkosi.conf b/mkosi.images/system/initrd/mkosi.conf new file mode 100644 index 0000000000..56bd4d0aa7 --- /dev/null +++ b/mkosi.images/system/initrd/mkosi.conf @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +[Content] +PostInstallationScripts=../mkosi.sanitizers.chroot +ExtraTrees=../leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions diff --git a/mkosi.images/system/leak-sanitizer-suppressions b/mkosi.images/system/leak-sanitizer-suppressions new file mode 100644 index 0000000000..639abb8f3f --- /dev/null +++ b/mkosi.images/system/leak-sanitizer-suppressions @@ -0,0 +1 @@ +leak:libselinux diff --git a/mkosi.images/system/mkosi.conf b/mkosi.images/system/mkosi.conf index 6455b0477e..5d33cba7ee 100644 --- a/mkosi.images/system/mkosi.conf +++ b/mkosi.images/system/mkosi.conf @@ -26,6 +26,14 @@ ExtraTrees= %O/minimal-1.root-%a-verity-sig.raw:/usr/share/minimal_1.verity.sig %O/minimal-base:/usr/share/TEST-13-NSPAWN-container-template %O/exitrd:/exitrd + leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions + +PostInstallationScripts=mkosi.sanitizers.chroot + +InitrdPackages= + findutils + grep + sed Packages= acl diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot index b2c56fda77..13cda4c9b8 100755 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot @@ -51,6 +51,8 @@ build() { IFS= # TODO: Replace meson_build and meson_install overrides with "--undefine __meson_verbose" once # https://github.com/mesonbuild/meson/pull/12835 is available. + # TODO: Replace __meson_auto_features override with meson_extra_configure_options once the suse spec + # starts to use it. # shellcheck disable=SC2046 rpmbuild \ -bb \ @@ -71,7 +73,7 @@ build() { --define "build_cflags $(rpm --eval %build_cflags) $EXTRA_CFLAGS" \ --define "meson_build %{shrink:%{__meson} compile -C %{_vpath_builddir} -j %{_smp_build_ncpus} %{nil}}" \ --define "meson_install %{shrink:DESTDIR=%{buildroot} %{__meson} install -C %{_vpath_builddir} --no-rebuild --quiet %{nil}}" \ - --define "meson_extra_configure_options -D mode=developer -D b_sanitize=${SANITIZERS:-none}" \ + --define "__meson_auto_features auto -D mode=developer -D b_sanitize=${SANITIZERS:-none}" \ --define "__os_install_post /usr/lib/rpm/brp-suse %{nil}" \ --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ --define "__script_requires %{nil}" \ diff --git a/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf b/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf new file mode 100644 index 0000000000..ebf7899a78 --- /dev/null +++ b/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +# The iscsi-init.service calls `sh` which might, in certain circumstances, pull in instrumented systemd NSS +# modules causing `sh` to fail. Avoid the issue by setting LD_PRELOAD to load the sanitizer libraries if +# needed. +[Service] +EnvironmentFile=-/usr/lib/systemd/systemd-asan-env diff --git a/mkosi.images/system/mkosi.postinst.chroot b/mkosi.images/system/mkosi.postinst.chroot index 15f268a20a..397884b720 100755 --- a/mkosi.images/system/mkosi.postinst.chroot +++ b/mkosi.images/system/mkosi.postinst.chroot @@ -2,48 +2,6 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -e -if [ -n "$SANITIZERS" ]; then - LD_PRELOAD=$(ldd /usr/lib/systemd/systemd | grep libasan.so | awk '{print $3}') - - mkdir -p /etc/systemd/system.conf.d - - cat >/etc/systemd/system.conf.d/10-asan.conf </etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf </etc/systemd/system/console-getty.service.d/10-no-vhangup.conf < 50s when built with sanitizers so let's not run it by default. - systemctl mask systemd-hwdb-update.service -fi - if command -v authselect >/dev/null; then # authselect 1.5.0 renamed the minimal profile to the local profile without keeping backwards compat so # let's use the new name if it exists. diff --git a/mkosi.images/system/mkosi.sanitizers.chroot b/mkosi.images/system/mkosi.sanitizers.chroot new file mode 100755 index 0000000000..48c5d147aa --- /dev/null +++ b/mkosi.images/system/mkosi.sanitizers.chroot @@ -0,0 +1,130 @@ +#!/bin/bash +# SPDX-License-Identifier: LGPL-2.1-or-later +set -e + +if [[ -z "$SANITIZERS" ]]; then + exit 0 +fi + +# Sanitizers log to stderr by default. However, journald's stderr is connected to /dev/null, so we lose +# all the sanitizer logs. To rectify that, let's connect journald's stdout to kmsg so that the sanitizer +# failures end up in the journal. +mkdir -p /etc/systemd/system/systemd-journald.service.d +cat >/etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf < 50s when built with sanitizers so let's not run it by default. +systemctl mask systemd-hwdb-update.service + +ASAN_RT_PATH="$(grep libasan.so < <(ldd /usr/lib/systemd/systemd) | cut -d ' ' -f 3)" +if [[ -z "$ASAN_RT_PATH" ]]; then + ASAN_RT_PATH="$(grep libclang_rt.asan < <(ldd /usr/lib/systemd/systemd) | cut -d ' ' -f 3)" + + # As clang's ASan DSO is usually in a non-standard path, let's check if + # the environment is set accordingly. If not, warn the user and exit. + # We're not setting the LD_LIBRARY_PATH automagically here, because + # user should encounter (and fix) the same issue when running the unit + # tests (meson test) + if ldd /usr/lib/systemd/systemd | grep -q "libclang_rt.asan.*not found"; then + echo >&2 "clang's ASan DSO libclang_rt.asan is not present in the runtime library path" + exit 1 + fi +fi +if [[ -z "$ASAN_RT_PATH" ]]; then + echo >&2 "systemd is not linked against the ASan DSO" + echo >&2 "gcc does this by default, for clang compile with -shared-libasan" + exit 1 +fi + +wrap=( + /usr/lib/polkit-1/polkitd + /usr/libexec/polkit-1/polkitd + agetty + btrfs + capsh + chgrp + chown + cryptsetup + curl + dbus-broker-launch + dbus-daemon + delv + dhcpd + dig + dmsetup + dnsmasq + findmnt + getent + getfacl + id + integritysetup + iscsid + kpartx + logger + login + ls + lsblk + lvm + mdadm + mkfs.btrfs + mkfs.erofs + mkfs.ext4 + mkfs.vfat + mkfs.xfs + mksquashfs + mkswap + multipath + multipathd + nvme + p11-kit + pkill + ps + setfacl + setpriv + sshd + stat + su + tar + tgtd + useradd + userdel + veritysetup +) + +for bin in "${wrap[@]}"; do + if ! command -v "$bin" >/dev/null; then + continue + fi + + if [[ "$bin" == getent ]]; then + enable_lsan=1 + else + enable_lsan=0 + fi + + target="$(command -v "$bin")" + + mv "$target" "$target.orig" + + cat >"$target" </usr/lib/systemd/systemd-asan-env < Date: Fri, 31 May 2024 17:01:27 +0200 Subject: [PATCH 4/6] TEST-69-SHUTDOWN: Order after systemd-user-sessions.service Make sure that we're allowed to login by ordering ourselves after systemd-user-sessions.service. --- test/TEST-69-SHUTDOWN/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/TEST-69-SHUTDOWN/meson.build b/test/TEST-69-SHUTDOWN/meson.build index 75c0446872..c1b1ab557f 100644 --- a/test/TEST-69-SHUTDOWN/meson.build +++ b/test/TEST-69-SHUTDOWN/meson.build @@ -4,5 +4,9 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), 'unit' : files('TEST-69-SHUTDOWN.service'), + 'configuration' : integration_test_template['configuration'] + { + 'wants' : '@0@ systemd-user-sessions.service'.format(integration_test_template['configuration']['wants']), + 'after' : '@0@ systemd-user-sessions.service'.format(integration_test_template['configuration']['after']), + }, }, ] From 605d72cb145ca792b1a78b5c554bc3922c5df0d5 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 31 May 2024 17:15:47 +0200 Subject: [PATCH 5/6] TEST-21-DFUZZER: Stop marking as slow This was a workaround to not run TEST-21-DFUZZER without sanitizers. Now that we're going to get sanitizer support, let's run it by default again. --- test/TEST-21-DFUZZER/meson.build | 1 - test/meson.build | 9 --------- 2 files changed, 10 deletions(-) diff --git a/test/TEST-21-DFUZZER/meson.build b/test/TEST-21-DFUZZER/meson.build index 29c8e434ca..932f0c5f0e 100644 --- a/test/TEST-21-DFUZZER/meson.build +++ b/test/TEST-21-DFUZZER/meson.build @@ -5,7 +5,6 @@ integration_tests += [ 'name' : fs.name(meson.current_source_dir()), 'timeout' : 3600, 'priority' : 50, - 'slow' : true, 'vm' : true, }, ] diff --git a/test/meson.build b/test/meson.build index 92edd079b4..9da211fec1 100644 --- a/test/meson.build +++ b/test/meson.build @@ -281,7 +281,6 @@ integration_test_template = { 'storage' : 'volatile', 'priority' : 0, 'firmware' : 'linux', - 'slow' : false, 'enabled' : true, 'configuration' : { 'memory-accounting' : 'no', @@ -404,10 +403,6 @@ foreach integration_test : integration_tests install_data(integration_test_unit, install_dir : testdata_dir / 'units') endif - if integration_test['slow'] - integration_test_args += ['--slow'] - endif - if integration_test['vm'] integration_test_args += ['--vm'] endif @@ -440,10 +435,6 @@ foreach integration_test : integration_tests integration_test_env += {'SYSTEMD_INTEGRATION_TESTS': '1'} endif - if want_slow_tests - integration_test_env += {'SYSTEMD_SLOW_TESTS': '1'} - endif - if not integration_test['enabled'] continue endif From ebec3c88c3228cfbdc30bfc9cc9d5e24a6e167bc Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 16 May 2024 17:20:24 +0200 Subject: [PATCH 6/6] ci: Build Fedora rawhide with sanitizers in mkosi Let's make sure one build has sanitizers enabled for extra coverage. --- .github/workflows/mkosi.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml index 583f287de2..e4e233b236 100644 --- a/.github/workflows/mkosi.yml +++ b/.github/workflows/mkosi.yml @@ -56,18 +56,25 @@ jobs: include: - distro: arch release: rolling + sanitizers: "" - distro: debian release: testing + sanitizers: "" - distro: ubuntu release: noble + sanitizers: "" - distro: fedora release: "40" + sanitizers: "" - distro: fedora release: rawhide + sanitizers: address,undefined - distro: opensuse release: tumbleweed + sanitizers: "" - distro: centos release: "9" + sanitizers: "" steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 @@ -115,6 +122,7 @@ jobs: WITH_DEBUG=1 # Enabling optimizations significantly speeds up integration tests. OPTIMIZATION=g + SANITIZERS=${{ matrix.sanitizers }} [Host] ToolsTree=default