test: add test for systemd upgrade

Add a basic test to check that systemd works after an upgrade

mkosi/mkosi.conf.d/centos-fedora/mkosi.conf

@@ -51,6 +51,8 @@ Packages=
         polkit
         procps-ng
         python3-pexpect
+        # needed to upgrade and downgrade systemd-ukify in tests
+        python3-zstd
         quota
         rpm
         softhsm

mkosi/mkosi.conf.d/centos-fedora/mkosi.prepare

@@ -0,0 +1,34 @@
+#!/bin/bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -e
+
+if [[ "$1" == "build" ]]; then
+    exit 0
+fi
+
+pkgdir=/usr/host-pkgs
+distrover=$(dnf repoquery --disablerepo mkosi --latest-limit=1 --queryformat='%{evr}' systemd)
+develver=$(rpm -q --queryformat='%{evr}' systemd)
+distromajor=$(dnf repoquery --disablerepo mkosi --latest-limit=1 --queryformat='%{version}' systemd)
+develmajor=$(rpm -q --queryformat='%{v}' systemd)
+
+distromajor=$(grep -o '^[0-9]\+' <<<"$distromajor")
+develmajor=$(grep -o '^[0-9]\+' <<<"$develmajor")
+deltaver=$((develmajor - distromajor))
+
+if [[ $deltaver -gt 2 ]]; then
+    echo "Skipping upgrade test, distro version is too old: $distrover"
+    exit 0
+elif [[ $deltaver -le 0 ]]; then
+    echo "Skipping upgrade test, distro version is newer or equal: $distrover"
+    exit 0
+fi
+
+# Filter only noarch and arch packages otherwise on CentOS 9 we could end having i686 packages too
+distropkgs=$(dnf repoquery 'systemd*' | grep -F "$distrover" | grep -e noarch -e "$DISTRIBUTION_ARCHITECTURE" | grep -v -e -debuginfo -e -debugsource -e standalone)
+develpkgs=$(dnf repoquery 'systemd*' | grep -F "$develver" | grep -e noarch -e "$DISTRIBUTION_ARCHITECTURE" | grep -v -e -debuginfo -e -debugsource -e standalone)
+
+# shellcheck disable=SC2086
+dnf download --arch noarch --arch "$DISTRIBUTION_ARCHITECTURE" --destdir "$BUILDROOT$pkgdir/distro" $distropkgs
+# shellcheck disable=SC2086
+dnf download --arch noarch --arch "$DISTRIBUTION_ARCHITECTURE" --destdir "$BUILDROOT$pkgdir/devel" $develpkgs

mkosi/mkosi.sanitizers/mkosi.postinst

@@ -10,7 +10,16 @@ if [[ ! -f "$BUILDROOT/$LIBSYSTEMD" ]]; then
 fi
 
 # ASAN and syscall filters aren't compatible with each other.
-find "$BUILDROOT"/usr "$BUILDROOT"/etc -name '*.service' -type f -exec sed -i 's/^\(MemoryDeny\|SystemCall\)/# \1/' {} +
+find "$BUILDROOT"/usr "$BUILDROOT"/etc -name '*.service' -type f | while read -r unit; do
+    if grep -q -e MemoryDeny -e SystemCall "$unit" ; then
+        mkdir -p "$unit.d"
+        cat > "$unit.d/sanitizer-compat.conf" <<EOF
+[Service]
+MemoryDenyWriteExecute=no
+SystemCallFilter=
+EOF
+    fi
+done
 
 # 'systemd-hwdb update' takes > 50s when built with sanitizers so let's not run it by default.
 systemctl --root="$BUILDROOT" mask systemd-hwdb-update.service

test/integration-tests/TEST-88-UPGRADE/meson.build

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+integration_tests += [
+        integration_test_template + {
+                'name' : fs.name(meson.current_source_dir()),
+        },
+]

test/integration-tests/meson.build

@@ -100,6 +100,7 @@ foreach dirname : [
         'TEST-85-NETWORK',
         'TEST-86-MULTI-PROFILE-UKI',
         'TEST-87-AUX-UTILS-VM',
+        'TEST-88-UPGRADE',
 ]
         subdir(dirname)
 endforeach

test/units/TEST-88-UPGRADE.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
+# ex: ts=8 sw=4 sts=4 et filetype=sh
+set -eux
+set -o pipefail
+
+pkgdir=/usr/host-pkgs
+
+if ! [[ -d $pkgdir ]]; then
+    echo "Distro packages not found in $pkgdir" >/skipped
+    exit 77
+fi
+
+if ! command -v dnf >/dev/null; then
+    echo 'dnf not found, skipping test.' >/skipped
+    exit 77
+fi
+
+minor=$(systemctl --version | awk '/^systemd/{print$2}')
+networkd=
+unitscmd='systemctl list-units --failed *systemd*'
+
+if [[ $($unitscmd --output json | jq length) -gt 0 ]]; then
+    echo 'Systemd failed units found before the test:'
+    $unitscmd
+    $unitscmd --output json | jq -r '.[].unit' >/tmp/failed-units
+fi
+
+check_sd() {
+    local unit fail=0
+    for unit in $($unitscmd --output json | jq -r '.[].unit'); do
+        if ! grep -sxqF "$unit" /tmp/failed-units; then
+            fail=1
+            systemctl status "$unit"
+        fi
+    done
+
+    if [[ $fail -eq 1 ]]; then
+        echo 'Systemd units above failed after the test!'
+    fi
+
+    if [[ -n $networkd ]]; then
+        if ! networkctl status; then
+            echo 'Networkd failed after the test!'
+            fail=1
+        fi
+    fi
+
+    if ! loginctl list-sessions; then
+        echo 'Loginctl failed after the test!'
+        fail=1
+    fi
+
+    [[ $fail -eq 0 ]]
+}
+
+# Copy the unit in /run so systemd finds it after the downgrade
+cp /usr/lib/systemd/tests/testdata/units/TEST-88-UPGRADE.service /run/systemd/system
+
+dnf downgrade -y --allowerasing --disablerepo '*' "$pkgdir"/distro/*.rpm
+
+# Some distros don't ship networkd, so the test will always fail
+if command -v networkctl >/dev/null; then
+    networkd=1
+fi
+
+newminor=$(systemctl --version | awk '/^systemd/{print$2}')
+
+if [[ $newminor -lt $minor ]]; then
+    echo "Downgrade to $newminor was successful."
+else
+    echo "Downgrade failed. Current version is still $newminor."
+    exit 1
+fi
+
+# TODO: sanity checks
+
+check_sd
+
+# Finally test the upgrade
+dnf -y upgrade --disablerepo '*' "$pkgdir"/devel/*.rpm
+
+# TODO: sanity checks
+check_sd
+
+touch /testok