mkosi: Various improvements and fixes (#36931)

.github/workflows/coverage.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@03d1a1b4cdc474c55166a2f48be244748011206d
+      - uses: systemd/mkosi@660874417bbe355121d705d2a049af39ddf28b58
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location

.github/workflows/mkosi.yml

@@ -120,7 +120,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@03d1a1b4cdc474c55166a2f48be244748011206d
+      - uses: systemd/mkosi@660874417bbe355121d705d2a049af39ddf28b58
 
       # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
       # immediately, we remove the files in the background. However, we first move them to a different location

.packit.yml

@@ -19,7 +19,7 @@ actions:
   post-upstream-clone:
     # Use the Fedora Rawhide specfile
     - git clone https://src.fedoraproject.org/rpms/systemd .packit_rpm
-    - bash -c 'git -C .packit_rpm checkout "$(grep GIT_COMMIT= mkosi.conf.d/10-centos-fedora/mkosi.conf | cut -d= -f2)"'
+    - bash -c 'git -C .packit_rpm checkout "$(grep GIT_COMMIT= mkosi.conf.d/centos-fedora/mkosi.conf.d/pkgenv.conf | cut -d= -f2)"'
     - bash -c 'echo "%bcond upstream 1" > .packit_rpm/systemd.spec.new'
     - bash -c 'echo "%define meson_extra_configure_options --werror" >> .packit_rpm/systemd.spec.new'
     - bash -c 'cat .packit_rpm/systemd.spec >> .packit_rpm/systemd.spec.new'

mkosi.conf

@@ -23,11 +23,6 @@ PassEnvironment=
         ASAN_OPTIONS
         COVERAGE
         VCS_TAG
-        GIT_URL
-        GIT_SUBDIR
-        GIT_BRANCH
-        GIT_COMMIT
-        PKG_SUBDIR
 
 [Output]
 RepartDirectories=mkosi.repart

mkosi.conf.d/05-tools/mkosi.conf.d/arch.conf

@@ -1,14 +0,0 @@
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-[Match]
-ToolsTreeDistribution=arch
-
-[Build]
-ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/arch/mkosi.prepare
-ToolsTreePackages=
-        base-devel
-        github-cli
-        lcov
-        mypy
-        ruff
-        shellcheck

mkosi.conf.d/05-tools/mkosi.conf.d/centos-fedora.conf

@@ -1,11 +0,0 @@
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-[Match]
-ToolsTreeDistribution=|fedora
-ToolsTreeDistribution=|centos
-
-[Build]
-ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.prepare
-ToolsTreePackages=
-        python3-mypy
-        rpm-build

mkosi.conf.d/05-tools/mkosi.conf.d/debian-ubuntu.conf

@@ -1,13 +0,0 @@
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-[Match]
-ToolsTreeDistribution=|debian
-ToolsTreeDistribution=|ubuntu
-
-[Build]
-ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/debian-ubuntu/mkosi.prepare
-ToolsTreePackages=
-        gh
-        lcov
-        mypy
-        shellcheck

mkosi.conf.d/05-tools/mkosi.conf.d/opensuse.conf

@@ -1,14 +0,0 @@
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-[Match]
-ToolsTreeDistribution=opensuse
-
-[Build]
-ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/opensuse/mkosi.prepare
-ToolsTreePackages=
-        gh
-        lcov
-        mypy
-        python3-ruff
-        rpm-build
-        ShellCheck

mkosi.conf.d/10-centos/mkosi.conf.d/10-orphan-file.conf

@@ -1,13 +0,0 @@
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-[Match]
-ToolsTreeDistribution=|!centos
-ToolsTreeRelease=|!9
-
-[Build]
-# The kernel versions in CentOS Stream 9 doesn't support orphan_file, but later versions of
-# mkfs.ext4 enabled it by default, so we disable it explicitly. Except that older versions
-# of mkfs.ext4 don't know about "orphan_file" so we skip it on some tools tree variants where
-# we know "orphan_file" is not supported.
-Environment=
-        SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file"

mkosi.conf.d/arch/mkosi.conf

@@ -3,13 +3,6 @@
 [Match]
 Distribution=arch
 
-[Build]
-Environment=
-        GIT_URL=https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git
-        GIT_BRANCH=main
-        GIT_COMMIT=1b4994657df1c7495607f0c2f20db73550b94f34
-        PKG_SUBDIR=arch
-
 [Content]
 VolatilePackages=
         systemd

mkosi.conf.d/arch/mkosi.conf.d/pkgenv.conf

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=arch
+
+[Build]
+Environment=
+        GIT_URL=https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git
+        GIT_BRANCH=main
+        GIT_COMMIT=1b4994657df1c7495607f0c2f20db73550b94f34
+        PKG_SUBDIR=arch

mkosi.conf.d/centos-fedora/mkosi.conf

@@ -4,13 +4,6 @@
 Distribution=|centos
 Distribution=|fedora
 
-[Build]
-Environment=
-        GIT_URL=https://src.fedoraproject.org/rpms/systemd.git
-        GIT_BRANCH=rawhide
-        GIT_COMMIT=617952132de4ed882b9b00ceeed077f843130f91
-        PKG_SUBDIR=fedora
-
 [Content]
 VolatilePackages=
         systemd

mkosi.conf.d/centos-fedora/mkosi.conf.d/pkgenv.conf

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=|centos
+Distribution=|fedora
+
+[Build]
+Environment=
+        GIT_URL=https://src.fedoraproject.org/rpms/systemd.git
+        GIT_BRANCH=rawhide
+        GIT_COMMIT=617952132de4ed882b9b00ceeed077f843130f91
+        PKG_SUBDIR=fedora

mkosi.conf.d/debian-ubuntu/mkosi.conf

@@ -4,14 +4,6 @@
 Distribution=|debian
 Distribution=|ubuntu
 
-[Build]
-Environment=
-        GIT_URL=https://salsa.debian.org/systemd-team/systemd.git
-        GIT_SUBDIR=debian
-        GIT_BRANCH=debian/master
-        GIT_COMMIT=d8c7f8f7f461b1edc3bf5040509e697ea574c990
-        PKG_SUBDIR=debian
-
 [Content]
 VolatilePackages=
         libnss-myhostname

mkosi.conf.d/debian-ubuntu/mkosi.conf.d/pkgenv.conf

@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=|debian
+Distribution=|ubuntu
+
+[Build]
+Environment=
+        GIT_URL=https://salsa.debian.org/systemd-team/systemd.git
+        GIT_SUBDIR=debian
+        GIT_BRANCH=debian/master
+        GIT_COMMIT=d8c7f8f7f461b1edc3bf5040509e697ea574c990
+        PKG_SUBDIR=debian

mkosi.conf.d/opensuse/mkosi.conf

@@ -9,12 +9,6 @@ Repositories=non-oss
 
 [Build]
 SandboxTrees=macros.db_backend:/etc/rpm/macros.db_backend
-Environment=
-        GIT_URL=https://github.com/bmwiedemann/openSUSE
-        GIT_SUBDIR=packages/s/systemd
-        GIT_BRANCH=master
-        GIT_COMMIT=9002a773f0ab185908dc8fb6c831c1cd9fdf11f0
-        PKG_SUBDIR=opensuse
 
 [Content]
 VolatilePackages=

mkosi.conf.d/opensuse/mkosi.conf.d/pkgenv.conf

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=opensuse
+
+[Build]
+Environment=
+        GIT_URL=https://github.com/bmwiedemann/openSUSE
+        GIT_SUBDIR=packages/s/systemd
+        GIT_BRANCH=master
+        GIT_COMMIT=9002a773f0ab185908dc8fb6c831c1cd9fdf11f0
+        PKG_SUBDIR=opensuse

mkosi.conf.d/ubuntu/mkosi.conf.d/non-x86.conf

@@ -7,4 +7,4 @@ Architecture=!x86
 Release=noble
 
 [Build]
-SandboxTrees=noble-backports-ports.sources:/etc/apt/sources.list.d/noble-backports-ports.sources
+SandboxTrees=%D/mkosi.conf.d/ubuntu/noble-backports-ports.sources:/etc/apt/sources.list.d/noble-backports-ports.sources

mkosi.conf.d/ubuntu/mkosi.conf.d/x86.conf

@@ -7,4 +7,4 @@ Architecture=|x86
 Release=noble
 
 [Build]
-SandboxTrees=noble-backports.sources:/etc/apt/sources.list.d/noble-backports.sources
+SandboxTrees=%D/mkosi.conf.d/ubuntu/noble-backports.sources:/etc/apt/sources.list.d/noble-backports.sources

mkosi.images/build/mkosi.conf.d/arch/mkosi.conf

@@ -3,6 +3,9 @@
 [Match]
 Distribution=arch
 
+[Include]
+Include=%D/mkosi.conf.d/arch/mkosi.conf.d/pkgenv.conf
+
 [Content]
 Packages=
         base

mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.conf

@@ -4,6 +4,9 @@
 Distribution=|centos
 Distribution=|fedora
 
+[Include]
+Include=%D/mkosi.conf.d/centos-fedora/mkosi.conf.d/pkgenv.conf
+
 [Content]
 Packages=
         clang-devel

mkosi.images/build/mkosi.conf.d/debian-ubuntu/mkosi.conf

@@ -4,6 +4,9 @@
 Distribution=|debian
 Distribution=|ubuntu
 
+[Include]
+Include=%D/mkosi.conf.d/debian-ubuntu/mkosi.conf.d/pkgenv.conf
+
 [Content]
 Packages=
         apt

mkosi.images/build/mkosi.conf.d/opensuse/mkosi.conf

@@ -3,6 +3,9 @@
 [Match]
 Distribution=opensuse
 
+[Include]
+Include=%D/mkosi.conf.d/opensuse/mkosi.conf.d/pkgenv.conf
+
 [Content]
 Packages=
         clang

mkosi.tools.conf/mkosi.conf

@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
-[Build]
-ToolsTreeSyncScripts=%D/mkosi.sync
-ToolsTreePackages=
+[Content]
+SyncScripts=%D/mkosi.sync
+Packages=
         gdb
         llvm
         meson # Also needed in the NO_BUILD case so we list it explicitly.

mkosi.tools.conf/mkosi.conf.d/arch.conf

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=arch
+
+[Include]
+Include=%D/mkosi.conf.d/arch/mkosi.conf.d/pkgenv.conf
+
+[Content]
+PrepareScripts=%D/mkosi.images/build/mkosi.conf.d/arch/mkosi.prepare
+Packages=
+        base-devel
+        github-cli
+        lcov
+        mypy
+        ruff
+        shellcheck

mkosi.tools.conf/mkosi.conf.d/centos-fedora.conf

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=|fedora
+Distribution=|centos
+
+[Include]
+Include=%D/mkosi.conf.d/centos-fedora/mkosi.conf.d/pkgenv.conf
+
+[Content]
+PrepareScripts=%D/mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.prepare
+Packages=
+        python3-mypy
+        rpm-build

mkosi.tools.conf/mkosi.conf.d/debian-ubuntu.conf

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=|debian
+Distribution=|ubuntu
+
+[Include]
+Include=%D/mkosi.conf.d/debian-ubuntu/mkosi.conf.d/pkgenv.conf
+
+[Content]
+PrepareScripts=%D/mkosi.images/build/mkosi.conf.d/debian-ubuntu/mkosi.prepare
+Packages=
+        gh
+        lcov
+        mypy
+        shellcheck

mkosi.tools.conf/mkosi.conf.d/fedora.conf

@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 [Match]
-ToolsTreeDistribution=fedora
+Distribution=fedora
 
-[Build]
-ToolsTreePackages=
+[Content]
+Packages=
         lcov
         gh
         ruff

mkosi.tools.conf/mkosi.conf.d/opensuse.conf

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Distribution=opensuse
+
+[Include]
+Include=%D/mkosi.conf.d/opensuse/mkosi.conf.d/pkgenv.conf
+
+[Content]
+PrepareScripts=%D/mkosi.images/build/mkosi.conf.d/opensuse/mkosi.prepare
+Packages=
+        gh
+        lcov
+        mypy
+        python3-ruff
+        rpm-build
+        ShellCheck

mkosi.tools.conf/mkosi.conf.d/ubuntu/mkosi.conf

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 [Match]
-ToolsTreeDistribution=ubuntu
+Distribution=ubuntu

mkosi.tools.conf/mkosi.conf.d/ubuntu/mkosi.conf.d/non-x86.conf

@@ -4,7 +4,7 @@
 [Match]
 Architecture=!x86-64
 Architecture=!x86
-ToolsTreeRelease=noble
+Release=noble
 
 [Build]
-ToolsTreeSandboxTrees=%D/mkosi.conf.d/10-ubuntu/noble-backports-ports.sources:/etc/apt/sources.list.d/noble-backports-ports.sources
+SandboxTrees=%D/mkosi.conf.d/ubuntu/noble-backports-ports.sources:/etc/apt/sources.list.d/noble-backports-ports.sources

mkosi.tools.conf/mkosi.conf.d/ubuntu/mkosi.conf.d/x86.conf

@@ -4,7 +4,7 @@
 [Match]
 Architecture=|x86-64
 Architecture=|x86
-ToolsTreeRelease=noble
+Release=noble
 
 [Build]
-ToolsTreeSandboxTrees=%D/mkosi.conf.d/10-ubuntu/noble-backports.sources:/etc/apt/sources.list.d/noble-backports.sources
+SandboxTrees=%D/mkosi.conf.d/ubuntu/noble-backports.sources:/etc/apt/sources.list.d/noble-backports.sources

test/integration-tests/README.md

@@ -169,9 +169,9 @@ ToolsTree=
 
 On the other hand, if the rpms are available but not installed on the host
 system, you'll want to make sure they're installed into the mkosi tools tree so
-that they're used to build the image by using either
-`ToolsTreePackageDirectories=` or `ToolsTreeSandboxTrees=` similarly to
-`PackageDirectories=` or `SandboxTrees=` mentioned above.
+that they're used to build the image by again using either
+`PackageDirectories=` or `SandboxTrees=`, but this time in
+`mkosi.tools.conf/mkosi.local.conf` instead of `mkosi.local.conf`.
 
 Finally, we'll make use of the standalone mode of running the integration tests
 to avoid having to install any build dependencies.

tools/fetch-distro.py

@@ -127,7 +127,7 @@ def update_distro(args, distro: str, config: dict):
     changes = subprocess.check_output(cmd, text=True).strip()
 
     conf_dir = Path('mkosi.conf.d')
-    files = conf_dir.glob('*/*.conf')
+    files = conf_dir.glob('**/pkgenv.conf')
     for file in files:
         s = file.read_text()
         if old_commit in s: