import-generator: optionally create loopback devices after download

This is useful for booting from a freshly downloaded disk image: just
specify

    rd.systemd.pull=verify=no,machine,blockdev,raw:image:https://192.168.100.1:8081/image.raw
    root=/dev/disk/by-loop-ref/image.raw-part2

on the kernel command line, and we'll download that in the initrd and boot from it.

(note the above disables download-time verification, putting trust in
verity and image policy that this won#t do harm)

Here's a more complete example. From a git checkout do:

    ninja -C build && mkosi -f -T serve

and then from another terminal do within the same checkout:

    ./build/systemd-vmspawn \
            --ram=16G \
            --register=no \
            -n \
            -i ./build/mkosi.output/image.raw \
            rd.systemd.pull=verify=no,machine,blockdev,raw:image:http://192.168.100.1:8081/image.raw \
            root=/dev/disk/by-loop-ref/image.raw-part2 \
            rootflags=x-systemd.device-timeout=infinity \
            ip=any

This will then boot via the ESP of the specified image, then download
the image via HTTP from the mkosi instance running in the first
terminal, attach it to a loopback block device, and then use its second
partition as root fs, and boot into it.

(this assumes your host is 192.168.100.1, of course)

Note that downloading the full image takes a bit of time (this downloads
it uncompressed after all), hence we turn off the timeout to wait for
the device.

This also introduces a new "imports.target" unit (and associated
"imports-pre.target") between imports are grouped, and which ensure the
imports actually are ordered correctly both on the host and in the
initrd.

man/systemd-import-generator.xml

@@ -117,6 +117,16 @@
 
             <xi:include href="version-info.xml" xpointer="v257"/></listitem>
           </varlistentry>
+
+          <varlistentry>
+            <term>blockdev</term>
+
+            <listitem><para>If this option is specified the downloaded image is attached to a loopback block
+            device (via <filename>systemd-loop@.service</filename>) after completion. This permits booting
+            from downloaded disk images. This is only supported for <literal>raw</literal> disk images.</para>
+
+            <xi:include href="version-info.xml" xpointer="v258"/></listitem>
+          </varlistentry>
         </variablelist>
 
         <xi:include href="version-info.xml" xpointer="v257"/></listitem>
@@ -183,6 +193,16 @@
       validation. This is useful for development purposes in virtual machines and containers. Warning: do not
       deploy a system with validation disabled like this!</para>
     </example>
+
+    <example>
+      <title>Download root disk image (raw) into memory, for booting into it</title>
+
+      <programlisting>rd.systemd.pull=raw,machine,verify=no,blockdev:image:https://example.com/image.raw.xz root=/dev/disk/by-loop-ref/image.raw-part2</programlisting>
+
+      <para>This downloads the specified disk image, saving it locally under the name
+      <literal>image</literal>, and attaches it to a loopback block device on completion. It then boots from
+      the 2nd partition in the image.</para>
+    </example>
   </refsect1>
 
   <refsect1>
@@ -193,6 +213,7 @@
       <member><citerefentry><refentrytitle>kernel-command-line</refentrytitle><manvolnum>7</manvolnum></citerefentry></member>
       <member><citerefentry><refentrytitle>systemd.system-credentials</refentrytitle><manvolnum>7</manvolnum></citerefentry></member>
       <member><citerefentry><refentrytitle>importctl</refentrytitle><manvolnum>1</manvolnum></citerefentry></member>
+      <member><citerefentry><refentrytitle>systemd-loop@.service</refentrytitle><manvolnum>8</manvolnum></citerefentry></member>
     </simplelist></para>
   </refsect1>
 </refentry>

man/systemd.special.xml

@@ -386,6 +386,16 @@
             directly.</para>
           </listitem>
         </varlistentry>
+        <varlistentry>
+          <term><filename>imports.target</filename></term>
+          <listitem>
+            <para>A target unit that pulls in all disk image download jobs to execute on system boot. This is
+            used by
+            <citerefentry><refentrytitle>systemd-import-generator</refentrytitle><manvolnum>8</manvolnum></citerefentry>.</para>
+
+            <xi:include href="version-info.xml" xpointer="v258"/>
+          </listitem>
+        </varlistentry>
         <varlistentry>
           <term><filename>init.scope</filename></term>
           <listitem>
@@ -1075,6 +1085,16 @@
             <xi:include href="version-info.xml" xpointer="v235"/>
           </listitem>
         </varlistentry>
+        <varlistentry>
+          <term><filename>imports-pre.target</filename></term>
+          <listitem>
+            <para>A passive unit that is ordered before all disk image download jobs to execute on system
+            boot. This is used by
+            <citerefentry><refentrytitle>systemd-import-generator</refentrytitle><manvolnum>8</manvolnum></citerefentry>.</para>
+
+            <xi:include href="version-info.xml" xpointer="v258"/>
+          </listitem>
+        </varlistentry>
         <varlistentry>
           <term><filename>local-fs-pre.target</filename></term>
           <listitem>

src/import/import-generator.c

@@ -8,20 +8,42 @@
 #include "fileio.h"
 #include "generator.h"
 #include "import-util.h"
+#include "initrd-util.h"
 #include "json-util.h"
 #include "proc-cmdline.h"
+#include "special.h"
 #include "specifier.h"
+#include "unit-name.h"
 #include "web-util.h"
 
+typedef struct Transfer {
+        ImageClass class;
+        ImportType type;
+        char *local;
+        char *remote;
+        bool blockdev;
+        sd_json_variant *json;
+} Transfer;
+
 static const char *arg_dest = NULL;
 static char *arg_success_action = NULL;
 static char *arg_failure_action = NULL;
-static sd_json_variant **arg_transfers = NULL;
+static Transfer *arg_transfers = NULL;
 static size_t arg_n_transfers = 0;
 
+static void transfer_destroy_many(Transfer *transfers, size_t n) {
+        FOREACH_ARRAY(t, transfers, n) {
+                free(t->local);
+                free(t->remote);
+                sd_json_variant_unref(t->json);
+        }
+
+        free(transfers);
+}
+
 STATIC_DESTRUCTOR_REGISTER(arg_success_action, freep);
 STATIC_DESTRUCTOR_REGISTER(arg_failure_action, freep);
-STATIC_ARRAY_DESTRUCTOR_REGISTER(arg_transfers, arg_n_transfers, sd_json_variant_unref_many);
+STATIC_ARRAY_DESTRUCTOR_REGISTER(arg_transfers, arg_n_transfers, transfer_destroy_many);
 
 static int parse_pull_expression(const char *v) {
         const char *p = v;
@@ -57,7 +79,7 @@ static int parse_pull_expression(const char *v) {
         ImportType type = _IMPORT_TYPE_INVALID;
         ImageClass class = _IMAGE_CLASS_INVALID;
         ImportVerify verify = IMPORT_VERIFY_SIGNATURE;
-        bool ro = false;
+        bool ro = false, blockdev = false;
 
         const char *o = options;
         for (;;) {
@@ -75,6 +97,8 @@ static int parse_pull_expression(const char *v) {
                         ro = true;
                 else if (streq(opt, "rw"))
                         ro = false;
+                else if (streq(opt, "blockdev"))
+                        blockdev = true;
                 else if ((suffix = startswith(opt, "verify="))) {
 
                         ImportVerify w = import_verify_from_string(suffix);
@@ -105,6 +129,35 @@ static int parse_pull_expression(const char *v) {
         if (class < 0)
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No image class (machine, portable, sysext, confext) specified in pull expression, refusing: %s", v);
 
+        if (!local) {
+                _cleanup_free_ char *c = NULL;
+                r = import_url_last_component(remote, &c);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to generate local name from URL '%s': %m", remote);
+
+                switch (type) {
+
+                case IMPORT_RAW:
+                        r = raw_strip_suffixes(c, &local);
+                        break;
+
+                case IMPORT_TAR:
+                        r = tar_strip_suffixes(c, &local);
+                        break;
+
+                default:
+                        assert_not_reached();
+                        break;
+                }
+                if (r < 0)
+                        return log_error_errno(r, "Failed to strip suffix from URL '%s': %m", remote);
+
+                log_info("Saving downloaded file under local name '%s'.", local);
+        }
+
+        if (blockdev && type != IMPORT_RAW)
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Option 'blockdev' only available for raw images, refusing: %s", v);
+
         if (!GREEDY_REALLOC(arg_transfers, arg_n_transfers + 1))
                 return log_oom();
 
@@ -113,7 +166,7 @@ static int parse_pull_expression(const char *v) {
         r = sd_json_buildo(
                         &j,
                         SD_JSON_BUILD_PAIR("remote", SD_JSON_BUILD_STRING(remote)),
-                        SD_JSON_BUILD_PAIR_CONDITION(!!local, "local", SD_JSON_BUILD_STRING(local)),
+                        SD_JSON_BUILD_PAIR("local", SD_JSON_BUILD_STRING(local)),
                         SD_JSON_BUILD_PAIR("class", JSON_BUILD_STRING_UNDERSCORIFY(image_class_to_string(class))),
                         SD_JSON_BUILD_PAIR("type", JSON_BUILD_STRING_UNDERSCORIFY(import_type_to_string(type))),
                         SD_JSON_BUILD_PAIR("readOnly", SD_JSON_BUILD_BOOLEAN(ro)),
@@ -121,7 +174,15 @@ static int parse_pull_expression(const char *v) {
         if (r < 0)
                 return log_error_errno(r, "Failed to build import JSON object: %m");
 
-        arg_transfers[arg_n_transfers++] = TAKE_PTR(j);
+        arg_transfers[arg_n_transfers++] = (Transfer) {
+                .class = class,
+                .type = type,
+                .local = TAKE_PTR(local),
+                .remote = TAKE_PTR(remote),
+                .json = TAKE_PTR(j),
+                .blockdev = blockdev,
+        };
+
         return 0;
 }
 
@@ -191,10 +252,10 @@ static int parse_credentials(void) {
         return 0;
 }
 
-static int transfer_generate(sd_json_variant *v, size_t c) {
+static int transfer_generate(const Transfer *t, size_t c) {
         int r;
 
-        assert(v);
+        assert(t);
 
         _cleanup_free_ char *service = NULL;
         if (asprintf(&service, "import%zu.service", c) < 0)
@@ -205,19 +266,17 @@ static int transfer_generate(sd_json_variant *v, size_t c) {
         if (r < 0)
                 return r;
 
-        const char *remote = sd_json_variant_string(sd_json_variant_by_key(v, "remote"));
-
         fprintf(f,
                 "[Unit]\n"
                 "Description=Download of %s\n"
                 "Documentation=man:systemd-import-generator(8)\n"
                 "SourcePath=/proc/cmdline\n"
                 "Requires=systemd-importd.socket\n"
-                "After=systemd-importd.socket\n"
+                "After=imports-pre.target systemd-importd.socket\n"
                 "Conflicts=shutdown.target\n"
-                "Before=shutdown.target\n"
+                "Before=imports.target shutdown.target\n"
                 "DefaultDependencies=no\n",
-                remote);
+                t->remote);
 
         if (arg_success_action)
                 fprintf(f, "SuccessAction=%s\n",
@@ -227,24 +286,39 @@ static int transfer_generate(sd_json_variant *v, size_t c) {
                 fprintf(f, "FailureAction=%s\n",
                         arg_failure_action);
 
-        const char *class = sd_json_variant_string(sd_json_variant_by_key(v, "class"));
-        if (streq_ptr(class, "sysext"))
+        if (t->class == IMAGE_SYSEXT)
                 fputs("Before=systemd-sysext.service\n", f);
-        else if (streq_ptr(class, "confext"))
+        else if (t->class == IMAGE_CONFEXT)
                 fputs("Before=systemd-confext.service\n", f);
 
         /* Assume network resource unless URL is file:// */
-        if (!file_url_is_valid(remote))
+        if (!file_url_is_valid(t->remote))
                 fputs("Wants=network-online.target\n"
                       "After=network-online.target\n", f);
 
+        _cleanup_free_ char *local_path = NULL, *loop_service = NULL;
+        if (t->blockdev) {
+                assert(t->type == IMPORT_RAW);
+
+                local_path = strjoin(image_root_to_string(t->class), "/", t->local, ".raw");
+                if (!local_path)
+                        return log_oom();
+
+                r = unit_name_from_path_instance("systemd-loop", local_path, ".service", &loop_service);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to build systemd-loop@.service instance name from path '%s': %m", local_path);
+
+                /* Make sure download completes before the loopback service is activated */
+                fprintf(f, "Before=%s\n", loop_service);
+        }
+
         fputs("\n"
               "[Service]\n"
               "Type=oneshot\n"
               "NotifyAccess=main\n", f);
 
         _cleanup_free_ char *formatted = NULL;
-        r = sd_json_variant_format(v, /* flags= */ 0, &formatted);
+        r = sd_json_variant_format(t->json, /* flags= */ 0, &formatted);
         if (r < 0)
                 return log_error_errno(r, "Failed to format import JSON data: %m");
 
@@ -259,7 +333,17 @@ static int transfer_generate(sd_json_variant *v, size_t c) {
         if (r < 0)
                 return log_error_errno(r, "Failed to write unit %s: %m", service);
 
-        return generator_add_symlink(arg_dest, "multi-user.target", "wants", service);
+        r = generator_add_symlink(arg_dest, "imports.target", "wants", service);
+        if (r < 0)
+                return r;
+
+        if (loop_service) {
+                r = generator_add_symlink(arg_dest, "imports.target", "wants", loop_service);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
 }
 
 static int generate(void) {
@@ -267,7 +351,7 @@ static int generate(void) {
         int r = 0;
 
         FOREACH_ARRAY(i, arg_transfers, arg_n_transfers)
-                RET_GATHER(r, transfer_generate(*i, c++));
+                RET_GATHER(r, transfer_generate(i, c++));
 
         return r;
 }

units/imports-pre.target

@@ -0,0 +1,14 @@
+#  SPDX-License-Identifier: LGPL-2.1-or-later
+#
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=Image Downloads (Pre)
+Documentation=man:systemd.special(7)
+Before=imports.target
+RefuseManualStart=yes

units/imports.target

@@ -0,0 +1,12 @@
+#  SPDX-License-Identifier: LGPL-2.1-or-later
+#
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=Image Downloads
+Documentation=man:systemd.special(7)

units/meson.build

@@ -382,6 +382,15 @@ units = [
           'conditions' : ['ENABLE_IMPORTD'],
           'symlinks' : ['sockets.target.wants/'],
         },
+        {
+          'file' : 'imports-pre.target',
+          'conditions' : ['ENABLE_IMPORTD'],
+        },
+        {
+          'file' : 'imports.target',
+          'conditions' : ['ENABLE_IMPORTD'],
+          'symlinks' : ['sysinit.target.wants/'],
+        },
         {
           'file' : 'systemd-initctl.service.in',
           'conditions' : ['HAVE_SYSV_COMPAT'],