diff --git a/man/importctl.xml b/man/importctl.xml index 56a4c9055c..f67565bc05 100644 --- a/man/importctl.xml +++ b/man/importctl.xml @@ -182,11 +182,12 @@ archive, possibly compressed with xz1, gzip1, + zstd1, or bzip21. It will then be unpacked into its own subvolume/directory. When import-raw is used, the file should be a qcow2 or raw - disk image, possibly compressed with xz, gzip or bzip2. If the second argument (the resulting image + disk image, possibly compressed with xz, gzip, zstd or bzip2. If the second argument (the resulting image name) is not specified, it is automatically derived from the file name. If the filename is passed as -, the image is read from standard input, in which case the second argument is mandatory. @@ -222,6 +223,8 @@ gzip1, if it ends in .xz, with xz1, + if it ends in .zst, with + zstd1, and if it ends in .bz2, with bzip21. If the path ends in neither, the file is left uncompressed. If the second argument is missing, the image @@ -315,8 +318,8 @@ When used with the or commands, specifies the compression format to use for the resulting file. Takes one of uncompressed, xz, gzip, - bzip2. By default, the format is determined automatically from the output image - file name passed. + zst, bzip2. By default, the format is determined + automatically from the output image file name passed. @@ -450,6 +453,7 @@ tar1 xz1 gzip1 + zstd1 bzip21 diff --git a/man/machinectl.xml b/man/machinectl.xml index 7e6161b31c..eab1a7d99e 100644 --- a/man/machinectl.xml +++ b/man/machinectl.xml @@ -872,6 +872,7 @@ xz1 gzip1 bzip21 + zstd1 diff --git a/man/org.freedesktop.import1.xml b/man/org.freedesktop.import1.xml index 146adf6bcf..cece4cad0b 100644 --- a/man/org.freedesktop.import1.xml +++ b/man/org.freedesktop.import1.xml @@ -214,12 +214,13 @@ node /org/freedesktop/import1 { to the tar or raw file to import. It should reference a file on disk, a pipe or a socket. When ImportTar()/ImportTarEx() is used the file descriptor should refer to a tar file, optionally compressed with gzip1, + zstd1, bzip21, or xz1. systemd-importd will detect the used compression scheme (if any) automatically. When ImportRaw()/ImportRawEx() is used the file descriptor should refer to a raw or qcow2 disk image containing an MBR or GPT disk label, also optionally compressed with - gzip, bzip2 or xz. In either case, if the file is specified as a file descriptor on disk, progress + gzip, zstd, bzip2 or xz. In either case, if the file is specified as a file descriptor on disk, progress information is generated for the import operation (as in that case we know the total size on disk). If a socket or pipe is specified, progress information is not available. The file descriptor argument is followed by a local name for the image. This should be a name suitable as a hostname and will be used @@ -250,9 +251,9 @@ node /org/freedesktop/import1 { name to export as their first parameter, followed by a file descriptor (opened for writing) where the tar or raw file will be written. It may either reference a file on disk or a pipe/socket. The third argument specifies in which compression format to write the image. It takes one of - uncompressed, xz, bzip2 or - gzip, depending on which compression scheme is required. The image written to the - specified file descriptor will be a tar file in case of + uncompressed, xz, bzip2, + gzip or zstd, depending on which compression scheme is required. + The image written to the specified file descriptor will be a tar file in case of ExportTar()/ExportTarEx() or a raw disk image in case of ExportRaw()/ExportRawEx(). Note that currently raw disk images may not be exported as tar files, and vice versa. This restriction might be lifted @@ -267,8 +268,8 @@ node /org/freedesktop/import1 { PullRaw()/PullRawEx() may be used to download, verify and import a system image from a URL. They take a URL argument which should point to a tar or raw file on the http:// or https:// protocols, possibly compressed with xz, - bzip2 or gzip. The second argument is a local name for the image. It should be suitable as a hostname, - similarly to the matching argument of the + bzip2, gzip or zstd. The second argument is a local name for the image. It should be suitable as a + hostname, similarly to the matching argument of the ImportTar()/ImportTarEx() and ImportRaw()/ImportRawEx() methods above. The third argument indicates the verification mode for the image. It may be one of no, diff --git a/shell-completion/bash/importctl b/shell-completion/bash/importctl index d8187d0fff..0a13842445 100644 --- a/shell-completion/bash/importctl +++ b/shell-completion/bash/importctl @@ -73,7 +73,7 @@ _importctl() { comps='no checksum signature' ;; --format) - comps='uncompressed xz gzip bzip2' + comps='uncompressed xz gzip bzip2 zstd' ;; --class) comps='machine portable sysext confext' diff --git a/shell-completion/bash/machinectl b/shell-completion/bash/machinectl index b3c2e50164..10a59fc923 100644 --- a/shell-completion/bash/machinectl +++ b/shell-completion/bash/machinectl @@ -85,7 +85,7 @@ _machinectl() { comps=$( machinectl --verify=help 2>/dev/null ) ;; --format) - comps='uncompressed xz gzip bzip2' + comps='uncompressed xz gzip bzip2 zstd' ;; esac COMPREPLY=( $(compgen -W '$comps' -- "$cur") ) diff --git a/src/import/export.c b/src/import/export.c index 346aca9446..0ee2a1db71 100644 --- a/src/import/export.c +++ b/src/import/export.c @@ -43,6 +43,8 @@ static void determine_compression_from_filename(const char *p) { arg_compress = IMPORT_COMPRESS_GZIP; else if (endswith(p, ".bz2")) arg_compress = IMPORT_COMPRESS_BZIP2; + else if (endswith(p, ".zst")) + arg_compress = IMPORT_COMPRESS_ZSTD; else arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; } @@ -254,6 +256,8 @@ static int parse_argv(int argc, char *argv[]) { arg_compress = IMPORT_COMPRESS_GZIP; else if (streq(optarg, "bzip2")) arg_compress = IMPORT_COMPRESS_BZIP2; + else if (streq(optarg, "zstd")) + arg_compress = IMPORT_COMPRESS_ZSTD; else return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Unknown format: %s", optarg); diff --git a/src/import/import-compress.c b/src/import/import-compress.c index 28cf6f841a..67dd446dda 100644 --- a/src/import/import-compress.c +++ b/src/import/import-compress.c @@ -19,6 +19,16 @@ void import_compress_free(ImportCompress *c) { BZ2_bzCompressEnd(&c->bzip2); else BZ2_bzDecompressEnd(&c->bzip2); +#endif +#if HAVE_ZSTD + } else if (c->type == IMPORT_COMPRESS_ZSTD) { + if (c->encoding) { + ZSTD_freeCCtx(c->c_zstd); + c->c_zstd = NULL; + } else { + ZSTD_freeDCtx(c->d_zstd); + c->d_zstd = NULL; + } #endif } @@ -35,6 +45,9 @@ int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) { static const uint8_t bzip2_signature[] = { 'B', 'Z', 'h' }; + static const uint8_t zstd_signature[] = { + 0x28, 0xb5, 0x2f, 0xfd + }; int r; @@ -43,8 +56,9 @@ int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) { if (c->type != IMPORT_COMPRESS_UNKNOWN) return 1; - if (size < MAX3(sizeof(xz_signature), + if (size < MAX4(sizeof(xz_signature), sizeof(gzip_signature), + sizeof(zstd_signature), sizeof(bzip2_signature))) return 0; @@ -73,6 +87,14 @@ int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) { return -EIO; c->type = IMPORT_COMPRESS_BZIP2; +#endif +#if HAVE_ZSTD + } else if (memcmp(data, zstd_signature, sizeof(zstd_signature)) == 0) { + c->d_zstd = ZSTD_createDCtx(); + if (!c->d_zstd) + return -ENOMEM; + + c->type = IMPORT_COMPRESS_ZSTD; #endif } else c->type = IMPORT_COMPRESS_UNCOMPRESSED; @@ -187,6 +209,35 @@ int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCo break; #endif +#if HAVE_ZSTD + case IMPORT_COMPRESS_ZSTD: { + ZSTD_inBuffer input = { + .src = (void*) data, + .size = size, + }; + + while (input.pos < input.size) { + uint8_t buffer[16 * 1024]; + ZSTD_outBuffer output = { + .dst = buffer, + .size = sizeof(buffer), + }; + size_t res; + + res = ZSTD_decompressStream(c->d_zstd, &output, &input); + if (ZSTD_isError(res)) + return -EIO; + + if (output.pos > 0) { + r = callback(output.dst, output.pos, userdata); + if (r < 0) + return r; + } + } + + break; + } +#endif default: assert_not_reached(); @@ -231,6 +282,20 @@ int import_compress_init(ImportCompress *c, ImportCompressType t) { break; #endif +#if HAVE_ZSTD + case IMPORT_COMPRESS_ZSTD: + c->c_zstd = ZSTD_createCCtx(); + if (!c->c_zstd) + return -ENOMEM; + + r = ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT); + if (ZSTD_isError(r)) + return -EIO; + + c->type = IMPORT_COMPRESS_ZSTD; + break; +#endif + case IMPORT_COMPRESS_UNCOMPRESSED: c->type = IMPORT_COMPRESS_UNCOMPRESSED; break; @@ -351,6 +416,35 @@ int import_compress(ImportCompress *c, const void *data, size_t size, void **buf break; #endif +#if HAVE_ZSTD + case IMPORT_COMPRESS_ZSTD: { + ZSTD_inBuffer input = { + .src = data, + .size = size, + }; + + while (input.pos < input.size) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + ZSTD_outBuffer output = { + .dst = ((uint8_t *) *buffer + *buffer_size), + .size = *buffer_allocated - *buffer_size, + }; + size_t res; + + res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue); + if (ZSTD_isError(res)) + return -EIO; + + *buffer_size += output.pos; + } + + break; + } +#endif + case IMPORT_COMPRESS_UNCOMPRESSED: if (*buffer_allocated < size) { @@ -455,6 +549,32 @@ int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size break; #endif +#if HAVE_ZSTD + case IMPORT_COMPRESS_ZSTD: { + ZSTD_inBuffer input = {}; + size_t res; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated); + if (r < 0) + return r; + + ZSTD_outBuffer output = { + .dst = ((uint8_t *) *buffer + *buffer_size), + .size = *buffer_allocated - *buffer_size, + }; + + res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end); + if (ZSTD_isError(res)) + return -EIO; + + *buffer_size += output.pos; + } while (res != 0); + + break; + } +#endif + case IMPORT_COMPRESS_UNCOMPRESSED: break; @@ -473,6 +593,9 @@ static const char* const import_compress_type_table[_IMPORT_COMPRESS_TYPE_MAX] = #if HAVE_BZIP2 [IMPORT_COMPRESS_BZIP2] = "bzip2", #endif +#if HAVE_ZSTD + [IMPORT_COMPRESS_ZSTD] = "zstd", +#endif }; DEFINE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType); diff --git a/src/import/import-compress.h b/src/import/import-compress.h index 0a4210378a..6ad6cf7a67 100644 --- a/src/import/import-compress.h +++ b/src/import/import-compress.h @@ -7,6 +7,10 @@ #include #include #include +#if HAVE_ZSTD +#include +#include +#endif #include "macro.h" @@ -16,6 +20,7 @@ typedef enum ImportCompressType { IMPORT_COMPRESS_XZ, IMPORT_COMPRESS_GZIP, IMPORT_COMPRESS_BZIP2, + IMPORT_COMPRESS_ZSTD, _IMPORT_COMPRESS_TYPE_MAX, _IMPORT_COMPRESS_TYPE_INVALID = -EINVAL, } ImportCompressType; @@ -28,6 +33,10 @@ typedef struct ImportCompress { z_stream gzip; #if HAVE_BZIP2 bz_stream bzip2; +#endif +#if HAVE_ZSTD + ZSTD_CCtx *c_zstd; + ZSTD_DCtx *d_zstd; #endif }; } ImportCompress; diff --git a/src/import/importctl.c b/src/import/importctl.c index 1ddba76b09..97a95d3612 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -491,6 +491,8 @@ static void determine_compression_from_filename(const char *p) { arg_format = "gzip"; else if (endswith(p, ".bz2")) arg_format = "bzip2"; + else if (endswith(p, ".zst")) + arg_format = "zstd"; } static int export_tar(int argc, char *argv[], void *userdata) { @@ -1018,7 +1020,8 @@ static int help(int argc, char *argv[], void *userdata) { " otherwise\n" " --verify=MODE Verification mode for downloaded images (no,\n" " checksum, signature)\n" - " --format=xz|gzip|bzip2 Desired output format for export\n" + " --format=xz|gzip|bzip2|zstd\n" + " Desired output format for export\n" " --force Install image even if already exists\n" " -m --class=machine Install as machine image\n" " -P --class=portable Install as portable service image\n" @@ -1139,7 +1142,7 @@ static int parse_argv(int argc, char *argv[]) { break; case ARG_FORMAT: - if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2")) + if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2", "zstd")) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Unknown format: %s", optarg); diff --git a/src/import/meson.build b/src/import/meson.build index 45500edb43..a9dc051394 100644 --- a/src/import/meson.build +++ b/src/import/meson.build @@ -47,6 +47,7 @@ lib_import_common = static_library( libbzip2, libxz, libz, + libzstd, userspace, ], build_by_default : false) @@ -61,6 +62,7 @@ common_deps = [ libcurl, libxz, libz, + libzstd, ] executables += [ diff --git a/src/machine/machinectl.c b/src/machine/machinectl.c index 366446c306..5a6052549d 100644 --- a/src/machine/machinectl.c +++ b/src/machine/machinectl.c @@ -2334,7 +2334,7 @@ static int parse_argv(int argc, char *argv[]) { break; case ARG_FORMAT: - if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2")) + if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2", "zstd")) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Unknown format: %s", optarg); diff --git a/src/shared/import-util.c b/src/shared/import-util.c index 1a396a6570..0c61ea449f 100644 --- a/src/shared/import-util.c +++ b/src/shared/import-util.c @@ -152,6 +152,8 @@ int tar_strip_suffixes(const char *name, char **ret) { e = endswith(name, ".tar.gz"); if (!e) e = endswith(name, ".tar.bz2"); + if (!e) + e = endswith(name, ".tar.zst"); if (!e) e = endswith(name, ".tgz"); if (!e) @@ -174,6 +176,7 @@ int raw_strip_suffixes(const char *p, char **ret) { ".xz\0" ".gz\0" ".bz2\0" + ".zst\0" ".sysext.raw\0" ".confext.raw\0" ".raw\0" diff --git a/test/units/TEST-13-NSPAWN.importctl.sh b/test/units/TEST-13-NSPAWN.importctl.sh index e4931a9bfd..080b42625c 100755 --- a/test/units/TEST-13-NSPAWN.importctl.sh +++ b/test/units/TEST-13-NSPAWN.importctl.sh @@ -84,6 +84,10 @@ systemctl daemon-reload systemctl start systemd-import@var-lib-confexts-importtest9.service cmp /var/tmp/importtest /var/lib/confexts/importtest9/importtest +importctl export-raw --class=confext --format zstd importtest /var/tmp/importtest10.raw.zst +importctl import-raw --class=confext /var/tmp/importtest10.raw.zst +cmp /var/tmp/importtest /var/lib/confexts/importtest10.raw + # Verify generic service calls, too varlinkctl call --more /run/systemd/io.systemd.Import io.systemd.service.Ping '{}' varlinkctl call --more /run/systemd/io.systemd.Import io.systemd.service.SetLogLevel '{"level":"7"}'