From fbc6fecf1adbd34bd541c04d04ceef2695caa80a Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 13 Feb 2025 19:38:45 +0000 Subject: [PATCH 1/4] ukify: switch from zstd to zstandard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zstd library does not support stream decompression, and it requires the zstd header to contain extra metadata, that the kernel build does not append: $ file -k vmlinuz-6.13+unreleased-cloud-arm64 vmlinuz-6.13+unreleased-cloud-arm64: PE32+ executable (EFI application) Aarch64 (stripped to external PDB), for MS Windows, 2 sections\012- data $ ukify build --linux vmlinuz-6.13+unreleased-cloud-arm64 --initrd /boot/initrd.img-6.12.12-amd64 --output uki Kernel version not specified, starting autodetection 😖. Real-Mode Kernel Header magic not found + readelf --notes vmlinuz-6.13+unreleased-cloud-arm64 readelf: Error: Not an ELF file - it has the wrong magic bytes at the start Traceback (most recent call last): File "/home/bluca/git/systemd/src/ukify/ukify.py", line 2508, in main() ~~~~^^ File "/home/bluca/git/systemd/src/ukify/ukify.py", line 2497, in main make_uki(opts) ~~~~~~~~^^^^^^ File "/home/bluca/git/systemd/src/ukify/ukify.py", line 1326, in make_uki opts.uname = Uname.scrape(linux, opts=opts) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ File "/home/bluca/git/systemd/src/ukify/ukify.py", line 382, in scrape version = func(filename, opts=opts) File "/home/bluca/git/systemd/src/ukify/ukify.py", line 372, in scrape_generic text = maybe_decompress(filename) File "/home/bluca/git/systemd/src/ukify/ukify.py", line 219, in maybe_decompress return get_zboot_kernel(f) File "/home/bluca/git/systemd/src/ukify/ukify.py", line 199, in get_zboot_kernel return cast(bytes, zstd.uncompress(f.read(size))) ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^ zstd.Error: Input data invalid or missing content size in frame header. This appears to be by design: https://github.com/sergey-dryabzhinsky/python-zstd/issues/53 Switch to python3-zstandard, which works. --- src/ukify/ukify.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ukify/ukify.py b/src/ukify/ukify.py index 9013e64b62..85b8d612f5 100755 --- a/src/ukify/ukify.py +++ b/src/ukify/ukify.py @@ -195,8 +195,8 @@ def get_zboot_kernel(f: IO[bytes]) -> bytes: elif comp_type.startswith(b'xzkern'): raise NotImplementedError('xzkern decompression not implemented') elif comp_type.startswith(b'zstd22'): - zstd = try_import('zstd') - return cast(bytes, zstd.uncompress(f.read(size))) + zstd = try_import('zstandard') + return cast(bytes, zstd.ZstdDecompressor().stream_reader(f.read(size)).read()) raise NotImplementedError(f'unknown compressed type: {comp_type!r}') @@ -226,8 +226,8 @@ def maybe_decompress(filename: Union[str, Path]) -> bytes: return cast(bytes, gzip.open(f).read()) if start.startswith(b'\x28\xb5\x2f\xfd'): - zstd = try_import('zstd') - return cast(bytes, zstd.uncompress(f.read())) + zstd = try_import('zstandard') + return cast(bytes, zstd.ZstdDecompressor().stream_reader(f.read()).read()) if start.startswith(b'\x02\x21\x4c\x18'): lz4 = try_import('lz4.frame', 'lz4') From a6d51ae582c863c01c581f1e31492910d53b0427 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 13 Feb 2025 19:43:00 +0000 Subject: [PATCH 2/4] ukify: fix zboot parsing with zstd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The header starts with 'zstd', not 'zstd22': $ ukify build --linux vmlinuz-6.13+unreleased-cloud-arm64 --initrd /boot/initrd.img-6.12.12-amd64 --output uki Kernel version not specified, starting autodetection 😖. Real-Mode Kernel Header magic not found + readelf --notes vmlinuz-6.13+unreleased-cloud-arm64 readelf: Error: Not an ELF file - it has the wrong magic bytes at the start Traceback (most recent call last): File "/home/bluca/git/systemd/src/ukify/ukify.py", line 2510, in main() ~~~~^^ File "/home/bluca/git/systemd/src/ukify/ukify.py", line 2499, in main make_uki(opts) ~~~~~~~~^^^^^^ File "/home/bluca/git/systemd/src/ukify/ukify.py", line 1328, in make_uki opts.uname = Uname.scrape(linux, opts=opts) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ File "/home/bluca/git/systemd/src/ukify/ukify.py", line 384, in scrape version = func(filename, opts=opts) File "/home/bluca/git/systemd/src/ukify/ukify.py", line 374, in scrape_generic text = maybe_decompress(filename) File "/home/bluca/git/systemd/src/ukify/ukify.py", line 221, in maybe_decompress return get_zboot_kernel(f) File "/home/bluca/git/systemd/src/ukify/ukify.py", line 201, in get_zboot_kernel raise NotImplementedError(f'unknown compressed type: {comp_type!r}') NotImplementedError: unknown compressed type: b'zstd\x00\x00' --- src/ukify/ukify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ukify/ukify.py b/src/ukify/ukify.py index 85b8d612f5..001ab956da 100755 --- a/src/ukify/ukify.py +++ b/src/ukify/ukify.py @@ -194,7 +194,7 @@ def get_zboot_kernel(f: IO[bytes]) -> bytes: raise NotImplementedError('lzo decompression not implemented') elif comp_type.startswith(b'xzkern'): raise NotImplementedError('xzkern decompression not implemented') - elif comp_type.startswith(b'zstd22'): + elif comp_type.startswith(b'zstd'): zstd = try_import('zstandard') return cast(bytes, zstd.ZstdDecompressor().stream_reader(f.read(size)).read()) From 0dd03215f1e402092f6c6da213708045e445a9ed Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 13 Feb 2025 19:44:12 +0000 Subject: [PATCH 3/4] ukify: if the specified kernel is not a valid PE file try to decompress it On some distros on some architectures (e.g.: Ubuntu arm64) the kernel is shipped as a gzipped file, which the UEFI firmware does not understand. If pefile fails to parse it, try to decompress it. --- src/ukify/ukify.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/ukify/ukify.py b/src/ukify/ukify.py index 001ab956da..6400618648 100755 --- a/src/ukify/ukify.py +++ b/src/ukify/ukify.py @@ -1297,7 +1297,22 @@ def make_uki(opts: UkifyConfig) -> None: linux = opts.linux combined_sigs = '{}' - if opts.linux and sign_args_present: + # On some distros, on some architectures, the vmlinuz is a gzip file, so we need to decompress it + # if it's not a valid PE file, as it will fail to be booted by the firmware. + if linux: + try: + pefile.PE(linux, fast_load=True) + except pefile.PEFormatError: + try: + decompressed = maybe_decompress(linux) + except NotImplementedError: + print(f'{linux} is not a valid PE file and cannot be decompressed either', file=sys.stderr) + else: + print(f'{linux} is compressed and cannot be loaded by UEFI, decompressing', file=sys.stderr) + linux = Path(tempfile.NamedTemporaryFile(prefix='linux-decompressed').name) + linux.write_bytes(decompressed) + + if linux and sign_args_present: assert opts.signtool is not None signtool = SignTool.from_string(opts.signtool) @@ -1307,12 +1322,12 @@ def make_uki(opts: UkifyConfig) -> None: if sign_kernel: linux_signed = tempfile.NamedTemporaryFile(prefix='linux-signed') + signtool.sign(os.fspath(linux), os.fspath(Path(linux_signed.name)), opts=opts) linux = Path(linux_signed.name) - signtool.sign(os.fspath(opts.linux), os.fspath(linux), opts=opts) - if opts.uname is None and opts.linux is not None: + if opts.uname is None and linux is not None: print('Kernel version not specified, starting autodetection 😖.', file=sys.stderr) - opts.uname = Uname.scrape(opts.linux, opts=opts) + opts.uname = Uname.scrape(linux, opts=opts) uki = UKI(opts.join_pcrsig if opts.join_pcrsig else opts.stub) initrd = join_initrds(opts.initrd) From cdedc90cafe3ecdf2772ad7db66fbf822b70495c Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Fri, 14 Feb 2025 02:05:48 +0000 Subject: [PATCH 4/4] ukify: do not insist on a stub being available when joining pcrsigs It is not used in this case, so skip it, otherwise it will need to be installed even if it is not used Follow-up for 9876e88e23ad1ecbffd7c69b2e0a4cbff283f681 --- src/ukify/ukify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ukify/ukify.py b/src/ukify/ukify.py index 6400618648..097a7ee0c6 100755 --- a/src/ukify/ukify.py +++ b/src/ukify/ukify.py @@ -2389,7 +2389,7 @@ def finalize_options(opts: argparse.Namespace) -> None: if opts.efi_arch is None: opts.efi_arch = guess_efi_arch() - if opts.stub is None: + if opts.stub is None and not opts.join_pcrsig: if opts.linux is not None: opts.stub = Path(f'/usr/lib/systemd/boot/efi/linux{opts.efi_arch}.efi.stub') else: