stages: add coreos.live-artifacts.mono stage

This adds a new `org.osbuild.coreos.live-artifacts.mono` stage to build
CoreOS Live ISO/PXE artifacts. The code is heavily based on the
`cmd-buildextend-live` script from coreos-assembler [1], but a lot of
things had to be adapted:
- the stage is provided the deployed oscontainer tree, metal, and
  metal4k images as inputs
- we use chroot instead of supermin to execute some commands in the
  context of the target oscontainer
- a bunch of calls that were wrapped by libguestfs for us (e.g.
  mkfs.vfat, mksquashfs), we now have to call ourselves; to retain
  maximum compatibility, we ensured that we still effectively use the
  same args that libguestfs passed

And various other minor adjustments.

Of course, this is not really in line with the OSBuild philosophy
of having smaller-scoped stages. We have labeled this with a .mono
suffix to denote it is monolithic, similar to the existing
`org.osbuild.bootiso.mono` stage today.

Eventually we may be able to break this stage down if we find it worth
the effort. Alternatively the need for it may go away as we align more
with Image Mode.

[1] 43a9c80e1f/src/cmd-buildextend-live

Co-authored-by: Dusty Mabe <dusty@dustymabe.com>
Co-authored-by: Renata Ravanelli <renata.ravanelli@gmail.com>

stages/org.osbuild.coreos.live-artifacts.mono

@@ -0,0 +1,845 @@
+#!/usr/bin/python3
+
+# This stage is based on coreos-assembler's `cmd-buildextend-live`. It
+# builds the CoreOS Live ISO and PXE (kernel, initramfs, and rootfs)
+# based on the provided metal and metal4k disk images that are provided
+# as inputs.
+#
+# For historical context and to see git history, refer to the original source:
+# https://github.com/coreos/coreos-assembler/blob/43a9c80e1f548269d71d6d586f0d5754c60f6144/src/cmd-buildextend-live
+
+import glob
+import hashlib
+import json
+import os
+import re
+import shutil
+import struct
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+import yaml
+
+import osbuild.api
+import osbuild.remoteloop as remoteloop
+from osbuild.util import checksum, osrelease
+from osbuild.util.chroot import Chroot
+
+# Size of file used to embed an Ignition config within a CPIO.
+IGNITION_IMG_SIZE = 256 * 1024
+
+# Size of the file used to embed miniso data.
+MINISO_DATA_FILE_SIZE = 16 * 1024
+
+LIVE_EXCLUDE_KARGS = set([
+    '$ignition_firstboot',   # unsubstituted variable in grub config
+    'console',               # no serial console by default on ISO
+    'ignition.platform.id',  # we hardcode "metal"
+    'ostree',                # dracut finds the tree automatically
+])
+
+
+# The kernel requires that uncompressed cpio archives appended to an initrd
+# start on a 4-byte boundary. If there's misalignment, it stops unpacking
+# and says:
+#
+#     Initramfs unpacking failed: invalid magic at start of compressed archive
+#
+# Append NUL bytes to destf until its size is a multiple of 4 bytes.
+#
+# https://www.kernel.org/doc/Documentation/early-userspace/buffer-format.txt
+# https://github.com/torvalds/linux/blob/47ec5303/init/initramfs.c#L463
+def align_initrd_for_uncompressed_append(destf):
+    offset = destf.tell()
+    if offset % 4:
+        destf.write(b'\0' * (4 - offset % 4))
+
+
+# Return OS features table for features.json, which is read by
+# coreos-installer {iso|pxe} customize
+def get_os_features(tree):
+    features = {
+        # coreos-installer >= 0.12.0
+        'installer-config': True,
+        # coreos/fedora-coreos-config@3edd2f28
+        'live-initrd-network': True,
+    }
+    file = os.path.join(tree, 'usr/share/coreos-installer/example-config.yaml')
+    with open(file, encoding='utf8') as f:
+        example_config_yaml = yaml.safe_load(f)
+    features['installer-config-directives'] = {
+        k: True for k in example_config_yaml
+    }
+    return features
+
+
+# https://www.kernel.org/doc/html/latest/admin-guide/initrd.html#compressed-cpio-images
+def mkinitrd_pipe(tmproot, destf, compress=True):
+    if not compress:
+        align_initrd_for_uncompressed_append(destf)
+    files = subprocess.check_output(['find', '.', '-mindepth', '1', '-print0'],
+                                    cwd=tmproot)
+    file_list = files.split(b'\0')
+    # If there's a root.squashfs, it _must_ be the first file in the cpio
+    # archive, since the dracut 20live module assumes its contents are at
+    # a fixed offset in the archive.
+    squashfs = b'./root.squashfs'
+    if squashfs in file_list:
+        file_list.remove(squashfs)
+        file_list.insert(0, squashfs)
+    cpioproc = subprocess.Popen(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
+                                 '--quiet', '--reproducible', '--force-local', '--null',
+                                 '-D', tmproot], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    if compress:
+        gzipargs = ['gzip', '-9']
+    else:
+        gzipargs = ['cat']
+    gzipproc = subprocess.Popen(gzipargs, stdin=cpioproc.stdout, stdout=destf)
+    cpioproc.stdin.write(b'\0'.join(file_list))
+    cpioproc.stdin.close()
+    assert cpioproc.wait() == 0, f"cpio exited with {cpioproc.returncode}"
+    assert gzipproc.wait() == 0, f"gzip exited with {gzipproc.returncode}"
+    # Fix up padding so the user can append the rootfs afterward
+    align_initrd_for_uncompressed_append(destf)
+
+
+def extend_initramfs(initramfs, tree, compress=True):
+    with open(initramfs, 'ab') as fdst:
+        mkinitrd_pipe(tree, fdst, compress=compress)
+
+
+def cp_reflink(src, dest):
+    subprocess.check_call(['cp', '--reflink=auto', src, dest])
+
+
+# Make stream hash for `rdcore stream-hash`
+# https://github.com/coreos/coreos-installer/blob/a8d6f50dea6e/src/bin/rdcore/stream_hash.rs#L26-L41
+def make_stream_hash(src, dest):
+    bufsize = 2 * 1024 * 1024
+    with open(src, 'rb') as inf:
+        with open(dest, 'w', encoding='utf8') as outf:
+            outf.write(f'stream-hash sha256 {bufsize}\n')
+            while True:
+                buf = inf.read(bufsize)
+                if not buf:
+                    break
+                outf.write(hashlib.sha256(buf).hexdigest() + '\n')
+
+
+def get_os_name(tree):
+    file = os.path.join(tree, 'usr/share/rpm-ostree/treefile.json')
+    with open(file, encoding='utf8') as f:
+        treefile = json.load(f)
+    return treefile['metadata']['name']
+
+
+def ensure_glob(pathname, **kwargs):
+    '''Call glob.glob(), and fail if there are no results.'''
+    ret = glob.glob(pathname, **kwargs)
+    if not ret:
+        raise ValueError(f'No matches for {pathname}')
+    return ret
+
+
+# This creates efiboot.img, which is a FAT filesystem.
+def make_efi_bootfile(loop_client, input_tarball, output_efiboot_img):
+    # Create the efiboot image file. Determine the size we should make
+    # it by taking the tarball size and adding 2MiB for fs overhead.
+    size = os.path.getsize(input_tarball) + 2 * 1024 * 1024
+    with open(output_efiboot_img, "wb") as out:
+        out.truncate(size)
+    # Make loopback device; mkfs; populate with files
+    with loop_client.device(output_efiboot_img) as loopdev:
+        # On RHEL 8, when booting from a disk device (rather than a CD),
+        # https://github.com/systemd/systemd/issues/14408 causes the
+        # hybrid ESP to race with the ISO9660 filesystem for the
+        # /dev/disk/by-label symlink unless the ESP has its own label,
+        # so set EFI-SYSTEM for consistency with the metal image.
+        # This should not be needed on Fedora or RHEL 9, but seems like
+        # a good thing to do anyway.
+        label = 'EFI-SYSTEM'
+        # NOTE: the arguments to mkfs here match how virt-make-fs calls mkfs
+        subprocess.check_call(['mkfs', '-t', 'vfat', '-I', '--mbr=n', '-n', label, loopdev])
+        with tempfile.TemporaryDirectory() as d:
+            try:
+                subprocess.check_call(['mount', '-o', 'utf8', loopdev, d])
+                subprocess.check_call(['tar', '-C', d, '-xf', input_tarball])
+            finally:
+                subprocess.check_call(['umount', d])
+
+
+def parse_metal_inputs(inputs):
+    def get_filepath_from_input(name):
+        files = inputs[name]["data"]["files"]
+        assert len(files) == 1
+        filename, _ = files.popitem()
+        filepath = os.path.join(inputs[name]["path"], filename)
+        return filepath
+    metal_file = get_filepath_from_input('metal')
+    metal4k_file = get_filepath_from_input('metal4k')
+    return metal_file, metal4k_file
+
+
+# Here we will generate the 4 Live/PXE CoreOS artifacts.
+# There are four files created and exported from this stage.
+#
+# 1. live-iso
+# 2. live-kernel    (i.e. images/pxeboot/vmlinuz)
+# 3. live-initramfs (i.e. images/pxeboot/initrd.img)
+# 4. live-rootfs    (i.e. images/pxeboot/rootfs.img)
+#
+# The live-iso itself contains the other 3 artifacts inside of
+# it. A rough approximation of the structure looks like:
+#
+#   live-iso.iso
+#     -> images/pxeboot/vmlinuz
+#     -> images/pxeboot/initrd.img
+#     -> images/pxeboot/rootfs.img
+#       -> metal.osmet
+#       -> metal4k.osmet
+#       -> root.squashfs
+#         -> full filesystem of metal image
+#
+# Where the ISO contains the kernel and initrd and the rootfs. The
+# rootfs contains the osmet file and the root.squashfs, which itself
+# contains all the files from a full CoreOS system.
+# pylint: disable=too-many-statements,too-many-branches
+def main(workdir, tree, inputs, options, loop_client):
+    squashfs_compression = 'zstd'
+    basearch = os.uname().machine
+    filenames = options['filenames']
+    output_iso = os.path.join(tree, filenames['live-iso'])
+    output_kernel = os.path.join(tree, filenames['live-kernel'])
+    output_rootfs = os.path.join(tree, filenames['live-rootfs'])
+    output_initramfs = os.path.join(tree, filenames['live-initramfs'])
+    img_metal, img_metal4k = parse_metal_inputs(inputs)
+    # The deployed tree input is a deployment tree just as it would
+    # appear on a booted OSTree system. We copy some files out of this
+    # input tree since it is easier and they match what is in the metal
+    # images anyway. We also use it as a chroot when we run `coreos-installer`.
+    deployed_tree = inputs['deployed-tree']['path']
+
+    # For dev/test purposes it is useful to create test fixture ISO
+    # images for the coreos-installer CI framework to use [1]. This test
+    # fixture is much smaller than an actual ISO and can be stored in git.
+    # Determine if the user wants a test fixture ISO created by checking
+    # for the coreos-installer-test-fixture file baked in the tree.
+    # [1] https://github.com/coreos/coreos-installer/tree/main/fixtures/iso
+    test_fixture = os.path.exists(
+        os.path.join(
+            deployed_tree,
+            'usr/share/coreos-assembler/coreos-installer-test-fixture'))
+
+    # Determine some basic information about the CoreOS we are operating on.
+    base_name = get_os_name(tree=deployed_tree)
+    os_release = osrelease.parse_files(os.path.join(deployed_tree, 'etc', 'os-release'))
+    version = os_release['OSTREE_VERSION']
+    name_version = f'{base_name}-{version}'
+    # The short volume ID can only be 32 characters (bytes probably). We may in
+    # the future want to shorten this more intelligently, otherwise we truncate the
+    # version which may impede uniqueness.
+    volid = name_version[0:32]
+
+    tmpisofile = os.path.join(workdir, 'live.iso')
+    tmpisoroot = os.path.join(workdir, 'iso')
+    tmpisocoreos = os.path.join(tmpisoroot, 'coreos')
+    tmpisoimages = os.path.join(tmpisoroot, 'images')
+    tmpisoimagespxe = os.path.join(tmpisoimages, 'pxeboot')
+    tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux')
+    # contents of initramfs on both PXE and ISO
+    tmpinitrd_base = os.path.join(workdir, 'initrd')
+    # contents of rootfs image
+    tmpinitrd_rootfs = os.path.join(workdir, 'initrd-rootfs')
+
+    for d in (tmpisoroot, tmpisocoreos, tmpisoimages, tmpisoimagespxe,
+              tmpisoisolinux, tmpinitrd_base, tmpinitrd_rootfs):
+        os.mkdir(d)
+
+    # convention for kernel and initramfs names and a few others
+    kernel_img = 'vmlinuz'
+    initrd_img = 'initrd.img'
+    rootfs_img = 'rootfs.img'
+    kargs_file = 'kargs.json'
+    igninfo_file = 'igninfo.json'
+
+    # Find the directory under `/usr/lib/modules/<kver>` where the
+    # kernel/initrd live. It will be the only entity in there.
+    modules_dir = os.path.join(deployed_tree, 'usr/lib/modules')
+    modules_dirents = os.listdir(modules_dir)
+    if len(modules_dirents) != 1:
+        raise ValueError(f"expected unique entry in modules dir, found: {modules_dirents}")
+    moduledir = modules_dirents[0]
+
+    # copy those files out of the ostree into the iso root dir
+    initramfs_img = 'initramfs.img'
+    for file in [kernel_img, initramfs_img]:
+        src = os.path.join(modules_dir, moduledir, file)
+        dst = os.path.join(tmpisoimagespxe, file)
+        if file == initramfs_img:
+            dst = os.path.join(tmpisoimagespxe, initrd_img)
+        shutil.copyfile(src, dst)
+        # initramfs isn't world readable by default so let's open up perms
+        os.chmod(dst, 0o644)
+
+    # Generate initramfs stamp file indicating that this is a live
+    # initramfs.  Store the build ID in it.
+    stamppath = os.path.join(tmpinitrd_base, 'etc/coreos-live-initramfs')
+    os.makedirs(os.path.dirname(stamppath), exist_ok=True)
+    with open(stamppath, 'w', encoding='utf8') as fh:
+        fh.write(version + '\n')
+
+    # Generate rootfs stamp file with the build ID, indicating that the
+    # rootfs has been appended and confirming that initramfs and rootfs are
+    # from the same build.
+    stamppath = os.path.join(tmpinitrd_rootfs, 'etc/coreos-live-rootfs')
+    os.makedirs(os.path.dirname(stamppath), exist_ok=True)
+    with open(stamppath, 'w', encoding='utf8') as fh:
+        fh.write(version + '\n')
+
+    # Add placeholder for Ignition CPIO file.  This allows an external tool,
+    # `coreos-installer iso ignition embed`, to modify an existing ISO image
+    # to embed a user's custom Ignition config.  The tool wraps the Ignition
+    # config in a cpio.xz and write it directly into this file in the ISO
+    # image.  The cpio.xz will be read into the initramfs filesystem at
+    # runtime and the Ignition Dracut module will ensure that the config is
+    # moved where Ignition will see it. We only handle !s390x here since that's
+    # the simple case (where layered initrds are supported). The s390x case is
+    # handled lower down
+    if basearch != 's390x':
+        with open(os.path.join(tmpisoimages, 'ignition.img'), 'wb') as fdst:
+            fdst.write(bytes(IGNITION_IMG_SIZE))
+        igninfo_json = {'file': 'images/ignition.img'}
+
+    # Generate JSON file that lists OS features available to
+    # coreos-installer {iso|pxe} customize.  Put it in the initramfs for
+    # pxe customize and the ISO for iso customize.
+    features = json.dumps(get_os_features(tree=deployed_tree), indent=2, sort_keys=True) + '\n'
+    featurespath = os.path.join(tmpinitrd_base, 'etc/coreos/features.json')
+    os.makedirs(os.path.dirname(featurespath), exist_ok=True)
+    with open(featurespath, 'w', encoding='utf8') as fh:
+        fh.write(features)
+    with open(os.path.join(tmpisocoreos, 'features.json'), 'w', encoding='utf8') as fh:
+        fh.write(features)
+
+    # Add osmet files. Use a chroot here to use the same coreos-installer as is
+    # in the artifacts we are building. Use the deployed_tree as the chroot since
+    # that's the easiest thing to do.
+    for img, sector_size in [(img_metal, 512), (img_metal4k, 4096)]:
+        with loop_client.device(img, partscan=True, read_only=True, sector_size=sector_size) as loopdev:
+            img_name = os.path.basename(img)
+            print(f'Generating osmet file for {img_name} image')
+            img_checksum = checksum.hexdigest_file(loopdev, "sha256")
+            img_osmet_file = os.path.join(tmpinitrd_rootfs, f'{img_name}.osmet')
+            with Chroot(deployed_tree, bind_mounts=["/run", "/tmp"]) as chroot:
+                cmd = ['coreos-installer', 'pack', 'osmet', loopdev,
+                       '--description', os_release['PRETTY_NAME'],
+                       '--checksum', img_checksum, '--output', img_osmet_file]
+                chroot.run(cmd, check=True)
+
+    tmp_squashfs_dir = os.path.join(workdir, 'tmp-squashfs-dir')
+    os.mkdir(tmp_squashfs_dir)
+
+    # Since inputs are read-only and we want to modify we'll make a
+    # copy of the metal.raw image and then mount that.
+    tmp_img_metal = os.path.join(workdir, os.path.basename(img_metal))
+    cp_reflink(img_metal, tmp_img_metal)
+    with loop_client.device(tmp_img_metal, partscan=True) as loopdev:
+        # So we can temporarily see the loopXpX devices
+        subprocess.check_call(['mount', '-t', 'devtmpfs', 'devtmpfs', '/dev/'])
+        # Mount manually to avoid conflicts with osmet.
+        # If mounted via the manifest, the stage begins with mounts already in place,
+        # but osmet also performs a mount operation, leading to conflicts due to duplicate
+        # filesystem UUIDs. Perform the manual mount only after the osmet stage
+        subprocess.check_call(['mount', '-o', 'rw', loopdev + 'p4', tmp_squashfs_dir])
+        subprocess.check_call(['mount', '-o', 'rw', loopdev + 'p3',
+                               os.path.join(tmp_squashfs_dir, 'boot')])
+        if basearch in ['x86_64', 'aarch64']:
+            subprocess.check_call(['mount', '-o', 'rw', loopdev + 'p2',
+                                   os.path.join(tmp_squashfs_dir, 'boot/efi')])
+        squashfs_dir_umount_needed = True
+
+        # Immplements necessary CoreOS adjustments
+        # including creating hardlinks in the /boot/ filesystem
+        # and modifying the read-only flag in OSTree configurations
+        # Where the contents of rootfs image are stored
+        # Make sure to create it, if it is not created yet.
+        tmpinitrd_rootfs = os.path.join(workdir, 'initrd-rootfs')
+        os.makedirs(tmpinitrd_rootfs, exist_ok=True)
+
+        # Remove the sysroot=readonly flag, see https://github.com/coreos/fedora-coreos-tracker/issues/589
+        subprocess.check_call(['sed', '-i', '/readonly=true/d', f'{tmp_squashfs_dir}/ostree/repo/config'])
+
+        # And ensure that the kernel binary and hmac file is in the place that dracut
+        # expects it to be; xref https://issues.redhat.com/browse/OCPBUGS-15843
+
+        kernel_binary = glob.glob(f"{tmp_squashfs_dir}/boot/ostree/*/vmlinuz*")[0]
+        kernel_hmac = glob.glob(f"{tmp_squashfs_dir}/boot/ostree/*/.*.hmac")[0]
+        kernel_binary_basename = os.path.basename(kernel_binary)
+        kernel_hmac_basename = os.path.basename(kernel_hmac)
+
+        # Create hard links in the /boot directory
+        os.link(kernel_hmac, f"{tmp_squashfs_dir}/boot/{kernel_hmac_basename}")
+        os.link(kernel_binary, f"{tmp_squashfs_dir}/boot/{kernel_binary_basename}")
+
+        print(f"Kernel binary linked: {tmp_squashfs_dir}/boot/{kernel_binary_basename}")
+        print(f"Kernel HMAC linked: {tmp_squashfs_dir}/boot/{kernel_hmac_basename}")
+        # Generate root squashfs
+        print(f'Compressing squashfs with {squashfs_compression}')
+
+        try:
+            # Name must be exactly "root.squashfs" because the 20live dracut module
+            # makes assumptions about the length of the name in sysroot.mount
+            tmp_squashfs = os.path.join(tmpinitrd_rootfs, 'root.squashfs')
+            # this matches the set of flags we implicitly passed when doing this
+            # through libguestfs' mksquashfs command
+            subprocess.check_call(['mksquashfs', tmp_squashfs_dir, tmp_squashfs,
+                                   '-root-becomes', tmp_squashfs_dir, '-wildcards', '-no-recovery',
+                                   '-comp', squashfs_compression])
+
+            # while it's mounted here, also get the kargs
+            blsentry = ensure_glob(os.path.join(tmp_squashfs_dir, 'boot/loader/entries/*.conf'))
+            if len(blsentry) != 1:
+                raise ValueError(f'Found != 1 BLS entries: {blsentry}')
+            blsentry = blsentry[0]
+            blsentry_kargs = []
+            with open(blsentry, encoding='utf8') as f:
+                for line in f:
+                    if line.startswith('options '):
+                        blsentry_kargs = line.split(' ', 1)[1].strip().split(' ')
+                        break
+            if len(blsentry_kargs) == 0:
+                raise ValueError("found no kargs in metal image")
+        finally:
+            if squashfs_dir_umount_needed:
+                subprocess.check_call(['umount', '-R', tmp_squashfs_dir])
+            subprocess.check_call(['umount', '/dev/'])
+
+    # Generate rootfs image
+    iso_rootfs = os.path.join(tmpisoimagespxe, rootfs_img)
+    # The rootfs must be uncompressed because the ISO mounts root.squashfs
+    # directly from the middle of the file
+    extend_initramfs(initramfs=iso_rootfs, tree=tmpinitrd_rootfs, compress=False)
+    # Check that the root.squashfs magic number is in the offset hardcoded
+    # in sysroot.mount in 20live/live-generator
+    with open(iso_rootfs, 'rb') as fh:
+        fh.seek(124)
+        if fh.read(4) != b'hsqs':
+            raise ValueError("root.squashfs not at expected offset in rootfs image")
+    # Save stream hash of rootfs for verifying out-of-band fetches
+    os.makedirs(os.path.join(tmpinitrd_base, 'etc'), exist_ok=True)
+    make_stream_hash(iso_rootfs, os.path.join(tmpinitrd_base, 'etc/coreos-live-want-rootfs'))
+    # Add common content
+    iso_initramfs = os.path.join(tmpisoimagespxe, initrd_img)
+    extend_initramfs(initramfs=iso_initramfs, tree=tmpinitrd_base)
+
+    # Filter kernel arguments for substituting into ISO bootloader
+    kargs_array = [karg for karg in blsentry_kargs
+                   if karg.split('=')[0] not in LIVE_EXCLUDE_KARGS]
+    kargs_array.append(f"coreos.liveiso={volid}")
+    kargs = ' '.join(kargs_array)
+    print(f'Substituting ISO kernel arguments: {kargs}')
+
+    kargs_json = {'files': []}
+    cmdline = ''
+    karg_embed_area_length = 0
+    srcdir_prefix = os.path.join(deployed_tree, 'usr/share/coreos-assembler/live/')
+    # Grab all the contents from the live dir from the configs
+    for srcdir, _, filenames in os.walk(srcdir_prefix):
+        dir_suffix = srcdir.replace(srcdir_prefix, '', 1)
+        dstdir = os.path.join(tmpisoroot, dir_suffix)
+        if not os.path.exists(dstdir):
+            os.mkdir(dstdir)
+        for filename in filenames:
+            # Skip development readmes to avoid confusing users
+            if filename == 'README-devel.md':
+                continue
+            srcfile = os.path.join(srcdir, filename)
+            dstfile = os.path.join(dstdir, filename)
+            # Assumes all files are text
+            with open(srcfile, encoding='utf8') as fh:
+                buf = fh.read()
+            newbuf = buf.replace('@@KERNEL-ARGS@@', kargs)
+            # if we injected kargs, also check for an embed area
+            if buf != newbuf:
+                karg_area_start = re.search(r'@@KERNEL-ARGS@@', buf)
+                buf = newbuf
+                karg_area_end = re.search(r'(#+)# COREOS_KARG_EMBED_AREA\n', buf)
+                if karg_area_end is not None:
+                    file_kargs = buf[karg_area_start.start():karg_area_end.start()]
+                    if len(cmdline) == 0:
+                        cmdline = file_kargs
+                    elif cmdline != file_kargs:
+                        raise ValueError(f'Default cmdline is different: "{cmdline}" != "{file_kargs}"')
+
+                    length = karg_area_end.start() + len(karg_area_end[1]) - karg_area_start.start()
+                    kargs_json['files'].append({
+                        'path': os.path.join(dir_suffix, filename),
+                        'offset': karg_area_start.start(),
+                        'pad': '#',
+                        'end': '\n',
+                    })
+                    if karg_embed_area_length == 0:
+                        karg_embed_area_length = length
+                    elif length != karg_embed_area_length:
+                        raise ValueError(f"Karg embed areas of varying length {kargs_json['files']}")
+            with open(dstfile, 'w', encoding='utf8') as fh:
+                fh.write(buf)
+            shutil.copystat(srcfile, dstfile)
+            print(f'{srcfile} -> {dstfile}')
+
+    if karg_embed_area_length > 0:
+        assert (karg_embed_area_length > len(cmdline))
+        kargs_json.update(
+            size=karg_embed_area_length,
+            default=cmdline.strip(),
+        )
+
+    # These sections are based on lorax templates
+    # see https://github.com/weldr/lorax/tree/master/share/templates.d/99-generic
+
+    # Generate the ISO image. Lots of good info here:
+    # https://fedoraproject.org/wiki/User:Pjones/BootableCDsForBIOSAndUEFI
+    genisoargs = ['/usr/bin/genisoimage', '-verbose',
+                  '-V', volid,
+                  '-volset', f"{name_version}",
+                  # For  greater portability, consider using both
+                  # Joliet and Rock Ridge extensions. Umm, OK :)
+                  '-rational-rock', '-J', '-joliet-long']
+
+    # For x86_64 legacy boot (BIOS) booting
+    if basearch == "x86_64":
+        # Install binaries from syslinux package
+        isolinuxfiles = [('/usr/share/syslinux/isolinux.bin', 0o755),
+                         ('/usr/share/syslinux/ldlinux.c32', 0o755),
+                         ('/usr/share/syslinux/libcom32.c32', 0o755),
+                         ('/usr/share/syslinux/libutil.c32', 0o755),
+                         ('/usr/share/syslinux/vesamenu.c32', 0o755)]
+        for src, mode in isolinuxfiles:
+            dst = os.path.join(tmpisoisolinux, os.path.basename(src))
+            shutil.copyfile(src, dst)
+            os.chmod(dst, mode)
+
+        # for legacy bios boot AKA eltorito boot
+        genisoargs += ['-eltorito-boot', 'isolinux/isolinux.bin',
+                       '-eltorito-catalog', 'isolinux/boot.cat',
+                       '-no-emul-boot',
+                       '-boot-load-size', '4',
+                       '-boot-info-table']
+
+    elif basearch == "ppc64le":
+        os.makedirs(os.path.join(tmpisoroot, 'boot/grub'))
+        # can be EFI/fedora or EFI/redhat
+        grubpath = ensure_glob(os.path.join(tmpisoroot, 'EFI/*/grub.cfg'))
+        if len(grubpath) != 1:
+            raise ValueError(f'Found != 1 grub.cfg files: {grubpath}')
+        shutil.move(grubpath[0], os.path.join(tmpisoroot, 'boot/grub/grub.cfg'))
+        for f in kargs_json['files']:
+            if re.match('^EFI/.*/grub.cfg$', f['path']):
+                f['path'] = 'boot/grub/grub.cfg'
+
+        # safely remove things we don't need in the final ISO tree
+        for d in ['EFI', 'isolinux']:
+            shutil.rmtree(os.path.join(tmpisoroot, d))
+
+        # grub2-mkrescue is a wrapper around xorriso
+        genisoargs = ['grub2-mkrescue', '-volid', volid]
+    elif basearch == "s390x":
+        # Reserve 32MB for the kernel, starting memory address of the initramfs
+        # See https://github.com/weldr/lorax/blob/master/share/templates.d/99-generic/s390.tmpl
+        INITRD_ADDRESS = '0x02000000'
+        lorax_templates = '/usr/share/lorax/templates.d/99-generic/config_files/s390'
+        shutil.copy(os.path.join(lorax_templates, 'redhat.exec'), tmpisoimages)
+        with open(os.path.join(lorax_templates, 'generic.ins'), 'r', encoding='utf8') as fp1:
+            with open(os.path.join(tmpisoroot, 'generic.ins'), 'w', encoding='utf8') as fp2:
+                _ = [fp2.write(line.replace('@INITRD_LOAD_ADDRESS@', INITRD_ADDRESS)) for line in fp1]
+        for prmfile in ['cdboot.prm', 'genericdvd.prm', 'generic.prm']:
+            with open(os.path.join(tmpisoimages, prmfile), 'w', encoding='utf8') as fp1:
+                with open(os.path.join(tmpisoroot, 'zipl.prm'), 'r', encoding='utf8') as fp2:
+                    fp1.write(fp2.read().strip())
+
+        # s390x's z/VM CMS files are limited to 8 char for filenames and extensions
+        # Also it is nice to keep naming convetion with Fedora/RHEL for existing users and code
+        kernel_dest = os.path.join(tmpisoimagespxe, 'kernel.img')
+        shutil.move(os.path.join(tmpisoimagespxe, kernel_img), kernel_dest)
+        kernel_img = 'kernel.img'
+
+        if test_fixture:
+            # truncate it to 128k so it includes the offsets to the initrd and kargs
+            # https://github.com/ibm-s390-linux/s390-tools/blob/032304d5034e/netboot/mk-s390image#L21-L24
+            with open(kernel_dest, 'rb+') as f:
+                f.truncate(128 * 1024)
+            with open(iso_initramfs, 'rb+') as f:
+                f.truncate(1024)
+
+        # On s390x, we reserve space for the Ignition config in the initrd
+        # image directly since the bootloader doesn't support multiple initrds.
+        # We do this by inflating the initramfs just for the duration of the
+        # `mk-s390image` call.
+        initramfs_size = os.stat(iso_initramfs).st_size
+        # sanity-check it's 4-byte aligned (see align_initrd_for_uncompressed_append)
+        assert initramfs_size % 4 == 0
+
+        # combine kernel, initramfs and cmdline using the mk-s390image tool
+        os.truncate(iso_initramfs, initramfs_size + IGNITION_IMG_SIZE)
+        subprocess.check_call(['/usr/bin/mk-s390image',
+                               kernel_dest,
+                               os.path.join(tmpisoimages, 'cdboot.img'),
+                               '-r', iso_initramfs,
+                               '-p', os.path.join(tmpisoimages, 'cdboot.prm')])
+        os.truncate(iso_initramfs, initramfs_size)
+
+        # Get the kargs and initramfs offsets in the cdboot.img. For more info, see:
+        # https://github.com/ibm-s390-linux/s390-tools/blob/032304d5034e/netboot/mk-s390image#L21-L23
+        CDBOOT_IMG_OFFS_INITRD_START_BYTES = 66568
+        CDBOOT_IMG_OFFS_KARGS_START_BYTES = 66688
+        CDBOOT_IMG_OFFS_KARGS_MAX_SIZE = 896
+        with open(os.path.join(tmpisoimages, 'cdboot.img'), 'rb') as f:
+            f.seek(CDBOOT_IMG_OFFS_INITRD_START_BYTES)
+            offset = struct.unpack(">Q", f.read(8))[0]
+
+            # sanity-check we're at the right spot by comparing a few bytes
+            f.seek(offset)
+            with open(iso_initramfs, 'rb') as canonical:
+                if f.read(1024) != canonical.read(1024):
+                    raise ValueError(f"expected initrd at offset {offset}")
+
+            igninfo_json = {
+                'file': 'images/cdboot.img',
+                'offset': offset + initramfs_size,
+                'length': IGNITION_IMG_SIZE,
+            }
+
+            # kargs are part of 'images/cdboot.img' blob
+            kargs_json['files'].append({
+                'path': 'images/cdboot.img',
+                'offset': CDBOOT_IMG_OFFS_KARGS_START_BYTES,
+                'pad': '\0',
+                'end': '\0',
+            })
+            kargs_json.update(
+                size=CDBOOT_IMG_OFFS_KARGS_MAX_SIZE,
+            )
+        # generate .addrsize file for LPAR
+        with open(os.path.join(tmpisoimages, 'initrd.addrsize'), 'wb') as addrsize:
+            addrsize_data = struct.pack(">iiii", 0, int(INITRD_ADDRESS, 16), 0,
+                                        os.stat(iso_initramfs).st_size)
+            addrsize.write(addrsize_data)
+
+        # safely remove things we don't need in the final ISO tree
+        for d in ['EFI', 'isolinux']:
+            shutil.rmtree(os.path.join(tmpisoroot, d))
+
+        genisoargs = ['/usr/bin/xorrisofs', '-verbose',
+                      '-volid', volid,
+                      '-volset', f"{name_version}",
+                      '-rational-rock', '-J', '-joliet-long',
+                      '-no-emul-boot', '-eltorito-boot',
+                      os.path.join(os.path.relpath(tmpisoimages, tmpisoroot), 'cdboot.img')]
+
+    # Drop zipl.prm as it was either unused (!s390x) or is now no longer
+    # needed (s390x)
+    os.unlink(os.path.join(tmpisoroot, 'zipl.prm'))
+
+    # For x86_64 and aarch64 UEFI booting
+    if basearch in ("x86_64", "aarch64"):
+        # Create the efiboot.img file. This is a fat32 formatted
+        # filesystem that contains all the files needed for EFI boot
+        # from an ISO.
+        with tempfile.TemporaryDirectory():
+
+            # In restrictive environments, setgid, setuid and ownership changes
+            # may be restricted. This sets the file ownership to root and
+            # removes the setgid and setuid bits in the tarball.
+            def strip(tarinfo):
+                tarinfo.uid = 0
+                tarinfo.gid = 0
+                if tarinfo.isdir():
+                    tarinfo.mode = 0o755
+                elif tarinfo.isfile():
+                    tarinfo.mode = 0o0644
+                return tarinfo
+
+            tmpimageefidir = os.path.join(workdir, "efi")
+            shutil.copytree(os.path.join(deployed_tree, 'usr/lib/bootupd/updates/EFI'), tmpimageefidir)
+
+            # Find name of vendor directory
+            vendor_ids = [n for n in os.listdir(tmpimageefidir) if n != "BOOT"]
+            if len(vendor_ids) != 1:
+                raise ValueError(f"did not find exactly one EFI vendor ID: {vendor_ids}")
+            vendor_id = vendor_ids[0]
+
+            # Always replace live/EFI/{vendor} to actual live/EFI/{vendor_id}
+            # https://github.com/openshift/os/issues/954
+            dfd = os.open(tmpisoroot, os.O_RDONLY)
+            grubfilepath = ensure_glob('EFI/*/grub.cfg', dir_fd=dfd)
+            if len(grubfilepath) != 1:
+                raise ValueError(f'Found != 1 grub.cfg files: {grubfilepath}')
+            srcpath = os.path.dirname(grubfilepath[0])
+            if srcpath != f'EFI/{vendor_id}':
+                print(f"Renaming '{srcpath}' to 'EFI/{vendor_id}'")
+                os.rename(srcpath, f"EFI/{vendor_id}", src_dir_fd=dfd, dst_dir_fd=dfd)
+                # And update kargs.json
+                for file in kargs_json['files']:
+                    if file['path'] == grubfilepath[0]:
+                        file['path'] = f'EFI/{vendor_id}/grub.cfg'
+            os.close(dfd)
+
+            # Delete fallback and its CSV file.  Its purpose is to create
+            # EFI boot variables, which we don't want when booting from
+            # removable media.
+            #
+            # A future shim release will merge fallback.efi into the main
+            # shim binary and enable the fallback behavior when the CSV
+            # exists.  But for now, fail if fallback.efi is missing.
+            for path in ensure_glob(os.path.join(tmpimageefidir, "BOOT", "fb*.efi")):
+                os.unlink(path)
+            for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "BOOT*.CSV")):
+                os.unlink(path)
+
+            # Drop vendor copies of shim; we already have it in BOOT*.EFI in
+            # BOOT
+            for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "shim*.efi")):
+                os.unlink(path)
+
+            # Consolidate remaining files into BOOT.  shim needs GRUB to be
+            # there, and the rest doesn't hurt.
+            for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "*")):
+                shutil.move(path, os.path.join(tmpimageefidir, "BOOT"))
+            os.rmdir(os.path.join(tmpimageefidir, vendor_id))
+
+            # Inject a stub grub.cfg pointing to the one in the main ISO image.
+            #
+            # When booting via El Torito, this stub is not used; GRUB reads
+            # the ISO image directly using its own ISO support.  This
+            # happens when booting from a CD device, or when the ISO is
+            # copied to a USB stick and booted on EFI firmware which prefers
+            # to boot a hard disk from an El Torito image if it has one.
+            # EDK II in QEMU behaves this way.
+            #
+            # This stub is used with EFI firmware which prefers to boot a
+            # hard disk from an ESP, or which cannot boot a hard disk via El
+            # Torito at all.  In that case, GRUB thinks it booted from a
+            # partition of the disk (a fake ESP created by isohybrid,
+            # pointing to efiboot.img) and needs a grub.cfg there.
+            with open(os.path.join(tmpimageefidir, "BOOT", "grub.cfg"), "w", encoding='utf8') as fh:
+                fh.write(f'''search --label "{volid}" --set root --no-floppy
+set prefix=($root)/EFI/{vendor_id}
+echo "Booting via ESP..."
+configfile $prefix/grub.cfg
+boot
+''')
+
+            # Install binaries from boot partition
+            # Manually construct the tarball to ensure proper permissions and ownership
+            efitarfile = tempfile.NamedTemporaryFile(suffix=".tar")
+            with tarfile.open(efitarfile.name, "w:", dereference=True) as tar:
+                tar.add(tmpimageefidir, arcname="/EFI", filter=strip)
+
+            # Create the efiboot.img file in the images/ dir
+            efibootfile = os.path.join(tmpisoimages, 'efiboot.img')
+            make_efi_bootfile(loop_client, input_tarball=efitarfile.name, output_efiboot_img=efibootfile)
+
+        genisoargs += ['-eltorito-alt-boot',
+                       '-efi-boot', 'images/efiboot.img',
+                       '-no-emul-boot']
+
+    # We've done everything that might affect kargs, so filter out any files
+    # that no longer exist and write out the kargs JSON if it lists any files
+    kargs_json['files'] = [f for f in kargs_json['files']
+                           if os.path.exists(os.path.join(tmpisoroot, f['path']))]
+    kargs_json['files'].sort(key=lambda f: f['path'])
+    if kargs_json['files']:
+        # Store the location of "karg embed areas" for use by
+        # `coreos-installer iso kargs modify`
+        with open(os.path.join(tmpisocoreos, kargs_file), 'w', encoding='utf8') as fh:
+            json.dump(kargs_json, fh, indent=2, sort_keys=True)
+            fh.write('\n')
+
+    # Write out the igninfo.json file. This is used by coreos-installer to know
+    # how to embed the Ignition config.
+    with open(os.path.join(tmpisocoreos, igninfo_file), 'w', encoding='utf8') as fh:
+        json.dump(igninfo_json, fh, indent=2, sort_keys=True)  # pylint: disable=E0601
+        fh.write('\n')
+
+    # Define inputs and outputs
+    genisoargs_final = genisoargs + ['-o', tmpisofile, tmpisoroot]
+
+    miniso_data = os.path.join(tmpisocoreos, "miniso.dat")
+    with open(miniso_data, 'wb') as f:
+        f.truncate(MINISO_DATA_FILE_SIZE)
+
+    if test_fixture:
+        # Replace or delete anything irrelevant to coreos-installer
+        with open(os.path.join(tmpisoimages, 'efiboot.img'), 'w', encoding='utf8') as fh:
+            fh.write('efiboot.img\n')
+        with open(os.path.join(tmpisoimagespxe, 'rootfs.img'), 'w', encoding='utf8') as fh:
+            fh.write('rootfs data\n')
+        with open(os.path.join(tmpisoimagespxe, 'initrd.img'), 'w', encoding='utf8') as fh:
+            fh.write('initrd data\n')
+        with open(os.path.join(tmpisoimagespxe, 'vmlinuz'), 'w', encoding='utf8') as fh:
+            fh.write('the kernel\n')
+        # this directory doesn't exist on s390x
+        if os.path.isdir(tmpisoisolinux):
+            with open(os.path.join(tmpisoisolinux, 'isolinux.bin'), 'rb+') as fh:
+                flen = fh.seek(0, 2)
+                fh.truncate(0)
+                fh.truncate(flen)
+                fh.seek(64)
+                # isohybrid checks for this magic
+                fh.write(b'\xfb\xc0\x78\x70')
+            for f in ensure_glob(os.path.join(tmpisoisolinux, '*.c32')):
+                os.unlink(f)
+            for f in ensure_glob(os.path.join(tmpisoisolinux, '*.msg')):
+                os.unlink(f)
+
+    subprocess.check_call(genisoargs_final)
+
+    # Add MBR, and GPT with ESP, for x86_64 BIOS/UEFI boot when ISO is
+    # copied to a USB stick
+    if basearch == "x86_64":
+        subprocess.check_call(['/usr/bin/isohybrid', '--uefi', tmpisofile])
+
+    # Copy to final output locations the things that won't change from
+    # this point forward. We do it here because we unlink the the rootfs below.
+    cp_reflink(os.path.join(tmpisoimagespxe, kernel_img), output_kernel)
+    cp_reflink(os.path.join(tmpisoimagespxe, initrd_img), output_initramfs)
+    cp_reflink(iso_rootfs, output_rootfs)
+
+    # Here we generate a minimal ISO purely for the purpose of
+    # calculating some data such that, given the full live ISO in
+    # future, coreos-installer can regenerate a minimal ISO.
+    #
+    # We first generate the minimal ISO and then call coreos-installer
+    # pack with both the minimal and full ISO as arguments. This will
+    # delete the minimal ISO (i.e. --consume) and update in place the
+    # full Live ISO.
+    #
+    # The only difference in the minimal ISO is that we drop two files.
+    # We keep everything else the same to maximize file matching between
+    # the two versions so we can get the smallest delta. E.g. we keep the
+    # `coreos.liveiso` karg, even though the miniso doesn't need it.
+    # coreos-installer takes care of removing it.
+    os.unlink(iso_rootfs)
+    os.unlink(miniso_data)
+    subprocess.check_call(genisoargs + ['-o', f'{tmpisofile}.minimal', tmpisoroot])
+    if basearch == "x86_64":
+        subprocess.check_call(['/usr/bin/isohybrid', '--uefi', f'{tmpisofile}.minimal'])
+    with Chroot(deployed_tree, bind_mounts=["/run"]) as chroot:
+        chroot.run(['coreos-installer', 'pack', 'minimal-iso', tmpisofile,
+                   f'{tmpisofile}.minimal', '--consume'], check=True)
+
+    # Copy the ISO to the final output location
+    shutil.move(tmpisofile, output_iso)
+
+
+if __name__ == "__main__":
+    args = osbuild.api.arguments()
+    with tempfile.TemporaryDirectory(dir=args["tree"]) as tmdir:
+        r = main(workdir=tmdir,
+                 tree=args["tree"],
+                 inputs=args["inputs"],
+                 options=args["options"],
+                 loop_client=remoteloop.LoopClient("/run/osbuild/api/remoteloop"))
+    sys.exit(r)

stages/org.osbuild.coreos.live-artifacts.mono.meta.json

@@ -0,0 +1,68 @@
+{
+  "summary": "Build CoreOS Live ISO and PXE kernel,initramfs,rootfs",
+  "description": [
+    "This stage builds the CoreOS Live ISO and PXE kernel,initramfs,rootfs",
+    "artifacts. Input to this stage are metal and metal4k raw disk images",
+    "that are then used to generate the Live artifacts."
+  ],
+  "capabilities": [
+    "CAP_MAC_ADMIN"
+  ],
+  "schema_2": {
+    "inputs": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "deployed-tree",
+        "metal",
+        "metal4k"
+      ],
+      "properties": {
+        "deployed-tree": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "metal": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "metal4k": {
+          "type": "object",
+          "additionalProperties": true
+        }
+      }
+    },
+    "options": {
+      "additionalProperties": false,
+      "required": [
+        "filenames"
+      ],
+      "properties": {
+        "filenames": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "live-iso",
+            "live-kernel",
+            "live-initramfs",
+            "live-rootfs"
+          ],
+          "properties": {
+            "live-iso": {
+              "type": "string"
+            },
+            "live-kernel": {
+              "type": "string"
+            },
+            "live-initramfs": {
+              "type": "string"
+            },
+            "live-rootfs": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    }
+  }
+}

test/data/manifests/fedora-coreos-container.json

@@ -1635,6 +1635,50 @@
           }
         }
       ]
+    },
+    {
+      "name": "live",
+      "build": "name:build",
+      "stages": [
+        {
+          "type": "org.osbuild.coreos.live-artifacts.mono",
+          "inputs": {
+            "deployed-tree": {
+              "type": "org.osbuild.tree",
+              "origin": "org.osbuild.pipeline",
+              "references": [
+                "name:deployed-tree"
+              ]
+            },
+            "metal": {
+              "type": "org.osbuild.files",
+              "origin": "org.osbuild.pipeline",
+              "references": {
+                "name:metal": {
+                  "file": "/metal.raw"
+                }
+              }
+            },
+            "metal4k": {
+              "type": "org.osbuild.files",
+              "origin": "org.osbuild.pipeline",
+              "references": {
+                "name:metal4k": {
+                  "file": "/metal4k.raw"
+                }
+              }
+            }
+          },
+          "options": {
+            "filenames": {
+              "live-iso": "coreos-live-iso.x86_64.iso",
+              "live-kernel": "coreos-live-kernel.x86_64",
+              "live-initramfs": "coreos-live-initramfs.x86_64.img",
+              "live-rootfs": "coreos-live-rootfs.x86_64.img"
+            }
+          }
+        }
+      ]
     }
   ],
   "sources": {

test/data/manifests/fedora-coreos-container.mpp.yaml

@@ -873,3 +873,36 @@ pipelines:
             type: qcow2
             compression: false
             compat: '1.1'
+  - name: live
+    build:
+      mpp-format-string: '{buildroot}'
+    stages:
+      - type: org.osbuild.coreos.live-artifacts.mono
+        inputs:
+          deployed-tree:
+            type: org.osbuild.tree
+            origin: org.osbuild.pipeline
+            references:
+              - name:deployed-tree
+          metal:
+            type: org.osbuild.files
+            origin: org.osbuild.pipeline
+            references:
+              name:metal:
+                file: /metal.raw
+          metal4k:
+            type: org.osbuild.files
+            origin: org.osbuild.pipeline
+            references:
+              name:metal4k:
+                file: /metal4k.raw
+        options:
+          filenames:
+            live-iso:
+              mpp-format-string: 'coreos-live-iso.{arch}.iso'
+            live-kernel:
+              mpp-format-string: 'coreos-live-kernel.{arch}'
+            live-initramfs:
+              mpp-format-string: 'coreos-live-initramfs.{arch}.img'
+            live-rootfs:
+              mpp-format-string: 'coreos-live-rootfs.{arch}.img'