Introduce runners

We've been using a generic `osbuild-run`, which sets up the build
environment (and works around bugs) for all build roots. It is already
getting unwieldy, because it tries to detect the OS for some things it
configures. It's also about to cause problems for RHEL, which doesn't
currently support a python3 shebang without having /etc around.

This patch changes the `build` key in a pipeline to not be a pipeline
itself, but an object with `runner` and `pipeline` keys. `pipeline` is
the build pipeline, as before. `runner` is the name of the runner to
use. Runners are programs in the `runners` subdirectory.

Three runners are included in this patch. They're copies of osbuild-run
for now (except some additions for rhel82). The idea is that each of
them only contains the minimal setup code necessary for an OS, and that
we can review what's needed when updating a build root.

Also modify the `--build-pipeline` command line switch to accept such a
build object (instead of a pipeline) and rename it accordingly, to
`--build-env`.

Correspondingly, `OSBUILD_TEST_BUILD_PIPELINE` → `OSBUILD_TEST_BUILD_ENV`.

.travis.yml

@@ -8,7 +8,7 @@ jobs:
   include:
     - name: pylint
       install: pip install pylint==2.4.1
-      script: pylint osbuild osbuild-run assemblers/* stages/*
+      script: pylint osbuild runners/* assemblers/* stages/*
     - name: unit-tests
       script: python3 -m unittest test.test_osbuild
     - name: rpm
@@ -21,14 +21,14 @@ jobs:
     - name: pipeline-noop
       before_install: sudo apt-get install -y systemd-container
       script:
-        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json
-        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json
+        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --build-env samples/ubuntu1804.json samples/noop.json
+        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --build-env samples/ubuntu1804.json samples/noop.json
     - name: f30-boot
       before_install: sudo apt-get install -y systemd-container yum qemu-kvm
-      script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_PIPELINE=samples/build-from-yum.json" python3 -m unittest -v test.test_boot
+      script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_ENV=samples/f27-build-from-ubuntu1804.json" python3 -m unittest -v test.test_boot
     - name: assemblers
       before_install: sudo apt-get install -y systemd-container yum tar qemu-utils
-      script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_PIPELINE=samples/build-from-yum.json" python3 -m unittest -v test.test_assemblers
+      script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_ENV=samples/f27-build-from-ubuntu1804.json" python3 -m unittest -v test.test_assemblers
     - name: stage-tests
       before_install: sudo apt-get install -y systemd-container yum
-      script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_PIPELINE=samples/build-from-yum.json" python3 -m unittest -v test.test_stages
+      script: sudo env "PATH=$PATH" "OSBUILD_TEST_BUILD_ENV=samples/f27-build-from-ubuntu1804.json" python3 -m unittest -v test.test_stages

README.md

@@ -71,24 +71,25 @@ The above pipeline has no base and produces a qcow2 image.
 ## Running
 
 ```
-usage: python3 -m osbuild [-h] [--build-pipeline PIPELINE] [--store DIRECTORY]
-                   [-l DIRECTORY]
+usage: __main__.py [-h] [--build-env ENV] [--store DIRECTORY] [-l DIRECTORY]
+                   [--json]
                    PIPELINE
 
 Build operating system images
 
 positional arguments:
-  PIPELINE              json file containing the pipeline that should be built
+  PIPELINE              json file containing the pipeline that should be
+                        built, or a '-' to read from stdin
 
 optional arguments:
   -h, --help            show this help message and exit
-  --build-pipeline PIPELINE
-                        json file containing the pipeline to create a build
+  --build-env ENV       json file containing a description of the build
                         environment
   --store DIRECTORY     the directory where intermediary os trees are stored
   -l DIRECTORY, --libdir DIRECTORY
                         the directory containing stages, assemblers, and the
                         osbuild library
+  --json                output results in JSON format
 ```
 
 ### Running example

osbuild.spec

@@ -54,7 +54,8 @@ install -p -m 0755 $(find stages -type f) %{buildroot}%{pkgdir}/stages/
 mkdir -p %{buildroot}%{pkgdir}/assemblers
 install -p -m 0755 $(find assemblers -type f) %{buildroot}%{pkgdir}/assemblers/
 
-install -p -m 0755 osbuild-run %{buildroot}%{pkgdir}/
+mkdir -p %{buildroot}%{pkgdir}/runners
+install -p -m 0755 $(find runners -type f) %{buildroot}%{pkgdir}/runners
 
 %check
 exit 0

osbuild/__init__.py

@@ -1,11 +1,12 @@
 
-from .pipeline import Assembler, AssemblerFailed, load, Pipeline, Stage, StageFailed
+from .pipeline import Assembler, AssemblerFailed, load, load_build, Pipeline, Stage, StageFailed
 
 
 __all__ = [
     "Assembler",
     "AssemblerFailed",
     "load",
+    "load_build",
     "Pipeline",
     "Stage",
     "StageFailed",

osbuild/__main__.py

@@ -14,8 +14,8 @@ def main():
     parser = argparse.ArgumentParser(description="Build operating system images")
     parser.add_argument("pipeline_path", metavar="PIPELINE",
                         help="json file containing the pipeline that should be built, or a '-' to read from stdin")
-    parser.add_argument("--build-pipeline", metavar="PIPELINE", type=os.path.abspath,
-                        help="json file containing the pipeline to create a build environment")
+    parser.add_argument("--build-env", metavar="ENV", type=os.path.abspath,
+                        help="json file containing a description of the build environment")
     parser.add_argument("--store", metavar="DIRECTORY", type=os.path.abspath,
                         default=".osbuild",
                         help="the directory where intermediary os trees are stored")
@@ -32,10 +32,10 @@ def main():
     pipeline = osbuild.load(json.load(f))
     f.close()
 
-    if args.build_pipeline:
-        with open(args.build_pipeline) as f:
-            build = osbuild.load(json.load(f))
-        pipeline.prepend_build_pipeline(build)
+    if args.build_env:
+        with open(args.build_env) as f:
+            build_pipeline, runner = osbuild.load_build(json.load(f))
+        pipeline.prepend_build_env(build_pipeline, runner)
 
     try:
         pipeline.run(args.store, interactive=not args.json, libdir=args.libdir)

osbuild/buildroot.py

@@ -13,12 +13,13 @@ __all__ = [
 
 
 class BuildRoot:
-    def __init__(self, root, path="/run/osbuild", libdir=None):
+    def __init__(self, root, runner, path="/run/osbuild", libdir=None):
         self.root = tempfile.mkdtemp(prefix="osbuild-buildroot-", dir=path)
         self.api = tempfile.mkdtemp(prefix="osbuild-api-", dir=path)
         self.var = tempfile.mkdtemp(prefix="osbuild-var-", dir="/var/tmp")
         self.mounts = []
         self.libdir = libdir or "/usr/lib/osbuild"
+        self.runner = runner
 
         self.mount_root(root)
         self.mount_var()
@@ -82,7 +83,7 @@ class BuildRoot:
             f"--bind-ro={self.libdir}:/run/osbuild/lib",
             *[f"--bind={b}" for b in (binds or [])],
             *[f"--bind-ro={b}" for b in [f"{self.api}:/run/osbuild/api"] + (readonly_binds or [])],
-            f"/run/osbuild/lib/osbuild-run"
+            f"/run/osbuild/lib/runners/{self.runner}"
             ] + argv, check=check, **kwargs)
 
     @contextlib.contextmanager

osbuild/pipeline.py

@@ -62,8 +62,8 @@ class Stage:
             description["options"] = self.options
         return description
 
-    def run(self, tree, build_tree, interactive=False, check=True, libdir=None):
-        with buildroot.BuildRoot(build_tree, libdir) as build_root:
+    def run(self, tree, runner, build_tree, interactive=False, check=True, libdir=None):
+        with buildroot.BuildRoot(build_tree, runner, libdir=libdir) as build_root:
             if interactive:
                 print_header(f"{self.name}: {self.id}", self.options)
 
@@ -108,8 +108,8 @@ class Assembler:
             description["options"] = self.options
         return description
 
-    def run(self, tree, build_tree, output_dir=None, interactive=False, check=True, libdir=None):
-        with buildroot.BuildRoot(build_tree, libdir) as build_root:
+    def run(self, tree, runner, build_tree, output_dir=None, interactive=False, check=True, libdir=None):
+        with buildroot.BuildRoot(build_tree, runner, libdir=libdir) as build_root:
             if interactive:
                 print_header(f"Assembler {self.name}: {self.id}", self.options)
 
@@ -148,8 +148,9 @@ class Assembler:
 
 
 class Pipeline:
-    def __init__(self, build=None):
+    def __init__(self, runner=None, build=None):
         self.build = build
+        self.runner = runner
         self.stages = []
         self.assembler = None
 
@@ -172,16 +173,20 @@ class Pipeline:
         build = self.build.tree_id if self.build else None
         self.assembler = Assembler(name, build, self.tree_id, options or {})
 
-    def prepend_build_pipeline(self, build):
+    def prepend_build_env(self, build_pipeline, runner):
         pipeline = self
         while pipeline.build:
             pipeline = pipeline.build
-        pipeline.build = build
+        pipeline.build = build_pipeline
+        pipeline.runner = runner
 
     def description(self):
         description = {}
         if self.build:
-            description["build"] = self.build.description()
+            description["build"] = {
+                "pipeline": self.build.description(),
+                "runner": self.runner
+            }
         if self.stages:
             description["stages"] = [s.description() for s in self.stages]
         if self.assembler:
@@ -228,6 +233,7 @@ class Pipeline:
                     with object_store.new(self.tree_id, base_id=base) as tree:
                         for stage in self.stages[base_idx + 1:]:
                             if not stage.run(tree,
+                                             self.runner,
                                              build_tree,
                                              interactive=interactive,
                                              check=check,
@@ -238,6 +244,7 @@ class Pipeline:
                 with object_store.get(self.tree_id) as tree, \
                     object_store.new(self.output_id) as output_dir:
                     if not self.assembler.run(tree,
+                                              self.runner,
                                               build_tree,
                                               output_dir=output_dir,
                                               interactive=interactive,
@@ -248,13 +255,24 @@ class Pipeline:
         return True
 
 
-def load(description):
-    build_description = description.get("build")
-    if build_description:
-        build = load(build_description)
+def load_build(description):
+    pipeline = description.get("pipeline")
+    if pipeline:
+        build_pipeline = load(pipeline)
     else:
-        build = None
-    pipeline = Pipeline(build)
+        build_pipeline = None
+
+    return build_pipeline, description["runner"]
+
+
+def load(description):
+    build = description.get("build")
+    if build:
+        build_pipeline, runner = load_build(build)
+    else:
+        build_pipeline, runner = None, None
+
+    pipeline = Pipeline(runner, build_pipeline)
 
     for s in description.get("stages", []):
         pipeline.add_stage(s["name"], s.get("options", {}))

runners/org.osbuild.fedora30

@@ -0,0 +1,112 @@
+#!/usr/bin/python3
+
+import array
+import json
+import shutil
+import os
+import socket
+import subprocess
+import sys
+
+
+# copied from remoteloop.py
+def load_fds(sock, msglen):
+    fds = array.array("i")   # Array of ints
+    msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
+    for cmsg_level, cmsg_type, cmsg_data in ancdata:
+        if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
+            # Append data, ignoring any truncated integers at the end.
+            fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+    return json.loads(msg), list(fds), addr
+
+def ldconfig():
+    # ld.so.conf must exist, or `ldconfig` throws a warning
+    subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
+    subprocess.run(["ldconfig"], check=True)
+
+
+def sysusers():
+    try:
+        subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    except subprocess.CalledProcessError as error:
+        sys.stderr.write(error.stdout)
+        sys.exit(1)
+
+
+def update_ca_trust():
+    if not shutil.which("update-ca-trust"):
+        return
+
+    # generate /etc/pki/tls/certs/ca-bundle.crt
+    os.makedirs("/etc/pki/ca-trust/extracted/pem")
+    os.makedirs("/etc/pki/tls/certs")
+    os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
+
+    # allow to fail, because it sometimes mysteriously does
+    subprocess.run(["update-ca-trust", "extract"], check=False)
+
+
+def append_certs(cert_conf, dir_fd, parents=b""):
+    for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
+        if entry.is_file():
+            line = os.path.join(parents, entry.name)
+            cert_conf.write(line)
+            cert_conf.write(b"\n")
+        elif entry.is_dir():
+            append_certs(cert_conf,
+                         os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
+                         os.path.join(parents, entry.name))
+
+
+def update_ca_certificates():
+    if not shutil.which("update-ca-certificates"):
+        return
+
+    # generate /etc/ssl/certs/ca-certificates.crt
+    os.makedirs("/etc/ssl/certs")
+    with open("/etc/ca-certificates.conf", "wb") as f:
+        append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
+    subprocess.run(["update-ca-certificates"], check=True)
+
+
+def tmpfiles():
+    # Allow systemd-tmpfiles to return non-0. Some packages want to create
+    # directories owned by users that are not set up with systemd-sysusers.
+    subprocess.run(["systemd-tmpfiles", "--create"], check=False)
+
+
+def nsswitch():
+    # the default behavior is fine, but using nss-resolve does not
+    # necessarily work in a non-booted container, so make sure that
+    # is not configured.
+    try:
+        os.remove("/etc/nsswitch.conf")
+    except FileNotFoundError:
+        pass
+
+
+def setup_stdio():
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+        sock.connect("/run/osbuild/api/osbuild")
+        req = {'method': 'setup-stdio'}
+        sock.send(json.dumps(req).encode('utf-8'))
+        msg, fds, _ = load_fds(sock, 1024)
+        for io in ['stdin', 'stdout', 'stderr']:
+            target = getattr(sys, io)
+            source = fds[msg[io]]
+            os.dup2(source, target.fileno())
+            os.close(source)
+
+
+if __name__ == "__main__":
+    setup_stdio()
+    ldconfig()
+    sysusers()
+    update_ca_trust()
+    update_ca_certificates()
+    tmpfiles()
+    nsswitch()
+
+    r = subprocess.run(sys.argv[1:], check=False)
+    sys.exit(r.returncode)

runners/org.osbuild.rhel82

@@ -0,0 +1,147 @@
+#!/usr/bin/python3.6
+
+import array
+import json
+import shutil
+import os
+import socket
+import subprocess
+import sys
+
+
+# copied from remoteloop.py
+def load_fds(sock, msglen):
+    fds = array.array("i")   # Array of ints
+    msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
+    for cmsg_level, cmsg_type, cmsg_data in ancdata:
+        if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
+            # Append data, ignoring any truncated integers at the end.
+            fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+    return json.loads(msg), list(fds), addr
+
+def ldconfig():
+    # ld.so.conf must exist, or `ldconfig` throws a warning
+    subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
+    subprocess.run(["ldconfig"], check=True)
+
+
+def sysusers():
+    try:
+        subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    except subprocess.CalledProcessError as error:
+        sys.stderr.write(error.stdout)
+        sys.exit(1)
+
+
+def update_ca_trust():
+    if not shutil.which("update-ca-trust"):
+        return
+
+    # generate /etc/pki/tls/certs/ca-bundle.crt
+    os.makedirs("/etc/pki/ca-trust/extracted/pem")
+    os.makedirs("/etc/pki/tls/certs")
+    os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
+
+    # allow to fail, because it sometimes mysteriously does
+    subprocess.run(["update-ca-trust", "extract"], check=False)
+
+
+def append_certs(cert_conf, dir_fd, parents=b""):
+    for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
+        if entry.is_file():
+            line = os.path.join(parents, entry.name)
+            cert_conf.write(line)
+            cert_conf.write(b"\n")
+        elif entry.is_dir():
+            append_certs(cert_conf,
+                         os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
+                         os.path.join(parents, entry.name))
+
+
+def update_ca_certificates():
+    if not shutil.which("update-ca-certificates"):
+        return
+
+    # generate /etc/ssl/certs/ca-certificates.crt
+    os.makedirs("/etc/ssl/certs")
+    with open("/etc/ca-certificates.conf", "wb") as f:
+        append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
+    subprocess.run(["update-ca-certificates"], check=True)
+
+
+def tmpfiles():
+    # Allow systemd-tmpfiles to return non-0. Some packages want to create
+    # directories owned by users that are not set up with systemd-sysusers.
+    subprocess.run(["systemd-tmpfiles", "--create"], check=False)
+
+
+def nsswitch():
+    # the default behavior is fine, but using nss-resolve does not
+    # necessarily work in a non-booted container, so make sure that
+    # is not configured.
+    try:
+        os.remove("/etc/nsswitch.conf")
+    except FileNotFoundError:
+        pass
+
+
+def setup_stdio():
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+        sock.connect("/run/osbuild/api/osbuild")
+        req = {'method': 'setup-stdio'}
+        sock.send(json.dumps(req).encode('utf-8'))
+        msg, fds, _ = load_fds(sock, 1024)
+        for io in ['stdin', 'stdout', 'stderr']:
+            target = getattr(sys, io)
+            source = fds[msg[io]]
+            os.dup2(source, target.fileno())
+            os.close(source)
+
+def os_release():
+    """/usr/lib/os-release doesn't exist. The `redhat-release` package
+    generates `/etc/os-release directly. To work around this, do the same here.
+
+    https://bugzilla.redhat.com/show_bug.cgi?id=1766754
+    """
+
+    # remove the symlink that systemd-nspawn creates
+    os.remove("/etc/os-release")
+    with open("/etc/os-release", "w") as f:
+        f.write('NAME="Red Hat Enterprise Linux"\n')
+        f.write('VERSION="8.2 (Ootpa)"\n')
+        f.write('ID="rhel"\n')
+        f.write('ID_LIKE="fedora"\n')
+        f.write('VERSION_ID="8.2"\n')
+        f.write('PLATFORM_ID="platform:el8"\n')
+        f.write('PRETTY_NAME="Red Hat Enterprise Linux 8.2 Beta (Ootpa)"\n')
+        f.write('ANSI_COLOR="0;31"\n')
+        f.write('CPE_NAME="cpe:/o:redhat:enterprise_linux:8.2:beta"\n')
+        f.write('HOME_URL="https://www.redhat.com/"\n')
+        f.write('BUG_REPORT_URL="https://bugzilla.redhat.com/"\n')
+
+
+def python_alternatives():
+    """/usr/bin/python3 is a symlink to /etc/alternatives/python3, which points
+    to /usr/bin/python3.6 by default. Recreate the link in /etc, so that
+    shebang lines in stages and assemblers work.
+    """
+    os.makedirs("/etc/alternatives", exist_ok=True)
+    try:
+        os.symlink("/usr/bin/python3.6", "/etc/alternatives/python3")
+    except FileExistsError:
+        pass
+
+if __name__ == "__main__":
+    setup_stdio()
+    ldconfig()
+    sysusers()
+    update_ca_trust()
+    update_ca_certificates()
+    tmpfiles()
+    nsswitch()
+    os_release()
+    python_alternatives()
+
+    r = subprocess.run(sys.argv[1:], check=False)
+    sys.exit(r.returncode)

runners/org.osbuild.ubuntu1804

@@ -0,0 +1,112 @@
+#!/usr/bin/python3
+
+import array
+import json
+import shutil
+import os
+import socket
+import subprocess
+import sys
+
+
+# copied from remoteloop.py
+def load_fds(sock, msglen):
+    fds = array.array("i")   # Array of ints
+    msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
+    for cmsg_level, cmsg_type, cmsg_data in ancdata:
+        if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
+            # Append data, ignoring any truncated integers at the end.
+            fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+    return json.loads(msg), list(fds), addr
+
+def ldconfig():
+    # ld.so.conf must exist, or `ldconfig` throws a warning
+    subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
+    subprocess.run(["ldconfig"], check=True)
+
+
+def sysusers():
+    try:
+        subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    except subprocess.CalledProcessError as error:
+        sys.stderr.write(error.stdout)
+        sys.exit(1)
+
+
+def update_ca_trust():
+    if not shutil.which("update-ca-trust"):
+        return
+
+    # generate /etc/pki/tls/certs/ca-bundle.crt
+    os.makedirs("/etc/pki/ca-trust/extracted/pem")
+    os.makedirs("/etc/pki/tls/certs")
+    os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
+
+    # allow to fail, because it sometimes mysteriously does
+    subprocess.run(["update-ca-trust", "extract"], check=False)
+
+
+def append_certs(cert_conf, dir_fd, parents=b""):
+    for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
+        if entry.is_file():
+            line = os.path.join(parents, entry.name)
+            cert_conf.write(line)
+            cert_conf.write(b"\n")
+        elif entry.is_dir():
+            append_certs(cert_conf,
+                         os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
+                         os.path.join(parents, entry.name))
+
+
+def update_ca_certificates():
+    if not shutil.which("update-ca-certificates"):
+        return
+
+    # generate /etc/ssl/certs/ca-certificates.crt
+    os.makedirs("/etc/ssl/certs")
+    with open("/etc/ca-certificates.conf", "wb") as f:
+        append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
+    subprocess.run(["update-ca-certificates"], check=True)
+
+
+def tmpfiles():
+    # Allow systemd-tmpfiles to return non-0. Some packages want to create
+    # directories owned by users that are not set up with systemd-sysusers.
+    subprocess.run(["systemd-tmpfiles", "--create"], check=False)
+
+
+def nsswitch():
+    # the default behavior is fine, but using nss-resolve does not
+    # necessarily work in a non-booted container, so make sure that
+    # is not configured.
+    try:
+        os.remove("/etc/nsswitch.conf")
+    except FileNotFoundError:
+        pass
+
+
+def setup_stdio():
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+        sock.connect("/run/osbuild/api/osbuild")
+        req = {'method': 'setup-stdio'}
+        sock.send(json.dumps(req).encode('utf-8'))
+        msg, fds, _ = load_fds(sock, 1024)
+        for io in ['stdin', 'stdout', 'stderr']:
+            target = getattr(sys, io)
+            source = fds[msg[io]]
+            os.dup2(source, target.fileno())
+            os.close(source)
+
+
+if __name__ == "__main__":
+    setup_stdio()
+    ldconfig()
+    sysusers()
+    update_ca_trust()
+    update_ca_certificates()
+    tmpfiles()
+    nsswitch()
+
+    r = subprocess.run(sys.argv[1:], check=False)
+    sys.exit(r.returncode)

samples/build-from-yum.json

@@ -1,23 +0,0 @@
-{
-  "stages": [
-    {
-      "name": "org.osbuild.yum",
-      "options": {
-        "releasever": "27",
-        "basearch": "x86_64",
-        "repos": [
-          {
-            "baseurl": "https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/$releasever/Everything/$basearch/os/",
-            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFiskqMBEADTbsoAXpDPk+FtcwBEPZQVe0YQYdOqavfQQVD/RYAcHnJW/K1b\nZhQusBjUIec9SfGi3uBNNmbziAvpd/ycMKyWHuWQLmBgbImrqnPbBPMXmxeNGnZj\nA1hjVDp0pzj2+gZQhqYWSf6kQy9u9A1mSU63Kl/tfw7+hX7Tc3I8feGAFCHcFQgE\nSxUib8Mw/OOGR3Am9fKdA+K1kJeQIiZvXMcNFx+3CfoavhFdicuoT2KbcSuzRm76\nduKNHlLaP6/IbZxNiDWh8SDVpFaFPlqR/R/+wibA6e9wMf6CZ4vfUY7NKYf4tYBs\n0EYdkn3j/KhJJxdb+M46Q/xwq9ovZo7XIhLrIUPuMw91X9cbvkU/a9kE1ffdpNmF\n1fDnUcEkuuEqOl+aMVsUBEbAQ86yrwpDfL4XT9vwnDIkggKZyvDTZ6q00XKg3Ger\nKuZtQBl/YcHDXuBlB1fzpGl8a8hq/+GeT2sVxjlYwPXjrsKd1NeP6ctQsR6gHuP3\nW5ohP4rArtM6ONN8rlTiodDLVGHBpUzIRdgr7RCL1AqB9vrdQ2MVZMasTcUnUvKo\n0H4mps+x05jGao0b0Z0TJc6wr6ybHH38NVG06VX5rJZlfZchGwkWzmYxai55/7ln\n1vJmk+kbdS7pK0jfmeVJ8A77XLCL36oJFCiCrYjV+ZGgvB+z08Jzwc7sgwARAQAB\ntCxGZWRvcmEgMjcgKDI3KSA8ZmVkb3JhLTI3QGZlZG9yYXByb2plY3Qub3JnPokC\nOAQTAQIAIgUCWKySowIbDwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQ9V50\nMPUoLuTcww/+LFPVEyVguMeU/QABEsE5FEN7kcDReZtdwq7p/aKC29mzCxeHggit\nYOGlrINkJ26Aq6p+oW6w7JxBWJnKoTBYJDFzNIbp/6GbG4oKcEnWQZfTnRLTr5au\nkVdWBFevzC0huraobKz3joYRIX826VUzS/A418zpnDVPtpo3x86V9f292rqi2tn9\nQ5cQC5Ck3/cjQDEwkN9gHz4j/c1oa6zBOcHbKkaZdWA2dIs6XOxCIHg78i6VQwMM\ns+vfm2vbV3ACCcOVnd3d6NxIQuDLEQwdtdB2zI8R74bjacosrcafK+F2DnkM7WrL\nSCiTKLJBMRDx+X2nOjT5pLuts4FC/XYRO23SMtPAMzQ852Z1lkjsaVDsjzNqCasU\nB0vDPHLQE8aj0zchNBSzuHoKpXNYTyJztekWL/QXkUsXu0x7N5WhBlZ+lni6LtZU\n51l7BJd1n+ZKnQ4gmjQ1ffVLbgzb9Z1MNje0s61FdKmUJQGULYqh32W4GV+RLvtI\n6AJV/EmFCUEfRJ3eA8tJiyKe512wiim/WDhvzFuuPBup2Z3TufiaJQOysLlN5+HU\nQitTtcd7j8ZgpsIAIZtSWOMxbIAJWbzn8gjfIj4ZDeo0ZZXH0VgDkXtpv1g8R2aR\nAz6ob5YYW5VnI2UEr53x1Z7lmUhv/TUZn26IeU16jCJ80k7pJvQSF8Y=\n=xpp1\n-----END PGP PUBLIC KEY BLOCK-----\n"
-          }
-        ],
-        "packages": [
-          "dnf",
-          "systemd",
-          "tar",
-          "gnupg"
-        ]
-      }
-    }
-  ]
-}

samples/f27-build-from-ubuntu1804.json

@@ -0,0 +1,29 @@
+{
+  "pipeline": {
+    "build": {
+      "runner": "org.osbuild.ubuntu1804"
+    },
+    "stages": [
+      {
+        "name": "org.osbuild.yum",
+        "options": {
+          "releasever": "27",
+          "basearch": "x86_64",
+          "repos": [
+            {
+              "baseurl": "https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/$releasever/Everything/$basearch/os/",
+              "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFiskqMBEADTbsoAXpDPk+FtcwBEPZQVe0YQYdOqavfQQVD/RYAcHnJW/K1b\nZhQusBjUIec9SfGi3uBNNmbziAvpd/ycMKyWHuWQLmBgbImrqnPbBPMXmxeNGnZj\nA1hjVDp0pzj2+gZQhqYWSf6kQy9u9A1mSU63Kl/tfw7+hX7Tc3I8feGAFCHcFQgE\nSxUib8Mw/OOGR3Am9fKdA+K1kJeQIiZvXMcNFx+3CfoavhFdicuoT2KbcSuzRm76\nduKNHlLaP6/IbZxNiDWh8SDVpFaFPlqR/R/+wibA6e9wMf6CZ4vfUY7NKYf4tYBs\n0EYdkn3j/KhJJxdb+M46Q/xwq9ovZo7XIhLrIUPuMw91X9cbvkU/a9kE1ffdpNmF\n1fDnUcEkuuEqOl+aMVsUBEbAQ86yrwpDfL4XT9vwnDIkggKZyvDTZ6q00XKg3Ger\nKuZtQBl/YcHDXuBlB1fzpGl8a8hq/+GeT2sVxjlYwPXjrsKd1NeP6ctQsR6gHuP3\nW5ohP4rArtM6ONN8rlTiodDLVGHBpUzIRdgr7RCL1AqB9vrdQ2MVZMasTcUnUvKo\n0H4mps+x05jGao0b0Z0TJc6wr6ybHH38NVG06VX5rJZlfZchGwkWzmYxai55/7ln\n1vJmk+kbdS7pK0jfmeVJ8A77XLCL36oJFCiCrYjV+ZGgvB+z08Jzwc7sgwARAQAB\ntCxGZWRvcmEgMjcgKDI3KSA8ZmVkb3JhLTI3QGZlZG9yYXByb2plY3Qub3JnPokC\nOAQTAQIAIgUCWKySowIbDwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQ9V50\nMPUoLuTcww/+LFPVEyVguMeU/QABEsE5FEN7kcDReZtdwq7p/aKC29mzCxeHggit\nYOGlrINkJ26Aq6p+oW6w7JxBWJnKoTBYJDFzNIbp/6GbG4oKcEnWQZfTnRLTr5au\nkVdWBFevzC0huraobKz3joYRIX826VUzS/A418zpnDVPtpo3x86V9f292rqi2tn9\nQ5cQC5Ck3/cjQDEwkN9gHz4j/c1oa6zBOcHbKkaZdWA2dIs6XOxCIHg78i6VQwMM\ns+vfm2vbV3ACCcOVnd3d6NxIQuDLEQwdtdB2zI8R74bjacosrcafK+F2DnkM7WrL\nSCiTKLJBMRDx+X2nOjT5pLuts4FC/XYRO23SMtPAMzQ852Z1lkjsaVDsjzNqCasU\nB0vDPHLQE8aj0zchNBSzuHoKpXNYTyJztekWL/QXkUsXu0x7N5WhBlZ+lni6LtZU\n51l7BJd1n+ZKnQ4gmjQ1ffVLbgzb9Z1MNje0s61FdKmUJQGULYqh32W4GV+RLvtI\n6AJV/EmFCUEfRJ3eA8tJiyKe512wiim/WDhvzFuuPBup2Z3TufiaJQOysLlN5+HU\nQitTtcd7j8ZgpsIAIZtSWOMxbIAJWbzn8gjfIj4ZDeo0ZZXH0VgDkXtpv1g8R2aR\nAz6ob5YYW5VnI2UEr53x1Z7lmUhv/TUZn26IeU16jCJ80k7pJvQSF8Y=\n=xpp1\n-----END PGP PUBLIC KEY BLOCK-----\n"
+            }
+          ],
+          "packages": [
+            "dnf",
+            "systemd",
+            "tar",
+            "gnupg"
+          ]
+        }
+      }
+    ]
+  },
+  "runner": "org.osbuild.fedora27"
+}

samples/ubuntu1804.json

@@ -0,0 +1,3 @@
+{
+  "runner": "org.osbuild.ubuntu1804"
+}

test/osbuildtest.py

@@ -35,10 +35,10 @@ class TestCase(unittest.TestCase):
     def run_osbuild(self, pipeline, input=None):
         osbuild_cmd = ["python3", "-m", "osbuild", "--json", "--store", self.store, "--libdir", ".", pipeline]
 
-        build_pipeline = os.getenv("OSBUILD_TEST_BUILD_PIPELINE", None)
-        if build_pipeline:
-            osbuild_cmd.append("--build-pipeline")
-            osbuild_cmd.append(build_pipeline)
+        build_env = os.getenv("OSBUILD_TEST_BUILD_ENV", None)
+        if build_env:
+            osbuild_cmd.append("--build-env")
+            osbuild_cmd.append(build_env)
 
         stdin = subprocess.PIPE if input else None
 

test/pipelines/f30-base.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -26,6 +27,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/pipelines/f30-boot.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -26,6 +27,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/chrony/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/chrony/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/firewall/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/firewall/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/fstab/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/fstab/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/groups/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/groups/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/hostname/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/hostname/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/keymap/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/keymap/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/locale/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/locale/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/systemd/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/systemd/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/timezone/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/timezone/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/users/a.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/stages_tests/users/b.json

@@ -1,5 +1,6 @@
 {
   "build": {
+    "pipeline": {
       "stages": [
         {
           "name": "org.osbuild.dnf",
@@ -21,6 +22,8 @@
         }
       ]
     },
+    "runner": "org.osbuild.fedora30"
+  },
   "stages": [
     {
       "name": "org.osbuild.dnf",

test/test_osbuild.py

@@ -44,15 +44,16 @@ class TestDescriptions(unittest.TestCase):
                 self.assertEqual(assembler.description(), description)
 
     def test_pipeline(self):
-        build = osbuild.Pipeline()
+        build = osbuild.Pipeline("org.osbuild.test")
         build.add_stage("org.osbuild.test", { "one": 1 })
 
-        pipeline = osbuild.Pipeline(build)
+        pipeline = osbuild.Pipeline("org.osbuild.test", build)
         pipeline.add_stage("org.osbuild.test", { "one": 2 })
         pipeline.set_assembler("org.osbuild.test")
 
         self.assertEqual(pipeline.description(), {
               "build": {
+                "pipeline": {
                   "stages": [
                     {
                       "name": "org.osbuild.test",
@@ -60,6 +61,8 @@ class TestDescriptions(unittest.TestCase):
                     }
                   ]
                 },
+                "runner": "org.osbuild.test"
+              },
               "stages": [
                 {
                   "name": "org.osbuild.test",