diff --git a/containers/osbuild-composer/Dockerfile b/containers/osbuild-composer/Dockerfile new file mode 100644 index 000000000..2d49bb6da --- /dev/null +++ b/containers/osbuild-composer/Dockerfile @@ -0,0 +1,71 @@ +# +# osbuild-composer - Containerized OSBuild Composer +# +# This container provides a minimal fedora image with the osbuild-composer +# application installed and configured as default entrypoint. +# +# Build Arguments: +# +# * OSB_FROM +# This specifies the host image to use. It must be an RPM-based +# distribution image with all osbuild-composer requirements +# pre-installed. +# +# Example: "docker.io/library/fedora:latest" +# +# * OSB_RPMREPO +# Base URL of an RPM repository from which to install osbuild-composer +# from. +# +# Example: "https://dl01.fedoraproject.org/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/" +# + +# Image arguments must be imported before `FROM`. +ARG OSB_FROM="docker.io/library/fedora:latest" + +# Prepare our host environment. +FROM "${OSB_FROM}" AS base + +# Import build parameters. +ARG OSB_RPMREPO="https://dl01.fedoraproject.org/pub/fedora/linux/releases/\$releasever/Everything/\$basearch/os/" + +# Create our state directory and use it as anchor. +WORKDIR "/var/lib/osb" + +# Create and switch into our src directory, which we use as temporary storage +# for all sources during the install. +WORKDIR "./src" + +# Install all global dependencies. +RUN \ + dnf \ + -y \ + "--repofrompath=ephemeral0,${OSB_RPMREPO}" \ + "--setopt=ephemeral0.gpgcheck=0" \ + "--setopt=ephemeral0.priority=10" \ + install "osbuild-composer" \ + && dnf clean all + +# Copy all our local sources, so we can access them from within the container +# build. They will be cleaned in a later step. +COPY "." "." + +# Prepare the runtime configuration and state. +RUN mkdir -p "../bin" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/run/weldr/" +RUN mkdir -p "/var/cache/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-worker/" +RUN mkdir -p "/var/lib/osbuild-composer/" + +# Install all required sources into the persistent directory. +RUN cp "entrypoint.py" "../bin/" + +# Leave and delete our temporary source directory. +WORKDIR ".." +RUN rm -rf "./src" + +# Prepare the runtime entrypoint and empty working directory. +WORKDIR "./workdir" +ENTRYPOINT ["python3", "../bin/entrypoint.py"] diff --git a/containers/osbuild-composer/entrypoint.py b/containers/osbuild-composer/entrypoint.py new file mode 100644 index 000000000..7645bdf99 --- /dev/null +++ b/containers/osbuild-composer/entrypoint.py @@ -0,0 +1,287 @@ +"""entrypoint - Containerized OSBuild Composer + +This provides the entrypoint for a containerized osbuild-composer image. It +spawns `osbuild-composer` on start and manages it until it exits. The main +purpose of this entrypoint is to prepare everything to be usable from within +a container. +""" + +import argparse +import contextlib +import os +import socket +import subprocess +import sys + + +class Cli(contextlib.AbstractContextManager): + """Command Line Interface""" + + def __init__(self, argv): + self.args = None + self._argv = argv + self._exitstack = None + self._parser = None + + def _parse_args(self): + self._parser = argparse.ArgumentParser( + add_help=True, + allow_abbrev=False, + argument_default=None, + description="Containerized OSBuild Composer", + prog="container/osbuild-composer", + ) + + # --[no-]builtin-worker + self._parser.add_argument( + "--builtin-worker", + action="store_true", + dest="builtin_worker", + help="Enable built-in local worker", + ) + self._parser.add_argument( + "--no-builtin-worker", + action="store_false", + dest="builtin_worker", + help="Disable built-in local worker", + ) + + # --[no-]composer-api + self._parser.add_argument( + "--composer-api", + action="store_true", + dest="composer_api", + help="Enable the composer-API", + ) + self._parser.add_argument( + "--no-composer-api", + action="store_false", + dest="composer_api", + help="Disable the composer-API", + ) + + # --[no-]local-worker-api + self._parser.add_argument( + "--local-worker-api", + action="store_true", + dest="local_worker_api", + help="Enable the local-worker-API", + ) + self._parser.add_argument( + "--no-local-worker-api", + action="store_false", + dest="local_worker_api", + help="Disable the local-worker-API", + ) + + # --[no-]remote-worker-api + self._parser.add_argument( + "--remote-worker-api", + action="store_true", + dest="remote_worker_api", + help="Enable the remote-worker-API", + ) + self._parser.add_argument( + "--no-remote-worker-api", + action="store_false", + dest="remote_worker_api", + help="Disable the remote-worker-API", + ) + + # --[no-]weldr-api + self._parser.add_argument( + "--weldr-api", + action="store_true", + dest="weldr_api", + help="Enable the weldr-API", + ) + self._parser.add_argument( + "--no-weldr-api", + action="store_false", + dest="weldr_api", + help="Disable the weldr-API", + ) + + self._parser.set_defaults( + builtin_worker=False, + composer_api=False, + local_worker_api=False, + remote_worker_api=False, + weldr_api=False, + ) + + return self._parser.parse_args(self._argv[1:]) + + def __enter__(self): + self._exitstack = contextlib.ExitStack() + self.args = self._parse_args() + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self._exitstack.close() + self._exitstack = None + + def _prepare_sockets(self): + # Prepare all the API sockets that osbuild-composer expectes, and make + # sure to pass them according to the systemd socket-activation API. + # + # Note that we rely on this being called early, so we get the correct + # FD numbers assigned. We need FD-#3 onwards for compatibility with + # socket activation (because python `subprocess.Popen` does not support + # renumbering the sockets we pass down). + + index = 3 + sockets = [] + names = [] + + # osbuild-composer.socket + if self.args.weldr_api: + print("Create weldr-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/weldr/api.socket") + sock.listen() + sockets.append(sock) + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-composer-api.socket + if self.args.composer_api: + print("Create composer-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind(("::", 443)) + sock.listen() + sockets.append(sock) + names.append("osbuild-composer-api.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-local-worker.socket + if self.args.local_worker_api: + print("Create local-worker-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/osbuild-composer/job.socket") + sock.listen() + sockets.append(sock) + names.append("osbuild-local-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-remote-worker.socket + if self.args.remote_worker_api: + print("Create remote-worker-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind(("::", 8700)) + sock.listen(256) + sockets.append(sock) + names.append("osbuild-remote-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # Prepare FD environment for the child process. + os.environ["LISTEN_FDS"] = str(len(sockets)) + os.environ["LISTEN_FDNAMES"] = ":".join(names) + + return sockets + + @staticmethod + def _spawn_worker(): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-worker", + "-unix", + "/run/osbuild-composer/job.socket", + ] + + env = os.environ.copy() + env["CACHE_DIRECTORY"] = "/var/cache/osbuild-worker" + env["STATE_DIRECTORY"] = "/var/lib/osbuild-worker" + + return subprocess.Popen( + cmd, + cwd="/", + env=env, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + + @staticmethod + def _spawn_composer(sockets): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-composer", + "-v", + ] + + # Prepare the environment for osbuild-composer. Note that we cannot use + # the `env` parameter of `subprocess.Popen()`, because it conflicts + # with the `preexec_fn=` parameter. Therefore, we have to modify the + # caller's environment. + os.environ["CACHE_DIRECTORY"] = "/var/cache/osbuild-composer" + os.environ["STATE_DIRECTORY"] = "/var/lib/osbuild-composer" + + # We need to set `LISTEN_PID=` to the target PID. The only way python + # allows us to do this is to hook into `preexec_fn=`, which is executed + # by `subprocess.Popen()` after forking, but before executing the new + # executable. + preexec_setenv = lambda: os.putenv("LISTEN_PID", str(os.getpid())) + + return subprocess.Popen( + cmd, + cwd="/usr/libexec/osbuild-composer", + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + pass_fds=[sock.fileno() for sock in sockets], + preexec_fn=preexec_setenv, + ) + + def run(self): + """Program Runtime""" + + proc_composer = None + proc_worker = None + res = 0 + sockets = self._prepare_sockets() + + try: + if self.args.builtin_worker: + proc_worker = self._spawn_worker() + + proc_composer = self._spawn_composer(sockets) + + res = proc_composer.wait() + if proc_worker: + proc_worker.terminate() + proc_worker.wait() + + return res + except KeyboardInterrupt: + if proc_worker: + proc_worker.terminate() + proc_worker.wait() + if proc_composer: + proc_composer.terminate() + res = proc_composer.wait() + except: + if proc_worker: + proc_worker.kill() + if proc_composer: + proc_composer.kill() + raise + + return res + + +if __name__ == "__main__": + with Cli(sys.argv) as global_main: + sys.exit(global_main.run()) diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile index 6ab850079..12aa4dd4e 100644 --- a/schutzbot/Jenkinsfile +++ b/schutzbot/Jenkinsfile @@ -111,6 +111,23 @@ pipeline { } } + stage("Container build") { + parallel { + stage('aarch64') { + agent { label "f33cloudbase && aarch64 && aws" } + steps { + sh "schutzbot/containerbuild.sh" + } + } + stage('x86_64') { + agent { label "f33cloudbase && x86_64 && aws" } + steps { + sh "schutzbot/containerbuild.sh" + } + } + } + } + stage("Testing 🍌") { parallel { diff --git a/schutzbot/containerbuild.sh b/schutzbot/containerbuild.sh new file mode 100755 index 000000000..192f88cd4 --- /dev/null +++ b/schutzbot/containerbuild.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + + +# Query host information. +echo "Query host" + +ARCH=$(uname -m) +COMMIT=$(git rev-parse HEAD) + + +# Populate our build matrix. +IMG_TAGS=( + "quay.io/osbuild/osbuild-composer:f32-${COMMIT}" + "quay.io/osbuild/osbuild-composer:f33-${COMMIT}" +) +IMG_PATHS=( + "./containers/osbuild-composer/" + "./containers/osbuild-composer/" +) +IMG_FROMS=( + "docker.io/library/fedora:32" + "docker.io/library/fedora:33" +) +IMG_RPMREPOS=( + "http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com/osbuild-composer/fedora-32/${ARCH}/${COMMIT}" + "http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com/osbuild-composer/fedora-33/${ARCH}/${COMMIT}" +) +IMG_COUNT=${#IMG_TAGS[*]} + + +# Prepare host system. +echo "Prepare host system" + +sudo dnf -y install podman + + +# Build the entire matrix. +echo "Build containers" + +for ((i=0; i