schutzbot: add koji integration tests

This sets up containers running koji and supporting infrastructure, and
calls the osbuild-composer-koji API to build and image and push it into
our testing instance.

koji-compose.py and various fixes by Christian Kellner.

Signed-off-by: Tom Gundersen <teg@jklm.no>

internal/upload/koji/run-koji-container.sh

@@ -89,7 +89,10 @@ koji_start() {
     quay.io/osbuild/koji:v1
 
   # TODO: we need to wait for the database to be initialized here. A better method should be used.
-  sleep 2
+  sleep 10
+
+  ${CONTAINER_RUNTIME} logs org.osbuild.koji.postgres
+  ${CONTAINER_RUNTIME} logs org.osbuild.koji.koji
 
   # create koji users
   # kojiadmin/kojipass    - admin

osbuild-composer.spec

@@ -232,6 +232,7 @@ systemctl stop "osbuild-worker@*.service" "osbuild-remote-worker@*.service"
 %package tests
 Summary:    Integration tests
 Requires:   %{name} = %{version}-%{release}
+Requires:   %{name}-koji = %{version}-%{release}
 Requires:   composer-cli
 Requires:   createrepo_c
 Requires:   genisoimage

schutzbot/Jenkinsfile

@@ -552,6 +552,12 @@ void run_tests(test_type) {
     }
 
     if (test_type == 'integration') {
+        // Run Koji tests.
+        sh (
+            label: "Koji tests",
+            script: "test/image-tests/koji.sh"
+        )
+
         // Run the qcow2 test.
         sh (
             label: "Integration test: QCOW2",

schutzbot/deploy.sh

@@ -55,7 +55,7 @@ sudo dnf repository-packages osbuild-mock list
 
 # Install the Image Builder packages.
 # Note: installing only -tests to catch missing dependencies
-retry sudo dnf -y install osbuild-composer-tests
+retry sudo dnf -y install osbuild-composer-tests make
 
 # Set up a directory to hold repository overrides.
 sudo mkdir -p /etc/osbuild-composer/repositories
@@ -69,8 +69,13 @@ if [[ -f "rhel-8-beta.json" ]]; then
     sudo cp rhel-8-beta.json /etc/osbuild-composer/repositories/
 fi
 
+# Generate SSL certificates
+sudo make composer-key-pair
+sudo make worker-key-pair
+
 # Start services.
 sudo systemctl enable --now osbuild-composer.socket
+sudo systemctl enable --now osbuild-composer-koji.socket
 
 # Verify that the API is running.
 sudo composer-cli status show

schutzbot/vendor/87-podman-bridge.conflist

@@ -0,0 +1,41 @@
+{
+  "cniVersion": "0.4.0",
+  "name": "podman",
+  "plugins": [
+    {
+      "type": "bridge",
+      "bridge": "cni-podman0",
+      "isGateway": true,
+      "ipMasq": true,
+      "hairpinMode": true,
+      "ipam": {
+        "type": "host-local",
+        "routes": [{ "dst": "0.0.0.0/0" }],
+        "ranges": [
+          [
+            {
+              "subnet": "10.88.0.0/16",
+              "gateway": "10.88.0.1"
+            }
+          ]
+        ]
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+        "portMappings": true
+      }
+    },
+    {
+      "type": "firewall"
+    },
+    {
+      "type": "tuning"
+    },
+    {
+      "domainName": "dns.podman",
+      "type": "dnsname"
+    }
+  ]
+}

schutzbot/vendor/README.md

@@ -0,0 +1,9 @@
+This directory includes 3rd party modules, needed in CI.
+
+ - [`dnsname`](https://github.com/containers/dnsname) plugin for podman,
+   needed to translate host names of containers into IPs. It is shipped
+   in Fedora, but missing in RHEL 8, see
+   [rhgbz#1877865](https://bugzilla.redhat.com/show_bug.cgi?id=1877865).
+   The `87-podman-bridge.conflist` file contains the corresponding config,
+   where the `{"domainName": "dns.podman", "type": "dnsname"}` bit is the
+   newly added part.

test/image-tests/koji-compose.py

@@ -0,0 +1,86 @@
+#!/usr/bin/python3
+import json
+import sys
+import time
+
+import requests
+
+DISTRO_BASEURLS = {
+    "fedora-31": ["http://download.fedoraproject.org/pub/fedora/linux/releases/31/Everything/x86_64/os/"],
+    "fedora-32": ["http://download.fedoraproject.org/pub/fedora/linux/releases/32/Everything/x86_64/os/"],
+    "rhel-8": [
+        "http://download.devel.redhat.com/released/RHEL-8/8.2.0/BaseOS/x86_64/os/",
+        "http://download.devel.redhat.com/released/RHEL-8/8.2.0/AppStream/x86_64/os/",
+
+    ]
+}
+
+
+def compose_request(distro, koji):
+    repositories = [{"baseurl": baseurl} for baseurl in DISTRO_BASEURLS[distro]]
+
+    req = {
+        "name": "name",
+        "version": "version",
+        "release": "release",
+        "distribution": distro,
+        "koji": {
+            "server": koji,
+            "task_id": 1
+        },
+        "image_requests": [{
+            "architecture": "x86_64",
+            "image_type": "qcow2",
+            "repositories": repositories
+        }]
+    }
+
+    return req
+
+
+def main(distro):
+    cr = compose_request(distro, "https://localhost/kojihub")
+    print(json.dumps(cr))
+
+    r = requests.post("https://localhost:8701/compose", json=cr,
+                      cert=("/etc/osbuild-composer/worker-crt.pem", "/etc/osbuild-composer/worker-key.pem"),
+                      verify="/etc/osbuild-composer/ca-crt.pem")
+    if r.status_code != 201:
+        print("Failed to create compose")
+        print(r.text)
+        sys.exit(1)
+
+    print(r.text)
+    compose_id = r.json()["id"]
+
+    while True:
+        r = requests.get(f"https://localhost:8701/compose/{compose_id}",
+                         cert=("/etc/osbuild-composer/worker-crt.pem", "/etc/osbuild-composer/worker-key.pem"),
+                         verify="/etc/osbuild-composer/ca-crt.pem")
+        if r.status_code != 200:
+            print("Failed to get compose status")
+            print(r.text)
+            sys.exit(1)
+        status = r.json()["status"]
+        print(status)
+        if status == "success":
+            print("Compose worked!")
+            print(r.text)
+            break
+        elif status == "failure":
+            print("compose failed!")
+            print(r.text)
+            sys.exit(1)
+        elif status != "pending" and status != "running":
+            print(f"unexpected status: {status}")
+            print(r.text)
+            sys.exit(1)
+
+        time.sleep(10)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print(f"usage: {sys.argv[0]} DISTRO", file=sys.stderr)
+        sys.exit(1)
+    main(sys.argv[1])

test/image-tests/koji.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -euo pipefail
+
+# Get OS data.
+source /etc/os-release
+ARCH=$(uname -m)
+
+# Colorful output.
+function greenprint {
+    echo -e "\033[1;32m${1}\033[0m"
+}
+
+if [[ $ID == rhel ]] && ! rpm -q epel-release; then
+    greenprint "📦 Setting up EPEL repository"
+    curl -Ls --retry 5 --output /tmp/epel.rpm \
+        https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+    sudo rpm -Uvh /tmp/epel.rpm
+fi
+
+greenprint "Installing required packages"
+sudo dnf -y install \
+    container-selinux \
+    dnsmasq \
+    krb5-workstation \
+    koji \
+    podman \
+    python3 \
+    sssd-krb5
+
+if [[ $ID == rhel ]]; then
+  greenprint "Tweaking podman, maybe."
+  sudo cp schutzbot/vendor/87-podman-bridge.conflist /etc/cni/net.d/
+  sudo cp schutzbot/vendor/dnsname /usr/libexec/cni/
+fi
+
+greenprint "Starting containers"
+sudo ./internal/upload/koji/run-koji-container.sh start
+
+greenprint "Adding kerberos config"
+sudo cp \
+    /tmp/osbuild-composer-koji-test/client.keytab \
+    /etc/krb5.keytab
+sudo cp \
+    test/image-tests/krb5-local.conf \
+    /etc/krb5.conf.d/local
+
+greenprint "Initializing Kerberos"
+kinit osbuild-krb@LOCAL -k
+sudo -u _osbuild-composer kinit osbuild-krb@LOCAL -k
+
+greenprint "Adding generated CA cert for Koji"
+sudo cp \
+    /tmp/osbuild-composer-koji-test/ca-crt.pem \
+    /etc/pki/ca-trust/source/anchors/koji-ca-crt.pem
+sudo update-ca-trust
+
+greenprint "Restarting composer to pick up new certs"
+sudo systemctl restart osbuild-composer
+
+greenprint "Testing Koji"
+koji --server=http://localhost/kojihub --user=osbuild --password=osbuildpass --authtype=password hello
+koji --server=http://localhost/kojihub hello
+sudo -u _osbuild-composer koji --server=http://localhost/kojihub hello
+
+greenprint "Creating Koji task"
+koji --server=http://localhost/kojihub --user kojiadmin --password kojipass --authtype=password make-task image
+
+greenprint "Pushing compose to Koji"
+sudo ./test/image-tests/koji-compose.py "${ID}-${VERSION_ID%.*}"
+
+greenprint "Show Koji task"
+koji --server=http://localhost/kojihub taskinfo 1
+koji --server=http://localhost/kojihub buildinfo 1
+
+greenprint "Stopping containers"
+sudo ./internal/upload/koji/run-koji-container.sh stop
+
+greenprint "Removing generated CA cert"
+sudo rm \
+    /etc/pki/ca-trust/source/anchors/koji-ca-crt.pem
+sudo update-ca-trust

test/image-tests/krb5-local.conf

@@ -0,0 +1,8 @@
+[realms]
+    LOCAL = {
+        kdc = localhost
+        admin_server = localhost
+    }
+
+[domain_realm]
+    localhost = LOCAL