upload/koji: add a simple test

This commit adds a test which uploads a random file to Koji and runs CGImport.
The result is checked using the koji cli client.

.github/koji.conf

@@ -0,0 +1,68 @@
+; Koji installed from pip is missing the default config.
+; This one is taken from Fedora 32 and installed in Github Actions.
+[koji]
+
+;configuration for koji cli tool
+
+;url of XMLRPC server
+;server = http://hub.example.com/kojihub
+server = https://koji.fedoraproject.org/kojihub
+
+;url of web interface
+;weburl = http://www.example.com/koji
+weburl = https://koji.fedoraproject.org/koji
+
+;url of package download site
+;pkgurl = http://www.example.com/packages
+
+;path to the koji top directory
+;topdir = /mnt/koji
+topurl = https://kojipkgs.fedoraproject.org/
+
+;configuration for Kerberos authentication
+authtype = kerberos
+krb_rdns = false
+
+;the service name of the principal being used by the hub
+;krbservice = host
+
+;the principal to auth as for automated clients
+;principal = client@EXAMPLE.COM
+
+;the keytab to auth as for automated clients
+;keytab = /etc/krb5.keytab
+
+;enable to lookup dns canonical hostname for krb auth
+;krb_canon_host = no
+
+;The realm of server principal. Using client's realm if not set
+;krb_server_realm = EXAMPLE.COM
+
+
+;configuration for SSL authentication
+
+;client certificate
+;cert = ~/.koji/client.crt
+
+;certificate of the CA that issued the HTTP server certificate
+;serverca = ~/.koji/serverca.crt
+
+;plugin paths, separated by ':' as the same as the shell's PATH
+;koji_cli_plugins module and ~/.koji/plugins are always loaded in advance,
+;and then be overridden by this option
+;plugin_paths = ~/.koji/plugins
+
+;[not_implemented_yet]
+;enabled plugins for CLI, runroot and save_failed_tree are available
+;plugins =
+; runroot plugin is enabled by default in fedora
+plugins = runroot
+
+;timeout of XMLRPC requests by seconds, default: 60 * 60 * 12 = 43200
+;timeout = 43200
+
+;timeout of GSSAPI/SSL authentication by seconds, default: 60
+;auth_timeout = 60
+
+; use the fast upload feature of koji by default
+use_fast_upload = yes

.github/workflows/tests.yml

@@ -136,3 +136,54 @@ jobs:
 
       - name: "🔎 Test RPM installation"
         run: dnf -y install $(ls rpms/*.x86_64.rpm)
+
+  koji:
+    name: "麹 Koji test"
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: docker.io/library/postgres:12-alpine
+        env:
+          POSTGRES_USER:     koji
+          POSTGRES_PASSWORD: kojipass
+          POSTGRES_DB:       koji
+      koji:
+        image: quay.io/osbuild/ghci-koji:v1
+        env:
+          POSTGRES_USER:     koji
+          POSTGRES_PASSWORD: kojipass
+          POSTGRES_DB:       koji
+          POSTGRES_HOST:     postgres
+        ports:
+          - 8080:80
+    steps:
+      - name: Set up Go 1.12
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.12
+        id: go
+
+        # We need python for koji client.
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+        # Koji from pip is missing its config file.
+        # To fix this, the default Fedora 32 config is included in the repo
+        # and installed here. See the last line of the script.
+      - name: Install koji client
+        run: |
+          sudo apt-get install -y libkrb5-dev
+          python -m pip install --upgrade pip
+          pip install koji
+          sudo cp .github/koji.conf /etc/koji.conf
+
+      - name: Run unit tests
+        run: go test -v -race -covermode atomic -coverprofile=coverage.txt -tags koji_test ./internal/upload/koji
+
+      - name: Send coverage to codecov.io
+        run: bash <(curl -s https://codecov.io/bash)

internal/upload/koji/README.md

@@ -0,0 +1,22 @@
+## How to run the Koji test
+
+Firstly, you need to start the koji container:
+
+```
+sudo ./internal/upload/koji/run-koji-container.sh
+```
+
+This command starts a kojihub instance available at
+http://localhost:8080/kojihub . You can test that it started successfully
+by running:
+```
+koji --server=http://localhost:8080/kojihub --user=osbuild --password=osbuildpass --authtype=password hello
+```
+
+Now, you can run the koji test using:
+```
+go test -v -tags koji_test ./internal/upload/koji
+```
+
+The test is run on each PR in the Github CI. See `.github/workflows/tests.yml`
+for more details.

internal/upload/koji/koji_test.go

@@ -0,0 +1,132 @@
+//+build koji_test
+
+package koji_test
+
+import (
+	"crypto/rand"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/osbuild/osbuild-composer/internal/upload/koji"
+)
+
+func TestKojiImport(t *testing.T) {
+	// define constants
+	server := "http://localhost:8080/kojihub"
+	user := "osbuild"
+	password := "osbuildpass"
+	filename := "image.qcow2"
+	filesize := 1024
+	// you cannot create two build with a same name, let's create a random one each time
+	buildName := "osbuild-image-" + uuid.Must(uuid.NewRandom()).String()
+	// koji needs to specify a directory to which the upload should happen, let's reuse the build name
+	uploadDirectory := buildName
+
+	// authenticate
+	k, err := koji.New(server)
+	require.NoError(t, err)
+	err = k.Login(user, password)
+	require.NoError(t, err)
+
+	defer func() {
+		err := k.Logout()
+		if err != nil {
+			require.NoError(t, err)
+		}
+	}()
+
+	// Create a random file
+	f, err := ioutil.TempFile("", "osbuild-koji-test-*.qcow2")
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, f.Close())
+		assert.NoError(t, os.Remove(f.Name()))
+	}()
+
+	_, err = io.CopyN(f, rand.Reader, int64(filesize))
+	require.NoError(t, err)
+	_, err = f.Seek(0, io.SeekStart)
+	require.NoError(t, err)
+
+	// Upload the file
+	hash, _, err := k.Upload(f, uploadDirectory, filename)
+	require.NoError(t, err)
+
+	// Import the build
+	build := koji.Build{
+		Name:      buildName,
+		Version:   "1",
+		Release:   "1",
+		StartTime: time.Now().Unix(),
+		EndTime:   time.Now().Unix(),
+	}
+	buildRoots := []koji.BuildRoot{
+		{
+			ID: 1,
+			Host: koji.Host{
+				Os:   "RHEL8",
+				Arch: "noarch",
+			},
+			ContentGenerator: koji.ContentGenerator{
+				Name:    "osbuild",
+				Version: "1",
+			},
+			Container: koji.Container{
+				Type: "nspawn",
+				Arch: "noarch",
+			},
+			Tools:      []koji.Tool{},
+			Components: []koji.Component{},
+		},
+	}
+	output := []koji.Output{
+		{
+			BuildRootID:  1,
+			Filename:     filename,
+			FileSize:     uint64(filesize),
+			Arch:         "noarch",
+			ChecksumType: "md5",
+			MD5:          hash,
+			Type:         "image",
+			Components:   []koji.Component{},
+			Extra: koji.OutputExtra{
+				Image: koji.OutputExtraImageInfo{
+					Arch: "noarch",
+				},
+			},
+		},
+	}
+
+	result, err := k.CGImport(build, buildRoots, output, uploadDirectory)
+	require.NoError(t, err)
+
+	// check if the build is really there:
+	cmd := exec.Command(
+		"koji",
+		"--server", server,
+		"--user", user,
+		"--password", password,
+		"--authtype", "password",
+		"list-builds",
+		"--buildid", strconv.Itoa(result.BuildID),
+	)
+
+	// sample output:
+	// Build                                                    Built by          State
+	// -------------------------------------------------------  ----------------  ----------------
+	// osbuild-image-92882b90-4bd9-4422-8b8a-40863f94535a-1-1   osbuild           COMPLETE
+	out, err := cmd.CombinedOutput()
+	assert.NoError(t, err)
+
+	// let's check for COMPLETE, koji will exit with non-zero status code if the build doesn't exist
+	assert.Contains(t, string(out), "COMPLETE")
+}

internal/upload/koji/run-koji-container.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -eu
+
+if [ $UID != 0 ]; then
+  echo must be run as root
+  exit 1
+fi
+
+clean_up () {
+    EXIT_CODE=$?
+
+    echo "Shutting down containers, please wait..."
+
+    podman stop org.osbuild.koji.koji || true
+    podman rm org.osbuild.koji.koji || true
+
+    podman stop org.osbuild.koji.postgres || true
+    podman rm org.osbuild.koji.postgres || true
+
+    podman network rm -f org.osbuild.koji || true
+
+    exit $EXIT_CODE
+}
+
+trap clean_up EXIT
+
+podman network create org.osbuild.koji
+podman run -d --name org.osbuild.koji.postgres --network org.osbuild.koji \
+  -e POSTGRES_USER=koji \
+  -e POSTGRES_PASSWORD=kojipass \
+  -e POSTGRES_DB=koji \
+  docker.io/library/postgres:12-alpine
+
+podman run -d --name org.osbuild.koji.koji --network org.osbuild.koji \
+  -p 8080:80 \
+  -e POSTGRES_USER=koji \
+  -e POSTGRES_PASSWORD=kojipass \
+  -e POSTGRES_DB=koji \
+  -e POSTGRES_HOST=org.osbuild.koji.postgres \
+  quay.io/osbuild/ghci-koji:rc1
+
+echo "Running, press CTRL+C to stop..."
+sleep infinity