koji_test.py: test upload to cloud with AWS

Extend the integration test with a new case, testing that direct upload
to the cloud works for Koji composes. Test this using a single cloud
provider, specifically AWS.

The test case submits a new osbuild-image build using Koji CLI,
determines the image information once the build finishes and then checks
that such image exists in AWS. The image is then deleted as part of the
test case tear-down.

The AWS credentials are now configured in the worker's configuration, if
the appropriate environment variables are set.

Update the SPEC file with a new test dependency and update the required
osbuild-composer version.

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Install test dependencies
-      run: dnf -y install python3-flexmock python3-httpretty python3-jsonschema python3-koji python3-pylint python3-requests
+      run: dnf -y install python3-boto3 python3-flexmock python3-httpretty python3-jsonschema python3-koji python3-pylint python3-requests
 
     - name: Check out code
       uses: actions/checkout@v3

koji-osbuild.spec

@@ -143,10 +143,11 @@ Requires:       jq
 Requires:       koji
 Requires:       krb5-workstation
 Requires:       openssl
-Requires:       osbuild-composer >= 22
+Requires:       osbuild-composer >= 58
 Requires:       osbuild-composer-tests
 Requires:       podman
 Requires:       podman-plugins
+Requires:       python3-boto3
 
 %description tests
 Integration tests for koji-osbuild. To be run on a dedicated system.

test/copy-creds.sh

@@ -33,6 +33,25 @@ cp ${TEST_DATA}/osbuild-worker.toml \
 
 echo "koji" > /etc/osbuild-worker/oauth-secret
 
+# if AWS credentials are defined in the ENV, add them to the worker's configuration
+# This is needed to test the upload to the cloud
+V2_AWS_ACCESS_KEY_ID="${V2_AWS_ACCESS_KEY_ID:-}"
+V2_AWS_SECRET_ACCESS_KEY="${V2_AWS_SECRET_ACCESS_KEY:-}"
+if [[ -n "$V2_AWS_ACCESS_KEY_ID" && -n "$V2_AWS_SECRET_ACCESS_KEY" ]]; then
+   echo "Adding AWS credentials to the worker's configuration"
+   sudo tee /etc/osbuild-worker/aws-credentials.toml > /dev/null << EOF
+[default]
+aws_access_key_id = "$V2_AWS_ACCESS_KEY_ID"
+aws_secret_access_key = "$V2_AWS_SECRET_ACCESS_KEY"
+EOF
+   sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF
+
+[aws]
+credentials = "/etc/osbuild-worker/aws-credentials.toml"
+bucket = "${AWS_BUCKET}"
+EOF
+fi
+
 echo "Copying system kerberos configuration"
 cp ${TEST_DATA}/krb5.local.conf \
    /etc/krb5.conf.d/local

test/integration.sh

@@ -39,7 +39,7 @@ greenprint "Testing Koji hub API access"
 koji --server=http://localhost:8080/kojihub --user=osbuild --password=osbuildpass --authtype=password hello
 
 greenprint "Copying credentials, certificates and configuration files"
-sudo /usr/libexec/koji-osbuild-tests/copy-creds.sh /usr/share/koji-osbuild-tests
+sudo -E /usr/libexec/koji-osbuild-tests/copy-creds.sh /usr/share/koji-osbuild-tests
 
 greenprint "Starting mock OpenID server"
 sudo /usr/libexec/koji-osbuild-tests/run-openid.sh start
@@ -63,6 +63,9 @@ greenprint "Creating Koji tag infrastructure"
 /usr/libexec/koji-osbuild-tests/make-tags.sh
 
 greenprint "Running integration tests"
+# export environment variables for the Boto3 client to work out of the box
+AWS_ACCESS_KEY_ID="${V2_AWS_ACCESS_KEY_ID:-}" \
+AWS_SECRET_ACCESS_KEY="${V2_AWS_SECRET_ACCESS_KEY:-}" \
 python3 -m unittest discover -v /usr/libexec/koji-osbuild-tests/integration/
 
 greenprint "Stopping koji builder"

test/integration/test_koji.py

@@ -4,19 +4,38 @@
 
 
 import functools
+import json
+import logging
+import os
 import platform
-import unittest
+import re
+import shutil
 import string
 import subprocess
+import tempfile
+import unittest
+
+import boto3
+from botocore.config import Config as BotoConfig
+from botocore.exceptions import ClientError as BotoClientError
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(format = '%(asctime)s %(levelname)s: %(message)s', level = logging.INFO)
 
 
 def koji_command(*args, _input=None, _globals=None, **kwargs):
+    return koji_command_cwd(*args, _input=_input, _globals=_globals, **kwargs)
+
+
+def koji_command_cwd(*args, cwd=None, _input=None, _globals=None, **kwargs):
     args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()]
     if _globals:
         args = [f'--{k}={v}' for k, v in _globals.items()] + args
     cmd = ["koji"] + args
-    print(cmd)
+    logger.info("Running %s", str(cmd))
     return subprocess.run(cmd,
+                          cwd=cwd,
                           encoding="utf-8",
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT,
@@ -92,16 +111,28 @@ class SutInfo:
 
 
 class TestIntegration(unittest.TestCase):
+    logger = logging.getLogger(__name__)
 
     def setUp(self):
-        global_args = dict(
+        self.koji_global_args = dict(
             server="http://localhost:8080/kojihub",
+            topurl="http://localhost:8080/kojifiles",
             user="kojiadmin",
             password="kojipass",
             authtype="password")
         self.koji = functools.partial(koji_command,
                                       "osbuild-image",
-                                      _globals=global_args)
+                                      _globals=self.koji_global_args)
+
+        self.workdir = tempfile.mkdtemp()
+        # EC2 image ID to clean up in tearDown() if set to a value
+        self.ec2_image_id = None
+
+    def tearDown(self):
+        shutil.rmtree(self.workdir)
+        if self.ec2_image_id is not None:
+            self.delete_ec2_image(self.ec2_image_id)
+            self.ec2_image_id = None
 
     def check_res(self, res: subprocess.CompletedProcess):
         if res.returncode != 0:
@@ -117,6 +148,55 @@ class TestIntegration(unittest.TestCase):
                    "\n error: " + res.stdout)
             self.fail(msg)
 
+    def task_id_from_res(self, res: subprocess.CompletedProcess) -> str:
+        """
+        Extract the Task ID from `koji osbuild-image` command output and return it.
+        """
+        r = re.compile(r'^Created task:[ \t]+(\d+)$', re.MULTILINE)
+        m = r.search(res.stdout)
+        if not m:
+            self.fail("Could not find task id in output")
+        return m.group(1)
+
+    @staticmethod
+    def get_ec2_client():
+        aws_region = os.getenv("AWS_REGION")
+        return boto3.client('ec2', config=BotoConfig(region_name=aws_region))
+
+    def check_ec2_image_exists(self, image_id: str) -> None:
+        """
+        Check if an EC2 image with the given ID exists.
+        If not, fail the test case.
+        """
+        client = self.get_ec2_client()
+        try:
+            resp = client.describe_images(ImageIds=[image_id])
+        except BotoClientError as e:
+            self.fail(str(e))
+        self.assertEqual(len(resp["Images"]), 1)
+
+    def delete_ec2_image(self, image_id: str) -> None:
+        client = self.get_ec2_client()
+        # first get the snapshot ID associated with the image
+        try:
+            resp = client.describe_images(ImageIds=[image_id])
+        except BotoClientError as e:
+            self.fail(str(e))
+        self.assertEqual(len(resp["Images"]), 1)
+
+        snapshot_id = resp["Images"][0]["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"]
+        # deregister the image
+        try:
+            resp = client.deregister_image(ImageId=image_id)
+        except BotoClientError as e:
+            self.logger.warning("Failed to deregister image %s: %s", image_id, str(e))
+
+        # delete the associated snapshot
+        try:
+            resp = client.delete_snapshot(SnapshotId=snapshot_id)
+        except BotoClientError as e:
+            self.logger.warning("Failed to delete snapshot %s: %s", snapshot_id, str(e))
+
     def test_compose(self):
         """Successful compose"""
         # Simple test of a successful compose of RHEL
@@ -155,3 +235,67 @@ class TestIntegration(unittest.TestCase):
                         "UNKNOWNTAG",
                         sut_info.os_arch)
         self.check_fail(res)
+
+    def test_cloud_upload_aws(self):
+        """Successful compose with cloud upload to AWS"""
+        sut_info = SutInfo()
+
+        repos = []
+        for repo in sut_info.testing_repos():
+            url = repo["url"]
+            package_sets = repo.get("package_sets")
+            repos += ["--repo", url]
+            if package_sets:
+                repos += ["--repo-package-sets", package_sets]
+
+        package = "aws"
+        aws_region = os.getenv("AWS_REGION")
+
+        upload_options = {
+            "region": aws_region,
+            "share_with_accounts": [os.getenv("AWS_API_TEST_SHARE_ACCOUNT")]
+        }
+
+        upload_options_file = os.path.join(self.workdir, "upload_options.json")
+        with open(upload_options_file, "w", encoding="utf-8") as f:
+            json.dump(upload_options, f)
+
+        res = self.koji(package,
+                        sut_info.os_version_major,
+                        sut_info.composer_distro_name,
+                        sut_info.koji_tag,
+                        sut_info.os_arch,
+                        "--wait",
+                        *repos,
+                        f"--image-type={package}",
+                        f"--upload-options={upload_options_file}")
+        self.check_res(res)
+
+        task_id = self.task_id_from_res(res)
+        # Download files uploaded by osbuild plugins to the Koji build task.
+        # requires koji client of version >= 1.29.1
+        res_download = koji_command_cwd(
+            "download-task", "--all", task_id, cwd=self.workdir, _globals=self.koji_global_args
+        )
+        self.check_res(res_download)
+
+        # Extract information about the uploaded AMI from compose status response.
+        compose_status_file = os.path.join(self.workdir, "compose-status.noarch.json")
+        with open(compose_status_file, "r", encoding="utf-8") as f:
+            compose_status = json.load(f)
+
+        self.assertEqual(compose_status["status"], "success")
+        image_statuses = compose_status["image_statuses"]
+        self.assertEqual(len(image_statuses), 1)
+
+        upload_status = image_statuses[0]["upload_status"]
+        self.assertEqual(upload_status["status"], "success")
+        self.assertEqual(upload_status["type"], "aws")
+
+        upload_options = upload_status["options"]
+        self.assertEqual(upload_options["region"], aws_region)
+
+        image_id = upload_options["ami"]
+        self.assertNotEqual(len(image_id), 0)
+        self.ec2_image_id = image_id
+        self.check_ec2_image_exists(image_id)

test/make-tags.sh

@@ -26,4 +26,6 @@ $KOJI add-pkg --owner kojiadmin "${TAG_CANDIDATE}" rhel-guest
 
 $KOJI add-pkg --owner kojiadmin "${TAG_CANDIDATE}" fedora-iot
 
+$KOJI add-pkg --owner kojiadmin "${TAG_CANDIDATE}" aws
+
 $KOJI regen-repo "${TAG_BUILD}"