From d1e064aec32d3f6ab5a83e6a1c472e5a5625bad9 Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Tue, 9 Aug 2022 11:35:06 +0200 Subject: [PATCH] koji_test.py: test upload to cloud with AWS Extend the integration test with a new case, testing that direct upload to the cloud works for Koji composes. Test this using a single cloud provider, specifically AWS. The test case submits a new osbuild-image build using Koji CLI, determines the image information once the build finishes and then checks that such image exists in AWS. The image is then deleted as part of the test case tear-down. The AWS credentials are now configured in the worker's configuration, if the appropriate environment variables are set. Update the SPEC file with a new test dependency and update the required osbuild-composer version. --- .github/workflows/ci.yml | 2 +- koji-osbuild.spec | 3 +- test/copy-creds.sh | 19 +++++ test/integration.sh | 5 +- test/integration/test_koji.py | 152 +++++++++++++++++++++++++++++++++- test/make-tags.sh | 2 + 6 files changed, 176 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e5f8ce..7f47e85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Install test dependencies - run: dnf -y install python3-flexmock python3-httpretty python3-jsonschema python3-koji python3-pylint python3-requests + run: dnf -y install python3-boto3 python3-flexmock python3-httpretty python3-jsonschema python3-koji python3-pylint python3-requests - name: Check out code uses: actions/checkout@v3 diff --git a/koji-osbuild.spec b/koji-osbuild.spec index a742486..8fc9fbf 100644 --- a/koji-osbuild.spec +++ b/koji-osbuild.spec @@ -143,10 +143,11 @@ Requires: jq Requires: koji Requires: krb5-workstation Requires: openssl -Requires: osbuild-composer >= 22 +Requires: osbuild-composer >= 58 Requires: osbuild-composer-tests Requires: podman Requires: podman-plugins +Requires: python3-boto3 %description tests Integration tests for koji-osbuild. To be run on a dedicated system. diff --git a/test/copy-creds.sh b/test/copy-creds.sh index 592d431..0e1c7a7 100755 --- a/test/copy-creds.sh +++ b/test/copy-creds.sh @@ -33,6 +33,25 @@ cp ${TEST_DATA}/osbuild-worker.toml \ echo "koji" > /etc/osbuild-worker/oauth-secret +# if AWS credentials are defined in the ENV, add them to the worker's configuration +# This is needed to test the upload to the cloud +V2_AWS_ACCESS_KEY_ID="${V2_AWS_ACCESS_KEY_ID:-}" +V2_AWS_SECRET_ACCESS_KEY="${V2_AWS_SECRET_ACCESS_KEY:-}" +if [[ -n "$V2_AWS_ACCESS_KEY_ID" && -n "$V2_AWS_SECRET_ACCESS_KEY" ]]; then + echo "Adding AWS credentials to the worker's configuration" + sudo tee /etc/osbuild-worker/aws-credentials.toml > /dev/null << EOF +[default] +aws_access_key_id = "$V2_AWS_ACCESS_KEY_ID" +aws_secret_access_key = "$V2_AWS_SECRET_ACCESS_KEY" +EOF + sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF + +[aws] +credentials = "/etc/osbuild-worker/aws-credentials.toml" +bucket = "${AWS_BUCKET}" +EOF +fi + echo "Copying system kerberos configuration" cp ${TEST_DATA}/krb5.local.conf \ /etc/krb5.conf.d/local diff --git a/test/integration.sh b/test/integration.sh index 1a7b939..afde1a9 100755 --- a/test/integration.sh +++ b/test/integration.sh @@ -39,7 +39,7 @@ greenprint "Testing Koji hub API access" koji --server=http://localhost:8080/kojihub --user=osbuild --password=osbuildpass --authtype=password hello greenprint "Copying credentials, certificates and configuration files" -sudo /usr/libexec/koji-osbuild-tests/copy-creds.sh /usr/share/koji-osbuild-tests +sudo -E /usr/libexec/koji-osbuild-tests/copy-creds.sh /usr/share/koji-osbuild-tests greenprint "Starting mock OpenID server" sudo /usr/libexec/koji-osbuild-tests/run-openid.sh start @@ -63,6 +63,9 @@ greenprint "Creating Koji tag infrastructure" /usr/libexec/koji-osbuild-tests/make-tags.sh greenprint "Running integration tests" +# export environment variables for the Boto3 client to work out of the box +AWS_ACCESS_KEY_ID="${V2_AWS_ACCESS_KEY_ID:-}" \ +AWS_SECRET_ACCESS_KEY="${V2_AWS_SECRET_ACCESS_KEY:-}" \ python3 -m unittest discover -v /usr/libexec/koji-osbuild-tests/integration/ greenprint "Stopping koji builder" diff --git a/test/integration/test_koji.py b/test/integration/test_koji.py index 948ac4d..523d8b5 100644 --- a/test/integration/test_koji.py +++ b/test/integration/test_koji.py @@ -4,19 +4,38 @@ import functools +import json +import logging +import os import platform -import unittest +import re +import shutil import string import subprocess +import tempfile +import unittest + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import ClientError as BotoClientError + + +logger = logging.getLogger(__name__) +logging.basicConfig(format = '%(asctime)s %(levelname)s: %(message)s', level = logging.INFO) def koji_command(*args, _input=None, _globals=None, **kwargs): + return koji_command_cwd(*args, _input=_input, _globals=_globals, **kwargs) + + +def koji_command_cwd(*args, cwd=None, _input=None, _globals=None, **kwargs): args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()] if _globals: args = [f'--{k}={v}' for k, v in _globals.items()] + args cmd = ["koji"] + args - print(cmd) + logger.info("Running %s", str(cmd)) return subprocess.run(cmd, + cwd=cwd, encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -92,16 +111,28 @@ class SutInfo: class TestIntegration(unittest.TestCase): + logger = logging.getLogger(__name__) def setUp(self): - global_args = dict( + self.koji_global_args = dict( server="http://localhost:8080/kojihub", + topurl="http://localhost:8080/kojifiles", user="kojiadmin", password="kojipass", authtype="password") self.koji = functools.partial(koji_command, "osbuild-image", - _globals=global_args) + _globals=self.koji_global_args) + + self.workdir = tempfile.mkdtemp() + # EC2 image ID to clean up in tearDown() if set to a value + self.ec2_image_id = None + + def tearDown(self): + shutil.rmtree(self.workdir) + if self.ec2_image_id is not None: + self.delete_ec2_image(self.ec2_image_id) + self.ec2_image_id = None def check_res(self, res: subprocess.CompletedProcess): if res.returncode != 0: @@ -117,6 +148,55 @@ class TestIntegration(unittest.TestCase): "\n error: " + res.stdout) self.fail(msg) + def task_id_from_res(self, res: subprocess.CompletedProcess) -> str: + """ + Extract the Task ID from `koji osbuild-image` command output and return it. + """ + r = re.compile(r'^Created task:[ \t]+(\d+)$', re.MULTILINE) + m = r.search(res.stdout) + if not m: + self.fail("Could not find task id in output") + return m.group(1) + + @staticmethod + def get_ec2_client(): + aws_region = os.getenv("AWS_REGION") + return boto3.client('ec2', config=BotoConfig(region_name=aws_region)) + + def check_ec2_image_exists(self, image_id: str) -> None: + """ + Check if an EC2 image with the given ID exists. + If not, fail the test case. + """ + client = self.get_ec2_client() + try: + resp = client.describe_images(ImageIds=[image_id]) + except BotoClientError as e: + self.fail(str(e)) + self.assertEqual(len(resp["Images"]), 1) + + def delete_ec2_image(self, image_id: str) -> None: + client = self.get_ec2_client() + # first get the snapshot ID associated with the image + try: + resp = client.describe_images(ImageIds=[image_id]) + except BotoClientError as e: + self.fail(str(e)) + self.assertEqual(len(resp["Images"]), 1) + + snapshot_id = resp["Images"][0]["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"] + # deregister the image + try: + resp = client.deregister_image(ImageId=image_id) + except BotoClientError as e: + self.logger.warning("Failed to deregister image %s: %s", image_id, str(e)) + + # delete the associated snapshot + try: + resp = client.delete_snapshot(SnapshotId=snapshot_id) + except BotoClientError as e: + self.logger.warning("Failed to delete snapshot %s: %s", snapshot_id, str(e)) + def test_compose(self): """Successful compose""" # Simple test of a successful compose of RHEL @@ -155,3 +235,67 @@ class TestIntegration(unittest.TestCase): "UNKNOWNTAG", sut_info.os_arch) self.check_fail(res) + + def test_cloud_upload_aws(self): + """Successful compose with cloud upload to AWS""" + sut_info = SutInfo() + + repos = [] + for repo in sut_info.testing_repos(): + url = repo["url"] + package_sets = repo.get("package_sets") + repos += ["--repo", url] + if package_sets: + repos += ["--repo-package-sets", package_sets] + + package = "aws" + aws_region = os.getenv("AWS_REGION") + + upload_options = { + "region": aws_region, + "share_with_accounts": [os.getenv("AWS_API_TEST_SHARE_ACCOUNT")] + } + + upload_options_file = os.path.join(self.workdir, "upload_options.json") + with open(upload_options_file, "w", encoding="utf-8") as f: + json.dump(upload_options, f) + + res = self.koji(package, + sut_info.os_version_major, + sut_info.composer_distro_name, + sut_info.koji_tag, + sut_info.os_arch, + "--wait", + *repos, + f"--image-type={package}", + f"--upload-options={upload_options_file}") + self.check_res(res) + + task_id = self.task_id_from_res(res) + # Download files uploaded by osbuild plugins to the Koji build task. + # requires koji client of version >= 1.29.1 + res_download = koji_command_cwd( + "download-task", "--all", task_id, cwd=self.workdir, _globals=self.koji_global_args + ) + self.check_res(res_download) + + # Extract information about the uploaded AMI from compose status response. + compose_status_file = os.path.join(self.workdir, "compose-status.noarch.json") + with open(compose_status_file, "r", encoding="utf-8") as f: + compose_status = json.load(f) + + self.assertEqual(compose_status["status"], "success") + image_statuses = compose_status["image_statuses"] + self.assertEqual(len(image_statuses), 1) + + upload_status = image_statuses[0]["upload_status"] + self.assertEqual(upload_status["status"], "success") + self.assertEqual(upload_status["type"], "aws") + + upload_options = upload_status["options"] + self.assertEqual(upload_options["region"], aws_region) + + image_id = upload_options["ami"] + self.assertNotEqual(len(image_id), 0) + self.ec2_image_id = image_id + self.check_ec2_image_exists(image_id) diff --git a/test/make-tags.sh b/test/make-tags.sh index 2f6cc9b..6f4cf0c 100755 --- a/test/make-tags.sh +++ b/test/make-tags.sh @@ -26,4 +26,6 @@ $KOJI add-pkg --owner kojiadmin "${TAG_CANDIDATE}" rhel-guest $KOJI add-pkg --owner kojiadmin "${TAG_CANDIDATE}" fedora-iot +$KOJI add-pkg --owner kojiadmin "${TAG_CANDIDATE}" aws + $KOJI regen-repo "${TAG_BUILD}"