Run AWS test using osbuild worker code

The `osbuild-image-tests` code does a good job testing image builds for AWS, but it doesn't use the same code paths that a customer would when using `composer-cli`. Add a test that builds a compose and uploads the image using osbuild-composer and its worker. Also, build an instance in AWS based on the image we imported and check to see if the smoke test file is present. Signed-off-by: Major Hayden <major@redhat.com>
2020-06-08 09:47:47 -05:00 · 2020-06-08 09:47:47 -05:00 · 5903cc42fe
commit 5903cc42fe
parent 540ca29c0a
8 changed files with 324 additions and 28 deletions
--- a/schutzbot/Jenkinsfile
+++ b/schutzbot/Jenkinsfile
@ -68,7 +68,7 @@ pipeline {
                // NOTE(mhayden): Disabling this for now since we don't have
                // access to these repositories in AWS.
                // stage('RHEL 8.3 Nightly') {
-                //     agent { label "rhel83" }
+                //     agent { label "rhel83 && psi" }
                //     environment {
                //         AWS_CREDS = credentials('aws-credentials-osbuildci')
                //     }
@ -196,12 +196,12 @@ void run_tests() {
 void preserve_logs(test_slug) {

    // Make a directory for the log files and move the logs there.
-    sh "mkdir ${test_slug} && mv *.log ${test_slug}/"
+    sh "mkdir ${test_slug} && mv *.log *.jpg ${test_slug}/ || true"

    // Artifact the logs.
    archiveArtifacts (
        allowEmptyArchive: true,
-        artifacts: "${test_slug}/*.log"
+        artifacts: "${test_slug}/*.log,${test_slug}/*.jpg"
    )

 }
--- a/schutzbot/run_tests.sh
+++ b/schutzbot/run_tests.sh
@ -36,8 +36,12 @@ ansible-playbook \
  -e install_source=os \
  ansible-osbuild/playbook.yml

+# Run the AWS test.
+if [[ ${TEST_TYPE:-} == image ]]; then
+  test/image-tests/aws.sh
+fi
+
 # Run the tests.
-echo "The AWS_BUCKET env var is set to: ${AWS_BUCKET:-}"
 ansible-playbook \
  -e workspace=${WORKSPACE} \
  -e test_type=${TEST_TYPE:-base} \
--- a/schutzbot/test.yml
+++ b/schutzbot/test.yml
@ -33,18 +33,6 @@
      when:
        - test_type == 'image'

-    - name: Run osbuild-composer AWS tests
-      include_tasks: test_runner_image.yml
-      loop: "{{ osbuild_composer_aws_test_cases }}"
-      vars:
-        env_vars: "{{ osbuild_composer_aws_env_vars }}"
-      loop_control:
-        loop_var: test_case
-      when:
-        - test_type == 'image'
-        # Don't run the AWS test if the bucket env var is not set.
-        - lookup('env', 'AWS_BUCKET') | length > 0
-
    - name: Show failed and passed tests
      debug:
        msg: |
--- a/schutzbot/vars.yml
+++ b/schutzbot/vars.yml
@ -33,15 +33,4 @@ osbuild_composer_image_test_cases:
  - vmdk-boot.json

 # Environment variables for image tests.
-osbuild_composer_image_env_vars: {}
-
-# List of AWS test cases
-osbuild_composer_aws_test_cases:
-  - ami-boot.json
-
-# Environment variables for AWS tests.
-osbuild_composer_aws_env_vars:
-  AWS_BUCKET: "{{ lookup('env', 'AWS_BUCKET') }}"
-  AWS_REGION: "{{ lookup('env', 'AWS_REGION') }}"
-  AWS_ACCESS_KEY_ID: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}"
-  AWS_SECRET_ACCESS_KEY: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}"
+osbuild_composer_image_env_vars: {}
--- a/test/cloud-init/user-data
+++ b/test/cloud-init/user-data
@ -2,3 +2,10 @@
 user: redhat
 ssh_authorized_keys:
  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC61wMCjOSHwbVb4VfVyl5sn497qW4PsdQ7Ty7aD6wDNZ/QjjULkDV/yW5WjDlDQ7UqFH0Sr7vywjqDizUAqK7zM5FsUKsUXWHWwg/ehKg8j9xKcMv11AkFoUoujtfAujnKODkk58XSA9whPr7qcw3vPrmog680pnMSzf9LC7J6kXfs6lkoKfBh9VnlxusCrw2yg0qI1fHAZBLPx7mW6+me71QZsS6sVz8v8KXyrXsKTdnF50FjzHcK9HXDBtSJS5wA3fkcRYymJe0o6WMWNdgSRVpoSiWaHHmFgdMUJaYoCfhXzyl7LtNb3Q+Sveg+tJK7JaRXBLMUllOlJ6ll5Hod
+
+write_files:
+  - path: "/etc/smoke-test.txt"
+    content: "c21va2UtdGVzdAo="
+    encoding: "b64"
+    owner: "root:root"
+    permissions: "0644"
--- a/test/image-tests/aws.sh
+++ b/test/image-tests/aws.sh
@ -0,0 +1,256 @@
+#!/bin/bash
+set -euo pipefail
+
+source /etc/os-release
+
+# Fedora 31's composer-cli doesn't support doing image uploads.
+if [[ $ID == fedora ]] && [[ $VERSION_ID == 31 ]]; then
+    echo "Fedora 31 does not support image uploads with composer-cli."
+    exit 0
+fi
+
+# Colorful output.
+function greenprint {
+    echo -e "\033[1;32m${1}\033[0m"
+}
+
+# Apply lorax patch to work around pytoml issues in RHEL 8.x.
+# See BZ 1843704 or https://github.com/weldr/lorax/pull/1030 for more details.
+if [[ $ID == rhel ]]; then
+    sudo sed -r -i 's#toml.load\(args\[3\]\)#toml.load(open(args[3]))#' \
+        /usr/lib/python3.6/site-packages/composer/cli/compose.py
+    sudo rm -f /usr/lib/python3.6/site-packages/composer/cli/compose.pyc
+fi
+
+# We need jq for parsing composer-cli output.
+if ! hash jq; then
+    greenprint "Installing jq"
+    sudo dnf -qy install jq
+fi
+
+# Check for awscli.
+if ! hash aws; then
+    greenprint "Installing awscli"
+    sudo pip3 -qq install awscli
+fi
+
+TEST_UUID=$(uuidgen)
+IMAGE_KEY=osbuild-composer-aws-test-${TEST_UUID}
+AWS_CMD="aws --region $AWS_REGION --output json --color on"
+
+# Jenkins sets WORKSPACE to the job workspace, but if this script runs
+# outside of Jenkins, we can set up a temporary directory instead.
+if [[ ${WORKSPACE:-empty} == empty ]]; then
+    WORKSPACE=$(mktemp -d)
+fi
+
+# Set up temporary files.
+TEMPDIR=$(mktemp -d)
+AWS_CONFIG=${TEMPDIR}/aws.toml
+BLUEPRINT_FILE=${TEMPDIR}/blueprint.toml
+AWS_INSTANCE_JSON=${TEMPDIR}/aws-instance.json
+COMPOSE_START=${TEMPDIR}/compose-start-${IMAGE_KEY}.json
+COMPOSE_INFO=${TEMPDIR}/compose-info-${IMAGE_KEY}.json
+AMI_DATA=${TEMPDIR}/ami-data-${IMAGE_KEY}.json
+INSTANCE_DATA=${TEMPDIR}/instance-data-${IMAGE_KEY}.json
+INSTANCE_CONSOLE=${TEMPDIR}/instance-console-${IMAGE_KEY}.json
+
+# Check for the smoke test file on the AWS instance that we start.
+smoke_test_check () {
+    # Ensure the ssh key has restricted permissions.
+    SSH_KEY=${WORKSPACE}/test/keyring/id_rsa
+    chmod 0600 $SSH_KEY
+
+    SMOKE_TEST=$(ssh -i ${SSH_KEY} redhat@${1} 'cat /etc/smoke-test.txt')
+    if [[ $SMOKE_TEST == smoke-test ]]; then
+        echo 1
+    else
+        echo 0
+    fi
+}
+
+# Get the console screenshot from the AWS instance.
+store_instance_screenshot () {
+    INSTANCE_ID=${1}
+    LOOP_COUNTER=${2}
+    SCREENSHOT_FILE=${WORKSPACE}/console-screenshot-${ID}-${VERSION_ID}-${LOOP_COUNTER}.jpg
+
+    $AWS_CMD ec2 get-console-screenshot --instance-id ${1} > $INSTANCE_CONSOLE
+    jq -r '.ImageData' | base64 -d - > $SCREENSHOT_FILE
+}
+
+# Write an AWS TOML file
+tee $AWS_CONFIG > /dev/null << EOF
+provider = "aws"
+
+[settings]
+accessKeyID = "${AWS_ACCESS_KEY_ID}"
+secretAccessKey = "${AWS_SECRET_ACCESS_KEY}"
+bucket = "${AWS_BUCKET}"
+region = "${AWS_REGION}"
+key = "${IMAGE_KEY}"
+EOF
+
+# Write a basic blueprint for our image.
+tee $BLUEPRINT_FILE > /dev/null << EOF
+name = "bash"
+description = "A base system with bash"
+version = "0.0.1"
+
+[[packages]]
+name = "bash"
+
+[customizations.services]
+enabled = ["sshd", "cloud-init", "cloud-init-local", "cloud-config", "cloud-final"]
+EOF
+
+# Copy the internal Fedora repository into place.
+if curl -fs http://download.devel.redhat.com > /dev/null; then
+    sudo cp ${WORKSPACE}/test/internal-repos/fedora-31.json \
+        /usr/share/osbuild-composer/repositories/fedora-31.json
+    sudo cp ${WORKSPACE}/test/internal-repos/fedora-32.json \
+        /usr/share/osbuild-composer/repositories/fedora-32.json
+    sudo systemctl stop osbuild-composer.service
+fi
+
+# Prepare the blueprint for the compose.
+greenprint "📋 Preparing blueprint"
+sudo composer-cli blueprints push $BLUEPRINT_FILE
+sudo composer-cli blueprints depsolve bash
+
+# Get worker unit file so we can watch the journal.
+WORKER_UNIT=$(sudo systemctl list-units | egrep -o "osbuild.*worker.*\.service")
+sudo journalctl -af -n 1 -u ${WORKER_UNIT} &
+WORKER_JOURNAL_PID=$!
+
+# Start the compose and upload to AWS.
+greenprint "🚀 Starting compose"
+sudo composer-cli --json compose start bash ami $IMAGE_KEY $AWS_CONFIG | tee $COMPOSE_START > /dev/null
+COMPOSE_ID=$(jq -r '.build_id' $COMPOSE_START)
+
+# Wait for the compose to finish.
+greenprint "⏱ Waiting for compose to finish: ${COMPOSE_ID}"
+while true; do
+    sudo composer-cli --json compose info ${COMPOSE_ID} | tee $COMPOSE_INFO > /dev/null
+    COMPOSE_STATUS=$(jq -r '.queue_status' $COMPOSE_INFO)
+
+    # Is the compose finished?
+    if [[ $COMPOSE_STATUS != RUNNING ]] && [[ $COMPOSE_STATUS != WAITING ]]; then
+        break
+    fi
+
+    # Wait 30 seconds and try again.
+    sleep 30
+done
+
+# Did the compose finish with success?
+if [[ $COMPOSE_STATUS != FINISHED ]]; then
+    echo "Something went wrong with the compose. 😢"
+    exit 1
+fi
+
+# Find the image that we made in AWS.
+greenprint "🔍 Search for created AMI"
+$AWS_CMD ec2 describe-images \
+    --owners self \
+    --filters Name=name,Values=${IMAGE_KEY} \
+    | tee $AMI_DATA > /dev/null
+
+AMI_IMAGE_ID=$(jq -r '.Images[].ImageId' $AMI_DATA)
+
+# Stop watching the worker journal.
+sudo kill ${WORKER_JOURNAL_PID}
+
+# NOTE(mhayden): Getting TagSpecifications to play along with bash's
+# parsing of curly braces and square brackets is nuts, so we just write some
+# json and pass it to the aws command.
+tee $AWS_INSTANCE_JSON > /dev/null << EOF
+{
+    "TagSpecifications": [
+        {
+            "ResourceType": "instance",
+            "Tags": [
+                {
+                    "Key": "Name",
+                    "Value": "${IMAGE_KEY}"
+                }
+            ]
+        }
+    ]
+}
+EOF
+
+# Build instance in AWS with our image.
+greenprint "👷🏻 Building instance in AWS"
+$AWS_CMD ec2 run-instances \
+    --associate-public-ip-address \
+    --key-name personal_servers \
+    --image-id ${AMI_IMAGE_ID} \
+    --instance-type t3a.micro \
+    --user-data file://${WORKSPACE}/test/cloud-init/user-data \
+    --cli-input-json file://${AWS_INSTANCE_JSON} > /dev/null
+
+# Wait for the instance to finish building.
+greenprint "⏱ Waiting for AWS instance to be marked as running"
+while true; do
+    $AWS_CMD ec2 describe-instances \
+        --filters Name=image-id,Values=${AMI_IMAGE_ID} \
+        | tee $INSTANCE_DATA > /dev/null
+
+    INSTANCE_STATUS=$(jq -r '.Reservations[].Instances[].State.Name' $INSTANCE_DATA)
+
+    # Break the loop if our instance is running.
+    if [[ $INSTANCE_STATUS == running ]]; then
+        break
+    fi
+
+    # Sleep for 10 seconds and try again.
+    sleep 10
+
+done
+
+# Get data about the instance we built.
+INSTANCE_ID=$(jq -r '.Reservations[].Instances[].InstanceId' $INSTANCE_DATA)
+PUBLIC_IP=$(jq -r '.Reservations[].Instances[].PublicIpAddress' $INSTANCE_DATA)
+
+# Wait for the node to come online.
+greenprint "⏱ Waiting for AWS instance to respond to ssh"
+for LOOP_COUNTER in {0..30}; do
+    if ssh-keyscan $PUBLIC_IP; then
+      ssh-keyscan $PUBLIC_IP >> ~/.ssh/known_hosts
+      break
+    fi
+
+    # Get a screenshot of the instance console.
+    store_instance_screenshot $INSTANCE_ID $LOOP_COUNTER || true
+
+    # ssh-keyscan has a 5 second timeout by default, so the pause per loop
+    # is 10 seconds when you include the following `sleep`.
+    sleep 5
+done
+
+# Check for our smoke test file.
+greenprint "🛃 Checking for smoke test file"
+RESULTS="$(smoke_test_check $PUBLIC_IP)"
+if [[ $RESULTS == "1" ]]; then
+    echo "Smoke test passed! 🥳"
+else
+    echo "Smoke test failed! 😭"
+fi
+
+# Clean up our mess.
+greenprint "🧼 Cleaning up"
+SNAPSHOT_ID=$(jq -r '.Images[].BlockDeviceMappings[].Ebs.SnapshotId' $AMI_DATA)
+$AWS_CMD ec2 terminate-instances --instance-id ${INSTANCE_ID}
+$AWS_CMD ec2 deregister-image --image-id ${AMI_IMAGE_ID}
+$AWS_CMD ec2 delete-snapshot --snapshot-id ${SNAPSHOT_ID}
+
+# Use the return code of the smoke test to determine if we passed or failed.
+if [[ $RESULTS == 1 ]]; then
+  greenprint "💚 Success"
+else
+  greenprint "❌ Failed"
+  exit 1
+fi
+
+exit 0
--- a/test/internal-repos/fedora-31.json
+++ b/test/internal-repos/fedora-31.json
@ -0,0 +1,26 @@
+{
+    "x86_64": [
+        {
+            "name": "fedora",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/releases/31/Everything/x86_64/os/",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFxq3QMBEADUhGfCfP1ijiggBuVbR/pBDSWMC3TWbfC8pt7fhZkYrilzfWUM\nfTsikPymSriScONXP6DNyZ5r7tgrIVdVrJvRIqIFRO0mufp9HyfWKDO//Ctyp7OQ\nzYw6NVthO/aWpyFfJpj6s4iZsYGqf9gByV8brBB8v8jEsCtVOj1BU3bMbLkMsRI9\n+WiLjDYyvopqNBQuIe8ogxSxpYdbUz6+jxzfvhRoBzWdjITd//Gjd90kkrBOMWkO\nLTqO133OD1WMT08G5NuQ4KhjYsVvSbBpfdkTcNuP8gBP9LxCQDc+e1eAhZ95g3qk\nXLeKEK9j+F+wuG/OrEAxBsscCxXRUB38QH6CFe3UxGoSMnBi+jEhicudo+ItpFOy\n7rPaYyRh4Pmu4QHcC83bNjp8NI6zTHrBmVuPqkxMn07GMAQav9ezBXj6umqTX4cU\ndsJUavJrJ3u7rT0lhBdiGrQ9zPbL07u2Kn+OXPAC3dKSf7G8TvwNAdry9esGSpi3\n8aa1myQYVZvAlsIBkbN3fb1wvDJE5czVhzwQ77V2t66jxeg0o9/2OZVH3CozD2Zj\nv28LHuW/jnQHtsQ0fUyQYRmHxNEVkW10GGM7fQwxzpxFFS1O/2XEnfMu7yBHZsgL\nSojfUct0FhLhEN/g/IINX9ZCVrzK5/De27CNjYE1cgYD/lTmQ0SyjfKVwwARAQAB\ntDFGZWRvcmEgKDMxKSA8ZmVkb3JhLTMxLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI+BBMBAgAoAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAUCXGrkTQUJ\nEs8P/QAKCRBQyzkLPDNZxBmDD/90IFwAfFcQq5ENl7/o2CYQ9k2adTHbV5RoIOWC\n/o9I5/btn1y8WDhPOUNmsgbUqRqz6srlVplg+LkpIj67PVLDBwpVbCJC8o1fztd2\nMryVqdvu562WVhUorII+iW7nfqD0yv55nH9b/JR1qloUa8LpeKw84JgvxF5wVfyR\nid1WjI0DBk2taFR4xCfU5Tb262fbdFj5iB9xskP7oNeS29+SfDjlnybtlFoqr9UA\nnY1uvhBPkGmj45SJkpfP+L+kGYXVaUd29M/q/Pt46X1KDvr6Z0l8bSUEk3zfcNdj\nuEhtHBqSy1UPPAikGX1Q5wGdu7R7+mv/ARqfI6OC44ipoOMNK1Aiu6+slbPYphwX\nighSz9yYuG0EdWt7akfKR0R04Kuej4LXLWcxTR4l8XDzThYgPP0g+z0XQJrAkVhi\nSrzICeC3K1GPSiUtNAxSTL+qWWgwvQyAPNoPV/OYmY+wUxUnKCZpEWPkL79lh6CM\nbJx/zlrOMzRumSzaOnKW9AOliviH4Rj89OmDifBEsQ0CewdHN9ly6g4ZFJJGYXJ5\nHTb5jdButTC3tDfvH8Z7dtXKdC4iqJCIxj698Xn8UjVefZQ2nbv5eXcZLfHtvbNB\nTTv1vvBV4G7aiHKYRSj7HmxhLBZC8Y/nmFAemOoOYDpR5eUmPmSbFayoLfRsFXmC\nHLs7cw==\n=6hRW\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        },
+        {
+            "name": "updates",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/updates/31/Everything/x86_64/",
+            "metadata_expire": "6h",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFxq3QMBEADUhGfCfP1ijiggBuVbR/pBDSWMC3TWbfC8pt7fhZkYrilzfWUM\nfTsikPymSriScONXP6DNyZ5r7tgrIVdVrJvRIqIFRO0mufp9HyfWKDO//Ctyp7OQ\nzYw6NVthO/aWpyFfJpj6s4iZsYGqf9gByV8brBB8v8jEsCtVOj1BU3bMbLkMsRI9\n+WiLjDYyvopqNBQuIe8ogxSxpYdbUz6+jxzfvhRoBzWdjITd//Gjd90kkrBOMWkO\nLTqO133OD1WMT08G5NuQ4KhjYsVvSbBpfdkTcNuP8gBP9LxCQDc+e1eAhZ95g3qk\nXLeKEK9j+F+wuG/OrEAxBsscCxXRUB38QH6CFe3UxGoSMnBi+jEhicudo+ItpFOy\n7rPaYyRh4Pmu4QHcC83bNjp8NI6zTHrBmVuPqkxMn07GMAQav9ezBXj6umqTX4cU\ndsJUavJrJ3u7rT0lhBdiGrQ9zPbL07u2Kn+OXPAC3dKSf7G8TvwNAdry9esGSpi3\n8aa1myQYVZvAlsIBkbN3fb1wvDJE5czVhzwQ77V2t66jxeg0o9/2OZVH3CozD2Zj\nv28LHuW/jnQHtsQ0fUyQYRmHxNEVkW10GGM7fQwxzpxFFS1O/2XEnfMu7yBHZsgL\nSojfUct0FhLhEN/g/IINX9ZCVrzK5/De27CNjYE1cgYD/lTmQ0SyjfKVwwARAQAB\ntDFGZWRvcmEgKDMxKSA8ZmVkb3JhLTMxLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI+BBMBAgAoAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAUCXGrkTQUJ\nEs8P/QAKCRBQyzkLPDNZxBmDD/90IFwAfFcQq5ENl7/o2CYQ9k2adTHbV5RoIOWC\n/o9I5/btn1y8WDhPOUNmsgbUqRqz6srlVplg+LkpIj67PVLDBwpVbCJC8o1fztd2\nMryVqdvu562WVhUorII+iW7nfqD0yv55nH9b/JR1qloUa8LpeKw84JgvxF5wVfyR\nid1WjI0DBk2taFR4xCfU5Tb262fbdFj5iB9xskP7oNeS29+SfDjlnybtlFoqr9UA\nnY1uvhBPkGmj45SJkpfP+L+kGYXVaUd29M/q/Pt46X1KDvr6Z0l8bSUEk3zfcNdj\nuEhtHBqSy1UPPAikGX1Q5wGdu7R7+mv/ARqfI6OC44ipoOMNK1Aiu6+slbPYphwX\nighSz9yYuG0EdWt7akfKR0R04Kuej4LXLWcxTR4l8XDzThYgPP0g+z0XQJrAkVhi\nSrzICeC3K1GPSiUtNAxSTL+qWWgwvQyAPNoPV/OYmY+wUxUnKCZpEWPkL79lh6CM\nbJx/zlrOMzRumSzaOnKW9AOliviH4Rj89OmDifBEsQ0CewdHN9ly6g4ZFJJGYXJ5\nHTb5jdButTC3tDfvH8Z7dtXKdC4iqJCIxj698Xn8UjVefZQ2nbv5eXcZLfHtvbNB\nTTv1vvBV4G7aiHKYRSj7HmxhLBZC8Y/nmFAemOoOYDpR5eUmPmSbFayoLfRsFXmC\nHLs7cw==\n=6hRW\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        },
+        {
+            "name": "fedora-modular",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/releases/31/Modular/x86_64/os/",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFxq3QMBEADUhGfCfP1ijiggBuVbR/pBDSWMC3TWbfC8pt7fhZkYrilzfWUM\nfTsikPymSriScONXP6DNyZ5r7tgrIVdVrJvRIqIFRO0mufp9HyfWKDO//Ctyp7OQ\nzYw6NVthO/aWpyFfJpj6s4iZsYGqf9gByV8brBB8v8jEsCtVOj1BU3bMbLkMsRI9\n+WiLjDYyvopqNBQuIe8ogxSxpYdbUz6+jxzfvhRoBzWdjITd//Gjd90kkrBOMWkO\nLTqO133OD1WMT08G5NuQ4KhjYsVvSbBpfdkTcNuP8gBP9LxCQDc+e1eAhZ95g3qk\nXLeKEK9j+F+wuG/OrEAxBsscCxXRUB38QH6CFe3UxGoSMnBi+jEhicudo+ItpFOy\n7rPaYyRh4Pmu4QHcC83bNjp8NI6zTHrBmVuPqkxMn07GMAQav9ezBXj6umqTX4cU\ndsJUavJrJ3u7rT0lhBdiGrQ9zPbL07u2Kn+OXPAC3dKSf7G8TvwNAdry9esGSpi3\n8aa1myQYVZvAlsIBkbN3fb1wvDJE5czVhzwQ77V2t66jxeg0o9/2OZVH3CozD2Zj\nv28LHuW/jnQHtsQ0fUyQYRmHxNEVkW10GGM7fQwxzpxFFS1O/2XEnfMu7yBHZsgL\nSojfUct0FhLhEN/g/IINX9ZCVrzK5/De27CNjYE1cgYD/lTmQ0SyjfKVwwARAQAB\ntDFGZWRvcmEgKDMxKSA8ZmVkb3JhLTMxLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI+BBMBAgAoAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAUCXGrkTQUJ\nEs8P/QAKCRBQyzkLPDNZxBmDD/90IFwAfFcQq5ENl7/o2CYQ9k2adTHbV5RoIOWC\n/o9I5/btn1y8WDhPOUNmsgbUqRqz6srlVplg+LkpIj67PVLDBwpVbCJC8o1fztd2\nMryVqdvu562WVhUorII+iW7nfqD0yv55nH9b/JR1qloUa8LpeKw84JgvxF5wVfyR\nid1WjI0DBk2taFR4xCfU5Tb262fbdFj5iB9xskP7oNeS29+SfDjlnybtlFoqr9UA\nnY1uvhBPkGmj45SJkpfP+L+kGYXVaUd29M/q/Pt46X1KDvr6Z0l8bSUEk3zfcNdj\nuEhtHBqSy1UPPAikGX1Q5wGdu7R7+mv/ARqfI6OC44ipoOMNK1Aiu6+slbPYphwX\nighSz9yYuG0EdWt7akfKR0R04Kuej4LXLWcxTR4l8XDzThYgPP0g+z0XQJrAkVhi\nSrzICeC3K1GPSiUtNAxSTL+qWWgwvQyAPNoPV/OYmY+wUxUnKCZpEWPkL79lh6CM\nbJx/zlrOMzRumSzaOnKW9AOliviH4Rj89OmDifBEsQ0CewdHN9ly6g4ZFJJGYXJ5\nHTb5jdButTC3tDfvH8Z7dtXKdC4iqJCIxj698Xn8UjVefZQ2nbv5eXcZLfHtvbNB\nTTv1vvBV4G7aiHKYRSj7HmxhLBZC8Y/nmFAemOoOYDpR5eUmPmSbFayoLfRsFXmC\nHLs7cw==\n=6hRW\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        },
+        {
+            "name": "updates-modular",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/updates/31/Modular/x86_64/",
+            "metadata_expire": "6h",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFxq3QMBEADUhGfCfP1ijiggBuVbR/pBDSWMC3TWbfC8pt7fhZkYrilzfWUM\nfTsikPymSriScONXP6DNyZ5r7tgrIVdVrJvRIqIFRO0mufp9HyfWKDO//Ctyp7OQ\nzYw6NVthO/aWpyFfJpj6s4iZsYGqf9gByV8brBB8v8jEsCtVOj1BU3bMbLkMsRI9\n+WiLjDYyvopqNBQuIe8ogxSxpYdbUz6+jxzfvhRoBzWdjITd//Gjd90kkrBOMWkO\nLTqO133OD1WMT08G5NuQ4KhjYsVvSbBpfdkTcNuP8gBP9LxCQDc+e1eAhZ95g3qk\nXLeKEK9j+F+wuG/OrEAxBsscCxXRUB38QH6CFe3UxGoSMnBi+jEhicudo+ItpFOy\n7rPaYyRh4Pmu4QHcC83bNjp8NI6zTHrBmVuPqkxMn07GMAQav9ezBXj6umqTX4cU\ndsJUavJrJ3u7rT0lhBdiGrQ9zPbL07u2Kn+OXPAC3dKSf7G8TvwNAdry9esGSpi3\n8aa1myQYVZvAlsIBkbN3fb1wvDJE5czVhzwQ77V2t66jxeg0o9/2OZVH3CozD2Zj\nv28LHuW/jnQHtsQ0fUyQYRmHxNEVkW10GGM7fQwxzpxFFS1O/2XEnfMu7yBHZsgL\nSojfUct0FhLhEN/g/IINX9ZCVrzK5/De27CNjYE1cgYD/lTmQ0SyjfKVwwARAQAB\ntDFGZWRvcmEgKDMxKSA8ZmVkb3JhLTMxLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI+BBMBAgAoAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAUCXGrkTQUJ\nEs8P/QAKCRBQyzkLPDNZxBmDD/90IFwAfFcQq5ENl7/o2CYQ9k2adTHbV5RoIOWC\n/o9I5/btn1y8WDhPOUNmsgbUqRqz6srlVplg+LkpIj67PVLDBwpVbCJC8o1fztd2\nMryVqdvu562WVhUorII+iW7nfqD0yv55nH9b/JR1qloUa8LpeKw84JgvxF5wVfyR\nid1WjI0DBk2taFR4xCfU5Tb262fbdFj5iB9xskP7oNeS29+SfDjlnybtlFoqr9UA\nnY1uvhBPkGmj45SJkpfP+L+kGYXVaUd29M/q/Pt46X1KDvr6Z0l8bSUEk3zfcNdj\nuEhtHBqSy1UPPAikGX1Q5wGdu7R7+mv/ARqfI6OC44ipoOMNK1Aiu6+slbPYphwX\nighSz9yYuG0EdWt7akfKR0R04Kuej4LXLWcxTR4l8XDzThYgPP0g+z0XQJrAkVhi\nSrzICeC3K1GPSiUtNAxSTL+qWWgwvQyAPNoPV/OYmY+wUxUnKCZpEWPkL79lh6CM\nbJx/zlrOMzRumSzaOnKW9AOliviH4Rj89OmDifBEsQ0CewdHN9ly6g4ZFJJGYXJ5\nHTb5jdButTC3tDfvH8Z7dtXKdC4iqJCIxj698Xn8UjVefZQ2nbv5eXcZLfHtvbNB\nTTv1vvBV4G7aiHKYRSj7HmxhLBZC8Y/nmFAemOoOYDpR5eUmPmSbFayoLfRsFXmC\nHLs7cw==\n=6hRW\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        }
+    ]
+}
--- a/test/internal-repos/fedora-32.json
+++ b/test/internal-repos/fedora-32.json
@ -0,0 +1,26 @@
+{
+    "x86_64": [
+        {
+            "name": "fedora",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/releases/32/Everything/x86_64/os/",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBF1RVqsBEADWMBqYv/G1r4PwyiPQCfg5fXFGXV1FCZ32qMi9gLUTv1CX7rYy\nH4Inj93oic+lt1kQ0kQCkINOwQczOkm6XDkEekmMrHknJpFLwrTK4AS28bYF2RjL\nM+QJ/dGXDMPYsP0tkLvoxaHr9WTRq89A+AmONcUAQIMJg3JxXAAafBi2UszUUEPI\nU35MyufFt2ePd1k/6hVAO8S2VT72TxXSY7Ha4X2J0pGzbqQ6Dq3AVzogsnoIi09A\n7fYutYZPVVAEGRUqavl0th8LyuZShASZ38CdAHBMvWV4bVZghd/wDV5ev3LXUE0o\nitLAqNSeiDJ3grKWN6v0qdU0l3Ya60sugABd3xaE+ROe8kDCy3WmAaO51Q880ZA2\niXOTJFObqkBTP9j9+ZeQ+KNE8SBoiH1EybKtBU8HmygZvu8ZC1TKUyL5gwGUJt8v\nergy5Bw3Q7av520sNGD3cIWr4fBAVYwdBoZT8RcsnU1PP67NmOGFcwSFJ/LpiOMC\npZ1IBvjOC7KyKEZY2/63kjW73mB7OHOd18BHtGVkA3QAdVlcSule/z68VOAy6bih\nE6mdxP28D4INsts8w6yr4G+3aEIN8u0qRQq66Ri5mOXTyle+ONudtfGg3U9lgicg\nz6oVk17RT0jV9uL6K41sGZ1sH/6yTXQKagdAYr3w1ix2L46JgzC+/+6SSwARAQAB\ntDFGZWRvcmEgKDMyKSA8ZmVkb3JhLTMyLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJdUVarAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRBsEwJtEslE0LdAD/wKdAMtfzr7O2y06/sOPnrb3D39Y2DXbB8y0iEmRdBL29Bq\n5btxwmAka7JZRJVFxPsOVqZ6KARjS0/oCBmJc0jCRANFCtM4UjVHTSsxrJfuPkel\nvrlNE9tcR6OCRpuj/PZgUa39iifF/FTUfDgh4Q91xiQoLqfBxOJzravQHoK9VzrM\nNTOu6J6l4zeGzY/ocj6DpT+5fdUO/3HgGFNiNYPC6GVzeiA3AAVR0sCyGENuqqdg\nwUxV3BIht05M5Wcdvxg1U9x5I3yjkLQw+idvX4pevTiCh9/0u+4g80cT/21Cxsdx\n7+DVHaewXbF87QQIcOAing0S5QE67r2uPVxmWy/56TKUqDoyP8SNsV62lT2jutsj\nLevNxUky011g5w3bc61UeaeKrrurFdRs+RwBVkXmtqm/i6g0ZTWZyWGO6gJd+HWA\nqY1NYiq4+cMvNLatmA2sOoCsRNmE9q6jM/ESVgaH8hSp8GcLuzt9/r4PZZGl5CvU\neldOiD221u8rzuHmLs4dsgwJJ9pgLT0cUAsOpbMPI0JpGIPQ2SG6yK7LmO6HFOxb\nAkz7IGUt0gy1MzPTyBvnB+WgD1I+IQXXsJbhP5+d+d3mOnqsd6oDM/grKBzrhoUe\noNadc9uzjqKlOrmrdIR3Bz38SSiWlde5fu6xPqJdmGZRNjXtcyJlbSPVDIloxw==\n=QWRO\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        },
+        {
+            "name": "updates",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/updates/32/Everything/x86_64/",
+            "metadata_expire": "6h",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBF1RVqsBEADWMBqYv/G1r4PwyiPQCfg5fXFGXV1FCZ32qMi9gLUTv1CX7rYy\nH4Inj93oic+lt1kQ0kQCkINOwQczOkm6XDkEekmMrHknJpFLwrTK4AS28bYF2RjL\nM+QJ/dGXDMPYsP0tkLvoxaHr9WTRq89A+AmONcUAQIMJg3JxXAAafBi2UszUUEPI\nU35MyufFt2ePd1k/6hVAO8S2VT72TxXSY7Ha4X2J0pGzbqQ6Dq3AVzogsnoIi09A\n7fYutYZPVVAEGRUqavl0th8LyuZShASZ38CdAHBMvWV4bVZghd/wDV5ev3LXUE0o\nitLAqNSeiDJ3grKWN6v0qdU0l3Ya60sugABd3xaE+ROe8kDCy3WmAaO51Q880ZA2\niXOTJFObqkBTP9j9+ZeQ+KNE8SBoiH1EybKtBU8HmygZvu8ZC1TKUyL5gwGUJt8v\nergy5Bw3Q7av520sNGD3cIWr4fBAVYwdBoZT8RcsnU1PP67NmOGFcwSFJ/LpiOMC\npZ1IBvjOC7KyKEZY2/63kjW73mB7OHOd18BHtGVkA3QAdVlcSule/z68VOAy6bih\nE6mdxP28D4INsts8w6yr4G+3aEIN8u0qRQq66Ri5mOXTyle+ONudtfGg3U9lgicg\nz6oVk17RT0jV9uL6K41sGZ1sH/6yTXQKagdAYr3w1ix2L46JgzC+/+6SSwARAQAB\ntDFGZWRvcmEgKDMyKSA8ZmVkb3JhLTMyLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJdUVarAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRBsEwJtEslE0LdAD/wKdAMtfzr7O2y06/sOPnrb3D39Y2DXbB8y0iEmRdBL29Bq\n5btxwmAka7JZRJVFxPsOVqZ6KARjS0/oCBmJc0jCRANFCtM4UjVHTSsxrJfuPkel\nvrlNE9tcR6OCRpuj/PZgUa39iifF/FTUfDgh4Q91xiQoLqfBxOJzravQHoK9VzrM\nNTOu6J6l4zeGzY/ocj6DpT+5fdUO/3HgGFNiNYPC6GVzeiA3AAVR0sCyGENuqqdg\nwUxV3BIht05M5Wcdvxg1U9x5I3yjkLQw+idvX4pevTiCh9/0u+4g80cT/21Cxsdx\n7+DVHaewXbF87QQIcOAing0S5QE67r2uPVxmWy/56TKUqDoyP8SNsV62lT2jutsj\nLevNxUky011g5w3bc61UeaeKrrurFdRs+RwBVkXmtqm/i6g0ZTWZyWGO6gJd+HWA\nqY1NYiq4+cMvNLatmA2sOoCsRNmE9q6jM/ESVgaH8hSp8GcLuzt9/r4PZZGl5CvU\neldOiD221u8rzuHmLs4dsgwJJ9pgLT0cUAsOpbMPI0JpGIPQ2SG6yK7LmO6HFOxb\nAkz7IGUt0gy1MzPTyBvnB+WgD1I+IQXXsJbhP5+d+d3mOnqsd6oDM/grKBzrhoUe\noNadc9uzjqKlOrmrdIR3Bz38SSiWlde5fu6xPqJdmGZRNjXtcyJlbSPVDIloxw==\n=QWRO\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        },
+        {
+            "name": "fedora-modular",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/releases/32/Modular/x86_64/os/",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBF1RVqsBEADWMBqYv/G1r4PwyiPQCfg5fXFGXV1FCZ32qMi9gLUTv1CX7rYy\nH4Inj93oic+lt1kQ0kQCkINOwQczOkm6XDkEekmMrHknJpFLwrTK4AS28bYF2RjL\nM+QJ/dGXDMPYsP0tkLvoxaHr9WTRq89A+AmONcUAQIMJg3JxXAAafBi2UszUUEPI\nU35MyufFt2ePd1k/6hVAO8S2VT72TxXSY7Ha4X2J0pGzbqQ6Dq3AVzogsnoIi09A\n7fYutYZPVVAEGRUqavl0th8LyuZShASZ38CdAHBMvWV4bVZghd/wDV5ev3LXUE0o\nitLAqNSeiDJ3grKWN6v0qdU0l3Ya60sugABd3xaE+ROe8kDCy3WmAaO51Q880ZA2\niXOTJFObqkBTP9j9+ZeQ+KNE8SBoiH1EybKtBU8HmygZvu8ZC1TKUyL5gwGUJt8v\nergy5Bw3Q7av520sNGD3cIWr4fBAVYwdBoZT8RcsnU1PP67NmOGFcwSFJ/LpiOMC\npZ1IBvjOC7KyKEZY2/63kjW73mB7OHOd18BHtGVkA3QAdVlcSule/z68VOAy6bih\nE6mdxP28D4INsts8w6yr4G+3aEIN8u0qRQq66Ri5mOXTyle+ONudtfGg3U9lgicg\nz6oVk17RT0jV9uL6K41sGZ1sH/6yTXQKagdAYr3w1ix2L46JgzC+/+6SSwARAQAB\ntDFGZWRvcmEgKDMyKSA8ZmVkb3JhLTMyLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJdUVarAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRBsEwJtEslE0LdAD/wKdAMtfzr7O2y06/sOPnrb3D39Y2DXbB8y0iEmRdBL29Bq\n5btxwmAka7JZRJVFxPsOVqZ6KARjS0/oCBmJc0jCRANFCtM4UjVHTSsxrJfuPkel\nvrlNE9tcR6OCRpuj/PZgUa39iifF/FTUfDgh4Q91xiQoLqfBxOJzravQHoK9VzrM\nNTOu6J6l4zeGzY/ocj6DpT+5fdUO/3HgGFNiNYPC6GVzeiA3AAVR0sCyGENuqqdg\nwUxV3BIht05M5Wcdvxg1U9x5I3yjkLQw+idvX4pevTiCh9/0u+4g80cT/21Cxsdx\n7+DVHaewXbF87QQIcOAing0S5QE67r2uPVxmWy/56TKUqDoyP8SNsV62lT2jutsj\nLevNxUky011g5w3bc61UeaeKrrurFdRs+RwBVkXmtqm/i6g0ZTWZyWGO6gJd+HWA\nqY1NYiq4+cMvNLatmA2sOoCsRNmE9q6jM/ESVgaH8hSp8GcLuzt9/r4PZZGl5CvU\neldOiD221u8rzuHmLs4dsgwJJ9pgLT0cUAsOpbMPI0JpGIPQ2SG6yK7LmO6HFOxb\nAkz7IGUt0gy1MzPTyBvnB+WgD1I+IQXXsJbhP5+d+d3mOnqsd6oDM/grKBzrhoUe\noNadc9uzjqKlOrmrdIR3Bz38SSiWlde5fu6xPqJdmGZRNjXtcyJlbSPVDIloxw==\n=QWRO\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        },
+        {
+            "name": "updates-modular",
+            "baseurl": "https://download-rdu01.fedoraproject.org/pub/fedora/linux/updates/32/Modular/x86_64/",
+            "metadata_expire": "6h",
+            "gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBF1RVqsBEADWMBqYv/G1r4PwyiPQCfg5fXFGXV1FCZ32qMi9gLUTv1CX7rYy\nH4Inj93oic+lt1kQ0kQCkINOwQczOkm6XDkEekmMrHknJpFLwrTK4AS28bYF2RjL\nM+QJ/dGXDMPYsP0tkLvoxaHr9WTRq89A+AmONcUAQIMJg3JxXAAafBi2UszUUEPI\nU35MyufFt2ePd1k/6hVAO8S2VT72TxXSY7Ha4X2J0pGzbqQ6Dq3AVzogsnoIi09A\n7fYutYZPVVAEGRUqavl0th8LyuZShASZ38CdAHBMvWV4bVZghd/wDV5ev3LXUE0o\nitLAqNSeiDJ3grKWN6v0qdU0l3Ya60sugABd3xaE+ROe8kDCy3WmAaO51Q880ZA2\niXOTJFObqkBTP9j9+ZeQ+KNE8SBoiH1EybKtBU8HmygZvu8ZC1TKUyL5gwGUJt8v\nergy5Bw3Q7av520sNGD3cIWr4fBAVYwdBoZT8RcsnU1PP67NmOGFcwSFJ/LpiOMC\npZ1IBvjOC7KyKEZY2/63kjW73mB7OHOd18BHtGVkA3QAdVlcSule/z68VOAy6bih\nE6mdxP28D4INsts8w6yr4G+3aEIN8u0qRQq66Ri5mOXTyle+ONudtfGg3U9lgicg\nz6oVk17RT0jV9uL6K41sGZ1sH/6yTXQKagdAYr3w1ix2L46JgzC+/+6SSwARAQAB\ntDFGZWRvcmEgKDMyKSA8ZmVkb3JhLTMyLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJdUVarAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRBsEwJtEslE0LdAD/wKdAMtfzr7O2y06/sOPnrb3D39Y2DXbB8y0iEmRdBL29Bq\n5btxwmAka7JZRJVFxPsOVqZ6KARjS0/oCBmJc0jCRANFCtM4UjVHTSsxrJfuPkel\nvrlNE9tcR6OCRpuj/PZgUa39iifF/FTUfDgh4Q91xiQoLqfBxOJzravQHoK9VzrM\nNTOu6J6l4zeGzY/ocj6DpT+5fdUO/3HgGFNiNYPC6GVzeiA3AAVR0sCyGENuqqdg\nwUxV3BIht05M5Wcdvxg1U9x5I3yjkLQw+idvX4pevTiCh9/0u+4g80cT/21Cxsdx\n7+DVHaewXbF87QQIcOAing0S5QE67r2uPVxmWy/56TKUqDoyP8SNsV62lT2jutsj\nLevNxUky011g5w3bc61UeaeKrrurFdRs+RwBVkXmtqm/i6g0ZTWZyWGO6gJd+HWA\nqY1NYiq4+cMvNLatmA2sOoCsRNmE9q6jM/ESVgaH8hSp8GcLuzt9/r4PZZGl5CvU\neldOiD221u8rzuHmLs4dsgwJJ9pgLT0cUAsOpbMPI0JpGIPQ2SG6yK7LmO6HFOxb\nAkz7IGUt0gy1MzPTyBvnB+WgD1I+IQXXsJbhP5+d+d3mOnqsd6oDM/grKBzrhoUe\noNadc9uzjqKlOrmrdIR3Bz38SSiWlde5fu6xPqJdmGZRNjXtcyJlbSPVDIloxw==\n=QWRO\n-----END PGP PUBLIC KEY BLOCK-----\n"
+        }
+    ]
+}