#!/usr/bin/bash # # Test osbuild-composer's main API endpoint by building a sample image and # uploading it to the appropriate cloud provider. The test currently supports # AWS and GCP. # # This script sets `-x` and is meant to always be run like that. This is # simpler than adding extensive error reporting, which would make this script # considerably more complex. Also, the full trace this produces is very useful # for the primary audience: developers of osbuild-composer looking at the log # from a run on a remote continuous integration system. # # # Cloud provider / target names # CLOUD_PROVIDER_AWS="aws" CLOUD_PROVIDER_GCP="gcp" CLOUD_PROVIDER_AZURE="azure" CLOUD_PROVIDER_AWS_S3="aws.s3" CLOUD_PROVIDER_GENERIC_S3="generic.s3" CLOUD_PROVIDER_CONTAINER_IMAGE_REGISTRY="container" # # Supported Image type names # export IMAGE_TYPE_AWS="aws" export IMAGE_TYPE_AZURE="azure" export IMAGE_TYPE_EDGE_COMMIT="edge-commit" export IMAGE_TYPE_EDGE_CONTAINER="edge-container" export IMAGE_TYPE_EDGE_INSTALLER="edge-installer" export IMAGE_TYPE_GCP="gcp" export IMAGE_TYPE_IMAGE_INSTALLER="image-installer" export IMAGE_TYPE_GUEST="guest-image" export IMAGE_TYPE_VSPHERE="vsphere" export IMAGE_TYPE_IOT_COMMIT="iot-commit" if (( $# > 2 )); then echo "$0 does not support more than two arguments" exit 1 fi if (( $# == 0 )); then echo "$0 requires that you set the image type to build" exit 1 fi set -euxo pipefail IMAGE_TYPE="$1" # select cloud provider based on image type # # the supported image types are listed in the api spec (internal/cloudapi/v2/openapi.v2.yml) case ${IMAGE_TYPE} in "$IMAGE_TYPE_AWS") CLOUD_PROVIDER="${CLOUD_PROVIDER_AWS}" ;; "$IMAGE_TYPE_AZURE") CLOUD_PROVIDER="${CLOUD_PROVIDER_AZURE}" ;; "$IMAGE_TYPE_GCP") CLOUD_PROVIDER="${CLOUD_PROVIDER_GCP}" ;; "$IMAGE_TYPE_EDGE_CONTAINER") CLOUD_PROVIDER="${CLOUD_PROVIDER_CONTAINER_IMAGE_REGISTRY}" ;; "$IMAGE_TYPE_EDGE_COMMIT"|"$IMAGE_TYPE_IOT_COMMIT"|"$IMAGE_TYPE_EDGE_INSTALLER"|"$IMAGE_TYPE_IMAGE_INSTALLER"|"$IMAGE_TYPE_GUEST"|"$IMAGE_TYPE_VSPHERE") # blobby image types: upload to s3 and provide download link CLOUD_PROVIDER="${2:-$CLOUD_PROVIDER_AWS_S3}" if [ "${CLOUD_PROVIDER}" != "${CLOUD_PROVIDER_AWS_S3}" ] && [ "${CLOUD_PROVIDER}" != "${CLOUD_PROVIDER_GENERIC_S3}" ]; then echo "${IMAGE_TYPE} can only be uploaded to either ${CLOUD_PROVIDER_AWS_S3} or ${CLOUD_PROVIDER_GENERIC_S3}" exit 1 fi ;; *) echo "Unknown image type: ${IMAGE_TYPE}" exit 1 esac ARTIFACTS="${ARTIFACTS:-/tmp/artifacts}" source /usr/libexec/osbuild-composer-test/set-env-variables.sh source /usr/libexec/tests/osbuild-composer/shared_lib.sh # Container image used for cloud provider CLI tools export CONTAINER_IMAGE_CLOUD_TOOLS="quay.io/osbuild/cloud-tools:latest" # # Provision the software under test. # /usr/libexec/osbuild-composer-test/provision.sh # # Set up the database queue # if which podman 2>/dev/null >&2; then CONTAINER_RUNTIME=podman elif which docker 2>/dev/null >&2; then CONTAINER_RUNTIME=docker else echo No container runtime found, install podman or docker. exit 2 fi # Start the db DB_CONTAINER_NAME="osbuild-composer-db" sudo "${CONTAINER_RUNTIME}" run -d --name "${DB_CONTAINER_NAME}" \ --health-cmd "pg_isready -U postgres -d osbuildcomposer" --health-interval 2s \ --health-timeout 2s --health-retries 10 \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=foobar \ -e POSTGRES_DB=osbuildcomposer \ -p 5432:5432 \ quay.io/osbuild/postgres:13-alpine # Dump the logs once to have a little more output sudo "${CONTAINER_RUNTIME}" logs osbuild-composer-db # Initialize a module in a temp dir so we can get tern without introducing # vendoring inconsistency pushd "$(mktemp -d)" sudo dnf install -y go go mod init temp go install github.com/jackc/tern@latest PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PGPORT=5432 \ "$(go env GOPATH)"/bin/tern migrate -m /usr/share/tests/osbuild-composer/schemas popd cat < /dev/null function dump_db() { # Disable -x for these commands to avoid printing the whole result and manifest into the log set +x # Save the result, including the manifest, for the job, straight from the db sudo "${CONTAINER_RUNTIME}" exec "${DB_CONTAINER_NAME}" psql -U postgres -d osbuildcomposer -c "SELECT result FROM jobs WHERE type='manifest-id-only'" \ | sudo tee "${ARTIFACTS}/build-result.txt" set -x } WORKDIR=$(mktemp -d) KILL_PIDS=() function cleanups() { set +eu cleanup # dump the DB here to ensure that it gets dumped even if the test fails dump_db sudo "${CONTAINER_RUNTIME}" kill "${DB_CONTAINER_NAME}" sudo "${CONTAINER_RUNTIME}" rm "${DB_CONTAINER_NAME}" sudo rm -rf "$WORKDIR" for P in "${KILL_PIDS[@]}"; do sudo pkill -P "$P" done set -eu } trap cleanups EXIT # make a dummy rpm and repo to test payload_repositories sudo dnf install -y rpm-build createrepo DUMMYRPMDIR=$(mktemp -d) DUMMYSPECFILE="$DUMMYRPMDIR/dummy.spec" export PAYLOAD_REPO_PORT="9999" export PAYLOAD_REPO_URL="http://localhost:9999" pushd "$DUMMYRPMDIR" cat < "$DUMMYSPECFILE" #----------- spec file starts --------------- Name: dummy Version: 1.0.0 Release: 0 BuildArch: noarch Vendor: dummy Summary: Provides %{name} License: BSD Provides: dummy %description %{summary} %files EOF mkdir -p "DUMMYRPMDIR/rpmbuild" rpmbuild --quiet --define "_topdir $DUMMYRPMDIR/rpmbuild" -bb "$DUMMYSPECFILE" mkdir -p "$DUMMYRPMDIR/repo" cp "$DUMMYRPMDIR"/rpmbuild/RPMS/noarch/*rpm "$DUMMYRPMDIR/repo" pushd "$DUMMYRPMDIR/repo" createrepo . sudo python3 -m http.server "$PAYLOAD_REPO_PORT" & KILL_PIDS+=("$!") popd popd # # Install the necessary cloud provider client tools # installClient # # Make sure /openapi and endpoints return success # curl \ --silent \ --show-error \ --cacert /etc/osbuild-composer/ca-crt.pem \ --key /etc/osbuild-composer/client-key.pem \ --cert /etc/osbuild-composer/client-crt.pem \ https://localhost/api/image-builder-composer/v2/openapi | jq . # # Prepare a request to be sent to the composer API. # REQUEST_FILE="${WORKDIR}/compose_request.json" IMG_COMPOSE_REQ_FILE="${WORKDIR}/img_compose_request.json" ARCH=$(uname -m) SSH_USER= TEST_ID="$(uuidgen)" # Generate a string, which can be used as a predictable resource name, # especially when running the test in CI where we may need to clean up # resources in case the test unexpectedly fails or is canceled CI="${CI:-false}" if [[ "$CI" == true ]]; then # in CI, imitate GenerateCIArtifactName() from internal/test/helpers.go TEST_ID="$DISTRO_CODE-$ARCH-$CI_COMMIT_BRANCH-$CI_JOB_ID" fi export TEST_ID if [[ "$ID" == "fedora" ]]; then # fedora uses fedora for everything SSH_USER="fedora" elif [[ "$CLOUD_PROVIDER" == "$CLOUD_PROVIDER_AWS" ]]; then # RHEL and centos use ec2-user for AWS SSH_USER="ec2-user" else # RHEL and centos use cloud-user for other clouds SSH_USER="cloud-user" fi export SSH_USER # This removes dot from VERSION_ID. # ID == rhel && VERSION_ID == 8.6 => DISTRO == rhel-86 # ID == centos && VERSION_ID == 8 => DISTRO == centos-8 # ID == fedora && VERSION_ID == 35 => DISTRO == fedora-35 export DISTRO="$ID-${VERSION_ID//./}" SUBSCRIPTION_BLOCK= # Only RHEL need subscription block. if [[ "$ID" == "rhel" ]]; then SUBSCRIPTION_BLOCK=$(cat < "$REQUEST_FILE2" sendCompose "$REQUEST_FILE2" waitForState "failure" # crashed/stopped/killed worker should result in the job being retried sendCompose "$REQUEST_FILE" waitForState "building" sudo systemctl stop "osbuild-remote-worker@*" RETRIED=0 for RETRY in {1..10}; do ROWS=$(sudo "${CONTAINER_RUNTIME}" exec "${DB_CONTAINER_NAME}" psql -U postgres -d osbuildcomposer -c \ "SELECT retries FROM jobs WHERE id = '$COMPOSE_ID' AND retries = 1") if grep -q "1 row" <<< "$ROWS"; then RETRIED=1 break else echo "Waiting until job is retried ($RETRY/10)" sleep 30 fi done if [ "$RETRIED" != 1 ]; then echo "Job $COMPOSE_ID wasn't retried after killing the worker" exit 1 fi # remove the job from the queue so the worker doesn't pick it up again sudo "${CONTAINER_RUNTIME}" exec "${DB_CONTAINER_NAME}" psql -U postgres -d osbuildcomposer -c \ "DELETE FROM jobs WHERE id = '$COMPOSE_ID'" sudo systemctl start "osbuild-remote-worker@localhost:8700.service" # full integration case INIT_COMPOSES="$(collectMetrics)" sendCompose "$REQUEST_FILE" waitForState SUBS_COMPOSES="$(collectMetrics)" test "$UPLOAD_STATUS" = "success" EXPECTED_UPLOAD_TYPE="$CLOUD_PROVIDER" if [ "${CLOUD_PROVIDER}" == "${CLOUD_PROVIDER_GENERIC_S3}" ]; then EXPECTED_UPLOAD_TYPE="${CLOUD_PROVIDER_AWS_S3}" fi test "$UPLOAD_TYPE" = "$EXPECTED_UPLOAD_TYPE" test $((INIT_COMPOSES+1)) = "$SUBS_COMPOSES" if [ -s "$IMG_COMPOSE_REQ_FILE" ]; then sendImgFromCompose "$IMG_COMPOSE_REQ_FILE" waitForImgState fi # # Verify the Cloud-provider specific upload_status options # checkUploadStatusOptions # # Verify the image landed in the appropriate cloud provider, and delete it. # verify # Verify selected package (postgresql) is included in package list function verifyPackageList() { # Save build metadata to artifacts directory for troubleshooting curl --silent \ --show-error \ --cacert /etc/osbuild-composer/ca-crt.pem \ --key /etc/osbuild-composer/client-key.pem \ --cert /etc/osbuild-composer/client-crt.pem \ https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json" local PACKAGENAMES PACKAGENAMES=$(jq -rM '.packages[].name' "${ARTIFACTS}/metadata.json") if ! grep -q postgresql <<< "${PACKAGENAMES}"; then echo "'postgresql' not found in compose package list 😠" exit 1 fi } verifyPackageList # # Verify oauth2 # cat <