From 2fe5e01d4d421d4d41e31f217c9b952d2e644aff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Fri, 12 Jan 2024 22:39:11 +0100 Subject: [PATCH] Test/api.sh: test distro aliases and names without dot-notation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the `api.sh` to allow testing compose requests with distro name which does not use dot-notation (specifically when the dot is removed from the distro name as it used to be in the past). In addition to that, allow also testing the distro alias using distro name without the minor version in compose requests. Enable these two new test variants in the CI. Signed-off-by: Tomáš Hozza --- .gitlab-ci.yml | 40 ++++++++ test/cases/api.sh | 132 +++++++++++++++++++++------ test/cases/api/aws.sh | 2 +- test/cases/api/azure.sh | 2 +- test/cases/api/common/common.sh | 10 ++ test/cases/api/common/s3.sh | 6 +- test/cases/api/container.registry.sh | 2 +- test/cases/api/gcp.sh | 2 +- test/cases/api/oci.sh | 2 +- tools/provision.sh | 19 +++- 10 files changed, 180 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86b98fddc..de93f459b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -568,6 +568,46 @@ API-module-hotfixes: INTERNAL_NETWORK: ["true"] TEST_MODULE_HOTFIXES: [1] +API-no-dot-notation: + stage: test + extends: .terraform + rules: + - !reference [.upstream_rules_all, rules] + # note: cloud API is not supported for on-prem installations so + # don't run this test case for nightly trees + script: + - schutzbot/deploy.sh + - /usr/libexec/tests/osbuild-composer/api.sh "${IMAGE_TYPE}" + parallel: + matrix: + - IMAGE_TYPE: + - aws + RUNNER: + - aws/rhel-8.9-ga-x86_64 + - aws/rhel-9.3-ga-x86_64 + INTERNAL_NETWORK: ["true"] + TEST_NO_DOT_NOTATION: [1] + +API-distro-alias: + stage: test + extends: .terraform + rules: + - !reference [.upstream_rules_all, rules] + # note: cloud API is not supported for on-prem installations so + # don't run this test case for nightly trees + script: + - schutzbot/deploy.sh + - /usr/libexec/tests/osbuild-composer/api.sh "${IMAGE_TYPE}" + parallel: + matrix: + - IMAGE_TYPE: + - aws-sap-rhui + RUNNER: + - aws/rhel-8.9-ga-x86_64 + - aws/rhel-9.3-ga-x86_64 + INTERNAL_NETWORK: ["true"] + TEST_DISTRO_ALIAS: [1] + .libvirt_integration: stage: test extends: .terraform/gcp diff --git a/test/cases/api.sh b/test/cases/api.sh index e9c26436a..12d0aeed3 100755 --- a/test/cases/api.sh +++ b/test/cases/api.sh @@ -22,7 +22,9 @@ CLOUD_PROVIDER_OCI="oci" # Supported Image type names # export IMAGE_TYPE_AWS="aws" +export IMAGE_TYPE_AWS_SAP_RHUI="aws-sap-rhui" export IMAGE_TYPE_AZURE="azure" +export IMAGE_TYPE_AZURE_SAP_RHUI="azure-sap-rhui" export IMAGE_TYPE_EDGE_COMMIT="edge-commit" export IMAGE_TYPE_EDGE_CONTAINER="edge-container" export IMAGE_TYPE_EDGE_INSTALLER="edge-installer" @@ -50,14 +52,50 @@ IMAGE_TYPE="$1" # set TEST_MODULE_HOTFIXES to 1 to enable module hotfixes for the test TEST_MODULE_HOTFIXES="${TEST_MODULE_HOTFIXES:-0}" +# set TEST_NO_DOT_NOTATION to 1 to disable dot notation in the test +# Not using the dot notation means that the dot separating the major and minor +# version in the distro name will be removed. +# +# This test variant is mutually exclusive with TEST_DISTRO_ALIAS. +TEST_NO_DOT_NOTATION="${TEST_NO_DOT_NOTATION:-0}" + +# set TEST_DISTRO_ALIAS to 1 to test configuring and using a distro alias +# when submitting composer request. +# +# This test variant deliberately uses the SAP image to test that the alias +# works as expected. The reason is that the SAP image will always have DNF +# configuration pinned down to a specific OS version and the version is the +# same as the distro definition used to generate the manifest. +# Specifically, the `/etc/dnf/vars/releasever` file will contain `X.Y`. +# +# This test variant is mutually exclusive with TEST_NO_DOT_NOTATION. +export TEST_DISTRO_ALIAS="${TEST_DISTRO_ALIAS:-0}" + +# TEST_NO_DOT_NOTATION and TEST_DISTRO_ALIAS are mutually exclusive +if [[ "$TEST_NO_DOT_NOTATION" == "1" && "$TEST_DISTRO_ALIAS" == "1" ]]; then + echo "TEST_NO_DOT_NOTATION and TEST_DISTRO_ALIAS are mutually exclusive" + exit 1 +fi + +# TEST_DISTRO_ALIAS requires that the IMAGE_TYPE is a SAP image +if [[ "$TEST_DISTRO_ALIAS" == "1" ]]; then + case "${IMAGE_TYPE}" in + "$IMAGE_TYPE_AWS_SAP_RHUI"|"IMAGE_TYPE_AZURE_SAP_RHUI") + ;; + *) + echo "TEST_DISTRO_ALIAS can only be used with SAP images" + exit 1 + esac +fi + # select cloud provider based on image type # # the supported image types are listed in the api spec (internal/cloudapi/v2/openapi.v2.yml) case ${IMAGE_TYPE} in - "$IMAGE_TYPE_AWS") + "$IMAGE_TYPE_AWS"|"$IMAGE_TYPE_AWS_SAP_RHUI") CLOUD_PROVIDER="${CLOUD_PROVIDER_AWS}" ;; - "$IMAGE_TYPE_AZURE") + "$IMAGE_TYPE_AZURE"|"IMAGE_TYPE_AZURE_SAP_RHUI") CLOUD_PROVIDER="${CLOUD_PROVIDER_AZURE}" ;; "$IMAGE_TYPE_GCP") @@ -88,14 +126,54 @@ ARTIFACTS="${ARTIFACTS:-/tmp/artifacts}" source /usr/libexec/osbuild-composer-test/set-env-variables.sh source /usr/libexec/tests/osbuild-composer/shared_lib.sh +export DISTRO="${DISTRO_CODE}" + # Container image used for cloud provider CLI tools export CONTAINER_IMAGE_CLOUD_TOOLS="quay.io/osbuild/cloud-tools:latest" +WORKDIR=$(mktemp -d) +KILL_PIDS=() +function cleanups() { + greenprint "Cleaning up" + set +eu + + cleanup + + # dump the DB here to ensure that it gets dumped even if the test fails + dump_db + + sudo "${CONTAINER_RUNTIME}" kill "${DB_CONTAINER_NAME}" + sudo "${CONTAINER_RUNTIME}" rm "${DB_CONTAINER_NAME}" + + sudo rm -rf "$WORKDIR" + + for P in "${KILL_PIDS[@]}"; do + sudo pkill -P "$P" + done + set -eu +} +trap cleanups EXIT + # # Provision the software under test. # -/usr/libexec/osbuild-composer-test/provision.sh +# Path to a file with extra composer configuration +EXTRA_COMPOSER_CONF="" + +# Configure distro alias for the host distro in composer without the minor +# version. +if [[ "${TEST_DISTRO_ALIAS}" == "1" ]]; then + DISTRO_ALIAS="${ID}-${VERSION_ID%.*}" + EXTRA_COMPOSER_CONF="$(mktemp -p "${WORKDIR}")" + + cat </dev/null +[distro_aliases] +${DISTRO_ALIAS} = "${DISTRO}" +EOF +fi + +/usr/libexec/osbuild-composer-test/provision.sh tls "${EXTRA_COMPOSER_CONF}" # # Set up the database queue @@ -133,7 +211,9 @@ PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PG "$(go env GOPATH)"/bin/tern migrate -m /usr/share/tests/osbuild-composer/schemas popd -cat < DISTRO == rhel-86 + # ID == centos && VERSION_ID == 8 => DISTRO == centos-8 + # ID == fedora && VERSION_ID == 35 => DISTRO == fedora-35 + DISTRO_NAME="$ID-${VERSION_ID//./}" +elif [[ "$TEST_DISTRO_ALIAS" == "1" ]]; then + DISTRO_NAME="$DISTRO_ALIAS" +else + DISTRO_NAME="$DISTRO" +fi +export DISTRO_NAME + SUBSCRIPTION_BLOCK= # Only RHEL need subscription block. @@ -490,6 +563,9 @@ function collectMetrics(){ } function sendCompose() { + echo "Compose request:" + cat "$1" + OUTPUT=$(mktemp) HTTPSTATUS=$(curl \ --silent \ diff --git a/test/cases/api/aws.sh b/test/cases/api/aws.sh index beb82d264..1a3a30bfe 100644 --- a/test/cases/api/aws.sh +++ b/test/cases/api/aws.sh @@ -36,7 +36,7 @@ function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/test/cases/api/azure.sh b/test/cases/api/azure.sh index dd8e11e53..59aa0e8bd 100644 --- a/test/cases/api/azure.sh +++ b/test/cases/api/azure.sh @@ -68,7 +68,7 @@ function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/test/cases/api/common/common.sh b/test/cases/api/common/common.sh index f2051415c..3e022b459 100644 --- a/test/cases/api/common/common.sh +++ b/test/cases/api/common/common.sh @@ -75,6 +75,16 @@ function _instanceCheck() { else echo "firewalld not available on host, that's fine" fi + + if [[ "${TEST_DISTRO_ALIAS}" == "1" ]]; then + echo "✔️ Checking DNF config of an image built using distro alias" + DNF_VARS_RELEASEVER=$($_ssh cat /etc/dnf/vars/releasever) + if [[ "$DNF_VARS_RELEASEVER" != "$VERSION_ID" ]]; then + echo "DNF config of an image built using distro alias has wrong releasever: $DNF_VARS_RELEASEVER" + echo "Expected: $VERSION_ID" + exit 1 + fi + fi } WORKER_REFRESH_TOKEN_PATH="/etc/osbuild-worker/token" diff --git a/test/cases/api/common/s3.sh b/test/cases/api/common/s3.sh index 5d7d72233..bde8587a8 100644 --- a/test/cases/api/common/s3.sh +++ b/test/cases/api/common/s3.sh @@ -16,7 +16,7 @@ function createReqFileEdge() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { @@ -62,7 +62,7 @@ EOF function createReqFileGuest() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { @@ -103,7 +103,7 @@ EOF function createReqFileVSphere() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { diff --git a/test/cases/api/container.registry.sh b/test/cases/api/container.registry.sh index 0a6ce8760..ea2ff7147 100644 --- a/test/cases/api/container.registry.sh +++ b/test/cases/api/container.registry.sh @@ -42,7 +42,7 @@ EOF function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { diff --git a/test/cases/api/gcp.sh b/test/cases/api/gcp.sh index f37c4575c..99e95faa4 100644 --- a/test/cases/api/gcp.sh +++ b/test/cases/api/gcp.sh @@ -55,7 +55,7 @@ function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/test/cases/api/oci.sh b/test/cases/api/oci.sh index a1ee19eaa..6ae41a2b3 100644 --- a/test/cases/api/oci.sh +++ b/test/cases/api/oci.sh @@ -60,7 +60,7 @@ function installClient() { function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/tools/provision.sh b/tools/provision.sh index 8a78313eb..293607233 100755 --- a/tools/provision.sh +++ b/tools/provision.sh @@ -15,6 +15,17 @@ AUTH_METHOD_NONE="none" # default to TLS for now AUTH_METHOD="${1:-$AUTH_METHOD_TLS}" +COMPOSER_CONFIG="/etc/osbuild-composer/osbuild-composer.toml" + +# Path to a file with additional configuration for composer. +# The content of this file will be appended to the default configuration. +EXTRA_COMPOSER_CONFIG="${2:-}" + +if [[ -n "${EXTRA_COMPOSER_CONFIG}" && ! -f "${EXTRA_COMPOSER_CONFIG}" ]]; then + echo "ERROR: File '${EXTRA_COMPOSER_CONFIG}' with extra configuration for composer does not exist." + exit 1 +fi + # koji and ansible are not in RHEL repositories. Depending on them in the spec # file breaks RHEL gating (see OSCI-1541). Therefore, we need to enable epel # and install koji and ansible here. @@ -83,7 +94,7 @@ EOF WORKER_TEST_CONFIG="/usr/share/tests/osbuild-composer/worker/osbuild-worker-tls.toml" fi - sudo cp -a "$COMPOSER_TEST_CONFIG" /etc/osbuild-composer/osbuild-composer.toml + sudo cp -a "$COMPOSER_TEST_CONFIG" "$COMPOSER_CONFIG" sudo cp -a "$WORKER_TEST_CONFIG" /etc/osbuild-worker/osbuild-worker.toml # if GCP credentials are defined in the ENV, add them to the worker's configuration @@ -209,6 +220,12 @@ else # AUTH_METHOD_NONE fi fi +# Append the extra configuration to the default configuration +if [[ -n "${EXTRA_COMPOSER_CONFIG}" ]]; then + echo "INFO: Appending extra composer configuration from '${EXTRA_COMPOSER_CONFIG}'" + cat "${EXTRA_COMPOSER_CONFIG}" | sudo tee -a "${COMPOSER_CONFIG}" +fi + # start appropriate units case "${AUTH_METHOD}" in "${AUTH_METHOD_JWT}" | "${AUTH_METHOD_TLS}")