diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86b98fddc..de93f459b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -568,6 +568,46 @@ API-module-hotfixes: INTERNAL_NETWORK: ["true"] TEST_MODULE_HOTFIXES: [1] +API-no-dot-notation: + stage: test + extends: .terraform + rules: + - !reference [.upstream_rules_all, rules] + # note: cloud API is not supported for on-prem installations so + # don't run this test case for nightly trees + script: + - schutzbot/deploy.sh + - /usr/libexec/tests/osbuild-composer/api.sh "${IMAGE_TYPE}" + parallel: + matrix: + - IMAGE_TYPE: + - aws + RUNNER: + - aws/rhel-8.9-ga-x86_64 + - aws/rhel-9.3-ga-x86_64 + INTERNAL_NETWORK: ["true"] + TEST_NO_DOT_NOTATION: [1] + +API-distro-alias: + stage: test + extends: .terraform + rules: + - !reference [.upstream_rules_all, rules] + # note: cloud API is not supported for on-prem installations so + # don't run this test case for nightly trees + script: + - schutzbot/deploy.sh + - /usr/libexec/tests/osbuild-composer/api.sh "${IMAGE_TYPE}" + parallel: + matrix: + - IMAGE_TYPE: + - aws-sap-rhui + RUNNER: + - aws/rhel-8.9-ga-x86_64 + - aws/rhel-9.3-ga-x86_64 + INTERNAL_NETWORK: ["true"] + TEST_DISTRO_ALIAS: [1] + .libvirt_integration: stage: test extends: .terraform/gcp diff --git a/test/cases/api.sh b/test/cases/api.sh index e9c26436a..12d0aeed3 100755 --- a/test/cases/api.sh +++ b/test/cases/api.sh @@ -22,7 +22,9 @@ CLOUD_PROVIDER_OCI="oci" # Supported Image type names # export IMAGE_TYPE_AWS="aws" +export IMAGE_TYPE_AWS_SAP_RHUI="aws-sap-rhui" export IMAGE_TYPE_AZURE="azure" +export IMAGE_TYPE_AZURE_SAP_RHUI="azure-sap-rhui" export IMAGE_TYPE_EDGE_COMMIT="edge-commit" export IMAGE_TYPE_EDGE_CONTAINER="edge-container" export IMAGE_TYPE_EDGE_INSTALLER="edge-installer" @@ -50,14 +52,50 @@ IMAGE_TYPE="$1" # set TEST_MODULE_HOTFIXES to 1 to enable module hotfixes for the test TEST_MODULE_HOTFIXES="${TEST_MODULE_HOTFIXES:-0}" +# set TEST_NO_DOT_NOTATION to 1 to disable dot notation in the test +# Not using the dot notation means that the dot separating the major and minor +# version in the distro name will be removed. +# +# This test variant is mutually exclusive with TEST_DISTRO_ALIAS. +TEST_NO_DOT_NOTATION="${TEST_NO_DOT_NOTATION:-0}" + +# set TEST_DISTRO_ALIAS to 1 to test configuring and using a distro alias +# when submitting composer request. +# +# This test variant deliberately uses the SAP image to test that the alias +# works as expected. The reason is that the SAP image will always have DNF +# configuration pinned down to a specific OS version and the version is the +# same as the distro definition used to generate the manifest. +# Specifically, the `/etc/dnf/vars/releasever` file will contain `X.Y`. +# +# This test variant is mutually exclusive with TEST_NO_DOT_NOTATION. +export TEST_DISTRO_ALIAS="${TEST_DISTRO_ALIAS:-0}" + +# TEST_NO_DOT_NOTATION and TEST_DISTRO_ALIAS are mutually exclusive +if [[ "$TEST_NO_DOT_NOTATION" == "1" && "$TEST_DISTRO_ALIAS" == "1" ]]; then + echo "TEST_NO_DOT_NOTATION and TEST_DISTRO_ALIAS are mutually exclusive" + exit 1 +fi + +# TEST_DISTRO_ALIAS requires that the IMAGE_TYPE is a SAP image +if [[ "$TEST_DISTRO_ALIAS" == "1" ]]; then + case "${IMAGE_TYPE}" in + "$IMAGE_TYPE_AWS_SAP_RHUI"|"IMAGE_TYPE_AZURE_SAP_RHUI") + ;; + *) + echo "TEST_DISTRO_ALIAS can only be used with SAP images" + exit 1 + esac +fi + # select cloud provider based on image type # # the supported image types are listed in the api spec (internal/cloudapi/v2/openapi.v2.yml) case ${IMAGE_TYPE} in - "$IMAGE_TYPE_AWS") + "$IMAGE_TYPE_AWS"|"$IMAGE_TYPE_AWS_SAP_RHUI") CLOUD_PROVIDER="${CLOUD_PROVIDER_AWS}" ;; - "$IMAGE_TYPE_AZURE") + "$IMAGE_TYPE_AZURE"|"IMAGE_TYPE_AZURE_SAP_RHUI") CLOUD_PROVIDER="${CLOUD_PROVIDER_AZURE}" ;; "$IMAGE_TYPE_GCP") @@ -88,14 +126,54 @@ ARTIFACTS="${ARTIFACTS:-/tmp/artifacts}" source /usr/libexec/osbuild-composer-test/set-env-variables.sh source /usr/libexec/tests/osbuild-composer/shared_lib.sh +export DISTRO="${DISTRO_CODE}" + # Container image used for cloud provider CLI tools export CONTAINER_IMAGE_CLOUD_TOOLS="quay.io/osbuild/cloud-tools:latest" +WORKDIR=$(mktemp -d) +KILL_PIDS=() +function cleanups() { + greenprint "Cleaning up" + set +eu + + cleanup + + # dump the DB here to ensure that it gets dumped even if the test fails + dump_db + + sudo "${CONTAINER_RUNTIME}" kill "${DB_CONTAINER_NAME}" + sudo "${CONTAINER_RUNTIME}" rm "${DB_CONTAINER_NAME}" + + sudo rm -rf "$WORKDIR" + + for P in "${KILL_PIDS[@]}"; do + sudo pkill -P "$P" + done + set -eu +} +trap cleanups EXIT + # # Provision the software under test. # -/usr/libexec/osbuild-composer-test/provision.sh +# Path to a file with extra composer configuration +EXTRA_COMPOSER_CONF="" + +# Configure distro alias for the host distro in composer without the minor +# version. +if [[ "${TEST_DISTRO_ALIAS}" == "1" ]]; then + DISTRO_ALIAS="${ID}-${VERSION_ID%.*}" + EXTRA_COMPOSER_CONF="$(mktemp -p "${WORKDIR}")" + + cat </dev/null +[distro_aliases] +${DISTRO_ALIAS} = "${DISTRO}" +EOF +fi + +/usr/libexec/osbuild-composer-test/provision.sh tls "${EXTRA_COMPOSER_CONF}" # # Set up the database queue @@ -133,7 +211,9 @@ PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PG "$(go env GOPATH)"/bin/tern migrate -m /usr/share/tests/osbuild-composer/schemas popd -cat < DISTRO == rhel-86 + # ID == centos && VERSION_ID == 8 => DISTRO == centos-8 + # ID == fedora && VERSION_ID == 35 => DISTRO == fedora-35 + DISTRO_NAME="$ID-${VERSION_ID//./}" +elif [[ "$TEST_DISTRO_ALIAS" == "1" ]]; then + DISTRO_NAME="$DISTRO_ALIAS" +else + DISTRO_NAME="$DISTRO" +fi +export DISTRO_NAME + SUBSCRIPTION_BLOCK= # Only RHEL need subscription block. @@ -490,6 +563,9 @@ function collectMetrics(){ } function sendCompose() { + echo "Compose request:" + cat "$1" + OUTPUT=$(mktemp) HTTPSTATUS=$(curl \ --silent \ diff --git a/test/cases/api/aws.sh b/test/cases/api/aws.sh index beb82d264..1a3a30bfe 100644 --- a/test/cases/api/aws.sh +++ b/test/cases/api/aws.sh @@ -36,7 +36,7 @@ function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/test/cases/api/azure.sh b/test/cases/api/azure.sh index dd8e11e53..59aa0e8bd 100644 --- a/test/cases/api/azure.sh +++ b/test/cases/api/azure.sh @@ -68,7 +68,7 @@ function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/test/cases/api/common/common.sh b/test/cases/api/common/common.sh index f2051415c..3e022b459 100644 --- a/test/cases/api/common/common.sh +++ b/test/cases/api/common/common.sh @@ -75,6 +75,16 @@ function _instanceCheck() { else echo "firewalld not available on host, that's fine" fi + + if [[ "${TEST_DISTRO_ALIAS}" == "1" ]]; then + echo "✔️ Checking DNF config of an image built using distro alias" + DNF_VARS_RELEASEVER=$($_ssh cat /etc/dnf/vars/releasever) + if [[ "$DNF_VARS_RELEASEVER" != "$VERSION_ID" ]]; then + echo "DNF config of an image built using distro alias has wrong releasever: $DNF_VARS_RELEASEVER" + echo "Expected: $VERSION_ID" + exit 1 + fi + fi } WORKER_REFRESH_TOKEN_PATH="/etc/osbuild-worker/token" diff --git a/test/cases/api/common/s3.sh b/test/cases/api/common/s3.sh index 5d7d72233..bde8587a8 100644 --- a/test/cases/api/common/s3.sh +++ b/test/cases/api/common/s3.sh @@ -16,7 +16,7 @@ function createReqFileEdge() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { @@ -62,7 +62,7 @@ EOF function createReqFileGuest() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { @@ -103,7 +103,7 @@ EOF function createReqFileVSphere() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { diff --git a/test/cases/api/container.registry.sh b/test/cases/api/container.registry.sh index 0a6ce8760..ea2ff7147 100644 --- a/test/cases/api/container.registry.sh +++ b/test/cases/api/container.registry.sh @@ -42,7 +42,7 @@ EOF function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "payload_repositories": [ { diff --git a/test/cases/api/gcp.sh b/test/cases/api/gcp.sh index f37c4575c..99e95faa4 100644 --- a/test/cases/api/gcp.sh +++ b/test/cases/api/gcp.sh @@ -55,7 +55,7 @@ function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/test/cases/api/oci.sh b/test/cases/api/oci.sh index a1ee19eaa..6ae41a2b3 100644 --- a/test/cases/api/oci.sh +++ b/test/cases/api/oci.sh @@ -60,7 +60,7 @@ function installClient() { function createReqFile() { cat > "$REQUEST_FILE" << EOF { - "distribution": "$DISTRO", + "distribution": "$DISTRO_NAME", "customizations": { "filesystem": [ { diff --git a/tools/provision.sh b/tools/provision.sh index 8a78313eb..293607233 100755 --- a/tools/provision.sh +++ b/tools/provision.sh @@ -15,6 +15,17 @@ AUTH_METHOD_NONE="none" # default to TLS for now AUTH_METHOD="${1:-$AUTH_METHOD_TLS}" +COMPOSER_CONFIG="/etc/osbuild-composer/osbuild-composer.toml" + +# Path to a file with additional configuration for composer. +# The content of this file will be appended to the default configuration. +EXTRA_COMPOSER_CONFIG="${2:-}" + +if [[ -n "${EXTRA_COMPOSER_CONFIG}" && ! -f "${EXTRA_COMPOSER_CONFIG}" ]]; then + echo "ERROR: File '${EXTRA_COMPOSER_CONFIG}' with extra configuration for composer does not exist." + exit 1 +fi + # koji and ansible are not in RHEL repositories. Depending on them in the spec # file breaks RHEL gating (see OSCI-1541). Therefore, we need to enable epel # and install koji and ansible here. @@ -83,7 +94,7 @@ EOF WORKER_TEST_CONFIG="/usr/share/tests/osbuild-composer/worker/osbuild-worker-tls.toml" fi - sudo cp -a "$COMPOSER_TEST_CONFIG" /etc/osbuild-composer/osbuild-composer.toml + sudo cp -a "$COMPOSER_TEST_CONFIG" "$COMPOSER_CONFIG" sudo cp -a "$WORKER_TEST_CONFIG" /etc/osbuild-worker/osbuild-worker.toml # if GCP credentials are defined in the ENV, add them to the worker's configuration @@ -209,6 +220,12 @@ else # AUTH_METHOD_NONE fi fi +# Append the extra configuration to the default configuration +if [[ -n "${EXTRA_COMPOSER_CONFIG}" ]]; then + echo "INFO: Appending extra composer configuration from '${EXTRA_COMPOSER_CONFIG}'" + cat "${EXTRA_COMPOSER_CONFIG}" | sudo tee -a "${COMPOSER_CONFIG}" +fi + # start appropriate units case "${AUTH_METHOD}" in "${AUTH_METHOD_JWT}" | "${AUTH_METHOD_TLS}")