diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile index 0bf110791..f0661ec16 100644 --- a/schutzbot/Jenkinsfile +++ b/schutzbot/Jenkinsfile @@ -13,6 +13,10 @@ pipeline { BUILD_CAUSE = detect_build_cause() GCP_BUCKET = "osbuild-composer-testing" GCP_REGION = "us-east4" + AZURE_TENANT_ID = "1710d22c-ccf0-4421-8ba7-0135cfaecb90" + AZURE_SUBSCRIPTION_ID = "8d026bb1-2a65-454d-a88f-c896db94c4f8" + AZURE_RESOURCE_GROUP = "sharing-research" + AZURE_LOCATION = "westeurope" } options { @@ -276,6 +280,8 @@ pipeline { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') + AZURE_CLIENT_ID = credentials('azure-client-id') + AZURE_CLIENT_SECRET = credentials('azure-client-secret') } steps { run_tests('integration') @@ -381,6 +387,8 @@ pipeline { AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') + AZURE_CLIENT_ID = credentials('azure-client-id') + AZURE_CLIENT_SECRET = credentials('azure-client-secret') } steps { run_tests('integration') @@ -667,6 +675,8 @@ pipeline { VCENTER_CREDS = credentials('vmware-vcenter-credentials') GOOGLE_APPLICATION_CREDENTIALS = credentials('gcp-credentials-osbuildci') GCP_API_TEST_SHARE_ACCOUNT = credentials('gcp-credentials-share-account') + AZURE_CLIENT_ID = credentials('azure-client-id') + AZURE_CLIENT_SECRET = credentials('azure-client-secret') } steps { run_tests('integration') @@ -764,6 +774,8 @@ pipeline { AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') GOOGLE_APPLICATION_CREDENTIALS = credentials('gcp-credentials-osbuildci') GCP_API_TEST_SHARE_ACCOUNT = credentials('gcp-credentials-share-account') + AZURE_CLIENT_ID = credentials('azure-client-id') + AZURE_CLIENT_SECRET = credentials('azure-client-secret') } steps { run_tests('integration') @@ -902,6 +914,11 @@ void run_tests(test_type) { ) } + sh ( + label: "Integration test: API (Azure)", + script: "/usr/libexec/tests/osbuild-composer/api.sh azure" + ) + if (env.VCENTER_CREDS) { // Run the VMWare test. sh ( diff --git a/test/cases/api.sh b/test/cases/api.sh index eefe2a0a2..703b6a0f0 100755 --- a/test/cases/api.sh +++ b/test/cases/api.sh @@ -27,6 +27,7 @@ set -euxo pipefail CLOUD_PROVIDER_AWS="aws" CLOUD_PROVIDER_GCP="gcp" +CLOUD_PROVIDER_AZURE="azure" CLOUD_PROVIDER=${1:-$CLOUD_PROVIDER_AWS} @@ -37,6 +38,9 @@ case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_GCP") echo "Testing Google Cloud Platform" ;; + "$CLOUD_PROVIDER_AZURE") + echo "Testing Azure" + ;; *) echo "Unknown cloud provider '$CLOUD_PROVIDER'. Supported are '$CLOUD_PROVIDER_AWS', '$CLOUD_PROVIDER_GCP'" exit 1 @@ -57,6 +61,11 @@ function checkEnvGCP() { printenv GOOGLE_APPLICATION_CREDENTIALS GCP_BUCKET GCP_REGION GCP_API_TEST_SHARE_ACCOUNT > /dev/null } +# Check that needed variables are set to access Azure. +function checkEnvAzure() { + printenv AZURE_TENANT_ID AZURE_SUBSCRIPTION_ID AZURE_RESOURCE_GROUP AZURE_LOCATION AZURE_CLIENT_ID AZURE_CLIENT_SECRET > /dev/null +} + case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_AWS") checkEnvAWS @@ -64,6 +73,9 @@ case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_GCP") checkEnvGCP ;; + "$CLOUD_PROVIDER_AZURE") + checkEnvAzure + ;; esac # @@ -102,6 +114,23 @@ function cleanupGCP() { fi } +function cleanupAzure() { + # since this function can be called at any time, ensure that we don't expand unbound variables + AZURE_CMD="${AZURE_CMD:-}" + AZURE_IMAGE_NAME="${AZURE_IMAGE_NAME:-}" + + # do not run clean-up if the image name is not yet defined + if [[ -n "$AZURE_CMD" && -n "$AZURE_IMAGE_NAME" ]]; then + set +e + $AZURE_CMD image delete --resource-group sharing-research --name "$AZURE_IMAGE_NAME" + + # find a storage account by its tag + AZURE_STORAGE_ACCOUNT=$($AZURE_CMD resource list --tag imageBuilderStorageAccount=location="$AZURE_LOCATION" | jq -r .[0].name) + $AZURE_CMD storage blob delete --container-name imagebuilder --name "$AZURE_IMAGE_NAME".vhd --account-name "$AZURE_STORAGE_ACCOUNT" + set -e + fi +} + WORKDIR=$(mktemp -d) function cleanup() { case $CLOUD_PROVIDER in @@ -111,6 +140,9 @@ function cleanup() { "$CLOUD_PROVIDER_GCP") cleanupGCP ;; + "$CLOUD_PROVIDER_AZURE") + cleanupAzure + ;; esac rm -rf "$WORKDIR" @@ -162,6 +194,24 @@ EOM $GCP_CMD --version } +function installClientAzure() { + if ! hash az; then + # this installation method is taken from the official docs: + # https://docs.microsoft.com/cs-cz/cli/azure/install-azure-cli-linux?pivots=dnf + sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc + echo -e "[azure-cli] +name=Azure CLI +baseurl=https://packages.microsoft.com/yumrepos/azure-cli +enabled=1 +gpgcheck=1 +gpgkey=https://packages.microsoft.com/keys/microsoft.asc" | sudo tee /etc/yum.repos.d/azure-cli.repo + fi + + sudo dnf install -y azure-cli + AZURE_CMD="az" + $AZURE_CMD version +} + case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_AWS") installClientAWS @@ -169,6 +219,9 @@ case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_GCP") installClientGCP ;; + "$CLOUD_PROVIDER_AZURE") + installClientAzure + ;; esac # @@ -296,6 +349,35 @@ function createReqFileGCP() { EOF } +function createReqFileAzure() { + AZURE_IMAGE_NAME="osbuild-composer-api-test-$(uuidgen)" + + cat > "$REQUEST_FILE" << EOF +{ + "distribution": "$DISTRO", + "image_requests": [ + { + "architecture": "$ARCH", + "image_type": "vhd", + "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json), + "upload_requests": [ + { + "type": "azure", + "options": { + "tenant_id": "${AZURE_TENANT_ID}", + "subscription_id": "${AZURE_SUBSCRIPTION_ID}", + "resource_group": "${AZURE_RESOURCE_GROUP}", + "location": "${AZURE_LOCATION}", + "image_name": "${AZURE_IMAGE_NAME}" + } + } + ] + } + ] +} +EOF +} + case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_AWS") createReqFileAWS @@ -303,6 +385,9 @@ case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_GCP") createReqFileGCP ;; + "$CLOUD_PROVIDER_AZURE") + createReqFileAzure + ;; esac # @@ -377,6 +462,13 @@ function checkUploadStatusOptionsGCP() { test "$PROJECT_ID" = "$GCP_PROJECT" } +function checkUploadStatusOptionsAzure() { + local IMAGE_NAME + IMAGE_NAME=$(echo "$UPLOAD_OPTIONS" | jq -r '.image_name') + + test "$IMAGE_NAME" = "$AZURE_IMAGE_NAME" +} + case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_AWS") checkUploadStatusOptionsAWS @@ -384,6 +476,9 @@ case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_GCP") checkUploadStatusOptionsGCP ;; + "$CLOUD_PROVIDER_AZURE") + checkUploadStatusOptionsAzure + ;; esac # @@ -504,6 +599,21 @@ function verifyInGCP() { ssh -oStrictHostKeyChecking=no -i "$GCP_SSH_KEY" "$SSH_USER"@"$HOST" rpm -q postgresql } +# Verify image in Azure +function verifyInAzure() { + set +x + $AZURE_CMD login --service-principal --username "${AZURE_CLIENT_ID}" --password "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" + set -x + + # verify that the image exists + $AZURE_CMD image show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_IMAGE_NAME}" + + # Boot testing is currently blocked due to + # https://github.com/Azure/azure-cli/issues/17123 + # Without this issue fixed or worked around, I'm not able to delete the disk + # attached to the VM. +} + case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_AWS") verifyInAWS @@ -511,6 +621,9 @@ case $CLOUD_PROVIDER in "$CLOUD_PROVIDER_GCP") verifyInGCP ;; + "$CLOUD_PROVIDER_AZURE") + verifyInAzure + ;; esac exit 0 diff --git a/tools/provision.sh b/tools/provision.sh index a254fbea6..cfc35a451 100755 --- a/tools/provision.sh +++ b/tools/provision.sh @@ -23,12 +23,29 @@ sudo cp -a /usr/share/tests/osbuild-composer/worker/osbuild-worker.toml \ GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-}" if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then # The credentials file must be copied to a different location. Jenkins places - # it into /tmp and as a restult, the worker would not see it due to using PrivateTmp=true. + # it into /tmp and as a result, the worker would not see it due to using PrivateTmp=true. GCP_CREDS_WORKER_PATH="/etc/osbuild-worker/gcp-credentials.json" sudo cp "$GOOGLE_APPLICATION_CREDENTIALS" "$GCP_CREDS_WORKER_PATH" echo -e "\n[gcp]\ncredentials = \"$GCP_CREDS_WORKER_PATH\"\n" | sudo tee -a /etc/osbuild-worker/osbuild-worker.toml fi +# if Azure credentials are defined in the env, create the credentials file +AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}" +AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET:-}" +if [[ -n "$AZURE_CLIENT_ID" && -n "$AZURE_CLIENT_SECRET" ]]; then + set +x + sudo tee /etc/osbuild-worker/azure-credentials.toml > /dev/null << EOF +client_id = "$AZURE_CLIENT_ID" +client_secret = "$AZURE_CLIENT_SECRET" +EOF + sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF + +[azure] +credentials = "/etc/osbuild-worker/azure-credentials.toml" +EOF + set -x +fi + # Copy rpmrepo snapshots for use in weldr tests sudo mkdir -p /etc/osbuild-composer/repositories # Copy all fedora repo overrides