particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
debian-forge-composer/test/cases/api.sh
Tomas Hozza 5acaaabc1d test/api.sh: do not print to stdout when dumping the DB 
Ensure that the content of the database is not printed to the console
when dumped at the end of the test case. The output is still preserved
as a CI run artifact.

Signed-off-by: Tomas Hozza <thozza@redhat.com>
2022-04-02 08:47:02 +01:00

1592 lines
49 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/bash
#
# Test osbuild-composer's main API endpoint by building a sample image and
# uploading it to the appropriate cloud provider. The test currently supports
# AWS and GCP.
#
# This script sets `-x` and is meant to always be run like that. This is
# simpler than adding extensive error reporting, which would make this script
# considerably more complex. Also, the full trace this produces is very useful
# for the primary audience: developers of osbuild-composer looking at the log
# from a run on a remote continuous integration system.
#
if (( $# != 1 )); then
echo "$0 requires exactly one argument"
echo "Please specify an image type to build"
exit 1
fi
set -euxo pipefail
IMAGE_TYPE="$1"
# Colorful timestamped output.
function greenprint {
echo -e "\033[1;32m[$(date -Isecond)] ${1}\033[0m"
}
ARTIFACTS=ci-artifacts
mkdir -p "${ARTIFACTS}"
source /usr/libexec/osbuild-composer-test/set-env-variables.sh
# Container image used for cloud provider CLI tools
CONTAINER_IMAGE_CLOUD_TOOLS="quay.io/osbuild/cloud-tools:latest"
#
# Provision the software under test.
#
/usr/libexec/osbuild-composer-test/provision.sh
#
# Set up the database queue
#
if which podman 2>/dev/null >&2; then
CONTAINER_RUNTIME=podman
elif which docker 2>/dev/null >&2; then
CONTAINER_RUNTIME=docker
else
echo No container runtime found, install podman or docker.
exit 2
fi
# Start the db
DB_CONTAINER_NAME="osbuild-composer-db"
sudo ${CONTAINER_RUNTIME} run -d --name "${DB_CONTAINER_NAME}" \
--health-cmd "pg_isready -U postgres -d osbuildcomposer" --health-interval 2s \
--health-timeout 2s --health-retries 10 \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=foobar \
-e POSTGRES_DB=osbuildcomposer \
-p 5432:5432 \
quay.io/osbuild/postgres:13-alpine
# Dump the logs once to have a little more output
sudo ${CONTAINER_RUNTIME} logs osbuild-composer-db
# Initialize a module in a temp dir so we can get tern without introducing
# vendoring inconsistency
pushd "$(mktemp -d)"
sudo dnf install -y go
go mod init temp
go get github.com/jackc/tern
PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PGPORT=5432 \
go run github.com/jackc/tern migrate -m /usr/share/tests/osbuild-composer/schemas
popd
cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
log_level = "debug"
[koji]
allowed_domains = [ "localhost", "client.osbuild.org" ]
ca = "/etc/osbuild-composer/ca-crt.pem"
[koji.aws_config]
bucket = "${AWS_BUCKET}"
[worker]
allowed_domains = [ "localhost", "worker.osbuild.org" ]
ca = "/etc/osbuild-composer/ca-crt.pem"
pg_host = "localhost"
pg_port = "5432"
pg_database = "osbuildcomposer"
pg_user = "postgres"
pg_password = "foobar"
pg_ssl_mode = "disable"
pg_max_conns = 10
EOF
sudo systemctl restart osbuild-composer
#
# Cloud provider / target names
#
CLOUD_PROVIDER_AWS="aws"
CLOUD_PROVIDER_GCP="gcp"
CLOUD_PROVIDER_AZURE="azure"
CLOUD_PROVIDER_AWS_S3="aws.s3"
#
# Supported Image type names
#
IMAGE_TYPE_AWS="aws"
IMAGE_TYPE_AZURE="azure"
IMAGE_TYPE_EDGE_COMMIT="edge-commit"
IMAGE_TYPE_EDGE_CONTAINER="edge-container"
IMAGE_TYPE_EDGE_INSTALLER="edge-installer"
IMAGE_TYPE_GCP="gcp"
IMAGE_TYPE_IMAGE_INSTALLER="image-installer"
IMAGE_TYPE_GUEST="guest-image"
IMAGE_TYPE_VSPHERE="vsphere"
# select cloud provider based on image type
#
# the supported image types are listed in the api spec (internal/cloudapi/v2/openapi.v2.yml)
case ${IMAGE_TYPE} in
"$IMAGE_TYPE_AWS")
CLOUD_PROVIDER="${CLOUD_PROVIDER_AWS}"
;;
"$IMAGE_TYPE_AZURE")
CLOUD_PROVIDER="${CLOUD_PROVIDER_AZURE}"
;;
"$IMAGE_TYPE_GCP")
CLOUD_PROVIDER="${CLOUD_PROVIDER_GCP}"
;;
"$IMAGE_TYPE_EDGE_COMMIT"|"$IMAGE_TYPE_EDGE_CONTAINER"|"$IMAGE_TYPE_EDGE_INSTALLER"|"$IMAGE_TYPE_IMAGE_INSTALLER"|"$IMAGE_TYPE_GUEST"|"$IMAGE_TYPE_VSPHERE")
# blobby image types: upload to s3 and provide download link
CLOUD_PROVIDER="${CLOUD_PROVIDER_AWS_S3}"
;;
*)
echo "Unknown image type: ${IMAGE_TYPE}"
exit 1
esac
greenprint "Using Cloud Provider / Target ${CLOUD_PROVIDER} for Image Type ${IMAGE_TYPE}"
#
# Verify that this script is running in the right environment.
#
# Check that needed variables are set to access AWS.
function checkEnvAWS() {
printenv AWS_REGION AWS_BUCKET V2_AWS_ACCESS_KEY_ID V2_AWS_SECRET_ACCESS_KEY AWS_API_TEST_SHARE_ACCOUNT > /dev/null
}
# Check that needed variables are set to access GCP.
function checkEnvGCP() {
printenv GOOGLE_APPLICATION_CREDENTIALS GCP_BUCKET GCP_REGION GCP_API_TEST_SHARE_ACCOUNT > /dev/null
}
# Check that needed variables are set to access Azure.
function checkEnvAzure() {
printenv AZURE_TENANT_ID AZURE_SUBSCRIPTION_ID AZURE_RESOURCE_GROUP AZURE_LOCATION V2_AZURE_CLIENT_ID V2_AZURE_CLIENT_SECRET > /dev/null
}
# Check that needed variables are set to register to RHSM (RHEL only)
function checkEnvSubscription() {
printenv API_TEST_SUBSCRIPTION_ORG_ID API_TEST_SUBSCRIPTION_ACTIVATION_KEY > /dev/null
}
function checkEnvVSphere() {
printenv GOVMOMI_USERNAME GOVMOMI_PASSWORD GOVMOMI_URL GOVMOMI_CLUSTER GOVC_DATACENTER GOVMOMI_DATASTORE GOVMOMI_FOLDER GOVMOMI_NETWORK > /dev/null
}
case $CLOUD_PROVIDER in
"$CLOUD_PROVIDER_AWS" | "$CLOUD_PROVIDER_AWS_S3")
checkEnvAWS
[[ "${IMAGE_TYPE}" == "${IMAGE_TYPE_VSPHERE}" ]] && checkEnvVSphere
;;
"$CLOUD_PROVIDER_GCP")
checkEnvGCP
;;
"$CLOUD_PROVIDER_AZURE")
checkEnvAzure
;;
esac
[[ "$ID" == "rhel" ]] && checkEnvSubscription
#
# Create a temporary directory and ensure it gets deleted when this script
# terminates in any way.
#
function cleanupAWS() {
# since this function can be called at any time, ensure that we don't expand unbound variables
AWS_CMD="${AWS_CMD:-}"
AWS_INSTANCE_ID="${AWS_INSTANCE_ID:-}"
AMI_IMAGE_ID="${AMI_IMAGE_ID:-}"
AWS_SNAPSHOT_ID="${AWS_SNAPSHOT_ID:-}"
if [ -n "$AWS_CMD" ]; then
$AWS_CMD ec2 terminate-instances --instance-ids "$AWS_INSTANCE_ID"
$AWS_CMD ec2 deregister-image --image-id "$AMI_IMAGE_ID"
$AWS_CMD ec2 delete-snapshot --snapshot-id "$AWS_SNAPSHOT_ID"
$AWS_CMD ec2 delete-key-pair --key-name "key-for-$AMI_IMAGE_ID"
fi
}
function cleanupAWSS3() {
local S3_URL
S3_URL=$(echo "$UPLOAD_OPTIONS" | jq -r '.url')
# extract filename component from URL
local S3_FILENAME
S3_FILENAME=$(echo "${S3_URL}" | grep -oP '(?<=/)[^/]+(?=\?)')
# prepend bucket
local S3_URI
S3_URI="s3://${AWS_BUCKET}/${S3_FILENAME}"
# since this function can be called at any time, ensure that we don't expand unbound variables
AWS_CMD="${AWS_CMD:-}"
if [ -n "$AWS_CMD" ]; then
$AWS_CMD s3 rm "${S3_URI}"
fi
}
function cleanupGCP() {
# since this function can be called at any time, ensure that we don't expand unbound variables
GCP_CMD="${GCP_CMD:-}"
GCP_IMAGE_NAME="${GCP_IMAGE_NAME:-}"
GCP_INSTANCE_NAME="${GCP_INSTANCE_NAME:-}"
GCP_ZONE="${GCP_ZONE:-}"
if [ -n "$GCP_CMD" ]; then
$GCP_CMD compute instances delete --zone="$GCP_ZONE" "$GCP_INSTANCE_NAME"
$GCP_CMD compute images delete "$GCP_IMAGE_NAME"
fi
}
function cleanupAzure() {
# since this function can be called at any time, ensure that we don't expand unbound variables
AZURE_CMD="${AZURE_CMD:-}"
AZURE_IMAGE_NAME="${AZURE_IMAGE_NAME:-}"
AZURE_INSTANCE_NAME="${AZURE_INSTANCE_NAME:-}"
# do not run clean-up if the image name is not yet defined
if [[ -n "$AZURE_CMD" && -n "$AZURE_IMAGE_NAME" ]]; then
# Re-get the vm_details in case the VM creation is failed.
[ -f "$WORKDIR/vm_details.json" ] || $AZURE_CMD vm show --name "$AZURE_INSTANCE_NAME" --resource-group "$AZURE_RESOURCE_GROUP" --show-details > "$WORKDIR/vm_details.json"
# Get all the resources ids
VM_ID=$(jq -r '.id' "$WORKDIR"/vm_details.json)
OSDISK_ID=$(jq -r '.storageProfile.osDisk.managedDisk.id' "$WORKDIR"/vm_details.json)
NIC_ID=$(jq -r '.networkProfile.networkInterfaces[0].id' "$WORKDIR"/vm_details.json)
$AZURE_CMD network nic show --ids "$NIC_ID" > "$WORKDIR"/nic_details.json
NSG_ID=$(jq -r '.networkSecurityGroup.id' "$WORKDIR"/nic_details.json)
PUBLICIP_ID=$(jq -r '.ipConfigurations[0].publicIpAddress.id' "$WORKDIR"/nic_details.json)
# Delete resources. Some resources must be removed in order:
# - Delete VM prior to any other resources
# - Delete NIC prior to NSG, public-ip
# Left Virtual Network and Storage Account there because other tests in the same resource group will reuse them
for id in "$VM_ID" "$OSDISK_ID" "$NIC_ID" "$NSG_ID" "$PUBLICIP_ID"; do
echo "Deleting $id..."
$AZURE_CMD resource delete --ids "$id"
done
# Delete image after VM deleting.
$AZURE_CMD image delete --resource-group "$AZURE_RESOURCE_GROUP" --name "$AZURE_IMAGE_NAME"
# find a storage account by its tag
AZURE_STORAGE_ACCOUNT=$($AZURE_CMD resource list --tag imageBuilderStorageAccount=location="$AZURE_LOCATION" | jq -r .[0].name)
AZURE_CONNECTION_STRING=$($AZURE_CMD storage account show-connection-string --name "$AZURE_STORAGE_ACCOUNT" | jq -r .connectionString)
$AZURE_CMD storage blob delete --container-name imagebuilder --name "$AZURE_IMAGE_NAME".vhd --account-name "$AZURE_STORAGE_ACCOUNT" --connection-string "$AZURE_CONNECTION_STRING"
fi
}
function cleanupVSphere() {
# since this function can be called at any time, ensure that we don't expand unbound variables
GOVC_CMD="${GOVC_CMD:-}"
VSPHERE_VM_NAME="${VSPHERE_VM_NAME:-}"
VSPHERE_CIDATA_ISO_PATH="${VSPHERE_CIDATA_ISO_PATH:-}"
greenprint "🧹 Cleaning up the VSphere VM"
$GOVC_CMD vm.destroy \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
"${VSPHERE_VM_NAME}"
greenprint "🧹 Cleaning up the VSphere Datastore"
$GOVC_CMD datastore.rm \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-ds="${GOVMOMI_DATASTORE}" \
-f \
"${VSPHERE_CIDATA_ISO_PATH}"
$GOVC_CMD datastore.rm \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-ds="${GOVMOMI_DATASTORE}" \
-f \
"${VSPHERE_VM_NAME}"
}
function dump_db() {
# Disable -x for these commands to avoid printing the whole result and manifest into the log
set +x
# Make sure we get 3 job entries in the db per compose (depsolve + manifest + build)
sudo ${CONTAINER_RUNTIME} exec "${DB_CONTAINER_NAME}" psql -U postgres -d osbuildcomposer -c "SELECT * FROM jobs;" | grep "9 rows" > /dev/null
# Save the result, including the manifest, for the job, straight from the db
sudo ${CONTAINER_RUNTIME} exec "${DB_CONTAINER_NAME}" psql -U postgres -d osbuildcomposer -c "SELECT result FROM jobs WHERE type='manifest-id-only'" \
| gpg --batch --yes --passphrase "${GPG_SYMMETRIC_PASSPHRASE}" -o "${ARTIFACTS}/build-result.gpg" --symmetric -
set -x
}
WORKDIR=$(mktemp -d)
KILL_PIDS=()
function cleanup() {
set +eu
case $CLOUD_PROVIDER in
"$CLOUD_PROVIDER_AWS")
cleanupAWS
;;
"$CLOUD_PROVIDER_AWS_S3")
cleanupAWSS3
[[ "${IMAGE_TYPE}" == "${IMAGE_TYPE_VSPHERE}" ]] && cleanupVSphere
;;
"$CLOUD_PROVIDER_GCP")
cleanupGCP
;;
"$CLOUD_PROVIDER_AZURE")
cleanupAzure
;;
esac
# dump the DB here to ensure that it gets dumped even if the test fails
dump_db
sudo ${CONTAINER_RUNTIME} kill "${DB_CONTAINER_NAME}"
sudo ${CONTAINER_RUNTIME} rm "${DB_CONTAINER_NAME}"
sudo rm -rf "$WORKDIR"
for P in "${KILL_PIDS[@]}"; do
sudo pkill -P "$P"
done
set -eu
}
trap cleanup EXIT
# make a dummy rpm and repo to test payload_repositories
sudo dnf install -y rpm-build createrepo
DUMMYRPMDIR=$(mktemp -d)
DUMMYSPECFILE="$DUMMYRPMDIR/dummy.spec"
PAYLOAD_REPO_PORT="9999"
PAYLOAD_REPO_URL="http://localhost:9999"
pushd "$DUMMYRPMDIR"
cat <<EOF > "$DUMMYSPECFILE"
#----------- spec file starts ---------------
Name: dummy
Version: 1.0.0
Release: 0
BuildArch: noarch
Vendor: dummy
Summary: Provides %{name}
License: BSD
Provides: dummy
%description
%{summary}
%files
EOF
mkdir -p "DUMMYRPMDIR/rpmbuild"
rpmbuild --quiet --define "_topdir $DUMMYRPMDIR/rpmbuild" -bb "$DUMMYSPECFILE"
mkdir -p "$DUMMYRPMDIR/repo"
cp "$DUMMYRPMDIR"/rpmbuild/RPMS/noarch/*rpm "$DUMMYRPMDIR/repo"
pushd "$DUMMYRPMDIR/repo"
createrepo .
sudo python3 -m http.server "$PAYLOAD_REPO_PORT" &
KILL_PIDS+=("$!")
popd
popd
#
# Install the necessary cloud provider client tools
#
function installClientAWS() {
if ! hash aws; then
echo "Using 'awscli' from a container"
sudo ${CONTAINER_RUNTIME} pull ${CONTAINER_IMAGE_CLOUD_TOOLS}
AWS_CMD="sudo ${CONTAINER_RUNTIME} run --rm \
-e AWS_ACCESS_KEY_ID=${V2_AWS_ACCESS_KEY_ID} \
-e AWS_SECRET_ACCESS_KEY=${V2_AWS_SECRET_ACCESS_KEY} \
-v ${WORKDIR}:${WORKDIR}:Z \
${CONTAINER_IMAGE_CLOUD_TOOLS} aws --region $AWS_REGION --output json --color on"
else
echo "Using pre-installed 'aws' from the system"
AWS_CMD="aws --region $AWS_REGION --output json --color on"
fi
$AWS_CMD --version
}
function installClientGCP() {
if ! hash gcloud; then
echo "Using 'gcloud' from a container"
sudo ${CONTAINER_RUNTIME} pull ${CONTAINER_IMAGE_CLOUD_TOOLS}
# directory mounted to the container, in which gcloud stores the credentials after logging in
GCP_CMD_CREDS_DIR="${WORKDIR}/gcloud_credentials"
mkdir "${GCP_CMD_CREDS_DIR}"
GCP_CMD="sudo ${CONTAINER_RUNTIME} run --rm \
-v ${GCP_CMD_CREDS_DIR}:/root/.config/gcloud:Z \
-v ${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS}:Z \
-v ${WORKDIR}:${WORKDIR}:Z \
${CONTAINER_IMAGE_CLOUD_TOOLS} gcloud --format=json"
else
echo "Using pre-installed 'gcloud' from the system"
GCP_CMD="gcloud --format=json --quiet"
fi
$GCP_CMD --version
}
function installClientAzure() {
if ! hash az; then
echo "Using 'azure-cli' from a container"
sudo ${CONTAINER_RUNTIME} pull ${CONTAINER_IMAGE_CLOUD_TOOLS}
# directory mounted to the container, in which azure-cli stores the credentials after logging in
AZURE_CMD_CREDS_DIR="${WORKDIR}/azure-cli_credentials"
mkdir "${AZURE_CMD_CREDS_DIR}"
AZURE_CMD="sudo ${CONTAINER_RUNTIME} run --rm \
-v ${AZURE_CMD_CREDS_DIR}:/root/.azure:Z \
-v ${WORKDIR}:${WORKDIR}:Z \
${CONTAINER_IMAGE_CLOUD_TOOLS} az"
else
echo "Using pre-installed 'azure-cli' from the system"
AZURE_CMD="az"
fi
$AZURE_CMD version
}
function installClientVSphere() {
if ! hash govc; then
greenprint "Installing govc"
pushd "${WORKDIR}"
curl -Ls --retry 5 --output govc.gz \
https://github.com/vmware/govmomi/releases/download/v0.24.0/govc_linux_amd64.gz
gunzip -f govc.gz
GOVC_CMD="${WORKDIR}/govc"
chmod +x "${GOVC_CMD}"
popd
else
echo "Using pre-installed 'govc' from the system"
GOVC_CMD="govc"
fi
$GOVC_CMD version
}
case $CLOUD_PROVIDER in
"$CLOUD_PROVIDER_AWS" | "$CLOUD_PROVIDER_AWS_S3")
installClientAWS
[[ "${IMAGE_TYPE}" == "${IMAGE_TYPE_VSPHERE}" ]] && installClientVSphere
;;
"$CLOUD_PROVIDER_GCP")
installClientGCP
;;
"$CLOUD_PROVIDER_AZURE")
installClientAzure
;;
esac
#
# Make sure /openapi and endpoints return success
#
curl \
--silent \
--show-error \
--cacert /etc/osbuild-composer/ca-crt.pem \
--key /etc/osbuild-composer/client-key.pem \
--cert /etc/osbuild-composer/client-crt.pem \
https://localhost/api/image-builder-composer/v2/openapi | jq .
#
# Prepare a request to be sent to the composer API.
#
REQUEST_FILE="${WORKDIR}/request.json"
ARCH=$(uname -m)
SSH_USER=
# Generate a string, which can be used as a predictable resource name,
# especially when running the test in CI where we may need to clean up
# resources in case the test unexpectedly fails or is canceled
CI="${CI:-false}"
if [[ "$CI" == true ]]; then
# in CI, imitate GenerateCIArtifactName() from internal/test/helpers.go
TEST_ID="$DISTRO_CODE-$ARCH-$CI_COMMIT_BRANCH-$CI_BUILD_ID"
else
# if not running in Jenkins, generate ID not relying on specific env variables
TEST_ID=$(uuidgen);
fi
if [[ "$ID" == "fedora" ]]; then
# fedora uses fedora for everything
SSH_USER="fedora"
elif [[ "$CLOUD_PROVIDER" == "$CLOUD_PROVIDER_AWS" ]]; then
# RHEL and centos use ec2-user for AWS
SSH_USER="ec2-user"
else
# RHEL and centos use cloud-user for other clouds
SSH_USER="cloud-user"
fi
# This removes dot from VERSION_ID.
# ID == rhel && VERSION_ID == 8.6 => DISTRO == rhel-86
# ID == centos && VERSION_ID == 8 => DISTRO == centos-8
# ID == fedora && VERSION_ID == 35 => DISTRO == fedora-35
DISTRO="$ID-${VERSION_ID//./}"
# Only RHEL need subscription block.
if [[ "$ID" == "rhel" ]]; then
SUBSCRIPTION_BLOCK=$(cat <<EndOfMessage
,
"subscription": {
"organization": "${API_TEST_SUBSCRIPTION_ORG_ID:-}",
"activation_key": "${API_TEST_SUBSCRIPTION_ACTIVATION_KEY:-}",
"base_url": "https://cdn.redhat.com/",
"server_url": "subscription.rhsm.redhat.com",
"insights": true
}
EndOfMessage
)
else
SUBSCRIPTION_BLOCK=''
fi
# generate a temp key for user tests
ssh-keygen -t rsa-sha2-512 -f "${WORKDIR}/usertest" -C "usertest" -N ""
function createReqFileAWS() {
AWS_SNAPSHOT_NAME=${TEST_ID}
cat > "$REQUEST_FILE" << EOF
{
"distribution": "$DISTRO",
"customizations": {
"filesystem": [
{
"mountpoint": "/var",
"min_size": 262144000
}
],
"payload_repositories": [
{
"baseurl": "$PAYLOAD_REPO_URL"
}
],
"packages": [
"postgresql",
"dummy"
]${SUBSCRIPTION_BLOCK},
"users":[
{
"name": "user1",
"groups": ["wheel"],
"key": "$(cat "${WORKDIR}/usertest.pub")"
},
{
"name": "user2",
"key": "$(cat "${WORKDIR}/usertest.pub")"
}
]
},
"image_request": {
"architecture": "$ARCH",
"image_type": "${IMAGE_TYPE}",
"repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
"upload_options": {
"region": "${AWS_REGION}",
"snapshot_name": "${AWS_SNAPSHOT_NAME}",
"share_with_accounts": ["${AWS_API_TEST_SHARE_ACCOUNT}"]
}
}
}
EOF
}
#
# Global var for ostree ref (only used in aws.s3 now)
#
OSTREE_REF="test/rhel/8/edge"
function createReqFileAWSS3() {
cat > "$REQUEST_FILE" << EOF
{
"distribution": "$DISTRO",
"customizations": {
"payload_repositories": [
{
"baseurl": "$PAYLOAD_REPO_URL"
}
],
"packages": [
"postgresql",
"dummy"
],
"users":[
{
"name": "user1",
"groups": ["wheel"],
"key": "$(cat "${WORKDIR}/usertest.pub")"
},
{
"name": "user2",
"key": "$(cat "${WORKDIR}/usertest.pub")"
}
]
},
"image_request": {
"architecture": "$ARCH",
"image_type": "${IMAGE_TYPE}",
"repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
"ostree": {
"ref": "${OSTREE_REF}"
},
"upload_options": {
"region": "${AWS_REGION}"
}
}
}
EOF
}
# the VSphere test case does not create any additional users,
# since this is not supported by the service UI
function createReqFileAWSS3VSphere() {
cat > "$REQUEST_FILE" << EOF
{
"distribution": "$DISTRO",
"customizations": {
"payload_repositories": [
{
"baseurl": "$PAYLOAD_REPO_URL"
}
],
"packages": [
"postgresql",
"dummy"
]${SUBSCRIPTION_BLOCK}
},
"image_request": {
"architecture": "$ARCH",
"image_type": "${IMAGE_TYPE}",
"repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
"upload_options": {
"region": "${AWS_REGION}"
}
}
}
EOF
}
function createReqFileGCP() {
# constrains for GCP resource IDs:
# - max 62 characters
# - must be a match of regex '[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?|[1-9][0-9]{0,19}'
#
# use sha224sum to get predictable 56 characters long testID without invalid characters
GCP_TEST_ID_HASH="$(echo -n "$TEST_ID" | sha224sum - | sed -E 's/([a-z0-9])\s+-/\1/')"
GCP_IMAGE_NAME="image-$GCP_TEST_ID_HASH"
cat > "$REQUEST_FILE" << EOF
{
"distribution": "$DISTRO",
"customizations": {
"filesystem": [
{
"mountpoint": "/var",
"min_size": 262144000
}
],
"payload_repositories": [
{
"baseurl": "$PAYLOAD_REPO_URL"
}
],
"packages": [
"postgresql",
"dummy"
]${SUBSCRIPTION_BLOCK}
},
"image_request": {
"architecture": "$ARCH",
"image_type": "${IMAGE_TYPE}",
"repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
"upload_options": {
"bucket": "${GCP_BUCKET}",
"region": "${GCP_REGION}",
"image_name": "${GCP_IMAGE_NAME}",
"share_with_accounts": ["${GCP_API_TEST_SHARE_ACCOUNT}"]
}
}
}
EOF
}
function createReqFileAzure() {
AZURE_IMAGE_NAME="image-$TEST_ID"
cat > "$REQUEST_FILE" << EOF
{
"distribution": "$DISTRO",
"customizations": {
"filesystem": [
{
"mountpoint": "/var",
"min_size": 262144000
}
],
"payload_repositories": [
{
"baseurl": "$PAYLOAD_REPO_URL"
}
],
"packages": [
"postgresql",
"dummy"
]${SUBSCRIPTION_BLOCK}
},
"image_request": {
"architecture": "$ARCH",
"image_type": "${IMAGE_TYPE}",
"repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
"upload_options": {
"tenant_id": "${AZURE_TENANT_ID}",
"subscription_id": "${AZURE_SUBSCRIPTION_ID}",
"resource_group": "${AZURE_RESOURCE_GROUP}",
"location": "${AZURE_LOCATION}",
"image_name": "${AZURE_IMAGE_NAME}"
}
}
}
EOF
}
case $CLOUD_PROVIDER in
"$CLOUD_PROVIDER_AWS")
createReqFileAWS
;;
"$CLOUD_PROVIDER_AWS_S3")
if [[ "${IMAGE_TYPE}" == "${IMAGE_TYPE_VSPHERE}" ]]; then
createReqFileAWSS3VSphere
else
createReqFileAWSS3
fi
;;
"$CLOUD_PROVIDER_GCP")
createReqFileGCP
;;
"$CLOUD_PROVIDER_AZURE")
createReqFileAzure
;;
esac
#
# Send the request and wait for the job to finish.
#
# Separate `curl` and `jq` commands here, because piping them together hides
# the server's response in case of an error.
#
function collectMetrics(){
METRICS_OUTPUT=$(curl \
--cacert /etc/osbuild-composer/ca-crt.pem \
--key /etc/osbuild-composer/client-key.pem \
--cert /etc/osbuild-composer/client-crt.pem \
https://localhost/metrics)
echo "$METRICS_OUTPUT" | grep "^image_builder_composer_total_compose_requests" | cut -f2 -d' '
}
function sendCompose() {
OUTPUT=$(mktemp)
HTTPSTATUS=$(curl \
--silent \
--show-error \
--cacert /etc/osbuild-composer/ca-crt.pem \
--key /etc/osbuild-composer/client-key.pem \
--cert /etc/osbuild-composer/client-crt.pem \
--header 'Content-Type: application/json' \
--request POST \
--data @"$1" \
--write-out '%{http_code}' \
--output "$OUTPUT" \
https://localhost/api/image-builder-composer/v2/compose)
test "$HTTPSTATUS" = "201"
COMPOSE_ID=$(jq -r '.id' "$OUTPUT")
}
function waitForState() {
local DESIRED_STATE="${1:-success}"
while true
do
OUTPUT=$(curl \
--silent \
--show-error \
--cacert /etc/osbuild-composer/ca-crt.pem \
--key /etc/osbuild-composer/client-key.pem \
--cert /etc/osbuild-composer/client-crt.pem \
"https://localhost/api/image-builder-composer/v2/composes/$COMPOSE_ID")
COMPOSE_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.status')
UPLOAD_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.status')
UPLOAD_TYPE=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.type')
UPLOAD_OPTIONS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.options')
case "$COMPOSE_STATUS" in
"$DESIRED_STATE")
break
;;
# all valid status values for a compose which hasn't finished yet
"pending"|"building"|"uploading"|"registering")
;;
# default undesired state
"failure")
echo "Image compose failed"
exit 1
;;
*)
echo "API returned unexpected image_status.status value: '$COMPOSE_STATUS'"
exit 1
;;
esac
sleep 30
done
}
#
# Make sure that requesting a non existing paquet results in failure
#
REQUEST_FILE2="${WORKDIR}/request2.json"
jq '.customizations.packages = [ "jesuisunpaquetquinexistepas" ]' "$REQUEST_FILE" > "$REQUEST_FILE2"
sendCompose "$REQUEST_FILE2"
waitForState "failure"
# crashed/stopped/killed worker should result in a failed state
sendCompose "$REQUEST_FILE"
waitForState "building"
sudo systemctl stop "osbuild-worker@*"
waitForState "failure"
sudo systemctl start "osbuild-worker@1"
# full integration case
INIT_COMPOSES="$(collectMetrics)"
sendCompose "$REQUEST_FILE"
waitForState
SUBS_COMPOSES="$(collectMetrics)"
test "$UPLOAD_STATUS" = "success"
test "$UPLOAD_TYPE" = "$CLOUD_PROVIDER"
test $((INIT_COMPOSES+1)) = "$SUBS_COMPOSES"
#
# Verify the Cloud-provider specific upload_status options
#
function checkUploadStatusOptionsAWS() {
local AMI
AMI=$(echo "$UPLOAD_OPTIONS" | jq -r '.ami')
local REGION
REGION=$(echo "$UPLOAD_OPTIONS" | jq -r '.region')
# AWS ID consist of resource identifier followed by a 17-character string
echo "$AMI" | grep -e 'ami-[[:alnum:]]\{17\}' -
test "$REGION" = "$AWS_REGION"
}
function checkUploadStatusOptionsAWSS3() {
local S3_URL
S3_URL=$(echo "$UPLOAD_OPTIONS" | jq -r '.url')
# S3 URL contains region and bucket name
echo "$S3_URL" | grep -F "$AWS_BUCKET" -
echo "$S3_URL" | grep -F "$AWS_REGION" -
}
function checkUploadStatusOptionsGCP() {
GCP_PROJECT=$(jq -r '.project_id' "$GOOGLE_APPLICATION_CREDENTIALS")
local IMAGE_NAME
IMAGE_NAME=$(echo "$UPLOAD_OPTIONS" | jq -r '.image_name')
local PROJECT_ID
PROJECT_ID=$(echo "$UPLOAD_OPTIONS" | jq -r '.project_id')
test "$IMAGE_NAME" = "$GCP_IMAGE_NAME"
test "$PROJECT_ID" = "$GCP_PROJECT"
}
function checkUploadStatusOptionsAzure() {
local IMAGE_NAME
IMAGE_NAME=$(echo "$UPLOAD_OPTIONS" | jq -r '.image_name')
test "$IMAGE_NAME" = "$AZURE_IMAGE_NAME"
}
case $CLOUD_PROVIDER in
"$CLOUD_PROVIDER_AWS")
checkUploadStatusOptionsAWS
;;
"$CLOUD_PROVIDER_AWS_S3")
checkUploadStatusOptionsAWSS3
;;
"$CLOUD_PROVIDER_GCP")
checkUploadStatusOptionsGCP
;;
"$CLOUD_PROVIDER_AZURE")
checkUploadStatusOptionsAzure
;;
esac
#
# Verify the image landed in the appropriate cloud provider, and delete it.
#
# Reusable function, which waits for a given host to respond to SSH
function _instanceWaitSSH() {
local HOST="$1"
for LOOP_COUNTER in {0..30}; do
if ssh-keyscan "$HOST" > /dev/null 2>&1; then
echo "SSH is up!"
ssh-keyscan "$HOST" | sudo tee -a /root/.ssh/known_hosts
break
fi
echo "Retrying in 5 seconds... $LOOP_COUNTER"
sleep 5
done
}
function _instanceCheck() {
echo "✔️ Instance checking"
local _ssh="$1"
# Check if postgres is installed
$_ssh rpm -q postgresql dummy
# Verify subscribe status. Loop check since the system may not be registered such early(RHEL only)
if [[ "$ID" == "rhel" ]]; then
set +eu
for LOOP_COUNTER in {1..10}; do
subscribe_org_id=$($_ssh sudo subscription-manager identity | grep 'org ID')
if [[ "$subscribe_org_id" == "org ID: $API_TEST_SUBSCRIPTION_ORG_ID" ]]; then
echo "System is subscribed."
break
else
echo "System is not subscribed. Retrying in 30 seconds...($LOOP_COUNTER/10)"
sleep 30
fi
done
set -eu
[[ "$subscribe_org_id" == "org ID: $API_TEST_SUBSCRIPTION_ORG_ID" ]]
# Unregister subscription
$_ssh sudo subscription-manager unregister
else
echo "Not RHEL OS. Skip subscription check."
fi
}
# Create a cloud-int user-data file
#
# Returns:
# - path to the user-data file
#
# Arguments:
# $1 - default username
# $2 - path to the SSH public key to set as authorized for the user
function _createCIUserdata() {
local _user="$1"
local _ssh_pubkey_path="$2"
local _ci_userdata_dir
_ci_userdata_dir="$(mktemp -d -p "${WORKDIR}")"
local _ci_userdata_path="${_ci_userdata_dir}/user-data"
cat > "${_ci_userdata_path}" <<EOF
#cloud-config
users:
- name: "${_user}"
sudo: "ALL=(ALL) NOPASSWD:ALL"
ssh_authorized_keys:
- "$(cat "${_ssh_pubkey_path}")"
EOF
echo "${_ci_userdata_path}"
}
# Create a cloud-int meta-data file
#
# Returns:
# - path to the meta-data file
#
# Arguments:
# $1 - VM name
function _createCIMetadata() {
local _vm_name="$1"
local _ci_metadata_dir
_ci_metadata_dir="$(mktemp -d -p "${WORKDIR}")"
local _ci_metadata_path="${_ci_metadata_dir}/meta-data"
cat > "${_ci_metadata_path}" <<EOF
instance-id: ${_vm_name}
local-hostname: ${_vm_name}
EOF
echo "${_ci_metadata_path}"
}
# Create an ISO with the provided cloud-init user-data file
#
# Returns:
# - path to the created ISO file
#
# Arguments:
# $1 - path to the cloud-init user-data file
# $2 - path to the cloud-init meta-data file
function _createCIUserdataISO() {
local _ci_userdata_path="$1"
local _ci_metadata_path="$2"
local _iso_path
_iso_path="$(mktemp -p "${WORKDIR}" --suffix .iso)"
mkisofs \
-input-charset "utf-8" \
-output "${_iso_path}" \
-volid "cidata" \
-joliet \
-rock \
-quiet \
-graft-points \
"${_ci_userdata_path}" \
"${_ci_metadata_path}"
echo "${_iso_path}"
}
# Verify image in EC2 on AWS
function verifyInAWS() {
$AWS_CMD ec2 describe-images \
--owners self \
--filters Name=name,Values="$AWS_SNAPSHOT_NAME" \
> "$WORKDIR/ami.json"
AMI_IMAGE_ID=$(jq -r '.Images[].ImageId' "$WORKDIR/ami.json")
AWS_SNAPSHOT_ID=$(jq -r '.Images[].BlockDeviceMappings[].Ebs.SnapshotId' "$WORKDIR/ami.json")
# Tag image and snapshot with "gitlab-ci-test" tag
$AWS_CMD ec2 create-tags \
--resources "${AWS_SNAPSHOT_ID}" "${AMI_IMAGE_ID}" \
--tags Key=gitlab-ci-test,Value=true
SHARE_OK=1
# Verify that the ec2 snapshot was shared
$AWS_CMD ec2 describe-snapshot-attribute --snapshot-id "$AWS_SNAPSHOT_ID" --attribute createVolumePermission > "$WORKDIR/snapshot-attributes.json"
SHARED_ID=$(jq -r '.CreateVolumePermissions[0].UserId' "$WORKDIR/snapshot-attributes.json")
if [ "$AWS_API_TEST_SHARE_ACCOUNT" != "$SHARED_ID" ]; then
SHARE_OK=0
fi
# Verify that the ec2 ami was shared
$AWS_CMD ec2 describe-image-attribute --image-id "$AMI_IMAGE_ID" --attribute launchPermission > "$WORKDIR/ami-attributes.json"
SHARED_ID=$(jq -r '.LaunchPermissions[0].UserId' "$WORKDIR/ami-attributes.json")
if [ "$AWS_API_TEST_SHARE_ACCOUNT" != "$SHARED_ID" ]; then
SHARE_OK=0
fi
if [ "$SHARE_OK" != 1 ]; then
echo "EC2 snapshot wasn't shared with the AWS_API_TEST_SHARE_ACCOUNT. 😢"
exit 1
fi
# Create key-pair
$AWS_CMD ec2 create-key-pair --key-name "key-for-$AMI_IMAGE_ID" --query 'KeyMaterial' --output text > keypair.pem
chmod 400 ./keypair.pem
# Create an instance based on the ami
$AWS_CMD ec2 run-instances --image-id "$AMI_IMAGE_ID" --count 1 --instance-type t2.micro --key-name "key-for-$AMI_IMAGE_ID" --tag-specifications 'ResourceType=instance,Tags=[{Key=gitlab-ci-test,Value=true}]' > "$WORKDIR/instances.json"
AWS_INSTANCE_ID=$(jq -r '.Instances[].InstanceId' "$WORKDIR/instances.json")
$AWS_CMD ec2 wait instance-running --instance-ids "$AWS_INSTANCE_ID"
$AWS_CMD ec2 describe-instances --instance-ids "$AWS_INSTANCE_ID" > "$WORKDIR/instances.json"
HOST=$(jq -r '.Reservations[].Instances[].PublicIpAddress' "$WORKDIR/instances.json")
echo "⏱ Waiting for AWS instance to respond to ssh"
_instanceWaitSSH "$HOST"
# Verify image
_ssh="ssh -oStrictHostKeyChecking=no -i ./keypair.pem $SSH_USER@$HOST"
_instanceCheck "$_ssh"
# Check access to user1 and user2
check_groups=$(ssh -oStrictHostKeyChecking=no -i "${WORKDIR}/usertest" "user1@$HOST" -t 'groups')
if [[ $check_groups =~ "wheel" ]]; then
echo "✔️ user1 has the group wheel"
else
echo 'user1 should have the group wheel 😢'
exit 1
fi
check_groups=$(ssh -oStrictHostKeyChecking=no -i "${WORKDIR}/usertest" "user2@$HOST" -t 'groups')
if [[ $check_groups =~ "wheel" ]]; then
echo 'user2 should not have group wheel 😢'
exit 1
else
echo "✔️ user2 does not have the group wheel"
fi
}
# verify edge commit content
function verifyEdgeCommit() {
filename="$1"
greenprint "Verifying contents of ${filename}"
# extract tarball and save file list to artifacts directroy
local COMMIT_DIR
COMMIT_DIR="${WORKDIR}/edge-commit"
mkdir -p "${COMMIT_DIR}"
tar xvf "${filename}" -C "${COMMIT_DIR}" > "${ARTIFACTS}/edge-commit-filelist.txt"
# Verify that the commit contains the ref we defined in the request
sudo dnf install -y ostree
local COMMIT_REF
COMMIT_REF=$(ostree refs --repo "${COMMIT_DIR}/repo")
if [[ "${COMMIT_REF}" != "${OSTREE_REF}" ]]; then
echo "Commit ref in archive does not match request 😠"
exit 1
fi
local TAR_COMMIT_ID
TAR_COMMIT_ID=$(ostree rev-parse --repo "${COMMIT_DIR}/repo" "${OSTREE_REF}")
API_COMMIT_ID_V2=$(curl \
--silent \
--show-error \
--cacert /etc/osbuild-composer/ca-crt.pem \
--key /etc/osbuild-composer/client-key.pem \
--cert /etc/osbuild-composer/client-crt.pem \
https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata | jq -r '.ostree_commit')
if [[ "${API_COMMIT_ID_V2}" != "${TAR_COMMIT_ID}" ]]; then
echo "Commit ID returned from API does not match Commit ID in archive 😠"
exit 1
fi
}
# Verify image blobs from s3
function verifyDisk() {
filename="$1"
greenprint "Verifying contents of ${filename}"
infofile="${filename}-info.json"
sudo /usr/libexec/osbuild-composer-test/image-info "${filename}" | tee "${infofile}" > /dev/null
# save image info to artifacts
cp -v "${infofile}" "${ARTIFACTS}/image-info.json"
# check compose request users in passwd
if ! jq .passwd "${infofile}" | grep -q "user1"; then
greenprint "❌ user1 not found in passwd file"
exit 1
fi
if ! jq .passwd "${infofile}" | grep -q "user2"; then
greenprint "❌ user2 not found in passwd file"
exit 1
fi
# check packages for postgresql
if ! jq .packages "${infofile}" | grep -q "postgresql"; then
greenprint "❌ postgresql not found in packages"
exit 1
fi
greenprint "✅ ${filename} image info verified"
}
# Verify VMDK image in VSphere
function verifyInVSphere() {
local _filename="$1"
greenprint "Verifying VMDK image: ${_filename}"
# Create SSH keys to use
local _vsphere_ssh_key="${WORKDIR}/vsphere_ssh_key"
ssh-keygen -t rsa-sha2-512 -f "${_vsphere_ssh_key}" -C "${SSH_USER}" -N ""
VSPHERE_VM_NAME="osbuild-composer-vm-${TEST_ID}"
# create cloud-init ISO with the configuration
local _ci_userdata_path
_ci_userdata_path="$(_createCIUserdata "${SSH_USER}" "${_vsphere_ssh_key}.pub")"
local _ci_metadata_path
_ci_metadata_path="$(_createCIMetadata "${VSPHERE_VM_NAME}")"
greenprint "💿 Creating cloud-init user-data ISO"
local _ci_iso_path
_ci_iso_path="$(_createCIUserdataISO "${_ci_userdata_path}" "${_ci_metadata_path}")"
VSPHERE_IMAGE_NAME="${VSPHERE_VM_NAME}.vmdk"
# import the built VMDK image to VSphere
# import.vmdk seems to be creating the provided directory and
# if one with this name exists, it appends "_<number>" to the name
greenprint "🚧 Converting the downloaded VMDK image to be streamOptimized"
qemu-img convert -O vmdk -o subformat=streamOptimized "${_filename}" "${WORKDIR}/${VSPHERE_IMAGE_NAME}"
greenprint "💿 ⬆️ Importing the converted VMDK image to VSphere"
$GOVC_CMD import.vmdk \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-ds="${GOVMOMI_DATASTORE}" \
"${WORKDIR}/${VSPHERE_IMAGE_NAME}" \
"${VSPHERE_VM_NAME}"
# create the VM, but don't start it
greenprint "🖥️ Creating VM in VSphere"
$GOVC_CMD vm.create \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-pool="${GOVMOMI_CLUSTER}"/Resources \
-ds="${GOVMOMI_DATASTORE}" \
-folder="${GOVMOMI_FOLDER}" \
-net="${GOVMOMI_NETWORK}" \
-net.adapter=vmxnet3 \
-m=4096 -c=2 -g=rhel8_64Guest -on=true -firmware=bios \
-disk="${VSPHERE_VM_NAME}/${VSPHERE_IMAGE_NAME}" \
-disk.controller=ide \
-on=false \
"${VSPHERE_VM_NAME}"
# upload ISO, create CDROM device and insert the ISO in it
greenprint "💿 ⬆️ Uploading the cloud-init user-data ISO to VSphere"
VSPHERE_CIDATA_ISO_PATH="${VSPHERE_VM_NAME}/cidata.iso"
$GOVC_CMD datastore.upload \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-ds="${GOVMOMI_DATASTORE}" \
"${_ci_iso_path}" \
"${VSPHERE_CIDATA_ISO_PATH}"
local _cdrom_device
greenprint "🖥️ + 💿 Adding a CD-ROM device to the VM"
_cdrom_device="$($GOVC_CMD device.cdrom.add \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-vm "${VSPHERE_VM_NAME}")"
greenprint "💿 Inserting the cloud-init ISO into the CD-ROM device"
$GOVC_CMD device.cdrom.insert \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-ds="${GOVMOMI_DATASTORE}" \
-vm "${VSPHERE_VM_NAME}" \
-device "${_cdrom_device}" \
"${VSPHERE_CIDATA_ISO_PATH}"
# start the VM
greenprint "🔌 Powering up the VSphere VM"
$GOVC_CMD vm.power \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
-on "${VSPHERE_VM_NAME}"
HOST=$($GOVC_CMD vm.ip \
-u "${GOVMOMI_USERNAME}:${GOVMOMI_PASSWORD}@${GOVMOMI_URL}" \
-k=true \
-dc="${GOVC_DATACENTER}" \
"${VSPHERE_VM_NAME}")
greenprint "⏱ Waiting for the VSphere VM to respond to ssh"
_instanceWaitSSH "${HOST}"
_ssh="ssh -oStrictHostKeyChecking=no -i ${_vsphere_ssh_key} $SSH_USER@$HOST"
_instanceCheck "${_ssh}"
greenprint "✅ Successfully verified VSphere image with cloud-init"
}
# Verify s3 blobs
function verifyInAWSS3() {
local S3_URL
S3_URL=$(echo "$UPLOAD_OPTIONS" | jq -r '.url')
greenprint "Verifying S3 object at ${S3_URL}"
# Tag the resource as a test file
local S3_FILENAME
S3_FILENAME=$(echo "${S3_URL}" | grep -oP '(?<=/)[^/]+(?=\?)')
# tag the object, also verifying that it exists in the bucket as expected
$AWS_CMD s3api put-object-tagging \
--bucket "${AWS_BUCKET}" \
--key "${S3_FILENAME}" \
--tagging '{"TagSet": [{ "Key": "gitlab-ci-test", "Value": "true" }]}'
greenprint "✅ Successfully tagged S3 object"
# Download the object using the Presigned URL and inspect
case ${IMAGE_TYPE} in
"$IMAGE_TYPE_EDGE_COMMIT")
curl "${S3_URL}" --output "${WORKDIR}/edge-commit.tar"
verifyEdgeCommit "${WORKDIR}/edge-commit.tar"
;;
"${IMAGE_TYPE_GUEST}")
curl "${S3_URL}" --output "${WORKDIR}/disk.qcow2"
verifyDisk "${WORKDIR}/disk.qcow2"
;;
"${IMAGE_TYPE_VSPHERE}")
curl "${S3_URL}" --output "${WORKDIR}/disk.vmdk"
verifyInVSphere "${WORKDIR}/disk.vmdk"
;;
*)
greenprint "No validation method for image type ${IMAGE_TYPE}"
;;
esac
greenprint "✅ Successfully verified S3 object"
}
# Verify image in Compute Engine on GCP
function verifyInGCP() {
# Authenticate
$GCP_CMD auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
# Extract and set the default project to be used for commands
GCP_PROJECT=$(jq -r '.project_id' "$GOOGLE_APPLICATION_CREDENTIALS")
$GCP_CMD config set project "$GCP_PROJECT"
# Add "gitlab-ci-test" label to the image
$GCP_CMD compute images add-labels "$GCP_IMAGE_NAME" --labels=gitlab-ci-test=true
# Verify that the image was shared
SHARE_OK=1
$GCP_CMD compute images get-iam-policy "$GCP_IMAGE_NAME" > "$WORKDIR/image-iam-policy.json"
SHARED_ACCOUNT=$(jq -r '.bindings[0].members[0]' "$WORKDIR/image-iam-policy.json")
SHARED_ROLE=$(jq -r '.bindings[0].role' "$WORKDIR/image-iam-policy.json")
if [ "$SHARED_ACCOUNT" != "$GCP_API_TEST_SHARE_ACCOUNT" ] || [ "$SHARED_ROLE" != "roles/compute.imageUser" ]; then
SHARE_OK=0
fi
if [ "$SHARE_OK" != 1 ]; then
echo "GCP image wasn't shared with the GCP_API_TEST_SHARE_ACCOUNT. 😢"
exit 1
fi
# Verify that the image boots and have customizations applied
# Create SSH keys to use
GCP_SSH_KEY="$WORKDIR/id_google_compute_engine"
ssh-keygen -t rsa-sha2-512 -f "$GCP_SSH_KEY" -C "$SSH_USER" -N ""
GCP_SSH_METADATA_FILE="$WORKDIR/gcp-ssh-keys-metadata"
echo "${SSH_USER}:$(cat "$GCP_SSH_KEY".pub)" > "$GCP_SSH_METADATA_FILE"
# create the instance
# resource ID can have max 62 characters, the $GCP_TEST_ID_HASH contains 56 characters
GCP_INSTANCE_NAME="vm-$GCP_TEST_ID_HASH"
# Randomize the used GCP zone to prevent hitting "exhausted resources" error on each test re-run
# disable Shellcheck error as the suggested alternatives are less readable for this use case
# shellcheck disable=SC2207
local GCP_ZONES=($($GCP_CMD compute zones list --filter="region=$GCP_REGION" | jq '.[] | select(.status == "UP") | .name' | tr -d '"' | tr '\n' ' '))
GCP_ZONE=${GCP_ZONES[$((RANDOM % ${#GCP_ZONES[@]}))]}
$GCP_CMD compute instances create "$GCP_INSTANCE_NAME" \
--zone="$GCP_ZONE" \
--image-project="$GCP_PROJECT" \
--image="$GCP_IMAGE_NAME" \
--labels=gitlab-ci-test=true \
--metadata-from-file=ssh-keys="$GCP_SSH_METADATA_FILE"
HOST=$($GCP_CMD compute instances describe "$GCP_INSTANCE_NAME" --zone="$GCP_ZONE" --format='get(networkInterfaces[0].accessConfigs[0].natIP)')
echo "⏱ Waiting for GCP instance to respond to ssh"
_instanceWaitSSH "$HOST"
# Verify image
_ssh="ssh -oStrictHostKeyChecking=no -i $GCP_SSH_KEY $SSH_USER@$HOST"
_instanceCheck "$_ssh"
}
# Verify image in Azure
function verifyInAzure() {
set +x
$AZURE_CMD login --service-principal --username "${V2_AZURE_CLIENT_ID}" --password "${V2_AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}"
set -x
# verify that the image exists
$AZURE_CMD image show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_IMAGE_NAME}"
# Verify that the image boots and have customizations applied
# Create SSH keys to use
AZURE_SSH_KEY="$WORKDIR/id_azure"
ssh-keygen -t rsa-sha2-512 -f "$AZURE_SSH_KEY" -C "$SSH_USER" -N ""
# Create network resources with predictable names
$AZURE_CMD network nsg create --resource-group "$AZURE_RESOURCE_GROUP" --name "nsg-$TEST_ID" --location "$AZURE_LOCATION"
$AZURE_CMD network nsg rule create --resource-group "$AZURE_RESOURCE_GROUP" \
--nsg-name "nsg-$TEST_ID" \
--name SSH \
--priority 1001 \
--access Allow \
--protocol Tcp \
--destination-address-prefixes '*' \
--destination-port-ranges 22 \
--source-port-ranges '*' \
--source-address-prefixes '*'
$AZURE_CMD network vnet create --resource-group "$AZURE_RESOURCE_GROUP" --name "vnet-$TEST_ID" --subnet-name "snet-$TEST_ID" --location "$AZURE_LOCATION"
$AZURE_CMD network public-ip create --resource-group "$AZURE_RESOURCE_GROUP" --name "ip-$TEST_ID" --location "$AZURE_LOCATION"
$AZURE_CMD network nic create --resource-group "$AZURE_RESOURCE_GROUP" \
--name "iface-$TEST_ID" \
--subnet "snet-$TEST_ID" \
--vnet-name "vnet-$TEST_ID" \
--network-security-group "nsg-$TEST_ID" \
--public-ip-address "ip-$TEST_ID" \
--location "$AZURE_LOCATION"
# create the instance
AZURE_INSTANCE_NAME="vm-$TEST_ID"
$AZURE_CMD vm create --name "$AZURE_INSTANCE_NAME" \
--resource-group "$AZURE_RESOURCE_GROUP" \
--image "$AZURE_IMAGE_NAME" \
--size "Standard_B1s" \
--admin-username "$SSH_USER" \
--ssh-key-values "$AZURE_SSH_KEY.pub" \
--authentication-type "ssh" \
--location "$AZURE_LOCATION" \
--nics "iface-$TEST_ID" \
--os-disk-name "disk-$TEST_ID"
$AZURE_CMD vm show --name "$AZURE_INSTANCE_NAME" --resource-group "$AZURE_RESOURCE_GROUP" --show-details > "$WORKDIR/vm_details.json"
HOST=$(jq -r '.publicIps' "$WORKDIR/vm_details.json")
echo "⏱ Waiting for Azure instance to respond to ssh"
_instanceWaitSSH "$HOST"
# Verify image
_ssh="ssh -oStrictHostKeyChecking=no -i $AZURE_SSH_KEY $SSH_USER@$HOST"
_instanceCheck "$_ssh"
}
case $CLOUD_PROVIDER in
"$CLOUD_PROVIDER_AWS")
verifyInAWS
;;
"$CLOUD_PROVIDER_AWS_S3")
verifyInAWSS3
;;
"$CLOUD_PROVIDER_GCP")
verifyInGCP
;;
"$CLOUD_PROVIDER_AZURE")
verifyInAzure
;;
esac
# Verify selected package (postgresql) is included in package list
function verifyPackageList() {
# Save build metadata to artifacts directory for troubleshooting
curl --silent \
--show-error \
--cacert /etc/osbuild-composer/ca-crt.pem \
--key /etc/osbuild-composer/client-key.pem \
--cert /etc/osbuild-composer/client-crt.pem \
https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json"
local PACKAGENAMES
PACKAGENAMES=$(jq -rM '.packages[].name' "${ARTIFACTS}/metadata.json")
if ! grep -q postgresql <<< "${PACKAGENAMES}"; then
echo "'postgresql' not found in compose package list 😠"
exit 1
fi
}
verifyPackageList
#
# Verify oauth2
#
cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
[koji]
enable_tls = false
enable_mtls = false
enable_jwt = true
jwt_keys_urls = ["https://localhost:8080/certs"]
jwt_ca_file = "/etc/osbuild-composer/ca-crt.pem"
jwt_acl_file = ""
jwt_tenant_provider_fields = ["rh-org-id"]
[worker]
pg_host = "localhost"
pg_port = "5432"
enable_artifacts = false
pg_database = "osbuildcomposer"
pg_user = "postgres"
pg_password = "foobar"
pg_ssl_mode = "disable"
enable_tls = true
enable_mtls = false
enable_jwt = true
jwt_keys_urls = ["https://localhost:8080/certs"]
jwt_ca_file = "/etc/osbuild-composer/ca-crt.pem"
jwt_tenant_provider_fields = ["rh-org-id"]
EOF
cat <<EOF | sudo tee "/etc/osbuild-worker/token"
offlineToken
EOF
cat <<EOF | sudo tee "/etc/osbuild-worker/osbuild-worker.toml"
[authentication]
oauth_url = http://localhost:8081/token
client_id = "rhsm-api"
offline_token = "/etc/osbuild-worker/token"
EOF
# Spin up an https instance for the composer-api and worker-api; the auth handler needs to hit an ssl `/certs` endpoint
sudo /usr/libexec/osbuild-composer-test/osbuild-mock-openid-provider -rsaPubPem /etc/osbuild-composer/client-crt.pem -rsaPem /etc/osbuild-composer/client-key.pem -cert /etc/osbuild-composer/composer-crt.pem -key /etc/osbuild-composer/composer-key.pem &
KILL_PIDS+=("$!")
# Spin up an http instance for the worker client to bypass the need to specify an extra CA
sudo /usr/libexec/osbuild-composer-test/osbuild-mock-openid-provider -a localhost:8081 -rsaPubPem /etc/osbuild-composer/client-crt.pem -rsaPem /etc/osbuild-composer/client-key.pem &
KILL_PIDS+=("$!")
sudo systemctl restart osbuild-composer
until curl --output /dev/null --silent --fail localhost:8081/token; do
sleep 0.5
done
TOKEN="$(curl localhost:8081/token | jq -r .access_token)"
[ "$(curl \
--silent \
--output /dev/null \
--write-out '%{http_code}' \
--header "Authorization: Bearer $TOKEN" \
http://localhost:443/api/image-builder-composer/v2/openapi)" = "200" ]
# /openapi doesn't need auth
[ "$(curl \
--silent \
--output /dev/null \
--write-out '%{http_code}' \
--header "Authorization: Bearer badtoken" \
http://localhost:443/api/image-builder-composer/v2/openapi)" = "200" ]
# /composes/$ID does doesn't need auth
[ "$(curl \
--silent \
--output /dev/null \
--write-out '%{http_code}' \
--header "Authorization: Bearer badtoken" \
http://localhost:443/api/image-builder-composer/v2/composes/"$COMPOSE_ID")" = "401" ]
sudo systemctl start osbuild-remote-worker@localhost:8700.service
sudo systemctl is-active --quiet osbuild-remote-worker@localhost:8700.service
exit 0
Reference in a new issue View git blame Copy permalink