From 16e27dad1b801ac44e8d89082ac828a68031cc8d Mon Sep 17 00:00:00 2001 From: Xiaofeng Wang Date: Thu, 10 Aug 2023 23:31:57 +0800 Subject: [PATCH] test: update edge-ami test to support aarch64 --- test/cases/ostree-ami-image.sh | 260 ++++++++++++++-------------- test/data/ansible/check_ostree.yaml | 11 +- 2 files changed, 141 insertions(+), 130 deletions(-) diff --git a/test/cases/ostree-ami-image.sh b/test/cases/ostree-ami-image.sh index b6b27101b..da8db1254 100755 --- a/test/cases/ostree-ami-image.sh +++ b/test/cases/ostree-ami-image.sh @@ -4,10 +4,9 @@ set -euo pipefail # Provision the software under test. /usr/libexec/osbuild-composer-test/provision.sh none -source /usr/libexec/tests/osbuild-composer/shared_lib.sh - # Get OS data. -source /etc/os-release +source /usr/libexec/osbuild-composer-test/set-env-variables.sh +source /usr/libexec/tests/osbuild-composer/shared_lib.sh ARCH=$(uname -m) # Start libvirtd and test it. @@ -63,7 +62,7 @@ CONTAINER_FILENAME=container.tar AMI_IMAGE_TYPE=edge-ami AMI_IMAGE_FILENAME=image.raw OSTREE_OSNAME=redhat -BUCKET_NAME="test-bucket-${TEST_UUID}" +BUCKET_NAME="composer-ci-${TEST_UUID}" BUCKET_URL="s3://${BUCKET_NAME}" OBJECT_URL="http://${BUCKET_NAME}.s3.${AWS_DEFAULT_REGION}.amazonaws.com" @@ -195,7 +194,7 @@ clean_up () { # Remove prod repo sudo rm -rf "$PROD_REPO" - + # Remomve tmp dir. sudo rm -rf "$TEMPDIR" @@ -204,40 +203,18 @@ clean_up () { # Deregister edge AMI image aws ec2 deregister-image --image-id "${AMI_ID}" - + # Remove snapshot aws ec2 delete-snapshot --snapshot-id "${SNAPSHOT_ID}" - + # Delete Key Pair aws ec2 delete-key-pair --key-name "${AMI_KEY_NAME}" - + # Terminate running instance aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}" - - # Clean up local folder - sudo rm -rf "${CONTAINERS_FILE}" "${IMPORT_SNAPSHOT_INFO}" "${IMPORT_SNAPSHOT_TASK}" "${AMI_FILENAME}" "${REGISTERED_AMI_ID}" "${INSTANCE_OUT_INFO}" "${MARKET_OPTIONS}" "${IGW_OUTPUT}" "${RT_OUTPUT}" "${SG_OUTPUT}" "${SUBNET_OUTPUT}" "${VPC_OUTPUT}" - + # Remove bucket content and bucket itself quietly aws s3 rb "${BUCKET_URL}" --force > /dev/null - - # Remove subnet - aws ec2 delete-subnet --subnet-id "${SUBN_ID}" - - # Remove Security Groups - aws ec2 delete-security-group --group-id "${SEC_GROUP_ID}" - - # Remove Route Table - aws ec2 delete-route-table --route-table-id "${RT_ID}" - - # Detach Internet gateway from VPC - aws ec2 detach-internet-gateway --internet-gateway-id "${IGW_ID}" --vpc-id "${VPC_ID}" - - # Remove Internet gateway - aws ec2 delete-internet-gateway --internet-gateway-id "${IGW_ID}" - - # Delete VPC - aws ec2 delete-vpc --vpc-id "${VPC_ID}" - } # Test result checking @@ -252,42 +229,71 @@ check_result () { fi } -# AWS EC2 AMI tagging function -tag_ec2_ami () { - ami_id=$1 - - greenprint "Add custom tags to EC2 ami" - aws ec2 create-tags \ - --resources "${ami_id}" --tags Key=Project,Value=rhel-edge - aws ec2 create-tags \ - --resources "${ami_id}" --tags Key=ImageType,Value=edge-ami - aws ec2 create-tags \ - --resources "${ami_id}" --tags Key=BuildBy,Value=osbuild-composer +# Configure AWS EC2 network +add_vpc () { + # Network setup + greenprint "VPC Network setup." -} + # Create VPC + VPC_OUTPUT=vpc_output.json + aws ec2 create-vpc --tag-specification 'ResourceType=vpc,Tags=[{Key=Name,Value=kite-ci}]' --cidr-block 172.32.0.0/16 --region="${AWS_DEFAULT_REGION}" | tee "${VPC_OUTPUT}" > /dev/null + VPC_ID=$(jq -r '.Vpc.VpcId' < "${VPC_OUTPUT}") + rm -f "$VPC_OUTPUT" -# AWS EC2 instance tagging function -tag_ec2_instance () { - instance_id=$1 + # Create VPC Internet Gateway + IGW_OUTPUT=igw_output.json + aws ec2 create-internet-gateway --tag-specifications 'ResourceType=internet-gateway,Tags=[{Key=Name,Value=kite-ci}]' | tee "${IGW_OUTPUT}" > /dev/null + IGW_ID=$(jq -r '.InternetGateway.InternetGatewayId' < "${IGW_OUTPUT}") + rm -f "$IGW_OUTPUT" - aws ec2 create-tags \ - --resources "${instance_id}" --tags Key=Project,Value=rhel-edge -} + # Attach internet gateway + aws ec2 attach-internet-gateway --vpc-id "${VPC_ID}" --internet-gateway-id "${IGW_ID}" -tag_describe_resource () { - res_id=$1 - aws ec2 describe-tags \ - --filters "Name=resource-id,Values=${res_id}" + # Add default route in route table for all vpc subnets + # Create route table + RT_OUTPUT=route_table_out.json + aws ec2 create-route-table --vpc-id "${VPC_ID}" --tag-specifications 'ResourceType=route-table,Tags=[{Key=Name,Value=kite-ci}]' | tee "${RT_OUTPUT}" > /dev/null + RT_ID=$(jq -r '.RouteTable.RouteTableId' < "${RT_OUTPUT}") + aws ec2 create-route --route-table-id "${RT_ID}" --destination-cidr-block 0.0.0.0/0 --gateway-id "${IGW_ID}" + rm -f "$RT_OUTPUT" + + ALL_ZONES=( "us-east-1a" "us-east-1b" "us-east-1c" "us-east-1d" "us-east-1e" "us-east-1f" ) + LENGTH=${#ALL_ZONES[@]} + for (( j=0; j /dev/null << EOF [customizations.kernel] name = "kernel-rt" EOF +fi greenprint "๐Ÿ“„ container blueprint" cat "$BLUEPRINT_FILE" @@ -468,7 +479,7 @@ EOF sudo chmod +r "${IGNITION_CONFIG_SAMPLE_PATH}" "${IGNITION_CONFIG_PATH}" # Start AWS cli installation -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" unzip awscliv2.zip > /dev/null sudo ./aws/install --update aws --version @@ -536,6 +547,8 @@ sudo chmod +r "${AMI_FILENAME}" # Upload ami to AWS S3 bucket greenprint "๐Ÿ“‚ Upload raw ami to S3 bucket" aws s3 cp --quiet "${AMI_FILENAME}" "${BUCKET_URL}/" --acl public-read + +# Clean compose and blueprints greenprint "๐Ÿงน Clean up edge-ami compose and blueprint" sudo composer-cli compose delete "${COMPOSE_ID}" > /dev/null sudo composer-cli blueprints delete ami > /dev/null @@ -554,20 +567,21 @@ EOF # Import the image as an EBS snapshot into EC2 IMPORT_SNAPSHOT_INFO=output_snapshot_info.json aws ec2 import-snapshot --description "RHEL edge ami snapshot" --disk-container file://"${CONTAINERS_FILE}" > "${IMPORT_SNAPSHOT_INFO}" -IMPORT_TASK_ID=$(cat "${IMPORT_SNAPSHOT_INFO}" | jq -r '.ImportTaskId') +IMPORT_TASK_ID=$(jq -r '.ImportTaskId' < "${IMPORT_SNAPSHOT_INFO}") +rm -f "$IMPORT_SNAPSHOT_INFO" "$CONTAINERS_FILE" # Monitor snapshot status greenprint "Check import status of the snapshot" IMPORT_SNAPSHOT_TASK=output_snapshot_task.json while true; do aws ec2 describe-import-snapshot-tasks --import-task-ids "${IMPORT_TASK_ID}" | tee "${IMPORT_SNAPSHOT_TASK}" > /dev/null - IMPORT_STATUS=$(cat "${IMPORT_SNAPSHOT_TASK}" | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status') + IMPORT_STATUS=$(jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status' < "${IMPORT_SNAPSHOT_TASK}") # Has the snapshot finished? if [[ $IMPORT_STATUS != active ]]; then break fi - + # Wait 5 seconds and try again. sleep 5 done @@ -579,49 +593,52 @@ else greenprint "Snapshot imported successfully." fi +SNAPSHOT_ID=$(jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId' < "${IMPORT_SNAPSHOT_TASK}") +aws ec2 create-tags --resources "${SNAPSHOT_ID}" --tags Key=Name,Value=composer-ci +rm -f "$IMPORT_SNAPSHOT_TASK" + # Import keypair greenprint "Share ssh public key with AWS" AMI_KEY_NAME="edge-ami-key-${TEST_UUID}" # Clean previous configured keypair -aws ec2 delete-key-pair --key-name "${AMI_KEY_NAME}" -aws ec2 import-key-pair --key-name "${AMI_KEY_NAME}" --public-key-material fileb://"${SSH_KEY}".pub +aws ec2 import-key-pair --key-name "${AMI_KEY_NAME}" --public-key-material fileb://"${SSH_KEY}".pub --tag-specification 'ResourceType=key-pair,Tags=[{Key=Name,Value=composer-ci}]' -# Network setup -greenprint "VPC Network setup." +# Create ec2 network +EXISTED_VPC=$(aws ec2 describe-vpcs --filters="Name=tag:Name,Values=kite-ci" --output json --query "Vpcs") +if [[ "$EXISTED_VPC" == "[]" ]]; then + add_vpc +fi -# Create VPC -VPC_OUTPUT=vpc_output.json -aws ec2 create-vpc --cidr-block 172.32.0.0/16 --region="${AWS_DEFAULT_REGION}" | tee "${VPC_OUTPUT}" > /dev/null -VPC_ID=$(cat "${VPC_OUTPUT}" | jq -r '.Vpc.VpcId') +################################################################## +## +## Install and test edge EC2 instance with edge-ami image +## +################################################################## +# Create AMI image from EBS snapshot +greenprint "Register AMI, create image from snapshot." +REGISTERED_AMI_NAME="edge_ami-${TEST_UUID}" +REGISTERED_AMI_ID=output_ami_id.json -# Create VPC Internet Gateway -IGW_OUTPUT=igw_output.json -aws ec2 create-internet-gateway | tee "${IGW_OUTPUT}" > /dev/null -IGW_ID=$(cat "${IGW_OUTPUT}" | jq -r '.InternetGateway.InternetGatewayId') +if [[ "${ARCH}" == x86_64 ]]; then + IMG_ARCH="${ARCH}" +elif [[ "${ARCH}" == aarch64 ]]; then + IMG_ARCH=arm64 +fi -# Attach internet gateway -aws ec2 attach-internet-gateway --vpc-id "${VPC_ID}" --internet-gateway-id "${IGW_ID}" +aws ec2 register-image \ + --name "${REGISTERED_AMI_NAME}" \ + --root-device-name /dev/xvda \ + --architecture "${IMG_ARCH}" \ + --ena-support \ + --sriov-net-support simple \ + --virtualization-type hvm \ + --block-device-mappings DeviceName=/dev/xvda,Ebs=\{SnapshotId="${SNAPSHOT_ID}"\} DeviceName=/dev/xvdf,Ebs=\{VolumeSize=10\} \ + --boot-mode uefi-preferred \ + --output json > "${REGISTERED_AMI_ID}" -# Create Subnet for VPC -SUBNET_OUTPUT=sub_net_output.json -aws ec2 create-subnet --vpc-id "${VPC_ID}" --cidr-block 172.32.32.0/24 | tee "${SUBNET_OUTPUT}" -SUBN_ID=$(cat "${SUBNET_OUTPUT}" | jq -r '.Subnet.SubnetId') - -# Add default route in route table for all vpc subnets -# Create route table -RT_OUTPUT=route_table_out.json -aws ec2 create-route-table --vpc-id "${VPC_ID}" | tee "${RT_OUTPUT}" > /dev/null -RT_ID=$(cat "${RT_OUTPUT}" | jq -r '.RouteTable.RouteTableId') -aws ec2 create-route --route-table-id "${RT_ID}" --destination-cidr-block 0.0.0.0/0 --gateway-id "${IGW_ID}" -# Associate route table to subnet -aws ec2 associate-route-table --route-table-id "${RT_ID}" --subnet-id "${SUBN_ID}" - -# Security Group setup -SG_OUTPUT=sec_group.json -aws ec2 create-security-group --group-name mysecuritygroup --description "edge-ami security group" --vpc-id "${VPC_ID}" | tee "${SG_OUTPUT}" -SEC_GROUP_ID=$(cat "${SG_OUTPUT}" | jq -r '.GroupId') -# Allow inbound ssh connections -aws ec2 authorize-security-group-ingress --group-id "${SEC_GROUP_ID}" --protocol tcp --port 22 --cidr 0.0.0.0/0 +AMI_ID=$(jq -r '.ImageId' < "${REGISTERED_AMI_ID}") +aws ec2 create-tags --resources "${AMI_ID}" --tags Key=Name,Value=composer-ci +rm -f "$REGISTERED_AMI_ID" # Create instance market options MARKET_OPTIONS=spot-options.json @@ -636,47 +653,37 @@ tee "${MARKET_OPTIONS}" > /dev/null << EOF } EOF -################################################################## -## -## Install and test edge EC2 instance with edge-ami image -## -################################################################## -# Create AMI image from EBS snapshot -greenprint "Register AMI, create image from snapshot." -REGISTERED_AMI_NAME="edge_ami-${TEST_UUID}" -REGISTERED_AMI_ID=output_ami_id.json -SNAPSHOT_ID=$(cat "${IMPORT_SNAPSHOT_TASK}" | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId') -aws ec2 register-image \ - --name "${REGISTERED_AMI_NAME}" \ - --root-device-name /dev/xvda \ - --architecture "${ARCH}" \ - --ena-support \ - --sriov-net-support simple \ - --virtualization-type hvm \ - --block-device-mappings DeviceName=/dev/xvda,Ebs=\{SnapshotId="${SNAPSHOT_ID}"\} DeviceName=/dev/xvdf,Ebs=\{VolumeSize=10\} \ - --boot-mode uefi-preferred \ - --output json > "${REGISTERED_AMI_ID}" - -AMI_ID=$(cat "${REGISTERED_AMI_ID}" | jq -r '.ImageId') -tag_ec2_ami "${AMI_ID}" -tag_describe_resource "${AMI_ID}" - # Launch Instance greenprint "๐Ÿ’ป Launch instance from AMI" -for LOOP_COUNTER in $(seq 0 9); do +for _ in $(seq 0 9); do + RESULTS=0 INSTANCE_OUT_INFO=instance_output_info.json INSTANCE_TYPE=$(get_instance_type "${ARCH}") + + ZONE_LIST=$(aws ec2 describe-instance-type-offerings --location-type availability-zone --filters="Name=instance-type,Values=${INSTANCE_TYPE}" --query "InstanceTypeOfferings") + if [[ "$ZONE_LIST" == "[]" ]]; then + greenprint "No available $INSTANCE_TYPE in this region" + break + else + ZONE_NAME=$(echo "$ZONE_LIST" | jq -r ".[0].Location") + fi + SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=tag:Name,Values=kite-ci" "Name=availabilityZone,Values=${ZONE_NAME}" | jq -r ".Subnets[0].SubnetId") + SEC_GROUP_ID=$(aws ec2 describe-security-groups --filters="Name=tag:Name,Values=kite-ci" --output json | jq -r ".SecurityGroups[0].GroupId") + aws ec2 run-instances \ --image-id "${AMI_ID}" \ --count 1 \ --instance-type "${INSTANCE_TYPE}" \ + --placement AvailabilityZone="${ZONE_NAME}" \ + --tag-specification 'ResourceType=instance,Tags=[{Key=Name,Value=composer-ci}]' \ --instance-market-options file://"${MARKET_OPTIONS}" \ --key-name "${AMI_KEY_NAME}" \ --security-group-ids "${SEC_GROUP_ID}" \ - --subnet-id "${SUBN_ID}" \ + --subnet-id "${SUBNET_ID}" \ --associate-public-ip-address > "${INSTANCE_OUT_INFO}" 2>&1 || : if ! grep -iqE 'unsupported|InsufficientInstanceCapacity' "${INSTANCE_OUT_INFO}"; then echo "Instance type supported!" + RESULTS=1 break fi sleep 30 @@ -688,13 +695,12 @@ sleep 5 # get instance public ip INSTANCE_ID=$(jq -r '.Instances[].InstanceId' "${INSTANCE_OUT_INFO}") -tag_ec2_instance "${INSTANCE_ID}" -tag_describe_resource "${INSTANCE_ID}" PUBLIC_GUEST_ADDRESS=$(aws ec2 describe-instances --instance-ids "${INSTANCE_ID}" --query 'Reservations[*].Instances[*].PublicIpAddress' --output text) +rm -f "$MARKET_OPTIONS" "$INSTANCE_OUT_INFO" # Check for ssh ready to go. greenprint "๐Ÿ›ƒ Checking for SSH is ready to go" -for LOOP_COUNTER in $(seq 0 30); do +for _ in $(seq 0 30); do RESULTS=$(wait_for_ssh_up "${PUBLIC_GUEST_ADDRESS}") if [[ $RESULTS == 1 ]]; then echo "SSH is ready now! ๐Ÿฅณ" @@ -727,7 +733,6 @@ EOF # Test IoT/Edge OS sudo ansible-playbook -v -i "${TEMPDIR}"/inventory \ -e image_type="${OSTREE_OSNAME}" \ - -e skip_rollback_test="true" \ -e ignition="true" \ -e edge_type=edge-ami-image \ -e ostree_commit="${INSTALL_HASH}" \ @@ -768,10 +773,15 @@ description = "Administrator account" password = "\$6\$GRmb7S0p8vsYmXzH\$o0E020S.9JQGaHkszoog4ha4AQVs3sk8q0DvLjSMxoxHBKnB2FBXGQ/OkwZQfW/76ktHd0NX5nls2LPxPuUdl." home = "/home/admin/" groups = ["wheel"] +EOF +# Red Hat does not provide realtime kernel package for ARM +if [[ "${ARCH}" != aarch64 ]]; then + tee -a "$BLUEPRINT_FILE" > /dev/null << EOF [customizations.kernel] name = "kernel-rt" EOF +fi greenprint "๐Ÿ“„ upgrade blueprint" cat "$BLUEPRINT_FILE" @@ -849,8 +859,7 @@ sleep 10 # Check for ssh ready to go. greenprint "๐Ÿ›ƒ Checking for SSH is ready to go" -# shellcheck disable=SC2034 # Unused variables left for readability -for LOOP_COUNTER in $(seq 0 30); do +for _ in $(seq 0 30); do RESULTS=$(wait_for_ssh_up "${PUBLIC_GUEST_ADDRESS}") if [[ $RESULTS == 1 ]]; then echo "SSH is ready now! ๐Ÿฅณ" @@ -891,4 +900,3 @@ check_result clean_up exit 0 - diff --git a/test/data/ansible/check_ostree.yaml b/test/data/ansible/check_ostree.yaml index cfac7aeb6..f7f5ab344 100644 --- a/test/data/ansible/check_ostree.yaml +++ b/test/data/ansible/check_ostree.yaml @@ -99,7 +99,7 @@ - name: wait for FDO onboarding block: - wait_for: - path: "/etc/device-credentials" + path: "/etc/device-credentials" delay: 10 timeout: 600 state: present @@ -747,8 +747,9 @@ - name: check fdo-client-linuxapp logs command: journalctl -u fdo-client-linuxapp register: result_fdo_client_linuxapp_journalctl + when: fdo_credential == "true" - - debug: + - debug: var: result_fdo_client_linuxapp_journalctl # case: check rollback function if boot error found @@ -843,8 +844,9 @@ - name: check fdo-client-linuxapp logs command: journalctl -u fdo-client-linuxapp register: result_fdo_client_linuxapp_journalctl + when: fdo_credential == "true" - - debug: + - debug: var: result_fdo_client_linuxapp_journalctl - name: wait for FDO re-encryption @@ -866,8 +868,9 @@ - name: check fdo-client-linuxapp logs command: journalctl -u fdo-client-linuxapp register: result_fdo_client_linuxapp_journalctl + when: fdo_credential == "true" - - debug: + - debug: var: result_fdo_client_linuxapp_journalctl # case: check greenboot* services log again