test: update edge-ami test to support aarch64

This commit is contained in:
Xiaofeng Wang 2023-08-10 23:31:57 +08:00 committed by Achilleas Koutsou
parent 207e148303
commit 16e27dad1b
2 changed files with 141 additions and 130 deletions

View file

@ -4,10 +4,9 @@ set -euo pipefail
# Provision the software under test.
/usr/libexec/osbuild-composer-test/provision.sh none
source /usr/libexec/tests/osbuild-composer/shared_lib.sh
# Get OS data.
source /etc/os-release
source /usr/libexec/osbuild-composer-test/set-env-variables.sh
source /usr/libexec/tests/osbuild-composer/shared_lib.sh
ARCH=$(uname -m)
# Start libvirtd and test it.
@ -63,7 +62,7 @@ CONTAINER_FILENAME=container.tar
AMI_IMAGE_TYPE=edge-ami
AMI_IMAGE_FILENAME=image.raw
OSTREE_OSNAME=redhat
BUCKET_NAME="test-bucket-${TEST_UUID}"
BUCKET_NAME="composer-ci-${TEST_UUID}"
BUCKET_URL="s3://${BUCKET_NAME}"
OBJECT_URL="http://${BUCKET_NAME}.s3.${AWS_DEFAULT_REGION}.amazonaws.com"
@ -195,7 +194,7 @@ clean_up () {
# Remove prod repo
sudo rm -rf "$PROD_REPO"
# Remomve tmp dir.
sudo rm -rf "$TEMPDIR"
@ -204,40 +203,18 @@ clean_up () {
# Deregister edge AMI image
aws ec2 deregister-image --image-id "${AMI_ID}"
# Remove snapshot
aws ec2 delete-snapshot --snapshot-id "${SNAPSHOT_ID}"
# Delete Key Pair
aws ec2 delete-key-pair --key-name "${AMI_KEY_NAME}"
# Terminate running instance
aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}"
# Clean up local folder
sudo rm -rf "${CONTAINERS_FILE}" "${IMPORT_SNAPSHOT_INFO}" "${IMPORT_SNAPSHOT_TASK}" "${AMI_FILENAME}" "${REGISTERED_AMI_ID}" "${INSTANCE_OUT_INFO}" "${MARKET_OPTIONS}" "${IGW_OUTPUT}" "${RT_OUTPUT}" "${SG_OUTPUT}" "${SUBNET_OUTPUT}" "${VPC_OUTPUT}"
# Remove bucket content and bucket itself quietly
aws s3 rb "${BUCKET_URL}" --force > /dev/null
# Remove subnet
aws ec2 delete-subnet --subnet-id "${SUBN_ID}"
# Remove Security Groups
aws ec2 delete-security-group --group-id "${SEC_GROUP_ID}"
# Remove Route Table
aws ec2 delete-route-table --route-table-id "${RT_ID}"
# Detach Internet gateway from VPC
aws ec2 detach-internet-gateway --internet-gateway-id "${IGW_ID}" --vpc-id "${VPC_ID}"
# Remove Internet gateway
aws ec2 delete-internet-gateway --internet-gateway-id "${IGW_ID}"
# Delete VPC
aws ec2 delete-vpc --vpc-id "${VPC_ID}"
}
# Test result checking
@ -252,42 +229,71 @@ check_result () {
fi
}
# AWS EC2 AMI tagging function
tag_ec2_ami () {
ami_id=$1
greenprint "Add custom tags to EC2 ami"
aws ec2 create-tags \
--resources "${ami_id}" --tags Key=Project,Value=rhel-edge
aws ec2 create-tags \
--resources "${ami_id}" --tags Key=ImageType,Value=edge-ami
aws ec2 create-tags \
--resources "${ami_id}" --tags Key=BuildBy,Value=osbuild-composer
# Configure AWS EC2 network
add_vpc () {
# Network setup
greenprint "VPC Network setup."
}
# Create VPC
VPC_OUTPUT=vpc_output.json
aws ec2 create-vpc --tag-specification 'ResourceType=vpc,Tags=[{Key=Name,Value=kite-ci}]' --cidr-block 172.32.0.0/16 --region="${AWS_DEFAULT_REGION}" | tee "${VPC_OUTPUT}" > /dev/null
VPC_ID=$(jq -r '.Vpc.VpcId' < "${VPC_OUTPUT}")
rm -f "$VPC_OUTPUT"
# AWS EC2 instance tagging function
tag_ec2_instance () {
instance_id=$1
# Create VPC Internet Gateway
IGW_OUTPUT=igw_output.json
aws ec2 create-internet-gateway --tag-specifications 'ResourceType=internet-gateway,Tags=[{Key=Name,Value=kite-ci}]' | tee "${IGW_OUTPUT}" > /dev/null
IGW_ID=$(jq -r '.InternetGateway.InternetGatewayId' < "${IGW_OUTPUT}")
rm -f "$IGW_OUTPUT"
aws ec2 create-tags \
--resources "${instance_id}" --tags Key=Project,Value=rhel-edge
}
# Attach internet gateway
aws ec2 attach-internet-gateway --vpc-id "${VPC_ID}" --internet-gateway-id "${IGW_ID}"
tag_describe_resource () {
res_id=$1
aws ec2 describe-tags \
--filters "Name=resource-id,Values=${res_id}"
# Add default route in route table for all vpc subnets
# Create route table
RT_OUTPUT=route_table_out.json
aws ec2 create-route-table --vpc-id "${VPC_ID}" --tag-specifications 'ResourceType=route-table,Tags=[{Key=Name,Value=kite-ci}]' | tee "${RT_OUTPUT}" > /dev/null
RT_ID=$(jq -r '.RouteTable.RouteTableId' < "${RT_OUTPUT}")
aws ec2 create-route --route-table-id "${RT_ID}" --destination-cidr-block 0.0.0.0/0 --gateway-id "${IGW_ID}"
rm -f "$RT_OUTPUT"
ALL_ZONES=( "us-east-1a" "us-east-1b" "us-east-1c" "us-east-1d" "us-east-1e" "us-east-1f" )
LENGTH=${#ALL_ZONES[@]}
for (( j=0; j<LENGTH; j++ ))
do
# Create Subnet for VPC
SUBNET_OUTPUT=sub_net_output.json
aws ec2 create-subnet --vpc-id "${VPC_ID}" --cidr-block "172.32.3${j}.0/24" --availability-zone "${ALL_ZONES[$j]}" --tag-specifications 'ResourceType=subnet,Tags=[{Key=Name,Value=kite-ci}]' | tee "${SUBNET_OUTPUT}"
SUBN_ID=$(jq -r '.Subnet.SubnetId' < "${SUBNET_OUTPUT}")
rm -f "$SUBNET_OUTPUT"
# Associate route table to subnet
aws ec2 associate-route-table --route-table-id "${RT_ID}" --subnet-id "${SUBN_ID}"
done
# Security Group setup
SG_OUTPUT=sec_group.json
aws ec2 create-security-group --group-name kite-ci-sg --description "kite ci edge-ami security group" --vpc-id "${VPC_ID}" --tag-specifications 'ResourceType=security-group,Tags=[{Key=Name,Value=kite-ci}]' | tee "${SG_OUTPUT}"
SEC_GROUP_ID=$(jq -r '.GroupId' < "${SG_OUTPUT}")
# Allow inbound ssh connections
aws ec2 authorize-security-group-ingress --group-id "${SEC_GROUP_ID}" --protocol tcp --port 22 --cidr 0.0.0.0/0 --tag-specifications 'ResourceType=security-group-rule,Tags=[{Key=Name,Value=kite-ci}]'
rm -f "$SG_OUTPUT"
}
# Get instance type
get_instance_type () {
arch=$1
if [[ $arch == x86_64 ]]; then
allInstanceTypes=("t2.medium" \
"t3.medium" \
"m6a.large")
elif [[ $arch == aarch64 ]]; then
allInstanceTypes=("t4g.medium" \
"c7g.medium" \
"m6g.medium")
else
echo "Not supported Architecture"
exit 1
fi
RND_LINE=$((RANDOM % 3))
echo "${allInstanceTypes[$RND_LINE]}"
@ -334,10 +340,15 @@ groups = []
[[packages]]
name = "python3"
version = "*"
EOF
# Red Hat does not provide realtime kernel package for ARM
if [[ "${ARCH}" != aarch64 ]]; then
tee -a "$BLUEPRINT_FILE" > /dev/null << EOF
[customizations.kernel]
name = "kernel-rt"
EOF
fi
greenprint "📄 container blueprint"
cat "$BLUEPRINT_FILE"
@ -468,7 +479,7 @@ EOF
sudo chmod +r "${IGNITION_CONFIG_SAMPLE_PATH}" "${IGNITION_CONFIG_PATH}"
# Start AWS cli installation
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip"
unzip awscliv2.zip > /dev/null
sudo ./aws/install --update
aws --version
@ -536,6 +547,8 @@ sudo chmod +r "${AMI_FILENAME}"
# Upload ami to AWS S3 bucket
greenprint "📂 Upload raw ami to S3 bucket"
aws s3 cp --quiet "${AMI_FILENAME}" "${BUCKET_URL}/" --acl public-read
# Clean compose and blueprints
greenprint "🧹 Clean up edge-ami compose and blueprint"
sudo composer-cli compose delete "${COMPOSE_ID}" > /dev/null
sudo composer-cli blueprints delete ami > /dev/null
@ -554,20 +567,21 @@ EOF
# Import the image as an EBS snapshot into EC2
IMPORT_SNAPSHOT_INFO=output_snapshot_info.json
aws ec2 import-snapshot --description "RHEL edge ami snapshot" --disk-container file://"${CONTAINERS_FILE}" > "${IMPORT_SNAPSHOT_INFO}"
IMPORT_TASK_ID=$(cat "${IMPORT_SNAPSHOT_INFO}" | jq -r '.ImportTaskId')
IMPORT_TASK_ID=$(jq -r '.ImportTaskId' < "${IMPORT_SNAPSHOT_INFO}")
rm -f "$IMPORT_SNAPSHOT_INFO" "$CONTAINERS_FILE"
# Monitor snapshot status
greenprint "Check import status of the snapshot"
IMPORT_SNAPSHOT_TASK=output_snapshot_task.json
while true; do
aws ec2 describe-import-snapshot-tasks --import-task-ids "${IMPORT_TASK_ID}" | tee "${IMPORT_SNAPSHOT_TASK}" > /dev/null
IMPORT_STATUS=$(cat "${IMPORT_SNAPSHOT_TASK}" | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status')
IMPORT_STATUS=$(jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status' < "${IMPORT_SNAPSHOT_TASK}")
# Has the snapshot finished?
if [[ $IMPORT_STATUS != active ]]; then
break
fi
# Wait 5 seconds and try again.
sleep 5
done
@ -579,49 +593,52 @@ else
greenprint "Snapshot imported successfully."
fi
SNAPSHOT_ID=$(jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId' < "${IMPORT_SNAPSHOT_TASK}")
aws ec2 create-tags --resources "${SNAPSHOT_ID}" --tags Key=Name,Value=composer-ci
rm -f "$IMPORT_SNAPSHOT_TASK"
# Import keypair
greenprint "Share ssh public key with AWS"
AMI_KEY_NAME="edge-ami-key-${TEST_UUID}"
# Clean previous configured keypair
aws ec2 delete-key-pair --key-name "${AMI_KEY_NAME}"
aws ec2 import-key-pair --key-name "${AMI_KEY_NAME}" --public-key-material fileb://"${SSH_KEY}".pub
aws ec2 import-key-pair --key-name "${AMI_KEY_NAME}" --public-key-material fileb://"${SSH_KEY}".pub --tag-specification 'ResourceType=key-pair,Tags=[{Key=Name,Value=composer-ci}]'
# Network setup
greenprint "VPC Network setup."
# Create ec2 network
EXISTED_VPC=$(aws ec2 describe-vpcs --filters="Name=tag:Name,Values=kite-ci" --output json --query "Vpcs")
if [[ "$EXISTED_VPC" == "[]" ]]; then
add_vpc
fi
# Create VPC
VPC_OUTPUT=vpc_output.json
aws ec2 create-vpc --cidr-block 172.32.0.0/16 --region="${AWS_DEFAULT_REGION}" | tee "${VPC_OUTPUT}" > /dev/null
VPC_ID=$(cat "${VPC_OUTPUT}" | jq -r '.Vpc.VpcId')
##################################################################
##
## Install and test edge EC2 instance with edge-ami image
##
##################################################################
# Create AMI image from EBS snapshot
greenprint "Register AMI, create image from snapshot."
REGISTERED_AMI_NAME="edge_ami-${TEST_UUID}"
REGISTERED_AMI_ID=output_ami_id.json
# Create VPC Internet Gateway
IGW_OUTPUT=igw_output.json
aws ec2 create-internet-gateway | tee "${IGW_OUTPUT}" > /dev/null
IGW_ID=$(cat "${IGW_OUTPUT}" | jq -r '.InternetGateway.InternetGatewayId')
if [[ "${ARCH}" == x86_64 ]]; then
IMG_ARCH="${ARCH}"
elif [[ "${ARCH}" == aarch64 ]]; then
IMG_ARCH=arm64
fi
# Attach internet gateway
aws ec2 attach-internet-gateway --vpc-id "${VPC_ID}" --internet-gateway-id "${IGW_ID}"
aws ec2 register-image \
--name "${REGISTERED_AMI_NAME}" \
--root-device-name /dev/xvda \
--architecture "${IMG_ARCH}" \
--ena-support \
--sriov-net-support simple \
--virtualization-type hvm \
--block-device-mappings DeviceName=/dev/xvda,Ebs=\{SnapshotId="${SNAPSHOT_ID}"\} DeviceName=/dev/xvdf,Ebs=\{VolumeSize=10\} \
--boot-mode uefi-preferred \
--output json > "${REGISTERED_AMI_ID}"
# Create Subnet for VPC
SUBNET_OUTPUT=sub_net_output.json
aws ec2 create-subnet --vpc-id "${VPC_ID}" --cidr-block 172.32.32.0/24 | tee "${SUBNET_OUTPUT}"
SUBN_ID=$(cat "${SUBNET_OUTPUT}" | jq -r '.Subnet.SubnetId')
# Add default route in route table for all vpc subnets
# Create route table
RT_OUTPUT=route_table_out.json
aws ec2 create-route-table --vpc-id "${VPC_ID}" | tee "${RT_OUTPUT}" > /dev/null
RT_ID=$(cat "${RT_OUTPUT}" | jq -r '.RouteTable.RouteTableId')
aws ec2 create-route --route-table-id "${RT_ID}" --destination-cidr-block 0.0.0.0/0 --gateway-id "${IGW_ID}"
# Associate route table to subnet
aws ec2 associate-route-table --route-table-id "${RT_ID}" --subnet-id "${SUBN_ID}"
# Security Group setup
SG_OUTPUT=sec_group.json
aws ec2 create-security-group --group-name mysecuritygroup --description "edge-ami security group" --vpc-id "${VPC_ID}" | tee "${SG_OUTPUT}"
SEC_GROUP_ID=$(cat "${SG_OUTPUT}" | jq -r '.GroupId')
# Allow inbound ssh connections
aws ec2 authorize-security-group-ingress --group-id "${SEC_GROUP_ID}" --protocol tcp --port 22 --cidr 0.0.0.0/0
AMI_ID=$(jq -r '.ImageId' < "${REGISTERED_AMI_ID}")
aws ec2 create-tags --resources "${AMI_ID}" --tags Key=Name,Value=composer-ci
rm -f "$REGISTERED_AMI_ID"
# Create instance market options
MARKET_OPTIONS=spot-options.json
@ -636,47 +653,37 @@ tee "${MARKET_OPTIONS}" > /dev/null << EOF
}
EOF
##################################################################
##
## Install and test edge EC2 instance with edge-ami image
##
##################################################################
# Create AMI image from EBS snapshot
greenprint "Register AMI, create image from snapshot."
REGISTERED_AMI_NAME="edge_ami-${TEST_UUID}"
REGISTERED_AMI_ID=output_ami_id.json
SNAPSHOT_ID=$(cat "${IMPORT_SNAPSHOT_TASK}" | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId')
aws ec2 register-image \
--name "${REGISTERED_AMI_NAME}" \
--root-device-name /dev/xvda \
--architecture "${ARCH}" \
--ena-support \
--sriov-net-support simple \
--virtualization-type hvm \
--block-device-mappings DeviceName=/dev/xvda,Ebs=\{SnapshotId="${SNAPSHOT_ID}"\} DeviceName=/dev/xvdf,Ebs=\{VolumeSize=10\} \
--boot-mode uefi-preferred \
--output json > "${REGISTERED_AMI_ID}"
AMI_ID=$(cat "${REGISTERED_AMI_ID}" | jq -r '.ImageId')
tag_ec2_ami "${AMI_ID}"
tag_describe_resource "${AMI_ID}"
# Launch Instance
greenprint "💻 Launch instance from AMI"
for LOOP_COUNTER in $(seq 0 9); do
for _ in $(seq 0 9); do
RESULTS=0
INSTANCE_OUT_INFO=instance_output_info.json
INSTANCE_TYPE=$(get_instance_type "${ARCH}")
ZONE_LIST=$(aws ec2 describe-instance-type-offerings --location-type availability-zone --filters="Name=instance-type,Values=${INSTANCE_TYPE}" --query "InstanceTypeOfferings")
if [[ "$ZONE_LIST" == "[]" ]]; then
greenprint "No available $INSTANCE_TYPE in this region"
break
else
ZONE_NAME=$(echo "$ZONE_LIST" | jq -r ".[0].Location")
fi
SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=tag:Name,Values=kite-ci" "Name=availabilityZone,Values=${ZONE_NAME}" | jq -r ".Subnets[0].SubnetId")
SEC_GROUP_ID=$(aws ec2 describe-security-groups --filters="Name=tag:Name,Values=kite-ci" --output json | jq -r ".SecurityGroups[0].GroupId")
aws ec2 run-instances \
--image-id "${AMI_ID}" \
--count 1 \
--instance-type "${INSTANCE_TYPE}" \
--placement AvailabilityZone="${ZONE_NAME}" \
--tag-specification 'ResourceType=instance,Tags=[{Key=Name,Value=composer-ci}]' \
--instance-market-options file://"${MARKET_OPTIONS}" \
--key-name "${AMI_KEY_NAME}" \
--security-group-ids "${SEC_GROUP_ID}" \
--subnet-id "${SUBN_ID}" \
--subnet-id "${SUBNET_ID}" \
--associate-public-ip-address > "${INSTANCE_OUT_INFO}" 2>&1 || :
if ! grep -iqE 'unsupported|InsufficientInstanceCapacity' "${INSTANCE_OUT_INFO}"; then
echo "Instance type supported!"
RESULTS=1
break
fi
sleep 30
@ -688,13 +695,12 @@ sleep 5
# get instance public ip
INSTANCE_ID=$(jq -r '.Instances[].InstanceId' "${INSTANCE_OUT_INFO}")
tag_ec2_instance "${INSTANCE_ID}"
tag_describe_resource "${INSTANCE_ID}"
PUBLIC_GUEST_ADDRESS=$(aws ec2 describe-instances --instance-ids "${INSTANCE_ID}" --query 'Reservations[*].Instances[*].PublicIpAddress' --output text)
rm -f "$MARKET_OPTIONS" "$INSTANCE_OUT_INFO"
# Check for ssh ready to go.
greenprint "🛃 Checking for SSH is ready to go"
for LOOP_COUNTER in $(seq 0 30); do
for _ in $(seq 0 30); do
RESULTS=$(wait_for_ssh_up "${PUBLIC_GUEST_ADDRESS}")
if [[ $RESULTS == 1 ]]; then
echo "SSH is ready now! 🥳"
@ -727,7 +733,6 @@ EOF
# Test IoT/Edge OS
sudo ansible-playbook -v -i "${TEMPDIR}"/inventory \
-e image_type="${OSTREE_OSNAME}" \
-e skip_rollback_test="true" \
-e ignition="true" \
-e edge_type=edge-ami-image \
-e ostree_commit="${INSTALL_HASH}" \
@ -768,10 +773,15 @@ description = "Administrator account"
password = "\$6\$GRmb7S0p8vsYmXzH\$o0E020S.9JQGaHkszoog4ha4AQVs3sk8q0DvLjSMxoxHBKnB2FBXGQ/OkwZQfW/76ktHd0NX5nls2LPxPuUdl."
home = "/home/admin/"
groups = ["wheel"]
EOF
# Red Hat does not provide realtime kernel package for ARM
if [[ "${ARCH}" != aarch64 ]]; then
tee -a "$BLUEPRINT_FILE" > /dev/null << EOF
[customizations.kernel]
name = "kernel-rt"
EOF
fi
greenprint "📄 upgrade blueprint"
cat "$BLUEPRINT_FILE"
@ -849,8 +859,7 @@ sleep 10
# Check for ssh ready to go.
greenprint "🛃 Checking for SSH is ready to go"
# shellcheck disable=SC2034 # Unused variables left for readability
for LOOP_COUNTER in $(seq 0 30); do
for _ in $(seq 0 30); do
RESULTS=$(wait_for_ssh_up "${PUBLIC_GUEST_ADDRESS}")
if [[ $RESULTS == 1 ]]; then
echo "SSH is ready now! 🥳"
@ -891,4 +900,3 @@ check_result
clean_up
exit 0

View file

@ -99,7 +99,7 @@
- name: wait for FDO onboarding
block:
- wait_for:
path: "/etc/device-credentials"
path: "/etc/device-credentials"
delay: 10
timeout: 600
state: present
@ -747,8 +747,9 @@
- name: check fdo-client-linuxapp logs
command: journalctl -u fdo-client-linuxapp
register: result_fdo_client_linuxapp_journalctl
when: fdo_credential == "true"
- debug:
- debug:
var: result_fdo_client_linuxapp_journalctl
# case: check rollback function if boot error found
@ -843,8 +844,9 @@
- name: check fdo-client-linuxapp logs
command: journalctl -u fdo-client-linuxapp
register: result_fdo_client_linuxapp_journalctl
when: fdo_credential == "true"
- debug:
- debug:
var: result_fdo_client_linuxapp_journalctl
- name: wait for FDO re-encryption
@ -866,8 +868,9 @@
- name: check fdo-client-linuxapp logs
command: journalctl -u fdo-client-linuxapp
register: result_fdo_client_linuxapp_journalctl
when: fdo_credential == "true"
- debug:
- debug:
var: result_fdo_client_linuxapp_journalctl
# case: check greenboot* services log again