appsre-ansible: support aarch64

make ansible playbooks arch-agnostic
extract embedded bash script into separate file with parameters
update packer template to support aarch64
Convert parts of bash script to python code that can start multi-arch instances to build RPMS

templates/packer/ansible/inventory/rhel-8-aarch64/group_vars/all.yml

@@ -0,0 +1,5 @@
+---
+# this is just a template!
+# the actual content is generated by build/appsre-build-worker-packer.sh
+rpmrepo_distribution: distro
+osbuild_commit: abcdef

templates/packer/ansible/roles/common/tasks/packages.yml

@@ -96,7 +96,7 @@
     - rpmcopy
   ansible.posix.synchronize:
     mode: push
-    src: "{{ playbook_dir }}/roles/common/files/rpmbuild/RPMS"
+    src: "{{ playbook_dir }}/roles/common/files/rpmbuild/{{ ansible_architecture }}/RPMS"
     dest: /tmp/rpmbuild
 
 - name: Add repo config

templates/packer/worker.pkr.hcl

@@ -55,6 +55,36 @@ build {
     }
   }
 
+  source "amazon-ebs.image_builder" {
+    name = "rhel-8-aarch64"
+
+    # Use a static RHEL 8.6 Cloud Access Image.
+    source_ami = "ami-0c84d76d81209a0e2"
+    ssh_username = "ec2-user"
+    instance_type = "c6g.large"
+
+    # Set a name for the resulting AMI.
+    ami_name = "${var.image_name}"
+
+    # Apply tags to the resulting AMI/EBS snapshot.
+    tags = {
+      AppCode = "IMGB-001"
+      Name = "${var.image_name}"
+      composer_commit = "${var.composer_commit}"
+      os = "rhel"
+      os_version = "8"
+      arch = "aarch64"
+    }
+
+    # Ensure that the EBS snapshot used for the AMI meets our requirements.
+    launch_block_device_mappings {
+      delete_on_termination = "true"
+      device_name           = "/dev/sda1"
+      volume_size           = 10
+      volume_type           = "gp2"
+    }
+  }
+
   source "amazon-ebs.image_builder"  {
     name = "fedora-35-x86_64"
 

tools/appsre-ansible/rpmbuild.yml

@@ -3,8 +3,17 @@
   become: yes
   remote_user: ec2-user
   hosts: rpmbuilder
+  gather_facts: no
   tasks:
 
+  - name: Wait for hosts to be reachable
+    ansible.builtin.wait_for_connection:
+      delay: 60
+      timeout: 400
+
+  - name: Gather facts
+    ansible.builtin.setup:
+
   - name: Add EPEL
     dnf:
       name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
@@ -72,7 +81,7 @@
   - name: Mockbuild osbuild
     shell: >-
         mock
-        -r "rhel-8-x86_64"
+        -r "rhel-8-{{ ansible_architecture }}"
         --rebuild
         --define "commit {{ OSBUILD_COMMIT }}"
         --define "_rpmfilename %%{NAME}.rpm"
@@ -91,7 +100,7 @@
   - name: Mockbuild osbuild-composer
     shell: >-
         mock
-        -r "rhel-8-x86_64"
+        -r "rhel-8-{{ ansible_architecture }}"
         --rebuild
         --define "commit {{ COMPOSER_COMMIT }}"
         --define "_rpmfilename %%{NAME}.rpm"
@@ -105,4 +114,4 @@
     ansible.posix.synchronize:
       mode: pull
       src: /home/ec2-user/rpmbuild/RPMS
-      dest: /osbuild-composer/templates/packer/ansible/roles/common/files/rpmbuild
+      dest: /osbuild-composer/templates/packer/ansible/roles/common/files/rpmbuild/{{ ansible_architecture }}

tools/appsre-build-worker-packer.sh

@@ -7,6 +7,7 @@ COMMIT_SHA=$(git rev-parse HEAD)
 COMMIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 ON_JENKINS=true
 SKIP_CREATE_AMI=false
+BUILD_RPMS=false
 
 # Use gitlab CI variables if available
 if [ -n "$CI_COMMIT_SHA" ]; then
@@ -49,91 +50,20 @@ function cleanup {
 }
 trap cleanup EXIT
 
-# What we will cp and exec
-cat > worker-packer.sh<<'EOF'
-#!/bin/bash
-set -exv
-EOF
-chmod +x worker-packer.sh
-
-# warning: this is RHEL 8 x86_64 specific!
-function ec2_rpm_build {
-    cat >> worker-packer.sh <<'EOF'
-function cleanup {
-    set +e
-    if [ "$ON_JENKINS" = true ]; then
-        if [ -n "$AWS_INSTANCE_ID" ]; then
-            aws ec2 terminate-instances --instance-ids "$AWS_INSTANCE_ID"
-        fi
-        if [ -n "$KEY_NAME" ]; then
-            aws ec2 delete-key-pair --key-name "$KEY_NAME"
-        fi
-    fi
-}
-trap cleanup EXIT
-
-KEY_NAME=$(uuidgen)
-RPMBUILD_DIR="/osbuild-composer/templates/packer/ansible/roles/common/files/rpmbuild/RPMS"
-mkdir -p "$RPMBUILD_DIR"
-
-aws ec2 create-key-pair --key-name "$KEY_NAME" --query 'KeyMaterial' --output text > /osbuild-composer/keypair.pem
-chmod 600 /osbuild-composer/keypair.pem
-# rhel 8.5 AMI
-aws ec2 run-instances --image-id ami-03debf3ebf61b20cd --instance-type c5.large --key-name "$KEY_NAME" \
-    --tag-specifications "ResourceType=instance,Tags=[{Key=commit,Value=$COMMIT_SHA},{Key=name,Value=rpm-builder-$COMMIT_SHA}]" \
-    > ./rpminstance.json
-AWS_INSTANCE_ID=$(jq -r '.Instances[].InstanceId' "rpminstance.json")
-aws ec2 wait instance-running --instance-ids "$AWS_INSTANCE_ID"
-
-aws ec2 describe-instances --instance-ids "$AWS_INSTANCE_ID" > "instances.json"
-RPMBUILDER_HOST=$(jq -r '.Reservations[].Instances[].PublicIpAddress' "instances.json")
-for LOOP_COUNTER in {0..30}; do
-    if ssh -i /osbuild-composer/keypair.pem -o ConnectTimeout=5 -o StrictHostKeyChecking=no "ec2-user@$RPMBUILDER_HOST" true; then
-        break
-    fi
-    sleep 5
-    echo "sleeping, try #$LOOP_COUNTER"
-done
-
-cat > /osbuild-composer/tools/appsre-ansible/inventory <<EOF2
-[rpmbuilder]
-$RPMBUILDER_HOST ansible_ssh_private_key_file=/osbuild-composer/keypair.pem ansible_ssh_common_args='-o StrictHostKeyChecking=no -o ServerAliveInterval=5'
-EOF2
-
-ansible-playbook \
-    -i /osbuild-composer/tools/appsre-ansible/inventory \
-    /osbuild-composer/tools/appsre-ansible/rpmbuild.yml \
-    -e "COMPOSER_COMMIT=$COMMIT_SHA" \
-    -e "OSBUILD_COMMIT=$(jq -r '.["rhel-8.6"].dependencies.osbuild.commit' /osbuild-composer/Schutzfile)" \
-    -e "RH_ACTIVATION_KEY=$RH_ACTIVATION_KEY" \
-    -e "RH_ORG_ID=$RH_ORG_ID"
-EOF
-}
-
-
 # Use prebuilt rpms on CI
 SKIP_TAGS="rpmcopy"
 if [ "$ON_JENKINS" = true ]; then
-    # Append rpm build to script when running on AppSRE's infra
-    ec2_rpm_build
+    # Build RPMs when running on AppSRE's infra
+    BUILD_RPMS=true
     SKIP_TAGS="rpmrepo"
 fi
 
-# Format: PACKER_IMAGE_USERS="\"000000000000\",\"000000000001\""
-if [ -n "$PACKER_IMAGE_USERS" ]; then
-    cat >> worker-packer.sh <<'EOF'
-cat > /osbuild-composer/templates/packer/share.auto.pkrvars.hcl <<EOF2
-image_users = [$PACKER_IMAGE_USERS]
-EOF2
-EOF
-fi
-
 if [ "$ON_JENKINS" = true ]; then
     # jenkins on main: build rhel only
-    PACKER_ONLY_EXCEPT=--only=amazon-ebs.rhel-8-x86_64
+    PACKER_ONLY_EXCEPT=--only=amazon-ebs.rhel-8-x86_64,amazon-ebs.rhel-8-aarch64
 elif [ -n "$CI_COMMIT_BRANCH" ] && [ "$CI_COMMIT_BRANCH" == "main" ]; then
     # Schutzbot on main: build all except rhel
-    PACKER_ONLY_EXCEPT=--except=amazon-ebs.rhel-8-x86_64
+    PACKER_ONLY_EXCEPT=--except=amazon-ebs.rhel-8-x86_64,amazon-ebs.rhel-8-aarch64
 elif [ -n "$CI_COMMIT_BRANCH" ]; then
     # Schutzbot but not main, build everything (use dummy except)
     PACKER_ONLY_EXCEPT=--except=amazon-ebs.dummy
@@ -201,12 +131,15 @@ $CONTAINER_RUNTIME run --rm \
                    -e COMMIT_SHA="$COMMIT_SHA" \
                    -e ON_JENKINS="$ON_JENKINS" \
                    -e PACKER_IMAGE_USERS="$PACKER_IMAGE_USERS" \
+                   -e PACKER_ONLY_EXCEPT="$PACKER_ONLY_EXCEPT" \
                    -e RH_ACTIVATION_KEY="$RH_ACTIVATION_KEY" \
                    -e RH_ORG_ID="$RH_ORG_ID" \
+                   -e BUILD_RPMS="$BUILD_RPMS" \
                    -e PKR_VAR_aws_access_key="$PACKER_AWS_ACCESS_KEY_ID" \
                    -e PKR_VAR_aws_secret_key="$PACKER_AWS_SECRET_ACCESS_KEY" \
                    -e PKR_VAR_image_name="osbuild-composer-worker-$COMMIT_BRANCH-$COMMIT_SHA" \
                    -e PKR_VAR_composer_commit="$COMMIT_SHA" \
                    -e PKR_VAR_ansible_skip_tags="$SKIP_TAGS" \
                    -e PKR_VAR_skip_create_ami="$SKIP_CREATE_AMI" \
-                   "packer:$COMMIT_SHA" /osbuild-composer/worker-packer.sh
+                   -e PYTHONUNBUFFERED=1 \
+                   "packer:$COMMIT_SHA" /osbuild-composer/tools/appsre-worker-packer-container.sh

tools/appsre-worker-packer-container.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -exv
+
+if [ "$BUILD_RPMS" = true ]; then
+    OUTPUT_DIR=/osbuild-composer/
+
+    python3 -m pip install boto3
+
+    mkdir -p "$OUTPUT_DIR/templates/packer/ansible/roles/common/files/rpmbuild/x86_64/RPMS"
+    mkdir -p "$OUTPUT_DIR/templates/packer/ansible/roles/common/files/rpmbuild/aarch64/RPMS"
+
+    /osbuild-composer/tools/build-rpms.py --base-dir $OUTPUT_DIR --commit "$COMMIT_SHA" x86_64 aarch64
+fi
+
+# Format: PACKER_IMAGE_USERS="\"000000000000\",\"000000000001\""
+if [ -n "$PACKER_IMAGE_USERS" ]; then
+    cat > /osbuild-composer/templates/packer/share.auto.pkrvars.hcl <<EOF2
+image_users = [$PACKER_IMAGE_USERS]
+EOF2
+fi
+
+/usr/bin/packer build "$PACKER_ONLY_EXCEPT" /osbuild-composer/templates/packer

tools/build-rpms.py

@@ -0,0 +1,210 @@
+#!/usr/bin/python3
+
+""" Used by AppSRE indirectly when building AMIs.
+Builds osbuild-composer & osbuild rpms for one or more architectures. """
+
+import argparse
+import json
+import os
+import pathlib
+import subprocess
+import sys
+import uuid
+import boto3
+
+arch_info = {}
+arch_info["x86_64"] = {
+    "ImageId": "ami-03debf3ebf61b20cd",
+    "InstanceType": "c5.large"
+}
+arch_info["aarch64"] = {
+    "ImageId": "ami-0c84d76d81209a0e2",
+    "InstanceType": "c6g.large"
+}
+
+
+class fg:  # pylint: disable=too-few-public-methods
+    """Set of constants to print colored output in the terminal"""
+    BOLD = '\033[1m'  # bold
+    OK = '\033[32m'  # green
+    INFO = '\033[33m'  # yellow
+    ERROR = '\033[31m'  # red
+    RESET = '\033[0m'  # reset
+
+
+def msg_error(body):
+    print(f"{fg.ERROR}{fg.BOLD}Error:{fg.RESET} {body}")
+
+
+def msg_info(body):
+    print(f"{fg.INFO}{fg.BOLD}Info:{fg.RESET} {body}")
+
+
+def msg_ok(body):
+    print(f"{fg.OK}{fg.BOLD}OK:{fg.RESET} {body}")
+
+
+def run_command(argv):
+    subprocess.run(argv, check=True)
+
+
+def create_cleanup_function(name, f, *args):
+    def fun():
+        msg_info("Cleaning up: " + name)
+        try:
+            f(*args)
+        except Exception as ex:
+            msg_error("during cleanup: " + str(ex))
+
+    return fun
+
+
+def stage(name, params, fun, *args):
+    msg_info(name + ','.join(params))
+
+    try:
+        ret = fun(*args)
+    except Exception as e:
+        msg_error(f"{name} {','.join(params)} failed: {e}")
+        raise
+
+    msg_ok(f"{name} {','.join(params)}")
+    return ret
+
+
+def create_keypair(cleanup_actions):
+    ec2 = boto3.client('ec2')
+    keyname = f'rpm-builder-{uuid.uuid4()}'
+    response = ec2.create_key_pair(KeyName=keyname)
+    cleanup_actions += [
+        create_cleanup_function(
+            f"keypair {keyname}",
+            lambda k: ec2.delete_key_pair(
+                KeyName=k), keyname)]
+    return keyname, response['KeyMaterial']
+
+
+def create_ec2_instances(cleanup_actions, args, keypair):
+    ec2 = boto3.resource('ec2')
+
+    instances = []
+    for a in args.arch:
+        tags = [
+            {
+                "ResourceType": "instance",
+                "Tags": [
+                    {
+                        "Key": "name",
+                        "Value": f"rpm-builder-{uuid.uuid4()}"
+                    },
+                    {
+                        "Key": "commit",
+                        "Value": f"{args.commit}"
+                    },
+
+                ]
+            }
+        ]
+
+        instance = ec2.create_instances(
+            ImageId=arch_info[a]["ImageId"],
+            MinCount=1,
+            MaxCount=1,
+            InstanceType=arch_info[a]["InstanceType"],
+            KeyName=keypair,
+            TagSpecifications=tags
+        )
+        instances += instance
+
+    for i in instances:
+        cleanup_actions += [
+            create_cleanup_function(
+                f"instance {i.id}",
+                lambda x: x.terminate(),
+                i)]
+        i.wait_until_running()
+        i.reload()
+
+    return instances
+
+
+def setup_ansible(args, instances):
+    with open(os.path.join(args.base_dir, "tools", "appsre-ansible", "inventory"), 'w') as f:
+        f.write("[rpmbuilder]\n")
+        for i in instances:
+            f.write(f"{i.public_ip_address}\n")
+
+
+def run_ansible(args, key_material):
+    os.umask(0)
+    keypath = os.path.join(args.base_dir, "keypair.pem")
+
+    with open(os.open(keypath, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
+        f.write(key_material)
+
+    with open(args.base_dir / "Schutzfile") as f:
+        osbuild_commit = json.load(
+            f)["rhel-8.6"]["dependencies"]["osbuild"]["commit"]
+
+    return run_command(["ansible-playbook",
+                        "--ssh-extra-args", "-o ControlPersist=no -o StrictHostKeyChecking=no -o ServerAliveInterval=5",
+                        "-i", f"{args.base_dir}/tools/appsre-ansible/inventory",
+                        "--key-file", keypath,
+                        "-e", f"COMPOSER_COMMIT={args.commit}",
+                        "-e", f"OSBUILD_COMMIT={osbuild_commit}",
+                        "-e", f"RH_ACTIVATION_KEY={os.environ['RH_ACTIVATION_KEY']}",
+                        "-e", f"RH_ORG_ID={os.environ['RH_ORG_ID']}",
+                        f"{args.base_dir}/tools/appsre-ansible/rpmbuild.yml"])
+
+
+def stage_generate_rpms(cleanup_actions, args):
+    keyname, key_material = stage(
+        "Create keypair", (), create_keypair, cleanup_actions)
+    instances = stage("Create EC2 instances", (),
+                      create_ec2_instances, cleanup_actions, args, keyname)
+    stage("Setup ansible", (), setup_ansible, args, instances)
+    stage("Run Ansible playbook", (), run_ansible, args, key_material)
+
+
+def check_env():
+    required_envvars = ["RH_ORG_ID", "RH_ACTIVATION_KEY"]
+    if not all(i in os.environ for i in required_envvars):
+        msg_error(
+            f"At least one of the required environment variables is missing: {required_envvars}")
+        sys.exit(1)
+
+
+def check_params():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--base-dir', type=pathlib.Path, required=True,
+                        help='Base directory for creating files')
+
+    parser.add_argument('--commit', type=str, required=True,
+                        help='Commit SHA')
+
+    parser.add_argument('arch', type=str, nargs='+', choices=arch_info.keys(),
+                        help='Architectures to build images for')
+
+    args = parser.parse_args()
+
+    return args
+
+
+def print_params(p):
+    msg_info(f"Building {p.arch} RPMs with base-dir {p.base_dir}")
+
+
+if __name__ == "__main__":
+
+    cleanup_actions = []
+
+    check_env()
+    args = check_params()
+    print_params(args)
+
+    try:
+        stage_generate_rpms(cleanup_actions, args)
+    finally:
+        for c in cleanup_actions:
+            c()