From 9d0ae3bc1f8f1e67595b6021e4f572ac11f1d40c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Budai?= Date: Tue, 7 Dec 2021 15:22:48 +0100 Subject: [PATCH] packer: add initialization scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker needs quite a lot of configuration involving secrets. Baking them in the AMI is just awful so we need to fetch them during the instance startup. Previously, this was all done using cloud-init. This makes the cloud-init config huge and it is also very hard to test. This commit moves all the configuration scripts into the image itself. Cloud-init still needs to be used to push the secret variables into the instance. The configuration scripts are run after cloud-init. They pick up yhe secrets and initialize the worker correctly. These scripts were adopted from https://github.com/osbuild/image-builder-terraform/commit/75b752a1c00c4ce931962d902a2a61c9998948f2 (private repository). During the adoption, some changes has to be applied to make shellcheck happy. Signed-off-by: Ondřej Budai --- templates/packer/README.md | 95 +++++++++++++++++++ .../offline_token.sh | 14 +++ .../set_hostname.sh | 15 +++ .../subscription_manager.sh | 15 +++ .../worker-initialization-scripts/vector.sh | 22 +++++ .../worker_external_creds.sh | 40 ++++++++ .../worker_service.sh | 24 +++++ .../files/worker-initialization.service | 18 ++++ .../ansible/roles/common/tasks/main.yml | 3 + .../tasks/worker-initialization-service.yml | 26 +++++ 10 files changed, 272 insertions(+) create mode 100644 templates/packer/README.md create mode 100755 templates/packer/ansible/roles/common/files/worker-initialization-scripts/offline_token.sh create mode 100755 templates/packer/ansible/roles/common/files/worker-initialization-scripts/set_hostname.sh create mode 100755 templates/packer/ansible/roles/common/files/worker-initialization-scripts/subscription_manager.sh create mode 100755 templates/packer/ansible/roles/common/files/worker-initialization-scripts/vector.sh create mode 100755 templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_external_creds.sh create mode 100755 templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_service.sh create mode 100644 templates/packer/ansible/roles/common/files/worker-initialization.service create mode 100644 templates/packer/ansible/roles/common/tasks/worker-initialization-service.yml diff --git a/templates/packer/README.md b/templates/packer/README.md new file mode 100644 index 000000000..ae49db345 --- /dev/null +++ b/templates/packer/README.md @@ -0,0 +1,95 @@ +# osbuild-composer Packer configuration + +This directory contains a packer configuration for building osbuild-composer +worker AMIs based on RHEL. + +## Running packer locally + +Run the following command in the root directory of this repository: +``` +PKR_VAR_aws_access_key="" \ +PKR_VAR_aws_secret_key="" \ +PKR_VAR_image_name=YOUR_UNIQUE_IMAGE_NAME \ +PKR_VAR_composer_commit=OSBUILD_COMPOSER_COMMIT_SHA \ +PKR_VAR_osbuild_commit=OSBUILD_COMMIT_SHA \ + packer build templates/packer +``` + +## Launching an instance from the built AMI + +The AMI expects that cloud-init is used to create a `/tmp/cloud_init_vars` +file that contains configuration values for the particular instance. + +The following block shows an example of such a file. The order of the +key-value pairs is not fixed but all of them are required. + +``` +# Domain name of the composer instance that the worker connects to +COMPOSER_HOST=api.stage.openshift.com + +# Port number of the composer instance that the worker connects to +COMPOSER_PORT=443 + +# AWS ARN of a secret containing a OAuth offline token that is used to authenticate to composer +# The secret contains only one key "offline_token". Its value is the offline token to be used. +OFFLINE_TOKEN_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:offline-token-abcdef + +# AWS ARN of a secret containing a command to subscribe the instance using subscription-manager +# The secrets contains only one key "subscription_manager_command" that contains the subscription-manager command +SUBSCRIPTION_MANAGER_COMMAND_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:subscription-manager-command-abcdef + +# AWS ARN of a secret containing GCP service account credentials +# The secret contains a JSON key file, see https://cloud.google.com/docs/authentication/getting-started +GCP_SERVICE_ACCOUNT_IMAGE_BUILDER_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:gcp_service_account_image_builder-abcdef + +# AWS ARN of a secret containing Azure account credentials +# The secret contains two keys: "client_secret" and "client_id". +AZURE_ACCOUNT_IMAGE_BUILDER_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:azure_account_image_builder-abcdef + +# AWS ARN of a secret containing AWS account credentials +# The secret contains two keys: "access_key_id" and "secret_access_key". +AWS_ACCOUNT_IMAGE_BUILDER_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:aws_account_image_builder-abcdef + +# The auto-generated EC2 instance ID is prefixed with this string to simplify searching in logs +SYSTEM_HOSTNAME_PREFIX=staging-worker-aoc + +# Endpoint URL for AWS Secrets Manager +SECRETS_MANAGER_ENDPOINT_URL=https://secretsmanager.us-east-1.amazonaws.com/ + +# Endpoint URL for AWS Cloudwatch Logs +CLOUDWATCH_LOGS_ENDPOINT_URL=https://logs.us-east-1.amazonaws.com/ + +# AWS Cloudwatch log group that the instance logs into +CLOUDWATCH_LOG_GROUP=staging_workers_aoc +``` + +### IAM considerations +The instance must have a IAM policy attached that permits it: + +- to access all configured secrets +- to create new log streams in the configured log group and to put log entried in them + + +### Cloud-init example + +The simplest way is to inject the file is to just use cloud-init's +`write_files` directive: + +``` +#cloud-config + +write_files: + - path: /tmp/cloud_init_vars + content: | + COMPOSER_HOST=api.stage.openshift.com + COMPOSER_PORT=443 + OFFLINE_TOKEN_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:offline-token-abcdef + SUBSCRIPTION_MANAGER_COMMAND_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:subscription-manager-command-abcdef + GCP_SERVICE_ACCOUNT_IMAGE_BUILDER_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:gcp_service_account_image_builder-abcdef + AZURE_ACCOUNT_IMAGE_BUILDER_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:azure_account_image_builder-abcdef + AWS_ACCOUNT_IMAGE_BUILDER_ARN=arn:aws:secretsmanager:us-east-1:123456789012:secret:aws_account_image_builder-abcdef + SYSTEM_HOSTNAME_PREFIX=staging-worker-aoc + SECRETS_MANAGER_ENDPOINT_URL=https://secretsmanager.us-east-1.amazonaws.com/ + CLOUDWATCH_LOGS_ENDPOINT_URL=https://logs.us-east-1.amazonaws.com/ + CLOUDWATCH_LOG_GROUP=staging_workers_aoc +``` diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/offline_token.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/offline_token.sh new file mode 100755 index 000000000..1c7d84fad --- /dev/null +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/offline_token.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -euo pipefail +source /tmp/cloud_init_vars + +echo "Writing offline token." + +# get offline token +/usr/local/bin/aws secretsmanager get-secret-value \ + --endpoint-url "${SECRETS_MANAGER_ENDPOINT_URL}" \ + --secret-id "${OFFLINE_TOKEN_ARN}" | jq -r ".SecretString" > /tmp/offline-token.json + +mkdir /etc/osbuild-worker +jq -r ".offline_token" /tmp/offline-token.json > /etc/osbuild-worker/offline-token +rm -f /tmp/offline-token.json diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/set_hostname.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/set_hostname.sh new file mode 100755 index 000000000..1196b1411 --- /dev/null +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/set_hostname.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euo pipefail +source /tmp/cloud_init_vars + +# Get the instance ID. +INSTANCE_ID=$(curl -Ls http://169.254.169.254/latest/meta-data/instance-id) + +# Assemble hostname. +FULL_HOSTNAME="${SYSTEM_HOSTNAME_PREFIX}-${INSTANCE_ID}" + +# Print out the new hostname. +echo "Setting system hostname to ${FULL_HOSTNAME}." + +# Set the system hostname. +hostnamectl set-hostname "$FULL_HOSTNAME" diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/subscription_manager.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/subscription_manager.sh new file mode 100755 index 000000000..259efebb5 --- /dev/null +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/subscription_manager.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euo pipefail +source /tmp/cloud_init_vars + +echo "Subscribing instance to RHN." + +# Register the instance with RHN. +# TODO: don't store the command in a secret, only the key/org-id +/usr/local/bin/aws secretsmanager get-secret-value \ + --endpoint-url "${SECRETS_MANAGER_ENDPOINT_URL}" \ + --secret-id "${SUBSCRIPTION_MANAGER_COMMAND_ARN}" | jq -r ".SecretString" > /tmp/subscription_manager_command.json +jq -r ".subscription_manager_command" /tmp/subscription_manager_command.json | bash +rm -f /tmp/subscription_manager_command.json + +subscription-manager attach --auto diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/vector.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/vector.sh new file mode 100755 index 000000000..f61e56910 --- /dev/null +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/vector.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euo pipefail +source /tmp/cloud_init_vars + +echo "Writing vector config." + +sudo mkdir -p /etc/vector +sudo tee /etc/vector/vector.toml > /dev/null << EOF +[sources.journald] +type = "journald" +exclude_units = ["vector.service"] + +[sinks.out] +type = "aws_cloudwatch_logs" +inputs = [ "journald" ] +endpoint = "${CLOUDWATCH_LOGS_ENDPOINT_URL}" +group_name = "${CLOUDWATCH_LOG_GROUP}" +stream_name = "worker_syslog_{{ host }}" +encoding.codec = "json" +EOF + +sudo systemctl enable --now vector diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_external_creds.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_external_creds.sh new file mode 100755 index 000000000..949f84c54 --- /dev/null +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_external_creds.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -euo pipefail +source /tmp/cloud_init_vars + +echo "Deploy cloud credentials for workers." + +# Deploy the GCP Service Account credentials file. +/usr/local/bin/aws secretsmanager get-secret-value \ + --endpoint-url "${SECRETS_MANAGER_ENDPOINT_URL}" \ + --secret-id "${GCP_SERVICE_ACCOUNT_IMAGE_BUILDER_ARN}" | jq -r ".SecretString" > /etc/osbuild-worker/gcp_credentials.json + +# Deploy the Azure credentials file. +/usr/local/bin/aws secretsmanager get-secret-value \ + --endpoint-url "${SECRETS_MANAGER_ENDPOINT_URL}" \ + --secret-id "${AZURE_ACCOUNT_IMAGE_BUILDER_ARN}" | jq -r ".SecretString" > /tmp/azure_credentials.json +CLIENT_ID=$(jq -r ".client_id" /tmp/azure_credentials.json) +CLIENT_SECRET=$(jq -r ".client_secret" /tmp/azure_credentials.json) +rm /tmp/azure_credentials.json + +sudo tee /etc/osbuild-worker/azure_credentials.toml > /dev/null << EOF +client_id = "$CLIENT_ID" +client_secret = "$CLIENT_SECRET" +EOF + +# Deploy the AWS credentials file if the secret ARN was set. +if [[ -n "$AWS_ACCOUNT_IMAGE_BUILDER_ARN" ]]; then + /usr/local/bin/aws secretsmanager get-secret-value \ + --endpoint-url "${SECRETS_MANAGER_ENDPOINT_URL}" \ + --secret-id "${AWS_ACCOUNT_IMAGE_BUILDER_ARN}" | jq -r ".SecretString" > /tmp/aws_credentials.json + ACCESS_KEY_ID=$(jq -r ".access_key_id" /tmp/aws_credentials.json) + SECRET_ACCESS_KEY=$(jq -r ".secret_access_key" /tmp/aws_credentials.json) + rm /tmp/aws_credentials.json + + sudo tee /etc/osbuild-worker/aws_credentials.toml > /dev/null << EOF +[default] +aws_access_key_id = "$ACCESS_KEY_ID" +aws_secret_access_key = "$SECRET_ACCESS_KEY" +EOF + +fi diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_service.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_service.sh new file mode 100755 index 000000000..0eb0b7f48 --- /dev/null +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/worker_service.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail +source /tmp/cloud_init_vars + +echo "Setting up worker services." + +sudo tee /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF +base_path = "/api/image-builder-worker/v1" +[authentication] +oauth_url = "https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token" +offline_token = "/etc/osbuild-worker/offline-token" +[gcp] +credentials = "/etc/osbuild-worker/gcp_credentials.json" +[azure] +credentials = "/etc/osbuild-worker/azure_credentials.toml" +[aws] +credentials = "/etc/osbuild-worker/aws_credentials.toml" +EOF + +# Prepare osbuild-composer's remote worker services and sockets. +systemctl enable --now "osbuild-remote-worker@${COMPOSER_HOST}:${COMPOSER_PORT}" + +# Now that everything is configured, ensure monit is monitoring everything. +systemctl enable --now monit diff --git a/templates/packer/ansible/roles/common/files/worker-initialization.service b/templates/packer/ansible/roles/common/files/worker-initialization.service new file mode 100644 index 000000000..ef117d57a --- /dev/null +++ b/templates/packer/ansible/roles/common/files/worker-initialization.service @@ -0,0 +1,18 @@ +[Unit] +Description=Worker Initialization Service +ConditionPathExists=!/etc/worker-first-boot +Wants=cloud-final.service +After=cloud-final.service + +[Service] +Type=oneshot +ExecStart=touch /etc/worker-first-boot +ExecStart=/usr/local/libexec/worker-initialization-scripts/set_hostname.sh +ExecStart=/usr/local/libexec/worker-initialization-scripts/vector.sh +ExecStart=/usr/local/libexec/worker-initialization-scripts/offline_token.sh +ExecStart=/usr/local/libexec/worker-initialization-scripts/subscription_manager.sh +ExecStart=/usr/local/libexec/worker-initialization-scripts/worker_external_creds.sh +ExecStart=/usr/local/libexec/worker-initialization-scripts/worker_service.sh + +[Install] +WantedBy=multi-user.target diff --git a/templates/packer/ansible/roles/common/tasks/main.yml b/templates/packer/ansible/roles/common/tasks/main.yml index 40e729a05..8ceaafa99 100644 --- a/templates/packer/ansible/roles/common/tasks/main.yml +++ b/templates/packer/ansible/roles/common/tasks/main.yml @@ -6,5 +6,8 @@ # Configure monitoring. - include_tasks: monitoring.yml +# Configure worker initialization service. +- include_tasks: worker-initialization-service.yml + - name: Ensure SELinux contexts are updated command: restorecon -Rv /etc diff --git a/templates/packer/ansible/roles/common/tasks/worker-initialization-service.yml b/templates/packer/ansible/roles/common/tasks/worker-initialization-service.yml new file mode 100644 index 000000000..82db91069 --- /dev/null +++ b/templates/packer/ansible/roles/common/tasks/worker-initialization-service.yml @@ -0,0 +1,26 @@ +--- + +- name: Copy worker initialization service + copy: + src: "{{ playbook_dir }}/roles/common/files/worker-initialization.service" + dest: /etc/systemd/system/ + +- name: Enable worker initialization service + systemd: + name: worker-initialization.service + enabled: yes + daemon_reload: yes # make sure the new service is loaded before enabling it + +- name: Create a directory for initialization scripts + file: + path: /usr/local/libexec/worker-initialization-scripts + state: directory + +- name: Copy scripts used by the initialization service + copy: + src: "{{ item }}" + dest: /usr/local/libexec/worker-initialization-scripts + mode: preserve + with_fileglob: + - "{{ playbook_dir }}/roles/common/files/worker-initialization-scripts/*" +