From 44e2a9519be11a7973ae43082cc39e0b552bdf09 Mon Sep 17 00:00:00 2001 From: Gianluca Zuccarelli Date: Fri, 29 Sep 2023 11:24:15 +0100 Subject: [PATCH] devel: mv to osbuild/getting-started Move the docker backend to the `getting-started` repo in `osbuild` since it seems like a more appropriate place for the backend stack. Add a note to the README to point users to the new repository. --- devel/.env | 7 - devel/.gitignore | 3 - devel/README.md | 90 +--------- devel/config/backend/quotas.json | 6 - devel/config/composer/acl.yml | 4 - devel/config/composer/osbuild-composer.toml | 20 --- devel/config/grafana/dashboards/dashboard.yml | 10 -- .../config/grafana/datasources/datasource.yml | 16 -- devel/config/prometheus/prometheus.yml | 17 -- devel/config/spandx/local-frontend-and-api.js | 11 -- devel/config/worker/osbuild-worker.toml | 4 - devel/config/worker/secret | 1 - devel/config/x509/openssl.cnf | 85 --------- devel/docker-compose.yml | 165 ------------------ devel/gen-certs.sh | 97 ---------- devel/gen-dashboards | 32 ---- devel/setup.sh | 16 -- 17 files changed, 1 insertion(+), 583 deletions(-) delete mode 100644 devel/.env delete mode 100644 devel/.gitignore delete mode 100644 devel/config/backend/quotas.json delete mode 100644 devel/config/composer/acl.yml delete mode 100644 devel/config/composer/osbuild-composer.toml delete mode 100644 devel/config/grafana/dashboards/dashboard.yml delete mode 100644 devel/config/grafana/datasources/datasource.yml delete mode 100644 devel/config/prometheus/prometheus.yml delete mode 100644 devel/config/spandx/local-frontend-and-api.js delete mode 100644 devel/config/worker/osbuild-worker.toml delete mode 100644 devel/config/worker/secret delete mode 100644 devel/config/x509/openssl.cnf delete mode 100644 devel/docker-compose.yml delete mode 100755 devel/gen-certs.sh delete mode 100755 devel/gen-dashboards delete mode 100755 devel/setup.sh diff --git a/devel/.env b/devel/.env deleted file mode 100644 index c0ca9457..00000000 --- a/devel/.env +++ /dev/null @@ -1,7 +0,0 @@ -COMPOSE_PROJECT_NAME=image-builder -CERT_DIR=./state/x509 -COMPOSER_CONFIG_DIR=./config/composer -WORKER_CONFIG_DIR=./config/worker -SPANDX_CONFIG=./config/spandx/local-frontend-and-api.js -COMPOSER_CLIENT_ID=rhsm-api -COMPOSER_CLIENT_SECRET=supersecretsecretsquirrel diff --git a/devel/.gitignore b/devel/.gitignore deleted file mode 100644 index 000f54a0..00000000 --- a/devel/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -state - -config/grafana/dashboards/*.json diff --git a/devel/README.md b/devel/README.md index 66fa4599..d124bddc 100644 --- a/devel/README.md +++ b/devel/README.md @@ -1,91 +1,3 @@ # devtools -Development Tools for Image Builder - -## Setup - -To start local development, first clone the image builder stack: - -```bash -git clone git@github.com:osbuild/osbuild.git -git clone git@github.com:osbuild/osbuild-composer.git -git clone git@github.com:osbuild/image-builder.git -git clone git@github.com:osbuild/image-builder-frontend.git -``` - -The folder structure should look like: - -``` -. -├── image-builder -├── image-builder-frontend -├── osbuild -└── osbuild-composer -``` - -Secondly redirect a few domains to localhost. One for each environment -of cloud.redhat.com that exists. You only need the ones you will be -developing against. If you are outside the Red Hat VPN, only `prod` is -available: - -```bash -echo "127.0.0.1 prod.foo.redhat.com" >> /etc/hosts -echo "127.0.0.1 qa.foo.redhat.com" >> /etc/hosts -echo "127.0.0.1 ci.foo.redhat.com" >> /etc/hosts -echo "127.0.0.1 stage.foo.redhat.com" >> /etc/hosts -``` - -Lastly run the setup tool from image-builder-frontend to generate TLS certs -and extract the Grafana dashboards from the image-builder and osbuild-composer -repos. - -```bash -cd image-builder-frontend/devel -./setup.sh -``` - -## Environment Variables - -For the Image Builder backend to upload successfully, you need to set some environment variables. For AWS, for example, set the following environment variables in the `.env` file: - -``` -OSBUILD_AWS_REGION -OSBUILD_AWS_ACCESS_KEY_ID -OSBUILD_AWS_SECRET_ACCESS_KEY -OSBUILD_AWS_S3_BUCKET -``` - -And then add to the list of environment variables for the backend container in the `docker-compose.yml` file. - -The config variables for the Image Builder backend can be found [here](https://github.com/osbuild/image-builder/blob/main/internal/config/config.go). - -## Run - -To build the containers run the following command: - -```bash -docker compose build # (or docker-compose build) -``` - -To run the containers: - -```bash -docker compose up # (or docker-compose up) -``` - -Note: As per the [docker compose cli](https://docs.docker.com/compose/reference/) docs, the new syntax for running docker compose changed from -`docker-compose` to `docker compose` - -Access the service through the GUI: -[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or -directly through the API: -[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder). - -The metrics containers are only launched when explicitly required. The command for this is below: - -```bash -docker compose --profile metrics up -``` - -Access the Grafana dashboard on [https://localhost:3000](https://localhost:3000). The default username is `admin` and the password is set to `foobar`. -The prometheus instance can be accessed on [https://localhost:9000](https://localhost:9000) +The docker backend stack has been moved to the [osbuild/getting-started](https://github.com/osbuild/osbuild-getting-started) repo. diff --git a/devel/config/backend/quotas.json b/devel/config/backend/quotas.json deleted file mode 100644 index c3a2a04d..00000000 --- a/devel/config/backend/quotas.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "default":{ - "quota":200, - "slidingWindow":1209600000000000 - } -} diff --git a/devel/config/composer/acl.yml b/devel/config/composer/acl.yml deleted file mode 100644 index 4d294744..00000000 --- a/devel/config/composer/acl.yml +++ /dev/null @@ -1,4 +0,0 @@ -## hack since oauth server is -## using http -- claim: typ - pattern: ^Bearer$ diff --git a/devel/config/composer/osbuild-composer.toml b/devel/config/composer/osbuild-composer.toml deleted file mode 100644 index ce12276a..00000000 --- a/devel/config/composer/osbuild-composer.toml +++ /dev/null @@ -1,20 +0,0 @@ -[worker] -request_job_timeout = "40s" -allowed_domains = [ "localhost", "worker.osbuild.org" ] -enable_mtls = false -enable_jwt = true -jwt_keys_urls = ["http://fauxauth:8888/certs"] -jwt_acl_file = "/etc/osbuild-composer/acl.yml" -jwt_tenant_provider_fields = ["rh-org-id"] -ca = "/etc/osbuild-composer/ca-crt.pem" -client_id = "rhsm-api" - -[koji] -allowed_domains = [ "client.osbuild.org" ] -enable_mtls = false -enable_jwt = true -jwt_keys_urls = ["http://fauxauth:8888/certs"] -jwt_acl_file = "/etc/osbuild-composer/acl.yml" -jwt_tenant_provider_fields = ["rh-org-id"] -ca = "/etc/osbuild-composer/ca-crt.pem" -client_id = "rhsm-api" diff --git a/devel/config/grafana/dashboards/dashboard.yml b/devel/config/grafana/dashboards/dashboard.yml deleted file mode 100644 index f0d5761f..00000000 --- a/devel/config/grafana/dashboards/dashboard.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: 1 -providers: - - name: 'grafana-dashboard-insights-image-builder-general' - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /etc/grafana/provisioning/dashboards diff --git a/devel/config/grafana/datasources/datasource.yml b/devel/config/grafana/datasources/datasource.yml deleted file mode 100644 index c9ca3a33..00000000 --- a/devel/config/grafana/datasources/datasource.yml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: 1 -datasources: - - access: 'proxy' - editable: true - isDefault: true - name: 'crcp01ue1-prometheus' - orgId: 1 - type: 'prometheus' - url: 'http://prometheus:9090' - - access: 'proxy' - editable: true - isDefault: false - name: 'app-sre-prod-04-prometheus' - orgId: 1 - type: 'prometheus' - url: 'http://prometheus:9090' diff --git a/devel/config/prometheus/prometheus.yml b/devel/config/prometheus/prometheus.yml deleted file mode 100644 index 04e9f288..00000000 --- a/devel/config/prometheus/prometheus.yml +++ /dev/null @@ -1,17 +0,0 @@ - -global: - scrape_interval: 15s -scrape_configs: - - job_name: 'image-builder' - scrape_interval: 5s - static_configs: - - targets: ['backend:8086'] - - job_name: 'composer' - scrape_interval: 5s - scheme: https - tls_config: - insecure_skip_verify: true - cert_file: /etc/image-builder/client-crt.pem - key_file: /etc/image-builder/client-key.pem - static_configs: - - targets: ['composer:8080'] diff --git a/devel/config/spandx/local-frontend-and-api.js b/devel/config/spandx/local-frontend-and-api.js deleted file mode 100644 index 33988fbc..00000000 --- a/devel/config/spandx/local-frontend-and-api.js +++ /dev/null @@ -1,11 +0,0 @@ -/*global module*/ - -const API_PORT = 8086; - -const routes = { - "/api/image-builder": { - host: `http://backend:${API_PORT}` - } -}; - -module.exports = { routes }; diff --git a/devel/config/worker/osbuild-worker.toml b/devel/config/worker/osbuild-worker.toml deleted file mode 100644 index f69ae454..00000000 --- a/devel/config/worker/osbuild-worker.toml +++ /dev/null @@ -1,4 +0,0 @@ -[authentication] -oauth_url = "http://fauxauth:8888/token?refresh_token=42" -client_secret = "/etc/osbuild-worker/secret" -client_id = "rhsm-api" diff --git a/devel/config/worker/secret b/devel/config/worker/secret deleted file mode 100644 index 602dc891..00000000 --- a/devel/config/worker/secret +++ /dev/null @@ -1 +0,0 @@ -supersecretsecretsquirrel diff --git a/devel/config/x509/openssl.cnf b/devel/config/x509/openssl.cnf deleted file mode 100644 index 7ff0d5cc..00000000 --- a/devel/config/x509/openssl.cnf +++ /dev/null @@ -1,85 +0,0 @@ -# -# ca options -# - -[ca] -default_ca = osbuild_ca - -[osbuild_ca] -database = ./index.txt -new_certs_dir = ./certs -rand_serial = yes - -certificate = ca.cert.pem -private_key = private/ca.key.pem - -default_days = 3650 -default_md = sha256 - -x509_extensions = osbuild_ca_ext - -# See WARNINGS in `man openssl ca`. This is ok, becasue it only copies -# extensions that are not already specified in `osbuild_ca_ext`. -copy_extensions = copy - -preserve = no -policy = osbuild_ca_policy - -# We want to issue multiple certificates with the same subject in the -# testing environment. -unique_subject = no - - -[osbuild_ca_ext] -basicConstraints = critical, CA:TRUE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always, issuer:always -keyUsage = critical, digitalSignature, cRLSign, keyCertSign - - -[osbuild_ca_policy] -commonName = supplied -emailAddress = supplied - - -# -# Extensions for server certificates -# - -[osbuild_server_ext] -basicConstraints = critical, CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid, issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = serverAuth - - -# -# Extensions for client certificates -# - -[osbuild_client_ext] -basicConstraints = CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer -keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth - - -# -# req options -# - -[req] -default_md = sha256 -default_bits = 2048 -distinguished_name = osbuild_distinguished_name - - -# -# Only prompt for CN -# - -[osbuild_distinguished_name] -CN = Common Name -emailAddress = E-Mail Address diff --git a/devel/docker-compose.yml b/devel/docker-compose.yml deleted file mode 100644 index d69bc56f..00000000 --- a/devel/docker-compose.yml +++ /dev/null @@ -1,165 +0,0 @@ -version: '3.9' -services: - composer: - image: local/osbuild-composer - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-ubi - entrypoint: - [ - "python3", - "/opt/entrypoint.py", - "--remote-worker-api", - "--composer-api" - ] - volumes: - - ${COMPOSER_CONFIG_DIR}/osbuild-composer.toml:/etc/osbuild-composer/osbuild-composer.toml:z - - ${COMPOSER_CONFIG_DIR}/acl.yml:/etc/osbuild-composer/acl.yml:z - - ${CERT_DIR}/ca-crt.pem:/etc/osbuild-composer/ca-crt.pem:z - - ${CERT_DIR}/composer-crt.pem:/etc/osbuild-composer/composer-crt.pem:z - - ${CERT_DIR}/composer-key.pem:/etc/osbuild-composer/composer-key.pem:z - ports: - - 8080:8080 - - 8700:8700 - networks: - net: - ipv4_address: 172.31.0.10 - worker: - image: local/osbuild-worker - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-worker - # override the entrypoint to specify composer hostname and port - entrypoint: [ "/usr/libexec/osbuild-composer/osbuild-worker", "composer:8700" ] - volumes: - - ${CERT_DIR}/ca-crt.pem:/etc/osbuild-composer/ca-crt.pem:z - - ${WORKER_CONFIG_DIR}/osbuild-worker.toml:/etc/osbuild-worker/osbuild-worker.toml:z - - ${WORKER_CONFIG_DIR}/secret:/etc/osbuild-worker/secret:z - environment: - - CACHE_DIRECTORY=/var/cache/osbuild-composer - privileged: true - cap_add: - - MKNOD - - SYS_ADMIN - - NET_ADMIN - depends_on: - - "composer" - restart: on-failure - networks: - net: - ipv4_address: 172.31.0.20 - postgres: - image: docker.io/postgres:10.5 - healthcheck: - test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ] - interval: 2s - timeout: 2s - retries: 10 - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - volumes: - - ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:Z - networks: - net: - ipv4_address: 172.31.0.30 - backend: - image: local/image-builder - build: - context: ../../image-builder - dockerfile: ./distribution/Dockerfile-ubi - ports: - - "8086:8086" - healthcheck: - test: [ "CMD", "curl", "localhost:8086/status" ] - interval: 2s - timeout: 2s - retries: 10 - volumes: - - ./config/backend/quotas.json:/config/quotas.json:z - - ${CERT_DIR}/ca-crt.pem:/etc/image-builder/ca-crt.pem:z - - ${CERT_DIR}/client-crt.pem:/etc/image-builder/client-crt.pem:z - - ${CERT_DIR}/client-key.pem:/etc/image-builder/client-key.pem:z - environment: - - LISTEN_ADDRESS=backend:8086 - - LOG_LEVEL=DEBUG - - ALLOWED_ORG_IDS=* - - PGHOST=postgres - - PGPORT=5432 - - PGDATABASE=postgres - - PGUSER=postgres - - PGPASSWORD=postgres - - COMPOSER_URL=https://composer:8080 - - COMPOSER_TOKEN_URL=http://fauxauth:8888/token?refresh_token=42 - - COMPOSER_CLIENT_SECRET=${COMPOSER_CLIENT_SECRET} - - COMPOSER_CLIENT_ID=${COMPOSER_CLIENT_ID} - - COMPOSER_CA_PATH=/etc/image-builder/ca-crt.pem - - DISTRIBUTIONS_DIR=/app/distributions - - QUOTA_FILE=/config/quotas.json - depends_on: - - "composer" - - "postgres" - restart: on-failure - networks: - net: - ipv4_address: 172.31.0.40 - frontend: - image: local/image-builder-frontend - ports: - - "1337:1337" - build: - context: ../../image-builder-frontend - dockerfile: ./distribution/Dockerfile - environment: - - HOST=frontend - networks: - net: - ipv4_address: 172.31.0.50 - prometheus: - profiles: ["metrics"] - image: prom/prometheus:latest - ports: - - "9000:9090" - volumes: - - ./config/prometheus:/config - - ${CERT_DIR}/ca-crt.pem:/etc/image-builder/ca-crt.pem:z - - ${CERT_DIR}/client-crt.pem:/etc/image-builder/client-crt.pem:z - - ${CERT_DIR}/client-key.pem:/etc/image-builder/client-key.pem:z - restart: unless-stopped - command: - - "--config.file=/config/prometheus.yml" - networks: - net: - ipv4_address: 172.31.0.60 - grafana: - profiles: ["metrics"] - image: grafana/grafana:latest - ports: - - "3000:3000" - volumes: - - ./config/grafana:/etc/grafana/provisioning/ - restart: unless-stopped - environment: - - GF_SECURITY_ADMIN_PASSWORD=foobar - networks: - net: - ipv4_address: 172.31.0.70 - fauxauth: - image: local/osbuild-fauxauth - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-fauxauth - entrypoint: [ "/opt/fauxauth.py", "-a", "0.0.0.0", "-p", "8888" ] - volumes: - - ${CERT_DIR}/:/etc/osbuild-composer/:z - ports: - - "8888:8888" - networks: - net: - ipv4_address: 172.31.0.80 -networks: - net: - ipam: - driver: default - config: - - subnet: 172.31.0.0/16 diff --git a/devel/gen-certs.sh b/devel/gen-certs.sh deleted file mode 100755 index c6032799..00000000 --- a/devel/gen-certs.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -if (( $# != 3 )); then - echo "Usage: $0 " - echo - echo "Positional arguments" - echo " OpenSSL configuration file" - echo " Destination directory for the generated files" - echo " Working directory for the generation process" - exit 1 -fi - -set -euxo pipefail -# Generate all X.509 certificates for the tests -# The whole generation is done in a $CADIR to better represent how osbuild-ca -# it. -OPENSSL_CONFIG="$1" -CERTDIR="$2" -CADIR="$3" - -# The $CADIR might exist from a previous test (current Schutzbot's imperfection) -rm -rf "$CADIR" || true -mkdir -p "$CADIR" "$CERTDIR" - -# Convert the arguments to real paths so we can safely change working directory -OPENSSL_CONFIG="$(realpath "${OPENSSL_CONFIG}")" -CERTDIR="$(realpath "${CERTDIR}")" -CADIR="$(realpath "${CADIR}")" - -pushd "$CADIR" - mkdir certs private - touch index.txt - - # Generate a CA. - openssl req -config "$OPENSSL_CONFIG" \ - -keyout private/ca.key.pem \ - -new -nodes -x509 -extensions osbuild_ca_ext \ - -out ca.cert.pem -subj "/CN=osbuild.org" - - # Copy the private key to the location expected by the tests - cp ca.cert.pem "$CERTDIR"/ca-crt.pem - - # Generate a composer certificate. - openssl req -config "$OPENSSL_CONFIG" \ - -keyout "$CERTDIR"/composer-key.pem \ - -new -nodes \ - -out /tmp/composer-csr.pem \ - -subj "/CN=localhost/emailAddress=osbuild@example.com" \ - -addext "subjectAltName=DNS:localhost, DNS:composer" - - openssl ca -batch -config "$OPENSSL_CONFIG" \ - -extensions osbuild_server_ext \ - -in /tmp/composer-csr.pem \ - -out "$CERTDIR"/composer-crt.pem - - # Generate a worker certificate. - openssl req -config "$OPENSSL_CONFIG" \ - -keyout "$CERTDIR"/worker-key.pem \ - -new -nodes \ - -out /tmp/worker-csr.pem \ - -subj "/CN=localhost/emailAddress=osbuild@example.com" \ - -addext "subjectAltName=DNS:localhost, DNS:worker" - - openssl ca -batch -config "$OPENSSL_CONFIG" \ - -extensions osbuild_client_ext \ - -in /tmp/worker-csr.pem \ - -out "$CERTDIR"/worker-crt.pem - - # Generate a client certificate. - openssl req -config "$OPENSSL_CONFIG" \ - -keyout "$CERTDIR"/client-key.pem \ - -new -nodes \ - -out /tmp/client-csr.pem \ - -subj "/CN=client.osbuild.org/emailAddress=osbuild@example.com" \ - -addext "subjectAltName=DNS:client.osbuild.org" - - openssl ca -batch -config "$OPENSSL_CONFIG" \ - -extensions osbuild_client_ext \ - -in /tmp/client-csr.pem \ - -out "$CERTDIR"/client-crt.pem - - # Client keys are used by tests to access the composer APIs. Allow all users access. - chmod 644 "$CERTDIR"/client-key.pem - - # Generate a kojihub certificate. - openssl req -config "$OPENSSL_CONFIG" \ - -keyout "$CERTDIR"/kojihub-key.pem \ - -new -nodes \ - -out /tmp/kojihub-csr.pem \ - -subj "/CN=localhost/emailAddress=osbuild@example.com" \ - -addext "subjectAltName=DNS:localhost" - - openssl ca -batch -config "$OPENSSL_CONFIG" \ - -extensions osbuild_server_ext \ - -in /tmp/kojihub-csr.pem \ - -out "$CERTDIR"/kojihub-crt.pem - -popd diff --git a/devel/gen-dashboards b/devel/gen-dashboards deleted file mode 100755 index c88c2897..00000000 --- a/devel/gen-dashboards +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 -import argparse, sys, yaml - -TEMPLATE_DIR="../../image-builder/templates/dashboards/grafana-dashboard-insights-image-builder-general.configmap.yml" -OUTPUT_DIR="config/grafana/dashboards/" - -def load_config(config_filepath): - with open(config_filepath, "r") as stream: - try: - return yaml.safe_load(stream)['data']['grafana.json'] - except yaml.YAMLError as e: - print("Error parsing configmap: {}\n".format(str(e))) - sys.exit(1) - -def write_dashboard(dashboard, output_filepath): - with open(output_filepath, 'w') as f: - try: - f.write(dashboard) - f.close() - except Exception as e: - print("Error saving dashboard: {}\n".format(str(e))) - sys.exit(1) - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input", help="Path to the dashboard configmap", type=str) - parser.add_argument("-o", "--output", help="File path of the output", type=str) - args = parser.parse_args() - write_dashboard(load_config(args.input), args.output) - -if __name__ == "__main__": - main() diff --git a/devel/setup.sh b/devel/setup.sh deleted file mode 100755 index bf9c3c84..00000000 --- a/devel/setup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -./gen-certs.sh \ - config/x509/openssl.cnf \ - state/x509 \ - state/x509/ca - -# image-builder dashboard -./gen-dashboards \ - --input ../../image-builder/templates/dashboards/grafana-dashboard-insights-image-builder-general.configmap.yml \ - --output ./config/grafana/dashboards/insights-dashboard.json - -# composer dashboard -./gen-dashboards \ - --input ../../osbuild-composer/templates/dashboards/grafana-dashboard-image-builder-composer-general.configmap.yml \ - --output ./config/grafana/dashboards/composer-dashboard.json