devel: mv to osbuild/getting-started

Move the docker backend to the `getting-started` repo in `osbuild` since
it seems like a more appropriate place for the backend stack. Add a note
to the README to point users to the new repository.
This commit is contained in:
Gianluca Zuccarelli 2023-09-29 11:24:15 +01:00 committed by Thomas Lavocat
parent 5a34506eb6
commit 44e2a9519b
17 changed files with 1 additions and 583 deletions

View file

@ -1,7 +0,0 @@
COMPOSE_PROJECT_NAME=image-builder
CERT_DIR=./state/x509
COMPOSER_CONFIG_DIR=./config/composer
WORKER_CONFIG_DIR=./config/worker
SPANDX_CONFIG=./config/spandx/local-frontend-and-api.js
COMPOSER_CLIENT_ID=rhsm-api
COMPOSER_CLIENT_SECRET=supersecretsecretsquirrel

3
devel/.gitignore vendored
View file

@ -1,3 +0,0 @@
state
config/grafana/dashboards/*.json

View file

@ -1,91 +1,3 @@
# devtools # devtools
Development Tools for Image Builder The docker backend stack has been moved to the [osbuild/getting-started](https://github.com/osbuild/osbuild-getting-started) repo.
## Setup
To start local development, first clone the image builder stack:
```bash
git clone git@github.com:osbuild/osbuild.git
git clone git@github.com:osbuild/osbuild-composer.git
git clone git@github.com:osbuild/image-builder.git
git clone git@github.com:osbuild/image-builder-frontend.git
```
The folder structure should look like:
```
.
├── image-builder
├── image-builder-frontend
├── osbuild
└── osbuild-composer
```
Secondly redirect a few domains to localhost. One for each environment
of cloud.redhat.com that exists. You only need the ones you will be
developing against. If you are outside the Red Hat VPN, only `prod` is
available:
```bash
echo "127.0.0.1 prod.foo.redhat.com" >> /etc/hosts
echo "127.0.0.1 qa.foo.redhat.com" >> /etc/hosts
echo "127.0.0.1 ci.foo.redhat.com" >> /etc/hosts
echo "127.0.0.1 stage.foo.redhat.com" >> /etc/hosts
```
Lastly run the setup tool from image-builder-frontend to generate TLS certs
and extract the Grafana dashboards from the image-builder and osbuild-composer
repos.
```bash
cd image-builder-frontend/devel
./setup.sh
```
## Environment Variables
For the Image Builder backend to upload successfully, you need to set some environment variables. For AWS, for example, set the following environment variables in the `.env` file:
```
OSBUILD_AWS_REGION
OSBUILD_AWS_ACCESS_KEY_ID
OSBUILD_AWS_SECRET_ACCESS_KEY
OSBUILD_AWS_S3_BUCKET
```
And then add to the list of environment variables for the backend container in the `docker-compose.yml` file.
The config variables for the Image Builder backend can be found [here](https://github.com/osbuild/image-builder/blob/main/internal/config/config.go).
## Run
To build the containers run the following command:
```bash
docker compose build # (or docker-compose build)
```
To run the containers:
```bash
docker compose up # (or docker-compose up)
```
Note: As per the [docker compose cli](https://docs.docker.com/compose/reference/) docs, the new syntax for running docker compose changed from
`docker-compose` to `docker compose`
Access the service through the GUI:
[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or
directly through the API:
[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder).
The metrics containers are only launched when explicitly required. The command for this is below:
```bash
docker compose --profile metrics up
```
Access the Grafana dashboard on [https://localhost:3000](https://localhost:3000). The default username is `admin` and the password is set to `foobar`.
The prometheus instance can be accessed on [https://localhost:9000](https://localhost:9000)

View file

@ -1,6 +0,0 @@
{
"default":{
"quota":200,
"slidingWindow":1209600000000000
}
}

View file

@ -1,4 +0,0 @@
## hack since oauth server is
## using http
- claim: typ
pattern: ^Bearer$

View file

@ -1,20 +0,0 @@
[worker]
request_job_timeout = "40s"
allowed_domains = [ "localhost", "worker.osbuild.org" ]
enable_mtls = false
enable_jwt = true
jwt_keys_urls = ["http://fauxauth:8888/certs"]
jwt_acl_file = "/etc/osbuild-composer/acl.yml"
jwt_tenant_provider_fields = ["rh-org-id"]
ca = "/etc/osbuild-composer/ca-crt.pem"
client_id = "rhsm-api"
[koji]
allowed_domains = [ "client.osbuild.org" ]
enable_mtls = false
enable_jwt = true
jwt_keys_urls = ["http://fauxauth:8888/certs"]
jwt_acl_file = "/etc/osbuild-composer/acl.yml"
jwt_tenant_provider_fields = ["rh-org-id"]
ca = "/etc/osbuild-composer/ca-crt.pem"
client_id = "rhsm-api"

View file

@ -1,10 +0,0 @@
apiVersion: 1
providers:
- name: 'grafana-dashboard-insights-image-builder-general'
orgId: 1
folder: ''
type: file
disableDeletion: false
editable: true
options:
path: /etc/grafana/provisioning/dashboards

View file

@ -1,16 +0,0 @@
apiVersion: 1
datasources:
- access: 'proxy'
editable: true
isDefault: true
name: 'crcp01ue1-prometheus'
orgId: 1
type: 'prometheus'
url: 'http://prometheus:9090'
- access: 'proxy'
editable: true
isDefault: false
name: 'app-sre-prod-04-prometheus'
orgId: 1
type: 'prometheus'
url: 'http://prometheus:9090'

View file

@ -1,17 +0,0 @@
global:
scrape_interval: 15s
scrape_configs:
- job_name: 'image-builder'
scrape_interval: 5s
static_configs:
- targets: ['backend:8086']
- job_name: 'composer'
scrape_interval: 5s
scheme: https
tls_config:
insecure_skip_verify: true
cert_file: /etc/image-builder/client-crt.pem
key_file: /etc/image-builder/client-key.pem
static_configs:
- targets: ['composer:8080']

View file

@ -1,11 +0,0 @@
/*global module*/
const API_PORT = 8086;
const routes = {
"/api/image-builder": {
host: `http://backend:${API_PORT}`
}
};
module.exports = { routes };

View file

@ -1,4 +0,0 @@
[authentication]
oauth_url = "http://fauxauth:8888/token?refresh_token=42"
client_secret = "/etc/osbuild-worker/secret"
client_id = "rhsm-api"

View file

@ -1 +0,0 @@
supersecretsecretsquirrel

View file

@ -1,85 +0,0 @@
#
# ca options
#
[ca]
default_ca = osbuild_ca
[osbuild_ca]
database = ./index.txt
new_certs_dir = ./certs
rand_serial = yes
certificate = ca.cert.pem
private_key = private/ca.key.pem
default_days = 3650
default_md = sha256
x509_extensions = osbuild_ca_ext
# See WARNINGS in `man openssl ca`. This is ok, becasue it only copies
# extensions that are not already specified in `osbuild_ca_ext`.
copy_extensions = copy
preserve = no
policy = osbuild_ca_policy
# We want to issue multiple certificates with the same subject in the
# testing environment.
unique_subject = no
[osbuild_ca_ext]
basicConstraints = critical, CA:TRUE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer:always
keyUsage = critical, digitalSignature, cRLSign, keyCertSign
[osbuild_ca_policy]
commonName = supplied
emailAddress = supplied
#
# Extensions for server certificates
#
[osbuild_server_ext]
basicConstraints = critical, CA:FALSE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid, issuer:always
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
#
# Extensions for client certificates
#
[osbuild_client_ext]
basicConstraints = CA:FALSE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth
#
# req options
#
[req]
default_md = sha256
default_bits = 2048
distinguished_name = osbuild_distinguished_name
#
# Only prompt for CN
#
[osbuild_distinguished_name]
CN = Common Name
emailAddress = E-Mail Address

View file

@ -1,165 +0,0 @@
version: '3.9'
services:
composer:
image: local/osbuild-composer
build:
context: ../../osbuild-composer
dockerfile: ./distribution/Dockerfile-ubi
entrypoint:
[
"python3",
"/opt/entrypoint.py",
"--remote-worker-api",
"--composer-api"
]
volumes:
- ${COMPOSER_CONFIG_DIR}/osbuild-composer.toml:/etc/osbuild-composer/osbuild-composer.toml:z
- ${COMPOSER_CONFIG_DIR}/acl.yml:/etc/osbuild-composer/acl.yml:z
- ${CERT_DIR}/ca-crt.pem:/etc/osbuild-composer/ca-crt.pem:z
- ${CERT_DIR}/composer-crt.pem:/etc/osbuild-composer/composer-crt.pem:z
- ${CERT_DIR}/composer-key.pem:/etc/osbuild-composer/composer-key.pem:z
ports:
- 8080:8080
- 8700:8700
networks:
net:
ipv4_address: 172.31.0.10
worker:
image: local/osbuild-worker
build:
context: ../../osbuild-composer
dockerfile: ./distribution/Dockerfile-worker
# override the entrypoint to specify composer hostname and port
entrypoint: [ "/usr/libexec/osbuild-composer/osbuild-worker", "composer:8700" ]
volumes:
- ${CERT_DIR}/ca-crt.pem:/etc/osbuild-composer/ca-crt.pem:z
- ${WORKER_CONFIG_DIR}/osbuild-worker.toml:/etc/osbuild-worker/osbuild-worker.toml:z
- ${WORKER_CONFIG_DIR}/secret:/etc/osbuild-worker/secret:z
environment:
- CACHE_DIRECTORY=/var/cache/osbuild-composer
privileged: true
cap_add:
- MKNOD
- SYS_ADMIN
- NET_ADMIN
depends_on:
- "composer"
restart: on-failure
networks:
net:
ipv4_address: 172.31.0.20
postgres:
image: docker.io/postgres:10.5
healthcheck:
test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ]
interval: 2s
timeout: 2s
retries: 10
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
volumes:
- ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:Z
networks:
net:
ipv4_address: 172.31.0.30
backend:
image: local/image-builder
build:
context: ../../image-builder
dockerfile: ./distribution/Dockerfile-ubi
ports:
- "8086:8086"
healthcheck:
test: [ "CMD", "curl", "localhost:8086/status" ]
interval: 2s
timeout: 2s
retries: 10
volumes:
- ./config/backend/quotas.json:/config/quotas.json:z
- ${CERT_DIR}/ca-crt.pem:/etc/image-builder/ca-crt.pem:z
- ${CERT_DIR}/client-crt.pem:/etc/image-builder/client-crt.pem:z
- ${CERT_DIR}/client-key.pem:/etc/image-builder/client-key.pem:z
environment:
- LISTEN_ADDRESS=backend:8086
- LOG_LEVEL=DEBUG
- ALLOWED_ORG_IDS=*
- PGHOST=postgres
- PGPORT=5432
- PGDATABASE=postgres
- PGUSER=postgres
- PGPASSWORD=postgres
- COMPOSER_URL=https://composer:8080
- COMPOSER_TOKEN_URL=http://fauxauth:8888/token?refresh_token=42
- COMPOSER_CLIENT_SECRET=${COMPOSER_CLIENT_SECRET}
- COMPOSER_CLIENT_ID=${COMPOSER_CLIENT_ID}
- COMPOSER_CA_PATH=/etc/image-builder/ca-crt.pem
- DISTRIBUTIONS_DIR=/app/distributions
- QUOTA_FILE=/config/quotas.json
depends_on:
- "composer"
- "postgres"
restart: on-failure
networks:
net:
ipv4_address: 172.31.0.40
frontend:
image: local/image-builder-frontend
ports:
- "1337:1337"
build:
context: ../../image-builder-frontend
dockerfile: ./distribution/Dockerfile
environment:
- HOST=frontend
networks:
net:
ipv4_address: 172.31.0.50
prometheus:
profiles: ["metrics"]
image: prom/prometheus:latest
ports:
- "9000:9090"
volumes:
- ./config/prometheus:/config
- ${CERT_DIR}/ca-crt.pem:/etc/image-builder/ca-crt.pem:z
- ${CERT_DIR}/client-crt.pem:/etc/image-builder/client-crt.pem:z
- ${CERT_DIR}/client-key.pem:/etc/image-builder/client-key.pem:z
restart: unless-stopped
command:
- "--config.file=/config/prometheus.yml"
networks:
net:
ipv4_address: 172.31.0.60
grafana:
profiles: ["metrics"]
image: grafana/grafana:latest
ports:
- "3000:3000"
volumes:
- ./config/grafana:/etc/grafana/provisioning/
restart: unless-stopped
environment:
- GF_SECURITY_ADMIN_PASSWORD=foobar
networks:
net:
ipv4_address: 172.31.0.70
fauxauth:
image: local/osbuild-fauxauth
build:
context: ../../osbuild-composer
dockerfile: ./distribution/Dockerfile-fauxauth
entrypoint: [ "/opt/fauxauth.py", "-a", "0.0.0.0", "-p", "8888" ]
volumes:
- ${CERT_DIR}/:/etc/osbuild-composer/:z
ports:
- "8888:8888"
networks:
net:
ipv4_address: 172.31.0.80
networks:
net:
ipam:
driver: default
config:
- subnet: 172.31.0.0/16

View file

@ -1,97 +0,0 @@
#!/bin/bash
if (( $# != 3 )); then
echo "Usage: $0 <openssl-config> <certdir> <cadir>"
echo
echo "Positional arguments"
echo " <openssl-config> OpenSSL configuration file"
echo " <certdir> Destination directory for the generated files"
echo " <cadir> Working directory for the generation process"
exit 1
fi
set -euxo pipefail
# Generate all X.509 certificates for the tests
# The whole generation is done in a $CADIR to better represent how osbuild-ca
# it.
OPENSSL_CONFIG="$1"
CERTDIR="$2"
CADIR="$3"
# The $CADIR might exist from a previous test (current Schutzbot's imperfection)
rm -rf "$CADIR" || true
mkdir -p "$CADIR" "$CERTDIR"
# Convert the arguments to real paths so we can safely change working directory
OPENSSL_CONFIG="$(realpath "${OPENSSL_CONFIG}")"
CERTDIR="$(realpath "${CERTDIR}")"
CADIR="$(realpath "${CADIR}")"
pushd "$CADIR"
mkdir certs private
touch index.txt
# Generate a CA.
openssl req -config "$OPENSSL_CONFIG" \
-keyout private/ca.key.pem \
-new -nodes -x509 -extensions osbuild_ca_ext \
-out ca.cert.pem -subj "/CN=osbuild.org"
# Copy the private key to the location expected by the tests
cp ca.cert.pem "$CERTDIR"/ca-crt.pem
# Generate a composer certificate.
openssl req -config "$OPENSSL_CONFIG" \
-keyout "$CERTDIR"/composer-key.pem \
-new -nodes \
-out /tmp/composer-csr.pem \
-subj "/CN=localhost/emailAddress=osbuild@example.com" \
-addext "subjectAltName=DNS:localhost, DNS:composer"
openssl ca -batch -config "$OPENSSL_CONFIG" \
-extensions osbuild_server_ext \
-in /tmp/composer-csr.pem \
-out "$CERTDIR"/composer-crt.pem
# Generate a worker certificate.
openssl req -config "$OPENSSL_CONFIG" \
-keyout "$CERTDIR"/worker-key.pem \
-new -nodes \
-out /tmp/worker-csr.pem \
-subj "/CN=localhost/emailAddress=osbuild@example.com" \
-addext "subjectAltName=DNS:localhost, DNS:worker"
openssl ca -batch -config "$OPENSSL_CONFIG" \
-extensions osbuild_client_ext \
-in /tmp/worker-csr.pem \
-out "$CERTDIR"/worker-crt.pem
# Generate a client certificate.
openssl req -config "$OPENSSL_CONFIG" \
-keyout "$CERTDIR"/client-key.pem \
-new -nodes \
-out /tmp/client-csr.pem \
-subj "/CN=client.osbuild.org/emailAddress=osbuild@example.com" \
-addext "subjectAltName=DNS:client.osbuild.org"
openssl ca -batch -config "$OPENSSL_CONFIG" \
-extensions osbuild_client_ext \
-in /tmp/client-csr.pem \
-out "$CERTDIR"/client-crt.pem
# Client keys are used by tests to access the composer APIs. Allow all users access.
chmod 644 "$CERTDIR"/client-key.pem
# Generate a kojihub certificate.
openssl req -config "$OPENSSL_CONFIG" \
-keyout "$CERTDIR"/kojihub-key.pem \
-new -nodes \
-out /tmp/kojihub-csr.pem \
-subj "/CN=localhost/emailAddress=osbuild@example.com" \
-addext "subjectAltName=DNS:localhost"
openssl ca -batch -config "$OPENSSL_CONFIG" \
-extensions osbuild_server_ext \
-in /tmp/kojihub-csr.pem \
-out "$CERTDIR"/kojihub-crt.pem
popd

View file

@ -1,32 +0,0 @@
#!/usr/bin/env python3
import argparse, sys, yaml
TEMPLATE_DIR="../../image-builder/templates/dashboards/grafana-dashboard-insights-image-builder-general.configmap.yml"
OUTPUT_DIR="config/grafana/dashboards/"
def load_config(config_filepath):
with open(config_filepath, "r") as stream:
try:
return yaml.safe_load(stream)['data']['grafana.json']
except yaml.YAMLError as e:
print("Error parsing configmap: {}\n".format(str(e)))
sys.exit(1)
def write_dashboard(dashboard, output_filepath):
with open(output_filepath, 'w') as f:
try:
f.write(dashboard)
f.close()
except Exception as e:
print("Error saving dashboard: {}\n".format(str(e)))
sys.exit(1)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", help="Path to the dashboard configmap", type=str)
parser.add_argument("-o", "--output", help="File path of the output", type=str)
args = parser.parse_args()
write_dashboard(load_config(args.input), args.output)
if __name__ == "__main__":
main()

View file

@ -1,16 +0,0 @@
#!/bin/bash
./gen-certs.sh \
config/x509/openssl.cnf \
state/x509 \
state/x509/ca
# image-builder dashboard
./gen-dashboards \
--input ../../image-builder/templates/dashboards/grafana-dashboard-insights-image-builder-general.configmap.yml \
--output ./config/grafana/dashboards/insights-dashboard.json
# composer dashboard
./gen-dashboards \
--input ../../osbuild-composer/templates/dashboards/grafana-dashboard-image-builder-composer-general.configmap.yml \
--output ./config/grafana/dashboards/composer-dashboard.json