d3cd3197c0
Currently liveness and readiness was treated the same. However, their behaviour at shutdown is meant to be different. When a service is not read no new connections are made to it, and when a service is not live it can be cleaned up. By considering our service live if and only if it listens to HTTP requests we don't have the opportunity to clean up after we stop listening to new requests. Leave readiness probes as they are, and instead use a file in the filesystem to indicate when the service is live. It is created before composer is spawned and deleted once composer exits.
391 lines
12 KiB
YAML
391 lines
12 KiB
YAML
apiVersion: v1
|
|
kind: Template
|
|
metadata:
|
|
name: composer
|
|
annotations:
|
|
openshift.io/display-name: Image-Builder composer service
|
|
description: Composer component of the image-builder serivce
|
|
tags: golang
|
|
iconClass: icon-shadowman
|
|
template.openshift.io/provider-display-name: Red Hat, Inc.
|
|
labels:
|
|
template: composer
|
|
objects:
|
|
- apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
labels:
|
|
service: image-builder
|
|
name: composer
|
|
spec:
|
|
replicas: 3
|
|
selector:
|
|
matchLabels:
|
|
app: composer
|
|
strategy:
|
|
# Update pods 1 at a time
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
# Create at most 0 extra pod over .spec.replicas
|
|
maxSurge: 0
|
|
# At all times there should be .spec.replicas - 1 available
|
|
maxUnavailable: 1
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: composer
|
|
spec:
|
|
serviceAccountName: image-builder
|
|
containers:
|
|
- image: "${IMAGE_NAME}:${IMAGE_TAG}"
|
|
name: composer
|
|
livenessProbe:
|
|
failureThreshold: 3
|
|
exec:
|
|
command:
|
|
- cat
|
|
- /tmp/live
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 1
|
|
readinessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: ${READINESS_URI}
|
|
port: ${{COMPOSER_API_PORT}}
|
|
scheme: HTTP
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 1
|
|
resources:
|
|
requests:
|
|
cpu: "${CPU_REQUEST}"
|
|
memory: "${MEMORY_REQUEST}"
|
|
limits:
|
|
cpu: "${CPU_LIMIT}"
|
|
memory: "${MEMORY_LIMIT}"
|
|
env:
|
|
- name: PGHOST
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.host
|
|
- name: PGPORT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.port
|
|
- name: PGDATABASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.name
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.user
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.password
|
|
- name: PGSSLMODE
|
|
value: "${PGSSLMODE}"
|
|
- name: PGMAXCONNS
|
|
value: "${PGMAXCONNS}"
|
|
ports:
|
|
- name: composer-api
|
|
protocol: TCP
|
|
containerPort: ${{COMPOSER_API_PORT}}
|
|
- name: worker-api
|
|
protocol: TCP
|
|
containerPort: ${{WORKER_API_PORT}}
|
|
volumeMounts:
|
|
- name: composer-config
|
|
mountPath: "${COMPOSER_CONFIG_DIR}"
|
|
readOnly: true
|
|
- name: state-directory
|
|
mountPath: "/var/lib/osbuild-composer"
|
|
- name: cache-directory
|
|
mountPath: "/var/cache/osbuild-composer"
|
|
volumes:
|
|
- name: composer-config
|
|
configMap:
|
|
name: composer-config
|
|
- name: db-secrets
|
|
secret:
|
|
secretName: db
|
|
- name: state-directory
|
|
emptyDir: {}
|
|
- name: cache-directory
|
|
emptyDir: {}
|
|
initContainers:
|
|
- name: composer-migrate
|
|
image: "${IMAGE_NAME}:${IMAGE_TAG}"
|
|
command: [ "/opt/migrate/tern", "migrate", "-m", "/opt/migrate/schemas" ]
|
|
env:
|
|
- name: PGHOST
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.host
|
|
- name: PGPORT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.port
|
|
- name: PGDATABASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.name
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.user
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.password
|
|
- name: PGSSLMODE
|
|
value: "${PGSSLMODE}"
|
|
|
|
- apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: image-builder
|
|
imagePullSecrets:
|
|
- name: quay-cloudservices-pull
|
|
|
|
- apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: image-builder-composer
|
|
labels:
|
|
app: composer
|
|
port: composer-api
|
|
spec:
|
|
ports:
|
|
- name: composer-api
|
|
protocol: TCP
|
|
port: 80
|
|
targetPort: ${{COMPOSER_API_PORT}}
|
|
selector:
|
|
app: composer
|
|
|
|
- apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: image-builder-worker
|
|
labels:
|
|
app: composer
|
|
port: worker-api
|
|
spec:
|
|
ports:
|
|
- name: worker-api
|
|
protocol: TCP
|
|
port: 80
|
|
targetPort: ${{WORKER_API_PORT}}
|
|
selector:
|
|
app: composer
|
|
|
|
# This map should probably move to app-intf
|
|
- apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: composer-config
|
|
data:
|
|
acl.yml: |
|
|
- claim: user_id
|
|
pattern: ^(54629121|54629180|54597799|54676085|54968801)$
|
|
- claim: rh-org-id
|
|
pattern: ^(13826359|15842261|15877963)$
|
|
- claim: account_id
|
|
pattern: ^(15842261)$
|
|
osbuild-composer.toml: |
|
|
log_level = "info"
|
|
[koji]
|
|
enable_tls = false
|
|
enable_mtls = false
|
|
enable_jwt = true
|
|
jwt_keys_urls = ["${RH_SSO_BASE_URL}/protocol/openid-connect/certs", "${MAS_SSO_BASE_URL}/protocol/openid-connect/certs"]
|
|
jwt_acl_file = "${COMPOSER_CONFIG_DIR}/acl.yml"
|
|
jwt_tenant_provider_fields = ["rh-org-id", "account_id"]
|
|
[koji.aws_config]
|
|
bucket = "${COMPOSER_CONFIG_BUCKET_NAME}"
|
|
[worker]
|
|
request_job_timeout = "20s"
|
|
base_path = "/api/image-builder-worker/v1"
|
|
enable_artifacts = false
|
|
enable_tls = false
|
|
enable_mtls = false
|
|
enable_jwt = true
|
|
jwt_keys_urls = ["${RH_SSO_BASE_URL}/protocol/openid-connect/certs", "${MAS_SSO_BASE_URL}/protocol/openid-connect/certs"]
|
|
jwt_acl_file = "${COMPOSER_CONFIG_DIR}/acl.yml"
|
|
jwt_tenant_provider_fields = ["rh-org-id", "account_id"]
|
|
|
|
- apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
labels:
|
|
service: image-builder
|
|
name: composer-maintenance
|
|
spec:
|
|
# run maintenance job at midnight
|
|
schedule: 0 0 * * *
|
|
concurrencyPolicy: Forbid
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
spec:
|
|
serviceAccountName: image-builder
|
|
restartPolicy: Never
|
|
containers:
|
|
- image: "${MAINTENANCE_IMAGE_NAME}:${IMAGE_TAG}"
|
|
name: composer-maintenance
|
|
resources:
|
|
requests:
|
|
cpu: "${CPU_REQUEST}"
|
|
memory: "${MEMORY_REQUEST}"
|
|
limits:
|
|
cpu: "${CPU_LIMIT}"
|
|
memory: "${MEMORY_LIMIT}"
|
|
env:
|
|
- name: GCP_AUTH_PROVIDER_X509_CERT_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: auth_provider_x509_cert_url
|
|
- name: GCP_AUTH_URI
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: auth_uri
|
|
- name: GCP_CLIENT_EMAIL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: client_email
|
|
- name: GCP_CLIENT_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: client_id
|
|
- name: GCP_CLIENT_X509_CERT_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: client_x509_cert_url
|
|
- name: GCP_PRIVATE_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: private_key
|
|
- name: GCP_PRIVATE_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: private_key_id
|
|
- name: GCP_PROJECT_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: project_id
|
|
- name: GCP_TOKEN_URI
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: token_uri
|
|
- name: GCP_TYPE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gcp-service-account
|
|
key: type
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: aws-account
|
|
key: access_key_id
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: aws-account
|
|
key: secret_access_key
|
|
- name: DRY_RUN
|
|
value: "${MAINTENANCE_DRY_RUN}"
|
|
- name: MAX_CONCURRENT_REQUESTS
|
|
value: "${MAINTENANCE_MAX_CONCURRENT_REQUESTS}"
|
|
|
|
parameters:
|
|
- description: composer image name
|
|
name: IMAGE_NAME
|
|
value: quay.io/app-sre/composer
|
|
required: true
|
|
- description: composer image tag
|
|
name: IMAGE_TAG
|
|
required: true
|
|
- description: postgres sslmode to use when connecting to the db
|
|
name: PGSSLMODE
|
|
value: "require"
|
|
- description: postgres maximum connections per pod
|
|
name: PGMAXCONNS
|
|
value: "20"
|
|
- description: base sso url
|
|
name: RH_SSO_BASE_URL
|
|
required: true
|
|
value: "https://sso.redhat.com/auth/realms/redhat-external"
|
|
- description: base sso url
|
|
name: MAS_SSO_BASE_URL
|
|
required: true
|
|
value: "https://identity.api.openshift.com/auth/realms/rhoas"
|
|
- description: base sso url
|
|
name: COMPOSER_CONFIG_DIR
|
|
required: true
|
|
value: "/etc/osbuild-composer"
|
|
- description: Bucket to store aws artifacts
|
|
name: COMPOSER_CONFIG_BUCKET_NAME
|
|
required: true
|
|
value: "imagebuilder.service.staging"
|
|
- description: composer-api port
|
|
name: COMPOSER_API_PORT
|
|
required: true
|
|
value: "8080"
|
|
- description: worker-api port
|
|
name: WORKER_API_PORT
|
|
required: true
|
|
value: "8700"
|
|
- name: LIVENESS_URI
|
|
description: URI to query for the liveness check
|
|
value: "/api/image-builder-composer/v2/openapi"
|
|
- name: READINESS_URI
|
|
description: URI to query for the readiness check
|
|
value: "/api/image-builder-composer/v2/openapi"
|
|
- name: CPU_REQUEST
|
|
description: CPU request per container
|
|
value: "200m"
|
|
- name: CPU_LIMIT
|
|
description: CPU limit per container
|
|
value: "1"
|
|
- name: MEMORY_REQUEST
|
|
description: Memory request per container
|
|
value: "256Mi"
|
|
- name: MEMORY_LIMIT
|
|
description: Memory limit per container
|
|
value: "512Mi"
|
|
# maintenance image variables
|
|
- description: composer-maintenance image name
|
|
name: MAINTENANCE_IMAGE_NAME
|
|
value: quay.io/app-sre/composer-maintenance
|
|
required: true
|
|
- description: composer-maintenance dry run
|
|
name: MAINTENANCE_DRY_RUN
|
|
# don't change this value, overwrite it in app-interface for a specific namespace
|
|
value: "true"
|
|
required: true
|
|
- description: composer-maintenance max concurrent requests
|
|
name: MAINTENANCE_MAX_CONCURRENT_REQUESTS
|
|
value: "10"
|
|
required: true
|