By default, pgxpool.Pool has 4 connections (or number of cpus if higher). Currently, we have 3 replicas, that means max 3*4=12 DB connections. The dequeue operation is actually blocking - when a worker is waiting for a job, one connection is blocked. My theory is that with 16 workers, we just don't have enough connections that causes all sorts of weird slowdowns. This commit bumps the number of connection from one replica to 10, therefore we should be at 30 connections in total. Signed-off-by: Ondřej Budai <ondrej@budai.cz>
270 lines
7.3 KiB
YAML
270 lines
7.3 KiB
YAML
apiVersion: v1
|
|
kind: Template
|
|
metadata:
|
|
name: composer
|
|
annotations:
|
|
openshift.io/display-name: Image-Builder composer service
|
|
description: Composer component of the image-builder serivce
|
|
tags: golang
|
|
iconClass: icon-shadowman
|
|
template.openshift.io/provider-display-name: Red Hat, Inc.
|
|
labels:
|
|
template: composer
|
|
objects:
|
|
- apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
labels:
|
|
service: image-builder
|
|
name: composer
|
|
spec:
|
|
replicas: 3
|
|
selector:
|
|
matchLabels:
|
|
app: composer
|
|
strategy:
|
|
# Update pods 1 at a time
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
# Create at most 0 extra pod over .spec.replicas
|
|
maxSurge: 0
|
|
# At all times there should be .spec.replicas - 1 available
|
|
maxUnavailable: 1
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: composer
|
|
spec:
|
|
serviceAccountName: image-builder
|
|
containers:
|
|
- image: "${IMAGE_NAME}:${IMAGE_TAG}"
|
|
name: composer
|
|
livenessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: ${LIVENESS_URI}
|
|
port: ${{COMPOSER_API_PORT}}
|
|
scheme: HTTP
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 1
|
|
readinessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: ${READINESS_URI}
|
|
port: ${{COMPOSER_API_PORT}}
|
|
scheme: HTTP
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 1
|
|
resources:
|
|
requests:
|
|
cpu: "${CPU_REQUEST}"
|
|
memory: "${MEMORY_REQUEST}"
|
|
limits:
|
|
cpu: "${CPU_LIMIT}"
|
|
memory: "${MEMORY_LIMIT}"
|
|
env:
|
|
- name: PGHOST
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.host
|
|
- name: PGPORT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.port
|
|
- name: PGDATABASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.name
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.user
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.password
|
|
- name: PGSSLMODE
|
|
value: "${PGSSLMODE}"
|
|
- name: PGMAXCONNS
|
|
value: "${PGMAXCONNS}"
|
|
ports:
|
|
- name: composer-api
|
|
protocol: TCP
|
|
containerPort: ${{COMPOSER_API_PORT}}
|
|
- name: worker-api
|
|
protocol: TCP
|
|
containerPort: ${{WORKER_API_PORT}}
|
|
volumeMounts:
|
|
- name: composer-config
|
|
mountPath: "${COMPOSER_CONFIG_DIR}"
|
|
readOnly: true
|
|
- name: state-directory
|
|
mountPath: "/var/lib/osbuild-composer"
|
|
- name: cache-directory
|
|
mountPath: "/var/cache/osbuild-composer"
|
|
volumes:
|
|
- name: composer-config
|
|
configMap:
|
|
name: composer-config
|
|
- name: db-secrets
|
|
secret:
|
|
secretName: db
|
|
- name: state-directory
|
|
emptyDir: {}
|
|
- name: cache-directory
|
|
emptyDir: {}
|
|
initContainers:
|
|
- name: composer-migrate
|
|
image: "${IMAGE_NAME}:${IMAGE_TAG}"
|
|
command: [ "/opt/migrate/tern", "migrate", "-m", "/opt/migrate/schemas" ]
|
|
env:
|
|
- name: PGHOST
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.host
|
|
- name: PGPORT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.port
|
|
- name: PGDATABASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.name
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.user
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: composer-db
|
|
key: db.password
|
|
- name: PGSSLMODE
|
|
value: "${PGSSLMODE}"
|
|
|
|
- apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: image-builder
|
|
imagePullSecrets:
|
|
- name: quay-cloudservices-pull
|
|
|
|
- apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: image-builder-composer
|
|
labels:
|
|
app: composer
|
|
port: composer-api
|
|
spec:
|
|
ports:
|
|
- name: composer-api
|
|
protocol: TCP
|
|
port: 80
|
|
targetPort: ${{COMPOSER_API_PORT}}
|
|
selector:
|
|
app: composer
|
|
|
|
- apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: image-builder-worker
|
|
labels:
|
|
app: composer
|
|
port: worker-api
|
|
spec:
|
|
ports:
|
|
- name: worker-api
|
|
protocol: TCP
|
|
port: 80
|
|
targetPort: ${{WORKER_API_PORT}}
|
|
selector:
|
|
app: composer
|
|
|
|
# This map should probably move to app-intf
|
|
- apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: composer-config
|
|
data:
|
|
acl.yml: |
|
|
- claim: user_id
|
|
pattern: ^(54629121|54629180|54597799|54676085)$
|
|
osbuild-composer.toml: |
|
|
log_level = "info"
|
|
[koji]
|
|
enable_tls = false
|
|
enable_mtls = false
|
|
enable_jwt = true
|
|
jwt_keys_url = "${SSO_BASE_URL}/protocol/openid-connect/certs"
|
|
jwt_acl_file = "${COMPOSER_CONFIG_DIR}/acl.yml"
|
|
[koji.aws_config]
|
|
bucket = "imagebuilder.service.staging"
|
|
[worker]
|
|
request_job_timeout = "20s"
|
|
base_path = "/api/image-builder-worker/v1"
|
|
enable_tls = false
|
|
enable_mtls = false
|
|
enable_jwt = true
|
|
jwt_keys_url = "${SSO_BASE_URL}/protocol/openid-connect/certs"
|
|
jwt_acl_file = "${COMPOSER_CONFIG_DIR}/acl.yml"
|
|
|
|
parameters:
|
|
- description: composer image name
|
|
name: IMAGE_NAME
|
|
value: quay.io/app-sre/composer
|
|
required: true
|
|
- description: composer image tag
|
|
name: IMAGE_TAG
|
|
required: true
|
|
- description: postgres sslmode to use when connecting to the db
|
|
name: PGSSLMODE
|
|
value: "require"
|
|
- description: postgres maximum connections per pod
|
|
name: PGMAXCONNS
|
|
value: "10"
|
|
- description: base sso url
|
|
name: SSO_BASE_URL
|
|
required: true
|
|
value: "https://sso.redhat.com/auth/realms/redhat-external"
|
|
- description: base sso url
|
|
name: COMPOSER_CONFIG_DIR
|
|
required: true
|
|
value: "/etc/osbuild-composer"
|
|
- description: composer-api port
|
|
name: COMPOSER_API_PORT
|
|
required: true
|
|
value: "8080"
|
|
- description: worker-api port
|
|
name: WORKER_API_PORT
|
|
required: true
|
|
value: "8700"
|
|
- name: LIVENESS_URI
|
|
description: URI to query for the liveness check
|
|
value: "/api/image-builder-composer/v2/openapi"
|
|
- name: READINESS_URI
|
|
description: URI to query for the readiness check
|
|
value: "/api/image-builder-composer/v2/openapi"
|
|
- name: CPU_REQUEST
|
|
description: CPU request per container
|
|
value: "200m"
|
|
- name: CPU_LIMIT
|
|
description: CPU limit per container
|
|
value: "1"
|
|
- name: MEMORY_REQUEST
|
|
description: Memory request per container
|
|
value: "256Mi"
|
|
- name: MEMORY_LIMIT
|
|
description: Memory limit per container
|
|
value: "512Mi"
|