From 8398f27742510d8a55098c7732b4e9b4887276b3 Mon Sep 17 00:00:00 2001 From: Diaa Sami Date: Wed, 10 May 2023 12:06:49 +0200 Subject: [PATCH] internal/cloudapi: additional prometheus listener Listening on another port, while keeping the existing endpoint until transition is complete --- cmd/osbuild-composer/composer.go | 34 +++++++++++++++++-- cmd/osbuild-composer/main.go | 8 +++++ distribution/osbuild-composer-api.socket | 1 + .../osbuild-composer-prometheus.socket | 9 +++++ osbuild-composer.spec | 7 ++-- templates/composer.yml | 10 ++++++ test/cases/api.sh | 16 +++++++++ 7 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 distribution/osbuild-composer-prometheus.socket diff --git a/cmd/osbuild-composer/composer.go b/cmd/osbuild-composer/composer.go index f2341d01a..af4dae521 100644 --- a/cmd/osbuild-composer/composer.go +++ b/cmd/osbuild-composer/composer.go @@ -44,7 +44,7 @@ type Composer struct { weldr *weldr.API api *cloudapi.Server - weldrListener, localWorkerListener, workerListener, apiListener net.Listener + weldrListener, localWorkerListener, workerListener, apiListener, promListener net.Listener } func NewComposer(config *ComposerConfigFile, stateDir, cacheDir string) (*Composer, error) { @@ -130,6 +130,10 @@ func (c *Composer) InitWeldr(repoPaths []string, weldrListener net.Listener, return nil } +func (c *Composer) InitMetricsAPI(prometheus net.Listener) { + c.promListener = prometheus +} + func (c *Composer) InitAPI(cert, key string, enableTLS bool, enableMTLS bool, enableJWT bool, l net.Listener) error { config := v2.ServerConfig{ JWTEnabled: c.config.Koji.EnableJWT, @@ -218,7 +222,7 @@ func (c *Composer) Start() error { logrus.Fatal("neither the weldr API socket nor the composer API socket is enabled, osbuild-composer is useless without one of these APIs enabled") } - var localWorkerAPI, remoteWorkerAPI, composerAPI *http.Server + var localWorkerAPI, remoteWorkerAPI, composerAPI, prometheusAPI *http.Server if c.localWorkerListener != nil { localWorkerAPI = &http.Server{ @@ -313,6 +317,25 @@ func (c *Composer) Start() error { }() } + if c.promListener != nil { + // metrics listener on another port + metricsMux := http.NewServeMux() + metricsMux.Handle("/metrics", promhttp.Handler().(http.HandlerFunc)) + + prometheusAPI = &http.Server{ + ErrorLog: c.logger, + Handler: metricsMux, + ReadHeaderTimeout: 5 * time.Second, + } + + go func() { + err := prometheusAPI.Serve(c.promListener) + if err != nil && err != http.ErrServerClosed { + panic(err) + } + }() + } + if c.weldrListener != nil { go func() { err := c.weldr.Serve(c.weldrListener) @@ -341,6 +364,13 @@ func (c *Composer) Start() error { } } + if c.promListener != nil { + err := prometheusAPI.Shutdown(context.Background()) + if err != nil { + panic(err) + } + } + if c.localWorkerListener != nil { err := localWorkerAPI.Shutdown(context.Background()) if err != nil { diff --git a/cmd/osbuild-composer/main.go b/cmd/osbuild-composer/main.go index 04137dc93..0af8b71d6 100644 --- a/cmd/osbuild-composer/main.go +++ b/cmd/osbuild-composer/main.go @@ -108,6 +108,14 @@ func main() { composer.InitLocalWorker(l[0]) } + if l, exists := listeners["osbuild-composer-prometheus.socket"]; exists { + if len(l) != 1 { + logrus.Warn("The osbuild-composer-prometheus.socket unit is misconfigured. It should contain only one socket.") + } + + composer.InitMetricsAPI(l[0]) + } + if l, exists := listeners["osbuild-composer-api.socket"]; exists { if len(l) != 1 { logrus.Fatal("The osbuild-composer-api.socket unit is misconfigured. It should contain only one socket.") diff --git a/distribution/osbuild-composer-api.socket b/distribution/osbuild-composer-api.socket index 580f4eeb6..f7b100dde 100644 --- a/distribution/osbuild-composer-api.socket +++ b/distribution/osbuild-composer-api.socket @@ -1,5 +1,6 @@ [Unit] Description=OSBuild Composer API socket +Requires=osbuild-composer-prometheus.socket [Socket] Service=osbuild-composer.service diff --git a/distribution/osbuild-composer-prometheus.socket b/distribution/osbuild-composer-prometheus.socket new file mode 100644 index 000000000..7aba0d0fa --- /dev/null +++ b/distribution/osbuild-composer-prometheus.socket @@ -0,0 +1,9 @@ +[Unit] +Description=OSBuild Composer prometheus socket + +[Socket] +Service=osbuild-composer.service +ListenStream=8008 + +[Install] +WantedBy=sockets.target diff --git a/osbuild-composer.spec b/osbuild-composer.spec index 8e5bacd1e..206f15bb2 100644 --- a/osbuild-composer.spec +++ b/osbuild-composer.spec @@ -262,13 +262,13 @@ cd $PWD/_build/src/%{goipath} %endif %post -%systemd_post osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-remote-worker.socket +%systemd_post osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-composer-prometheus.socket osbuild-remote-worker.socket %preun -%systemd_preun osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-remote-worker.socket +%systemd_preun osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-composer-prometheus.socket osbuild-remote-worker.socket %postun -%systemd_postun_with_restart osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-remote-worker.socket +%systemd_postun_with_restart osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-composer-prometheus.socket osbuild-remote-worker.socket %files %license LICENSE @@ -277,6 +277,7 @@ cd $PWD/_build/src/%{goipath} %{_unitdir}/osbuild-composer.service %{_unitdir}/osbuild-composer.socket %{_unitdir}/osbuild-composer-api.socket +%{_unitdir}/osbuild-composer-prometheus.socket %{_unitdir}/osbuild-local-worker.socket %{_unitdir}/osbuild-remote-worker.socket %{_sysusersdir}/osbuild-composer.conf diff --git a/templates/composer.yml b/templates/composer.yml index 104c429c2..8a15637c7 100644 --- a/templates/composer.yml +++ b/templates/composer.yml @@ -109,6 +109,9 @@ objects: - name: composer-api protocol: TCP containerPort: ${{COMPOSER_API_PORT}} + - name: prometheus + protocol: TCP + containerPort: ${{PROMETHEUS_PORT}} - name: worker-api protocol: TCP containerPort: ${{WORKER_API_PORT}} @@ -217,6 +220,10 @@ objects: protocol: TCP port: 80 targetPort: ${{COMPOSER_API_PORT}} + - name: prometheus + protocol: TCP + port: 8008 + targetPort: ${{PROMETHEUS_PORT}} selector: app: composer @@ -455,6 +462,9 @@ parameters: name: COMPOSER_API_PORT required: true value: "8080" + - description: prometheus port + name: PROMETHEUS_PORT + value: "8008" - description: worker-api port name: WORKER_API_PORT required: true diff --git a/test/cases/api.sh b/test/cases/api.sh index dc44ad027..7d322f9eb 100755 --- a/test/cases/api.sh +++ b/test/cases/api.sh @@ -416,6 +416,20 @@ function collectMetrics(){ echo "$METRICS_OUTPUT" | grep "^image_builder_composer_total_compose_requests" | cut -f2 -d' ' } +function testNewMetricsPort(){ + METRICS_OUTPUT1=$(curl \ + --cacert /etc/osbuild-composer/ca-crt.pem \ + --key /etc/osbuild-composer/client-key.pem \ + --cert /etc/osbuild-composer/client-crt.pem \ + https://localhost/metrics) + METRICS_OUTPUT2=$(curl http://localhost:8008/metrics) + + COMPOSES1=$(echo "$METRICS_OUTPUT1" | grep "^image_builder_composer_total_compose_requests" | cut -f2 -d' ') + COMPOSES2=$(echo "$METRICS_OUTPUT2" | grep "^image_builder_composer_total_compose_requests" | cut -f2 -d' ') + + test "$COMPOSES1" = "$COMPOSES2" +} + function sendCompose() { OUTPUT=$(mktemp) HTTPSTATUS=$(curl \ @@ -570,6 +584,8 @@ sudo "${CONTAINER_RUNTIME}" exec "${DB_CONTAINER_NAME}" psql -U postgres -d osbu "DELETE FROM jobs WHERE id = '$COMPOSE_ID'" sudo systemctl start "osbuild-remote-worker@localhost:8700.service" +testNewMetricsPort + # full integration case INIT_COMPOSES="$(collectMetrics)" sendCompose "$REQUEST_FILE"