internal/cloudapi: additional prometheus listener

Listening on another port, while keeping the existing endpoint until
transition is complete

cmd/osbuild-composer/composer.go

@@ -44,7 +44,7 @@ type Composer struct {
 	weldr   *weldr.API
 	api     *cloudapi.Server
 
-	weldrListener, localWorkerListener, workerListener, apiListener net.Listener
+	weldrListener, localWorkerListener, workerListener, apiListener, promListener net.Listener
 }
 
 func NewComposer(config *ComposerConfigFile, stateDir, cacheDir string) (*Composer, error) {
@@ -130,6 +130,10 @@ func (c *Composer) InitWeldr(repoPaths []string, weldrListener net.Listener,
 	return nil
 }
 
+func (c *Composer) InitMetricsAPI(prometheus net.Listener) {
+	c.promListener = prometheus
+}
+
 func (c *Composer) InitAPI(cert, key string, enableTLS bool, enableMTLS bool, enableJWT bool, l net.Listener) error {
 	config := v2.ServerConfig{
 		JWTEnabled:           c.config.Koji.EnableJWT,
@@ -218,7 +222,7 @@ func (c *Composer) Start() error {
 		logrus.Fatal("neither the weldr API socket nor the composer API socket is enabled, osbuild-composer is useless without one of these APIs enabled")
 	}
 
-	var localWorkerAPI, remoteWorkerAPI, composerAPI *http.Server
+	var localWorkerAPI, remoteWorkerAPI, composerAPI, prometheusAPI *http.Server
 
 	if c.localWorkerListener != nil {
 		localWorkerAPI = &http.Server{
@@ -313,6 +317,25 @@ func (c *Composer) Start() error {
 		}()
 	}
 
+	if c.promListener != nil {
+		// metrics listener on another port
+		metricsMux := http.NewServeMux()
+		metricsMux.Handle("/metrics", promhttp.Handler().(http.HandlerFunc))
+
+		prometheusAPI = &http.Server{
+			ErrorLog:          c.logger,
+			Handler:           metricsMux,
+			ReadHeaderTimeout: 5 * time.Second,
+		}
+
+		go func() {
+			err := prometheusAPI.Serve(c.promListener)
+			if err != nil && err != http.ErrServerClosed {
+				panic(err)
+			}
+		}()
+	}
+
 	if c.weldrListener != nil {
 		go func() {
 			err := c.weldr.Serve(c.weldrListener)
@@ -341,6 +364,13 @@ func (c *Composer) Start() error {
 		}
 	}
 
+	if c.promListener != nil {
+		err := prometheusAPI.Shutdown(context.Background())
+		if err != nil {
+			panic(err)
+		}
+	}
+
 	if c.localWorkerListener != nil {
 		err := localWorkerAPI.Shutdown(context.Background())
 		if err != nil {

cmd/osbuild-composer/main.go

@@ -108,6 +108,14 @@ func main() {
 		composer.InitLocalWorker(l[0])
 	}
 
+	if l, exists := listeners["osbuild-composer-prometheus.socket"]; exists {
+		if len(l) != 1 {
+			logrus.Warn("The osbuild-composer-prometheus.socket unit is misconfigured. It should contain only one socket.")
+		}
+
+		composer.InitMetricsAPI(l[0])
+	}
+
 	if l, exists := listeners["osbuild-composer-api.socket"]; exists {
 		if len(l) != 1 {
 			logrus.Fatal("The osbuild-composer-api.socket unit is misconfigured. It should contain only one socket.")

distribution/osbuild-composer-api.socket

@@ -1,5 +1,6 @@
 [Unit]
 Description=OSBuild Composer API socket
+Requires=osbuild-composer-prometheus.socket
 
 [Socket]
 Service=osbuild-composer.service

distribution/osbuild-composer-prometheus.socket

@@ -0,0 +1,9 @@
+[Unit]
+Description=OSBuild Composer prometheus socket
+
+[Socket]
+Service=osbuild-composer.service
+ListenStream=8008
+
+[Install]
+WantedBy=sockets.target

osbuild-composer.spec

@@ -262,13 +262,13 @@ cd $PWD/_build/src/%{goipath}
 %endif
 
 %post
-%systemd_post osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-remote-worker.socket
+%systemd_post osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-composer-prometheus.socket osbuild-remote-worker.socket
 
 %preun
-%systemd_preun osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-remote-worker.socket
+%systemd_preun osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-composer-prometheus.socket osbuild-remote-worker.socket
 
 %postun
-%systemd_postun_with_restart osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-remote-worker.socket
+%systemd_postun_with_restart osbuild-composer.service osbuild-composer.socket osbuild-composer-api.socket osbuild-composer-prometheus.socket osbuild-remote-worker.socket
 
 %files
 %license LICENSE
@@ -277,6 +277,7 @@ cd $PWD/_build/src/%{goipath}
 %{_unitdir}/osbuild-composer.service
 %{_unitdir}/osbuild-composer.socket
 %{_unitdir}/osbuild-composer-api.socket
+%{_unitdir}/osbuild-composer-prometheus.socket
 %{_unitdir}/osbuild-local-worker.socket
 %{_unitdir}/osbuild-remote-worker.socket
 %{_sysusersdir}/osbuild-composer.conf

templates/composer.yml

@@ -109,6 +109,9 @@ objects:
           - name: composer-api
             protocol: TCP
             containerPort: ${{COMPOSER_API_PORT}}
+          - name: prometheus
+            protocol: TCP
+            containerPort: ${{PROMETHEUS_PORT}}
           - name: worker-api
             protocol: TCP
             containerPort: ${{WORKER_API_PORT}}
@@ -217,6 +220,10 @@ objects:
         protocol: TCP
         port: 80
         targetPort: ${{COMPOSER_API_PORT}}
+      - name: prometheus
+        protocol: TCP
+        port: 8008
+        targetPort: ${{PROMETHEUS_PORT}}
     selector:
       app: composer
 
@@ -455,6 +462,9 @@ parameters:
     name: COMPOSER_API_PORT
     required: true
     value: "8080"
+  - description: prometheus port
+    name: PROMETHEUS_PORT
+    value: "8008"
   - description: worker-api port
     name: WORKER_API_PORT
     required: true

test/cases/api.sh

@@ -416,6 +416,20 @@ function collectMetrics(){
     echo "$METRICS_OUTPUT" | grep "^image_builder_composer_total_compose_requests" | cut -f2 -d' '
 }
 
+function testNewMetricsPort(){
+    METRICS_OUTPUT1=$(curl \
+                          --cacert /etc/osbuild-composer/ca-crt.pem \
+                          --key /etc/osbuild-composer/client-key.pem \
+                          --cert /etc/osbuild-composer/client-crt.pem \
+                          https://localhost/metrics)
+    METRICS_OUTPUT2=$(curl http://localhost:8008/metrics)
+
+    COMPOSES1=$(echo "$METRICS_OUTPUT1" | grep "^image_builder_composer_total_compose_requests" | cut -f2 -d' ')
+    COMPOSES2=$(echo "$METRICS_OUTPUT2" | grep "^image_builder_composer_total_compose_requests" | cut -f2 -d' ')
+
+    test "$COMPOSES1" = "$COMPOSES2"
+}
+
 function sendCompose() {
     OUTPUT=$(mktemp)
     HTTPSTATUS=$(curl \
@@ -570,6 +584,8 @@ sudo "${CONTAINER_RUNTIME}" exec "${DB_CONTAINER_NAME}" psql -U postgres -d osbu
      "DELETE FROM jobs WHERE id = '$COMPOSE_ID'"
 sudo systemctl start "osbuild-remote-worker@localhost:8700.service"
 
+testNewMetricsPort
+
 # full integration case
 INIT_COMPOSES="$(collectMetrics)"
 sendCompose "$REQUEST_FILE"