prometheus: split off request timing information into separate mw

Tracks the worker api in addition to the composer api.

internal/cloudapi/v2/server.go

@@ -93,6 +93,7 @@ func (s *Server) Handler(path string) http.Handler {
 
 	mws := []echo.MiddlewareFunc{
 		prometheus.StatusMiddleware(prometheus.ComposerSubsystem),
+		prometheus.HTTPDurationMiddleware(prometheus.ComposerSubsystem),
 	}
 	if s.config.JWTEnabled {
 		mws = append(mws, auth.TenantChannelMiddleware(s.config.TenantProviderFields, HTTPError(ErrorTenantNotFound)))

internal/prometheus/http_metrics.go

@@ -23,12 +23,24 @@ var (
 	})
 )
 
-var (
-	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+func HTTPDurationHisto(subsystem string) *prometheus.HistogramVec {
+	reg := prometheus.NewRegistry()
+	histo := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
 		Name:      "http_duration_seconds",
 		Namespace: Namespace,
-		Subsystem: ComposerSubsystem,
+		Subsystem: subsystem,
 		Help:      "Duration of HTTP requests.",
 		Buckets:   []float64{.025, .05, .075, .1, .2, .5, .75, 1, 1.5, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20},
 	}, []string{"path", "tenant"})
-)
+
+	err := prometheus.Register(histo)
+	if err != nil {
+		registered, ok := err.(prometheus.AlreadyRegisteredError)
+		if !ok {
+			panic(err)
+		}
+		// return existing counter if metrics already registered
+		return registered.ExistingCollector.(*prometheus.HistogramVec)
+	}
+	return histo
+}

internal/prometheus/middleware.go

@@ -17,15 +17,23 @@ func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
 		if strings.HasSuffix(ctx.Path(), "/compose") {
 			ComposeRequests.Inc()
 		}
-
-		// leave tenant empty in case it's not in the context
-		tenant, _ := ctx.Get(auth.TenantCtxKey).(string)
-		timer := prometheus.NewTimer(httpDuration.WithLabelValues(ctx.Path(), tenant))
-		defer timer.ObserveDuration()
 		return next(ctx)
 	}
 }
 
+func HTTPDurationMiddleware(subsystem string) func(next echo.HandlerFunc) echo.HandlerFunc {
+	histogram := HTTPDurationHisto(subsystem)
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(ctx echo.Context) error {
+			// leave tenant empty in case it's not in the context
+			tenant, _ := ctx.Get(auth.TenantCtxKey).(string)
+			timer := prometheus.NewTimer(histogram.WithLabelValues(ctx.Path(), tenant))
+			defer timer.ObserveDuration()
+			return next(ctx)
+		}
+	}
+}
+
 func StatusMiddleware(subsystem string) func(next echo.HandlerFunc) echo.HandlerFunc {
 	counter := StatusRequestsCounter(subsystem)
 	return func(next echo.HandlerFunc) echo.HandlerFunc {

internal/worker/server.go

@@ -102,6 +102,7 @@ func (s *Server) Handler() http.Handler {
 
 	mws := []echo.MiddlewareFunc{
 		prometheus.StatusMiddleware(prometheus.WorkerSubsystem),
+		prometheus.HTTPDurationMiddleware(prometheus.WorkerSubsystem),
 	}
 	if s.config.JWTEnabled {
 		mws = append(mws, auth.TenantChannelMiddleware(s.config.TenantProviderFields, api.HTTPError(api.ErrorTenantNotFound)))