particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

prometheus/job: measure time spent pending rather than queued

Browse source 
We are interested in the time it takes from a job could be dequeued
until it is, but if a job has dependencies that are not yet finished, it
cannot be dequeued.

Change the logic to measure the time since the last dependency was
dequeued rather than when the job was queued.

The purpose of this metric is to have an alert fire in case we have too
few workers processing jobs.
This commit is contained in:
Tom Gundersen 2022-03-15 20:17:55 +00:00
parent 4621768c14
commit 4eeaebd40b
2 changed files with 15 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
internal/prometheus/job_metrics.go
View file

@ -61,9 +61,9 @@ func EnqueueJobMetrics(jobType string) {
PendingJobs.WithLabelValues(jobType).Inc()
}
func DequeueJobMetrics(queued time.Time, started time.Time, jobType string) {
if !started.IsZero() && !queued.IsZero() {
diff := started.Sub(queued).Seconds()
func DequeueJobMetrics(pending time.Time, started time.Time, jobType string) {
if !started.IsZero() && !pending.IsZero() {
diff := started.Sub(pending).Seconds()
JobWaitDuration.WithLabelValues(jobType).Observe(diff)
PendingJobs.WithLabelValues(jobType).Dec()
RunningJobs.WithLabelValues(jobType).Inc()

14
internal/worker/server.go
View file

@ -455,13 +455,23 @@ func (s *Server) requestJob(ctx context.Context, arch string, jobTypes []string,
return
}
// Record how long the job has been pending for, that is either how
// long it has been queued for, in case it has no dependencies, or
// how long it has been since all its dependencies finished, if it
// has any.
pending := status.Queued
for _, depID := range depIDs {
// TODO: include type of arguments
var result json.RawMessage
_, result, _, _, _, _, _, err = s.jobs.JobStatus(depID)
var finished time.Time
_, result, _, _, finished, _, _, err = s.jobs.JobStatus(depID)
if err != nil {
return
}
if finished.After(pending) {
pending = finished
}
dynamicArgs = append(dynamicArgs, result)
}
@ -474,7 +484,7 @@ func (s *Server) requestJob(ctx context.Context, arch string, jobTypes []string,
// TODO: Drop the ':$architecture' for metrics too, first prometheus queries for alerts and
// dashboards need to be adjusted.
prometheus.DequeueJobMetrics(status.Queued, status.Started, jobType)
prometheus.DequeueJobMetrics(pending, status.Started, jobType)
if jobType == "osbuild:"+arch {
jobType = "osbuild"
} else if jobType == "osbuild-koji:"+arch {