apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-image-builder-worker-general labels: grafana_dashboard: "true" annotations: grafana-folder: /grafana-dashboard-definitions/Image-Builder data: grafana.json: |- { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "iteration": 1645525186957, "links": [], "panels": [ { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 212, "panels": [], "title": "Build Job Success Rate", "type": "row" }, { "datasource": "${datasource}", "description": "The percentage of successful osbuild jobs for the selected time range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0%" } }, "type": "special" } ], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#EAB839", "value": "0.95" }, { "color": "green", "value": "0.955" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 5, "x": 0, "y": 1 }, "id": 197, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "1 - (\n (\n sum(increase(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$__range]))\n /\n sum(increase(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[$__range])) \n ) OR on() vector(0) # set a fallback if the query result is empty\n)", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Build Job Success Rate", "type": "stat" }, { "datasource": "${datasource}", "description": "The throughput rate of osbuild job errors and non-errors over time for the selected time range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 11, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "success/sec" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 11, "x": 5, "y": 1 }, "id": 202, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "(sum(rate(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[$interval])) OR on() vector(0))\n- \n(sum(rate(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$interval])) OR on() vector(0))", "hide": false, "interval": "", "legendFormat": "success/sec", "refId": "A" }, { "exemplar": true, "expr": "(sum(rate(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$interval])) OR on() vector(0))", "hide": false, "interval": "", "legendFormat": "errors/sec", "refId": "B" } ], "title": "Build Job Throughput Rate", "type": "timeseries" }, { "datasource": "${datasource}", "description": "The number of osbuild job errors (as a percentage) over time for the selected time range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, "id": 228, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "(\n sum(rate(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$interval]))\n /\n sum(rate(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[$interval]))\n)\nOR on() vector(0) # set fallback incase the above query result is empty", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Build Job Error Rate", "type": "timeseries" }, { "cacheTimeout": 1, "datasource": "${datasource}", "description": "How long will it take to consume all our budget if our error consumption remains at the current rate for the selected date range.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [ { "options": { "match": "null", "result": { "color": "green", "index": 0, "text": "∞" } }, "type": "special" }, { "options": { "from": 672, "result": { "index": 1, "text": "∞" }, "to": 3360100 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#EAB839", "value": 40 }, { "color": "green", "value": 50 } ] }, "unit": "h" }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 0, "y": 9 }, "id": 238, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 80 }, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "28 * 24 * (1 - $stability_slo)\n/ \n(\n (\n sum(rate(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[28d]))\n / \n sum(rate(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[28d]))\n ) OR on() vector(0.01) # set fallback incase the above query result is empty\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Remaining", "type": "stat" }, { "cacheTimeout": null, "datasource": "${datasource}", "description": "The percentage of error budget consumed for the selected time range. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 0.95 }, { "color": "red", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 20, "x": 4, "y": 9 }, "id": 239, "links": [], "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "1 - (\n (\n 1 - $stability_slo - (\n (\n sum(increase(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[28d]))\n /\n sum(increase(image_builder_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[28d]))\n ) OR on() vector(0) # set fallback for empty query result\n ) \n ) \n)\n/ \n(1 - $stability_slo)", "instant": false, "interval": "", "intervalFactor": 10, "legendFormat": "errorbudget", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Consumed", "type": "timeseries" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "id": 129, "panels": [], "title": "Build Job Duration", "type": "row" }, { "datasource": "${datasource}", "description": "The duration of 90% of osbuild jobs", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": "1228" }, { "color": "red", "value": "1536" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 5, "x": 0, "y": 18 }, "id": 200, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$__range])) by (le))", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Build Job Duration", "type": "stat" }, { "datasource": "${datasource}", "description": "The request Duration for osbuild jobs over the selected date range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "seconds", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 35, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": "175" }, { "color": "red", "value": "200" } ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byName", "options": "99p" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "95p" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "90p" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "50p" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p90" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 11, "x": 5, "y": 18 }, "id": 201, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.5, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))", "interval": "", "legendFormat": "p50", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))", "hide": false, "interval": "", "legendFormat": "p90", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))", "hide": false, "interval": "", "legendFormat": "p99", "refId": "C" } ], "title": "Build Job Duration", "type": "timeseries" }, { "datasource": "${datasource}", "description": "Percent of requests exceeding Duration allowed by SLO", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 18 }, "id": 210, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "1 - sum(rate(image_builder_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[$interval]))/sum(rate(image_builder_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[$interval]))", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Slow Request Rate", "type": "timeseries" }, { "cacheTimeout": 1, "datasource": "${datasource}", "description": "How long will it take to consume all our budget if our error consumption remains at the current rate for the selected date range.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "1.40 days" } }, "type": "special" }, { "options": { "from": 672, "result": { "index": 1, "text": "∞" }, "to": 3360100 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#EAB839", "value": 40 }, { "color": "green", "value": 50 } ] }, "unit": "h" }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 0, "y": 26 }, "id": 198, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 80 }, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "28 * 24 * (1 - $latency_slo) \n/ \n( \n 1.001 - ( \n (\n sum(rate(image_builder_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[$__range]))\n /\n sum(rate(image_builder_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[$__range]))\n ) OR on() vector(1) # set fallback incase the above query result is empty\n )\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Remaining", "type": "stat" }, { "cacheTimeout": null, "datasource": "${datasource}", "description": "The percentage of error budget consumed for the selected time range. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 0.95 }, { "color": "red", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 20, "x": 4, "y": 26 }, "id": 199, "links": [], "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "1 - (\n (\n (\n sum(increase(image_builder_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[28d]))\n /\n sum(increase(image_builder_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[28d])) \n ) OR on() vector(1) \n ) - $latency_slo\n)\n/\n(1 - $latency_slo)", "instant": false, "interval": "", "intervalFactor": 10, "legendFormat": "errorbudget", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Consumed", "type": "timeseries" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, "id": 214, "panels": [], "title": "Depsolve Job Success Rate", "type": "row" }, { "datasource": "${datasource}", "description": "The percentage of successful depsolve jobs for the selected time range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0%" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#EAB839", "value": "0.95" }, { "color": "green", "value": "0.955" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 5, "x": 0, "y": 35 }, "id": 225, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "1 - (\n (\n sum(increase(image_builder_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[$__range]))\n /\n sum(increase(image_builder_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[$__range])) \n ) OR on() vector(0) # set a fallback if the query result is empty\n)", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Depsolve Job Success Rate", "type": "stat" }, { "datasource": "${datasource}", "description": "The throughput rate of depsolve job errors and non-errors over time for the selected time range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 11, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "success/sec" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 11, "x": 5, "y": 35 }, "id": 227, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "(sum(rate(image_builder_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[$interval])) OR on() vector(0)) \n- \n(sum(rate(image_builder_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[$interval])) OR on() vector(0))", "hide": false, "interval": "", "legendFormat": "success/sec", "refId": "A" }, { "exemplar": true, "expr": "(sum(rate(image_builder_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[$interval])) OR on() vector(0))", "hide": false, "interval": "", "legendFormat": "errors/sec", "refId": "B" } ], "title": "Depsolve Job Throughput Rate", "type": "timeseries" }, { "datasource": "${datasource}", "description": "The number of depsolve job errors (as a percentage) over time for the selected time range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 3, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 35 }, "id": 194, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "(\n sum(rate(image_builder_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[$interval]))\n / \n sum(rate(image_builder_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[$interval]))\n)\nOR on() vector(0) # set fallback incase the above query result is empty", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Depsolve Job Error Rate", "type": "timeseries" }, { "cacheTimeout": 1, "datasource": "${datasource}", "description": "How long will it take to consume all our budget if our error consumption remains at the current rate for the selected date range.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [ { "options": { "match": "null", "result": { "color": "green", "index": 0, "text": "∞" } }, "type": "special" }, { "options": { "from": 672, "result": { "index": 1, "text": "∞" }, "to": 3360100 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#EAB839", "value": 40 }, { "color": "green", "value": 50 } ] }, "unit": "h" }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 0, "y": 43 }, "id": 223, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 80 }, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "28 * 24 * (1 - $stability_slo) \n/ \n(\n (\n sum(rate(image_builder_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[28d]))\n / \n sum(rate(image_builder_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[28d]))\n ) OR on() vector(0.01) # set fallback incase the above query result is empty\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Remaining", "type": "stat" }, { "cacheTimeout": null, "datasource": "${datasource}", "description": "The percentage of error budget consumed for the selected time range. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 0.95 }, { "color": "red", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 20, "x": 4, "y": 43 }, "id": 224, "links": [], "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "1 - (\n (\n 1 - $stability_slo - (\n ( \n sum(increase(image_builder_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[28d]))\n /\n sum(increase(image_builder_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[28d]))\n ) OR on() vector(0)\n )\n )\n)\n/ \n(1 - $stability_slo)", "instant": false, "interval": "", "intervalFactor": 10, "legendFormat": "errorbudget", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Consumed", "type": "timeseries" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "id": 128, "panels": [], "title": "Depsolve Duration", "type": "row" }, { "datasource": "${datasource}", "description": "The duration of 90% of depsolve jobs", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": "25" }, { "color": "red", "value": "30" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 5, "x": 0, "y": 52 }, "id": 237, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$__range])) by (le))", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Depsolve Job Duration", "type": "stat" }, { "datasource": "${datasource}", "description": "The request Duration for depsolve jobs over the selected date range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "seconds", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 35, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": "175" }, { "color": "red", "value": "200" } ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byName", "options": "99p" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "95p" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "90p" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "50p" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p90" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 11, "x": 5, "y": 52 }, "id": 205, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.5, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))", "interval": "", "legendFormat": "p50", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))", "hide": false, "interval": "", "legendFormat": "p90", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(image_builder_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))", "hide": false, "interval": "", "legendFormat": "p99", "refId": "C" } ], "title": "Depsolve Job Duration", "type": "timeseries" }, { "datasource": "${datasource}", "description": "Percent of requests exceeding Duration allowed by SLO", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 52 }, "id": 234, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "1 - sum(rate(image_builder_worker_job_duration_seconds_bucket{le=\"32\", type=\"depsolve\"}[$interval]))/sum(rate(image_builder_worker_job_duration_seconds_count{type=\"depsolve\"}[$interval]))", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Slow Request Rate", "type": "timeseries" }, { "cacheTimeout": 1, "datasource": "${datasource}", "description": "How long will it take to consume all our budget if our error consumption remains at the current rate for the selected date range.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "1.40 days" } }, "type": "special" }, { "options": { "from": 672, "result": { "index": 1, "text": "∞" }, "to": 3360100 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "#EAB839", "value": 40 }, { "color": "green", "value": 50 } ] }, "unit": "h" }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 0, "y": 60 }, "id": 115, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 80 }, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "28 * 24 * (1 - $latency_slo) \n/ \n(\n 1.001 - (\n (\n sum(rate(image_builder_worker_job_duration_seconds_bucket{le=\"32\",type=\"depsolve\"}[$__range]))\n /\n sum(rate(image_builder_worker_job_duration_seconds_count{type=\"depsolve\"}[$__range]))\n ) OR on() vector(1) # set fallback incase the above query result is empty\n )\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Remaining", "type": "stat" }, { "cacheTimeout": null, "datasource": "${datasource}", "description": "The percentage of error budget consumed for the selected time range. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 0.95 }, { "color": "red", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 20, "x": 4, "y": 60 }, "id": 119, "links": [], "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "1 - (\n (\n (\n sum(increase(image_builder_worker_job_duration_seconds_bucket{le=\"32\", type=\"depsolve\"}[28d]))\n /\n sum(increase(image_builder_worker_job_duration_seconds_count{type=\"depsolve\"}[28d])) \n ) OR on() vector(1)\n ) - $latency_slo\n)\n/ (1 - $latency_slo)", "instant": false, "interval": "", "intervalFactor": 10, "legendFormat": "errorbudget", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Error Budget Consumed", "type": "timeseries" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, "id": 207, "panels": [], "title": "Job Wait Duration", "type": "row" }, { "datasource": "${datasource}", "description": "The duration of 90% of jobs waiting for execution in the job queue", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": "1228" }, { "color": "red", "value": "1536" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 5, "x": 0, "y": 69 }, "id": 208, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.1.5", "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(image_builder_worker_job_wait_duration_seconds_bucket[$__range])) by (le))", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Job Wait Duration", "type": "stat" }, { "datasource": "${datasource}", "description": "The duration for jobs waiting in the job queue over the selected date range", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisLabel": "seconds", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 35, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": "175" }, { "color": "red", "value": "200" } ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byName", "options": "99p" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "95p" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "90p" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "50p" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p90" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 11, "x": 5, "y": 69 }, "id": 209, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.5, sum(rate(image_builder_worker_job_wait_duration_seconds_bucket[$interval])) by (le))", "interval": "", "legendFormat": "p50", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(image_builder_worker_job_wait_duration_seconds_bucket[$interval])) by (le))", "hide": false, "interval": "", "legendFormat": "p90", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(image_builder_worker_job_wait_duration_seconds_bucket[$interval])) by (le))", "hide": false, "interval": "", "legendFormat": "p99", "refId": "C" } ], "title": "Job Wait Duration", "type": "timeseries" }, { "datasource": "${datasource}", "description": "Percent of requests exceeding duration allowed by SLO", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 69 }, "id": 204, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "1 - sum(rate(image_builder_worker_job_wait_duration_seconds_bucket{le=\"1536\"}[$interval]))/sum(rate(image_builder_worker_job_wait_duration_seconds_count[$interval]))", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Slow Request Rate", "type": "timeseries" } ], "refresh": false, "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "app-sre-prod-04-prometheus", "value": "app-sre-prod-04-prometheus" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "/app-sre-(prod-04|stage-01)-prometheus/", "skipUrlSync": false, "type": "datasource" }, { "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "text": "28d", "value": "28d" }, "description": null, "error": null, "hide": 0, "label": null, "name": "interval", "options": [ { "selected": false, "text": "5m", "value": "5m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "3d", "value": "3d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "14d", "value": "14d" }, { "selected": true, "text": "28d", "value": "28d" } ], "query": "5m,30m,1h,6h,12h,1d,3d,7d,14d,28d", "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "interval" }, { "description": "Compose stability SLO target", "error": null, "hide": 2, "label": null, "name": "stability_slo", "query": "0.95", "skipUrlSync": false, "type": "constant" }, { "description": "Compose Duration SLO target", "error": null, "hide": 2, "label": null, "name": "latency_slo", "query": "0.9", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-28d", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "28d" ] }, "timezone": "", "title": "Image Builder Worker", "uid": "image-builder-worker", "version": 4 }