From de7d5ba702bcc2495fa12931d24dac5f70e56eeb Mon Sep 17 00:00:00 2001 From: Gianluca Zuccarelli Date: Wed, 29 Sep 2021 12:26:26 +0100 Subject: [PATCH] devel: prometheus-grafana integration Add prometheus and grafana integration to the devel container stack. This should make it easier to run and configure grafana dashboards locally. --- devel/README.md | 4 +- devel/config/grafana/dashboards/dashboard.yml | 10 + .../dashboards/insights-dashboard.json | 518 ++++++++++++++++++ .../config/grafana/datasources/datasource.yml | 9 + devel/config/prometheus/prometheus.yml | 8 + devel/docker-compose.yml | 25 +- 6 files changed, 572 insertions(+), 2 deletions(-) create mode 100644 devel/config/grafana/dashboards/dashboard.yml create mode 100644 devel/config/grafana/dashboards/insights-dashboard.json create mode 100644 devel/config/grafana/datasources/datasource.yml create mode 100644 devel/config/prometheus/prometheus.yml diff --git a/devel/README.md b/devel/README.md index 3af8960b..6a2d9162 100644 --- a/devel/README.md +++ b/devel/README.md @@ -51,10 +51,12 @@ The config variables for the Image Builder backend can be found [here](https://g ## Run ```bash -docker-compose up +docker-compose up --build ``` Access the service through the GUI: [https://prod.foo.redhat.com:1337/beta/insights/image-builder](https://prod.foo.redhat.com:1337/beta/insights/image-builder), or directly through the API: [https://prod.foo.redhat.com:1337/docs/api/image-builder](https://prod.foo.redhat.com:1337/docs/api/image-builder). + +Access the Grafana dashboard on [https://localhost:3000](https://localhost:3000). The default username is `admin` and the password is set to `foobar`. \ No newline at end of file diff --git a/devel/config/grafana/dashboards/dashboard.yml b/devel/config/grafana/dashboards/dashboard.yml new file mode 100644 index 00000000..f0d5761f --- /dev/null +++ b/devel/config/grafana/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: 'grafana-dashboard-insights-image-builder-general' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/devel/config/grafana/dashboards/insights-dashboard.json b/devel/config/grafana/dashboards/insights-dashboard.json new file mode 100644 index 00000000..151776ca --- /dev/null +++ b/devel/config/grafana/dashboards/insights-dashboard.json @@ -0,0 +1,518 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1624534605950, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of successful compose requests over time. If nothing shows up, try increasing the interval variable on top.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1- (sum by (job) (increase(image_builder_compose_errors[$interval])) / sum by (job) (increase(image_builder_compose_requests_total[$interval])))) * 100", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Successful compose requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:252", + "decimals": 0, + "format": "percent", + "label": null, + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "$$hashKey": "object:253", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average response time in seconds.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum by (job) (rate(image_builder_http_duration_seconds_sum{path=~\".*compose\"}[$interval])) / sum by (job) (rate(image_builder_http_duration_seconds_count{path=~\".*compose\"}[$interval])))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compose request response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:326", + "format": "short", + "label": "seconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:327", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum by (job) (rate(image_builder_http_duration_seconds_sum{path!~\".*compose\"}[$interval])) / sum by (job) (rate(image_builder_http_duration_seconds_count{path!~\".*compose\"}[$interval]))) * 1000", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Non-compose request response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:439", + "format": "short", + "label": "milliseconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:440", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "expr": "sum(up{job=\"image-builder\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current pod count", + "type": "stat" + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "crcs02ue1-prometheus", + "value": "crcs02ue1-prometheus" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "/crc(p01|s02)ue1-prometheus/", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "12h", + "value": "12h" + }, + "hide": 0, + "label": null, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": true, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Image Builder", + "uid": "91_RmVCMk", + "version": 3 +} diff --git a/devel/config/grafana/datasources/datasource.yml b/devel/config/grafana/datasources/datasource.yml new file mode 100644 index 00000000..558f3f96 --- /dev/null +++ b/devel/config/grafana/datasources/datasource.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +datasources: + - access: 'proxy' + editable: true + isDefault: true + name: 'crcs02ue1-prometheus' + orgId: 1 + type: 'prometheus' + url: 'http://prometheus:9090' diff --git a/devel/config/prometheus/prometheus.yml b/devel/config/prometheus/prometheus.yml new file mode 100644 index 00000000..1830b0d0 --- /dev/null +++ b/devel/config/prometheus/prometheus.yml @@ -0,0 +1,8 @@ + +global: + scrape_interval: 15s +scrape_configs: + - job_name: 'image-builder' + scrape_interval: 5s + static_configs: + - targets: ['backend:8086'] diff --git a/devel/docker-compose.yml b/devel/docker-compose.yml index d7afd9ec..547d9610 100644 --- a/devel/docker-compose.yml +++ b/devel/docker-compose.yml @@ -117,7 +117,30 @@ services: depends_on: - "backend" - "frontend" - + prometheus: + image: prom/prometheus:latest + ports: + - "9000:9090" + volumes: + - ./config/prometheus:/config + restart: unless-stopped + networks: + net: + ipv4_address: 172.31.0.70 + command: + - "--config.file=/config/prometheus.yml" + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" + volumes: + - ./config/grafana:/etc/grafana/provisioning/ + restart: unless-stopped + networks: + net: + ipv4_address: 172.31.0.80 + environment: + - GF_SECURITY_ADMIN_PASSWORD=foobar networks: net: ipam: