| { | |
| "annotations": { | |
| "list": [ | |
| { | |
| "builtIn": 1, | |
| "datasource": "-- Grafana --", | |
| "enable": true, | |
| "hide": true, | |
| "iconColor": "rgba(0, 211, 255, 1)", | |
| "name": "Annotations & Alerts", | |
| "target": { | |
| "limit": 100, | |
| "matchAny": false, | |
| "tags": [ ], | |
| "type": "dashboard" | |
| }, | |
| "type": "dashboard" | |
| }, | |
| { | |
| "datasource": "${datasource}", | |
| "enable": false, | |
| "expr": "sum by (tenant) (changes(loki_ruler_wal_prometheus_tsdb_wal_truncations_total{tenant=~\"${tenant}\"}[$__rate_interval]))", | |
| "iconColor": "red", | |
| "name": "WAL Truncations", | |
| "target": { | |
| "queryType": "Azure Monitor", | |
| "refId": "Anno" | |
| }, | |
| "titleFormat": "{{tenant}}" | |
| } | |
| ] | |
| }, | |
| "editable": true, | |
| "fiscalYearStartMonth": 0, | |
| "gnetId": null, | |
| "graphTooltip": 0, | |
| "iteration": 1635347545534, | |
| "links": [ ], | |
| "liveNow": false, | |
| "panels": [ | |
| { | |
| "datasource": "${datasource}", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "mappings": [ ], | |
| "noValue": "0", | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 1 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 2, | |
| "x": 0, | |
| "y": 0 | |
| }, | |
| "id": 2, | |
| "options": { | |
| "colorMode": "value", | |
| "graphMode": "area", | |
| "justifyMode": "auto", | |
| "orientation": "auto", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "textMode": "auto" | |
| }, | |
| "pluginVersion": "8.3.0-38205pre", | |
| "targets": [ | |
| { | |
| "datasource": "${datasource}", | |
| "exemplar": false, | |
| "expr": "sum(loki_ruler_wal_appender_ready) by (pod, tenant) == 0", | |
| "instant": true, | |
| "interval": "", | |
| "legendFormat": "", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Appenders Not Ready", | |
| "type": "stat" | |
| }, | |
| { | |
| "datasource": "${datasource}", | |
| "description": "", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [ ], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 11, | |
| "x": 2, | |
| "y": 0 | |
| }, | |
| "id": 4, | |
| "options": { | |
| "legend": { | |
| "calcs": [ ], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltip": { | |
| "mode": "single" | |
| } | |
| }, | |
| "targets": [ | |
| { | |
| "datasource": "${datasource}", | |
| "exemplar": true, | |
| "expr": "sum(rate(loki_ruler_wal_samples_appended_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", | |
| "interval": "", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Samples Appended to WAL per Second", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${datasource}", | |
| "description": "Series are unique combinations of labels", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [ ], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 11, | |
| "x": 13, | |
| "y": 0 | |
| }, | |
| "id": 5, | |
| "options": { | |
| "legend": { | |
| "calcs": [ ], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltip": { | |
| "mode": "single" | |
| } | |
| }, | |
| "targets": [ | |
| { | |
| "datasource": "${datasource}", | |
| "exemplar": true, | |
| "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", | |
| "interval": "", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Series Created per Second", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${datasource}", | |
| "description": "Difference between highest timestamp appended to WAL and highest timestamp successfully written to remote storage", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [ ], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 12, | |
| "x": 0, | |
| "y": 10 | |
| }, | |
| "id": 6, | |
| "options": { | |
| "legend": { | |
| "calcs": [ ], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltip": { | |
| "mode": "single" | |
| } | |
| }, | |
| "targets": [ | |
| { | |
| "datasource": "${datasource}", | |
| "exemplar": true, | |
| "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{tenant=~\"${tenant}\"}\n- on (tenant)\n (\n loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{tenant=~\"${tenant}\"}\n or vector(0)\n )", | |
| "interval": "", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Write Behind", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${datasource}", | |
| "description": "", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [ ], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 12, | |
| "x": 12, | |
| "y": 10 | |
| }, | |
| "id": 7, | |
| "options": { | |
| "legend": { | |
| "calcs": [ ], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltip": { | |
| "mode": "single" | |
| } | |
| }, | |
| "targets": [ | |
| { | |
| "datasource": "${datasource}", | |
| "exemplar": true, | |
| "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", | |
| "interval": "", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Samples Sent per Second", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${datasource}", | |
| "description": "\n", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [ ], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| }, | |
| "unit": "bytes" | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 12, | |
| "x": 0, | |
| "y": 20 | |
| }, | |
| "id": 8, | |
| "options": { | |
| "legend": { | |
| "calcs": [ ], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltip": { | |
| "mode": "single" | |
| } | |
| }, | |
| "targets": [ | |
| { | |
| "datasource": "${datasource}", | |
| "exemplar": true, | |
| "expr": "sum by (tenant) (loki_ruler_wal_disk_size{tenant=~\"${tenant}\"})", | |
| "interval": "", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "WAL Disk Size", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": "${datasource}", | |
| "description": "Some number of pending samples is expected, but if remote-write is failing this value will remain high", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [ ], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": null | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 12, | |
| "x": 12, | |
| "y": 20 | |
| }, | |
| "id": 9, | |
| "options": { | |
| "legend": { | |
| "calcs": [ ], | |
| "displayMode": "list", | |
| "placement": "bottom" | |
| }, | |
| "tooltip": { | |
| "mode": "single" | |
| } | |
| }, | |
| "targets": [ | |
| { | |
| "datasource": "${datasource}", | |
| "exemplar": true, | |
| "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{tenant=~\"${tenant}\"}) by (tenant,pod) > 0", | |
| "interval": "", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Pending Samples", | |
| "type": "timeseries" | |
| } | |
| ], | |
| "schemaVersion": 31, | |
| "style": "dark", | |
| "tags": [ ], | |
| "templating": { | |
| "list": [ | |
| { | |
| "description": null, | |
| "error": null, | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": "Datasource", | |
| "multi": false, | |
| "name": "datasource", | |
| "options": [ ], | |
| "query": "prometheus", | |
| "queryValue": "", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "type": "datasource" | |
| }, | |
| { | |
| "allValue": null, | |
| "datasource": "${datasource}", | |
| "definition": "label_values(loki_ruler_wal_samples_appended_total, tenant)", | |
| "description": null, | |
| "error": null, | |
| "hide": 0, | |
| "includeAll": true, | |
| "label": "Tenant", | |
| "multi": true, | |
| "name": "tenant", | |
| "options": [ ], | |
| "query": { | |
| "query": "label_values(loki_ruler_wal_samples_appended_total, tenant)", | |
| "refId": "StandardVariableQuery" | |
| }, | |
| "refresh": 2, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "sort": 0, | |
| "type": "query" | |
| } | |
| ] | |
| }, | |
| "time": { | |
| "from": "now-6h", | |
| "to": "now" | |
| }, | |
| "timepicker": { }, | |
| "timezone": "", | |
| "title": "Recording Rules", | |
| "uid": "2xKA_ZK7k", | |
| "version": 9, | |
| "weekStart": "" | |
| } |