| { | |
| "annotations": { | |
| "list": [ ] | |
| }, | |
| "editable": true, | |
| "gnetId": null, | |
| "graphTooltip": 0, | |
| "hideControls": false, | |
| "links": [ | |
| { | |
| "asDropdown": true, | |
| "icon": "external link", | |
| "includeVars": true, | |
| "keepTime": true, | |
| "tags": [ | |
| "loki" | |
| ], | |
| "targetBlank": false, | |
| "title": "Loki Dashboards", | |
| "type": "dashboards" | |
| } | |
| ], | |
| "refresh": "10s", | |
| "rows": [ | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 1, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "series", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Series", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 2, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "chunks", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Chunks per series", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Active Series / Chunks", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 3, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "99th Percentile", | |
| "refId": "A", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "50th Percentile", | |
| "refId": "B", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "Average", | |
| "refId": "C", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Utilization", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "percentunit", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 4, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "99th Percentile", | |
| "refId": "A", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "50th Percentile", | |
| "refId": "B", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "Average", | |
| "refId": "C", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Age", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "ms", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Flush Stats", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 5, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "99th Percentile", | |
| "refId": "A", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "50th Percentile", | |
| "refId": "B", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "Average", | |
| "refId": "C", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Log Entries Per Chunk", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 6, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "Index Entries", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Index Entries Per Chunk", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Flush Stats", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 7, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "loki_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"} or cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Queue Length", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { | |
| "1xx": "#EAB839", | |
| "2xx": "#7EB26D", | |
| "3xx": "#6ED0E0", | |
| "4xx": "#EF843C", | |
| "5xx": "#E24D42", | |
| "error": "#E24D42", | |
| "success": "#7EB26D" | |
| }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 10, | |
| "id": 8, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 0, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": true, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{status}}", | |
| "refId": "A", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Flush Rate", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Flush Stats", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 9, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Chunks Flushed/Second", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 10, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": true, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{reason}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Chunk Flush Reason", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": 1, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": 1, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Flush Stats", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "cards": { | |
| "cardPadding": null, | |
| "cardRound": null | |
| }, | |
| "color": { | |
| "cardColor": "#b4ff00", | |
| "colorScale": "sqrt", | |
| "colorScheme": "interpolateSpectral", | |
| "exponent": 0.5, | |
| "mode": "spectrum" | |
| }, | |
| "dataFormat": "tsbuckets", | |
| "datasource": "$datasource", | |
| "heatmap": { }, | |
| "hideZeroBuckets": false, | |
| "highlightCards": true, | |
| "id": 11, | |
| "legend": { | |
| "show": true | |
| }, | |
| "span": 12, | |
| "targets": [ | |
| { | |
| "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", | |
| "format": "heatmap", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{le}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Chunk Utilization", | |
| "tooltip": { | |
| "show": true, | |
| "showHistogram": true | |
| }, | |
| "type": "heatmap", | |
| "xAxis": { | |
| "show": true | |
| }, | |
| "xBucketNumber": null, | |
| "xBucketSize": null, | |
| "yAxis": { | |
| "decimals": 0, | |
| "format": "percentunit", | |
| "show": true, | |
| "splitFactor": null | |
| }, | |
| "yBucketBound": "auto" | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Utilization", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "cards": { | |
| "cardPadding": null, | |
| "cardRound": null | |
| }, | |
| "color": { | |
| "cardColor": "#b4ff00", | |
| "colorScale": "sqrt", | |
| "colorScheme": "interpolateSpectral", | |
| "exponent": 0.5, | |
| "mode": "spectrum" | |
| }, | |
| "dataFormat": "tsbuckets", | |
| "datasource": "$datasource", | |
| "heatmap": { }, | |
| "hideZeroBuckets": false, | |
| "highlightCards": true, | |
| "id": 12, | |
| "legend": { | |
| "show": true | |
| }, | |
| "span": 12, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)", | |
| "format": "heatmap", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{le}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Chunk Size Bytes", | |
| "tooltip": { | |
| "show": true, | |
| "showHistogram": true | |
| }, | |
| "type": "heatmap", | |
| "xAxis": { | |
| "show": true | |
| }, | |
| "xBucketNumber": null, | |
| "xBucketSize": null, | |
| "yAxis": { | |
| "decimals": 0, | |
| "format": "bytes", | |
| "show": true, | |
| "splitFactor": null | |
| }, | |
| "yBucketBound": "auto" | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Utilization", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 13, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 12, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "p99", | |
| "legendLink": null, | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "p90", | |
| "legendLink": null, | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "p50", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Chunk Size Quantiles", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "bytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Utilization", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 14, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 12, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "p50", | |
| "legendLink": null, | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "p99", | |
| "legendLink": null, | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "avg", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Chunk Duration hours (end-start)", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Duration", | |
| "titleSize": "h6" | |
| } | |
| ], | |
| "schemaVersion": 14, | |
| "style": "dark", | |
| "tags": [ | |
| "loki" | |
| ], | |
| "templating": { | |
| "list": [ | |
| { | |
| "current": { | |
| "text": "default", | |
| "value": "default" | |
| }, | |
| "hide": 0, | |
| "label": "Data Source", | |
| "name": "datasource", | |
| "options": [ ], | |
| "query": "prometheus", | |
| "refresh": 1, | |
| "regex": "", | |
| "type": "datasource" | |
| }, | |
| { | |
| "allValue": null, | |
| "current": { | |
| "text": "prod", | |
| "value": "prod" | |
| }, | |
| "datasource": "$datasource", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": "cluster", | |
| "multi": false, | |
| "name": "cluster", | |
| "options": [ ], | |
| "query": "label_values(loki_build_info, cluster)", | |
| "refresh": 1, | |
| "regex": "", | |
| "sort": 2, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": null, | |
| "current": { | |
| "text": "prod", | |
| "value": "prod" | |
| }, | |
| "datasource": "$datasource", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": "namespace", | |
| "multi": false, | |
| "name": "namespace", | |
| "options": [ ], | |
| "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", | |
| "refresh": 1, | |
| "regex": "", | |
| "sort": 2, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| } | |
| ] | |
| }, | |
| "time": { | |
| "from": "now-1h", | |
| "to": "now" | |
| }, | |
| "timepicker": { | |
| "refresh_intervals": [ | |
| "5s", | |
| "10s", | |
| "30s", | |
| "1m", | |
| "5m", | |
| "15m", | |
| "30m", | |
| "1h", | |
| "2h", | |
| "1d" | |
| ], | |
| "time_options": [ | |
| "5m", | |
| "15m", | |
| "1h", | |
| "6h", | |
| "12h", | |
| "24h", | |
| "2d", | |
| "7d", | |
| "30d" | |
| ] | |
| }, | |
| "timezone": "utc", | |
| "title": "Loki / Chunks", | |
| "uid": "chunks", | |
| "version": 0 | |
| } |