| { | |
| "annotations": { | |
| "list": [ ] | |
| }, | |
| "editable": true, | |
| "gnetId": null, | |
| "graphTooltip": 0, | |
| "hideControls": false, | |
| "links": [ | |
| { | |
| "asDropdown": true, | |
| "icon": "external link", | |
| "includeVars": true, | |
| "keepTime": true, | |
| "tags": [ | |
| "loki" | |
| ], | |
| "targetBlank": false, | |
| "title": "Loki Dashboards", | |
| "type": "dashboards" | |
| } | |
| ], | |
| "refresh": "10s", | |
| "rows": [ | |
| { | |
| "collapse": false, | |
| "collapsed": false, | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "gridPos": { }, | |
| "id": 1, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "In-memory streams", | |
| "tooltip": { | |
| "sort": 2 | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "gridPos": { }, | |
| "id": 2, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ | |
| { | |
| "alias": "limit", | |
| "color": "#E02F44", | |
| "fill": 0 | |
| } | |
| ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}}", | |
| "legendLink": null, | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "limit", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "CPU", | |
| "tooltip": { | |
| "sort": 2 | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "gridPos": { }, | |
| "id": 3, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ | |
| { | |
| "alias": "limit", | |
| "color": "#E02F44", | |
| "fill": 0 | |
| } | |
| ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}}", | |
| "legendLink": null, | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "limit", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Memory (workingset)", | |
| "tooltip": { | |
| "sort": 2 | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "bytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "gridPos": { }, | |
| "id": 4, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Memory (go heap inuse)", | |
| "tooltip": { | |
| "sort": 2 | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "bytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 10, | |
| "gridPos": { }, | |
| "id": 5, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 0, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": true, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}} - {{device}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Disk Writes", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "Bps", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 10, | |
| "gridPos": { }, | |
| "id": 6, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 0, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": true, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{pod}} - {{device}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Disk Reads", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "Bps", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "gridPos": { }, | |
| "id": 7, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{persistentvolumeclaim}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Disk Space Utilization", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "percentunit", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Write path", | |
| "titleSize": "h6", | |
| "type": "row" | |
| } | |
| ], | |
| "schemaVersion": 14, | |
| "style": "dark", | |
| "tags": [ | |
| "loki" | |
| ], | |
| "templating": { | |
| "list": [ | |
| { | |
| "current": { | |
| "text": "default", | |
| "value": "default" | |
| }, | |
| "hide": 0, | |
| "label": "Data Source", | |
| "name": "datasource", | |
| "options": [ ], | |
| "query": "prometheus", | |
| "refresh": 1, | |
| "regex": "", | |
| "type": "datasource" | |
| }, | |
| { | |
| "allValue": null, | |
| "current": { | |
| "text": "prod", | |
| "value": "prod" | |
| }, | |
| "datasource": "$datasource", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": "cluster", | |
| "multi": false, | |
| "name": "cluster", | |
| "options": [ ], | |
| "query": "label_values(loki_build_info, cluster)", | |
| "refresh": 1, | |
| "regex": "", | |
| "sort": 2, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": null, | |
| "current": { | |
| "text": "prod", | |
| "value": "prod" | |
| }, | |
| "datasource": "$datasource", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": "namespace", | |
| "multi": false, | |
| "name": "namespace", | |
| "options": [ ], | |
| "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", | |
| "refresh": 1, | |
| "regex": "", | |
| "sort": 2, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| } | |
| ] | |
| }, | |
| "time": { | |
| "from": "now-1h", | |
| "to": "now" | |
| }, | |
| "timepicker": { | |
| "refresh_intervals": [ | |
| "5s", | |
| "10s", | |
| "30s", | |
| "1m", | |
| "5m", | |
| "15m", | |
| "30m", | |
| "1h", | |
| "2h", | |
| "1d" | |
| ], | |
| "time_options": [ | |
| "5m", | |
| "15m", | |
| "1h", | |
| "6h", | |
| "12h", | |
| "24h", | |
| "2d", | |
| "7d", | |
| "30d" | |
| ] | |
| }, | |
| "timezone": "utc", | |
| "title": "Loki / Writes Resources", | |
| "uid": "writes-resources", | |
| "version": 0 | |
| } |