blob: 1b68bd340b5eafa6b88931ef02c6ec2af860d695 [file] [log] [blame]
Mohammed Naser8a2c8fb2023-02-19 17:23:55 +00001{
2 "annotations": {
3 "list": [ ]
4 },
5 "editable": true,
6 "gnetId": null,
7 "graphTooltip": 0,
8 "hideControls": false,
9 "links": [
10 {
11 "asDropdown": true,
12 "icon": "external link",
13 "includeVars": true,
14 "keepTime": true,
15 "tags": [
16 "loki"
17 ],
18 "targetBlank": false,
19 "title": "Loki Dashboards",
20 "type": "dashboards"
21 }
22 ],
23 "refresh": "10s",
24 "rows": [
25 {
26 "collapse": false,
27 "collapsed": false,
28 "panels": [
29 {
30 "aliasColors": { },
31 "bars": false,
32 "dashLength": 10,
33 "dashes": false,
34 "datasource": "$datasource",
35 "fill": 1,
36 "gridPos": { },
37 "id": 1,
38 "legend": {
39 "avg": false,
40 "current": false,
41 "max": false,
42 "min": false,
43 "show": true,
44 "total": false,
45 "values": false
46 },
47 "lines": true,
48 "linewidth": 1,
49 "links": [ ],
50 "nullPointMode": "null as zero",
51 "percentage": false,
52 "pointradius": 5,
53 "points": false,
54 "renderer": "flot",
55 "seriesOverrides": [ ],
56 "spaceLength": 10,
57 "span": 6,
58 "stack": false,
59 "steppedLine": false,
60 "targets": [
61 {
62 "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})",
63 "format": "time_series",
64 "intervalFactor": 2,
65 "legendFormat": "{{pod}}",
66 "legendLink": null,
67 "step": 10
68 }
69 ],
70 "thresholds": [ ],
71 "timeFrom": null,
72 "timeShift": null,
73 "title": "In-memory streams",
74 "tooltip": {
75 "sort": 2
76 },
77 "type": "graph",
78 "xaxis": {
79 "buckets": null,
80 "mode": "time",
81 "name": null,
82 "show": true,
83 "values": [ ]
84 },
85 "yaxes": [
86 {
87 "format": "short",
88 "label": null,
89 "logBase": 1,
90 "max": null,
91 "min": 0,
92 "show": true
93 },
94 {
95 "format": "short",
96 "label": null,
97 "logBase": 1,
98 "max": null,
99 "min": null,
100 "show": false
101 }
102 ]
103 },
104 {
105 "aliasColors": { },
106 "bars": false,
107 "dashLength": 10,
108 "dashes": false,
109 "datasource": "$datasource",
110 "fill": 1,
111 "gridPos": { },
112 "id": 2,
113 "legend": {
114 "avg": false,
115 "current": false,
116 "max": false,
117 "min": false,
118 "show": true,
119 "total": false,
120 "values": false
121 },
122 "lines": true,
123 "linewidth": 1,
124 "links": [ ],
125 "nullPointMode": "null as zero",
126 "percentage": false,
127 "pointradius": 5,
128 "points": false,
129 "renderer": "flot",
130 "seriesOverrides": [
131 {
132 "alias": "limit",
133 "color": "#E02F44",
134 "fill": 0
135 }
136 ],
137 "spaceLength": 10,
138 "span": 6,
139 "stack": false,
140 "steppedLine": false,
141 "targets": [
142 {
143 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))",
144 "format": "time_series",
145 "intervalFactor": 2,
146 "legendFormat": "{{pod}}",
147 "legendLink": null,
148 "step": 10
149 },
150 {
151 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})",
152 "format": "time_series",
153 "intervalFactor": 2,
154 "legendFormat": "limit",
155 "legendLink": null,
156 "step": 10
157 }
158 ],
159 "thresholds": [ ],
160 "timeFrom": null,
161 "timeShift": null,
162 "title": "CPU",
163 "tooltip": {
164 "sort": 2
165 },
166 "type": "graph",
167 "xaxis": {
168 "buckets": null,
169 "mode": "time",
170 "name": null,
171 "show": true,
172 "values": [ ]
173 },
174 "yaxes": [
175 {
176 "format": "short",
177 "label": null,
178 "logBase": 1,
179 "max": null,
180 "min": 0,
181 "show": true
182 },
183 {
184 "format": "short",
185 "label": null,
186 "logBase": 1,
187 "max": null,
188 "min": null,
189 "show": false
190 }
191 ]
192 },
193 {
194 "aliasColors": { },
195 "bars": false,
196 "dashLength": 10,
197 "dashes": false,
198 "datasource": "$datasource",
199 "fill": 1,
200 "gridPos": { },
201 "id": 3,
202 "legend": {
203 "avg": false,
204 "current": false,
205 "max": false,
206 "min": false,
207 "show": true,
208 "total": false,
209 "values": false
210 },
211 "lines": true,
212 "linewidth": 1,
213 "links": [ ],
214 "nullPointMode": "null as zero",
215 "percentage": false,
216 "pointradius": 5,
217 "points": false,
218 "renderer": "flot",
219 "seriesOverrides": [
220 {
221 "alias": "limit",
222 "color": "#E02F44",
223 "fill": 0
224 }
225 ],
226 "spaceLength": 10,
227 "span": 6,
228 "stack": false,
229 "steppedLine": false,
230 "targets": [
231 {
232 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})",
233 "format": "time_series",
234 "intervalFactor": 2,
235 "legendFormat": "{{pod}}",
236 "legendLink": null,
237 "step": 10
238 },
239 {
240 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)",
241 "format": "time_series",
242 "intervalFactor": 2,
243 "legendFormat": "limit",
244 "legendLink": null,
245 "step": 10
246 }
247 ],
248 "thresholds": [ ],
249 "timeFrom": null,
250 "timeShift": null,
251 "title": "Memory (workingset)",
252 "tooltip": {
253 "sort": 2
254 },
255 "type": "graph",
256 "xaxis": {
257 "buckets": null,
258 "mode": "time",
259 "name": null,
260 "show": true,
261 "values": [ ]
262 },
263 "yaxes": [
264 {
265 "format": "bytes",
266 "label": null,
267 "logBase": 1,
268 "max": null,
269 "min": 0,
270 "show": true
271 },
272 {
273 "format": "short",
274 "label": null,
275 "logBase": 1,
276 "max": null,
277 "min": null,
278 "show": false
279 }
280 ]
281 },
282 {
283 "aliasColors": { },
284 "bars": false,
285 "dashLength": 10,
286 "dashes": false,
287 "datasource": "$datasource",
288 "fill": 1,
289 "gridPos": { },
290 "id": 4,
291 "legend": {
292 "avg": false,
293 "current": false,
294 "max": false,
295 "min": false,
296 "show": true,
297 "total": false,
298 "values": false
299 },
300 "lines": true,
301 "linewidth": 1,
302 "links": [ ],
303 "nullPointMode": "null as zero",
304 "percentage": false,
305 "pointradius": 5,
306 "points": false,
307 "renderer": "flot",
308 "seriesOverrides": [ ],
309 "spaceLength": 10,
310 "span": 6,
311 "stack": false,
312 "steppedLine": false,
313 "targets": [
314 {
315 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})",
316 "format": "time_series",
317 "intervalFactor": 2,
318 "legendFormat": "{{pod}}",
319 "legendLink": null,
320 "step": 10
321 }
322 ],
323 "thresholds": [ ],
324 "timeFrom": null,
325 "timeShift": null,
326 "title": "Memory (go heap inuse)",
327 "tooltip": {
328 "sort": 2
329 },
330 "type": "graph",
331 "xaxis": {
332 "buckets": null,
333 "mode": "time",
334 "name": null,
335 "show": true,
336 "values": [ ]
337 },
338 "yaxes": [
339 {
340 "format": "bytes",
341 "label": null,
342 "logBase": 1,
343 "max": null,
344 "min": 0,
345 "show": true
346 },
347 {
348 "format": "short",
349 "label": null,
350 "logBase": 1,
351 "max": null,
352 "min": null,
353 "show": false
354 }
355 ]
356 },
357 {
358 "aliasColors": { },
359 "bars": false,
360 "dashLength": 10,
361 "dashes": false,
362 "datasource": "$datasource",
363 "fill": 10,
364 "gridPos": { },
365 "id": 5,
366 "legend": {
367 "avg": false,
368 "current": false,
369 "max": false,
370 "min": false,
371 "show": true,
372 "total": false,
373 "values": false
374 },
375 "lines": true,
376 "linewidth": 0,
377 "links": [ ],
378 "nullPointMode": "null as zero",
379 "percentage": false,
380 "pointradius": 5,
381 "points": false,
382 "renderer": "flot",
383 "seriesOverrides": [ ],
384 "spaceLength": 10,
385 "span": 6,
386 "stack": true,
387 "steppedLine": false,
388 "targets": [
389 {
390 "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
391 "format": "time_series",
392 "intervalFactor": 2,
393 "legendFormat": "{{pod}} - {{device}}",
394 "legendLink": null,
395 "step": 10
396 }
397 ],
398 "thresholds": [ ],
399 "timeFrom": null,
400 "timeShift": null,
401 "title": "Disk Writes",
402 "tooltip": {
403 "shared": true,
404 "sort": 2,
405 "value_type": "individual"
406 },
407 "type": "graph",
408 "xaxis": {
409 "buckets": null,
410 "mode": "time",
411 "name": null,
412 "show": true,
413 "values": [ ]
414 },
415 "yaxes": [
416 {
417 "format": "Bps",
418 "label": null,
419 "logBase": 1,
420 "max": null,
421 "min": 0,
422 "show": true
423 },
424 {
425 "format": "short",
426 "label": null,
427 "logBase": 1,
428 "max": null,
429 "min": null,
430 "show": false
431 }
432 ]
433 },
434 {
435 "aliasColors": { },
436 "bars": false,
437 "dashLength": 10,
438 "dashes": false,
439 "datasource": "$datasource",
440 "fill": 10,
441 "gridPos": { },
442 "id": 6,
443 "legend": {
444 "avg": false,
445 "current": false,
446 "max": false,
447 "min": false,
448 "show": true,
449 "total": false,
450 "values": false
451 },
452 "lines": true,
453 "linewidth": 0,
454 "links": [ ],
455 "nullPointMode": "null as zero",
456 "percentage": false,
457 "pointradius": 5,
458 "points": false,
459 "renderer": "flot",
460 "seriesOverrides": [ ],
461 "spaceLength": 10,
462 "span": 6,
463 "stack": true,
464 "steppedLine": false,
465 "targets": [
466 {
467 "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n",
468 "format": "time_series",
469 "intervalFactor": 2,
470 "legendFormat": "{{pod}} - {{device}}",
471 "legendLink": null,
472 "step": 10
473 }
474 ],
475 "thresholds": [ ],
476 "timeFrom": null,
477 "timeShift": null,
478 "title": "Disk Reads",
479 "tooltip": {
480 "shared": true,
481 "sort": 2,
482 "value_type": "individual"
483 },
484 "type": "graph",
485 "xaxis": {
486 "buckets": null,
487 "mode": "time",
488 "name": null,
489 "show": true,
490 "values": [ ]
491 },
492 "yaxes": [
493 {
494 "format": "Bps",
495 "label": null,
496 "logBase": 1,
497 "max": null,
498 "min": 0,
499 "show": true
500 },
501 {
502 "format": "short",
503 "label": null,
504 "logBase": 1,
505 "max": null,
506 "min": null,
507 "show": false
508 }
509 ]
510 },
511 {
512 "aliasColors": { },
513 "bars": false,
514 "dashLength": 10,
515 "dashes": false,
516 "datasource": "$datasource",
517 "fill": 1,
518 "gridPos": { },
519 "id": 7,
520 "legend": {
521 "avg": false,
522 "current": false,
523 "max": false,
524 "min": false,
525 "show": true,
526 "total": false,
527 "values": false
528 },
529 "lines": true,
530 "linewidth": 1,
531 "links": [ ],
532 "nullPointMode": "null as zero",
533 "percentage": false,
534 "pointradius": 5,
535 "points": false,
536 "renderer": "flot",
537 "seriesOverrides": [ ],
538 "spaceLength": 10,
539 "span": 6,
540 "stack": false,
541 "steppedLine": false,
542 "targets": [
543 {
544 "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})",
545 "format": "time_series",
546 "intervalFactor": 2,
547 "legendFormat": "{{persistentvolumeclaim}}",
548 "legendLink": null,
549 "step": 10
550 }
551 ],
552 "thresholds": [ ],
553 "timeFrom": null,
554 "timeShift": null,
555 "title": "Disk Space Utilization",
556 "tooltip": {
557 "shared": true,
558 "sort": 2,
559 "value_type": "individual"
560 },
561 "type": "graph",
562 "xaxis": {
563 "buckets": null,
564 "mode": "time",
565 "name": null,
566 "show": true,
567 "values": [ ]
568 },
569 "yaxes": [
570 {
571 "format": "percentunit",
572 "label": null,
573 "logBase": 1,
574 "max": null,
575 "min": 0,
576 "show": true
577 },
578 {
579 "format": "short",
580 "label": null,
581 "logBase": 1,
582 "max": null,
583 "min": null,
584 "show": false
585 }
586 ]
587 }
588 ],
589 "repeat": null,
590 "repeatIteration": null,
591 "repeatRowId": null,
592 "showTitle": true,
593 "title": "Write path",
594 "titleSize": "h6",
595 "type": "row"
596 }
597 ],
598 "schemaVersion": 14,
599 "style": "dark",
600 "tags": [
601 "loki"
602 ],
603 "templating": {
604 "list": [
605 {
606 "current": {
607 "text": "default",
608 "value": "default"
609 },
610 "hide": 0,
611 "label": "Data Source",
612 "name": "datasource",
613 "options": [ ],
614 "query": "prometheus",
615 "refresh": 1,
616 "regex": "",
617 "type": "datasource"
618 },
619 {
620 "allValue": null,
621 "current": {
622 "text": "prod",
623 "value": "prod"
624 },
625 "datasource": "$datasource",
626 "hide": 0,
627 "includeAll": false,
628 "label": "cluster",
629 "multi": false,
630 "name": "cluster",
631 "options": [ ],
632 "query": "label_values(loki_build_info, cluster)",
633 "refresh": 1,
634 "regex": "",
635 "sort": 2,
636 "tagValuesQuery": "",
637 "tags": [ ],
638 "tagsQuery": "",
639 "type": "query",
640 "useTags": false
641 },
642 {
643 "allValue": null,
644 "current": {
645 "text": "prod",
646 "value": "prod"
647 },
648 "datasource": "$datasource",
649 "hide": 0,
650 "includeAll": false,
651 "label": "namespace",
652 "multi": false,
653 "name": "namespace",
654 "options": [ ],
655 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
656 "refresh": 1,
657 "regex": "",
658 "sort": 2,
659 "tagValuesQuery": "",
660 "tags": [ ],
661 "tagsQuery": "",
662 "type": "query",
663 "useTags": false
664 }
665 ]
666 },
667 "time": {
668 "from": "now-1h",
669 "to": "now"
670 },
671 "timepicker": {
672 "refresh_intervals": [
673 "5s",
674 "10s",
675 "30s",
676 "1m",
677 "5m",
678 "15m",
679 "30m",
680 "1h",
681 "2h",
682 "1d"
683 ],
684 "time_options": [
685 "5m",
686 "15m",
687 "1h",
688 "6h",
689 "12h",
690 "24h",
691 "2d",
692 "7d",
693 "30d"
694 ]
695 },
696 "timezone": "utc",
697 "title": "Loki / Writes Resources",
698 "uid": "writes-resources",
699 "version": 0
700 }