blob: 7fc99ecad0d4cd5b555d32b733fdcb69e56feb4c [file] [log] [blame]
Mohammed Naser8a2c8fb2023-02-19 17:23:55 +00001{
2 "annotations": {
3 "list": [ ]
4 },
5 "editable": true,
6 "gnetId": null,
7 "graphTooltip": 0,
8 "hideControls": false,
9 "links": [
10 {
11 "asDropdown": true,
12 "icon": "external link",
13 "includeVars": true,
14 "keepTime": true,
15 "tags": [
16 "loki"
17 ],
18 "targetBlank": false,
19 "title": "Loki Dashboards",
20 "type": "dashboards"
21 }
22 ],
23 "refresh": "10s",
24 "rows": [
25 {
26 "collapse": false,
27 "height": "250px",
28 "panels": [
29 {
30 "aliasColors": { },
31 "bars": false,
32 "dashLength": 10,
33 "dashes": false,
34 "datasource": "$datasource",
35 "fill": 1,
36 "id": 1,
37 "legend": {
38 "avg": false,
39 "current": false,
40 "max": false,
41 "min": false,
42 "show": true,
43 "total": false,
44 "values": false
45 },
46 "lines": true,
47 "linewidth": 1,
48 "links": [ ],
49 "nullPointMode": "null as zero",
50 "percentage": false,
51 "pointradius": 5,
52 "points": false,
53 "renderer": "flot",
54 "seriesOverrides": [
55 {
56 "alias": "limit",
57 "color": "#E02F44",
58 "fill": 0
59 }
60 ],
61 "spaceLength": 10,
62 "span": 4,
63 "stack": false,
64 "steppedLine": false,
65 "targets": [
66 {
67 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))",
68 "format": "time_series",
69 "intervalFactor": 2,
70 "legendFormat": "{{pod}}",
71 "legendLink": null,
72 "step": 10
73 },
74 {
75 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})",
76 "format": "time_series",
77 "intervalFactor": 2,
78 "legendFormat": "limit",
79 "legendLink": null,
80 "step": 10
81 }
82 ],
83 "thresholds": [ ],
84 "timeFrom": null,
85 "timeShift": null,
86 "title": "CPU",
87 "tooltip": {
88 "sort": 2
89 },
90 "type": "graph",
91 "xaxis": {
92 "buckets": null,
93 "mode": "time",
94 "name": null,
95 "show": true,
96 "values": [ ]
97 },
98 "yaxes": [
99 {
100 "format": "short",
101 "label": null,
102 "logBase": 1,
103 "max": null,
104 "min": 0,
105 "show": true
106 },
107 {
108 "format": "short",
109 "label": null,
110 "logBase": 1,
111 "max": null,
112 "min": null,
113 "show": false
114 }
115 ]
116 },
117 {
118 "aliasColors": { },
119 "bars": false,
120 "dashLength": 10,
121 "dashes": false,
122 "datasource": "$datasource",
123 "fill": 1,
124 "id": 2,
125 "legend": {
126 "avg": false,
127 "current": false,
128 "max": false,
129 "min": false,
130 "show": true,
131 "total": false,
132 "values": false
133 },
134 "lines": true,
135 "linewidth": 1,
136 "links": [ ],
137 "nullPointMode": "null as zero",
138 "percentage": false,
139 "pointradius": 5,
140 "points": false,
141 "renderer": "flot",
142 "seriesOverrides": [
143 {
144 "alias": "limit",
145 "color": "#E02F44",
146 "fill": 0
147 }
148 ],
149 "spaceLength": 10,
150 "span": 4,
151 "stack": false,
152 "steppedLine": false,
153 "targets": [
154 {
155 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})",
156 "format": "time_series",
157 "intervalFactor": 2,
158 "legendFormat": "{{pod}}",
159 "legendLink": null,
160 "step": 10
161 },
162 {
163 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)",
164 "format": "time_series",
165 "intervalFactor": 2,
166 "legendFormat": "limit",
167 "legendLink": null,
168 "step": 10
169 }
170 ],
171 "thresholds": [ ],
172 "timeFrom": null,
173 "timeShift": null,
174 "title": "Memory (workingset)",
175 "tooltip": {
176 "sort": 2
177 },
178 "type": "graph",
179 "xaxis": {
180 "buckets": null,
181 "mode": "time",
182 "name": null,
183 "show": true,
184 "values": [ ]
185 },
186 "yaxes": [
187 {
188 "format": "bytes",
189 "label": null,
190 "logBase": 1,
191 "max": null,
192 "min": 0,
193 "show": true
194 },
195 {
196 "format": "short",
197 "label": null,
198 "logBase": 1,
199 "max": null,
200 "min": null,
201 "show": false
202 }
203 ]
204 },
205 {
206 "aliasColors": { },
207 "bars": false,
208 "dashLength": 10,
209 "dashes": false,
210 "datasource": "$datasource",
211 "fill": 1,
212 "id": 3,
213 "legend": {
214 "avg": false,
215 "current": false,
216 "max": false,
217 "min": false,
218 "show": true,
219 "total": false,
220 "values": false
221 },
222 "lines": true,
223 "linewidth": 1,
224 "links": [ ],
225 "nullPointMode": "null as zero",
226 "percentage": false,
227 "pointradius": 5,
228 "points": false,
229 "renderer": "flot",
230 "seriesOverrides": [ ],
231 "spaceLength": 10,
232 "span": 4,
233 "stack": false,
234 "steppedLine": false,
235 "targets": [
236 {
237 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})",
238 "format": "time_series",
239 "intervalFactor": 2,
240 "legendFormat": "{{pod}}",
241 "legendLink": null,
242 "step": 10
243 }
244 ],
245 "thresholds": [ ],
246 "timeFrom": null,
247 "timeShift": null,
248 "title": "Memory (go heap inuse)",
249 "tooltip": {
250 "sort": 2
251 },
252 "type": "graph",
253 "xaxis": {
254 "buckets": null,
255 "mode": "time",
256 "name": null,
257 "show": true,
258 "values": [ ]
259 },
260 "yaxes": [
261 {
262 "format": "bytes",
263 "label": null,
264 "logBase": 1,
265 "max": null,
266 "min": 0,
267 "show": true
268 },
269 {
270 "format": "short",
271 "label": null,
272 "logBase": 1,
273 "max": null,
274 "min": null,
275 "show": false
276 }
277 ]
278 }
279 ],
280 "repeat": null,
281 "repeatIteration": null,
282 "repeatRowId": null,
283 "showTitle": true,
284 "title": "Resource Usage",
285 "titleSize": "h6"
286 },
287 {
288 "collapse": false,
289 "height": "250px",
290 "panels": [
291 {
292 "aliasColors": { },
293 "bars": false,
294 "dashLength": 10,
295 "dashes": false,
296 "datasource": "$datasource",
297 "fieldConfig": {
298 "defaults": {
299 "color": {
300 "fixedColor": "blue",
301 "mode": "fixed"
302 },
303 "custom": { },
304 "thresholds": {
305 "mode": "absolute",
306 "steps": [
307 {
308 "color": "green",
309 "value": null
310 }
311 ]
312 },
313 "unit": "dateTimeFromNow"
314 }
315 },
316 "fill": 1,
317 "id": 4,
318 "legend": {
319 "avg": false,
320 "current": false,
321 "max": false,
322 "min": false,
323 "show": true,
324 "total": false,
325 "values": false
326 },
327 "lines": true,
328 "linewidth": 1,
329 "links": [ ],
330 "nullPointMode": "null as zero",
331 "options": {
332 "colorMode": "value",
333 "graphMode": "area",
334 "justifyMode": "auto",
335 "orientation": "auto",
336 "reduceOptions": {
337 "calcs": [
338 "lastNotNull"
339 ],
340 "fields": "",
341 "values": false
342 },
343 "text": { },
344 "textMode": "auto"
345 },
346 "percentage": false,
347 "pointradius": 5,
348 "points": false,
349 "renderer": "flot",
350 "seriesOverrides": [ ],
351 "spaceLength": 10,
352 "span": 4,
353 "stack": false,
354 "steppedLine": false,
355 "targets": [
356 {
357 "expr": "loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"} * 1e3",
358 "format": "time_series",
359 "instant": true,
360 "refId": "A"
361 }
362 ],
363 "thresholds": [ ],
364 "timeFrom": null,
365 "timeShift": null,
366 "title": "Last Compact and Mark Operation Success",
367 "tooltip": {
368 "shared": true,
369 "sort": 2,
370 "value_type": "individual"
371 },
372 "type": "stat",
373 "xaxis": {
374 "buckets": null,
375 "mode": "time",
376 "name": null,
377 "show": true,
378 "values": [ ]
379 },
380 "yaxes": [
381 {
382 "format": "short",
383 "label": null,
384 "logBase": 1,
385 "max": null,
386 "min": 0,
387 "show": true
388 },
389 {
390 "format": "short",
391 "label": null,
392 "logBase": 1,
393 "max": null,
394 "min": null,
395 "show": false
396 }
397 ]
398 },
399 {
400 "aliasColors": { },
401 "bars": false,
402 "dashLength": 10,
403 "dashes": false,
404 "datasource": "$datasource",
405 "fill": 1,
406 "id": 5,
407 "legend": {
408 "avg": false,
409 "current": false,
410 "max": false,
411 "min": false,
412 "show": true,
413 "total": false,
414 "values": false
415 },
416 "lines": true,
417 "linewidth": 1,
418 "links": [ ],
419 "nullPointMode": "null as zero",
420 "percentage": false,
421 "pointradius": 5,
422 "points": false,
423 "renderer": "flot",
424 "seriesOverrides": [ ],
425 "spaceLength": 10,
426 "span": 4,
427 "stack": false,
428 "steppedLine": false,
429 "targets": [
430 {
431 "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
432 "format": "time_series",
433 "intervalFactor": 2,
434 "legendFormat": "duration",
435 "legendLink": null,
436 "step": 10
437 }
438 ],
439 "thresholds": [ ],
440 "timeFrom": null,
441 "timeShift": null,
442 "title": "Compact and Mark Operations Duration",
443 "tooltip": {
444 "shared": true,
445 "sort": 2,
446 "value_type": "individual"
447 },
448 "type": "graph",
449 "xaxis": {
450 "buckets": null,
451 "mode": "time",
452 "name": null,
453 "show": true,
454 "values": [ ]
455 },
456 "yaxes": [
457 {
458 "format": "s",
459 "label": null,
460 "logBase": 1,
461 "max": null,
462 "min": 0,
463 "show": true
464 },
465 {
466 "format": "short",
467 "label": null,
468 "logBase": 1,
469 "max": null,
470 "min": null,
471 "show": false
472 }
473 ]
474 },
475 {
476 "aliasColors": { },
477 "bars": false,
478 "dashLength": 10,
479 "dashes": false,
480 "datasource": "$datasource",
481 "fill": 1,
482 "id": 6,
483 "legend": {
484 "avg": false,
485 "current": false,
486 "max": false,
487 "min": false,
488 "show": true,
489 "total": false,
490 "values": false
491 },
492 "lines": true,
493 "linewidth": 1,
494 "links": [ ],
495 "nullPointMode": "null as zero",
496 "percentage": false,
497 "pointradius": 5,
498 "points": false,
499 "renderer": "flot",
500 "seriesOverrides": [ ],
501 "spaceLength": 10,
502 "span": 4,
503 "stack": false,
504 "steppedLine": false,
505 "targets": [
506 {
507 "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
508 "format": "time_series",
509 "intervalFactor": 2,
510 "legendFormat": "{{success}}",
511 "legendLink": null,
512 "step": 10
513 }
514 ],
515 "thresholds": [ ],
516 "timeFrom": null,
517 "timeShift": null,
518 "title": "Compact and Mark Operations Per Status",
519 "tooltip": {
520 "shared": true,
521 "sort": 2,
522 "value_type": "individual"
523 },
524 "type": "graph",
525 "xaxis": {
526 "buckets": null,
527 "mode": "time",
528 "name": null,
529 "show": true,
530 "values": [ ]
531 },
532 "yaxes": [
533 {
534 "format": "short",
535 "label": null,
536 "logBase": 1,
537 "max": null,
538 "min": 0,
539 "show": true
540 },
541 {
542 "format": "short",
543 "label": null,
544 "logBase": 1,
545 "max": null,
546 "min": null,
547 "show": false
548 }
549 ]
550 }
551 ],
552 "repeat": null,
553 "repeatIteration": null,
554 "repeatRowId": null,
555 "showTitle": true,
556 "title": "Compact and Mark",
557 "titleSize": "h6"
558 },
559 {
560 "collapse": false,
561 "height": "250px",
562 "panels": [
563 {
564 "aliasColors": { },
565 "bars": false,
566 "dashLength": 10,
567 "dashes": false,
568 "datasource": "$datasource",
569 "fill": 10,
570 "id": 7,
571 "legend": {
572 "avg": false,
573 "current": false,
574 "max": false,
575 "min": false,
576 "show": true,
577 "total": false,
578 "values": false
579 },
580 "lines": true,
581 "linewidth": 0,
582 "links": [ ],
583 "nullPointMode": "null as zero",
584 "percentage": false,
585 "pointradius": 5,
586 "points": false,
587 "renderer": "flot",
588 "seriesOverrides": [ ],
589 "spaceLength": 10,
590 "span": 4,
591 "stack": true,
592 "steppedLine": false,
593 "targets": [
594 {
595 "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
596 "format": "time_series",
597 "intervalFactor": 2,
598 "legendFormat": "{{action}}",
599 "legendLink": null,
600 "step": 10
601 }
602 ],
603 "thresholds": [ ],
604 "timeFrom": null,
605 "timeShift": null,
606 "title": "Processed Tables Per Action",
607 "tooltip": {
608 "shared": true,
609 "sort": 2,
610 "value_type": "individual"
611 },
612 "type": "graph",
613 "xaxis": {
614 "buckets": null,
615 "mode": "time",
616 "name": null,
617 "show": true,
618 "values": [ ]
619 },
620 "yaxes": [
621 {
622 "format": "short",
623 "label": null,
624 "logBase": 1,
625 "max": null,
626 "min": 0,
627 "show": true
628 },
629 {
630 "format": "short",
631 "label": null,
632 "logBase": 1,
633 "max": null,
634 "min": null,
635 "show": false
636 }
637 ]
638 },
639 {
640 "aliasColors": { },
641 "bars": false,
642 "dashLength": 10,
643 "dashes": false,
644 "datasource": "$datasource",
645 "fill": 10,
646 "id": 8,
647 "legend": {
648 "avg": false,
649 "current": false,
650 "max": false,
651 "min": false,
652 "show": true,
653 "total": false,
654 "values": false
655 },
656 "lines": true,
657 "linewidth": 0,
658 "links": [ ],
659 "nullPointMode": "null as zero",
660 "percentage": false,
661 "pointradius": 5,
662 "points": false,
663 "renderer": "flot",
664 "seriesOverrides": [ ],
665 "spaceLength": 10,
666 "span": 4,
667 "stack": true,
668 "steppedLine": false,
669 "targets": [
670 {
671 "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})",
672 "format": "time_series",
673 "intervalFactor": 2,
674 "legendFormat": "{{table}}-{{action}}",
675 "legendLink": null,
676 "step": 10
677 }
678 ],
679 "thresholds": [ ],
680 "timeFrom": null,
681 "timeShift": null,
682 "title": "Modified Tables",
683 "tooltip": {
684 "shared": true,
685 "sort": 2,
686 "value_type": "individual"
687 },
688 "type": "graph",
689 "xaxis": {
690 "buckets": null,
691 "mode": "time",
692 "name": null,
693 "show": true,
694 "values": [ ]
695 },
696 "yaxes": [
697 {
698 "format": "short",
699 "label": null,
700 "logBase": 1,
701 "max": null,
702 "min": 0,
703 "show": true
704 },
705 {
706 "format": "short",
707 "label": null,
708 "logBase": 1,
709 "max": null,
710 "min": null,
711 "show": false
712 }
713 ]
714 },
715 {
716 "aliasColors": { },
717 "bars": false,
718 "dashLength": 10,
719 "dashes": false,
720 "datasource": "$datasource",
721 "fill": 10,
722 "id": 9,
723 "legend": {
724 "avg": false,
725 "current": false,
726 "max": false,
727 "min": false,
728 "show": true,
729 "total": false,
730 "values": false
731 },
732 "lines": true,
733 "linewidth": 0,
734 "links": [ ],
735 "nullPointMode": "null as zero",
736 "percentage": false,
737 "pointradius": 5,
738 "points": false,
739 "renderer": "flot",
740 "seriesOverrides": [ ],
741 "spaceLength": 10,
742 "span": 4,
743 "stack": true,
744 "steppedLine": false,
745 "targets": [
746 {
747 "expr": "sum by (table)(rate(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >0",
748 "format": "time_series",
749 "intervalFactor": 2,
750 "legendFormat": "{{table}}",
751 "legendLink": null,
752 "step": 10
753 }
754 ],
755 "thresholds": [ ],
756 "timeFrom": null,
757 "timeShift": null,
758 "title": "Marks Creation Rate Per Table",
759 "tooltip": {
760 "shared": true,
761 "sort": 2,
762 "value_type": "individual"
763 },
764 "type": "graph",
765 "xaxis": {
766 "buckets": null,
767 "mode": "time",
768 "name": null,
769 "show": true,
770 "values": [ ]
771 },
772 "yaxes": [
773 {
774 "format": "short",
775 "label": null,
776 "logBase": 1,
777 "max": null,
778 "min": 0,
779 "show": true
780 },
781 {
782 "format": "short",
783 "label": null,
784 "logBase": 1,
785 "max": null,
786 "min": null,
787 "show": false
788 }
789 ]
790 }
791 ],
792 "repeat": null,
793 "repeatIteration": null,
794 "repeatRowId": null,
795 "showTitle": true,
796 "title": "Per Table Marker",
797 "titleSize": "h6"
798 },
799 {
800 "collapse": false,
801 "height": "250px",
802 "panels": [
803 {
804 "aliasColors": { },
805 "bars": false,
806 "dashLength": 10,
807 "dashes": false,
808 "datasource": "$datasource",
809 "fill": 1,
810 "format": "short",
811 "id": 10,
812 "legend": {
813 "avg": false,
814 "current": false,
815 "max": false,
816 "min": false,
817 "show": true,
818 "total": false,
819 "values": false
820 },
821 "lines": true,
822 "linewidth": 1,
823 "links": [ ],
824 "nullPointMode": "null as zero",
825 "percentage": false,
826 "pointradius": 5,
827 "points": false,
828 "renderer": "flot",
829 "seriesOverrides": [ ],
830 "spaceLength": 10,
831 "span": 6,
832 "stack": false,
833 "steppedLine": false,
834 "targets": [
835 {
836 "expr": "sum (increase(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))",
837 "format": "time_series",
838 "instant": true,
839 "intervalFactor": 2,
840 "refId": "A"
841 }
842 ],
843 "thresholds": "70,80",
844 "timeFrom": null,
845 "timeShift": null,
846 "title": "Marked Chunks (24h)",
847 "tooltip": {
848 "shared": true,
849 "sort": 2,
850 "value_type": "individual"
851 },
852 "type": "singlestat",
853 "xaxis": {
854 "buckets": null,
855 "mode": "time",
856 "name": null,
857 "show": true,
858 "values": [ ]
859 },
860 "yaxes": [
861 {
862 "format": "short",
863 "label": null,
864 "logBase": 1,
865 "max": null,
866 "min": 0,
867 "show": true
868 },
869 {
870 "format": "short",
871 "label": null,
872 "logBase": 1,
873 "max": null,
874 "min": null,
875 "show": false
876 }
877 ]
878 },
879 {
880 "aliasColors": { },
881 "bars": false,
882 "dashLength": 10,
883 "dashes": false,
884 "datasource": "$datasource",
885 "fill": 1,
886 "id": 11,
887 "legend": {
888 "avg": false,
889 "current": false,
890 "max": false,
891 "min": false,
892 "show": true,
893 "total": false,
894 "values": false
895 },
896 "lines": true,
897 "linewidth": 1,
898 "links": [ ],
899 "nullPointMode": "null as zero",
900 "percentage": false,
901 "pointradius": 5,
902 "points": false,
903 "renderer": "flot",
904 "seriesOverrides": [ ],
905 "spaceLength": 10,
906 "span": 6,
907 "stack": false,
908 "steppedLine": false,
909 "targets": [
910 {
911 "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
912 "format": "time_series",
913 "intervalFactor": 2,
914 "legendFormat": "99th Percentile",
915 "refId": "A",
916 "step": 10
917 },
918 {
919 "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
920 "format": "time_series",
921 "intervalFactor": 2,
922 "legendFormat": "50th Percentile",
923 "refId": "B",
924 "step": 10
925 },
926 {
927 "expr": "sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
928 "format": "time_series",
929 "intervalFactor": 2,
930 "legendFormat": "Average",
931 "refId": "C",
932 "step": 10
933 }
934 ],
935 "thresholds": [ ],
936 "timeFrom": null,
937 "timeShift": null,
938 "title": "Mark Table Latency",
939 "tooltip": {
940 "shared": true,
941 "sort": 2,
942 "value_type": "individual"
943 },
944 "type": "graph",
945 "xaxis": {
946 "buckets": null,
947 "mode": "time",
948 "name": null,
949 "show": true,
950 "values": [ ]
951 },
952 "yaxes": [
953 {
954 "format": "ms",
955 "label": null,
956 "logBase": 1,
957 "max": null,
958 "min": 0,
959 "show": true
960 },
961 {
962 "format": "short",
963 "label": null,
964 "logBase": 1,
965 "max": null,
966 "min": null,
967 "show": false
968 }
969 ]
970 }
971 ],
972 "repeat": null,
973 "repeatIteration": null,
974 "repeatRowId": null,
975 "showTitle": true,
976 "title": "",
977 "titleSize": "h6"
978 },
979 {
980 "collapse": false,
981 "height": "250px",
982 "panels": [
983 {
984 "aliasColors": { },
985 "bars": false,
986 "dashLength": 10,
987 "dashes": false,
988 "datasource": "$datasource",
989 "fill": 1,
990 "format": "short",
991 "id": 12,
992 "legend": {
993 "avg": false,
994 "current": false,
995 "max": false,
996 "min": false,
997 "show": true,
998 "total": false,
999 "values": false
1000 },
1001 "lines": true,
1002 "linewidth": 1,
1003 "links": [ ],
1004 "nullPointMode": "null as zero",
1005 "percentage": false,
1006 "pointradius": 5,
1007 "points": false,
1008 "renderer": "flot",
1009 "seriesOverrides": [ ],
1010 "spaceLength": 10,
1011 "span": 6,
1012 "stack": false,
1013 "steppedLine": false,
1014 "targets": [
1015 {
1016 "expr": "sum (increase(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))",
1017 "format": "time_series",
1018 "instant": true,
1019 "intervalFactor": 2,
1020 "refId": "A"
1021 }
1022 ],
1023 "thresholds": "70,80",
1024 "timeFrom": null,
1025 "timeShift": null,
1026 "title": "Delete Chunks (24h)",
1027 "tooltip": {
1028 "shared": true,
1029 "sort": 2,
1030 "value_type": "individual"
1031 },
1032 "type": "singlestat",
1033 "xaxis": {
1034 "buckets": null,
1035 "mode": "time",
1036 "name": null,
1037 "show": true,
1038 "values": [ ]
1039 },
1040 "yaxes": [
1041 {
1042 "format": "short",
1043 "label": null,
1044 "logBase": 1,
1045 "max": null,
1046 "min": 0,
1047 "show": true
1048 },
1049 {
1050 "format": "short",
1051 "label": null,
1052 "logBase": 1,
1053 "max": null,
1054 "min": null,
1055 "show": false
1056 }
1057 ]
1058 },
1059 {
1060 "aliasColors": { },
1061 "bars": false,
1062 "dashLength": 10,
1063 "dashes": false,
1064 "datasource": "$datasource",
1065 "fill": 1,
1066 "id": 13,
1067 "legend": {
1068 "avg": false,
1069 "current": false,
1070 "max": false,
1071 "min": false,
1072 "show": true,
1073 "total": false,
1074 "values": false
1075 },
1076 "lines": true,
1077 "linewidth": 1,
1078 "links": [ ],
1079 "nullPointMode": "null as zero",
1080 "percentage": false,
1081 "pointradius": 5,
1082 "points": false,
1083 "renderer": "flot",
1084 "seriesOverrides": [ ],
1085 "spaceLength": 10,
1086 "span": 6,
1087 "stack": false,
1088 "steppedLine": false,
1089 "targets": [
1090 {
1091 "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
1092 "format": "time_series",
1093 "intervalFactor": 2,
1094 "legendFormat": "99th Percentile",
1095 "refId": "A",
1096 "step": 10
1097 },
1098 {
1099 "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
1100 "format": "time_series",
1101 "intervalFactor": 2,
1102 "legendFormat": "50th Percentile",
1103 "refId": "B",
1104 "step": 10
1105 },
1106 {
1107 "expr": "sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
1108 "format": "time_series",
1109 "intervalFactor": 2,
1110 "legendFormat": "Average",
1111 "refId": "C",
1112 "step": 10
1113 }
1114 ],
1115 "thresholds": [ ],
1116 "timeFrom": null,
1117 "timeShift": null,
1118 "title": "Delete Latency",
1119 "tooltip": {
1120 "shared": true,
1121 "sort": 2,
1122 "value_type": "individual"
1123 },
1124 "type": "graph",
1125 "xaxis": {
1126 "buckets": null,
1127 "mode": "time",
1128 "name": null,
1129 "show": true,
1130 "values": [ ]
1131 },
1132 "yaxes": [
1133 {
1134 "format": "ms",
1135 "label": null,
1136 "logBase": 1,
1137 "max": null,
1138 "min": 0,
1139 "show": true
1140 },
1141 {
1142 "format": "short",
1143 "label": null,
1144 "logBase": 1,
1145 "max": null,
1146 "min": null,
1147 "show": false
1148 }
1149 ]
1150 }
1151 ],
1152 "repeat": null,
1153 "repeatIteration": null,
1154 "repeatRowId": null,
1155 "showTitle": true,
1156 "title": "Sweeper",
1157 "titleSize": "h6"
1158 },
1159 {
1160 "collapse": false,
1161 "height": "250px",
1162 "panels": [
1163 {
1164 "aliasColors": { },
1165 "bars": false,
1166 "dashLength": 10,
1167 "dashes": false,
1168 "datasource": "$datasource",
1169 "fill": 1,
1170 "id": 14,
1171 "legend": {
1172 "avg": false,
1173 "current": false,
1174 "max": false,
1175 "min": false,
1176 "show": true,
1177 "total": false,
1178 "values": false
1179 },
1180 "lines": true,
1181 "linewidth": 1,
1182 "links": [ ],
1183 "nullPointMode": "null as zero",
1184 "percentage": false,
1185 "pointradius": 5,
1186 "points": false,
1187 "renderer": "flot",
1188 "seriesOverrides": [ ],
1189 "spaceLength": 10,
1190 "span": 4,
1191 "stack": false,
1192 "steppedLine": false,
1193 "targets": [
1194 {
1195 "expr": "time() - (loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time{cluster=~\"$cluster\", namespace=~\"$namespace\"} > 0)",
1196 "format": "time_series",
1197 "intervalFactor": 2,
1198 "legendFormat": "lag",
1199 "legendLink": null,
1200 "step": 10
1201 }
1202 ],
1203 "thresholds": [ ],
1204 "timeFrom": null,
1205 "timeShift": null,
1206 "title": "Sweeper Lag",
1207 "tooltip": {
1208 "shared": true,
1209 "sort": 2,
1210 "value_type": "individual"
1211 },
1212 "type": "graph",
1213 "xaxis": {
1214 "buckets": null,
1215 "mode": "time",
1216 "name": null,
1217 "show": true,
1218 "values": [ ]
1219 },
1220 "yaxes": [
1221 {
1222 "format": "s",
1223 "label": null,
1224 "logBase": 1,
1225 "max": null,
1226 "min": null,
1227 "show": true
1228 },
1229 {
1230 "format": "short",
1231 "label": null,
1232 "logBase": 1,
1233 "max": null,
1234 "min": null,
1235 "show": false
1236 }
1237 ]
1238 },
1239 {
1240 "aliasColors": { },
1241 "bars": false,
1242 "dashLength": 10,
1243 "dashes": false,
1244 "datasource": "$datasource",
1245 "fill": 1,
1246 "id": 15,
1247 "legend": {
1248 "avg": false,
1249 "current": false,
1250 "max": false,
1251 "min": false,
1252 "show": true,
1253 "total": false,
1254 "values": false
1255 },
1256 "lines": true,
1257 "linewidth": 1,
1258 "links": [ ],
1259 "nullPointMode": "null as zero",
1260 "percentage": false,
1261 "pointradius": 5,
1262 "points": false,
1263 "renderer": "flot",
1264 "seriesOverrides": [ ],
1265 "spaceLength": 10,
1266 "span": 4,
1267 "stack": false,
1268 "steppedLine": false,
1269 "targets": [
1270 {
1271 "expr": "sum(loki_boltdb_shipper_retention_sweeper_marker_files_current{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
1272 "format": "time_series",
1273 "intervalFactor": 2,
1274 "legendFormat": "count",
1275 "legendLink": null,
1276 "step": 10
1277 }
1278 ],
1279 "thresholds": [ ],
1280 "timeFrom": null,
1281 "timeShift": null,
1282 "title": "Marks Files to Process",
1283 "tooltip": {
1284 "shared": true,
1285 "sort": 2,
1286 "value_type": "individual"
1287 },
1288 "type": "graph",
1289 "xaxis": {
1290 "buckets": null,
1291 "mode": "time",
1292 "name": null,
1293 "show": true,
1294 "values": [ ]
1295 },
1296 "yaxes": [
1297 {
1298 "format": "short",
1299 "label": null,
1300 "logBase": 1,
1301 "max": null,
1302 "min": 0,
1303 "show": true
1304 },
1305 {
1306 "format": "short",
1307 "label": null,
1308 "logBase": 1,
1309 "max": null,
1310 "min": null,
1311 "show": false
1312 }
1313 ]
1314 },
1315 {
1316 "aliasColors": { },
1317 "bars": false,
1318 "dashLength": 10,
1319 "dashes": false,
1320 "datasource": "$datasource",
1321 "fill": 1,
1322 "id": 16,
1323 "legend": {
1324 "avg": false,
1325 "current": false,
1326 "max": false,
1327 "min": false,
1328 "show": true,
1329 "total": false,
1330 "values": false
1331 },
1332 "lines": true,
1333 "linewidth": 1,
1334 "links": [ ],
1335 "nullPointMode": "null as zero",
1336 "percentage": false,
1337 "pointradius": 5,
1338 "points": false,
1339 "renderer": "flot",
1340 "seriesOverrides": [ ],
1341 "spaceLength": 10,
1342 "span": 4,
1343 "stack": false,
1344 "steppedLine": false,
1345 "targets": [
1346 {
1347 "expr": "sum by (status)(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
1348 "format": "time_series",
1349 "intervalFactor": 2,
1350 "legendFormat": "{{status}}",
1351 "legendLink": null,
1352 "step": 10
1353 }
1354 ],
1355 "thresholds": [ ],
1356 "timeFrom": null,
1357 "timeShift": null,
1358 "title": "Delete Rate Per Status",
1359 "tooltip": {
1360 "shared": true,
1361 "sort": 2,
1362 "value_type": "individual"
1363 },
1364 "type": "graph",
1365 "xaxis": {
1366 "buckets": null,
1367 "mode": "time",
1368 "name": null,
1369 "show": true,
1370 "values": [ ]
1371 },
1372 "yaxes": [
1373 {
1374 "format": "short",
1375 "label": null,
1376 "logBase": 1,
1377 "max": null,
1378 "min": 0,
1379 "show": true
1380 },
1381 {
1382 "format": "short",
1383 "label": null,
1384 "logBase": 1,
1385 "max": null,
1386 "min": null,
1387 "show": false
1388 }
1389 ]
1390 }
1391 ],
1392 "repeat": null,
1393 "repeatIteration": null,
1394 "repeatRowId": null,
1395 "showTitle": true,
1396 "title": "",
1397 "titleSize": "h6"
1398 },
1399 {
1400 "collapse": false,
1401 "height": "250px",
1402 "panels": [
1403 {
1404 "datasource": "$logs",
1405 "id": 17,
1406 "span": 12,
1407 "targets": [
1408 {
1409 "expr": "{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}",
1410 "refId": "A"
1411 }
1412 ],
1413 "title": "Compactor Logs",
1414 "type": "logs"
1415 }
1416 ],
1417 "repeat": null,
1418 "repeatIteration": null,
1419 "repeatRowId": null,
1420 "showTitle": true,
1421 "title": "Logs",
1422 "titleSize": "h6"
1423 }
1424 ],
1425 "schemaVersion": 14,
1426 "style": "dark",
1427 "tags": [
1428 "loki"
1429 ],
1430 "templating": {
1431 "list": [
1432 {
1433 "current": {
1434 "text": "default",
1435 "value": "default"
1436 },
1437 "hide": 0,
1438 "label": "Data Source",
1439 "name": "datasource",
1440 "options": [ ],
1441 "query": "prometheus",
1442 "refresh": 1,
1443 "regex": "",
1444 "type": "datasource"
1445 },
1446 {
1447 "allValue": null,
1448 "current": {
1449 "text": "prod",
1450 "value": "prod"
1451 },
1452 "datasource": "$datasource",
1453 "hide": 0,
1454 "includeAll": false,
1455 "label": "cluster",
1456 "multi": false,
1457 "name": "cluster",
1458 "options": [ ],
1459 "query": "label_values(loki_build_info, cluster)",
1460 "refresh": 1,
1461 "regex": "",
1462 "sort": 2,
1463 "tagValuesQuery": "",
1464 "tags": [ ],
1465 "tagsQuery": "",
1466 "type": "query",
1467 "useTags": false
1468 },
1469 {
1470 "allValue": null,
1471 "current": {
1472 "text": "prod",
1473 "value": "prod"
1474 },
1475 "datasource": "$datasource",
1476 "hide": 0,
1477 "includeAll": false,
1478 "label": "namespace",
1479 "multi": false,
1480 "name": "namespace",
1481 "options": [ ],
1482 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
1483 "refresh": 1,
1484 "regex": "",
1485 "sort": 2,
1486 "tagValuesQuery": "",
1487 "tags": [ ],
1488 "tagsQuery": "",
1489 "type": "query",
1490 "useTags": false
1491 },
1492 {
1493 "hide": 0,
1494 "label": null,
1495 "name": "logs",
1496 "options": [ ],
1497 "query": "loki",
1498 "refresh": 1,
1499 "regex": "",
1500 "type": "datasource"
1501 }
1502 ]
1503 },
1504 "time": {
1505 "from": "now-1h",
1506 "to": "now"
1507 },
1508 "timepicker": {
1509 "refresh_intervals": [
1510 "5s",
1511 "10s",
1512 "30s",
1513 "1m",
1514 "5m",
1515 "15m",
1516 "30m",
1517 "1h",
1518 "2h",
1519 "1d"
1520 ],
1521 "time_options": [
1522 "5m",
1523 "15m",
1524 "1h",
1525 "6h",
1526 "12h",
1527 "24h",
1528 "2d",
1529 "7d",
1530 "30d"
1531 ]
1532 },
1533 "timezone": "utc",
1534 "title": "Loki / Retention",
1535 "uid": "retention",
1536 "version": 0
1537 }