Gitiles
Code Review Sign In
review.vexxhost.dev / atmosphere / refs/heads/main / . / charts / loki / src / dashboards / loki-operational.json
blob: b69db2370a1b2d36c07553e98334b503be29fd30 [file] [log] [blame]
Mohammed Naser8a2c8fb2023-02-19 17:23:55 +0000[diff] [blame]1{
2 "annotations": {
3 "list": [ ]
4 },
5 "editable": true,
6 "gnetId": null,
7 "graphTooltip": 0,
8 "hideControls": false,
9 "id": 68,
10 "iteration": 1588704280892,
11 "links": [
12 {
13 "asDropdown": true,
14 "icon": "external link",
15 "includeVars": true,
16 "keepTime": true,
17 "tags": [
18 "loki"
19 ],
20 "targetBlank": false,
21 "title": "Loki Dashboards",
22 "type": "dashboards"
23 }
24 ],
25 "panels": [
26 {
27 "collapsed": false,
28 "datasource": null,
29 "gridPos": {
30 "h": 1,
31 "w": 24,
32 "x": 0,
33 "y": 0
34 },
35 "id": 17,
36 "panels": [ ],
37 "targets": [ ],
38 "title": "Main",
39 "type": "row"
40 },
41 {
42 "aliasColors": {
43 "5xx": "red"
44 },
45 "bars": false,
46 "dashLength": 10,
47 "dashes": false,
48 "datasource": "$datasource",
49 "fieldConfig": {
50 "defaults": {
51 "custom": { }
52 },
53 "overrides": [ ]
54 },
55 "fill": 1,
56 "fillGradient": 0,
57 "gridPos": {
58 "h": 5,
59 "w": 4,
60 "x": 0,
61 "y": 1
62 },
63 "hiddenSeries": false,
64 "id": 6,
65 "legend": {
66 "avg": false,
67 "current": false,
68 "max": false,
69 "min": false,
70 "show": true,
71 "total": false,
72 "values": false
73 },
74 "lines": true,
75 "linewidth": 1,
76 "nullPointMode": "null",
77 "options": {
78 "dataLinks": [ ]
79 },
80 "panels": [ ],
81 "percentage": false,
82 "pointradius": 2,
83 "points": false,
84 "renderer": "flot",
85 "seriesOverrides": [ ],
86 "spaceLength": 10,
87 "stack": false,
88 "steppedLine": false,
89 "targets": [
90 {
91 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)",
92 "legendFormat": "{{status}}",
93 "refId": "A"
94 }
95 ],
96 "thresholds": [ ],
97 "timeFrom": null,
98 "timeRegions": [ ],
99 "timeShift": null,
100 "title": "Queries/Second",
101 "tooltip": {
102 "shared": true,
103 "sort": 0,
104 "value_type": "individual"
105 },
106 "type": "graph",
107 "xaxis": {
108 "buckets": null,
109 "mode": "time",
110 "name": null,
111 "show": true,
112 "values": [ ]
113 },
114 "yaxes": [
115 {
116 "format": "short",
117 "label": null,
118 "logBase": 10,
119 "max": null,
120 "min": null,
121 "show": true
122 },
123 {
124 "format": "short",
125 "label": null,
126 "logBase": 1,
127 "max": null,
128 "min": null,
129 "show": true
130 }
131 ],
132 "yaxis": {
133 "align": false,
134 "alignLevel": null
135 }
136 },
137 {
138 "aliasColors": {
139 "5xx": "red"
140 },
141 "bars": false,
142 "dashLength": 10,
143 "dashes": false,
144 "datasource": "$datasource",
145 "fieldConfig": {
146 "defaults": {
147 "custom": { }
148 },
149 "overrides": [ ]
150 },
151 "fill": 1,
152 "fillGradient": 0,
153 "gridPos": {
154 "h": 5,
155 "w": 4,
156 "x": 4,
157 "y": 1
158 },
159 "hiddenSeries": false,
160 "id": 7,
161 "legend": {
162 "avg": false,
163 "current": false,
164 "max": false,
165 "min": false,
166 "show": true,
167 "total": false,
168 "values": false
169 },
170 "lines": true,
171 "linewidth": 1,
172 "nullPointMode": "null",
173 "options": {
174 "dataLinks": [ ]
175 },
176 "panels": [ ],
177 "percentage": false,
178 "pointradius": 2,
179 "points": false,
180 "renderer": "flot",
181 "seriesOverrides": [ ],
182 "spaceLength": 10,
183 "stack": false,
184 "steppedLine": false,
185 "targets": [
186 {
187 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
188 "legendFormat": "{{status}}",
189 "refId": "A"
190 }
191 ],
192 "thresholds": [ ],
193 "timeFrom": null,
194 "timeRegions": [ ],
195 "timeShift": null,
196 "title": "Pushes/Second",
197 "tooltip": {
198 "shared": true,
199 "sort": 0,
200 "value_type": "individual"
201 },
202 "type": "graph",
203 "xaxis": {
204 "buckets": null,
205 "mode": "time",
206 "name": null,
207 "show": true,
208 "values": [ ]
209 },
210 "yaxes": [
211 {
212 "format": "short",
213 "label": null,
214 "logBase": 10,
215 "max": null,
216 "min": null,
217 "show": true
218 },
219 {
220 "format": "short",
221 "label": null,
222 "logBase": 1,
223 "max": null,
224 "min": null,
225 "show": true
226 }
227 ],
228 "yaxis": {
229 "align": false,
230 "alignLevel": null
231 }
232 },
233 {
234 "aliasColors": { },
235 "bars": false,
236 "dashLength": 10,
237 "dashes": false,
238 "datasource": "$datasource",
239 "fieldConfig": {
240 "defaults": {
241 "custom": { }
242 },
243 "overrides": [ ]
244 },
245 "fill": 1,
246 "fillGradient": 0,
247 "gridPos": {
248 "h": 5,
249 "w": 4,
250 "x": 12,
251 "y": 1
252 },
253 "hiddenSeries": false,
254 "id": 2,
255 "interval": "",
256 "legend": {
257 "avg": false,
258 "current": false,
259 "max": false,
260 "min": false,
261 "show": false,
262 "total": false,
263 "values": false
264 },
265 "lines": true,
266 "linewidth": 1,
267 "nullPointMode": "null",
268 "options": {
269 "dataLinks": [ ]
270 },
271 "panels": [ ],
272 "percentage": false,
273 "pointradius": 2,
274 "points": false,
275 "renderer": "flot",
276 "seriesOverrides": [ ],
277 "spaceLength": 10,
278 "stack": false,
279 "steppedLine": false,
280 "targets": [
281 {
282 "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))",
283 "legendFormat": "{{tenant}}",
284 "refId": "A"
285 }
286 ],
287 "thresholds": [ ],
288 "timeFrom": null,
289 "timeRegions": [ ],
290 "timeShift": null,
291 "title": "Lines Per Tenant (top 10)",
292 "tooltip": {
293 "shared": false,
294 "sort": 0,
295 "value_type": "individual"
296 },
297 "type": "graph",
298 "xaxis": {
299 "buckets": null,
300 "mode": "time",
301 "name": null,
302 "show": true,
303 "values": [ ]
304 },
305 "yaxes": [
306 {
307 "format": "short",
308 "label": null,
309 "logBase": 1,
310 "max": null,
311 "min": null,
312 "show": true
313 },
314 {
315 "format": "short",
316 "label": null,
317 "logBase": 1,
318 "max": null,
319 "min": null,
320 "show": true
321 }
322 ],
323 "yaxis": {
324 "align": false,
325 "alignLevel": null
326 }
327 },
328 {
329 "aliasColors": { },
330 "bars": false,
331 "dashLength": 10,
332 "dashes": false,
333 "datasource": "$datasource",
334 "fieldConfig": {
335 "defaults": {
336 "custom": { }
337 },
338 "overrides": [ ]
339 },
340 "fill": 1,
341 "fillGradient": 0,
342 "gridPos": {
343 "h": 5,
344 "w": 4,
345 "x": 16,
346 "y": 1
347 },
348 "hiddenSeries": false,
349 "id": 4,
350 "legend": {
351 "avg": false,
352 "current": false,
353 "hideEmpty": true,
354 "hideZero": true,
355 "max": false,
356 "min": false,
357 "show": false,
358 "total": false,
359 "values": false
360 },
361 "lines": true,
362 "linewidth": 1,
363 "nullPointMode": "null",
364 "options": {
365 "dataLinks": [ ]
366 },
367 "panels": [ ],
368 "percentage": false,
369 "pointradius": 2,
370 "points": false,
371 "renderer": "flot",
372 "seriesOverrides": [ ],
373 "spaceLength": 10,
374 "stack": false,
375 "steppedLine": false,
376 "targets": [
377 {
378 "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024",
379 "legendFormat": "{{tenant}}",
380 "refId": "A"
381 }
382 ],
383 "thresholds": [ ],
384 "timeFrom": null,
385 "timeRegions": [ ],
386 "timeShift": null,
387 "title": "MBs Per Tenant (Top 10)",
388 "tooltip": {
389 "shared": false,
390 "sort": 0,
391 "value_type": "individual"
392 },
393 "type": "graph",
394 "xaxis": {
395 "buckets": null,
396 "mode": "time",
397 "name": null,
398 "show": true,
399 "values": [ ]
400 },
401 "yaxes": [
402 {
403 "format": "short",
404 "label": null,
405 "logBase": 1,
406 "max": null,
407 "min": null,
408 "show": true
409 },
410 {
411 "format": "short",
412 "label": null,
413 "logBase": 1,
414 "max": null,
415 "min": null,
416 "show": true
417 }
418 ],
419 "yaxis": {
420 "align": false,
421 "alignLevel": null
422 }
423 },
424 {
425 "aliasColors": { },
426 "bars": false,
427 "dashLength": 10,
428 "dashes": false,
429 "datasource": "$datasource",
430 "fieldConfig": {
431 "defaults": {
432 "custom": { }
433 },
434 "overrides": [ ]
435 },
436 "fill": 1,
437 "fillGradient": 0,
438 "gridPos": {
439 "h": 5,
440 "w": 4,
441 "x": 20,
442 "y": 1
443 },
444 "hiddenSeries": false,
445 "id": 24,
446 "legend": {
447 "avg": false,
448 "current": false,
449 "max": false,
450 "min": false,
451 "show": false,
452 "total": false,
453 "values": false
454 },
455 "lines": true,
456 "linewidth": 1,
457 "nullPointMode": "null",
458 "options": {
459 "dataLinks": [ ]
460 },
461 "panels": [ ],
462 "percentage": false,
463 "pointradius": 2,
464 "points": false,
465 "renderer": "flot",
466 "seriesOverrides": [ ],
467 "spaceLength": 10,
468 "stack": false,
469 "steppedLine": false,
470 "targets": [
471 {
472 "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0",
473 "hide": false,
474 "interval": "",
475 "legendFormat": "{{container}}-{{pod}}",
476 "refId": "B"
477 }
478 ],
479 "thresholds": [ ],
480 "timeFrom": null,
481 "timeRegions": [ ],
482 "timeShift": null,
483 "title": "Container Restarts",
484 "tooltip": {
485 "shared": true,
486 "sort": 2,
487 "value_type": "individual"
488 },
489 "type": "graph",
490 "xaxis": {
491 "buckets": null,
492 "mode": "time",
493 "name": null,
494 "show": true,
495 "values": [ ]
496 },
497 "yaxes": [
498 {
499 "format": "short",
500 "label": null,
501 "logBase": 1,
502 "max": null,
503 "min": null,
504 "show": true
505 },
506 {
507 "format": "short",
508 "label": null,
509 "logBase": 1,
510 "max": null,
511 "min": null,
512 "show": true
513 }
514 ],
515 "yaxis": {
516 "align": false,
517 "alignLevel": null
518 }
519 },
520 {
521 "aliasColors": { },
522 "bars": false,
523 "dashLength": 10,
524 "dashes": false,
525 "datasource": "$datasource",
526 "fieldConfig": {
527 "defaults": {
528 "custom": { }
529 },
530 "overrides": [ ]
531 },
532 "fill": 1,
533 "fillGradient": 0,
534 "gridPos": {
535 "h": 10,
536 "w": 12,
537 "x": 0,
538 "y": 6
539 },
540 "hiddenSeries": false,
541 "id": 9,
542 "legend": {
543 "alignAsTable": true,
544 "avg": false,
545 "current": false,
546 "max": false,
547 "min": false,
548 "rightSide": true,
549 "show": true,
550 "total": false,
551 "values": false
552 },
553 "lines": true,
554 "linewidth": 1,
555 "nullPointMode": "null",
556 "options": {
557 "dataLinks": [ ]
558 },
559 "panels": [ ],
560 "percentage": false,
561 "pointradius": 2,
562 "points": false,
563 "renderer": "flot",
564 "seriesOverrides": [ ],
565 "spaceLength": 10,
566 "stack": false,
567 "steppedLine": false,
568 "targets": [
569 {
570 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3",
571 "legendFormat": ".99",
572 "refId": "A"
573 },
574 {
575 "expr": "histogram_quantile(0.75, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3",
576 "legendFormat": ".9",
577 "refId": "B"
578 },
579 {
580 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3",
581 "legendFormat": ".5",
582 "refId": "C"
583 }
584 ],
585 "thresholds": [ ],
586 "timeFrom": null,
587 "timeRegions": [ ],
588 "timeShift": null,
589 "title": "Push Latency",
590 "tooltip": {
591 "shared": true,
592 "sort": 0,
593 "value_type": "individual"
594 },
595 "type": "graph",
596 "xaxis": {
597 "buckets": null,
598 "mode": "time",
599 "name": null,
600 "show": true,
601 "values": [ ]
602 },
603 "yaxes": [
604 {
605 "format": "short",
606 "label": null,
607 "logBase": 1,
608 "max": null,
609 "min": null,
610 "show": true
611 },
612 {
613 "format": "short",
614 "label": null,
615 "logBase": 1,
616 "max": null,
617 "min": null,
618 "show": true
619 }
620 ],
621 "yaxis": {
622 "align": false,
623 "alignLevel": null
624 }
625 },
626 {
627 "aliasColors": { },
628 "bars": false,
629 "dashLength": 10,
630 "dashes": false,
631 "datasource": "$datasource",
632 "fieldConfig": {
633 "defaults": {
634 "custom": { }
635 },
636 "overrides": [ ]
637 },
638 "fill": 1,
639 "fillGradient": 0,
640 "gridPos": {
641 "h": 5,
642 "w": 6,
643 "x": 12,
644 "y": 6
645 },
646 "hiddenSeries": false,
647 "id": 12,
648 "legend": {
649 "alignAsTable": true,
650 "avg": false,
651 "current": false,
652 "max": false,
653 "min": false,
654 "rightSide": true,
655 "show": false,
656 "total": false,
657 "values": false
658 },
659 "lines": true,
660 "linewidth": 1,
661 "nullPointMode": "null",
662 "options": {
663 "dataLinks": [ ]
664 },
665 "panels": [ ],
666 "percentage": false,
667 "pointradius": 2,
668 "points": false,
669 "renderer": "flot",
670 "seriesOverrides": [ ],
671 "spaceLength": 10,
672 "stack": false,
673 "steppedLine": false,
674 "targets": [
675 {
676 "expr": "histogram_quantile(0.99, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3",
677 "legendFormat": ".99",
678 "refId": "A"
679 },
680 {
681 "expr": "histogram_quantile(0.9, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3",
682 "legendFormat": ".9",
683 "refId": "B"
684 },
685 {
686 "expr": "histogram_quantile(0.5, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3",
687 "legendFormat": ".5",
688 "refId": "C"
689 }
690 ],
691 "thresholds": [ ],
692 "timeFrom": null,
693 "timeRegions": [ ],
694 "timeShift": null,
695 "title": "Distributor Latency",
696 "tooltip": {
697 "shared": true,
698 "sort": 0,
699 "value_type": "individual"
700 },
701 "type": "graph",
702 "xaxis": {
703 "buckets": null,
704 "mode": "time",
705 "name": null,
706 "show": true,
707 "values": [ ]
708 },
709 "yaxes": [
710 {
711 "format": "short",
712 "label": null,
713 "logBase": 1,
714 "max": null,
715 "min": null,
716 "show": true
717 },
718 {
719 "format": "short",
720 "label": null,
721 "logBase": 1,
722 "max": null,
723 "min": null,
724 "show": true
725 }
726 ],
727 "yaxis": {
728 "align": false,
729 "alignLevel": null
730 }
731 },
732 {
733 "aliasColors": { },
734 "bars": false,
735 "dashLength": 10,
736 "dashes": false,
737 "datasource": "$datasource",
738 "fieldConfig": {
739 "defaults": {
740 "custom": { }
741 },
742 "overrides": [ ]
743 },
744 "fill": 0,
745 "fillGradient": 0,
746 "gridPos": {
747 "h": 5,
748 "w": 6,
749 "x": 18,
750 "y": 6
751 },
752 "hiddenSeries": false,
753 "id": 71,
754 "legend": {
755 "alignAsTable": true,
756 "avg": false,
757 "current": false,
758 "max": false,
759 "min": false,
760 "rightSide": true,
761 "show": false,
762 "total": false,
763 "values": false
764 },
765 "lines": true,
766 "linewidth": 1,
767 "nullPointMode": "null",
768 "options": {
769 "dataLinks": [ ]
770 },
771 "panels": [ ],
772 "percentage": false,
773 "pointradius": 2,
774 "points": false,
775 "renderer": "flot",
776 "seriesOverrides": [ ],
777 "spaceLength": 10,
778 "stack": false,
779 "steppedLine": false,
780 "targets": [
781 {
782 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[5m])) by (route)",
783 "interval": "",
784 "legendFormat": "{{route}}",
785 "refId": "A"
786 }
787 ],
788 "thresholds": [ ],
789 "timeFrom": null,
790 "timeRegions": [ ],
791 "timeShift": null,
792 "title": "Distributor Success Rate",
793 "tooltip": {
794 "shared": true,
795 "sort": 0,
796 "value_type": "individual"
797 },
798 "type": "graph",
799 "xaxis": {
800 "buckets": null,
801 "mode": "time",
802 "name": null,
803 "show": true,
804 "values": [ ]
805 },
806 "yaxes": [
807 {
808 "decimals": null,
809 "format": "percentunit",
810 "label": "",
811 "logBase": 1,
812 "max": "1",
813 "min": null,
814 "show": true
815 },
816 {
817 "format": "short",
818 "label": null,
819 "logBase": 1,
820 "max": null,
821 "min": null,
822 "show": true
823 }
824 ],
825 "yaxis": {
826 "align": false,
827 "alignLevel": null
828 }
829 },
830 {
831 "aliasColors": { },
832 "bars": false,
833 "dashLength": 10,
834 "dashes": false,
835 "datasource": "$datasource",
836 "fieldConfig": {
837 "defaults": {
838 "custom": { }
839 },
840 "overrides": [ ]
841 },
842 "fill": 1,
843 "fillGradient": 0,
844 "gridPos": {
845 "h": 5,
846 "w": 6,
847 "x": 12,
848 "y": 11
849 },
850 "hiddenSeries": false,
851 "id": 13,
852 "legend": {
853 "alignAsTable": true,
854 "avg": false,
855 "current": false,
856 "max": false,
857 "min": false,
858 "rightSide": true,
859 "show": false,
860 "total": false,
861 "values": false
862 },
863 "lines": true,
864 "linewidth": 1,
865 "nullPointMode": "null",
866 "options": {
867 "dataLinks": [ ]
868 },
869 "panels": [ ],
870 "percentage": false,
871 "pointradius": 2,
872 "points": false,
873 "renderer": "flot",
874 "seriesOverrides": [ ],
875 "spaceLength": 10,
876 "stack": false,
877 "steppedLine": false,
878 "targets": [
879 {
880 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3",
881 "legendFormat": ".99",
882 "refId": "A"
883 },
884 {
885 "expr": "histogram_quantile(0.9, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3",
886 "hide": false,
887 "legendFormat": ".9",
888 "refId": "B"
889 },
890 {
891 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3",
892 "hide": false,
893 "legendFormat": ".5",
894 "refId": "C"
895 }
896 ],
897 "thresholds": [ ],
898 "timeFrom": null,
899 "timeRegions": [ ],
900 "timeShift": null,
901 "title": "Ingester Latency Write",
902 "tooltip": {
903 "shared": true,
904 "sort": 0,
905 "value_type": "individual"
906 },
907 "type": "graph",
908 "xaxis": {
909 "buckets": null,
910 "mode": "time",
911 "name": null,
912 "show": true,
913 "values": [ ]
914 },
915 "yaxes": [
916 {
917 "format": "short",
918 "label": null,
919 "logBase": 1,
920 "max": null,
921 "min": null,
922 "show": true
923 },
924 {
925 "format": "short",
926 "label": null,
927 "logBase": 1,
928 "max": null,
929 "min": null,
930 "show": true
931 }
932 ],
933 "yaxis": {
934 "align": false,
935 "alignLevel": null
936 }
937 },
938 {
939 "aliasColors": { },
940 "bars": false,
941 "dashLength": 10,
942 "dashes": false,
943 "datasource": "$datasource",
944 "fieldConfig": {
945 "defaults": {
946 "custom": { }
947 },
948 "overrides": [ ]
949 },
950 "fill": 0,
951 "fillGradient": 0,
952 "gridPos": {
953 "h": 5,
954 "w": 6,
955 "x": 18,
956 "y": 11
957 },
958 "hiddenSeries": false,
959 "id": 72,
960 "legend": {
961 "alignAsTable": true,
962 "avg": false,
963 "current": false,
964 "max": false,
965 "min": false,
966 "rightSide": true,
967 "show": false,
968 "total": false,
969 "values": false
970 },
971 "lines": true,
972 "linewidth": 1,
973 "nullPointMode": "null",
974 "options": {
975 "dataLinks": [ ]
976 },
977 "panels": [ ],
978 "percentage": false,
979 "pointradius": 2,
980 "points": false,
981 "renderer": "flot",
982 "seriesOverrides": [ ],
983 "spaceLength": 10,
984 "stack": false,
985 "steppedLine": false,
986 "targets": [
987 {
988 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)",
989 "interval": "",
990 "legendFormat": "{{route}}",
991 "refId": "A"
992 }
993 ],
994 "thresholds": [ ],
995 "timeFrom": null,
996 "timeRegions": [ ],
997 "timeShift": null,
998 "title": "Ingester Success Rate Write",
999 "tooltip": {
1000 "shared": true,
1001 "sort": 0,
1002 "value_type": "individual"
1003 },
1004 "type": "graph",
1005 "xaxis": {
1006 "buckets": null,
1007 "mode": "time",
1008 "name": null,
1009 "show": true,
1010 "values": [ ]
1011 },
1012 "yaxes": [
1013 {
1014 "decimals": null,
1015 "format": "percentunit",
1016 "label": "",
1017 "logBase": 1,
1018 "max": "1",
1019 "min": null,
1020 "show": true
1021 },
1022 {
1023 "format": "short",
1024 "label": null,
1025 "logBase": 1,
1026 "max": null,
1027 "min": null,
1028 "show": true
1029 }
1030 ],
1031 "yaxis": {
1032 "align": false,
1033 "alignLevel": null
1034 }
1035 },
1036 {
1037 "aliasColors": { },
1038 "bars": false,
1039 "dashLength": 10,
1040 "dashes": false,
1041 "datasource": "$datasource",
1042 "fieldConfig": {
1043 "defaults": {
1044 "custom": { }
1045 },
1046 "overrides": [ ]
1047 },
1048 "fill": 1,
1049 "fillGradient": 0,
1050 "gridPos": {
1051 "h": 10,
1052 "w": 12,
1053 "x": 0,
1054 "y": 16
1055 },
1056 "hiddenSeries": false,
1057 "id": 10,
1058 "legend": {
1059 "alignAsTable": true,
1060 "avg": false,
1061 "current": false,
1062 "hideEmpty": true,
1063 "hideZero": false,
1064 "max": false,
1065 "min": false,
1066 "rightSide": true,
1067 "show": true,
1068 "total": false,
1069 "values": false
1070 },
1071 "lines": true,
1072 "linewidth": 1,
1073 "nullPointMode": "null",
1074 "options": {
1075 "dataLinks": [ ]
1076 },
1077 "panels": [ ],
1078 "percentage": false,
1079 "pointradius": 2,
1080 "points": false,
1081 "renderer": "flot",
1082 "seriesOverrides": [ ],
1083 "spaceLength": 10,
1084 "stack": false,
1085 "steppedLine": false,
1086 "targets": [
1087 {
1088 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))",
1089 "legendFormat": "{{route}}-.99",
1090 "refId": "A"
1091 },
1092 {
1093 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))",
1094 "legendFormat": "{{route}}-.9",
1095 "refId": "B"
1096 },
1097 {
1098 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))",
1099 "legendFormat": "{{route}}-.5",
1100 "refId": "C"
1101 }
1102 ],
1103 "thresholds": [ ],
1104 "timeFrom": null,
1105 "timeRegions": [ ],
1106 "timeShift": null,
1107 "title": "Query Latency",
1108 "tooltip": {
1109 "shared": true,
1110 "sort": 0,
1111 "value_type": "individual"
1112 },
1113 "type": "graph",
1114 "xaxis": {
1115 "buckets": null,
1116 "mode": "time",
1117 "name": null,
1118 "show": true,
1119 "values": [ ]
1120 },
1121 "yaxes": [
1122 {
1123 "format": "short",
1124 "label": null,
1125 "logBase": 1,
1126 "max": null,
1127 "min": null,
1128 "show": true
1129 },
1130 {
1131 "format": "short",
1132 "label": null,
1133 "logBase": 1,
1134 "max": null,
1135 "min": null,
1136 "show": true
1137 }
1138 ],
1139 "yaxis": {
1140 "align": false,
1141 "alignLevel": null
1142 }
1143 },
1144 {
1145 "aliasColors": { },
1146 "bars": false,
1147 "dashLength": 10,
1148 "dashes": false,
1149 "datasource": "$datasource",
1150 "fieldConfig": {
1151 "defaults": {
1152 "custom": { }
1153 },
1154 "overrides": [ ]
1155 },
1156 "fill": 1,
1157 "fillGradient": 0,
1158 "gridPos": {
1159 "h": 5,
1160 "w": 6,
1161 "x": 12,
1162 "y": 16
1163 },
1164 "hiddenSeries": false,
1165 "id": 14,
1166 "legend": {
1167 "alignAsTable": true,
1168 "avg": false,
1169 "current": false,
1170 "max": false,
1171 "min": false,
1172 "rightSide": true,
1173 "show": false,
1174 "total": false,
1175 "values": false
1176 },
1177 "lines": true,
1178 "linewidth": 1,
1179 "nullPointMode": "null",
1180 "options": {
1181 "dataLinks": [ ]
1182 },
1183 "panels": [ ],
1184 "percentage": false,
1185 "pointradius": 2,
1186 "points": false,
1187 "renderer": "flot",
1188 "seriesOverrides": [ ],
1189 "spaceLength": 10,
1190 "stack": false,
1191 "steppedLine": false,
1192 "targets": [
1193 {
1194 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3",
1195 "legendFormat": ".99-{{route}}",
1196 "refId": "A"
1197 },
1198 {
1199 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3",
1200 "legendFormat": ".9-{{route}}",
1201 "refId": "B"
1202 },
1203 {
1204 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3",
1205 "legendFormat": ".5-{{route}}",
1206 "refId": "C"
1207 }
1208 ],
1209 "thresholds": [ ],
1210 "timeFrom": null,
1211 "timeRegions": [ ],
1212 "timeShift": null,
1213 "title": "Querier Latency",
1214 "tooltip": {
1215 "shared": true,
1216 "sort": 0,
1217 "value_type": "individual"
1218 },
1219 "type": "graph",
1220 "xaxis": {
1221 "buckets": null,
1222 "mode": "time",
1223 "name": null,
1224 "show": true,
1225 "values": [ ]
1226 },
1227 "yaxes": [
1228 {
1229 "format": "short",
1230 "label": null,
1231 "logBase": 1,
1232 "max": null,
1233 "min": null,
1234 "show": true
1235 },
1236 {
1237 "format": "short",
1238 "label": null,
1239 "logBase": 1,
1240 "max": null,
1241 "min": null,
1242 "show": true
1243 }
1244 ],
1245 "yaxis": {
1246 "align": false,
1247 "alignLevel": null
1248 }
1249 },
1250 {
1251 "aliasColors": { },
1252 "bars": false,
1253 "dashLength": 10,
1254 "dashes": false,
1255 "datasource": "$datasource",
1256 "fieldConfig": {
1257 "defaults": {
1258 "custom": { }
1259 },
1260 "overrides": [ ]
1261 },
1262 "fill": 0,
1263 "fillGradient": 0,
1264 "gridPos": {
1265 "h": 5,
1266 "w": 6,
1267 "x": 18,
1268 "y": 16
1269 },
1270 "hiddenSeries": false,
1271 "id": 73,
1272 "legend": {
1273 "alignAsTable": true,
1274 "avg": false,
1275 "current": false,
1276 "max": false,
1277 "min": false,
1278 "rightSide": true,
1279 "show": false,
1280 "total": false,
1281 "values": false
1282 },
1283 "lines": true,
1284 "linewidth": 1,
1285 "nullPointMode": "null",
1286 "options": {
1287 "dataLinks": [ ]
1288 },
1289 "panels": [ ],
1290 "percentage": false,
1291 "pointradius": 2,
1292 "points": false,
1293 "renderer": "flot",
1294 "seriesOverrides": [ ],
1295 "spaceLength": 10,
1296 "stack": false,
1297 "steppedLine": false,
1298 "targets": [
1299 {
1300 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[5m])) by (route)",
1301 "interval": "",
1302 "legendFormat": "{{route}}",
1303 "refId": "A"
1304 }
1305 ],
1306 "thresholds": [ ],
1307 "timeFrom": null,
1308 "timeRegions": [ ],
1309 "timeShift": null,
1310 "title": "Querier Success Rate",
1311 "tooltip": {
1312 "shared": true,
1313 "sort": 0,
1314 "value_type": "individual"
1315 },
1316 "type": "graph",
1317 "xaxis": {
1318 "buckets": null,
1319 "mode": "time",
1320 "name": null,
1321 "show": true,
1322 "values": [ ]
1323 },
1324 "yaxes": [
1325 {
1326 "decimals": null,
1327 "format": "percentunit",
1328 "label": "",
1329 "logBase": 1,
1330 "max": "1",
1331 "min": null,
1332 "show": true
1333 },
1334 {
1335 "format": "short",
1336 "label": null,
1337 "logBase": 1,
1338 "max": null,
1339 "min": null,
1340 "show": true
1341 }
1342 ],
1343 "yaxis": {
1344 "align": false,
1345 "alignLevel": null
1346 }
1347 },
1348 {
1349 "aliasColors": { },
1350 "bars": false,
1351 "dashLength": 10,
1352 "dashes": false,
1353 "datasource": "$datasource",
1354 "description": "",
1355 "fieldConfig": {
1356 "defaults": {
1357 "custom": { }
1358 },
1359 "overrides": [ ]
1360 },
1361 "fill": 1,
1362 "fillGradient": 0,
1363 "gridPos": {
1364 "h": 5,
1365 "w": 6,
1366 "x": 12,
1367 "y": 21
1368 },
1369 "hiddenSeries": false,
1370 "id": 15,
1371 "legend": {
1372 "alignAsTable": true,
1373 "avg": false,
1374 "current": false,
1375 "max": false,
1376 "min": false,
1377 "rightSide": true,
1378 "show": false,
1379 "total": false,
1380 "values": false
1381 },
1382 "lines": true,
1383 "linewidth": 1,
1384 "nullPointMode": "null",
1385 "options": {
1386 "dataLinks": [ ]
1387 },
1388 "panels": [ ],
1389 "percentage": false,
1390 "pointradius": 2,
1391 "points": false,
1392 "renderer": "flot",
1393 "seriesOverrides": [ ],
1394 "spaceLength": 10,
1395 "stack": false,
1396 "steppedLine": false,
1397 "targets": [
1398 {
1399 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3",
1400 "legendFormat": ".99-{{route}}",
1401 "refId": "A"
1402 },
1403 {
1404 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3",
1405 "legendFormat": ".9-{{route}}",
1406 "refId": "B"
1407 },
1408 {
1409 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3",
1410 "legendFormat": ".5-{{route}}",
1411 "refId": "C"
1412 }
1413 ],
1414 "thresholds": [ ],
1415 "timeFrom": null,
1416 "timeRegions": [ ],
1417 "timeShift": null,
1418 "title": "Ingester Latency Read",
1419 "tooltip": {
1420 "shared": true,
1421 "sort": 0,
1422 "value_type": "individual"
1423 },
1424 "type": "graph",
1425 "xaxis": {
1426 "buckets": null,
1427 "mode": "time",
1428 "name": null,
1429 "show": true,
1430 "values": [ ]
1431 },
1432 "yaxes": [
1433 {
1434 "format": "short",
1435 "label": null,
1436 "logBase": 1,
1437 "max": null,
1438 "min": null,
1439 "show": true
1440 },
1441 {
1442 "format": "short",
1443 "label": null,
1444 "logBase": 1,
1445 "max": null,
1446 "min": null,
1447 "show": true
1448 }
1449 ],
1450 "yaxis": {
1451 "align": false,
1452 "alignLevel": null
1453 }
1454 },
1455 {
1456 "aliasColors": { },
1457 "bars": false,
1458 "dashLength": 10,
1459 "dashes": false,
1460 "datasource": "$datasource",
1461 "fieldConfig": {
1462 "defaults": {
1463 "custom": { }
1464 },
1465 "overrides": [ ]
1466 },
1467 "fill": 0,
1468 "fillGradient": 0,
1469 "gridPos": {
1470 "h": 5,
1471 "w": 6,
1472 "x": 18,
1473 "y": 21
1474 },
1475 "hiddenSeries": false,
1476 "id": 74,
1477 "legend": {
1478 "alignAsTable": true,
1479 "avg": false,
1480 "current": false,
1481 "max": false,
1482 "min": false,
1483 "rightSide": true,
1484 "show": false,
1485 "total": false,
1486 "values": false
1487 },
1488 "lines": true,
1489 "linewidth": 1,
1490 "nullPointMode": "null",
1491 "options": {
1492 "dataLinks": [ ]
1493 },
1494 "panels": [ ],
1495 "percentage": false,
1496 "pointradius": 2,
1497 "points": false,
1498 "renderer": "flot",
1499 "seriesOverrides": [ ],
1500 "spaceLength": 10,
1501 "stack": false,
1502 "steppedLine": false,
1503 "targets": [
1504 {
1505 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)",
1506 "interval": "",
1507 "legendFormat": "{{route}}",
1508 "refId": "A"
1509 }
1510 ],
1511 "thresholds": [ ],
1512 "timeFrom": null,
1513 "timeRegions": [ ],
1514 "timeShift": null,
1515 "title": "Ingester Success Rate Read",
1516 "tooltip": {
1517 "shared": true,
1518 "sort": 0,
1519 "value_type": "individual"
1520 },
1521 "type": "graph",
1522 "xaxis": {
1523 "buckets": null,
1524 "mode": "time",
1525 "name": null,
1526 "show": true,
1527 "values": [ ]
1528 },
1529 "yaxes": [
1530 {
1531 "decimals": null,
1532 "format": "percentunit",
1533 "label": "",
1534 "logBase": 1,
1535 "max": "1",
1536 "min": null,
1537 "show": true
1538 },
1539 {
1540 "format": "short",
1541 "label": null,
1542 "logBase": 1,
1543 "max": null,
1544 "min": null,
1545 "show": true
1546 }
1547 ],
1548 "yaxis": {
1549 "align": false,
1550 "alignLevel": null
1551 }
1552 },
1553 {
1554 "collapsed": true,
1555 "datasource": null,
1556 "gridPos": {
1557 "h": 1,
1558 "w": 24,
1559 "x": 0,
1560 "y": 26
1561 },
1562 "id": 110,
1563 "panels": [
1564 {
1565 "aliasColors": { },
1566 "bars": false,
1567 "dashLength": 10,
1568 "dashes": false,
1569 "datasource": "$datasource",
1570 "fill": 1,
1571 "fillGradient": 0,
1572 "gridPos": {
1573 "h": 8,
1574 "w": 12,
1575 "x": 0,
1576 "y": 27
1577 },
1578 "hiddenSeries": false,
1579 "id": 112,
1580 "legend": {
1581 "avg": false,
1582 "current": false,
1583 "max": false,
1584 "min": false,
1585 "show": false,
1586 "total": false,
1587 "values": false
1588 },
1589 "lines": true,
1590 "linewidth": 1,
1591 "nullPointMode": "null",
1592 "options": {
1593 "dataLinks": [ ]
1594 },
1595 "panels": [ ],
1596 "percentage": false,
1597 "pointradius": 2,
1598 "points": false,
1599 "renderer": "flot",
1600 "seriesOverrides": [ ],
1601 "spaceLength": 10,
1602 "stack": false,
1603 "steppedLine": false,
1604 "targets": [
1605 {
1606 "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))",
1607 "interval": "",
1608 "legendFormat": "{{ tenant }} - {{ reason }}",
1609 "refId": "A"
1610 }
1611 ],
1612 "thresholds": [ ],
1613 "timeFrom": null,
1614 "timeRegions": [ ],
1615 "timeShift": null,
1616 "title": "Discarded Lines",
1617 "tooltip": {
1618 "shared": false,
1619 "sort": 0,
1620 "value_type": "individual"
1621 },
1622 "type": "graph",
1623 "xaxis": {
1624 "buckets": null,
1625 "mode": "time",
1626 "name": null,
1627 "show": true,
1628 "values": [ ]
1629 },
1630 "yaxes": [
1631 {
1632 "format": "short",
1633 "label": null,
1634 "logBase": 1,
1635 "max": null,
1636 "min": null,
1637 "show": true
1638 },
1639 {
1640 "format": "short",
1641 "label": null,
1642 "logBase": 1,
1643 "max": null,
1644 "min": null,
1645 "show": true
1646 }
1647 ],
1648 "yaxis": {
1649 "align": false,
1650 "alignLevel": null
1651 }
1652 },
1653 {
1654 "columns": [ ],
1655 "datasource": "$datasource",
1656 "fontSize": "100%",
1657 "gridPos": {
1658 "h": 8,
1659 "w": 12,
1660 "x": 12,
1661 "y": 27
1662 },
1663 "id": 113,
1664 "pageSize": null,
1665 "panels": [ ],
1666 "showHeader": true,
1667 "sort": {
1668 "col": 3,
1669 "desc": true
1670 },
1671 "styles": [
1672 {
1673 "alias": "Time",
1674 "align": "auto",
1675 "dateFormat": "YYYY-MM-DD HH:mm:ss",
1676 "pattern": "Time",
1677 "type": "hidden"
1678 },
1679 {
1680 "alias": "",
1681 "align": "auto",
1682 "colorMode": null,
1683 "colors": [
1684 "rgba(245, 54, 54, 0.9)",
1685 "rgba(237, 129, 40, 0.89)",
1686 "rgba(50, 172, 45, 0.97)"
1687 ],
1688 "dateFormat": "YYYY-MM-DD HH:mm:ss",
1689 "decimals": 2,
1690 "mappingType": 1,
1691 "pattern": "tenant",
1692 "thresholds": [ ],
1693 "type": "string",
1694 "unit": "short"
1695 },
1696 {
1697 "alias": "",
1698 "align": "auto",
1699 "colorMode": null,
1700 "colors": [
1701 "rgba(245, 54, 54, 0.9)",
1702 "rgba(237, 129, 40, 0.89)",
1703 "rgba(50, 172, 45, 0.97)"
1704 ],
1705 "dateFormat": "YYYY-MM-DD HH:mm:ss",
1706 "decimals": 2,
1707 "mappingType": 1,
1708 "pattern": "reason",
1709 "thresholds": [ ],
1710 "type": "number",
1711 "unit": "short"
1712 },
1713 {
1714 "alias": "",
1715 "align": "right",
1716 "colorMode": null,
1717 "colors": [
1718 "rgba(245, 54, 54, 0.9)",
1719 "rgba(237, 129, 40, 0.89)",
1720 "rgba(50, 172, 45, 0.97)"
1721 ],
1722 "decimals": 2,
1723 "pattern": "/.*/",
1724 "thresholds": [ ],
1725 "type": "number",
1726 "unit": "short"
1727 }
1728 ],
1729 "targets": [
1730 {
1731 "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))",
1732 "format": "table",
1733 "instant": true,
1734 "interval": "",
1735 "legendFormat": "{{ tenant }} - {{ reason }}",
1736 "refId": "A"
1737 }
1738 ],
1739 "timeFrom": null,
1740 "timeShift": null,
1741 "title": "Discarded Lines Per Interval",
1742 "transform": "table",
1743 "type": "table-old"
1744 }
1745 ],
1746 "targets": [ ],
1747 "title": "Limits",
1748 "type": "row"
1749 },
1750 {
1751 "collapsed": true,
1752 "datasource": null,
1753 "gridPos": {
1754 "h": 1,
1755 "w": 24,
1756 "x": 0,
1757 "y": 27
1758 },
1759 "id": 23,
1760 "panels": [
1761 {
1762 "aliasColors": { },
1763 "bars": false,
1764 "dashLength": 10,
1765 "dashes": false,
1766 "datasource": "$datasource",
1767 "fill": 1,
1768 "fillGradient": 0,
1769 "gridPos": {
1770 "h": 7,
1771 "w": 6,
1772 "x": 0,
1773 "y": 28
1774 },
1775 "hiddenSeries": false,
1776 "id": 26,
1777 "legend": {
1778 "avg": false,
1779 "current": false,
1780 "max": false,
1781 "min": false,
1782 "show": false,
1783 "total": false,
1784 "values": false
1785 },
1786 "lines": false,
1787 "linewidth": 1,
1788 "nullPointMode": "null",
1789 "options": {
1790 "dataLinks": [ ]
1791 },
1792 "panels": [ ],
1793 "percentage": false,
1794 "pointradius": 1,
1795 "points": true,
1796 "renderer": "flot",
1797 "seriesOverrides": [ ],
1798 "spaceLength": 10,
1799 "stack": false,
1800 "steppedLine": false,
1801 "targets": [
1802 {
1803 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}",
1804 "intervalFactor": 3,
1805 "legendFormat": "{{pod}}-{{container}}",
1806 "refId": "A"
1807 }
1808 ],
1809 "thresholds": [ ],
1810 "timeFrom": null,
1811 "timeRegions": [ ],
1812 "timeShift": null,
1813 "title": "CPU Usage",
1814 "tooltip": {
1815 "shared": true,
1816 "sort": 2,
1817 "value_type": "individual"
1818 },
1819 "type": "graph",
1820 "xaxis": {
1821 "buckets": null,
1822 "mode": "time",
1823 "name": null,
1824 "show": true,
1825 "values": [ ]
1826 },
1827 "yaxes": [
1828 {
1829 "format": "short",
1830 "label": null,
1831 "logBase": 1,
1832 "max": null,
1833 "min": null,
1834 "show": true
1835 },
1836 {
1837 "format": "short",
1838 "label": null,
1839 "logBase": 1,
1840 "max": null,
1841 "min": null,
1842 "show": true
1843 }
1844 ],
1845 "yaxis": {
1846 "align": false,
1847 "alignLevel": null
1848 }
1849 },
1850 {
1851 "aliasColors": { },
1852 "bars": false,
1853 "dashLength": 10,
1854 "dashes": false,
1855 "datasource": "$datasource",
1856 "fill": 1,
1857 "fillGradient": 0,
1858 "gridPos": {
1859 "h": 7,
1860 "w": 6,
1861 "x": 6,
1862 "y": 28
1863 },
1864 "hiddenSeries": false,
1865 "id": 27,
1866 "legend": {
1867 "avg": false,
1868 "current": false,
1869 "hideEmpty": false,
1870 "hideZero": false,
1871 "max": false,
1872 "min": false,
1873 "show": false,
1874 "total": false,
1875 "values": false
1876 },
1877 "lines": false,
1878 "linewidth": 1,
1879 "nullPointMode": "null",
1880 "options": {
1881 "dataLinks": [ ]
1882 },
1883 "panels": [ ],
1884 "percentage": false,
1885 "pointradius": 1,
1886 "points": true,
1887 "renderer": "flot",
1888 "seriesOverrides": [ ],
1889 "spaceLength": 10,
1890 "stack": false,
1891 "steppedLine": false,
1892 "targets": [
1893 {
1894 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}",
1895 "instant": false,
1896 "intervalFactor": 3,
1897 "legendFormat": "{{pod}}",
1898 "refId": "A"
1899 }
1900 ],
1901 "thresholds": [ ],
1902 "timeFrom": null,
1903 "timeRegions": [ ],
1904 "timeShift": null,
1905 "title": "Memory Usage",
1906 "tooltip": {
1907 "shared": true,
1908 "sort": 2,
1909 "value_type": "individual"
1910 },
1911 "type": "graph",
1912 "xaxis": {
1913 "buckets": null,
1914 "mode": "time",
1915 "name": null,
1916 "show": true,
1917 "values": [ ]
1918 },
1919 "yaxes": [
1920 {
1921 "format": "bytes",
1922 "label": null,
1923 "logBase": 1,
1924 "max": null,
1925 "min": null,
1926 "show": true
1927 },
1928 {
1929 "format": "short",
1930 "label": null,
1931 "logBase": 1,
1932 "max": null,
1933 "min": null,
1934 "show": true
1935 }
1936 ],
1937 "yaxis": {
1938 "align": false,
1939 "alignLevel": null
1940 }
1941 },
1942 {
1943 "aliasColors": { },
1944 "bars": true,
1945 "dashLength": 10,
1946 "dashes": false,
1947 "datasource": "$logs",
1948 "fill": 1,
1949 "fillGradient": 0,
1950 "gridPos": {
1951 "h": 4,
1952 "w": 12,
1953 "x": 12,
1954 "y": 28
1955 },
1956 "hiddenSeries": false,
1957 "id": 31,
1958 "legend": {
1959 "avg": false,
1960 "current": false,
1961 "max": false,
1962 "min": false,
1963 "show": false,
1964 "total": false,
1965 "values": false
1966 },
1967 "lines": false,
1968 "linewidth": 1,
1969 "nullPointMode": "null",
1970 "options": {
1971 "dataLinks": [ ]
1972 },
1973 "panels": [ ],
1974 "percentage": false,
1975 "pointradius": 2,
1976 "points": false,
1977 "renderer": "flot",
1978 "seriesOverrides": [
1979 {
1980 "alias": "{}",
1981 "color": "#C4162A"
1982 }
1983 ],
1984 "spaceLength": 10,
1985 "stack": false,
1986 "steppedLine": false,
1987 "targets": [
1988 {
1989 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"[1m]))",
1990 "refId": "A"
1991 }
1992 ],
1993 "thresholds": [ ],
1994 "timeFrom": null,
1995 "timeRegions": [ ],
1996 "timeShift": null,
1997 "title": "Error Log Rate",
1998 "tooltip": {
1999 "shared": true,
2000 "sort": 2,
2001 "value_type": "individual"
2002 },
2003 "type": "graph",
2004 "xaxis": {
2005 "buckets": null,
2006 "mode": "time",
2007 "name": null,
2008 "show": false,
2009 "values": [ ]
2010 },
2011 "yaxes": [
2012 {
2013 "format": "short",
2014 "label": null,
2015 "logBase": 1,
2016 "max": null,
2017 "min": null,
2018 "show": true
2019 },
2020 {
2021 "format": "short",
2022 "label": null,
2023 "logBase": 1,
2024 "max": null,
2025 "min": null,
2026 "show": false
2027 }
2028 ],
2029 "yaxis": {
2030 "align": false,
2031 "alignLevel": null
2032 }
2033 },
2034 {
2035 "datasource": "$logs",
2036 "gridPos": {
2037 "h": 18,
2038 "w": 12,
2039 "x": 12,
2040 "y": 32
2041 },
2042 "id": 29,
2043 "options": {
2044 "showLabels": false,
2045 "showTime": false,
2046 "sortOrder": "Descending",
2047 "wrapLogMessage": true
2048 },
2049 "panels": [ ],
2050 "targets": [
2051 {
2052 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"",
2053 "refId": "A"
2054 }
2055 ],
2056 "timeFrom": null,
2057 "timeShift": null,
2058 "title": "Logs",
2059 "type": "logs"
2060 },
2061 {
2062 "aliasColors": { },
2063 "bars": false,
2064 "dashLength": 10,
2065 "dashes": false,
2066 "datasource": "$datasource",
2067 "fill": 0,
2068 "fillGradient": 0,
2069 "gridPos": {
2070 "h": 7,
2071 "w": 6,
2072 "x": 0,
2073 "y": 35
2074 },
2075 "hiddenSeries": false,
2076 "id": 33,
2077 "legend": {
2078 "avg": false,
2079 "current": false,
2080 "max": false,
2081 "min": false,
2082 "show": false,
2083 "total": false,
2084 "values": false
2085 },
2086 "lines": true,
2087 "linewidth": 1,
2088 "nullPointMode": "null",
2089 "options": {
2090 "dataLinks": [ ]
2091 },
2092 "panels": [ ],
2093 "percentage": false,
2094 "pointradius": 1,
2095 "points": false,
2096 "renderer": "flot",
2097 "seriesOverrides": [ ],
2098 "spaceLength": 10,
2099 "stack": false,
2100 "steppedLine": false,
2101 "targets": [
2102 {
2103 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[5m])) by (route)",
2104 "interval": "",
2105 "intervalFactor": 1,
2106 "legendFormat": "{{route}}",
2107 "refId": "A"
2108 }
2109 ],
2110 "thresholds": [ ],
2111 "timeFrom": null,
2112 "timeRegions": [ ],
2113 "timeShift": null,
2114 "title": "Success Rate",
2115 "tooltip": {
2116 "shared": true,
2117 "sort": 2,
2118 "value_type": "individual"
2119 },
2120 "type": "graph",
2121 "xaxis": {
2122 "buckets": null,
2123 "mode": "time",
2124 "name": null,
2125 "show": true,
2126 "values": [ ]
2127 },
2128 "yaxes": [
2129 {
2130 "format": "short",
2131 "label": null,
2132 "logBase": 1,
2133 "max": null,
2134 "min": null,
2135 "show": true
2136 },
2137 {
2138 "format": "short",
2139 "label": null,
2140 "logBase": 1,
2141 "max": null,
2142 "min": null,
2143 "show": true
2144 }
2145 ],
2146 "yaxis": {
2147 "align": false,
2148 "alignLevel": null
2149 }
2150 },
2151 {
2152 "aliasColors": { },
2153 "bars": false,
2154 "dashLength": 10,
2155 "dashes": false,
2156 "datasource": "$datasource",
2157 "fill": 1,
2158 "fillGradient": 0,
2159 "gridPos": {
2160 "h": 7,
2161 "w": 6,
2162 "x": 6,
2163 "y": 35
2164 },
2165 "hiddenSeries": false,
2166 "id": 32,
2167 "legend": {
2168 "avg": false,
2169 "current": false,
2170 "max": false,
2171 "min": false,
2172 "show": false,
2173 "total": false,
2174 "values": false
2175 },
2176 "lines": true,
2177 "linewidth": 1,
2178 "nullPointMode": "null",
2179 "options": {
2180 "dataLinks": [ ]
2181 },
2182 "panels": [ ],
2183 "percentage": false,
2184 "pointradius": 1,
2185 "points": false,
2186 "renderer": "flot",
2187 "seriesOverrides": [ ],
2188 "spaceLength": 10,
2189 "stack": false,
2190 "steppedLine": false,
2191 "targets": [
2192 {
2193 "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (ingester)",
2194 "intervalFactor": 1,
2195 "legendFormat": "{{ingester}}",
2196 "refId": "A"
2197 }
2198 ],
2199 "thresholds": [ ],
2200 "timeFrom": null,
2201 "timeRegions": [ ],
2202 "timeShift": null,
2203 "title": "Append Failures By Ingester",
2204 "tooltip": {
2205 "shared": true,
2206 "sort": 2,
2207 "value_type": "individual"
2208 },
2209 "type": "graph",
2210 "xaxis": {
2211 "buckets": null,
2212 "mode": "time",
2213 "name": null,
2214 "show": true,
2215 "values": [ ]
2216 },
2217 "yaxes": [
2218 {
2219 "format": "short",
2220 "label": null,
2221 "logBase": 1,
2222 "max": null,
2223 "min": null,
2224 "show": true
2225 },
2226 {
2227 "format": "short",
2228 "label": null,
2229 "logBase": 1,
2230 "max": null,
2231 "min": null,
2232 "show": true
2233 }
2234 ],
2235 "yaxis": {
2236 "align": false,
2237 "alignLevel": null
2238 }
2239 },
2240 {
2241 "aliasColors": { },
2242 "bars": false,
2243 "dashLength": 10,
2244 "dashes": false,
2245 "datasource": "$datasource",
2246 "fill": 1,
2247 "fillGradient": 0,
2248 "gridPos": {
2249 "h": 7,
2250 "w": 6,
2251 "x": 0,
2252 "y": 42
2253 },
2254 "hiddenSeries": false,
2255 "id": 34,
2256 "legend": {
2257 "avg": false,
2258 "current": false,
2259 "max": false,
2260 "min": false,
2261 "show": false,
2262 "total": false,
2263 "values": false
2264 },
2265 "lines": true,
2266 "linewidth": 1,
2267 "nullPointMode": "null",
2268 "options": {
2269 "dataLinks": [ ]
2270 },
2271 "panels": [ ],
2272 "percentage": false,
2273 "pointradius": 1,
2274 "points": false,
2275 "renderer": "flot",
2276 "seriesOverrides": [ ],
2277 "spaceLength": 10,
2278 "stack": false,
2279 "steppedLine": false,
2280 "targets": [
2281 {
2282 "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)",
2283 "intervalFactor": 1,
2284 "legendFormat": "{{pod}}",
2285 "refId": "A"
2286 }
2287 ],
2288 "thresholds": [ ],
2289 "timeFrom": null,
2290 "timeRegions": [ ],
2291 "timeShift": null,
2292 "title": "Bytes Received/Second",
2293 "tooltip": {
2294 "shared": true,
2295 "sort": 2,
2296 "value_type": "individual"
2297 },
2298 "type": "graph",
2299 "xaxis": {
2300 "buckets": null,
2301 "mode": "time",
2302 "name": null,
2303 "show": true,
2304 "values": [ ]
2305 },
2306 "yaxes": [
2307 {
2308 "format": "short",
2309 "label": null,
2310 "logBase": 1,
2311 "max": null,
2312 "min": null,
2313 "show": true
2314 },
2315 {
2316 "format": "short",
2317 "label": null,
2318 "logBase": 1,
2319 "max": null,
2320 "min": null,
2321 "show": true
2322 }
2323 ],
2324 "yaxis": {
2325 "align": false,
2326 "alignLevel": null
2327 }
2328 },
2329 {
2330 "aliasColors": { },
2331 "bars": false,
2332 "dashLength": 10,
2333 "dashes": false,
2334 "datasource": "$datasource",
2335 "fill": 1,
2336 "fillGradient": 0,
2337 "gridPos": {
2338 "h": 7,
2339 "w": 6,
2340 "x": 6,
2341 "y": 42
2342 },
2343 "hiddenSeries": false,
2344 "id": 35,
2345 "legend": {
2346 "avg": false,
2347 "current": false,
2348 "max": false,
2349 "min": false,
2350 "show": false,
2351 "total": false,
2352 "values": false
2353 },
2354 "lines": true,
2355 "linewidth": 1,
2356 "nullPointMode": "null",
2357 "options": {
2358 "dataLinks": [ ]
2359 },
2360 "panels": [ ],
2361 "percentage": false,
2362 "pointradius": 1,
2363 "points": false,
2364 "renderer": "flot",
2365 "seriesOverrides": [ ],
2366 "spaceLength": 10,
2367 "stack": false,
2368 "steppedLine": false,
2369 "targets": [
2370 {
2371 "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)",
2372 "intervalFactor": 1,
2373 "legendFormat": "{{pod}}",
2374 "refId": "A"
2375 }
2376 ],
2377 "thresholds": [ ],
2378 "timeFrom": null,
2379 "timeRegions": [ ],
2380 "timeShift": null,
2381 "title": "Lines Received/Second",
2382 "tooltip": {
2383 "shared": true,
2384 "sort": 2,
2385 "value_type": "individual"
2386 },
2387 "type": "graph",
2388 "xaxis": {
2389 "buckets": null,
2390 "mode": "time",
2391 "name": null,
2392 "show": true,
2393 "values": [ ]
2394 },
2395 "yaxes": [
2396 {
2397 "format": "short",
2398 "label": null,
2399 "logBase": 1,
2400 "max": null,
2401 "min": null,
2402 "show": true
2403 },
2404 {
2405 "format": "short",
2406 "label": null,
2407 "logBase": 1,
2408 "max": null,
2409 "min": null,
2410 "show": true
2411 }
2412 ],
2413 "yaxis": {
2414 "align": false,
2415 "alignLevel": null
2416 }
2417 }
2418 ],
2419 "targets": [ ],
2420 "title": "Write Path",
2421 "type": "row"
2422 },
2423 {
2424 "collapsed": true,
2425 "datasource": null,
2426 "gridPos": {
2427 "h": 1,
2428 "w": 24,
2429 "x": 0,
2430 "y": 29
2431 },
2432 "id": 104,
2433 "panels": [
2434 {
2435 "aliasColors": { },
2436 "bars": false,
2437 "dashLength": 10,
2438 "dashes": false,
2439 "datasource": "$datasource",
2440 "fill": 1,
2441 "fillGradient": 0,
2442 "gridPos": {
2443 "h": 8,
2444 "w": 12,
2445 "x": 0,
2446 "y": 30
2447 },
2448 "hiddenSeries": false,
2449 "id": 106,
2450 "legend": {
2451 "avg": false,
2452 "current": false,
2453 "hideEmpty": true,
2454 "hideZero": true,
2455 "max": false,
2456 "min": false,
2457 "show": false,
2458 "total": false,
2459 "values": false
2460 },
2461 "lines": true,
2462 "linewidth": 1,
2463 "nullPointMode": "null",
2464 "options": {
2465 "dataLinks": [ ]
2466 },
2467 "panels": [ ],
2468 "percentage": false,
2469 "pointradius": 2,
2470 "points": false,
2471 "renderer": "flot",
2472 "seriesOverrides": [ ],
2473 "spaceLength": 10,
2474 "stack": false,
2475 "steppedLine": false,
2476 "targets": [
2477 {
2478 "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}))",
2479 "interval": "",
2480 "legendFormat": "{{ tenant }}",
2481 "refId": "A"
2482 }
2483 ],
2484 "thresholds": [ ],
2485 "timeFrom": null,
2486 "timeRegions": [ ],
2487 "timeShift": null,
2488 "title": "Active Streams",
2489 "tooltip": {
2490 "shared": false,
2491 "sort": 0,
2492 "value_type": "individual"
2493 },
2494 "type": "graph",
2495 "xaxis": {
2496 "buckets": null,
2497 "mode": "time",
2498 "name": null,
2499 "show": true,
2500 "values": [ ]
2501 },
2502 "yaxes": [
2503 {
2504 "format": "short",
2505 "label": null,
2506 "logBase": 1,
2507 "max": null,
2508 "min": null,
2509 "show": true
2510 },
2511 {
2512 "format": "short",
2513 "label": null,
2514 "logBase": 1,
2515 "max": null,
2516 "min": null,
2517 "show": true
2518 }
2519 ],
2520 "yaxis": {
2521 "align": false,
2522 "alignLevel": null
2523 }
2524 },
2525 {
2526 "aliasColors": { },
2527 "bars": false,
2528 "dashLength": 10,
2529 "dashes": false,
2530 "datasource": "$datasource",
2531 "fill": 1,
2532 "fillGradient": 0,
2533 "gridPos": {
2534 "h": 8,
2535 "w": 12,
2536 "x": 12,
2537 "y": 30
2538 },
2539 "hiddenSeries": false,
2540 "id": 108,
2541 "legend": {
2542 "avg": false,
2543 "current": false,
2544 "hideEmpty": true,
2545 "hideZero": true,
2546 "max": false,
2547 "min": false,
2548 "show": false,
2549 "total": false,
2550 "values": false
2551 },
2552 "lines": true,
2553 "linewidth": 1,
2554 "nullPointMode": "null",
2555 "options": {
2556 "dataLinks": [ ]
2557 },
2558 "panels": [ ],
2559 "percentage": false,
2560 "pointradius": 2,
2561 "points": false,
2562 "renderer": "flot",
2563 "seriesOverrides": [ ],
2564 "spaceLength": 10,
2565 "stack": false,
2566 "steppedLine": false,
2567 "targets": [
2568 {
2569 "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]) > 0))",
2570 "interval": "",
2571 "legendFormat": "{{ tenant }}",
2572 "refId": "A"
2573 }
2574 ],
2575 "thresholds": [ ],
2576 "timeFrom": null,
2577 "timeRegions": [ ],
2578 "timeShift": null,
2579 "title": "Streams Created/Sec",
2580 "tooltip": {
2581 "shared": false,
2582 "sort": 0,
2583 "value_type": "individual"
2584 },
2585 "type": "graph",
2586 "xaxis": {
2587 "buckets": null,
2588 "mode": "time",
2589 "name": null,
2590 "show": true,
2591 "values": [ ]
2592 },
2593 "yaxes": [
2594 {
2595 "format": "short",
2596 "label": null,
2597 "logBase": 1,
2598 "max": null,
2599 "min": null,
2600 "show": true
2601 },
2602 {
2603 "format": "short",
2604 "label": null,
2605 "logBase": 1,
2606 "max": null,
2607 "min": null,
2608 "show": true
2609 }
2610 ],
2611 "yaxis": {
2612 "align": false,
2613 "alignLevel": null
2614 }
2615 }
2616 ],
2617 "targets": [ ],
2618 "title": "Streams",
2619 "type": "row"
2620 },
2621 {
2622 "collapsed": true,
2623 "datasource": null,
2624 "gridPos": {
2625 "h": 1,
2626 "w": 24,
2627 "x": 0,
2628 "y": 30
2629 },
2630 "id": 94,
2631 "panels": [
2632 {
2633 "aliasColors": { },
2634 "bars": false,
2635 "dashLength": 10,
2636 "dashes": false,
2637 "datasource": "$datasource",
2638 "fill": 1,
2639 "fillGradient": 0,
2640 "gridPos": {
2641 "h": 8,
2642 "w": 12,
2643 "x": 0,
2644 "y": 31
2645 },
2646 "hiddenSeries": false,
2647 "id": 102,
2648 "legend": {
2649 "avg": false,
2650 "current": false,
2651 "max": false,
2652 "min": false,
2653 "show": true,
2654 "total": false,
2655 "values": false
2656 },
2657 "lines": true,
2658 "linewidth": 1,
2659 "nullPointMode": "null",
2660 "options": {
2661 "dataLinks": [ ]
2662 },
2663 "panels": [ ],
2664 "percentage": false,
2665 "pointradius": 2,
2666 "points": false,
2667 "renderer": "flot",
2668 "seriesOverrides": [
2669 {
2670 "alias": "De-Dupe Ratio",
2671 "yaxis": 2
2672 }
2673 ],
2674 "spaceLength": 10,
2675 "stack": false,
2676 "steppedLine": false,
2677 "targets": [
2678 {
2679 "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))",
2680 "interval": "",
2681 "legendFormat": "Chunks",
2682 "refId": "A"
2683 },
2684 {
2685 "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) < 1",
2686 "interval": "",
2687 "legendFormat": "De-Dupe Ratio",
2688 "refId": "B"
2689 }
2690 ],
2691 "thresholds": [ ],
2692 "timeFrom": null,
2693 "timeRegions": [ ],
2694 "timeShift": null,
2695 "title": "Chunks Flushed/Sec",
2696 "tooltip": {
2697 "shared": true,
2698 "sort": 0,
2699 "value_type": "individual"
2700 },
2701 "type": "graph",
2702 "xaxis": {
2703 "buckets": null,
2704 "mode": "time",
2705 "name": null,
2706 "show": true,
2707 "values": [ ]
2708 },
2709 "yaxes": [
2710 {
2711 "format": "short",
2712 "label": null,
2713 "logBase": 1,
2714 "max": null,
2715 "min": null,
2716 "show": true
2717 },
2718 {
2719 "format": "short",
2720 "label": null,
2721 "logBase": 1,
2722 "max": null,
2723 "min": null,
2724 "show": true
2725 }
2726 ],
2727 "yaxis": {
2728 "align": false,
2729 "alignLevel": null
2730 }
2731 },
2732 {
2733 "cards": {
2734 "cardPadding": null,
2735 "cardRound": null
2736 },
2737 "color": {
2738 "cardColor": "#b4ff00",
2739 "colorScale": "sqrt",
2740 "colorScheme": "interpolateSpectral",
2741 "exponent": 0.5,
2742 "mode": "spectrum"
2743 },
2744 "dataFormat": "tsbuckets",
2745 "datasource": "$datasource",
2746 "gridPos": {
2747 "h": 8,
2748 "w": 12,
2749 "x": 12,
2750 "y": 31
2751 },
2752 "heatmap": { },
2753 "hideZeroBuckets": false,
2754 "highlightCards": true,
2755 "id": 100,
2756 "legend": {
2757 "show": true
2758 },
2759 "panels": [ ],
2760 "reverseYBuckets": false,
2761 "targets": [
2762 {
2763 "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) by (le)",
2764 "format": "heatmap",
2765 "instant": false,
2766 "interval": "",
2767 "legendFormat": "{{ le }}",
2768 "refId": "A"
2769 }
2770 ],
2771 "timeFrom": null,
2772 "timeShift": null,
2773 "title": "Chunk Size Bytes",
2774 "tooltip": {
2775 "show": true,
2776 "showHistogram": false
2777 },
2778 "type": "heatmap",
2779 "xAxis": {
2780 "show": true
2781 },
2782 "xBucketNumber": null,
2783 "xBucketSize": null,
2784 "yAxis": {
2785 "decimals": 0,
2786 "format": "bytes",
2787 "logBase": 1,
2788 "max": null,
2789 "min": null,
2790 "show": true,
2791 "splitFactor": null
2792 },
2793 "yBucketBound": "auto",
2794 "yBucketNumber": null,
2795 "yBucketSize": null
2796 },
2797 {
2798 "aliasColors": { },
2799 "bars": false,
2800 "dashLength": 10,
2801 "dashes": false,
2802 "datasource": "$datasource",
2803 "fill": 7,
2804 "fillGradient": 0,
2805 "gridPos": {
2806 "h": 9,
2807 "w": 12,
2808 "x": 0,
2809 "y": 39
2810 },
2811 "hiddenSeries": false,
2812 "id": 96,
2813 "legend": {
2814 "avg": false,
2815 "current": false,
2816 "max": false,
2817 "min": false,
2818 "show": true,
2819 "total": false,
2820 "values": false
2821 },
2822 "lines": true,
2823 "linewidth": 1,
2824 "nullPointMode": "null",
2825 "options": {
2826 "dataLinks": [ ]
2827 },
2828 "panels": [ ],
2829 "percentage": false,
2830 "pointradius": 2,
2831 "points": false,
2832 "renderer": "flot",
2833 "seriesOverrides": [ ],
2834 "spaceLength": 10,
2835 "stack": true,
2836 "steppedLine": false,
2837 "targets": [
2838 {
2839 "expr": "sum by(reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval]))",
2840 "interval": "",
2841 "legendFormat": "{{ reason }}"
2842 }
2843 ],
2844 "thresholds": [ ],
2845 "timeFrom": null,
2846 "timeRegions": [ ],
2847 "timeShift": null,
2848 "title": "Chunk Flush Reason %",
2849 "tooltip": {
2850 "shared": true,
2851 "sort": 0,
2852 "value_type": "individual"
2853 },
2854 "type": "graph",
2855 "xaxis": {
2856 "buckets": null,
2857 "mode": "time",
2858 "name": null,
2859 "show": true,
2860 "values": [ ]
2861 },
2862 "yaxes": [
2863 {
2864 "format": "percentunit",
2865 "label": null,
2866 "logBase": 1,
2867 "max": "1",
2868 "min": "0",
2869 "show": true
2870 },
2871 {
2872 "format": "short",
2873 "label": null,
2874 "logBase": 1,
2875 "max": null,
2876 "min": null,
2877 "show": true
2878 }
2879 ],
2880 "yaxis": {
2881 "align": false,
2882 "alignLevel": null
2883 }
2884 },
2885 {
2886 "cards": {
2887 "cardPadding": null,
2888 "cardRound": null
2889 },
2890 "color": {
2891 "cardColor": "#b4ff00",
2892 "colorScale": "sqrt",
2893 "colorScheme": "interpolateSpectral",
2894 "exponent": 0.5,
2895 "max": null,
2896 "min": null,
2897 "mode": "spectrum"
2898 },
2899 "dataFormat": "tsbuckets",
2900 "datasource": "$datasource",
2901 "gridPos": {
2902 "h": 9,
2903 "w": 12,
2904 "x": 12,
2905 "y": 39
2906 },
2907 "heatmap": { },
2908 "hideZeroBuckets": true,
2909 "highlightCards": true,
2910 "id": 98,
2911 "legend": {
2912 "show": true
2913 },
2914 "panels": [ ],
2915 "reverseYBuckets": false,
2916 "targets": [
2917 {
2918 "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))",
2919 "format": "heatmap",
2920 "instant": false,
2921 "interval": "",
2922 "legendFormat": "{{ le }}",
2923 "refId": "A"
2924 }
2925 ],
2926 "timeFrom": null,
2927 "timeShift": null,
2928 "title": "Chunk Utilization",
2929 "tooltip": {
2930 "show": true,
2931 "showHistogram": false
2932 },
2933 "type": "heatmap",
2934 "xAxis": {
2935 "show": true
2936 },
2937 "xBucketNumber": null,
2938 "xBucketSize": null,
2939 "yAxis": {
2940 "decimals": 0,
2941 "format": "percentunit",
2942 "logBase": 1,
2943 "max": null,
2944 "min": null,
2945 "show": true,
2946 "splitFactor": null
2947 },
2948 "yBucketBound": "auto",
2949 "yBucketNumber": null,
2950 "yBucketSize": null
2951 }
2952 ],
2953 "targets": [ ],
2954 "title": "Chunks",
2955 "type": "row"
2956 },
2957 {
2958 "collapsed": true,
2959 "datasource": null,
2960 "gridPos": {
2961 "h": 1,
2962 "w": 24,
2963 "x": 0,
2964 "y": 31
2965 },
2966 "id": 64,
2967 "panels": [
2968 {
2969 "aliasColors": { },
2970 "bars": false,
2971 "dashLength": 10,
2972 "dashes": false,
2973 "datasource": "$datasource",
2974 "fill": 1,
2975 "fillGradient": 0,
2976 "gridPos": {
2977 "h": 7,
2978 "w": 6,
2979 "x": 0,
2980 "y": 32
2981 },
2982 "hiddenSeries": false,
2983 "id": 68,
2984 "legend": {
2985 "avg": false,
2986 "current": false,
2987 "max": false,
2988 "min": false,
2989 "show": false,
2990 "total": false,
2991 "values": false
2992 },
2993 "lines": false,
2994 "linewidth": 1,
2995 "nullPointMode": "null",
2996 "options": {
2997 "dataLinks": [ ]
2998 },
2999 "panels": [ ],
3000 "percentage": false,
3001 "pointradius": 1,
3002 "points": true,
3003 "renderer": "flot",
3004 "seriesOverrides": [ ],
3005 "spaceLength": 10,
3006 "stack": false,
3007 "steppedLine": false,
3008 "targets": [
3009 {
3010 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}",
3011 "intervalFactor": 3,
3012 "legendFormat": "{{pod}}-{{container}}",
3013 "refId": "A"
3014 }
3015 ],
3016 "thresholds": [ ],
3017 "timeFrom": null,
3018 "timeRegions": [ ],
3019 "timeShift": null,
3020 "title": "CPU Usage",
3021 "tooltip": {
3022 "shared": true,
3023 "sort": 2,
3024 "value_type": "individual"
3025 },
3026 "type": "graph",
3027 "xaxis": {
3028 "buckets": null,
3029 "mode": "time",
3030 "name": null,
3031 "show": true,
3032 "values": [ ]
3033 },
3034 "yaxes": [
3035 {
3036 "format": "short",
3037 "label": null,
3038 "logBase": 1,
3039 "max": null,
3040 "min": null,
3041 "show": true
3042 },
3043 {
3044 "format": "short",
3045 "label": null,
3046 "logBase": 1,
3047 "max": null,
3048 "min": null,
3049 "show": true
3050 }
3051 ],
3052 "yaxis": {
3053 "align": false,
3054 "alignLevel": null
3055 }
3056 },
3057 {
3058 "aliasColors": { },
3059 "bars": false,
3060 "dashLength": 10,
3061 "dashes": false,
3062 "datasource": "$datasource",
3063 "fill": 1,
3064 "fillGradient": 0,
3065 "gridPos": {
3066 "h": 7,
3067 "w": 6,
3068 "x": 0,
3069 "y": 39
3070 },
3071 "hiddenSeries": false,
3072 "id": 69,
3073 "legend": {
3074 "avg": false,
3075 "current": false,
3076 "hideEmpty": false,
3077 "hideZero": false,
3078 "max": false,
3079 "min": false,
3080 "show": false,
3081 "total": false,
3082 "values": false
3083 },
3084 "lines": false,
3085 "linewidth": 1,
3086 "nullPointMode": "null",
3087 "options": {
3088 "dataLinks": [ ]
3089 },
3090 "panels": [ ],
3091 "percentage": false,
3092 "pointradius": 1,
3093 "points": true,
3094 "renderer": "flot",
3095 "seriesOverrides": [ ],
3096 "spaceLength": 10,
3097 "stack": false,
3098 "steppedLine": false,
3099 "targets": [
3100 {
3101 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}",
3102 "instant": false,
3103 "intervalFactor": 3,
3104 "legendFormat": "{{pod}}",
3105 "refId": "A"
3106 }
3107 ],
3108 "thresholds": [ ],
3109 "timeFrom": null,
3110 "timeRegions": [ ],
3111 "timeShift": null,
3112 "title": "Memory Usage",
3113 "tooltip": {
3114 "shared": true,
3115 "sort": 2,
3116 "value_type": "individual"
3117 },
3118 "type": "graph",
3119 "xaxis": {
3120 "buckets": null,
3121 "mode": "time",
3122 "name": null,
3123 "show": true,
3124 "values": [ ]
3125 },
3126 "yaxes": [
3127 {
3128 "format": "bytes",
3129 "label": null,
3130 "logBase": 1,
3131 "max": null,
3132 "min": null,
3133 "show": true
3134 },
3135 {
3136 "format": "short",
3137 "label": null,
3138 "logBase": 1,
3139 "max": null,
3140 "min": null,
3141 "show": true
3142 }
3143 ],
3144 "yaxis": {
3145 "align": false,
3146 "alignLevel": null
3147 }
3148 },
3149 {
3150 "aliasColors": { },
3151 "bars": true,
3152 "dashLength": 10,
3153 "dashes": false,
3154 "datasource": "$logs",
3155 "fill": 1,
3156 "fillGradient": 0,
3157 "gridPos": {
3158 "h": 3,
3159 "w": 18,
3160 "x": 12,
3161 "y": 32
3162 },
3163 "hiddenSeries": false,
3164 "id": 65,
3165 "legend": {
3166 "avg": false,
3167 "current": false,
3168 "max": false,
3169 "min": false,
3170 "show": false,
3171 "total": false,
3172 "values": false
3173 },
3174 "lines": false,
3175 "linewidth": 1,
3176 "nullPointMode": "null",
3177 "options": {
3178 "dataLinks": [ ]
3179 },
3180 "panels": [ ],
3181 "percentage": false,
3182 "pointradius": 2,
3183 "points": false,
3184 "renderer": "flot",
3185 "seriesOverrides": [
3186 {
3187 "alias": "{}",
3188 "color": "#F2495C"
3189 }
3190 ],
3191 "spaceLength": 10,
3192 "stack": false,
3193 "steppedLine": false,
3194 "targets": [
3195 {
3196 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"[1m]))",
3197 "refId": "A"
3198 }
3199 ],
3200 "thresholds": [ ],
3201 "timeFrom": null,
3202 "timeRegions": [ ],
3203 "timeShift": null,
3204 "title": "Error Log Rate",
3205 "tooltip": {
3206 "shared": true,
3207 "sort": 2,
3208 "value_type": "individual"
3209 },
3210 "type": "graph",
3211 "xaxis": {
3212 "buckets": null,
3213 "mode": "time",
3214 "name": null,
3215 "show": false,
3216 "values": [ ]
3217 },
3218 "yaxes": [
3219 {
3220 "format": "short",
3221 "label": null,
3222 "logBase": 1,
3223 "max": null,
3224 "min": null,
3225 "show": true
3226 },
3227 {
3228 "format": "short",
3229 "label": null,
3230 "logBase": 1,
3231 "max": null,
3232 "min": null,
3233 "show": false
3234 }
3235 ],
3236 "yaxis": {
3237 "align": false,
3238 "alignLevel": null
3239 }
3240 },
3241 {
3242 "datasource": "$logs",
3243 "gridPos": {
3244 "h": 18,
3245 "w": 18,
3246 "x": 12,
3247 "y": 35
3248 },
3249 "id": 66,
3250 "options": {
3251 "showLabels": false,
3252 "showTime": false,
3253 "sortOrder": "Descending",
3254 "wrapLogMessage": true
3255 },
3256 "panels": [ ],
3257 "targets": [
3258 {
3259 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"",
3260 "refId": "A"
3261 }
3262 ],
3263 "timeFrom": null,
3264 "timeShift": null,
3265 "title": "Logs",
3266 "type": "logs"
3267 },
3268 {
3269 "aliasColors": { },
3270 "bars": false,
3271 "dashLength": 10,
3272 "dashes": false,
3273 "datasource": "$datasource",
3274 "fill": 0,
3275 "fillGradient": 0,
3276 "gridPos": {
3277 "h": 7,
3278 "w": 6,
3279 "x": 0,
3280 "y": 46
3281 },
3282 "hiddenSeries": false,
3283 "id": 70,
3284 "legend": {
3285 "avg": false,
3286 "current": false,
3287 "max": false,
3288 "min": false,
3289 "show": false,
3290 "total": false,
3291 "values": false
3292 },
3293 "lines": true,
3294 "linewidth": 1,
3295 "nullPointMode": "null",
3296 "options": {
3297 "dataLinks": [ ]
3298 },
3299 "panels": [ ],
3300 "percentage": false,
3301 "pointradius": 1,
3302 "points": false,
3303 "renderer": "flot",
3304 "seriesOverrides": [ ],
3305 "spaceLength": 10,
3306 "stack": false,
3307 "steppedLine": false,
3308 "targets": [
3309 {
3310 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[1m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[1m])) by (route)",
3311 "interval": "",
3312 "intervalFactor": 1,
3313 "legendFormat": "{{route}}",
3314 "refId": "A"
3315 }
3316 ],
3317 "thresholds": [ ],
3318 "timeFrom": null,
3319 "timeRegions": [ ],
3320 "timeShift": null,
3321 "title": "Success Rate",
3322 "tooltip": {
3323 "shared": true,
3324 "sort": 2,
3325 "value_type": "individual"
3326 },
3327 "type": "graph",
3328 "xaxis": {
3329 "buckets": null,
3330 "mode": "time",
3331 "name": null,
3332 "show": true,
3333 "values": [ ]
3334 },
3335 "yaxes": [
3336 {
3337 "format": "short",
3338 "label": null,
3339 "logBase": 1,
3340 "max": null,
3341 "min": null,
3342 "show": true
3343 },
3344 {
3345 "format": "short",
3346 "label": null,
3347 "logBase": 1,
3348 "max": null,
3349 "min": null,
3350 "show": true
3351 }
3352 ],
3353 "yaxis": {
3354 "align": false,
3355 "alignLevel": null
3356 }
3357 }
3358 ],
3359 "targets": [ ],
3360 "title": "Read Path",
3361 "type": "row"
3362 },
3363 {
3364 "collapsed": true,
3365 "datasource": null,
3366 "gridPos": {
3367 "h": 1,
3368 "w": 24,
3369 "x": 0,
3370 "y": 32
3371 },
3372 "id": 52,
3373 "panels": [
3374 {
3375 "aliasColors": { },
3376 "bars": false,
3377 "dashLength": 10,
3378 "dashes": false,
3379 "datasource": "$datasource",
3380 "fill": 1,
3381 "fillGradient": 0,
3382 "gridPos": {
3383 "h": 8,
3384 "w": 24,
3385 "x": 0,
3386 "y": 30
3387 },
3388 "hiddenSeries": false,
3389 "id": 53,
3390 "interval": "",
3391 "legend": {
3392 "alignAsTable": true,
3393 "avg": false,
3394 "current": false,
3395 "max": false,
3396 "min": false,
3397 "rightSide": true,
3398 "show": true,
3399 "total": false,
3400 "values": false
3401 },
3402 "lines": true,
3403 "linewidth": 1,
3404 "nullPointMode": "null",
3405 "options": {
3406 "dataLinks": [ ]
3407 },
3408 "panels": [ ],
3409 "percentage": false,
3410 "pointradius": 1,
3411 "points": false,
3412 "renderer": "flot",
3413 "seriesOverrides": [ ],
3414 "spaceLength": 10,
3415 "stack": false,
3416 "steppedLine": false,
3417 "targets": [
3418 {
3419 "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))",
3420 "intervalFactor": 1,
3421 "legendFormat": "{{container}}: .99-{{method}}-{{name}}",
3422 "refId": "A"
3423 },
3424 {
3425 "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))",
3426 "hide": false,
3427 "legendFormat": "{{container}}: .9-{{method}}-{{name}}",
3428 "refId": "B"
3429 },
3430 {
3431 "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))",
3432 "hide": false,
3433 "legendFormat": "{{container}}: .5-{{method}}-{{name}}",
3434 "refId": "C"
3435 }
3436 ],
3437 "thresholds": [ ],
3438 "timeFrom": null,
3439 "timeRegions": [ ],
3440 "timeShift": null,
3441 "title": "Latency By Method",
3442 "tooltip": {
3443 "shared": true,
3444 "sort": 2,
3445 "value_type": "individual"
3446 },
3447 "type": "graph",
3448 "xaxis": {
3449 "buckets": null,
3450 "mode": "time",
3451 "name": null,
3452 "show": true,
3453 "values": [ ]
3454 },
3455 "yaxes": [
3456 {
3457 "format": "short",
3458 "label": null,
3459 "logBase": 1,
3460 "max": null,
3461 "min": null,
3462 "show": true
3463 },
3464 {
3465 "format": "short",
3466 "label": null,
3467 "logBase": 1,
3468 "max": null,
3469 "min": null,
3470 "show": true
3471 }
3472 ],
3473 "yaxis": {
3474 "align": false,
3475 "alignLevel": null
3476 }
3477 },
3478 {
3479 "aliasColors": { },
3480 "bars": false,
3481 "dashLength": 10,
3482 "dashes": false,
3483 "datasource": "$datasource",
3484 "fill": 1,
3485 "fillGradient": 0,
3486 "gridPos": {
3487 "h": 8,
3488 "w": 24,
3489 "x": 0,
3490 "y": 38
3491 },
3492 "hiddenSeries": false,
3493 "id": 54,
3494 "interval": "",
3495 "legend": {
3496 "alignAsTable": true,
3497 "avg": false,
3498 "current": false,
3499 "max": false,
3500 "min": false,
3501 "rightSide": true,
3502 "show": true,
3503 "total": false,
3504 "values": false
3505 },
3506 "lines": true,
3507 "linewidth": 1,
3508 "nullPointMode": "null",
3509 "options": {
3510 "dataLinks": [ ]
3511 },
3512 "panels": [ ],
3513 "percentage": false,
3514 "pointradius": 1,
3515 "points": false,
3516 "renderer": "flot",
3517 "seriesOverrides": [ ],
3518 "spaceLength": 10,
3519 "stack": false,
3520 "steppedLine": false,
3521 "targets": [
3522 {
3523 "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)",
3524 "intervalFactor": 1,
3525 "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}",
3526 "refId": "A"
3527 }
3528 ],
3529 "thresholds": [ ],
3530 "timeFrom": null,
3531 "timeRegions": [ ],
3532 "timeShift": null,
3533 "title": "Status By Method",
3534 "tooltip": {
3535 "shared": true,
3536 "sort": 2,
3537 "value_type": "individual"
3538 },
3539 "type": "graph",
3540 "xaxis": {
3541 "buckets": null,
3542 "mode": "time",
3543 "name": null,
3544 "show": true,
3545 "values": [ ]
3546 },
3547 "yaxes": [
3548 {
3549 "format": "short",
3550 "label": null,
3551 "logBase": 1,
3552 "max": null,
3553 "min": null,
3554 "show": true
3555 },
3556 {
3557 "format": "short",
3558 "label": null,
3559 "logBase": 1,
3560 "max": null,
3561 "min": null,
3562 "show": true
3563 }
3564 ],
3565 "yaxis": {
3566 "align": false,
3567 "alignLevel": null
3568 }
3569 }
3570 ],
3571 "targets": [ ],
3572 "title": "Memcached",
3573 "type": "row"
3574 },
3575 {
3576 "collapsed": true,
3577 "datasource": null,
3578 "gridPos": {
3579 "h": 1,
3580 "w": 24,
3581 "x": 0,
3582 "y": 33
3583 },
3584 "id": 57,
3585 "panels": [
3586 {
3587 "aliasColors": { },
3588 "bars": false,
3589 "dashLength": 10,
3590 "dashes": false,
3591 "datasource": "$datasource",
3592 "fill": 1,
3593 "fillGradient": 0,
3594 "gridPos": {
3595 "h": 8,
3596 "w": 24,
3597 "x": 0,
3598 "y": 31
3599 },
3600 "hiddenSeries": false,
3601 "id": 55,
3602 "interval": "",
3603 "legend": {
3604 "alignAsTable": true,
3605 "avg": false,
3606 "current": false,
3607 "max": false,
3608 "min": false,
3609 "rightSide": true,
3610 "show": true,
3611 "total": false,
3612 "values": false
3613 },
3614 "lines": true,
3615 "linewidth": 1,
3616 "nullPointMode": "null",
3617 "options": {
3618 "dataLinks": [ ]
3619 },
3620 "panels": [ ],
3621 "percentage": false,
3622 "pointradius": 1,
3623 "points": false,
3624 "renderer": "flot",
3625 "seriesOverrides": [ ],
3626 "spaceLength": 10,
3627 "stack": false,
3628 "steppedLine": false,
3629 "targets": [
3630 {
3631 "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
3632 "intervalFactor": 1,
3633 "legendFormat": ".99-{{operation}}",
3634 "refId": "A"
3635 },
3636 {
3637 "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
3638 "hide": false,
3639 "legendFormat": ".9-{{operation}}",
3640 "refId": "B"
3641 },
3642 {
3643 "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
3644 "hide": false,
3645 "legendFormat": ".5-{{operation}}",
3646 "refId": "C"
3647 }
3648 ],
3649 "thresholds": [ ],
3650 "timeFrom": null,
3651 "timeRegions": [ ],
3652 "timeShift": null,
3653 "title": "Latency By Operation",
3654 "tooltip": {
3655 "shared": true,
3656 "sort": 2,
3657 "value_type": "individual"
3658 },
3659 "type": "graph",
3660 "xaxis": {
3661 "buckets": null,
3662 "mode": "time",
3663 "name": null,
3664 "show": true,
3665 "values": [ ]
3666 },
3667 "yaxes": [
3668 {
3669 "format": "short",
3670 "label": null,
3671 "logBase": 1,
3672 "max": null,
3673 "min": null,
3674 "show": true
3675 },
3676 {
3677 "format": "short",
3678 "label": null,
3679 "logBase": 1,
3680 "max": null,
3681 "min": null,
3682 "show": true
3683 }
3684 ],
3685 "yaxis": {
3686 "align": false,
3687 "alignLevel": null
3688 }
3689 },
3690 {
3691 "aliasColors": { },
3692 "bars": false,
3693 "dashLength": 10,
3694 "dashes": false,
3695 "datasource": "$datasource",
3696 "fill": 1,
3697 "fillGradient": 0,
3698 "gridPos": {
3699 "h": 8,
3700 "w": 24,
3701 "x": 0,
3702 "y": 39
3703 },
3704 "hiddenSeries": false,
3705 "id": 58,
3706 "interval": "",
3707 "legend": {
3708 "alignAsTable": true,
3709 "avg": false,
3710 "current": false,
3711 "max": false,
3712 "min": false,
3713 "rightSide": true,
3714 "show": true,
3715 "total": false,
3716 "values": false
3717 },
3718 "lines": true,
3719 "linewidth": 1,
3720 "nullPointMode": "null",
3721 "options": {
3722 "dataLinks": [ ]
3723 },
3724 "panels": [ ],
3725 "percentage": false,
3726 "pointradius": 1,
3727 "points": false,
3728 "renderer": "flot",
3729 "seriesOverrides": [ ],
3730 "spaceLength": 10,
3731 "stack": false,
3732 "steppedLine": false,
3733 "targets": [
3734 {
3735 "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)",
3736 "intervalFactor": 1,
3737 "legendFormat": "{{status_code}}-{{operation}}",
3738 "refId": "A"
3739 }
3740 ],
3741 "thresholds": [ ],
3742 "timeFrom": null,
3743 "timeRegions": [ ],
3744 "timeShift": null,
3745 "title": "Status By Operation",
3746 "tooltip": {
3747 "shared": true,
3748 "sort": 2,
3749 "value_type": "individual"
3750 },
3751 "type": "graph",
3752 "xaxis": {
3753 "buckets": null,
3754 "mode": "time",
3755 "name": null,
3756 "show": true,
3757 "values": [ ]
3758 },
3759 "yaxes": [
3760 {
3761 "format": "short",
3762 "label": null,
3763 "logBase": 1,
3764 "max": null,
3765 "min": null,
3766 "show": true
3767 },
3768 {
3769 "format": "short",
3770 "label": null,
3771 "logBase": 1,
3772 "max": null,
3773 "min": null,
3774 "show": true
3775 }
3776 ],
3777 "yaxis": {
3778 "align": false,
3779 "alignLevel": null
3780 }
3781 }
3782 ],
3783 "targets": [ ],
3784 "title": "Consul",
3785 "type": "row"
3786 },
3787 {
3788 "collapsed": true,
3789 "datasource": null,
3790 "gridPos": {
3791 "h": 1,
3792 "w": 24,
3793 "x": 0,
3794 "y": 34
3795 },
3796 "id": 43,
3797 "panels": [
3798 {
3799 "aliasColors": { },
3800 "bars": false,
3801 "dashLength": 10,
3802 "dashes": false,
3803 "datasource": "$datasource",
3804 "fill": 1,
3805 "fillGradient": 0,
3806 "gridPos": {
3807 "h": 7,
3808 "w": 6,
3809 "x": 0,
3810 "y": 9
3811 },
3812 "hiddenSeries": false,
3813 "id": 41,
3814 "interval": "",
3815 "legend": {
3816 "avg": false,
3817 "current": false,
3818 "max": false,
3819 "min": false,
3820 "show": false,
3821 "total": false,
3822 "values": false
3823 },
3824 "lines": true,
3825 "linewidth": 1,
3826 "nullPointMode": "null",
3827 "options": {
3828 "dataLinks": [ ]
3829 },
3830 "panels": [ ],
3831 "percentage": false,
3832 "pointradius": 1,
3833 "points": false,
3834 "renderer": "flot",
3835 "seriesOverrides": [ ],
3836 "spaceLength": 10,
3837 "stack": false,
3838 "steppedLine": false,
3839 "targets": [
3840 {
3841 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))",
3842 "intervalFactor": 1,
3843 "legendFormat": ".9",
3844 "refId": "A"
3845 },
3846 {
3847 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))",
3848 "refId": "B"
3849 },
3850 {
3851 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))",
3852 "refId": "C"
3853 }
3854 ],
3855 "thresholds": [ ],
3856 "timeFrom": null,
3857 "timeRegions": [ ],
3858 "timeShift": null,
3859 "title": "MutateRows Latency",
3860 "tooltip": {
3861 "shared": true,
3862 "sort": 2,
3863 "value_type": "individual"
3864 },
3865 "type": "graph",
3866 "xaxis": {
3867 "buckets": null,
3868 "mode": "time",
3869 "name": null,
3870 "show": true,
3871 "values": [ ]
3872 },
3873 "yaxes": [
3874 {
3875 "format": "short",
3876 "label": null,
3877 "logBase": 1,
3878 "max": null,
3879 "min": null,
3880 "show": true
3881 },
3882 {
3883 "format": "short",
3884 "label": null,
3885 "logBase": 1,
3886 "max": null,
3887 "min": null,
3888 "show": true
3889 }
3890 ],
3891 "yaxis": {
3892 "align": false,
3893 "alignLevel": null
3894 }
3895 },
3896 {
3897 "aliasColors": { },
3898 "bars": false,
3899 "dashLength": 10,
3900 "dashes": false,
3901 "datasource": "$datasource",
3902 "fill": 1,
3903 "fillGradient": 0,
3904 "gridPos": {
3905 "h": 7,
3906 "w": 6,
3907 "x": 6,
3908 "y": 9
3909 },
3910 "hiddenSeries": false,
3911 "id": 46,
3912 "interval": "",
3913 "legend": {
3914 "avg": false,
3915 "current": false,
3916 "max": false,
3917 "min": false,
3918 "show": false,
3919 "total": false,
3920 "values": false
3921 },
3922 "lines": true,
3923 "linewidth": 1,
3924 "nullPointMode": "null",
3925 "options": {
3926 "dataLinks": [ ]
3927 },
3928 "panels": [ ],
3929 "percentage": false,
3930 "pointradius": 1,
3931 "points": false,
3932 "renderer": "flot",
3933 "seriesOverrides": [ ],
3934 "spaceLength": 10,
3935 "stack": false,
3936 "steppedLine": false,
3937 "targets": [
3938 {
3939 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))",
3940 "interval": "",
3941 "intervalFactor": 1,
3942 "legendFormat": "99%",
3943 "refId": "A"
3944 },
3945 {
3946 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))",
3947 "interval": "",
3948 "legendFormat": "90%",
3949 "refId": "B"
3950 },
3951 {
3952 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))",
3953 "interval": "",
3954 "legendFormat": "50%",
3955 "refId": "C"
3956 }
3957 ],
3958 "thresholds": [ ],
3959 "timeFrom": null,
3960 "timeRegions": [ ],
3961 "timeShift": null,
3962 "title": "ReadRows Latency",
3963 "tooltip": {
3964 "shared": true,
3965 "sort": 2,
3966 "value_type": "individual"
3967 },
3968 "type": "graph",
3969 "xaxis": {
3970 "buckets": null,
3971 "mode": "time",
3972 "name": null,
3973 "show": true,
3974 "values": [ ]
3975 },
3976 "yaxes": [
3977 {
3978 "format": "short",
3979 "label": null,
3980 "logBase": 1,
3981 "max": null,
3982 "min": null,
3983 "show": true
3984 },
3985 {
3986 "format": "short",
3987 "label": null,
3988 "logBase": 1,
3989 "max": null,
3990 "min": null,
3991 "show": true
3992 }
3993 ],
3994 "yaxis": {
3995 "align": false,
3996 "alignLevel": null
3997 }
3998 },
3999 {
4000 "aliasColors": { },
4001 "bars": false,
4002 "dashLength": 10,
4003 "dashes": false,
4004 "datasource": "$datasource",
4005 "fill": 1,
4006 "fillGradient": 0,
4007 "gridPos": {
4008 "h": 7,
4009 "w": 6,
4010 "x": 12,
4011 "y": 9
4012 },
4013 "hiddenSeries": false,
4014 "id": 44,
4015 "interval": "",
4016 "legend": {
4017 "avg": false,
4018 "current": false,
4019 "max": false,
4020 "min": false,
4021 "show": false,
4022 "total": false,
4023 "values": false
4024 },
4025 "lines": true,
4026 "linewidth": 1,
4027 "nullPointMode": "null",
4028 "options": {
4029 "dataLinks": [ ]
4030 },
4031 "panels": [ ],
4032 "percentage": false,
4033 "pointradius": 1,
4034 "points": false,
4035 "renderer": "flot",
4036 "seriesOverrides": [ ],
4037 "spaceLength": 10,
4038 "stack": false,
4039 "steppedLine": false,
4040 "targets": [
4041 {
4042 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))",
4043 "interval": "",
4044 "intervalFactor": 1,
4045 "legendFormat": "99%",
4046 "refId": "A"
4047 },
4048 {
4049 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))",
4050 "interval": "",
4051 "legendFormat": "90%",
4052 "refId": "B"
4053 },
4054 {
4055 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))",
4056 "interval": "",
4057 "legendFormat": "50%",
4058 "refId": "C"
4059 }
4060 ],
4061 "thresholds": [ ],
4062 "timeFrom": null,
4063 "timeRegions": [ ],
4064 "timeShift": null,
4065 "title": "GetTable Latency",
4066 "tooltip": {
4067 "shared": true,
4068 "sort": 2,
4069 "value_type": "individual"
4070 },
4071 "type": "graph",
4072 "xaxis": {
4073 "buckets": null,
4074 "mode": "time",
4075 "name": null,
4076 "show": true,
4077 "values": [ ]
4078 },
4079 "yaxes": [
4080 {
4081 "format": "short",
4082 "label": null,
4083 "logBase": 1,
4084 "max": null,
4085 "min": null,
4086 "show": true
4087 },
4088 {
4089 "format": "short",
4090 "label": null,
4091 "logBase": 1,
4092 "max": null,
4093 "min": null,
4094 "show": true
4095 }
4096 ],
4097 "yaxis": {
4098 "align": false,
4099 "alignLevel": null
4100 }
4101 },
4102 {
4103 "aliasColors": { },
4104 "bars": false,
4105 "dashLength": 10,
4106 "dashes": false,
4107 "datasource": "$datasource",
4108 "fill": 1,
4109 "fillGradient": 0,
4110 "gridPos": {
4111 "h": 7,
4112 "w": 6,
4113 "x": 18,
4114 "y": 9
4115 },
4116 "hiddenSeries": false,
4117 "id": 45,
4118 "interval": "",
4119 "legend": {
4120 "avg": false,
4121 "current": false,
4122 "max": false,
4123 "min": false,
4124 "show": false,
4125 "total": false,
4126 "values": false
4127 },
4128 "lines": true,
4129 "linewidth": 1,
4130 "nullPointMode": "null",
4131 "options": {
4132 "dataLinks": [ ]
4133 },
4134 "panels": [ ],
4135 "percentage": false,
4136 "pointradius": 1,
4137 "points": false,
4138 "renderer": "flot",
4139 "seriesOverrides": [ ],
4140 "spaceLength": 10,
4141 "stack": false,
4142 "steppedLine": false,
4143 "targets": [
4144 {
4145 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))",
4146 "intervalFactor": 1,
4147 "legendFormat": ".9",
4148 "refId": "A"
4149 },
4150 {
4151 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))",
4152 "refId": "B"
4153 },
4154 {
4155 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))",
4156 "refId": "C"
4157 }
4158 ],
4159 "thresholds": [ ],
4160 "timeFrom": null,
4161 "timeRegions": [ ],
4162 "timeShift": null,
4163 "title": "ListTables Latency",
4164 "tooltip": {
4165 "shared": true,
4166 "sort": 2,
4167 "value_type": "individual"
4168 },
4169 "type": "graph",
4170 "xaxis": {
4171 "buckets": null,
4172 "mode": "time",
4173 "name": null,
4174 "show": true,
4175 "values": [ ]
4176 },
4177 "yaxes": [
4178 {
4179 "format": "short",
4180 "label": null,
4181 "logBase": 1,
4182 "max": null,
4183 "min": null,
4184 "show": true
4185 },
4186 {
4187 "format": "short",
4188 "label": null,
4189 "logBase": 1,
4190 "max": null,
4191 "min": null,
4192 "show": true
4193 }
4194 ],
4195 "yaxis": {
4196 "align": false,
4197 "alignLevel": null
4198 }
4199 },
4200 {
4201 "aliasColors": { },
4202 "bars": false,
4203 "dashLength": 10,
4204 "dashes": false,
4205 "datasource": "$datasource",
4206 "fill": 1,
4207 "fillGradient": 0,
4208 "gridPos": {
4209 "h": 7,
4210 "w": 6,
4211 "x": 0,
4212 "y": 16
4213 },
4214 "hiddenSeries": false,
4215 "id": 47,
4216 "interval": "",
4217 "legend": {
4218 "avg": false,
4219 "current": false,
4220 "max": false,
4221 "min": false,
4222 "show": false,
4223 "total": false,
4224 "values": false
4225 },
4226 "lines": true,
4227 "linewidth": 1,
4228 "nullPointMode": "null",
4229 "options": {
4230 "dataLinks": [ ]
4231 },
4232 "panels": [ ],
4233 "percentage": false,
4234 "pointradius": 1,
4235 "points": false,
4236 "renderer": "flot",
4237 "seriesOverrides": [ ],
4238 "spaceLength": 10,
4239 "stack": false,
4240 "steppedLine": false,
4241 "targets": [
4242 {
4243 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)",
4244 "intervalFactor": 1,
4245 "legendFormat": "{{status_code}}",
4246 "refId": "A"
4247 }
4248 ],
4249 "thresholds": [ ],
4250 "timeFrom": null,
4251 "timeRegions": [ ],
4252 "timeShift": null,
4253 "title": "MutateRows Status",
4254 "tooltip": {
4255 "shared": true,
4256 "sort": 2,
4257 "value_type": "individual"
4258 },
4259 "type": "graph",
4260 "xaxis": {
4261 "buckets": null,
4262 "mode": "time",
4263 "name": null,
4264 "show": true,
4265 "values": [ ]
4266 },
4267 "yaxes": [
4268 {
4269 "format": "short",
4270 "label": null,
4271 "logBase": 1,
4272 "max": null,
4273 "min": null,
4274 "show": true
4275 },
4276 {
4277 "format": "short",
4278 "label": null,
4279 "logBase": 1,
4280 "max": null,
4281 "min": null,
4282 "show": true
4283 }
4284 ],
4285 "yaxis": {
4286 "align": false,
4287 "alignLevel": null
4288 }
4289 },
4290 {
4291 "aliasColors": { },
4292 "bars": false,
4293 "dashLength": 10,
4294 "dashes": false,
4295 "datasource": "$datasource",
4296 "fill": 1,
4297 "fillGradient": 0,
4298 "gridPos": {
4299 "h": 7,
4300 "w": 6,
4301 "x": 6,
4302 "y": 16
4303 },
4304 "hiddenSeries": false,
4305 "id": 50,
4306 "interval": "",
4307 "legend": {
4308 "avg": false,
4309 "current": false,
4310 "max": false,
4311 "min": false,
4312 "show": false,
4313 "total": false,
4314 "values": false
4315 },
4316 "lines": true,
4317 "linewidth": 1,
4318 "nullPointMode": "null",
4319 "options": {
4320 "dataLinks": [ ]
4321 },
4322 "panels": [ ],
4323 "percentage": false,
4324 "pointradius": 1,
4325 "points": false,
4326 "renderer": "flot",
4327 "seriesOverrides": [ ],
4328 "spaceLength": 10,
4329 "stack": false,
4330 "steppedLine": false,
4331 "targets": [
4332 {
4333 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)",
4334 "intervalFactor": 1,
4335 "legendFormat": "{{status_code}}",
4336 "refId": "A"
4337 }
4338 ],
4339 "thresholds": [ ],
4340 "timeFrom": null,
4341 "timeRegions": [ ],
4342 "timeShift": null,
4343 "title": "ReadRows Status",
4344 "tooltip": {
4345 "shared": true,
4346 "sort": 2,
4347 "value_type": "individual"
4348 },
4349 "type": "graph",
4350 "xaxis": {
4351 "buckets": null,
4352 "mode": "time",
4353 "name": null,
4354 "show": true,
4355 "values": [ ]
4356 },
4357 "yaxes": [
4358 {
4359 "format": "short",
4360 "label": null,
4361 "logBase": 1,
4362 "max": null,
4363 "min": null,
4364 "show": true
4365 },
4366 {
4367 "format": "short",
4368 "label": null,
4369 "logBase": 1,
4370 "max": null,
4371 "min": null,
4372 "show": true
4373 }
4374 ],
4375 "yaxis": {
4376 "align": false,
4377 "alignLevel": null
4378 }
4379 },
4380 {
4381 "aliasColors": { },
4382 "bars": false,
4383 "dashLength": 10,
4384 "dashes": false,
4385 "datasource": "$datasource",
4386 "fill": 1,
4387 "fillGradient": 0,
4388 "gridPos": {
4389 "h": 7,
4390 "w": 6,
4391 "x": 12,
4392 "y": 16
4393 },
4394 "hiddenSeries": false,
4395 "id": 48,
4396 "interval": "",
4397 "legend": {
4398 "avg": false,
4399 "current": false,
4400 "max": false,
4401 "min": false,
4402 "show": false,
4403 "total": false,
4404 "values": false
4405 },
4406 "lines": true,
4407 "linewidth": 1,
4408 "nullPointMode": "null",
4409 "options": {
4410 "dataLinks": [ ]
4411 },
4412 "panels": [ ],
4413 "percentage": false,
4414 "pointradius": 1,
4415 "points": false,
4416 "renderer": "flot",
4417 "seriesOverrides": [ ],
4418 "spaceLength": 10,
4419 "stack": false,
4420 "steppedLine": false,
4421 "targets": [
4422 {
4423 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)",
4424 "intervalFactor": 1,
4425 "legendFormat": "{{status_code}}",
4426 "refId": "A"
4427 }
4428 ],
4429 "thresholds": [ ],
4430 "timeFrom": null,
4431 "timeRegions": [ ],
4432 "timeShift": null,
4433 "title": "GetTable Status",
4434 "tooltip": {
4435 "shared": true,
4436 "sort": 2,
4437 "value_type": "individual"
4438 },
4439 "type": "graph",
4440 "xaxis": {
4441 "buckets": null,
4442 "mode": "time",
4443 "name": null,
4444 "show": true,
4445 "values": [ ]
4446 },
4447 "yaxes": [
4448 {
4449 "format": "short",
4450 "label": null,
4451 "logBase": 1,
4452 "max": null,
4453 "min": null,
4454 "show": true
4455 },
4456 {
4457 "format": "short",
4458 "label": null,
4459 "logBase": 1,
4460 "max": null,
4461 "min": null,
4462 "show": true
4463 }
4464 ],
4465 "yaxis": {
4466 "align": false,
4467 "alignLevel": null
4468 }
4469 },
4470 {
4471 "aliasColors": { },
4472 "bars": false,
4473 "dashLength": 10,
4474 "dashes": false,
4475 "datasource": "$datasource",
4476 "fill": 1,
4477 "fillGradient": 0,
4478 "gridPos": {
4479 "h": 7,
4480 "w": 6,
4481 "x": 18,
4482 "y": 16
4483 },
4484 "hiddenSeries": false,
4485 "id": 49,
4486 "interval": "",
4487 "legend": {
4488 "avg": false,
4489 "current": false,
4490 "max": false,
4491 "min": false,
4492 "show": false,
4493 "total": false,
4494 "values": false
4495 },
4496 "lines": true,
4497 "linewidth": 1,
4498 "nullPointMode": "null",
4499 "options": {
4500 "dataLinks": [ ]
4501 },
4502 "panels": [ ],
4503 "percentage": false,
4504 "pointradius": 1,
4505 "points": false,
4506 "renderer": "flot",
4507 "seriesOverrides": [ ],
4508 "spaceLength": 10,
4509 "stack": false,
4510 "steppedLine": false,
4511 "targets": [
4512 {
4513 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)",
4514 "intervalFactor": 1,
4515 "legendFormat": "{{status_code}}",
4516 "refId": "A"
4517 }
4518 ],
4519 "thresholds": [ ],
4520 "timeFrom": null,
4521 "timeRegions": [ ],
4522 "timeShift": null,
4523 "title": "ListTables Status",
4524 "tooltip": {
4525 "shared": true,
4526 "sort": 2,
4527 "value_type": "individual"
4528 },
4529 "type": "graph",
4530 "xaxis": {
4531 "buckets": null,
4532 "mode": "time",
4533 "name": null,
4534 "show": true,
4535 "values": [ ]
4536 },
4537 "yaxes": [
4538 {
4539 "format": "short",
4540 "label": null,
4541 "logBase": 1,
4542 "max": null,
4543 "min": null,
4544 "show": true
4545 },
4546 {
4547 "format": "short",
4548 "label": null,
4549 "logBase": 1,
4550 "max": null,
4551 "min": null,
4552 "show": true
4553 }
4554 ],
4555 "yaxis": {
4556 "align": false,
4557 "alignLevel": null
4558 }
4559 }
4560 ],
4561 "targets": [ ],
4562 "title": "Big Table",
4563 "type": "row"
4564 },
4565 {
4566 "collapsed": true,
4567 "datasource": null,
4568 "gridPos": {
4569 "h": 1,
4570 "w": 24,
4571 "x": 0,
4572 "y": 35
4573 },
4574 "id": 60,
4575 "panels": [
4576 {
4577 "aliasColors": { },
4578 "bars": false,
4579 "dashLength": 10,
4580 "dashes": false,
4581 "datasource": "$datasource",
4582 "fill": 1,
4583 "fillGradient": 0,
4584 "gridPos": {
4585 "h": 8,
4586 "w": 24,
4587 "x": 0,
4588 "y": 33
4589 },
4590 "hiddenSeries": false,
4591 "id": 61,
4592 "interval": "",
4593 "legend": {
4594 "alignAsTable": true,
4595 "avg": false,
4596 "current": false,
4597 "max": false,
4598 "min": false,
4599 "rightSide": true,
4600 "show": true,
4601 "total": false,
4602 "values": false
4603 },
4604 "lines": true,
4605 "linewidth": 1,
4606 "nullPointMode": "null",
4607 "options": {
4608 "dataLinks": [ ]
4609 },
4610 "panels": [ ],
4611 "percentage": false,
4612 "pointradius": 1,
4613 "points": false,
4614 "renderer": "flot",
4615 "seriesOverrides": [ ],
4616 "spaceLength": 10,
4617 "stack": false,
4618 "steppedLine": false,
4619 "targets": [
4620 {
4621 "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
4622 "intervalFactor": 1,
4623 "legendFormat": ".99-{{operation}}",
4624 "refId": "A"
4625 },
4626 {
4627 "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
4628 "hide": false,
4629 "legendFormat": ".9-{{operation}}",
4630 "refId": "B"
4631 },
4632 {
4633 "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
4634 "hide": false,
4635 "legendFormat": ".5-{{operation}}",
4636 "refId": "C"
4637 }
4638 ],
4639 "thresholds": [ ],
4640 "timeFrom": null,
4641 "timeRegions": [ ],
4642 "timeShift": null,
4643 "title": "Latency By Operation",
4644 "tooltip": {
4645 "shared": true,
4646 "sort": 2,
4647 "value_type": "individual"
4648 },
4649 "type": "graph",
4650 "xaxis": {
4651 "buckets": null,
4652 "mode": "time",
4653 "name": null,
4654 "show": true,
4655 "values": [ ]
4656 },
4657 "yaxes": [
4658 {
4659 "format": "short",
4660 "label": null,
4661 "logBase": 1,
4662 "max": null,
4663 "min": null,
4664 "show": true
4665 },
4666 {
4667 "format": "short",
4668 "label": null,
4669 "logBase": 1,
4670 "max": null,
4671 "min": null,
4672 "show": true
4673 }
4674 ],
4675 "yaxis": {
4676 "align": false,
4677 "alignLevel": null
4678 }
4679 },
4680 {
4681 "aliasColors": { },
4682 "bars": false,
4683 "dashLength": 10,
4684 "dashes": false,
4685 "datasource": "$datasource",
4686 "fill": 1,
4687 "fillGradient": 0,
4688 "gridPos": {
4689 "h": 8,
4690 "w": 24,
4691 "x": 0,
4692 "y": 41
4693 },
4694 "hiddenSeries": false,
4695 "id": 62,
4696 "interval": "",
4697 "legend": {
4698 "alignAsTable": true,
4699 "avg": false,
4700 "current": false,
4701 "max": false,
4702 "min": false,
4703 "rightSide": true,
4704 "show": true,
4705 "total": false,
4706 "values": false
4707 },
4708 "lines": true,
4709 "linewidth": 1,
4710 "nullPointMode": "null",
4711 "options": {
4712 "dataLinks": [ ]
4713 },
4714 "panels": [ ],
4715 "percentage": false,
4716 "pointradius": 1,
4717 "points": false,
4718 "renderer": "flot",
4719 "seriesOverrides": [ ],
4720 "spaceLength": 10,
4721 "stack": false,
4722 "steppedLine": false,
4723 "targets": [
4724 {
4725 "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
4726 "intervalFactor": 1,
4727 "legendFormat": "{{status_code}}-{{operation}}",
4728 "refId": "A"
4729 }
4730 ],
4731 "thresholds": [ ],
4732 "timeFrom": null,
4733 "timeRegions": [ ],
4734 "timeShift": null,
4735 "title": "Status By Method",
4736 "tooltip": {
4737 "shared": true,
4738 "sort": 2,
4739 "value_type": "individual"
4740 },
4741 "type": "graph",
4742 "xaxis": {
4743 "buckets": null,
4744 "mode": "time",
4745 "name": null,
4746 "show": true,
4747 "values": [ ]
4748 },
4749 "yaxes": [
4750 {
4751 "format": "short",
4752 "label": null,
4753 "logBase": 1,
4754 "max": null,
4755 "min": null,
4756 "show": true
4757 },
4758 {
4759 "format": "short",
4760 "label": null,
4761 "logBase": 1,
4762 "max": null,
4763 "min": null,
4764 "show": true
4765 }
4766 ],
4767 "yaxis": {
4768 "align": false,
4769 "alignLevel": null
4770 }
4771 }
4772 ],
4773 "targets": [ ],
4774 "title": "GCS",
4775 "type": "row"
4776 },
4777 {
4778 "collapsed": true,
4779 "datasource": null,
4780 "gridPos": {
4781 "h": 1,
4782 "w": 24,
4783 "x": 0,
4784 "y": 36
4785 },
4786 "id": 76,
4787 "panels": [
4788 {
4789 "aliasColors": { },
4790 "bars": false,
4791 "dashLength": 10,
4792 "dashes": false,
4793 "datasource": null,
4794 "fill": 1,
4795 "fillGradient": 0,
4796 "gridPos": {
4797 "h": 6,
4798 "w": 6,
4799 "x": 0,
4800 "y": 9
4801 },
4802 "id": 82,
4803 "legend": {
4804 "avg": false,
4805 "current": false,
4806 "max": false,
4807 "min": false,
4808 "show": true,
4809 "total": false,
4810 "values": false
4811 },
4812 "lines": true,
4813 "linewidth": 1,
4814 "nullPointMode": "null",
4815 "options": {
4816 "dataLinks": [ ]
4817 },
4818 "panels": [ ],
4819 "percentage": false,
4820 "pointradius": 2,
4821 "points": false,
4822 "renderer": "flot",
4823 "seriesOverrides": [ ],
4824 "spaceLength": 10,
4825 "stack": false,
4826 "steppedLine": false,
4827 "targets": [
4828 {
4829 "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
4830 "refId": "A"
4831 }
4832 ],
4833 "thresholds": [ ],
4834 "timeFrom": null,
4835 "timeRegions": [ ],
4836 "timeShift": null,
4837 "title": "Failure Rate",
4838 "tooltip": {
4839 "shared": true,
4840 "sort": 0,
4841 "value_type": "individual"
4842 },
4843 "type": "graph",
4844 "xaxis": {
4845 "buckets": null,
4846 "mode": "time",
4847 "name": null,
4848 "show": true,
4849 "values": [ ]
4850 },
4851 "yaxes": [
4852 {
4853 "format": "short",
4854 "label": null,
4855 "logBase": 1,
4856 "max": null,
4857 "min": null,
4858 "show": true
4859 },
4860 {
4861 "format": "short",
4862 "label": null,
4863 "logBase": 1,
4864 "max": null,
4865 "min": null,
4866 "show": true
4867 }
4868 ],
4869 "yaxis": {
4870 "align": false,
4871 "alignLevel": null
4872 }
4873 },
4874 {
4875 "aliasColors": { },
4876 "bars": false,
4877 "dashLength": 10,
4878 "dashes": false,
4879 "datasource": null,
4880 "fill": 1,
4881 "fillGradient": 0,
4882 "gridPos": {
4883 "h": 6,
4884 "w": 6,
4885 "x": 6,
4886 "y": 9
4887 },
4888 "id": 83,
4889 "legend": {
4890 "avg": false,
4891 "current": false,
4892 "max": false,
4893 "min": false,
4894 "show": true,
4895 "total": false,
4896 "values": false
4897 },
4898 "lines": true,
4899 "linewidth": 1,
4900 "nullPointMode": "null",
4901 "options": {
4902 "dataLinks": [ ]
4903 },
4904 "panels": [ ],
4905 "percentage": false,
4906 "pointradius": 2,
4907 "points": false,
4908 "renderer": "flot",
4909 "seriesOverrides": [ ],
4910 "spaceLength": 10,
4911 "stack": false,
4912 "steppedLine": false,
4913 "targets": [
4914 {
4915 "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
4916 "refId": "A"
4917 }
4918 ],
4919 "thresholds": [ ],
4920 "timeFrom": null,
4921 "timeRegions": [ ],
4922 "timeShift": null,
4923 "title": "Consumed Capacity Rate",
4924 "tooltip": {
4925 "shared": true,
4926 "sort": 0,
4927 "value_type": "individual"
4928 },
4929 "type": "graph",
4930 "xaxis": {
4931 "buckets": null,
4932 "mode": "time",
4933 "name": null,
4934 "show": true,
4935 "values": [ ]
4936 },
4937 "yaxes": [
4938 {
4939 "format": "short",
4940 "label": null,
4941 "logBase": 1,
4942 "max": null,
4943 "min": null,
4944 "show": true
4945 },
4946 {
4947 "format": "short",
4948 "label": null,
4949 "logBase": 1,
4950 "max": null,
4951 "min": null,
4952 "show": true
4953 }
4954 ],
4955 "yaxis": {
4956 "align": false,
4957 "alignLevel": null
4958 }
4959 },
4960 {
4961 "aliasColors": { },
4962 "bars": false,
4963 "dashLength": 10,
4964 "dashes": false,
4965 "datasource": null,
4966 "fill": 1,
4967 "fillGradient": 0,
4968 "gridPos": {
4969 "h": 6,
4970 "w": 6,
4971 "x": 12,
4972 "y": 9
4973 },
4974 "id": 84,
4975 "legend": {
4976 "avg": false,
4977 "current": false,
4978 "max": false,
4979 "min": false,
4980 "show": true,
4981 "total": false,
4982 "values": false
4983 },
4984 "lines": true,
4985 "linewidth": 1,
4986 "nullPointMode": "null",
4987 "options": {
4988 "dataLinks": [ ]
4989 },
4990 "panels": [ ],
4991 "percentage": false,
4992 "pointradius": 2,
4993 "points": false,
4994 "renderer": "flot",
4995 "seriesOverrides": [ ],
4996 "spaceLength": 10,
4997 "stack": false,
4998 "steppedLine": false,
4999 "targets": [
5000 {
5001 "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
5002 "refId": "A"
5003 }
5004 ],
5005 "thresholds": [ ],
5006 "timeFrom": null,
5007 "timeRegions": [ ],
5008 "timeShift": null,
5009 "title": "Throttled Rate",
5010 "tooltip": {
5011 "shared": true,
5012 "sort": 0,
5013 "value_type": "individual"
5014 },
5015 "type": "graph",
5016 "xaxis": {
5017 "buckets": null,
5018 "mode": "time",
5019 "name": null,
5020 "show": true,
5021 "values": [ ]
5022 },
5023 "yaxes": [
5024 {
5025 "format": "short",
5026 "label": null,
5027 "logBase": 1,
5028 "max": null,
5029 "min": null,
5030 "show": true
5031 },
5032 {
5033 "format": "short",
5034 "label": null,
5035 "logBase": 1,
5036 "max": null,
5037 "min": null,
5038 "show": true
5039 }
5040 ],
5041 "yaxis": {
5042 "align": false,
5043 "alignLevel": null
5044 }
5045 },
5046 {
5047 "aliasColors": { },
5048 "bars": false,
5049 "dashLength": 10,
5050 "dashes": false,
5051 "datasource": null,
5052 "fill": 1,
5053 "fillGradient": 0,
5054 "gridPos": {
5055 "h": 6,
5056 "w": 6,
5057 "x": 18,
5058 "y": 9
5059 },
5060 "id": 85,
5061 "legend": {
5062 "avg": false,
5063 "current": false,
5064 "max": false,
5065 "min": false,
5066 "show": true,
5067 "total": false,
5068 "values": false
5069 },
5070 "lines": true,
5071 "linewidth": 1,
5072 "nullPointMode": "null",
5073 "options": {
5074 "dataLinks": [ ]
5075 },
5076 "panels": [ ],
5077 "percentage": false,
5078 "pointradius": 2,
5079 "points": false,
5080 "renderer": "flot",
5081 "seriesOverrides": [ ],
5082 "spaceLength": 10,
5083 "stack": false,
5084 "steppedLine": false,
5085 "targets": [
5086 {
5087 "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))",
5088 "refId": "A"
5089 }
5090 ],
5091 "thresholds": [ ],
5092 "timeFrom": null,
5093 "timeRegions": [ ],
5094 "timeShift": null,
5095 "title": "Dropped Rate",
5096 "tooltip": {
5097 "shared": true,
5098 "sort": 0,
5099 "value_type": "individual"
5100 },
5101 "type": "graph",
5102 "xaxis": {
5103 "buckets": null,
5104 "mode": "time",
5105 "name": null,
5106 "show": true,
5107 "values": [ ]
5108 },
5109 "yaxes": [
5110 {
5111 "format": "short",
5112 "label": null,
5113 "logBase": 1,
5114 "max": null,
5115 "min": null,
5116 "show": true
5117 },
5118 {
5119 "format": "short",
5120 "label": null,
5121 "logBase": 1,
5122 "max": null,
5123 "min": null,
5124 "show": true
5125 }
5126 ],
5127 "yaxis": {
5128 "align": false,
5129 "alignLevel": null
5130 }
5131 },
5132 {
5133 "aliasColors": { },
5134 "bars": false,
5135 "dashLength": 10,
5136 "dashes": false,
5137 "datasource": null,
5138 "fill": 1,
5139 "fillGradient": 0,
5140 "gridPos": {
5141 "h": 6,
5142 "w": 6,
5143 "x": 0,
5144 "y": 15
5145 },
5146 "id": 86,
5147 "legend": {
5148 "avg": false,
5149 "current": false,
5150 "max": false,
5151 "min": false,
5152 "show": true,
5153 "total": false,
5154 "values": false
5155 },
5156 "lines": true,
5157 "linewidth": 1,
5158 "nullPointMode": "null",
5159 "options": {
5160 "dataLinks": [ ]
5161 },
5162 "panels": [ ],
5163 "percentage": false,
5164 "pointradius": 2,
5165 "points": false,
5166 "renderer": "flot",
5167 "seriesOverrides": [ ],
5168 "spaceLength": 10,
5169 "stack": false,
5170 "steppedLine": false,
5171 "targets": [
5172 {
5173 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
5174 "legendFormat": ".99",
5175 "refId": "A"
5176 },
5177 {
5178 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
5179 "legendFormat": ".9",
5180 "refId": "B"
5181 },
5182 {
5183 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))",
5184 "legendFormat": ".5",
5185 "refId": "C"
5186 }
5187 ],
5188 "thresholds": [ ],
5189 "timeFrom": null,
5190 "timeRegions": [ ],
5191 "timeShift": null,
5192 "title": "Query Pages",
5193 "tooltip": {
5194 "shared": true,
5195 "sort": 0,
5196 "value_type": "individual"
5197 },
5198 "type": "graph",
5199 "xaxis": {
5200 "buckets": null,
5201 "mode": "time",
5202 "name": null,
5203 "show": true,
5204 "values": [ ]
5205 },
5206 "yaxes": [
5207 {
5208 "format": "short",
5209 "label": null,
5210 "logBase": 1,
5211 "max": null,
5212 "min": null,
5213 "show": true
5214 },
5215 {
5216 "format": "short",
5217 "label": null,
5218 "logBase": 1,
5219 "max": null,
5220 "min": null,
5221 "show": true
5222 }
5223 ],
5224 "yaxis": {
5225 "align": false,
5226 "alignLevel": null
5227 }
5228 },
5229 {
5230 "aliasColors": { },
5231 "bars": false,
5232 "dashLength": 10,
5233 "dashes": false,
5234 "datasource": "$datasource",
5235 "fill": 1,
5236 "fillGradient": 0,
5237 "gridPos": {
5238 "h": 6,
5239 "w": 9,
5240 "x": 6,
5241 "y": 15
5242 },
5243 "id": 87,
5244 "interval": "",
5245 "legend": {
5246 "alignAsTable": true,
5247 "avg": false,
5248 "current": false,
5249 "max": false,
5250 "min": false,
5251 "rightSide": true,
5252 "show": true,
5253 "total": false,
5254 "values": false
5255 },
5256 "lines": true,
5257 "linewidth": 1,
5258 "nullPointMode": "null",
5259 "options": {
5260 "dataLinks": [ ]
5261 },
5262 "panels": [ ],
5263 "percentage": false,
5264 "pointradius": 1,
5265 "points": false,
5266 "renderer": "flot",
5267 "seriesOverrides": [ ],
5268 "spaceLength": 10,
5269 "stack": false,
5270 "steppedLine": false,
5271 "targets": [
5272 {
5273 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5274 "intervalFactor": 1,
5275 "legendFormat": ".99-{{operation}}",
5276 "refId": "A"
5277 },
5278 {
5279 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5280 "hide": false,
5281 "legendFormat": ".9-{{operation}}",
5282 "refId": "B"
5283 },
5284 {
5285 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5286 "hide": false,
5287 "legendFormat": ".5-{{operation}}",
5288 "refId": "C"
5289 }
5290 ],
5291 "thresholds": [ ],
5292 "timeFrom": null,
5293 "timeRegions": [ ],
5294 "timeShift": null,
5295 "title": "Latency By Operation",
5296 "tooltip": {
5297 "shared": true,
5298 "sort": 2,
5299 "value_type": "individual"
5300 },
5301 "type": "graph",
5302 "xaxis": {
5303 "buckets": null,
5304 "mode": "time",
5305 "name": null,
5306 "show": true,
5307 "values": [ ]
5308 },
5309 "yaxes": [
5310 {
5311 "format": "short",
5312 "label": null,
5313 "logBase": 1,
5314 "max": null,
5315 "min": null,
5316 "show": true
5317 },
5318 {
5319 "format": "short",
5320 "label": null,
5321 "logBase": 1,
5322 "max": null,
5323 "min": null,
5324 "show": true
5325 }
5326 ],
5327 "yaxis": {
5328 "align": false,
5329 "alignLevel": null
5330 }
5331 },
5332 {
5333 "aliasColors": { },
5334 "bars": false,
5335 "dashLength": 10,
5336 "dashes": false,
5337 "datasource": "$datasource",
5338 "fill": 1,
5339 "fillGradient": 0,
5340 "gridPos": {
5341 "h": 6,
5342 "w": 9,
5343 "x": 15,
5344 "y": 15
5345 },
5346 "id": 88,
5347 "interval": "",
5348 "legend": {
5349 "alignAsTable": true,
5350 "avg": false,
5351 "current": false,
5352 "max": false,
5353 "min": false,
5354 "rightSide": true,
5355 "show": true,
5356 "total": false,
5357 "values": false
5358 },
5359 "lines": true,
5360 "linewidth": 1,
5361 "nullPointMode": "null",
5362 "options": {
5363 "dataLinks": [ ]
5364 },
5365 "panels": [ ],
5366 "percentage": false,
5367 "pointradius": 1,
5368 "points": false,
5369 "renderer": "flot",
5370 "seriesOverrides": [ ],
5371 "spaceLength": 10,
5372 "stack": false,
5373 "steppedLine": false,
5374 "targets": [
5375 {
5376 "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
5377 "intervalFactor": 1,
5378 "legendFormat": "{{status_code}}-{{operation}}",
5379 "refId": "A"
5380 }
5381 ],
5382 "thresholds": [ ],
5383 "timeFrom": null,
5384 "timeRegions": [ ],
5385 "timeShift": null,
5386 "title": "Status By Method",
5387 "tooltip": {
5388 "shared": true,
5389 "sort": 2,
5390 "value_type": "individual"
5391 },
5392 "type": "graph",
5393 "xaxis": {
5394 "buckets": null,
5395 "mode": "time",
5396 "name": null,
5397 "show": true,
5398 "values": [ ]
5399 },
5400 "yaxes": [
5401 {
5402 "format": "short",
5403 "label": null,
5404 "logBase": 1,
5405 "max": null,
5406 "min": null,
5407 "show": true
5408 },
5409 {
5410 "format": "short",
5411 "label": null,
5412 "logBase": 1,
5413 "max": null,
5414 "min": null,
5415 "show": true
5416 }
5417 ],
5418 "yaxis": {
5419 "align": false,
5420 "alignLevel": null
5421 }
5422 }
5423 ],
5424 "targets": [ ],
5425 "title": "Dynamo",
5426 "type": "row"
5427 },
5428 {
5429 "collapsed": true,
5430 "datasource": null,
5431 "gridPos": {
5432 "h": 1,
5433 "w": 24,
5434 "x": 0,
5435 "y": 37
5436 },
5437 "id": 78,
5438 "panels": [
5439 {
5440 "aliasColors": { },
5441 "bars": false,
5442 "dashLength": 10,
5443 "dashes": false,
5444 "datasource": "$datasource",
5445 "fill": 1,
5446 "fillGradient": 0,
5447 "gridPos": {
5448 "h": 8,
5449 "w": 24,
5450 "x": 0,
5451 "y": 10
5452 },
5453 "id": 79,
5454 "interval": "",
5455 "legend": {
5456 "alignAsTable": true,
5457 "avg": false,
5458 "current": false,
5459 "max": false,
5460 "min": false,
5461 "rightSide": true,
5462 "show": true,
5463 "total": false,
5464 "values": false
5465 },
5466 "lines": true,
5467 "linewidth": 1,
5468 "nullPointMode": "null",
5469 "options": {
5470 "dataLinks": [ ]
5471 },
5472 "panels": [ ],
5473 "percentage": false,
5474 "pointradius": 1,
5475 "points": false,
5476 "renderer": "flot",
5477 "seriesOverrides": [ ],
5478 "spaceLength": 10,
5479 "stack": false,
5480 "steppedLine": false,
5481 "targets": [
5482 {
5483 "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5484 "intervalFactor": 1,
5485 "legendFormat": ".99-{{operation}}",
5486 "refId": "A"
5487 },
5488 {
5489 "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5490 "hide": false,
5491 "legendFormat": ".9-{{operation}}",
5492 "refId": "B"
5493 },
5494 {
5495 "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5496 "hide": false,
5497 "legendFormat": ".5-{{operation}}",
5498 "refId": "C"
5499 }
5500 ],
5501 "thresholds": [ ],
5502 "timeFrom": null,
5503 "timeRegions": [ ],
5504 "timeShift": null,
5505 "title": "Latency By Operation",
5506 "tooltip": {
5507 "shared": true,
5508 "sort": 2,
5509 "value_type": "individual"
5510 },
5511 "type": "graph",
5512 "xaxis": {
5513 "buckets": null,
5514 "mode": "time",
5515 "name": null,
5516 "show": true,
5517 "values": [ ]
5518 },
5519 "yaxes": [
5520 {
5521 "format": "short",
5522 "label": null,
5523 "logBase": 1,
5524 "max": null,
5525 "min": null,
5526 "show": true
5527 },
5528 {
5529 "format": "short",
5530 "label": null,
5531 "logBase": 1,
5532 "max": null,
5533 "min": null,
5534 "show": true
5535 }
5536 ],
5537 "yaxis": {
5538 "align": false,
5539 "alignLevel": null
5540 }
5541 },
5542 {
5543 "aliasColors": { },
5544 "bars": false,
5545 "dashLength": 10,
5546 "dashes": false,
5547 "datasource": "$datasource",
5548 "fill": 1,
5549 "fillGradient": 0,
5550 "gridPos": {
5551 "h": 8,
5552 "w": 24,
5553 "x": 0,
5554 "y": 18
5555 },
5556 "id": 80,
5557 "interval": "",
5558 "legend": {
5559 "alignAsTable": true,
5560 "avg": false,
5561 "current": false,
5562 "max": false,
5563 "min": false,
5564 "rightSide": true,
5565 "show": true,
5566 "total": false,
5567 "values": false
5568 },
5569 "lines": true,
5570 "linewidth": 1,
5571 "nullPointMode": "null",
5572 "options": {
5573 "dataLinks": [ ]
5574 },
5575 "panels": [ ],
5576 "percentage": false,
5577 "pointradius": 1,
5578 "points": false,
5579 "renderer": "flot",
5580 "seriesOverrides": [ ],
5581 "spaceLength": 10,
5582 "stack": false,
5583 "steppedLine": false,
5584 "targets": [
5585 {
5586 "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
5587 "intervalFactor": 1,
5588 "legendFormat": "{{status_code}}-{{operation}}",
5589 "refId": "A"
5590 }
5591 ],
5592 "thresholds": [ ],
5593 "timeFrom": null,
5594 "timeRegions": [ ],
5595 "timeShift": null,
5596 "title": "Status By Method",
5597 "tooltip": {
5598 "shared": true,
5599 "sort": 2,
5600 "value_type": "individual"
5601 },
5602 "type": "graph",
5603 "xaxis": {
5604 "buckets": null,
5605 "mode": "time",
5606 "name": null,
5607 "show": true,
5608 "values": [ ]
5609 },
5610 "yaxes": [
5611 {
5612 "format": "short",
5613 "label": null,
5614 "logBase": 1,
5615 "max": null,
5616 "min": null,
5617 "show": true
5618 },
5619 {
5620 "format": "short",
5621 "label": null,
5622 "logBase": 1,
5623 "max": null,
5624 "min": null,
5625 "show": true
5626 }
5627 ],
5628 "yaxis": {
5629 "align": false,
5630 "alignLevel": null
5631 }
5632 }
5633 ],
5634 "targets": [ ],
5635 "title": "S3",
5636 "type": "row"
5637 },
5638 {
5639 "collapsed": true,
5640 "datasource": null,
5641 "gridPos": {
5642 "h": 1,
5643 "w": 24,
5644 "x": 0,
5645 "y": 37
5646 },
5647 "id": 78,
5648 "panels": [
5649 {
5650 "aliasColors": { },
5651 "bars": false,
5652 "dashLength": 10,
5653 "dashes": false,
5654 "datasource": "$datasource",
5655 "fill": 1,
5656 "fillGradient": 0,
5657 "gridPos": {
5658 "h": 8,
5659 "w": 24,
5660 "x": 0,
5661 "y": 10
5662 },
5663 "id": 79,
5664 "interval": "",
5665 "legend": {
5666 "alignAsTable": true,
5667 "avg": false,
5668 "current": false,
5669 "max": false,
5670 "min": false,
5671 "rightSide": true,
5672 "show": true,
5673 "total": false,
5674 "values": false
5675 },
5676 "lines": true,
5677 "linewidth": 1,
5678 "nullPointMode": "null",
5679 "options": {
5680 "dataLinks": [ ]
5681 },
5682 "panels": [ ],
5683 "percentage": false,
5684 "pointradius": 1,
5685 "points": false,
5686 "renderer": "flot",
5687 "seriesOverrides": [ ],
5688 "spaceLength": 10,
5689 "stack": false,
5690 "steppedLine": false,
5691 "targets": [
5692 {
5693 "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5694 "intervalFactor": 1,
5695 "legendFormat": ".99-{{operation}}",
5696 "refId": "A"
5697 },
5698 {
5699 "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5700 "hide": false,
5701 "legendFormat": ".9-{{operation}}",
5702 "refId": "B"
5703 },
5704 {
5705 "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5706 "hide": false,
5707 "legendFormat": ".5-{{operation}}",
5708 "refId": "C"
5709 }
5710 ],
5711 "thresholds": [ ],
5712 "timeFrom": null,
5713 "timeRegions": [ ],
5714 "timeShift": null,
5715 "title": "Latency By Operation",
5716 "tooltip": {
5717 "shared": true,
5718 "sort": 2,
5719 "value_type": "individual"
5720 },
5721 "type": "graph",
5722 "xaxis": {
5723 "buckets": null,
5724 "mode": "time",
5725 "name": null,
5726 "show": true,
5727 "values": [ ]
5728 },
5729 "yaxes": [
5730 {
5731 "format": "short",
5732 "label": null,
5733 "logBase": 1,
5734 "max": null,
5735 "min": null,
5736 "show": true
5737 },
5738 {
5739 "format": "short",
5740 "label": null,
5741 "logBase": 1,
5742 "max": null,
5743 "min": null,
5744 "show": true
5745 }
5746 ],
5747 "yaxis": {
5748 "align": false,
5749 "alignLevel": null
5750 }
5751 },
5752 {
5753 "aliasColors": { },
5754 "bars": false,
5755 "dashLength": 10,
5756 "dashes": false,
5757 "datasource": "$datasource",
5758 "fill": 1,
5759 "fillGradient": 0,
5760 "gridPos": {
5761 "h": 8,
5762 "w": 24,
5763 "x": 0,
5764 "y": 18
5765 },
5766 "id": 80,
5767 "interval": "",
5768 "legend": {
5769 "alignAsTable": true,
5770 "avg": false,
5771 "current": false,
5772 "max": false,
5773 "min": false,
5774 "rightSide": true,
5775 "show": true,
5776 "total": false,
5777 "values": false
5778 },
5779 "lines": true,
5780 "linewidth": 1,
5781 "nullPointMode": "null",
5782 "options": {
5783 "dataLinks": [ ]
5784 },
5785 "panels": [ ],
5786 "percentage": false,
5787 "pointradius": 1,
5788 "points": false,
5789 "renderer": "flot",
5790 "seriesOverrides": [ ],
5791 "spaceLength": 10,
5792 "stack": false,
5793 "steppedLine": false,
5794 "targets": [
5795 {
5796 "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
5797 "intervalFactor": 1,
5798 "legendFormat": "{{status_code}}-{{operation}}",
5799 "refId": "A"
5800 }
5801 ],
5802 "thresholds": [ ],
5803 "timeFrom": null,
5804 "timeRegions": [ ],
5805 "timeShift": null,
5806 "title": "Status By Method",
5807 "tooltip": {
5808 "shared": true,
5809 "sort": 2,
5810 "value_type": "individual"
5811 },
5812 "type": "graph",
5813 "xaxis": {
5814 "buckets": null,
5815 "mode": "time",
5816 "name": null,
5817 "show": true,
5818 "values": [ ]
5819 },
5820 "yaxes": [
5821 {
5822 "format": "short",
5823 "label": null,
5824 "logBase": 1,
5825 "max": null,
5826 "min": null,
5827 "show": true
5828 },
5829 {
5830 "format": "short",
5831 "label": null,
5832 "logBase": 1,
5833 "max": null,
5834 "min": null,
5835 "show": true
5836 }
5837 ],
5838 "yaxis": {
5839 "align": false,
5840 "alignLevel": null
5841 }
5842 }
5843 ],
5844 "targets": [ ],
5845 "title": "Azure Blob",
5846 "type": "row"
5847 },
5848 {
5849 "collapsed": true,
5850 "datasource": null,
5851 "gridPos": {
5852 "h": 1,
5853 "w": 24,
5854 "x": 0,
5855 "y": 37
5856 },
5857 "id": 114,
5858 "panels": [
5859 {
5860 "aliasColors": { },
5861 "bars": false,
5862 "dashLength": 10,
5863 "dashes": false,
5864 "datasource": "$datasource",
5865 "fill": 1,
5866 "fillGradient": 0,
5867 "gridPos": {
5868 "h": 8,
5869 "w": 24,
5870 "x": 0,
5871 "y": 10
5872 },
5873 "id": 115,
5874 "interval": "",
5875 "legend": {
5876 "alignAsTable": true,
5877 "avg": false,
5878 "current": false,
5879 "max": false,
5880 "min": false,
5881 "rightSide": true,
5882 "show": true,
5883 "total": false,
5884 "values": false
5885 },
5886 "lines": true,
5887 "linewidth": 1,
5888 "nullPointMode": "null",
5889 "options": {
5890 "dataLinks": [ ]
5891 },
5892 "panels": [ ],
5893 "percentage": false,
5894 "pointradius": 1,
5895 "points": false,
5896 "renderer": "flot",
5897 "seriesOverrides": [ ],
5898 "spaceLength": 10,
5899 "stack": false,
5900 "steppedLine": false,
5901 "targets": [
5902 {
5903 "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5904 "intervalFactor": 1,
5905 "legendFormat": ".99-{{operation}}",
5906 "refId": "A"
5907 },
5908 {
5909 "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5910 "hide": false,
5911 "legendFormat": ".9-{{operation}}",
5912 "refId": "B"
5913 },
5914 {
5915 "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))",
5916 "hide": false,
5917 "legendFormat": ".5-{{operation}}",
5918 "refId": "C"
5919 }
5920 ],
5921 "thresholds": [ ],
5922 "timeFrom": null,
5923 "timeRegions": [ ],
5924 "timeShift": null,
5925 "title": "Latency By Operation",
5926 "tooltip": {
5927 "shared": true,
5928 "sort": 2,
5929 "value_type": "individual"
5930 },
5931 "type": "graph",
5932 "xaxis": {
5933 "buckets": null,
5934 "mode": "time",
5935 "name": null,
5936 "show": true,
5937 "values": [ ]
5938 },
5939 "yaxes": [
5940 {
5941 "format": "short",
5942 "label": null,
5943 "logBase": 1,
5944 "max": null,
5945 "min": null,
5946 "show": true
5947 },
5948 {
5949 "format": "short",
5950 "label": null,
5951 "logBase": 1,
5952 "max": null,
5953 "min": null,
5954 "show": true
5955 }
5956 ],
5957 "yaxis": {
5958 "align": false,
5959 "alignLevel": null
5960 }
5961 },
5962 {
5963 "aliasColors": { },
5964 "bars": false,
5965 "dashLength": 10,
5966 "dashes": false,
5967 "datasource": "$datasource",
5968 "fill": 1,
5969 "fillGradient": 0,
5970 "gridPos": {
5971 "h": 8,
5972 "w": 24,
5973 "x": 0,
5974 "y": 18
5975 },
5976 "id": 116,
5977 "interval": "",
5978 "legend": {
5979 "alignAsTable": true,
5980 "avg": false,
5981 "current": false,
5982 "max": false,
5983 "min": false,
5984 "rightSide": true,
5985 "show": true,
5986 "total": false,
5987 "values": false
5988 },
5989 "lines": true,
5990 "linewidth": 1,
5991 "nullPointMode": "null",
5992 "options": {
5993 "dataLinks": [ ]
5994 },
5995 "panels": [ ],
5996 "percentage": false,
5997 "pointradius": 1,
5998 "points": false,
5999 "renderer": "flot",
6000 "seriesOverrides": [ ],
6001 "spaceLength": 10,
6002 "stack": false,
6003 "steppedLine": false,
6004 "targets": [
6005 {
6006 "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)",
6007 "intervalFactor": 1,
6008 "legendFormat": "{{status_code}}-{{operation}}",
6009 "refId": "A"
6010 }
6011 ],
6012 "thresholds": [ ],
6013 "timeFrom": null,
6014 "timeRegions": [ ],
6015 "timeShift": null,
6016 "title": "Status By Method",
6017 "tooltip": {
6018 "shared": true,
6019 "sort": 2,
6020 "value_type": "individual"
6021 },
6022 "type": "graph",
6023 "xaxis": {
6024 "buckets": null,
6025 "mode": "time",
6026 "name": null,
6027 "show": true,
6028 "values": [ ]
6029 },
6030 "yaxes": [
6031 {
6032 "format": "short",
6033 "label": null,
6034 "logBase": 1,
6035 "max": null,
6036 "min": null,
6037 "show": true
6038 },
6039 {
6040 "format": "short",
6041 "label": null,
6042 "logBase": 1,
6043 "max": null,
6044 "min": null,
6045 "show": true
6046 }
6047 ],
6048 "yaxis": {
6049 "align": false,
6050 "alignLevel": null
6051 }
6052 }
6053 ],
6054 "targets": [ ],
6055 "title": "BoltDB Shipper",
6056 "type": "row"
6057 }
6058 ],
6059 "refresh": "10s",
6060 "rows": [ ],
6061 "schemaVersion": 14,
6062 "style": "dark",
6063 "tags": [
6064 "loki"
6065 ],
6066 "templating": {
6067 "list": [
6068 {
6069 "current": {
6070 "text": "default",
6071 "value": "default"
6072 },
6073 "hide": 0,
6074 "label": "Data Source",
6075 "name": "datasource",
6076 "options": [ ],
6077 "query": "prometheus",
6078 "refresh": 1,
6079 "regex": "",
6080 "type": "datasource"
6081 },
6082 {
6083 "hide": 0,
6084 "label": null,
6085 "name": "logs",
6086 "options": [ ],
6087 "query": "loki",
6088 "refresh": 1,
6089 "regex": "",
6090 "type": "datasource"
6091 },
6092 {
6093 "allValue": null,
6094 "current": {
6095 "text": "prod",
6096 "value": "prod"
6097 },
6098 "datasource": "$datasource",
6099 "hide": 0,
6100 "includeAll": false,
6101 "label": "cluster",
6102 "multi": false,
6103 "name": "cluster",
6104 "options": [ ],
6105 "query": "label_values(loki_build_info, cluster)",
6106 "refresh": 1,
6107 "regex": "",
6108 "sort": 2,
6109 "tagValuesQuery": "",
6110 "tags": [ ],
6111 "tagsQuery": "",
6112 "type": "query",
6113 "useTags": false
6114 },
6115 {
6116 "allValue": null,
6117 "current": {
6118 "text": "prod",
6119 "value": "prod"
6120 },
6121 "datasource": "$datasource",
6122 "hide": 0,
6123 "includeAll": false,
6124 "label": "namespace",
6125 "multi": false,
6126 "name": "namespace",
6127 "options": [ ],
6128 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
6129 "refresh": 1,
6130 "regex": "",
6131 "sort": 2,
6132 "tagValuesQuery": "",
6133 "tags": [ ],
6134 "tagsQuery": "",
6135 "type": "query",
6136 "useTags": false
6137 }
6138 ]
6139 },
6140 "time": {
6141 "from": "now-1h",
6142 "to": "now"
6143 },
6144 "timepicker": {
6145 "refresh_intervals": [
6146 "5s",
6147 "10s",
6148 "30s",
6149 "1m",
6150 "5m",
6151 "15m",
6152 "30m",
6153 "1h",
6154 "2h",
6155 "1d"
6156 ],
6157 "time_options": [
6158 "5m",
6159 "15m",
6160 "1h",
6161 "6h",
6162 "12h",
6163 "24h",
6164 "2d",
6165 "7d",
6166 "30d"
6167 ]
6168 },
6169 "timezone": "utc",
6170 "title": "Loki / Operational",
6171 "uid": "operational",
6172 "version": 0
6173 }