feat: add logging via vector + loki
diff --git a/charts/loki/src/alerts.yaml b/charts/loki/src/alerts.yaml
new file mode 100644
index 0000000..b1e9687
--- /dev/null
+++ b/charts/loki/src/alerts.yaml
@@ -0,0 +1,52 @@
+groups:
+- name: loki_alerts
+ rules:
+ - alert: LokiRequestErrors
+ annotations:
+ message: |
+ {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
+ expr: |
+ 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (namespace, job, route)
+ /
+ sum(rate(loki_request_duration_seconds_count[2m])) by (namespace, job, route)
+ > 10
+ for: 15m
+ labels:
+ severity: critical
+ - alert: LokiRequestPanics
+ annotations:
+ message: |
+ {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics.
+ expr: |
+ sum(increase(loki_panic_total[10m])) by (namespace, job) > 0
+ labels:
+ severity: critical
+ - alert: LokiRequestLatency
+ annotations:
+ message: |
+ {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
+ expr: |
+ namespace_job_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*"} > 1
+ for: 15m
+ labels:
+ severity: critical
+ - alert: LokiTooManyCompactorsRunning
+ annotations:
+ message: |
+ {{ $labels.cluster }} {{ $labels.namespace }} has had {{ printf "%.0f" $value }} compactors running for more than 5m. Only one compactor should run at a time.
+ expr: |
+ sum(loki_boltdb_shipper_compactor_running) by (namespace, cluster) > 1
+ for: 5m
+ labels:
+ severity: warning
+- name: 'loki_canaries_alerts'
+ rules:
+ - alert: 'LokiCanaryLatency'
+ annotations:
+ message: |
+ {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
+ expr: |
+ histogram_quantile(0.99, sum(rate(loki_canary_response_latency_seconds_bucket[5m])) by (le, namespace, job)) > 5
+ for: '15m'
+ labels:
+ severity: 'warning'