feat(monitoring): refactor (#555)

* chore(kube-prometheus-stack): update to latest

* fix: KubeJobFailed should be SEV-3

* chore: refactor softnet alerts

* chore(monitoring): migrate to using jsonnet

* chore: refactor alerts

* chore: major monitoring refactor

* fix: solve alerts

* fix: apiserver selector

* more cleanups

* switch from SEV- to P

* fix: improve port binding alerts

* fix admin state alert for neutron

* map some more alerts

* drop uuid

* Revert "drop uuid"

This reverts commit ad0f05d0e7564759e8259c2cc53c2e2f5c73e1b8.

* fix: drop recording rules

* switch alertmanager to jsonnet

* fix: idempotence for monitoring

* chore: fix linters
diff --git a/roles/defaults/vars/main.yml b/roles/defaults/vars/main.yml
index e4ca646..9d5f503 100644
--- a/roles/defaults/vars/main.yml
+++ b/roles/defaults/vars/main.yml
@@ -13,7 +13,7 @@
 # under the License.
 
 _atmosphere_images:
-  alertmanager: quay.io/prometheus/alertmanager:v0.24.0
+  alertmanager: quay.io/prometheus/alertmanager:v0.26.0
   barbican_api: quay.io/vexxhost/barbican@sha256:fde302ee731cca6019feaf87400f5a377c3e38f459bc88d4c7677f2967e0939b # image-source: quay.io/vexxhost/barbican:zed
   barbican_db_sync: quay.io/vexxhost/barbican@sha256:fde302ee731cca6019feaf87400f5a377c3e38f459bc88d4c7677f2967e0939b # image-source: quay.io/vexxhost/barbican:zed
   bootstrap: quay.io/vexxhost/heat@sha256:755225f9a63c0968f1ceeda3a2f06c66dd8d247ff00308f549e66496aa8f59d0 # image-source: quay.io/vexxhost/heat:zed
@@ -58,8 +58,8 @@
   glance_metadefs_load: quay.io/vexxhost/glance@sha256:32aaf33e83f7285e2ad04cb8e692068e072d9d852b6b2625742995724ec77508 # image-source: quay.io/vexxhost/glance:zed
   glance_registry: quay.io/vexxhost/glance@sha256:32aaf33e83f7285e2ad04cb8e692068e072d9d852b6b2625742995724ec77508 # image-source: quay.io/vexxhost/glance:zed
   glance_storage_init: quay.io/vexxhost/glance@sha256:32aaf33e83f7285e2ad04cb8e692068e072d9d852b6b2625742995724ec77508 # image-source: quay.io/vexxhost/glance:zed
-  grafana_sidecar: quay.io/kiwigrid/k8s-sidecar:1.19.2
-  grafana: docker.io/grafana/grafana:9.2.3
+  grafana_sidecar: quay.io/kiwigrid/k8s-sidecar:1.24.6
+  grafana: docker.io/grafana/grafana:10.1.0
   haproxy: docker.io/library/haproxy:2.5
   heat_api: quay.io/vexxhost/heat@sha256:755225f9a63c0968f1ceeda3a2f06c66dd8d247ff00308f549e66496aa8f59d0 # image-source: quay.io/vexxhost/heat:zed
   heat_cfn: quay.io/vexxhost/heat@sha256:755225f9a63c0968f1ceeda3a2f06c66dd8d247ff00308f549e66496aa8f59d0 # image-source: quay.io/vexxhost/heat:zed
@@ -91,7 +91,7 @@
   kube_etcd: registry.k8s.io/etcd:3.5.6-0
   kube_proxy: registry.k8s.io/kube-proxy:v1.22.17
   kube_scheduler: registry.k8s.io/kube-scheduler:v1.22.17
-  kube_state_metrics: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.6.0
+  kube_state_metrics: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.9.2
   kubectl: docker.io/bitnami/kubectl@sha256:bd420268ae3424b3ab3174e26b895fd8dc464589a8cd62654b9aa739d00ff280 # image-source: docker.io/bitnami/kubectl:latest
   libvirt: quay.io/vexxhost/libvirtd@sha256:d400204e0332dc815827e5902038a1c672446c58633ba97ede9e20f8ae9a2349 # image-source: quay.io/vexxhost/libvirtd:yoga-focal
   local_path_provisioner_helper: docker.io/library/busybox:1.36.0
@@ -161,17 +161,17 @@
   percona_xtradb_cluster: docker.io/percona/percona-xtradb-cluster:5.7.39-31.61
   placement_db_sync: quay.io/vexxhost/placement@sha256:ef2f95bdc3c88504b3a2297bf314728788126008f9bafba337e53d48a7dd19c3 # image-source: quay.io/vexxhost/placement:zed
   placement: quay.io/vexxhost/placement@sha256:ef2f95bdc3c88504b3a2297bf314728788126008f9bafba337e53d48a7dd19c3 # image-source: quay.io/vexxhost/placement:zed
-  prometheus_config_reloader: quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
+  prometheus_config_reloader: quay.io/prometheus-operator/prometheus-config-reloader:v0.67.1
   prometheus_ethtool_exporter: quay.io/vexxhost/ethtool-exporter:5f05120a743a71adcbceb9f8ee1d43ecc7c4183a
   prometheus_ipmi_exporter: us-docker.pkg.dev/vexxhost-infra/openstack/ipmi-exporter:1.4.0
   prometheus_memcached_exporter: quay.io/prometheus/memcached-exporter:v0.10.0
   prometheus_mysqld_exporter: quay.io/prometheus/mysqld-exporter:v0.14.0
-  prometheus_node_exporter: quay.io/prometheus/node-exporter:v1.3.1
+  prometheus_node_exporter: quay.io/prometheus/node-exporter:v1.6.1
   prometheus_openstack_exporter: ghcr.io/openstack-exporter/openstack-exporter:1.6.0
-  prometheus_operator_kube_webhook_certgen: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.3.0
-  prometheus_operator: quay.io/prometheus-operator/prometheus-operator:v0.60.1
+  prometheus_operator_kube_webhook_certgen: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6
+  prometheus_operator: quay.io/prometheus-operator/prometheus-operator:v0.67.1
   prometheus_pushgateway: docker.io/prom/pushgateway:v1.4.2
-  prometheus: quay.io/prometheus/prometheus:v2.39.1
+  prometheus: quay.io/prometheus/prometheus:v2.46.0
   rabbit_init: docker.io/library/rabbitmq:3.10.2-management
   rabbitmq_cluster_operator: docker.io/rabbitmqoperator/cluster-operator:1.13.1
   rabbitmq_credential_updater: docker.io/rabbitmqoperator/default-user-credential-updater:1.0.2