[stable/2023.1] Add build request failure monitoring [ATMOSPHERE-249] (#1432)
(cherry picked from commit 08ff881)
diff --git a/roles/defaults/vars/main.yml b/roles/defaults/vars/main.yml
index 959fff7..ca0658d 100644
--- a/roles/defaults/vars/main.yml
+++ b/roles/defaults/vars/main.yml
@@ -187,7 +187,7 @@
prometheus_memcached_exporter: quay.io/prometheus/memcached-exporter:v0.14.3
prometheus_mysqld_exporter: quay.io/prometheus/mysqld-exporter:v0.15.1
prometheus_node_exporter: quay.io/prometheus/node-exporter:v1.7.0
- prometheus_openstack_database_exporter: ghcr.io/vexxhost/openstack-database-exporter:v0.3.0
+ prometheus_openstack_database_exporter: ghcr.io/vexxhost/openstack-database-exporter:v0.4.2
prometheus_openstack_exporter: ghcr.io/openstack-exporter/openstack-exporter:1.7.0
prometheus_operator_kube_webhook_certgen: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20221220-controller-v1.5.1-58-g787ea74b6
prometheus_operator: quay.io/prometheus-operator/prometheus-operator:v0.73.0
diff --git a/roles/kube_prometheus_stack/files/jsonnet/openstack.libsonnet b/roles/kube_prometheus_stack/files/jsonnet/openstack.libsonnet
index e77077e..0824e07 100644
--- a/roles/kube_prometheus_stack/files/jsonnet/openstack.libsonnet
+++ b/roles/kube_prometheus_stack/files/jsonnet/openstack.libsonnet
@@ -1,4 +1,18 @@
{
+ prometheusRules+:: {
+ groups: [
+ {
+ name: 'recording',
+ rules:
+ [
+ {
+ record: 'nova:build_requests:sum',
+ expr: 'sum(openstack_nova_api_build_request)',
+ },
+ ],
+ },
+ ],
+ },
prometheusAlerts+: {
groups+: [
{
@@ -254,6 +268,35 @@
],
},
{
+ name: 'nova-build-requests',
+ rules: [
+ {
+ alert: 'NovaStuckBuildRequest',
+ annotations: {
+ summary: 'Nova build request stuck in queue for more than 1 hour',
+ description: 'Instance ID {{ $labels.instance_uuid }} (project: {{ $labels.project_id }}) has been stuck in build request state for more than 1 hour.',
+ },
+ expr: 'openstack_nova_api_build_request > 0',
+ 'for': '1h',
+ labels: {
+ severity: 'P4',
+ },
+ },
+ {
+ alert: 'NovaStuckBuildRequestIncreasing',
+ annotations: {
+ summary: 'Nova build request is increasing',
+ description: 'Build request count rate is increasing across the cluster.',
+ },
+ expr: 'rate(nova:build_requests:sum[5m]) > 0',
+ 'for': '15m',
+ labels: {
+ severity: 'P3',
+ },
+ },
+ ]
+ },
+ {
name: 'octavia',
rules:
[
diff --git a/roles/openstack_exporter/tasks/main.yml b/roles/openstack_exporter/tasks/main.yml
index 115445d..d6e067e 100644
--- a/roles/openstack_exporter/tasks/main.yml
+++ b/roles/openstack_exporter/tasks/main.yml
@@ -159,9 +159,10 @@
labels:
application: openstack-database-exporter
stringData:
- NEUTRON_DSN: "{{ _neutron_db_user.resources.0.data.DB_CONNECTION | b64decode | regex_replace('^.*//', '') | regex_replace('@(.*)/', '@tcp(\\1)/') }}"
- NOVA_DSN: "{{ _nova_db_user.resources.0.data.DB_CONNECTION | b64decode | regex_replace('^.*//', '') | regex_replace('@(.*)/', '@tcp(\\1)/') }}"
- OCTAVIA_DSN: "{{ _octavia_db_user.resources.0.data.DB_CONNECTION | b64decode | regex_replace('^.*//', '') | regex_replace('@(.*)/', '@tcp(\\1)/') }}"
+ NEUTRON_DSN: "{{ _neutron_db_user.resources.0.data.DB_CONNECTION | b64decode | regex_replace('^.*//', '') | regex_replace('@(.*)/', '@tcp(\\1)/') }}"
+ NOVA_DSN: "{{ _nova_db_user.resources.0.data.DB_CONNECTION | b64decode | regex_replace('^.*//', '') | regex_replace('@(.*)/', '@tcp(\\1)/') }}"
+ NOVA_API_DSN: "{{ _nova_db_user.resources.0.data.DB_CONNECTION | b64decode | regex_replace('^.*//', '') | regex_replace('@(.*)/', '@tcp(\\1)/') }}_api"
+ OCTAVIA_DSN: "{{ _octavia_db_user.resources.0.data.DB_CONNECTION | b64decode | regex_replace('^.*//', '') | regex_replace('@(.*)/', '@tcp(\\1)/') }}"
- name: Deploy service
run_once: true