Switch to RabbitMQ operator
Finally.
Sem-Ver: feature
Change-Id: Id03c7651ff964328cca1cd0a624b3f38f9b33634
diff --git a/doc/source/roles/openstack_helm_infra_rabbitmq/index.rst b/doc/source/roles/openstack_helm_infra_rabbitmq/index.rst
deleted file mode 100644
index 4ab8e1f..0000000
--- a/doc/source/roles/openstack_helm_infra_rabbitmq/index.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-.. Copyright (C) 2022 VEXXHOST, Inc.
-.. SPDX-License-Identifier: Apache-2.0
-
-``openstack_helm_infra_rabbitmq``
-=================================
-
-.. toctree::
- :maxdepth: 2
-
- defaults/main
\ No newline at end of file
diff --git a/playbooks/cleanup.yml b/playbooks/cleanup.yml
new file mode 100644
index 0000000..49031d4
--- /dev/null
+++ b/playbooks/cleanup.yml
@@ -0,0 +1,35 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Clean-up legacy RabbitMQ cluster
+ hosts: controllers[0]
+ become: true
+ gather_facts: false
+ tasks:
+ - name: Delete the Helm release
+ kubernetes.core.helm:
+ name: rabbitmq
+ namespace: openstack
+ kubeconfig: /etc/kubernetes/admin.conf
+ state: absent
+ wait: true
+
+ - name: Delete the PVCs
+ kubernetes.core.k8s:
+ state: absent
+ api_version: v1
+ kind: PersistentVolumeClaim
+ namespace: openstack
+ name: "rabbitmq-data-rabbitmq-rabbitmq-{{ item }}"
+ loop: "{{ range(0, 3) | list }}"
\ No newline at end of file
diff --git a/playbooks/openstack.yml b/playbooks/openstack.yml
index aff12a8..18a0aa7 100644
--- a/playbooks/openstack.yml
+++ b/playbooks/openstack.yml
@@ -64,9 +64,9 @@
tags:
- openstack-helm-infra-memcached
- - role: openstack_helm_infra_rabbitmq
+ - role: rabbitmq_operator
tags:
- - openstack-helm-infra-rabbitmq
+ - rabbitmq-operator
- role: openstack_helm_keystone
tags:
diff --git a/playbooks/site.yml b/playbooks/site.yml
index 2966d48..f1eaefb 100644
--- a/playbooks/site.yml
+++ b/playbooks/site.yml
@@ -15,3 +15,4 @@
- import_playbook: vexxhost.atmosphere.ceph
- import_playbook: vexxhost.atmosphere.kubernetes
- import_playbook: vexxhost.atmosphere.openstack
+- import_playbook: vexxhost.atmosphere.cleanup
\ No newline at end of file
diff --git a/releasenotes/notes/migrate-to-rabbitmq-operator-908ead4f29c82230.yaml b/releasenotes/notes/migrate-to-rabbitmq-operator-908ead4f29c82230.yaml
new file mode 100644
index 0000000..e931c1f
--- /dev/null
+++ b/releasenotes/notes/migrate-to-rabbitmq-operator-908ead4f29c82230.yaml
@@ -0,0 +1,7 @@
+---
+features:
+ - Introduce usage of RabbitMQ operator, remove usage of old RabbitMQ charts
+ and start to run a single replica of RabbitMQ for each OpenStack service.
+upgrade:
+ - The playbooks must all be ran in order, and once done make sure to have the
+ ``cleanup`` playbook run to clean up the old cluster.
diff --git a/roles/openstack_helm_endpoints/defaults/main.yml b/roles/openstack_helm_endpoints/defaults/main.yml
index 7e5987c..567c08b 100644
--- a/roles/openstack_helm_endpoints/defaults/main.yml
+++ b/roles/openstack_helm_endpoints/defaults/main.yml
@@ -23,12 +23,6 @@
openstack_helm_endpoints_region_name: "{{ undef(hint='You must specify an OpenStack region name') }}"
# ]]]
-# .. envvar:: openstack_helm_endpoints_rabbitmq_erlang_cookie [[[
-#
-# Erlang cookie for RabbitMQ
-openstack_helm_endpoints_rabbitmq_erlang_cookie: "{{ undef(hint='You must specify an RabbitMQ Erlang cookie') }}"
-
- # ]]]
# .. envvar:: openstack_helm_endpoints_rabbitmq_admin_password [[[
#
# RabbitMQ administrator password
diff --git a/roles/openstack_helm_endpoints/tasks/main.yml b/roles/openstack_helm_endpoints/tasks/main.yml
index a616f0d..7562a0f 100644
--- a/roles/openstack_helm_endpoints/tasks/main.yml
+++ b/roles/openstack_helm_endpoints/tasks/main.yml
@@ -26,6 +26,34 @@
when:
- openstack_helm_endpoints_list is not defined or openstack_helm_endpoints_list == None
+# NOTE(mnaser): Since we manage one-RabbitMQ per service, we create the RabbitMQ
+# cluster here and then append the necessary values to be used
+# inside the `oslo_messaging` section.
+- name: Configure oslo.messaging
+ block:
+ - name: Create RabbitMQ cluster
+ ansible.builtin.include_role:
+ name: rabbitmq
+ vars:
+ rabbitmq_cluster_name: "{{ openstack_helm_endpoints_chart }}"
+
+ - name: Grab RabbitMQ cluster secret
+ kubernetes.core.k8s_info:
+ api_version: v1
+ kind: Secret
+ name: "rabbitmq-{{ openstack_helm_endpoints_chart }}-default-user"
+ namespace: openstack
+ register: _openstack_helm_endpoints_rabbitmq_cluster_secret
+
+ - name: Cache fact with RabbitMQ cluster credentials
+ ansible.builtin.set_fact:
+ _openstack_helm_endpoints_rabbitmq_cluster_username: |-
+ {{ _openstack_helm_endpoints_rabbitmq_cluster_secret.resources[0]['data']['username'] | b64decode }}
+ _openstack_helm_endpoints_rabbitmq_cluster_password: |-
+ {{ _openstack_helm_endpoints_rabbitmq_cluster_secret.resources[0]['data']['password'] | b64decode }}
+ when:
+ - '"oslo_messaging" in openstack_helm_endpoints_list'
+
# NOTE(mnaser): Since we deploy the database using the operator and we let it
# generate the root password, we look it up if the fact has not
# been cached from a previous run.
diff --git a/roles/openstack_helm_endpoints/vars/main.yml b/roles/openstack_helm_endpoints/vars/main.yml
index bb90562..b41dbe9 100644
--- a/roles/openstack_helm_endpoints/vars/main.yml
+++ b/roles/openstack_helm_endpoints/vars/main.yml
@@ -37,16 +37,18 @@
_openstack_helm_endpoints_oslo_messaging:
oslo_messaging:
auth:
- erlang_cookie: "{{ openstack_helm_endpoints_rabbitmq_erlang_cookie }}"
user:
- password: "{{ openstack_helm_endpoints_rabbitmq_admin_password }}"
+ username: "{{ _openstack_helm_endpoints_rabbitmq_cluster_username }}"
+ password: "{{ _openstack_helm_endpoints_rabbitmq_cluster_password }}"
# NOTE(mnaser): The following is not actually used by the chart, however,
# since we are actually doing dynamic lookups to generate
# endpoints, we add it here.
admin:
- password: "{{ openstack_helm_endpoints_rabbitmq_admin_password }}"
- statefulset:
- replicas: 3
+ username: "{{ _openstack_helm_endpoints_rabbitmq_cluster_username }}"
+ password: "{{ _openstack_helm_endpoints_rabbitmq_cluster_password }}"
+ statefulset: null
+ hosts:
+ default: "rabbitmq-{{ openstack_helm_endpoints_chart }}"
_openstack_helm_endpoints_oslo_cache:
oslo_cache:
diff --git a/roles/openstack_helm_infra_rabbitmq/defaults/main.yml b/roles/openstack_helm_infra_rabbitmq/defaults/main.yml
deleted file mode 100644
index 355e1be..0000000
--- a/roles/openstack_helm_infra_rabbitmq/defaults/main.yml
+++ /dev/null
@@ -1,37 +0,0 @@
----
-# .. vim: foldmarker=[[[,]]]:foldmethod=marker
-
-# .. Copyright (C) 2022 VEXXHOST, Inc.
-# .. SPDX-License-Identifier: Apache-2.0
-
-# Default variables
-# =================
-
-# .. contents:: Sections
-# :local:
-
-
-# .. envvar:: openstack_helm_infra_rabbitmq_chart_repo_name [[[
-#
-# Helm repository name for the chart.
-openstack_helm_infra_rabbitmq_chart_repo_name: openstack-helm-infra
-
- # ]]]
-# .. envvar:: openstack_helm_infra_rabbitmq_chart_repo_url [[[
-#
-# Helm repository URL for the chart.
-openstack_helm_infra_rabbitmq_chart_repo_url: https://tarballs.opendev.org/openstack/openstack-helm-infra/
-
- # ]]]
-# .. envvar:: openstack_helm_infra_rabbitmq_chart_name [[[
-#
-# Helm chart name (will also be used for release name)
-openstack_helm_infra_rabbitmq_chart_name: rabbitmq
-
- # ]]]
-# .. envvar:: openstack_helm_infra_rabbitmq_image_repository [[[
-#
-# Image repository location to be prefixed for all images
-openstack_helm_infra_rabbitmq_image_repository: "{{ atmosphere_image_repository | default('us-docker.pkg.dev/vexxhost-infra/openstack') }}"
-
- # ]]]
diff --git a/roles/openstack_helm_infra_rabbitmq/tasks/main.yml b/roles/openstack_helm_infra_rabbitmq/tasks/main.yml
deleted file mode 100644
index 7e50b0e..0000000
--- a/roles/openstack_helm_infra_rabbitmq/tasks/main.yml
+++ /dev/null
@@ -1,159 +0,0 @@
-# Copyright (c) 2022 VEXXHOST, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-- name: Generate OpenStack-Helm endpoints
- ansible.builtin.include_role:
- name: openstack_helm_endpoints
- vars:
- openstack_helm_endpoints_repo_name: "{{ openstack_helm_infra_rabbitmq_chart_repo_name }}"
- openstack_helm_endpoints_repo_url: "{{ openstack_helm_infra_rabbitmq_chart_repo_url }}"
- openstack_helm_endpoints_chart: "{{ openstack_helm_infra_rabbitmq_chart_name }}"
-
-- name: Deploy Helm chart
- kubernetes.core.helm:
- name: "{{ openstack_helm_infra_rabbitmq_chart_name }}"
- chart_ref: "{{ openstack_helm_infra_rabbitmq_chart_repo_name }}/{{ openstack_helm_infra_rabbitmq_chart_name }}"
- chart_version: 0.1.15
- release_namespace: openstack
- kubeconfig: /etc/kubernetes/admin.conf
- values:
- endpoints: "{{ openstack_helm_endpoints }}"
- images:
- tags:
- dep_check: "{{ openstack_helm_infra_rabbitmq_image_repository }}/kubernetes-entrypoint:latest"
- rabbitmq_init: "{{ openstack_helm_infra_rabbitmq_image_repository }}/cli:latest"
- rabbitmq: "{{ openstack_helm_infra_rabbitmq_image_repository }}/rabbitmq:3.8.23"
- pod:
- replicas:
- server: 3
- conf:
- enabled_plugins:
- - rabbitmq_management
- - rabbitmq_peer_discovery_k8s
- - rabbitmq_prometheus
- volume:
- size: 10Gi
- manifests:
- ingress_management: false
-
-- name: Create Service for metrics
- kubernetes.core.k8s:
- state: present
- definition:
- apiVersion: v1
- kind: Service
- metadata:
- name: rabbitmq-metrics
- namespace: openstack
- labels:
- application: rabbitmq
- component: server
- spec:
- selector:
- application: rabbitmq
- component: server
- ports:
- - name: metrics
- port: 15692
- targetPort: 15692
-
-- name: Create ServiceMonitor
- kubernetes.core.k8s:
- state: present
- definition:
- apiVersion: monitoring.coreos.com/v1
- kind: ServiceMonitor
- metadata:
- name: rabbitmq
- namespace: monitoring
- labels:
- release: kube-prometheus-stack
- spec:
- jobLabel: application
- endpoints:
- - port: "metrics"
- path: "/metrics"
- relabelings:
- - sourceLabels: ["__meta_kubernetes_pod_name"]
- targetLabel: "instance"
- - action: "labeldrop"
- regex: "^(container|endpoint|namespace|pod|service)$"
- namespaceSelector:
- matchNames:
- - openstack
- selector:
- matchLabels:
- application: rabbitmq
- component: server
-
-- name: Create PrometheusRule
- kubernetes.core.k8s:
- state: present
- definition:
- apiVersion: monitoring.coreos.com/v1
- kind: PrometheusRule
- metadata:
- name: rabbitmq
- namespace: monitoring
- labels:
- release: kube-prometheus-stack
- spec:
- groups:
- - name: rules
- rules:
- - alert: RabbitmqDown
- expr: absent(rabbitmq_build_info)
- for: 1m
- labels:
- severity: critical
- - alert: RabbitmqNodeDown
- expr: sum(rabbitmq_build_info) < 3
- for: 1m
- labels:
- severity: critical
- - alert: RabbitmqNodeNotDistributed
- expr: erlang_vm_dist_node_state < 3
- for: 1m
- labels:
- severity: critical
- - alert: RabbitmqMemoryHigh
- expr: rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes > 0.80
- labels:
- severity: warning
- - alert: RabbitmqMemoryHigh
- expr: rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes > 0.95
- labels:
- severity: critical
- - alert: RabbitmqFileDescriptorsUsage
- expr: rabbitmq_process_open_fds / rabbitmq_process_max_fds > 0.80
- labels:
- severity: warning
- - alert: RabbitmqFileDescriptorsUsage
- expr: rabbitmq_process_open_fds / rabbitmq_process_max_fds > 0.95
- labels:
- severity: critical
- - alert: RabbitmqUnackedMessages
- expr: sum(rabbitmq_queue_messages_unacked) BY (queue) > 1000
- for: 5m
- labels:
- severity: warning
- - alert: RabbitmqUnackedMessages
- expr: sum(rabbitmq_queue_messages_unacked) BY (queue) > 1000
- for: 1h
- labels:
- severity: critical
- - alert: RabbitmqConnections
- expr: rabbitmq_connections > 1000
- labels:
- severity: warning
diff --git a/roles/openstack_helm_neutron/meta/main.yml b/roles/openstack_helm_neutron/meta/main.yml
index c11c9ec..ce58dc4 100644
--- a/roles/openstack_helm_neutron/meta/main.yml
+++ b/roles/openstack_helm_neutron/meta/main.yml
@@ -27,11 +27,4 @@
vars:
helm_repository_name: "{{ openstack_helm_neutron_chart_repo_name }}"
helm_repository_repo_url: "{{ openstack_helm_neutron_chart_repo_url }}"
- - cilium
- - openstack_namespace
- - percona_xtradb_cluster
- - openstack_helm_infra_memcached
- - openstack_helm_infra_rabbitmq
- - openstack_helm_infra_openvswitch
- - openstack_helm_keystone
- openstacksdk
diff --git a/roles/openstack_helm_senlin/tasks/main.yml b/roles/openstack_helm_senlin/tasks/main.yml
index 073bb7c..4fa5979 100644
--- a/roles/openstack_helm_senlin/tasks/main.yml
+++ b/roles/openstack_helm_senlin/tasks/main.yml
@@ -54,7 +54,7 @@
kubernetes.core.helm:
name: "{{ openstack_helm_senlin_chart_name }}"
chart_ref: "{{ openstack_helm_senlin_chart_repo_name }}/{{ openstack_helm_senlin_chart_name }}"
- chart_version: 0.2.3
+ chart_version: 0.2.6
release_namespace: openstack
kubeconfig: /etc/kubernetes/admin.conf
values: "{{ _openstack_helm_senlin_values }}"
diff --git a/roles/openstack_helm_infra_rabbitmq/meta/main.yml b/roles/rabbitmq/meta/main.yml
similarity index 79%
rename from roles/openstack_helm_infra_rabbitmq/meta/main.yml
rename to roles/rabbitmq/meta/main.yml
index 27cf12a..da0f4ee 100644
--- a/roles/openstack_helm_infra_rabbitmq/meta/main.yml
+++ b/roles/rabbitmq/meta/main.yml
@@ -23,7 +23,4 @@
- focal
dependencies:
- - role: helm_repository
- vars:
- helm_repository_name: "{{ openstack_helm_infra_rabbitmq_chart_repo_name }}"
- helm_repository_repo_url: "{{ openstack_helm_infra_rabbitmq_chart_repo_url }}"
+ - rabbitmq_operator
diff --git a/roles/rabbitmq/tasks/main.yml b/roles/rabbitmq/tasks/main.yml
new file mode 100644
index 0000000..36edf3f
--- /dev/null
+++ b/roles/rabbitmq/tasks/main.yml
@@ -0,0 +1,38 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Deploy cluster
+ kubernetes.core.k8s:
+ state: present
+ definition:
+ apiVersion: rabbitmq.com/v1beta1
+ kind: RabbitmqCluster
+ metadata:
+ name: "rabbitmq-{{ rabbitmq_cluster_name }}"
+ namespace: openstack
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: openstack-control-plane
+ operator: In
+ values:
+ - enabled
+ wait: true
+ wait_timeout: 600
+ wait_condition:
+ type: ClusterAvailable
+ status: "True"
diff --git a/roles/openstack_helm_infra_rabbitmq/meta/main.yml b/roles/rabbitmq_operator/meta/main.yml
similarity index 78%
copy from roles/openstack_helm_infra_rabbitmq/meta/main.yml
copy to roles/rabbitmq_operator/meta/main.yml
index 27cf12a..4ff64d0 100644
--- a/roles/openstack_helm_infra_rabbitmq/meta/main.yml
+++ b/roles/rabbitmq_operator/meta/main.yml
@@ -14,7 +14,7 @@
galaxy_info:
author: VEXXHOST, Inc.
- description: Ansible role for RabbitMQ
+ description: Ansible role for RabbitMQ operator
license: Apache-2.0
min_ansible_version: 5.5.0
platforms:
@@ -25,5 +25,5 @@
dependencies:
- role: helm_repository
vars:
- helm_repository_name: "{{ openstack_helm_infra_rabbitmq_chart_repo_name }}"
- helm_repository_repo_url: "{{ openstack_helm_infra_rabbitmq_chart_repo_url }}"
+ helm_repository_name: bitnami
+ helm_repository_repo_url: https://charts.bitnami.com/bitnami
diff --git a/roles/rabbitmq_operator/tasks/main.yml b/roles/rabbitmq_operator/tasks/main.yml
new file mode 100644
index 0000000..49d2e8b
--- /dev/null
+++ b/roles/rabbitmq_operator/tasks/main.yml
@@ -0,0 +1,119 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Deploy Helm chart
+ kubernetes.core.helm:
+ name: rabbitmq-cluster-operator
+ chart_ref: bitnami/rabbitmq-cluster-operator
+ chart_version: 2.5.2
+ release_namespace: openstack
+ kubeconfig: /etc/kubernetes/admin.conf
+ values:
+ rabbitmqImage:
+ repository: library/rabbitmq
+ tag: 3.10.2-management
+ credentialUpdaterImage:
+ repository: rabbitmqoperator/default-user-credential-updater
+ tag: 1.0.2
+ clusterOperator:
+ image:
+ repository: rabbitmqoperator/cluster-operator
+ tag: 1.13.1
+ fullnameOverride: rabbitmq-cluster-operator
+ nodeSelector:
+ openstack-control-plane: enabled
+ msgTopologyOperator:
+ image:
+ repository: rabbitmqoperator/messaging-topology-operator
+ tag: 1.6.0
+ fullnameOverride: rabbitmq-messaging-topology-operator
+ nodeSelector:
+ openstack-control-plane: enabled
+ useCertManager: true
+
+- name: Create PodMonitor
+ kubernetes.core.k8s:
+ state: present
+ definition:
+ apiVersion: monitoring.coreos.com/v1
+ kind: PodMonitor
+ metadata:
+ name: rabbitmq
+ namespace: monitoring
+ labels:
+ release: kube-prometheus-stack
+ spec:
+ jobLabel: app.kubernetes.io/component
+ podMetricsEndpoints:
+ - port: prometheus
+ path: /metrics
+ relabelings:
+ - sourceLabels: ["__meta_kubernetes_pod_name"]
+ targetLabel: "instance"
+ - action: "labeldrop"
+ regex: "^(container|endpoint|namespace|pod|service)$"
+ namespaceSelector:
+ matchNames:
+ - openstack
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: rabbitmq
+
+- name: Create PrometheusRule
+ kubernetes.core.k8s:
+ state: present
+ definition:
+ apiVersion: monitoring.coreos.com/v1
+ kind: PrometheusRule
+ metadata:
+ name: rabbitmq
+ namespace: monitoring
+ labels:
+ release: kube-prometheus-stack
+ spec:
+ groups:
+ - name: limits
+ rules:
+ - alert: RabbitmqMemoryHigh
+ expr: rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes > 0.80
+ labels:
+ severity: warning
+ - alert: RabbitmqMemoryHigh
+ expr: rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes > 0.95
+ labels:
+ severity: critical
+ - alert: RabbitmqFileDescriptorsUsage
+ expr: rabbitmq_process_open_fds / rabbitmq_process_max_fds > 0.80
+ labels:
+ severity: warning
+ - alert: RabbitmqFileDescriptorsUsage
+ expr: rabbitmq_process_open_fds / rabbitmq_process_max_fds > 0.95
+ labels:
+ severity: critical
+ - alert: RabbitmqConnections
+ expr: rabbitmq_connections > 1000
+ labels:
+ severity: warning
+ - name: msgs
+ rules:
+ - alert: RabbitmqUnackedMessages
+ expr: sum(rabbitmq_queue_messages_unacked) BY (queue) > 1000
+ for: 5m
+ labels:
+ severity: warning
+ - alert: RabbitmqUnackedMessages
+ expr: sum(rabbitmq_queue_messages_unacked) BY (queue) > 1000
+ for: 1h
+ labels:
+ severity: critical