feat(opsgenie): add integration
This commit adds a native ingration for adding OpsGenie to the
monitoring so you can easily and quickly have alerts go straight
into OpsGenie.
diff --git a/atmosphere/flows.py b/atmosphere/flows.py
index 3086da6..2301c4e 100644
--- a/atmosphere/flows.py
+++ b/atmosphere/flows.py
@@ -43,7 +43,8 @@
# monitoring
v1.ApplyNamespaceTask(name=constants.NAMESPACE_MONITORING),
*openstack_helm.kube_prometheus_stack_tasks_from_config(
- config.kube_prometheus_stack
+ config.kube_prometheus_stack,
+ opsgenie=config.opsgenie,
),
flux.ApplyHelmRepositoryTask(
namespace=constants.NAMESPACE_MONITORING,
diff --git a/atmosphere/models/config.py b/atmosphere/models/config.py
index 2bbdfa7..fada215 100644
--- a/atmosphere/models/config.py
+++ b/atmosphere/models/config.py
@@ -2,6 +2,7 @@
import tomli
from schematics import types
+from schematics.exceptions import ValidationError
from atmosphere.models import base
@@ -101,6 +102,22 @@
namespace = types.StringType(default="openstack", required=True)
+class OpsGenieConfig(base.Model):
+ enabled = types.BooleanType(default=False, required=True)
+ api_key = types.StringType()
+ heartbeat = types.StringType()
+
+ def validate_api_key(self, data, value):
+ if data["enabled"] and not value:
+ raise ValidationError(types.BaseType.MESSAGES["required"])
+ return value
+
+ def validate_heartbeat(self, data, value):
+ if data["enabled"] and not value:
+ raise ValidationError(types.BaseType.MESSAGES["required"])
+ return value
+
+
class Config(base.Model):
kube_prometheus_stack = types.ModelType(
KubePrometheusStackChartConfig, default=KubePrometheusStackChartConfig()
@@ -116,6 +133,7 @@
default=AcmeIssuerConfig(),
required=True,
)
+ opsgenie = types.ModelType(OpsGenieConfig, default=OpsGenieConfig())
@classmethod
def from_toml(cls, data, validate=True):
diff --git a/atmosphere/tasks/composite/openstack_helm.py b/atmosphere/tasks/composite/openstack_helm.py
index 6a09f89..c4ea65c 100644
--- a/atmosphere/tasks/composite/openstack_helm.py
+++ b/atmosphere/tasks/composite/openstack_helm.py
@@ -1,3 +1,5 @@
+import textwrap
+
import mergedeep
import pykube
import yaml
@@ -48,8 +50,90 @@
)
+def generate_alertmanager_config_for_opsgenie(
+ opsgenie: config.OpsGenieConfig,
+) -> dict:
+ return {
+ "route": {
+ "group_by": ["alertname", "severity"],
+ "receiver": "opsgenie",
+ "routes": [
+ {"receiver": "null", "matchers": ['alertname = "InfoInhibitor"']},
+ {
+ "receiver": "heartbeat",
+ "matchers": ['alertname = "Watchdog"'],
+ "group_wait": "0s",
+ "group_interval": "30s",
+ "repeat_interval": "15s",
+ },
+ ],
+ },
+ "receivers": [
+ {"name": "null"},
+ {
+ "name": "opsgenie",
+ "opsgenie_configs": [
+ {
+ "api_key": opsgenie.api_key,
+ "message": "{{ .GroupLabels.alertname }}",
+ "priority": textwrap.dedent(
+ """\
+ {{- if eq .GroupLabels.severity "critical" -}}
+ P1
+ {{- else if eq .GroupLabels.severity "warning" -}}
+ P2
+ {{- else if eq .GroupLabels.severity "info" -}}
+ P3
+ {{- else -}}
+ P4
+ {{- end -}}
+ """
+ ),
+ "description": textwrap.dedent(
+ """\
+ {{ if gt (len .Alerts.Firing) 0 -}}
+ Alerts Firing:
+ {{ range .Alerts.Firing }}
+ - Message: {{ .Annotations.message }}
+ Labels:
+ {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
+ {{ end }} Annotations:
+ {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
+ {{ end }} Source: {{ .GeneratorURL }}
+ {{ end }}
+ {{- end }}
+ {{ if gt (len .Alerts.Resolved) 0 -}}
+ Alerts Resolved:
+ {{ range .Alerts.Resolved }}
+ - Message: {{ .Annotations.message }}
+ Labels:
+ {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
+ {{ end }} Annotations:
+ {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
+ {{ end }} Source: {{ .GeneratorURL }}
+ {{ end }}
+ {{- end }}
+ """
+ ),
+ }
+ ],
+ },
+ {
+ "name": "heartbeat",
+ "webhook_configs": [
+ {
+ "url": f"https://api.opsgenie.com/v2/heartbeats/{opsgenie.heartbeat}/ping",
+ "send_resolved": False,
+ "http_config": {"basic_auth": {"password": opsgenie.api_key}},
+ }
+ ],
+ },
+ ],
+ }
+
+
def kube_prometheus_stack_tasks_from_config(
- config: config.KubePrometheusStackChartConfig,
+ config: config.KubePrometheusStackChartConfig, opsgenie: config.OpsGenieConfig
):
if not config.enabled:
return []
@@ -60,11 +144,16 @@
config.overrides,
)
+ if opsgenie.enabled:
+ values["alertmanager"]["config"] = generate_alertmanager_config_for_opsgenie(
+ opsgenie
+ )
+
return [
flux.ApplyHelmRepositoryTask(
namespace=constants.NAMESPACE_MONITORING,
name=constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
- url="https://prometheus-community.github.io/helm-charts",
+ url=constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY_URL,
),
flux.ApplyHelmReleaseTask(
namespace=config.namespace,
diff --git a/atmosphere/tasks/constants.py b/atmosphere/tasks/constants.py
index b121bff..4c737fc 100644
--- a/atmosphere/tasks/constants.py
+++ b/atmosphere/tasks/constants.py
@@ -23,7 +23,11 @@
HELM_REPOSITORY_OPENSTACK_HELM = "openstack-helm"
HELM_REPOSITORY_OPENSTACK_HELM_INFRA = "openstack-helm-infra"
HELM_REPOSITORY_PERCONA = "percona"
+
HELM_REPOSITORY_PROMETHEUS_COMMUINTY = "prometheus-community"
+HELM_REPOSITORY_PROMETHEUS_COMMUINTY_URL = (
+ "https://prometheus-community.github.io/helm-charts"
+)
PROMETHEUS_MONITOR_RELABELING_SET_NODE_NAME_TO_INSTANCE = {
"sourceLabels": ["__meta_kubernetes_pod_node_name"],
diff --git a/atmosphere/tests/unit/tasks/composite/test_openstack_helm.py b/atmosphere/tests/unit/tasks/composite/test_openstack_helm.py
index ca1febb..f94d970 100644
--- a/atmosphere/tests/unit/tasks/composite/test_openstack_helm.py
+++ b/atmosphere/tests/unit/tasks/composite/test_openstack_helm.py
@@ -1,5 +1,6 @@
import textwrap
+import mergedeep
import pytest
from atmosphere.models import config
@@ -20,6 +21,193 @@
"apiVersion": "source.toolkit.fluxcd.io/v1beta2",
"kind": "HelmRepository",
"metadata": {
+ "name": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
+ "namespace": constants.NAMESPACE_MONITORING,
+ },
+ "spec": {
+ "interval": "1m",
+ "url": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY_URL,
+ },
+ },
+ {
+ "apiVersion": "helm.toolkit.fluxcd.io/v2beta1",
+ "kind": "HelmRelease",
+ "metadata": {
+ "name": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_NAME,
+ "namespace": constants.NAMESPACE_MONITORING,
+ },
+ "spec": {
+ "chart": {
+ "spec": {
+ "chart": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_NAME,
+ "sourceRef": {
+ "kind": "HelmRepository",
+ "name": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
+ },
+ "version": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_VERSION,
+ }
+ },
+ "install": {"crds": "CreateReplace", "disableWait": True},
+ "interval": "60s",
+ "upgrade": {"crds": "CreateReplace", "disableWait": True},
+ "values": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_VALUES,
+ "valuesFrom": [],
+ },
+ },
+ ],
+ id="default",
+ ),
+ pytest.param(
+ textwrap.dedent(
+ """\
+ [kube_prometheus_stack.overrides]
+ foo = "bar"
+ """
+ ),
+ [
+ {
+ "apiVersion": "source.toolkit.fluxcd.io/v1beta2",
+ "kind": "HelmRepository",
+ "metadata": {
+ "name": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
+ "namespace": constants.NAMESPACE_MONITORING,
+ },
+ "spec": {
+ "interval": "1m",
+ "url": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY_URL,
+ },
+ },
+ {
+ "apiVersion": "helm.toolkit.fluxcd.io/v2beta1",
+ "kind": "HelmRelease",
+ "metadata": {
+ "name": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_NAME,
+ "namespace": constants.NAMESPACE_MONITORING,
+ },
+ "spec": {
+ "chart": {
+ "spec": {
+ "chart": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_NAME,
+ "sourceRef": {
+ "kind": "HelmRepository",
+ "name": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
+ },
+ "version": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_VERSION,
+ }
+ },
+ "install": {"crds": "CreateReplace", "disableWait": True},
+ "interval": "60s",
+ "upgrade": {"crds": "CreateReplace", "disableWait": True},
+ "values": mergedeep.merge(
+ {},
+ constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_VALUES,
+ {"foo": "bar"},
+ ),
+ "valuesFrom": [],
+ },
+ },
+ ],
+ id="overrides",
+ ),
+ pytest.param(
+ textwrap.dedent(
+ """\
+ [opsgenie]
+ enabled = true
+ api_key = "foobar"
+ heartbeat = "prod"
+ """
+ ),
+ [
+ {
+ "apiVersion": "source.toolkit.fluxcd.io/v1beta2",
+ "kind": "HelmRepository",
+ "metadata": {
+ "name": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
+ "namespace": constants.NAMESPACE_MONITORING,
+ },
+ "spec": {
+ "interval": "1m",
+ "url": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY_URL,
+ },
+ },
+ {
+ "apiVersion": "helm.toolkit.fluxcd.io/v2beta1",
+ "kind": "HelmRelease",
+ "metadata": {
+ "name": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_NAME,
+ "namespace": constants.NAMESPACE_MONITORING,
+ },
+ "spec": {
+ "chart": {
+ "spec": {
+ "chart": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_NAME,
+ "sourceRef": {
+ "kind": "HelmRepository",
+ "name": constants.HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
+ },
+ "version": constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_VERSION,
+ }
+ },
+ "install": {"crds": "CreateReplace", "disableWait": True},
+ "interval": "60s",
+ "upgrade": {"crds": "CreateReplace", "disableWait": True},
+ "values": mergedeep.merge(
+ {},
+ constants.HELM_RELEASE_KUBE_PROMETHEUS_STACK_VALUES,
+ {
+ "alertmanager": {
+ "config": openstack_helm.generate_alertmanager_config_for_opsgenie(
+ config.OpsGenieConfig(
+ {"api_key": "foobar", "heartbeat": "prod"}
+ )
+ )
+ }
+ },
+ ),
+ "valuesFrom": [],
+ },
+ },
+ ],
+ id="opsgenie",
+ ),
+ pytest.param(
+ textwrap.dedent(
+ """\
+ [kube_prometheus_stack]
+ enabled = false
+ """
+ ),
+ [],
+ id="disabled",
+ ),
+ ],
+)
+def test_kube_prometheus_stack_tasks_from_config(pykube, cfg_data, expected):
+ cfg = config.Config.from_string(cfg_data, validate=False)
+ cfg.kube_prometheus_stack.validate()
+
+ assert [
+ t.generate_object().obj
+ for t in openstack_helm.kube_prometheus_stack_tasks_from_config(
+ cfg.kube_prometheus_stack, opsgenie=cfg.opsgenie
+ )
+ ] == expected
+
+
+@pytest.mark.parametrize(
+ "cfg_data,expected",
+ [
+ pytest.param(
+ textwrap.dedent(
+ """\
+ """
+ ),
+ [
+ {
+ "apiVersion": "source.toolkit.fluxcd.io/v1beta2",
+ "kind": "HelmRepository",
+ "metadata": {
"name": constants.HELM_RELEASE_INGRESS_NGINX_NAME,
"namespace": constants.NAMESPACE_OPENSTACK,
},
diff --git a/docs/integrations.md b/docs/integrations.md
new file mode 100644
index 0000000..0678b63
--- /dev/null
+++ b/docs/integrations.md
@@ -0,0 +1,29 @@
+# Integrations
+
+## OpsGenie
+
+Atmosphere can be integrated with OpsGenie in order to send all alerts to it,
+this is useful if you want to have a single place to manage all your alerts.
+
+In order to get started, you will need to complete the following steps inside
+OpsGenie:
+
+1. Create an integration inside OpsGenie, you can do this by going to
+ _Settings_ > _Integrations_ > _Add Integration_ and selecting _Prometheus_.
+2. Copy the API key that is generated for you and setup correct assignment
+ rules inside OpsGenie.
+3. Create a new heartbeat inside OpsGenie, you can do this by going to
+ _Settings_ > _Heartbeats_ > _Create Heartbeat_. Set the interval to 1 minute.
+
+Afterwards, you can configure the following options for the Atmosphere config:
+
+```yaml
+atmosphere_config:
+ opsgenie:
+ enabled: true
+ api_key: <your-api-key>
+ heartbeat_name: <your-heartbeat-name>
+```
+
+Once this is done and deployed, you'll start to see alerts inside OpsGenie and
+you can also verify that the heartbeat is listed as _ACTIVE_.