feat: clean-up more code for helm repos
diff --git a/atmosphere/flows.py b/atmosphere/flows.py
index 73a3ea8..6c424eb 100644
--- a/atmosphere/flows.py
+++ b/atmosphere/flows.py
@@ -3,52 +3,91 @@
from atmosphere.config import CONF
from atmosphere.tasks import flux, kubernetes, openstack_helm
-HELM_REPOSITORIES_BY_NAMESPACE = {
- "kube-system": {
- "ceph": "https://ceph.github.io/csi-charts",
- },
- "openstack": {
- "bitnami": "https://charts.bitnami.com/bitnami",
- "coredns": "https://coredns.github.io/helm",
- "ingress-nginx": "https://kubernetes.github.io/ingress-nginx",
- "openstack-helm-infra": "https://tarballs.opendev.org/openstack/openstack-helm-infra/",
- "openstack-helm": "https://tarballs.opendev.org/openstack/openstack-helm/",
- "percona": "https://percona.github.io/percona-helm-charts/",
- },
-}
+NAMESPACE_CERT_MANAGER = "cert-manager"
+NAMESPACE_KUBE_SYSTEM = "kube-system"
+NAMESPACE_MONITORING = "monitoring"
+NAMESPACE_OPENSTACK = "openstack"
-OPENSTACK_HELM_CHARTS_BY_NAMESPACE = {
- "openstack": ["memcached"],
-}
-
-
-def generate_for_openstack_helm_chart(chart):
- flow = graph_flow.Flow(chart)
-
- if getattr(CONF, chart).enabled:
- flow.add(
- openstack_helm.GenerateReleaseSecretTask(inject={"chart": chart}),
- kubernetes.EnsureSecretTask(),
- )
-
- return flow
+HELM_REPOSITORY_BITNAMI = "bitnami"
+HELM_REPOSITORY_CEPH = "ceph"
+HELM_REPOSITORY_COREDNS = "coredns"
+HELM_REPOSITORY_INGRESS_NGINX = "ingress-nginx"
+HELM_REPOSITORY_JETSTACK = "jetstack"
+HELM_REPOSITORY_NODE_FEATURE_DISCOVERY = "node-feature-discovery"
+HELM_REPOSITORY_OPENSTACK_HELM = "openstack-helm"
+HELM_REPOSITORY_OPENSTACK_HELM_INFRA = "openstack-helm-infra"
+HELM_REPOSITORY_PERCONA = "percona"
+HELM_REPOSITORY_PROMETHEUS_COMMUINTY = "prometheus-community"
def get_deployment_flow():
- flow = graph_flow.Flow("deploy")
+ flow = graph_flow.Flow("deploy").add(
+ # kube-system
+ kubernetes.CreateOrUpdateNamespaceTask(name=NAMESPACE_KUBE_SYSTEM),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_KUBE_SYSTEM,
+ name=HELM_REPOSITORY_CEPH,
+ url="https://ceph.github.io/csi-charts",
+ ),
+ # cert-manager
+ kubernetes.CreateOrUpdateNamespaceTask(name=NAMESPACE_CERT_MANAGER),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_CERT_MANAGER,
+ name=HELM_REPOSITORY_JETSTACK,
+ url="https://charts.jetstack.io",
+ ),
+ # monitoring
+ kubernetes.CreateOrUpdateNamespaceTask(name=NAMESPACE_MONITORING),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_MONITORING,
+ name=HELM_REPOSITORY_PROMETHEUS_COMMUINTY,
+ url="https://prometheus-community.github.io/helm-charts",
+ ),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_MONITORING,
+ name=HELM_REPOSITORY_NODE_FEATURE_DISCOVERY,
+ url="https://kubernetes-sigs.github.io/node-feature-discovery/charts",
+ ),
+ # openstack
+ kubernetes.CreateOrUpdateNamespaceTask(name=NAMESPACE_OPENSTACK),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_OPENSTACK,
+ name=HELM_REPOSITORY_BITNAMI,
+ url="https://charts.bitnami.com/bitnami",
+ ),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_OPENSTACK,
+ name=HELM_REPOSITORY_PERCONA,
+ url="https://percona.github.io/percona-helm-charts/",
+ ),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_OPENSTACK,
+ name=HELM_REPOSITORY_INGRESS_NGINX,
+ url="https://kubernetes.github.io/ingress-nginx",
+ ),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_OPENSTACK,
+ name=HELM_REPOSITORY_OPENSTACK_HELM_INFRA,
+ url="https://tarballs.opendev.org/openstack/openstack-helm-infra/",
+ ),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_OPENSTACK,
+ name=HELM_REPOSITORY_COREDNS,
+ url="https://coredns.github.io/helm",
+ ),
+ flux.CreateOrUpdateHelmRepositoryTask(
+ namespace=NAMESPACE_OPENSTACK,
+ name=HELM_REPOSITORY_OPENSTACK_HELM,
+ url="https://tarballs.opendev.org/openstack/openstack-helm/",
+ ),
+ )
- for namespace, repos in HELM_REPOSITORIES_BY_NAMESPACE.items():
- for repo, url in repos.items():
- task = flux.EnsureHelmRepositoryTask(
- name=repo,
- inject={"namespace": namespace, "name": repo, "url": url},
- provides=f"helm-repository-{repo}",
- )
- flow.add(task)
-
- for namespace, charts in OPENSTACK_HELM_CHARTS_BY_NAMESPACE.items():
- for chart in charts:
- flow.add(generate_for_openstack_helm_chart(chart))
+ if CONF.memcached.enabled:
+ flow.add(
+ openstack_helm.CreateOrUpdateReleaseSecretTask(
+ namespace=NAMESPACE_OPENSTACK, chart="memcached"
+ ),
+ )
return flow
diff --git a/atmosphere/tasks/flux.py b/atmosphere/tasks/flux.py
index 4fe9a2c..ec6fdc0 100644
--- a/atmosphere/tasks/flux.py
+++ b/atmosphere/tasks/flux.py
@@ -1,7 +1,7 @@
import pykube
-from taskflow import task
-from atmosphere import clients, logger
+from atmosphere import logger
+from atmosphere.tasks import kubernetes
LOG = logger.get_logger()
@@ -12,20 +12,27 @@
kind = "HelmRepository"
-class EnsureHelmRepositoryTask(task.Task):
- def execute(self, namespace, name, url, *args, **kwargs):
- log = LOG.bind(kind="HelmRelease", namespace=namespace, name=name)
- api = clients.get_pykube_api()
+class CreateOrUpdateHelmRepositoryTask(kubernetes.CreateOrUpdateKubernetesObjectTask):
+ def __init__(self, namespace: str, name: str, url: str, *args, **kwargs):
+ super().__init__(
+ HelmRepository,
+ namespace,
+ name,
+ requires=set(["namespace", "name", "url"]),
+ inject={"name": name, "url": url},
+ *args,
+ **kwargs
+ )
- log.debug("Ensuring HelmRepository")
- repository = HelmRepository(
- api,
+ def generate_object(self, namespace, name, url, *args, **kwargs):
+ return HelmRepository(
+ self.api,
{
"apiVersion": "source.toolkit.fluxcd.io/v1beta2",
"kind": "HelmRepository",
"metadata": {
"name": name,
- "namespace": namespace,
+ "namespace": namespace.name,
},
"spec": {
"interval": "1m",
@@ -33,11 +40,3 @@
},
},
)
-
- if not repository.exists():
- log.debug("Resource does not exist, creating")
- repository.create()
- else:
- repository.update()
-
- log.info("Ensured resource")
diff --git a/atmosphere/tasks/kubernetes.py b/atmosphere/tasks/kubernetes.py
index 12bd512..a5adbe4 100644
--- a/atmosphere/tasks/kubernetes.py
+++ b/atmosphere/tasks/kubernetes.py
@@ -1,41 +1,120 @@
+import re
+
import pykube
from taskflow import task
from atmosphere import clients, logger
+CAMEL_CASE_PATTERN = re.compile(r"(?<!^)(?=[A-Z])")
LOG = logger.get_logger()
-class EnsureSecretTask(task.Task):
- def execute(self, secret_namespace, secret_name, secret_data, *args, **kwargs):
- log = LOG.bind(namespace=secret_namespace, name=secret_name)
- api = clients.get_pykube_api()
+class CreateOrUpdateKubernetesObjectTask(task.Task):
+ def __init__(
+ self, kind: pykube.objects.APIObject, namespace: str, name: str, *args, **kwargs
+ ):
+ self._obj_kind = kind
+ self._obj_namespace = namespace
+ self._obj_name = name
- log.debug("Ensuring secret")
- secret = pykube.Secret(
- api,
+ kwargs["name"] = CAMEL_CASE_PATTERN.sub("-", kind.__name__).lower()
+ if namespace:
+ kwargs["name"] += f"-{namespace}"
+ kwargs["name"] += f"-{name}"
+
+ if namespace:
+ # kwargs.setdefault("requires", [])
+ # kwargs["requires"] += [f"namespace-{namespace}"]
+ kwargs.setdefault("rebind", {})
+ kwargs["rebind"]["namespace"] = f"namespace-{namespace}"
+
+ kwargs.setdefault("provides", set())
+ kwargs["provides"] = kwargs["provides"].union(set([kwargs["name"]]))
+
+ super().__init__(*args, **kwargs)
+
+ @property
+ def api(self):
+ return clients.get_pykube_api()
+
+ @property
+ def logger(self):
+ log = LOG.bind(
+ kind=self._obj_kind.__name__,
+ name=self._obj_name,
+ )
+ if self._obj_namespace:
+ log = log.bind(namespace=self._obj_namespace)
+ return log
+
+ def generate_object(self, *args, **kwargs):
+ raise NotImplementedError
+
+ def ensure_object(self, resource, *args, **kwargs):
+ self.logger.debug("Ensuring resource")
+
+ if not resource.exists():
+ self.logger.debug("Resource does not exist, creating")
+ resource.create()
+ else:
+ resource.update()
+
+ self.logger.info("Ensured resource")
+
+ return {
+ self.name: resource,
+ }
+
+ def execute(self, *args, **kwargs):
+ resource = self.generate_object(*args, **kwargs)
+ return self.ensure_object(resource)
+
+
+class CreateOrUpdateNamespaceTask(CreateOrUpdateKubernetesObjectTask):
+ def __init__(self, name: str, *args, **kwargs):
+ super().__init__(
+ pykube.Namespace,
+ None,
+ name,
+ requires=set(["name"]),
+ inject={"name": name},
+ *args,
+ **kwargs,
+ )
+
+ def generate_object(self, name, *args, **kwargs):
+ return pykube.Namespace(
+ self.api,
+ {
+ "apiVersion": "v1",
+ "kind": "Namespace",
+ "metadata": {"name": name},
+ },
+ )
+
+
+class CreateOrUpdateSecretTask(CreateOrUpdateKubernetesObjectTask):
+ def __init__(self, namespace: str, name: str, data: str, *args, **kwargs):
+ super().__init__(
+ pykube.Secret,
+ namespace,
+ name,
+ requires=set(["namespace", "name", "data"]),
+ inject={"name": name, "data": data},
+ *args,
+ **kwargs,
+ )
+
+ def generate_object(self, namespace, name, data, *args, **kwargs):
+ return pykube.Secret(
+ self.api,
{
"apiVersion": "v1",
"kind": "Secret",
"metadata": {
- "name": secret_name,
- "namespace": secret_namespace,
+ "name": name,
+ "namespace": namespace.name,
},
- "data": secret_data,
+ "data": data,
},
)
-
- if not secret.exists():
- log.debug("Secret does not exist, creating")
- secret.create()
-
- secret.reload()
-
- if secret.obj["data"] != secret_data:
- log.info("Secret data has changed, updating")
- secret.obj["data"] = secret_data
- secret.update()
- else:
- log.debug("Secret is up to date")
-
- log.info("Ensured secret")
diff --git a/atmosphere/tasks/openstack_helm.py b/atmosphere/tasks/openstack_helm.py
index e5831bc..ee6c6f3 100644
--- a/atmosphere/tasks/openstack_helm.py
+++ b/atmosphere/tasks/openstack_helm.py
@@ -1,13 +1,13 @@
-from taskflow import task
-
from atmosphere.models.openstack_helm import values
+from atmosphere.tasks import kubernetes
-class GenerateReleaseSecretTask(task.Task):
- default_provides = ("secret_namespace", "secret_name", "secret_data")
-
- def execute(self, chart, *args, **kwargs):
- secret_name = f"atmosphere-{chart}"
- secret_data = values.Values.for_chart(chart).secret_data
-
- return "openstack", secret_name, secret_data
+class CreateOrUpdateReleaseSecretTask(kubernetes.CreateOrUpdateSecretTask):
+ def __init__(self, namespace: str, chart: str, *args, **kwargs):
+ super().__init__(
+ namespace,
+ f"atmosphere-{chart}",
+ values.Values.for_chart(chart).secret_data,
+ *args,
+ **kwargs,
+ )
diff --git a/roles/atmosphere/templates/cluster_role.yml b/roles/atmosphere/templates/cluster_role.yml
index 72f1301..4014eb2 100644
--- a/roles/atmosphere/templates/cluster_role.yml
+++ b/roles/atmosphere/templates/cluster_role.yml
@@ -4,6 +4,9 @@
metadata:
name: atmosphere
rules:
+ - apiGroups: [""]
+ resources: ["namespaces"]
+ verbs: ["get", "create", "patch"]
- apiGroups: ["source.toolkit.fluxcd.io"]
resources: ["helmrepositories"]
verbs: ["get", "create", "patch"]
diff --git a/roles/cert_manager/meta/main.yml b/roles/cert_manager/meta/main.yml
index 61d67be..97154e4 100644
--- a/roles/cert_manager/meta/main.yml
+++ b/roles/cert_manager/meta/main.yml
@@ -22,3 +22,6 @@
- name: Ubuntu
versions:
- focal
+
+dependencies:
+ - role: atmosphere
diff --git a/roles/cert_manager/tasks/main.yml b/roles/cert_manager/tasks/main.yml
index c2ff92d..e6516ab 100644
--- a/roles/cert_manager/tasks/main.yml
+++ b/roles/cert_manager/tasks/main.yml
@@ -12,28 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
-- name: Create namespace
- kubernetes.core.k8s:
- state: present
- definition:
- apiVersion: v1
- kind: Namespace
- metadata:
- name: cert-manager
-
- name: Deploy Helm chart
kubernetes.core.k8s:
state: present
definition:
- - apiVersion: source.toolkit.fluxcd.io/v1beta2
- kind: HelmRepository
- metadata:
- name: jetstack
- namespace: cert-manager
- spec:
- interval: 60s
- url: https://charts.jetstack.io
-
- apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
diff --git a/roles/kube_prometheus_stack/meta/main.yml b/roles/kube_prometheus_stack/meta/main.yml
index 4e2f96b..a6bf0d3 100644
--- a/roles/kube_prometheus_stack/meta/main.yml
+++ b/roles/kube_prometheus_stack/meta/main.yml
@@ -22,3 +22,6 @@
- name: Ubuntu
versions:
- focal
+
+dependencies:
+ - role: atmosphere
diff --git a/roles/kube_prometheus_stack/tasks/main.yml b/roles/kube_prometheus_stack/tasks/main.yml
index ce884a7..1731704 100644
--- a/roles/kube_prometheus_stack/tasks/main.yml
+++ b/roles/kube_prometheus_stack/tasks/main.yml
@@ -54,15 +54,6 @@
kubernetes.core.k8s:
state: present
definition:
- - apiVersion: source.toolkit.fluxcd.io/v1beta2
- kind: HelmRepository
- metadata:
- name: prometheus-community
- namespace: monitoring
- spec:
- interval: 60s
- url: https://prometheus-community.github.io/helm-charts
-
- apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
diff --git a/roles/node_feature_discovery/meta/main.yml b/roles/node_feature_discovery/meta/main.yml
index a14bf41..9bd6201 100644
--- a/roles/node_feature_discovery/meta/main.yml
+++ b/roles/node_feature_discovery/meta/main.yml
@@ -22,3 +22,6 @@
- name: Ubuntu
versions:
- focal
+
+dependencies:
+ - role: atmosphere
diff --git a/roles/node_feature_discovery/tasks/main.yml b/roles/node_feature_discovery/tasks/main.yml
index 39d3c14..c613dfd 100644
--- a/roles/node_feature_discovery/tasks/main.yml
+++ b/roles/node_feature_discovery/tasks/main.yml
@@ -16,15 +16,6 @@
kubernetes.core.k8s:
state: present
definition:
- - apiVersion: source.toolkit.fluxcd.io/v1beta2
- kind: HelmRepository
- metadata:
- name: node-feature-discovery
- namespace: monitoring
- spec:
- interval: 60s
- url: https://kubernetes-sigs.github.io/node-feature-discovery/charts
-
- apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
diff --git a/roles/prometheus_pushgateway/meta/main.yml b/roles/prometheus_pushgateway/meta/main.yml
index db1a3c7..3532b72 100644
--- a/roles/prometheus_pushgateway/meta/main.yml
+++ b/roles/prometheus_pushgateway/meta/main.yml
@@ -22,3 +22,6 @@
- name: Ubuntu
versions:
- focal
+
+dependencies:
+ - role: atmosphere
diff --git a/roles/prometheus_pushgateway/tasks/main.yml b/roles/prometheus_pushgateway/tasks/main.yml
index 20a1b63..84a7b8b 100644
--- a/roles/prometheus_pushgateway/tasks/main.yml
+++ b/roles/prometheus_pushgateway/tasks/main.yml
@@ -16,15 +16,6 @@
kubernetes.core.k8s:
state: present
definition:
- - apiVersion: source.toolkit.fluxcd.io/v1beta2
- kind: HelmRepository
- metadata:
- name: prometheus-community
- namespace: monitoring
- spec:
- interval: 60s
- url: https://prometheus-community.github.io/helm-charts
-
- apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata: