fix(libvirt): create sidecar to rotate creds (#865)
diff --git a/Earthfile b/Earthfile
index b85b9ca..888110c 100644
--- a/Earthfile
+++ b/Earthfile
@@ -1,5 +1,38 @@
VERSION --use-copy-link 0.7
+go.build:
+ FROM golang:1.21
+ WORKDIR /src
+ ARG GOOS=linux
+ ARG GOARCH=amd64
+ ARG VARIANT
+ COPY --dir go.mod go.sum ./
+ RUN go mod download
+
+libvirt-tls-sidecar.build:
+ FROM +go.build
+ ARG GOOS=linux
+ ARG GOARCH=amd64
+ ARG VARIANT
+ COPY --dir cmd internal ./
+ RUN GOARM=${VARIANT#"v"} go build -o main cmd/libvirt-tls-sidecar/main.go
+ SAVE ARTIFACT ./main
+
+libvirt-tls-sidecar.platform-image:
+ ARG TARGETPLATFORM
+ ARG TARGETARCH
+ ARG TARGETVARIANT
+ FROM --platform=$TARGETPLATFORM ./images/base+image
+ COPY \
+ --platform=linux/amd64 \
+ (+libvirt-tls-sidecar.build/main --GOARCH=$TARGETARCH --VARIANT=$TARGETVARIANT) /usr/bin/libvirt-tls-sidecar
+ ENTRYPOINT ["/usr/bin/libvirt-tls-sidecar"]
+ LABEL org.opencontainers.image.source=https://github.com/vexxhost/atmosphere
+ SAVE IMAGE --push ghcr.io/vexxhost/atmosphere/libvirt-tls-sidecar:latest
+
+libvirt-tls-sidecar.image:
+ BUILD --platform=linux/amd64 --platform=linux/arm64 +libvirt-tls-sidecar.platform-image
+
build.wheels:
FROM ./images/builder+image
COPY pyproject.toml poetry.lock ./
diff --git a/charts/libvirt/templates/bin/_libvirt.sh.tpl b/charts/libvirt/templates/bin/_libvirt.sh.tpl
index 0f96c0b..d87a126 100644
--- a/charts/libvirt/templates/bin/_libvirt.sh.tpl
+++ b/charts/libvirt/templates/bin/_libvirt.sh.tpl
@@ -16,31 +16,30 @@
set -ex
-# NOTE(mnaser): This will move the API certificates into the expected location.
-if [ -f /tmp/api.crt ]; then
- mkdir -p /etc/pki/CA /etc/pki/libvirt/private
+wait_for_file() {
+ local file=$1
- cp /tmp/api-ca.crt {{ .Values.conf.libvirt.ca_file }}
- cp /tmp/api-ca.crt /etc/pki/qemu/ca-cert.pem
+ while [ ! -f $file ]; do
+ sleep 1
+ done
+}
- cp /tmp/api.crt {{ .Values.conf.libvirt.cert_file }}
- cp /tmp/api.crt /etc/pki/libvirt/clientcert.pem
- cp /tmp/api.crt /etc/pki/qemu/server-cert.pem
- cp /tmp/api.crt /etc/pki/qemu/client-cert.pem
+wait_for_file {{ .Values.conf.libvirt.ca_file }}
+wait_for_file /etc/pki/qemu/ca-cert.pem
- cp /tmp/api.key {{ .Values.conf.libvirt.key_file }}
- cp /tmp/api.key /etc/pki/libvirt/private/clientkey.pem
- cp /tmp/api.key /etc/pki/qemu/server-key.pem
- cp /tmp/api.key /etc/pki/qemu/client-key.pem
-fi
+wait_for_file {{ .Values.conf.libvirt.cert_file }}
+wait_for_file /etc/pki/libvirt/clientcert.pem
+wait_for_file /etc/pki/qemu/server-cert.pem
+wait_for_file /etc/pki/qemu/client-cert.pem
-# NOTE(mnaser): This will move the VNC certificates into the expected location.
-if [ -f /tmp/vnc.crt ]; then
- mkdir -p /etc/pki/libvirt-vnc
- mv /tmp/vnc.key /etc/pki/libvirt-vnc/server-key.pem
- mv /tmp/vnc.crt /etc/pki/libvirt-vnc/server-cert.pem
- mv /tmp/vnc-ca.crt /etc/pki/libvirt-vnc/ca-cert.pem
-fi
+wait_for_file {{ .Values.conf.libvirt.key_file }}
+wait_for_file /etc/pki/libvirt/private/clientkey.pem
+wait_for_file /etc/pki/qemu/server-key.pem
+wait_for_file /etc/pki/qemu/client-key.pem
+
+wait_for_file /etc/pki/libvirt-vnc/ca-cert.pem
+wait_for_file /etc/pki/libvirt-vnc/server-cert.pem
+wait_for_file /etc/pki/libvirt-vnc/server-key.pem
# TODO: We disable cgroup functionality for cgroup v2, we should fix this in the future
if $(stat -fc %T /sys/fs/cgroup/ | grep -q cgroup2fs); then
diff --git a/charts/libvirt/templates/bin/_wait-for-libvirt.sh.tpl b/charts/libvirt/templates/bin/_wait-for-libvirt.sh.tpl
new file mode 100644
index 0000000..65eca54
--- /dev/null
+++ b/charts/libvirt/templates/bin/_wait-for-libvirt.sh.tpl
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+{{/*
+Copyright (c) 2023 VEXXHOST, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -xe
+
+# NOTE(mnaser): We use this script in the postStart hook of the libvirt
+# container to ensure that the libvirt daemon is running
+# before we start the exporter.
+until virsh list --all; do
+ echo "Waiting for libvirt to be ready..."
+ sleep 1
+done
diff --git a/charts/libvirt/templates/configmap-bin.yaml b/charts/libvirt/templates/configmap-bin.yaml
index 9b74179..99a0c81 100644
--- a/charts/libvirt/templates/configmap-bin.yaml
+++ b/charts/libvirt/templates/configmap-bin.yaml
@@ -26,10 +26,8 @@
{{- end }}
libvirt.sh: |
{{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- if or (eq .Values.conf.libvirt.listen_tls "1") (eq .Values.conf.qemu.vnc_tls "1") }}
- cert-init.sh: |
-{{ tpl .Values.scripts.cert_init_sh . | indent 4 }}
-{{- end }}
+ wait-for-libvirt.sh: |
+{{ tuple "bin/_wait-for-libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- if .Values.conf.ceph.enabled }}
ceph-keyring.sh: |
{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
diff --git a/charts/libvirt/templates/daemonset-libvirt.yaml b/charts/libvirt/templates/daemonset-libvirt.yaml
index 35cd6e1..51b3c1f 100644
--- a/charts/libvirt/templates/daemonset-libvirt.yaml
+++ b/charts/libvirt/templates/daemonset-libvirt.yaml
@@ -32,10 +32,6 @@
{{- $configMapName := index . 1 }}
{{- $serviceAccountName := index . 2 }}
{{- $envAll := index . 3 }}
-{{- $ssl_enabled := false }}
-{{- if eq $envAll.Values.conf.libvirt.listen_tls "1" }}
-{{- $ssl_enabled = true }}
-{{- end }}
{{- with $envAll }}
{{- $mounts_libvirt := .Values.pod.mounts.libvirt.libvirt }}
@@ -60,6 +56,7 @@
labels:
{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
+ kubectl.kubernetes.io/default-container: libvirt
{{- dict "envAll" $envAll "podName" "libvirt-libvirt-default" "containerNames" (list "libvirt") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
@@ -79,80 +76,6 @@
initContainers:
{{ tuple $envAll "pod_dependency" $mounts_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{ dict "envAll" $envAll | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" | indent 8 }}
-{{- if $ssl_enabled }}
- - name: cert-init-api
-{{ tuple $envAll "kubectl" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ dict "envAll" $envAll "application" "libvirt" "container" "cert_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
- command:
- - /tmp/cert-init.sh
- env:
- - name: TYPE
- value: api
- - name: ISSUER_KIND
- value: {{ .Values.issuers.libvirt.kind }}
- - name: ISSUER_NAME
- value: {{ .Values.issuers.libvirt.name }}
- - name: POD_UID
- valueFrom:
- fieldRef:
- fieldPath: metadata.uid
- - name: POD_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: POD_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- volumeMounts:
- - name: pod-tmp
- mountPath: /tmp
- - name: libvirt-bin
- mountPath: /tmp/cert-init.sh
- subPath: cert-init.sh
- readOnly: true
-{{- end }}
-{{- if eq .Values.conf.qemu.vnc_tls "1" }}
- - name: cert-init-vnc
-{{ tuple $envAll "kubectl" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ dict "envAll" $envAll "application" "libvirt" "container" "cert_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
- command:
- - /tmp/cert-init.sh
- env:
- - name: TYPE
- value: vnc
- - name: ISSUER_KIND
- value: {{ .Values.issuers.vencrypt.kind }}
- - name: ISSUER_NAME
- value: {{ .Values.issuers.vencrypt.name }}
- - name: POD_UID
- valueFrom:
- fieldRef:
- fieldPath: metadata.uid
- - name: POD_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: POD_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- volumeMounts:
- - name: pod-tmp
- mountPath: /tmp
- - name: libvirt-bin
- mountPath: /tmp/cert-init.sh
- subPath: cert-init.sh
- readOnly: true
-{{- end }}
{{- if .Values.conf.ceph.enabled }}
{{- if empty .Values.conf.ceph.cinder.keyring }}
- name: ceph-admin-keyring-placement
@@ -205,6 +128,44 @@
readOnly: true
{{- end }}
containers:
+ - name: tls-sidecar
+{{ tuple $envAll "libvirt_tls_sidecar" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.libvirt_tls_sidecar | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "libvirt" "container" "libvirt_tls_sidecar" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+ env:
+ - name: API_ISSUER_KIND
+ value: {{ .Values.issuers.libvirt.kind }}
+ - name: API_ISSUER_NAME
+ value: {{ .Values.issuers.libvirt.name }}
+ - name: VNC_ISSUER_KIND
+ value: {{ .Values.issuers.vencrypt.kind }}
+ - name: VNC_ISSUER_NAME
+ value: {{ .Values.issuers.vencrypt.name }}
+ - name: POD_UID
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.uid
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ volumeMounts:
+ - name: etc-pki-qemu
+ mountPath: /etc/pki/qemu
+ - name: etc-pki-ca
+ mountPath: /etc/pki/CA
+ - name: etc-pki-libvirt
+ mountPath: /etc/pki/libvirt
+ - name: etc-pki-libvirt-vnc
+ mountPath: /etc/pki/libvirt-vnc
- name: libvirt
{{ tuple $envAll "libvirt" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.libvirt | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -231,6 +192,10 @@
command:
- /tmp/libvirt.sh
lifecycle:
+ postStart:
+ exec:
+ command:
+ - /tmp/wait-for-libvirt.sh
preStop:
exec:
command:
@@ -239,16 +204,24 @@
- |-
kill $(cat /var/run/libvirtd.pid)
volumeMounts:
-{{- if $ssl_enabled }}
- name: etc-pki-qemu
mountPath: /etc/pki/qemu
-{{- end }}
+ - name: etc-pki-ca
+ mountPath: /etc/pki/CA
+ - name: etc-pki-libvirt
+ mountPath: /etc/pki/libvirt
+ - name: etc-pki-libvirt-vnc
+ mountPath: /etc/pki/libvirt-vnc
- name: pod-tmp
mountPath: /tmp
- name: libvirt-bin
mountPath: /tmp/libvirt.sh
subPath: libvirt.sh
readOnly: true
+ - name: libvirt-bin
+ mountPath: /tmp/wait-for-libvirt.sh
+ subPath: wait-for-libvirt.sh
+ readOnly: true
- name: libvirt-etc
mountPath: /etc/libvirt/libvirtd.conf
subPath: libvirtd.conf
@@ -328,11 +301,15 @@
{{- end }}
{{- end }}
volumes:
-{{- if $ssl_enabled }}
- name: etc-pki-qemu
hostPath:
path: /etc/pki/qemu
-{{- end }}
+ - name: etc-pki-ca
+ emptyDir: {}
+ - name: etc-pki-libvirt
+ emptyDir: {}
+ - name: etc-pki-libvirt-vnc
+ emptyDir: {}
- name: pod-tmp
emptyDir: {}
- name: libvirt-bin
diff --git a/charts/libvirt/templates/role-cert-manager.yaml b/charts/libvirt/templates/role-cert-manager.yaml
index b830690..c7f7b3c 100644
--- a/charts/libvirt/templates/role-cert-manager.yaml
+++ b/charts/libvirt/templates/role-cert-manager.yaml
@@ -48,7 +48,9 @@
- ""
verbs:
- get
+ - list
- patch
+ - watch
resources:
- secrets
-{{- end -}}
\ No newline at end of file
+{{- end -}}
diff --git a/charts/libvirt/values.yaml b/charts/libvirt/values.yaml
index 207b8fb..4ab2353 100644
--- a/charts/libvirt/values.yaml
+++ b/charts/libvirt/values.yaml
@@ -27,6 +27,7 @@
images:
tags:
libvirt: docker.io/openstackhelm/libvirt:latest-ubuntu_focal
+ libvirt_tls_sidecar: ghcr.io/vexxhost/atmosphere/libvirt-tls-sidecar:latest
libvirt_exporter: vexxhost/libvirtd-exporter:latest
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_18.2.0-1-20231013'
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
@@ -271,55 +272,6 @@
- endpoint: internal
service: local_image_registry
-scripts:
- # Script is included here (vs in bin/) to allow overriding.
- cert_init_sh: |
- #!/bin/bash
- set -x
-
- HOSTNAME_FQDN=$(hostname --fqdn)
-
- # Script to create certs for each libvirt pod based on pod IP (by default).
- cat <<EOF | kubectl apply -f -
- apiVersion: cert-manager.io/v1
- kind: Certificate
- metadata:
- name: ${POD_NAME}-${TYPE}
- namespace: ${POD_NAMESPACE}
- ownerReferences:
- - apiVersion: v1
- kind: Pod
- name: ${POD_NAME}
- uid: ${POD_UID}
- spec:
- secretName: ${POD_NAME}-${TYPE}
- commonName: ${POD_IP}
- usages:
- - client auth
- - server auth
- dnsNames:
- - ${HOSTNAME}
- - ${HOSTNAME_FQDN}
- ipAddresses:
- - ${POD_IP}
- issuerRef:
- kind: ${ISSUER_KIND}
- name: ${ISSUER_NAME}
- EOF
-
- kubectl -n ${POD_NAMESPACE} wait --for=condition=Ready --timeout=300s \
- certificate/${POD_NAME}-${TYPE}
-
- # NOTE(mnaser): cert-manager does not clean-up the secrets when the certificate
- # is deleted, so we should add an owner reference to the secret
- # to ensure that it is cleaned up when the pod is deleted.
- kubectl -n ${POD_NAMESPACE} patch secret ${POD_NAME}-${TYPE} \
- --type=json -p='[{"op": "add", "path": "/metadata/ownerReferences", "value": [{"apiVersion": "v1", "kind": "Pod", "name": "'${POD_NAME}'", "uid": "'${POD_UID}'"}]}]'
-
- kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.crt}' | base64 -d > /tmp/${TYPE}.crt
- kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.key}' | base64 -d > /tmp/${TYPE}.key
- kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/${TYPE}-ca.crt
-
manifests:
configmap_bin: true
configmap_etc: true
diff --git a/cmd/libvirt-tls-sidecar/main.go b/cmd/libvirt-tls-sidecar/main.go
new file mode 100644
index 0000000..c94f5c8
--- /dev/null
+++ b/cmd/libvirt-tls-sidecar/main.go
@@ -0,0 +1,97 @@
+// Copyright (c) 2024 VEXXHOST, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
+ log "github.com/sirupsen/logrus"
+ "k8s.io/client-go/rest"
+
+ "github.com/vexxhost/atmosphere/internal/tls"
+)
+
+const (
+ EnvVarApiIssuerKind = "API_ISSUER_KIND"
+ EnvVarApiIssuerName = "API_ISSUER_NAME"
+ EnvVarVncIssuerKind = "VNC_ISSUER_KIND"
+ EnvVarVncIssuerName = "VNC_ISSUER_NAME"
+)
+
+func main() {
+ config, err := rest.InClusterConfig()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ required := []string{
+ EnvVarApiIssuerKind,
+ EnvVarApiIssuerName,
+ EnvVarVncIssuerKind,
+ EnvVarVncIssuerName,
+ }
+
+ for _, env := range required {
+ if os.Getenv(env) == "" {
+ log.Fatal(fmt.Sprintf("missing required environment variable: %s", env))
+ }
+ }
+
+ ctx := context.Background()
+ go createCertificateSpec(ctx, config, tls.LibvirtCertificateTypeAPI)
+ go createCertificateSpec(ctx, config, tls.LibvirtCertificateTypeVNC)
+
+ <-ctx.Done()
+}
+
+func createCertificateSpec(ctx context.Context, config *rest.Config, certificateType tls.LibvirtCertificateType) {
+ var issuerRef cmmeta.ObjectReference
+ switch certificateType {
+ case tls.LibvirtCertificateTypeAPI:
+ issuerRef = cmmeta.ObjectReference{
+ Kind: os.Getenv(EnvVarApiIssuerKind),
+ Name: os.Getenv(EnvVarApiIssuerName),
+ }
+ case tls.LibvirtCertificateTypeVNC:
+ issuerRef = cmmeta.ObjectReference{
+ Kind: os.Getenv(EnvVarVncIssuerKind),
+ Name: os.Getenv(EnvVarVncIssuerName),
+ }
+ }
+
+ spec := &tls.LibvirtCertificateSpec{
+ Type: certificateType,
+ IssuerRef: issuerRef,
+ }
+
+ manager, err := tls.NewLibvirtManager(config, spec)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ err = manager.Create(ctx)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ log.WithFields(log.Fields{
+ "certificateType": certificateType,
+ }).Info("certificate created")
+
+ go manager.Watch(ctx)
+}
diff --git a/go.mod b/go.mod
index 90397d4..25ff713 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@
toolchain go1.21.4
require (
+ github.com/cert-manager/cert-manager v1.12.1
github.com/erikgeiser/promptkit v0.9.0
github.com/goccy/go-yaml v1.11.2
github.com/google/go-github/v57 v57.0.0
@@ -33,7 +34,6 @@
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
- github.com/cert-manager/cert-manager v1.12.1 // indirect
github.com/charmbracelet/bubbles v0.16.1 // indirect
github.com/charmbracelet/bubbletea v0.24.2 // indirect
github.com/charmbracelet/lipgloss v0.7.1 // indirect
@@ -54,6 +54,7 @@
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
+ github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
@@ -108,10 +109,12 @@
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
+ k8s.io/apiextensions-apiserver v0.28.4 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/controller-runtime v0.15.0 // indirect
+ sigs.k8s.io/gateway-api v0.7.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
diff --git a/go.sum b/go.sum
index bbbd768..e6d8b1f 100644
--- a/go.sum
+++ b/go.sum
@@ -263,10 +263,10 @@
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
-go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
-go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
-go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
+go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -420,6 +420,8 @@
k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU=
sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk=
+sigs.k8s.io/gateway-api v0.7.0 h1:/mG8yyJNBifqvuVLW5gwlI4CQs0NR/5q4BKUlf1bVdY=
+sigs.k8s.io/gateway-api v0.7.0/go.mod h1:Xv0+ZMxX0lu1nSSDIIPEfbVztgNZ+3cfiYrJsa2Ooso=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
diff --git a/internal/net/hostname.go b/internal/net/hostname.go
new file mode 100644
index 0000000..a2a7c96
--- /dev/null
+++ b/internal/net/hostname.go
@@ -0,0 +1,21 @@
+package net
+
+import (
+ "os"
+ "os/exec"
+ "strings"
+)
+
+func Hostname() (string, error) {
+ return os.Hostname()
+}
+
+func FQDN() (string, error) {
+ cmd := exec.Command("/bin/hostname", "--fqdn")
+ out, err := cmd.Output()
+ if err != nil {
+ return "", err
+ }
+
+ return strings.TrimSpace(string(out)), nil
+}
diff --git a/internal/tls/libvirt.go b/internal/tls/libvirt.go
new file mode 100644
index 0000000..cbcac1e
--- /dev/null
+++ b/internal/tls/libvirt.go
@@ -0,0 +1,317 @@
+package tls
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "time"
+
+ cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
+ cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
+ cmclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/typed/certmanager/v1"
+ log "github.com/sirupsen/logrus"
+ "github.com/vexxhost/atmosphere/internal/net"
+ v1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/fields"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/apimachinery/pkg/watch"
+ kubernetes "k8s.io/client-go/kubernetes/typed/core/v1"
+ "k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/cache"
+)
+
+type LibvirtCertificateType string
+
+const (
+ LibvirtCertificateTypeAPI LibvirtCertificateType = "api"
+ LibvirtCertificateTypeVNC LibvirtCertificateType = "vnc"
+)
+
+const (
+ EnvVarPodUID = "POD_UID"
+ EnvVarPodName = "POD_NAME"
+ EnvVarPodNamespace = "POD_NAMESPACE"
+ EnvVarPodIP = "POD_IP"
+)
+
+type LibvirtCertificateSpec struct {
+ Type LibvirtCertificateType
+ IssuerRef cmmeta.ObjectReference
+}
+
+type LibvirtManager struct {
+ logger *log.Entry
+ spec *LibvirtCertificateSpec
+ certificate *cmv1.Certificate
+ certificateName string
+ certificateClient cmclient.CertificateInterface
+ secretClient kubernetes.SecretInterface
+}
+
+func NewLibvirtManager(config *rest.Config, spec *LibvirtCertificateSpec) (*LibvirtManager, error) {
+ required := []string{
+ EnvVarPodName,
+ EnvVarPodNamespace,
+ EnvVarPodUID,
+ EnvVarPodIP,
+ }
+
+ for _, env := range required {
+ if os.Getenv(env) == "" {
+ return nil, fmt.Errorf("missing required environment variable: %s", env)
+ }
+ }
+
+ mgr := &LibvirtManager{}
+
+ hostname, err := net.Hostname()
+ if err != nil {
+ return nil, err
+ }
+
+ fqdn, err := net.FQDN()
+ if err != nil {
+ return nil, err
+ }
+
+ clientset, err := kubernetes.NewForConfig(config)
+ if err != nil {
+ return nil, err
+ }
+
+ cmClient, err := cmclient.NewForConfig(config)
+ if err != nil {
+ return nil, err
+ }
+
+ podUID := types.UID(os.Getenv(EnvVarPodUID))
+ podNamespace := os.Getenv(EnvVarPodNamespace)
+ podName := os.Getenv(EnvVarPodName)
+ podIP := os.Getenv(EnvVarPodIP)
+
+ mgr.spec = spec
+ mgr.secretClient = clientset.Secrets(podNamespace)
+ mgr.certificateClient = cmClient.Certificates(podNamespace)
+ mgr.certificateName = fmt.Sprintf("%s-%s", podName, spec.Type)
+
+ mgr.logger = log.WithFields(log.Fields{
+ "certificateName": mgr.certificateName,
+ "podName": podName,
+ "podNamespace": podNamespace,
+ "podUID": podUID,
+ "podIP": podIP,
+ "hostname": hostname,
+ "fqdn": fqdn,
+ "issuerKind": spec.IssuerRef.Kind,
+ "issuerName": spec.IssuerRef.Name,
+ })
+
+ mgr.certificate = &cmv1.Certificate{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: mgr.certificateName,
+ Namespace: podNamespace,
+ OwnerReferences: []metav1.OwnerReference{
+ {
+ APIVersion: "v1",
+ Kind: "Pod",
+ Name: podName,
+ UID: podUID,
+ },
+ },
+ },
+ Spec: cmv1.CertificateSpec{
+ SecretName: mgr.certificateName,
+ CommonName: podIP,
+ Usages: []cmv1.KeyUsage{
+ cmv1.UsageClientAuth,
+ cmv1.UsageServerAuth,
+ },
+ DNSNames: []string{hostname, fqdn},
+ IPAddresses: []string{podIP},
+ IssuerRef: spec.IssuerRef,
+ },
+ }
+
+ return mgr, nil
+}
+
+func (m *LibvirtManager) Create(ctx context.Context) error {
+ // Create certificate
+ _, err := m.certificateClient.Create(ctx, m.certificate, metav1.CreateOptions{})
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return err
+ }
+
+ m.logger.Info("certificate created")
+
+ // Wait for certificate to become ready
+ err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) {
+ certificate, err := m.certificateClient.Get(ctx, m.certificateName, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+
+ for _, condition := range certificate.Status.Conditions {
+ if condition.Type == cmv1.CertificateConditionReady {
+ if condition.Status == cmmeta.ConditionTrue {
+ return true, nil
+ }
+
+ m.logger.WithFields(log.Fields{
+ "reason": condition.Reason,
+ "message": condition.Message,
+ }).Info("certificate not ready")
+ }
+ }
+
+ return false, nil
+ })
+ if err != nil {
+ return err
+ }
+
+ m.logger.Info("certificate ready")
+
+ // Create patch with ownerReference so the secret is garbage collected
+ patch := []map[string]interface{}{
+ {
+ "op": "add",
+ "path": "/metadata/ownerReferences",
+ "value": m.certificate.OwnerReferences,
+ },
+ }
+ patchBytes, err := json.Marshal(patch)
+ if err != nil {
+ return err
+ }
+
+ m.logger.Info("patching secret")
+
+ // Patch secret with ownerReference
+ _, err = m.secretClient.Patch(ctx, m.certificateName, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+ return err
+}
+
+func (m *LibvirtManager) Watch(ctx context.Context) {
+ for {
+ m.watch(ctx)
+ m.logger.Info("watch closed or disconnected, retrying in 5 seconds")
+
+ time.Sleep(5 * time.Second)
+ }
+}
+
+func (m *LibvirtManager) watch(ctx context.Context) {
+ fieldSelector := fields.OneTermEqualSelector("metadata.name", m.certificateName).String()
+
+ listWatcher := &cache.ListWatch{
+ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+ options.FieldSelector = fieldSelector
+ return m.secretClient.List(ctx, options)
+ },
+ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+ options.FieldSelector = fieldSelector
+ return m.secretClient.Watch(ctx, options)
+ },
+ }
+
+ _, controller := cache.NewInformer(
+ listWatcher,
+ &v1.Secret{},
+ time.Minute,
+ cache.ResourceEventHandlerFuncs{
+ AddFunc: func(obj interface{}) {
+ secret := obj.(*v1.Secret)
+ m.write(secret)
+ },
+ UpdateFunc: func(oldObj, newObj interface{}) {
+ secret := newObj.(*v1.Secret)
+ m.write(secret)
+ },
+ DeleteFunc: func(obj interface{}) {
+ m.logger.Fatal("secret deleted")
+ },
+ },
+ )
+
+ stop := make(chan struct{})
+ defer close(stop)
+ controller.Run(stop)
+}
+
+func (m *LibvirtManager) write(secret *v1.Secret) {
+ switch m.spec.Type {
+ case LibvirtCertificateTypeAPI:
+ m.createDirectory("/etc/pki/libvirt/private")
+ m.writeFile("/etc/pki/CA/cacert.pem", secret.Data["ca.crt"])
+ m.writeFile("/etc/pki/libvirt/servercert.pem", secret.Data["tls.crt"])
+ m.writeFile("/etc/pki/libvirt/private/serverkey.pem", secret.Data["tls.key"])
+ m.writeFile("/etc/pki/libvirt/clientcert.pem", secret.Data["tls.crt"])
+ m.writeFile("/etc/pki/libvirt/private/clientkey.pem", secret.Data["ca.key"])
+ m.createDirectory("/etc/pki/qemu")
+ m.writeFile("/etc/pki/qemu/ca-cert.pem", secret.Data["ca.crt"])
+ m.writeFile("/etc/pki/qemu/server-cert.pem", secret.Data["tls.crt"])
+ m.writeFile("/etc/pki/qemu/server-key.pem", secret.Data["tls.key"])
+ m.writeFile("/etc/pki/qemu/client-cert.pem", secret.Data["tls.crt"])
+ m.writeFile("/etc/pki/qemu/client-key.pem", secret.Data["tls.key"])
+ case LibvirtCertificateTypeVNC:
+ m.createDirectory("/etc/pki/libvirt-vnc")
+ m.writeFile("/etc/pki/libvirt-vnc/ca-cert.pem", secret.Data["ca.crt"])
+ m.writeFile("/etc/pki/libvirt-vnc/server-cert.pem", secret.Data["tls.crt"])
+ m.writeFile("/etc/pki/libvirt-vnc/server-key.pem", secret.Data["tls.key"])
+ }
+}
+
+func (m *LibvirtManager) createDirectory(path string) {
+ if _, err := os.Stat(path); !os.IsNotExist(err) {
+ return
+ }
+
+ m.logger.WithFields(log.Fields{
+ "path": path,
+ }).Info("creating directory")
+
+ err := os.MkdirAll(path, 0755)
+ if err != nil {
+ m.logger.Fatal(err)
+ }
+}
+
+func (m *LibvirtManager) writeFile(path string, data []byte) {
+ log := m.logger.WithFields(log.Fields{
+ "path": path,
+ })
+
+ existingData, err := os.ReadFile(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ log.Info("file does not exist, creating file")
+
+ err = os.WriteFile(path, data, 0644)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ return
+ }
+
+ m.logger.Fatal(err)
+ }
+
+ if bytes.Equal(existingData, data) {
+ return
+ }
+
+ log.Info("file contents changed, updating file")
+
+ err = os.WriteFile(path, data, 0644)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/roles/defaults/vars/main.yml b/roles/defaults/vars/main.yml
index 36b4712..08f2787 100644
--- a/roles/defaults/vars/main.yml
+++ b/roles/defaults/vars/main.yml
@@ -95,6 +95,7 @@
kube_state_metrics: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.9.2@sha256:c30cae7072ffb03f3e7934516db89b3be6c9e5521c0d04d5bbc6e115c9bfa3a7
kubectl: docker.io/bitnami/kubectl:1.27.3@sha256:876cebc2d9272d9eb42c2128c9a08c7e7715dbfe4f2eb2f0b3612df977fdd6b7
libvirt: ghcr.io/vexxhost/atmosphere/libvirtd@sha256:68274a76b635cf78a513e0b9324e49efdc653714bf974161e5940ddfda206114
+ libvirt_tls_sidecar: ghcr.io/vexxhost/atmosphere/libvirt-tls-sidecar:latest@sha256:8b3567c9b7f4c942abf12971bc69c0f4b382955b27139c98247891228f825ae7
libvirt_exporter: docker.io/vexxhost/libvirtd-exporter:latest@sha256:1a0fdf89f80060bfdbb8cf45213295c5d9fb1f7ea7dbfe2b331f0649cc98df8e
local_path_provisioner_helper: docker.io/library/busybox:1.36.0@sha256:9e2bbca079387d7965c3a9cee6d0c53f4f4e63ff7637877a83c4c05f2a666112
local_path_provisioner: docker.io/rancher/local-path-provisioner:v0.0.24@sha256:5bb33992a4ec3034c28b5e0b3c4c2ac35d3613b25b79455eb4b1a95adc82cdc0
diff --git a/roles/libvirt/vars/main.yml b/roles/libvirt/vars/main.yml
index 56ece6a..4de2325 100644
--- a/roles/libvirt/vars/main.yml
+++ b/roles/libvirt/vars/main.yml
@@ -27,7 +27,6 @@
listen_tls: "1"
listen_addr: 0.0.0.0
qemu:
- default_tls_x509_cert_dir: /etc/pki/qemu
default_tls_x509_verify: "1"
vnc_tls: "1"
issuers: