| From 0e654ba688e7bcc8daeb8c1ce812c5ed743a89ad Mon Sep 17 00:00:00 2001 |
| From: root <okozachenko1203@gmail.com> |
| Date: Fri, 01 Sep 2023 01:10:50 +1000 |
| Subject: [PATCH] Issue certificates for libvirt in initContainer |
| |
| In this patch, certificates are generated in initContainer. |
| It allows us to issue certificates using pod ip per a replica. |
| This change is backward compatible by overriding |
| scripts.cert_init_sh in values. |
| https://review.opendev.org/c/openstack/openstack-helm-infra/+/893406/9 |
| |
| Change-Id: I54d0e57f363fcac07055cc97720505dc1641bbdc |
| --- |
| |
| diff --git a/libvirt/templates/bin/_libvirt.sh.tpl b/libvirt/templates/bin/_libvirt.sh.tpl |
| index 357bfe3..0f96c0b 100644 |
| --- a/libvirt/templates/bin/_libvirt.sh.tpl |
| +++ b/libvirt/templates/bin/_libvirt.sh.tpl |
| @@ -16,6 +16,24 @@ |
| |
| set -ex |
| |
| +# NOTE(mnaser): This will move the API certificates into the expected location. |
| +if [ -f /tmp/api.crt ]; then |
| + mkdir -p /etc/pki/CA /etc/pki/libvirt/private |
| + |
| + cp /tmp/api-ca.crt {{ .Values.conf.libvirt.ca_file }} |
| + cp /tmp/api-ca.crt /etc/pki/qemu/ca-cert.pem |
| + |
| + cp /tmp/api.crt {{ .Values.conf.libvirt.cert_file }} |
| + cp /tmp/api.crt /etc/pki/libvirt/clientcert.pem |
| + cp /tmp/api.crt /etc/pki/qemu/server-cert.pem |
| + cp /tmp/api.crt /etc/pki/qemu/client-cert.pem |
| + |
| + cp /tmp/api.key {{ .Values.conf.libvirt.key_file }} |
| + cp /tmp/api.key /etc/pki/libvirt/private/clientkey.pem |
| + cp /tmp/api.key /etc/pki/qemu/server-key.pem |
| + cp /tmp/api.key /etc/pki/qemu/client-key.pem |
| +fi |
| + |
| # NOTE(mnaser): This will move the VNC certificates into the expected location. |
| if [ -f /tmp/vnc.crt ]; then |
| mkdir -p /etc/pki/libvirt-vnc |
| diff --git a/libvirt/templates/configmap-bin.yaml b/libvirt/templates/configmap-bin.yaml |
| index ca1a7ec..9b74179 100644 |
| --- a/libvirt/templates/configmap-bin.yaml |
| +++ b/libvirt/templates/configmap-bin.yaml |
| @@ -26,9 +26,9 @@ |
| {{- end }} |
| libvirt.sh: | |
| {{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} |
| -{{- if eq .Values.conf.qemu.vnc_tls "1" }} |
| +{{- if or (eq .Values.conf.libvirt.listen_tls "1") (eq .Values.conf.qemu.vnc_tls "1") }} |
| cert-init.sh: | |
| -{{ tpl .Values.conf.vencrypt.cert_init_sh . | indent 4 }} |
| +{{ tpl .Values.scripts.cert_init_sh . | indent 4 }} |
| {{- end }} |
| {{- if .Values.conf.ceph.enabled }} |
| ceph-keyring.sh: | |
| diff --git a/libvirt/templates/daemonset-libvirt.yaml b/libvirt/templates/daemonset-libvirt.yaml |
| index 4a0b128..35cd6e1 100644 |
| --- a/libvirt/templates/daemonset-libvirt.yaml |
| +++ b/libvirt/templates/daemonset-libvirt.yaml |
| @@ -79,6 +79,43 @@ |
| initContainers: |
| {{ tuple $envAll "pod_dependency" $mounts_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} |
| {{ dict "envAll" $envAll | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" | indent 8 }} |
| +{{- if $ssl_enabled }} |
| + - name: cert-init-api |
| +{{ tuple $envAll "kubectl" | include "helm-toolkit.snippets.image" | indent 10 }} |
| +{{ dict "envAll" $envAll "application" "libvirt" "container" "cert_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} |
| + command: |
| + - /tmp/cert-init.sh |
| + env: |
| + - name: TYPE |
| + value: api |
| + - name: ISSUER_KIND |
| + value: {{ .Values.issuers.libvirt.kind }} |
| + - name: ISSUER_NAME |
| + value: {{ .Values.issuers.libvirt.name }} |
| + - name: POD_UID |
| + valueFrom: |
| + fieldRef: |
| + fieldPath: metadata.uid |
| + - name: POD_NAME |
| + valueFrom: |
| + fieldRef: |
| + fieldPath: metadata.name |
| + - name: POD_NAMESPACE |
| + valueFrom: |
| + fieldRef: |
| + fieldPath: metadata.namespace |
| + - name: POD_IP |
| + valueFrom: |
| + fieldRef: |
| + fieldPath: status.podIP |
| + volumeMounts: |
| + - name: pod-tmp |
| + mountPath: /tmp |
| + - name: libvirt-bin |
| + mountPath: /tmp/cert-init.sh |
| + subPath: cert-init.sh |
| + readOnly: true |
| +{{- end }} |
| {{- if eq .Values.conf.qemu.vnc_tls "1" }} |
| - name: cert-init-vnc |
| {{ tuple $envAll "kubectl" | include "helm-toolkit.snippets.image" | indent 10 }} |
| @@ -89,9 +126,9 @@ |
| - name: TYPE |
| value: vnc |
| - name: ISSUER_KIND |
| - value: {{ .Values.conf.vencrypt.issuer.kind }} |
| + value: {{ .Values.issuers.vencrypt.kind }} |
| - name: ISSUER_NAME |
| - value: {{ .Values.conf.vencrypt.issuer.name }} |
| + value: {{ .Values.issuers.vencrypt.name }} |
| - name: POD_UID |
| valueFrom: |
| fieldRef: |
| @@ -202,10 +239,10 @@ |
| - |- |
| kill $(cat /var/run/libvirtd.pid) |
| volumeMounts: |
| - {{ dict "enabled" $ssl_enabled "name" "ssl-client" "path" "/etc/pki/libvirt" "certs" (tuple "clientcert.pem" "clientkey.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} |
| - {{ dict "enabled" $ssl_enabled "name" "ssl-server-cert" "path" "/etc/pki/libvirt" "certs" (tuple "servercert.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} |
| - {{ dict "enabled" $ssl_enabled "name" "ssl-server-key" "path" "/etc/pki/libvirt/private" "certs" (tuple "serverkey.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} |
| - {{ dict "enabled" $ssl_enabled "name" "ssl-ca-cert" "path" "/etc/pki/CA" "certs" (tuple "cacert.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} |
| +{{- if $ssl_enabled }} |
| + - name: etc-pki-qemu |
| + mountPath: /etc/pki/qemu |
| +{{- end }} |
| - name: pod-tmp |
| mountPath: /tmp |
| - name: libvirt-bin |
| @@ -291,10 +328,11 @@ |
| {{- end }} |
| {{- end }} |
| volumes: |
| - {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.client "name" "ssl-client" "path" "/etc/pki/libvirt" "certs" (tuple "clientcert.pem" "clientkey.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} |
| - {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-server-cert" "path" "/etc/pki/libvirt" "certs" (tuple "servercert.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} |
| - {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-server-key" "path" "/etc/pki/libvirt/private" "certs" (tuple "serverkey.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} |
| - {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-ca-cert" "path" "/etc/pki/CA" "certs" (tuple "cacert.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} |
| +{{- if $ssl_enabled }} |
| + - name: etc-pki-qemu |
| + hostPath: |
| + path: /etc/pki/qemu |
| +{{- end }} |
| - name: pod-tmp |
| emptyDir: {} |
| - name: libvirt-bin |
| diff --git a/libvirt/values.yaml b/libvirt/values.yaml |
| index 0821d9c..207b8fb 100644 |
| --- a/libvirt/values.yaml |
| +++ b/libvirt/values.yaml |
| @@ -90,6 +90,17 @@ |
| configmap: ceph-etc |
| user_secret_name: pvc-ceph-client-key |
| |
| +# Issuers for TLS certificates |
| +issuers: |
| + # Issuer to issue a certificate for libvirt api when listen_tls is enabled |
| + libvirt: |
| + kind: ClusterIssuer |
| + name: ca-clusterissuer |
| + # Issuer to issue a certificate for vencrypt |
| + vencrypt: |
| + kind: ClusterIssuer |
| + name: ca-clusterissuer |
| + |
| conf: |
| ceph: |
| enabled: true |
| @@ -121,61 +132,9 @@ |
| stdio_handler: "file" |
| user: "nova" |
| group: "kvm" |
| + default_tls_x509_cert_dir: /etc/pki/qemu |
| kubernetes: |
| cgroup: "kubepods.slice" |
| - vencrypt: |
| - # Issuer to use for the vencrypt certs. |
| - issuer: |
| - kind: ClusterIssuer |
| - name: ca-clusterissuer |
| - # Script is included here (vs in bin/) to allow overriding, in the case that |
| - # communication happens over an IP other than the pod IP for some reason. |
| - cert_init_sh: | |
| - #!/bin/bash |
| - set -x |
| - |
| - HOSTNAME_FQDN=$(hostname --fqdn) |
| - |
| - # Script to create certs for each libvirt pod based on pod IP (by default). |
| - cat <<EOF | kubectl apply -f - |
| - apiVersion: cert-manager.io/v1 |
| - kind: Certificate |
| - metadata: |
| - name: ${POD_NAME}-${TYPE} |
| - namespace: ${POD_NAMESPACE} |
| - ownerReferences: |
| - - apiVersion: v1 |
| - kind: Pod |
| - name: ${POD_NAME} |
| - uid: ${POD_UID} |
| - spec: |
| - secretName: ${POD_NAME}-${TYPE} |
| - commonName: ${POD_IP} |
| - usages: |
| - - client auth |
| - - server auth |
| - dnsNames: |
| - - ${HOSTNAME} |
| - - ${HOSTNAME_FQDN} |
| - ipAddresses: |
| - - ${POD_IP} |
| - issuerRef: |
| - kind: ${ISSUER_KIND} |
| - name: ${ISSUER_NAME} |
| - EOF |
| - |
| - kubectl -n ${POD_NAMESPACE} wait --for=condition=Ready --timeout=300s \ |
| - certificate/${POD_NAME}-${TYPE} |
| - |
| - # NOTE(mnaser): cert-manager does not clean-up the secrets when the certificate |
| - # is deleted, so we should add an owner reference to the secret |
| - # to ensure that it is cleaned up when the pod is deleted. |
| - kubectl -n ${POD_NAMESPACE} patch secret ${POD_NAME}-${TYPE} \ |
| - --type=json -p='[{"op": "add", "path": "/metadata/ownerReferences", "value": [{"apiVersion": "v1", "kind": "Pod", "name": "'${POD_NAME}'", "uid": "'${POD_UID}'"}]}]' |
| - |
| - kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.crt}' | base64 -d > /tmp/${TYPE}.crt |
| - kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.key}' | base64 -d > /tmp/${TYPE}.key |
| - kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/${TYPE}-ca.crt |
| |
| pod: |
| probes: |
| @@ -312,6 +271,55 @@ |
| - endpoint: internal |
| service: local_image_registry |
| |
| +scripts: |
| + # Script is included here (vs in bin/) to allow overriding. |
| + cert_init_sh: | |
| + #!/bin/bash |
| + set -x |
| + |
| + HOSTNAME_FQDN=$(hostname --fqdn) |
| + |
| + # Script to create certs for each libvirt pod based on pod IP (by default). |
| + cat <<EOF | kubectl apply -f - |
| + apiVersion: cert-manager.io/v1 |
| + kind: Certificate |
| + metadata: |
| + name: ${POD_NAME}-${TYPE} |
| + namespace: ${POD_NAMESPACE} |
| + ownerReferences: |
| + - apiVersion: v1 |
| + kind: Pod |
| + name: ${POD_NAME} |
| + uid: ${POD_UID} |
| + spec: |
| + secretName: ${POD_NAME}-${TYPE} |
| + commonName: ${POD_IP} |
| + usages: |
| + - client auth |
| + - server auth |
| + dnsNames: |
| + - ${HOSTNAME} |
| + - ${HOSTNAME_FQDN} |
| + ipAddresses: |
| + - ${POD_IP} |
| + issuerRef: |
| + kind: ${ISSUER_KIND} |
| + name: ${ISSUER_NAME} |
| + EOF |
| + |
| + kubectl -n ${POD_NAMESPACE} wait --for=condition=Ready --timeout=300s \ |
| + certificate/${POD_NAME}-${TYPE} |
| + |
| + # NOTE(mnaser): cert-manager does not clean-up the secrets when the certificate |
| + # is deleted, so we should add an owner reference to the secret |
| + # to ensure that it is cleaned up when the pod is deleted. |
| + kubectl -n ${POD_NAMESPACE} patch secret ${POD_NAME}-${TYPE} \ |
| + --type=json -p='[{"op": "add", "path": "/metadata/ownerReferences", "value": [{"apiVersion": "v1", "kind": "Pod", "name": "'${POD_NAME}'", "uid": "'${POD_UID}'"}]}]' |
| + |
| + kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.crt}' | base64 -d > /tmp/${TYPE}.crt |
| + kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.key}' | base64 -d > /tmp/${TYPE}.key |
| + kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/${TYPE}-ca.crt |
| + |
| manifests: |
| configmap_bin: true |
| configmap_etc: true |
| @@ -327,4 +335,5 @@ |
| tls: |
| server: libvirt-tls-server |
| client: libvirt-tls-client |
| + |
| ... |