blob: c0f5f5414883a7ba8ca5d75cd1f52f9d0d0fde54 [file] [log] [blame]
From 0e654ba688e7bcc8daeb8c1ce812c5ed743a89ad Mon Sep 17 00:00:00 2001
From: root <okozachenko1203@gmail.com>
Date: Fri, 01 Sep 2023 01:10:50 +1000
Subject: [PATCH] Issue certificates for libvirt in initContainer
In this patch, certificates are generated in initContainer.
It allows us to issue certificates using pod ip per a replica.
This change is backward compatible by overriding
scripts.cert_init_sh in values.
https://review.opendev.org/c/openstack/openstack-helm-infra/+/893406/9
Change-Id: I54d0e57f363fcac07055cc97720505dc1641bbdc
---
diff --git a/libvirt/templates/bin/_libvirt.sh.tpl b/libvirt/templates/bin/_libvirt.sh.tpl
index 357bfe3..0f96c0b 100644
--- a/libvirt/templates/bin/_libvirt.sh.tpl
+++ b/libvirt/templates/bin/_libvirt.sh.tpl
@@ -16,6 +16,24 @@
set -ex
+# NOTE(mnaser): This will move the API certificates into the expected location.
+if [ -f /tmp/api.crt ]; then
+ mkdir -p /etc/pki/CA /etc/pki/libvirt/private
+
+ cp /tmp/api-ca.crt {{ .Values.conf.libvirt.ca_file }}
+ cp /tmp/api-ca.crt /etc/pki/qemu/ca-cert.pem
+
+ cp /tmp/api.crt {{ .Values.conf.libvirt.cert_file }}
+ cp /tmp/api.crt /etc/pki/libvirt/clientcert.pem
+ cp /tmp/api.crt /etc/pki/qemu/server-cert.pem
+ cp /tmp/api.crt /etc/pki/qemu/client-cert.pem
+
+ cp /tmp/api.key {{ .Values.conf.libvirt.key_file }}
+ cp /tmp/api.key /etc/pki/libvirt/private/clientkey.pem
+ cp /tmp/api.key /etc/pki/qemu/server-key.pem
+ cp /tmp/api.key /etc/pki/qemu/client-key.pem
+fi
+
# NOTE(mnaser): This will move the VNC certificates into the expected location.
if [ -f /tmp/vnc.crt ]; then
mkdir -p /etc/pki/libvirt-vnc
diff --git a/libvirt/templates/configmap-bin.yaml b/libvirt/templates/configmap-bin.yaml
index ca1a7ec..9b74179 100644
--- a/libvirt/templates/configmap-bin.yaml
+++ b/libvirt/templates/configmap-bin.yaml
@@ -26,9 +26,9 @@
{{- end }}
libvirt.sh: |
{{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- if eq .Values.conf.qemu.vnc_tls "1" }}
+{{- if or (eq .Values.conf.libvirt.listen_tls "1") (eq .Values.conf.qemu.vnc_tls "1") }}
cert-init.sh: |
-{{ tpl .Values.conf.vencrypt.cert_init_sh . | indent 4 }}
+{{ tpl .Values.scripts.cert_init_sh . | indent 4 }}
{{- end }}
{{- if .Values.conf.ceph.enabled }}
ceph-keyring.sh: |
diff --git a/libvirt/templates/daemonset-libvirt.yaml b/libvirt/templates/daemonset-libvirt.yaml
index 4a0b128..35cd6e1 100644
--- a/libvirt/templates/daemonset-libvirt.yaml
+++ b/libvirt/templates/daemonset-libvirt.yaml
@@ -79,6 +79,43 @@
initContainers:
{{ tuple $envAll "pod_dependency" $mounts_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{ dict "envAll" $envAll | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" | indent 8 }}
+{{- if $ssl_enabled }}
+ - name: cert-init-api
+{{ tuple $envAll "kubectl" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ dict "envAll" $envAll "application" "libvirt" "container" "cert_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+ command:
+ - /tmp/cert-init.sh
+ env:
+ - name: TYPE
+ value: api
+ - name: ISSUER_KIND
+ value: {{ .Values.issuers.libvirt.kind }}
+ - name: ISSUER_NAME
+ value: {{ .Values.issuers.libvirt.name }}
+ - name: POD_UID
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.uid
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ volumeMounts:
+ - name: pod-tmp
+ mountPath: /tmp
+ - name: libvirt-bin
+ mountPath: /tmp/cert-init.sh
+ subPath: cert-init.sh
+ readOnly: true
+{{- end }}
{{- if eq .Values.conf.qemu.vnc_tls "1" }}
- name: cert-init-vnc
{{ tuple $envAll "kubectl" | include "helm-toolkit.snippets.image" | indent 10 }}
@@ -89,9 +126,9 @@
- name: TYPE
value: vnc
- name: ISSUER_KIND
- value: {{ .Values.conf.vencrypt.issuer.kind }}
+ value: {{ .Values.issuers.vencrypt.kind }}
- name: ISSUER_NAME
- value: {{ .Values.conf.vencrypt.issuer.name }}
+ value: {{ .Values.issuers.vencrypt.name }}
- name: POD_UID
valueFrom:
fieldRef:
@@ -202,10 +239,10 @@
- |-
kill $(cat /var/run/libvirtd.pid)
volumeMounts:
- {{ dict "enabled" $ssl_enabled "name" "ssl-client" "path" "/etc/pki/libvirt" "certs" (tuple "clientcert.pem" "clientkey.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
- {{ dict "enabled" $ssl_enabled "name" "ssl-server-cert" "path" "/etc/pki/libvirt" "certs" (tuple "servercert.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
- {{ dict "enabled" $ssl_enabled "name" "ssl-server-key" "path" "/etc/pki/libvirt/private" "certs" (tuple "serverkey.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
- {{ dict "enabled" $ssl_enabled "name" "ssl-ca-cert" "path" "/etc/pki/CA" "certs" (tuple "cacert.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- if $ssl_enabled }}
+ - name: etc-pki-qemu
+ mountPath: /etc/pki/qemu
+{{- end }}
- name: pod-tmp
mountPath: /tmp
- name: libvirt-bin
@@ -291,10 +328,11 @@
{{- end }}
{{- end }}
volumes:
- {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.client "name" "ssl-client" "path" "/etc/pki/libvirt" "certs" (tuple "clientcert.pem" "clientkey.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
- {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-server-cert" "path" "/etc/pki/libvirt" "certs" (tuple "servercert.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
- {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-server-key" "path" "/etc/pki/libvirt/private" "certs" (tuple "serverkey.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
- {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-ca-cert" "path" "/etc/pki/CA" "certs" (tuple "cacert.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- if $ssl_enabled }}
+ - name: etc-pki-qemu
+ hostPath:
+ path: /etc/pki/qemu
+{{- end }}
- name: pod-tmp
emptyDir: {}
- name: libvirt-bin
diff --git a/libvirt/values.yaml b/libvirt/values.yaml
index 0821d9c..207b8fb 100644
--- a/libvirt/values.yaml
+++ b/libvirt/values.yaml
@@ -90,6 +90,17 @@
configmap: ceph-etc
user_secret_name: pvc-ceph-client-key
+# Issuers for TLS certificates
+issuers:
+ # Issuer to issue a certificate for libvirt api when listen_tls is enabled
+ libvirt:
+ kind: ClusterIssuer
+ name: ca-clusterissuer
+ # Issuer to issue a certificate for vencrypt
+ vencrypt:
+ kind: ClusterIssuer
+ name: ca-clusterissuer
+
conf:
ceph:
enabled: true
@@ -121,61 +132,9 @@
stdio_handler: "file"
user: "nova"
group: "kvm"
+ default_tls_x509_cert_dir: /etc/pki/qemu
kubernetes:
cgroup: "kubepods.slice"
- vencrypt:
- # Issuer to use for the vencrypt certs.
- issuer:
- kind: ClusterIssuer
- name: ca-clusterissuer
- # Script is included here (vs in bin/) to allow overriding, in the case that
- # communication happens over an IP other than the pod IP for some reason.
- cert_init_sh: |
- #!/bin/bash
- set -x
-
- HOSTNAME_FQDN=$(hostname --fqdn)
-
- # Script to create certs for each libvirt pod based on pod IP (by default).
- cat <<EOF | kubectl apply -f -
- apiVersion: cert-manager.io/v1
- kind: Certificate
- metadata:
- name: ${POD_NAME}-${TYPE}
- namespace: ${POD_NAMESPACE}
- ownerReferences:
- - apiVersion: v1
- kind: Pod
- name: ${POD_NAME}
- uid: ${POD_UID}
- spec:
- secretName: ${POD_NAME}-${TYPE}
- commonName: ${POD_IP}
- usages:
- - client auth
- - server auth
- dnsNames:
- - ${HOSTNAME}
- - ${HOSTNAME_FQDN}
- ipAddresses:
- - ${POD_IP}
- issuerRef:
- kind: ${ISSUER_KIND}
- name: ${ISSUER_NAME}
- EOF
-
- kubectl -n ${POD_NAMESPACE} wait --for=condition=Ready --timeout=300s \
- certificate/${POD_NAME}-${TYPE}
-
- # NOTE(mnaser): cert-manager does not clean-up the secrets when the certificate
- # is deleted, so we should add an owner reference to the secret
- # to ensure that it is cleaned up when the pod is deleted.
- kubectl -n ${POD_NAMESPACE} patch secret ${POD_NAME}-${TYPE} \
- --type=json -p='[{"op": "add", "path": "/metadata/ownerReferences", "value": [{"apiVersion": "v1", "kind": "Pod", "name": "'${POD_NAME}'", "uid": "'${POD_UID}'"}]}]'
-
- kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.crt}' | base64 -d > /tmp/${TYPE}.crt
- kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.key}' | base64 -d > /tmp/${TYPE}.key
- kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/${TYPE}-ca.crt
pod:
probes:
@@ -312,6 +271,55 @@
- endpoint: internal
service: local_image_registry
+scripts:
+ # Script is included here (vs in bin/) to allow overriding.
+ cert_init_sh: |
+ #!/bin/bash
+ set -x
+
+ HOSTNAME_FQDN=$(hostname --fqdn)
+
+ # Script to create certs for each libvirt pod based on pod IP (by default).
+ cat <<EOF | kubectl apply -f -
+ apiVersion: cert-manager.io/v1
+ kind: Certificate
+ metadata:
+ name: ${POD_NAME}-${TYPE}
+ namespace: ${POD_NAMESPACE}
+ ownerReferences:
+ - apiVersion: v1
+ kind: Pod
+ name: ${POD_NAME}
+ uid: ${POD_UID}
+ spec:
+ secretName: ${POD_NAME}-${TYPE}
+ commonName: ${POD_IP}
+ usages:
+ - client auth
+ - server auth
+ dnsNames:
+ - ${HOSTNAME}
+ - ${HOSTNAME_FQDN}
+ ipAddresses:
+ - ${POD_IP}
+ issuerRef:
+ kind: ${ISSUER_KIND}
+ name: ${ISSUER_NAME}
+ EOF
+
+ kubectl -n ${POD_NAMESPACE} wait --for=condition=Ready --timeout=300s \
+ certificate/${POD_NAME}-${TYPE}
+
+ # NOTE(mnaser): cert-manager does not clean-up the secrets when the certificate
+ # is deleted, so we should add an owner reference to the secret
+ # to ensure that it is cleaned up when the pod is deleted.
+ kubectl -n ${POD_NAMESPACE} patch secret ${POD_NAME}-${TYPE} \
+ --type=json -p='[{"op": "add", "path": "/metadata/ownerReferences", "value": [{"apiVersion": "v1", "kind": "Pod", "name": "'${POD_NAME}'", "uid": "'${POD_UID}'"}]}]'
+
+ kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.crt}' | base64 -d > /tmp/${TYPE}.crt
+ kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.key}' | base64 -d > /tmp/${TYPE}.key
+ kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/${TYPE}-ca.crt
+
manifests:
configmap_bin: true
configmap_etc: true
@@ -327,4 +335,5 @@
tls:
server: libvirt-tls-server
client: libvirt-tls-client
+
...