ci: use buildset registry (#1062)

diff --git a/roles/defaults/vars/main.yml b/roles/defaults/vars/main.yml
index 0077e26..da43a0d 100644
--- a/roles/defaults/vars/main.yml
+++ b/roles/defaults/vars/main.yml
@@ -14,8 +14,8 @@
 
 _atmosphere_images:
   alertmanager: quay.io/prometheus/alertmanager:v0.26.0@sha256:361db356b33041437517f1cd298462055580585f26555c317df1a3caf2868552
-  barbican_api: ghcr.io/vexxhost/atmosphere/barbican:2023.2@sha256:cb8f8ecb76051b7006785f45bddbc535ed357696e91c8fb935c6dfe8cafe056b
-  barbican_db_sync: ghcr.io/vexxhost/atmosphere/barbican:2023.2@sha256:cb8f8ecb76051b7006785f45bddbc535ed357696e91c8fb935c6dfe8cafe056b
+  barbican_api: registry.atmosphere.dev/library/barbican:2023.2@sha256:836d31f3d9b88d7da006478b9d0cd79390b5726042e31d27824599c7fe97acc9
+  barbican_db_sync: registry.atmosphere.dev/library/barbican:2023.2@sha256:836d31f3d9b88d7da006478b9d0cd79390b5726042e31d27824599c7fe97acc9
   bootstrap: ghcr.io/vexxhost/atmosphere/heat:2023.2@sha256:9a659c6a058f8c169affc5850a48870be179849f08e3586e3091e566cbc9543a
   ceph_config_helper: ghcr.io/vexxhost/atmosphere/libvirtd:zed@sha256:5f349c9842535c27edbf94be42e4b5c07aa0ff62358cec4b61b1357554e9cf9c
   ceph: quay.io/ceph/ceph:v16.2.11@sha256:1b9803c8984bef8b82f05e233e8fe8ed8f0bba8e5cc2c57f6efaccbeea682add
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index e5cafdc..9a33828 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -13,15 +13,15 @@
 # under the License.
 
 - job:
-    name: atmosphere-build-images
-    pre-run: zuul.d/playbooks/build-images/pre.yml
-    run: zuul.d/playbooks/build-images/run.yml
+    name: atmosphere-buildset-registry
+    pre-run: zuul.d/playbooks/buildset-registry/pre.yml
+    run: zuul.d/playbooks/buildset-registry/run.yml
     ansible-split-streams: true
 
 - job:
     name: atmosphere-upload-images
-    parent: atmosphere-build-images
-    run: zuul.d/playbooks/build-images/run.yml
+    parent: atmosphere-buildset-registry
+    run: zuul.d/playbooks/buildset-registry/run.yml
     secrets:
       - registry_credentials
       - cosign_key
@@ -33,6 +33,8 @@
     pre-run: zuul.d/playbooks/molecule/pre.yml
     run: zuul.d/playbooks/molecule/run.yml
     post-run: zuul.d/playbooks/molecule/post.yml
+    dependencies:
+      - atmosphere-buildset-registry
 
 - job:
     name: atmosphere-molecule-keycloak
diff --git a/zuul.d/playbooks/build-images/run.yml b/zuul.d/playbooks/build-images/run.yml
deleted file mode 100644
index 851154b..0000000
--- a/zuul.d/playbooks/build-images/run.yml
+++ /dev/null
@@ -1,82 +0,0 @@
-# Copyright (c) 2024 VEXXHOST, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-- name: Build images
-  hosts: all
-  tasks:
-    - name: Create builder
-      ansible.builtin.shell: docker buildx create --name=atmosphere --driver=docker-container
-
-    - name: Log into registry
-      when: zuul.pipeline == 'post'
-      docker_login:
-        registry: registry.atmosphere.dev
-        username: "{{ registry_credentials.username }}"
-        password: "{{ registry_credentials.password }}"
-
-    - name: Build images
-      ansible.builtin.shell: |
-        docker buildx bake --builder=atmosphere --provenance --sbom=true {% if zuul.pipeline == 'post' %}--push{% endif %}
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
-      environment:
-        PUSH_TO_CACHE: "{{ zuul.pipeline == 'post' }}"
-
-    - name: Get list of images built
-      ansible.builtin.shell: docker buildx bake --print
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
-      register: images_built_json
-
-    - name: Set fact with list of images
-      set_fact:
-        images_built: "{{ images_built_json.stdout | from_json | json_query('target.*.tags[?@] | []') }}"
-
-    - name: Sign images
-      when: zuul.pipeline == 'post'
-      block:
-        - name: Download cosign binary
-          become: true
-          ansible.builtin.get_url:
-            url: https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64
-            dest: /usr/local/bin/cosign
-            mode: 0755
-
-        - name: Copy the cosign private key
-          copy:
-            content: "{{ cosign_key.private }}"
-            dest: cosign.key
-
-        - name: Sign images
-          ansible.builtin.shell: |
-            cosign sign -y --recursive --key cosign.key {{ item }}
-          loop: "{{ images_built }}"
-
-        - name: Delete the cosign private key
-          file:
-            path: cosign.key
-            state: absent
-
-    - name: Return Zuul artifacts for images
-      zuul_return:
-        data:
-          zuul:
-            artifacts:
-              - name: "{{ item }}"
-                url: "docker://{{ item }}"
-                metadata:
-                  type: container_image
-                  repository: "{{ item.split(':')[0] }}"
-                  tag: "{{ item.split(':')[1] }}"
-      loop: "{{ images_built }}"
diff --git a/zuul.d/playbooks/build-images/pre.yml b/zuul.d/playbooks/buildset-registry/pre.yml
similarity index 91%
rename from zuul.d/playbooks/build-images/pre.yml
rename to zuul.d/playbooks/buildset-registry/pre.yml
index 6b51ca1..81304bb 100644
--- a/zuul.d/playbooks/build-images/pre.yml
+++ b/zuul.d/playbooks/buildset-registry/pre.yml
@@ -16,3 +16,5 @@
   hosts: all
   roles:
     - ensure-docker
+    - run-buildset-registry
+    - use-buildset-registry
diff --git a/zuul.d/playbooks/buildset-registry/run.yml b/zuul.d/playbooks/buildset-registry/run.yml
new file mode 100644
index 0000000..7853f2a
--- /dev/null
+++ b/zuul.d/playbooks/buildset-registry/run.yml
@@ -0,0 +1,141 @@
+# Copyright (c) 2024 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Build images
+  hosts: all
+  tasks:
+    # NOTE(mnaser): This can be removed once the following merges
+    #               https://review.opendev.org/c/zuul/zuul-jobs/+/915025
+    - name: Load "buildset_registry" fact
+      block:
+        - name: Check for results.json
+          stat:
+            path: "{{ zuul.executor.result_data_file }}"
+          register: result_json_stat
+          delegate_to: localhost
+        - name: Load information from zuul_return
+          no_log: true
+          set_fact:
+            buildset_registry: "{{ (lookup('file', zuul.executor.result_data_file) | from_json)['secret_data']['buildset_registry'] }}"
+          when:
+            - buildset_registry is not defined
+            - result_json_stat.stat.exists
+            - result_json_stat.stat.size > 0
+            - "'buildset_registry' in (lookup('file', zuul.executor.result_data_file) | from_json).get('secret_data')"
+
+    - name: Configure Buildkit certificates
+      when: buildset_registry is defined and buildset_registry.cert
+      become: true
+      block:
+        - name: Create a folder for the certificates
+          ansible.builtin.file:
+            path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}"
+            state: directory
+        - name: Copy the certificate
+          ansible.builtin.copy:
+            content: "{{ buildset_registry.cert }}"
+            dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
+        - name: Create a buildkitd.toml file
+          ansible.builtin.copy:
+            dest: /etc/buildkitd.toml
+            content: |
+              [registry."{{ buildset_registry.host }}:{{ buildset_registry.port }}"]
+                ca=["/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"]
+
+    - name: Create builder
+      ansible.builtin.shell: docker buildx create --name=atmosphere --driver=docker-container {% if buildset_registry.cert %}--config /etc/buildkitd.toml{% endif %}
+
+    - name: Point registry to Atmosphere if in post pipeline
+      when: zuul.pipeline == 'post'
+      no_log: true
+      ansible.builtin.set_fact:
+        buildset_registry:
+          host: registry.atmosphere.dev
+          port: 5000
+          username: "{{ registry_credentials.username }}"
+          password: "{{ registry_credentials.password }}"
+
+    - name: Log into registry
+      docker_login:
+        registry: "{{ buildset_registry.host }}:{{ buildset_registry.port }}"
+        username: "{{ buildset_registry.username }}"
+        password: "{{ buildset_registry.password }}"
+
+    - name: Build images
+      ansible.builtin.shell: |
+        docker buildx bake --builder=atmosphere --provenance --sbom=true --push
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        REGISTRY: "{{ buildset_registry.host }}:{{ buildset_registry.port }}/library"
+        PUSH_TO_CACHE: "{{ zuul.pipeline == 'post' }}"
+
+    - name: Get list of images built
+      ansible.builtin.shell: docker buildx bake --print
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        REGISTRY: "{{ buildset_registry.host }}:{{ buildset_registry.port }}/library"
+      register: images_built_json
+
+    - name: Set fact with list of images
+      set_fact:
+        images_built: "{{ images_built_json.stdout | from_json | json_query('target.*.tags[?@] | []') }}"
+
+    - name: Sign images
+      when: zuul.pipeline == 'post'
+      block:
+        - name: Download cosign binary
+          become: true
+          ansible.builtin.get_url:
+            url: https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64
+            dest: /usr/local/bin/cosign
+            mode: 0755
+
+        - name: Copy the cosign private key
+          copy:
+            content: "{{ cosign_key.private }}"
+            dest: cosign.key
+
+        - name: Sign images
+          ansible.builtin.shell: |
+            cosign sign -y --recursive --key cosign.key {{ item }}
+          loop: "{{ images_built }}"
+
+        - name: Delete the cosign private key
+          file:
+            path: cosign.key
+            state: absent
+
+    - name: Return Zuul artifacts for images
+      zuul_return:
+        data:
+          zuul:
+            artifacts:
+              - name: "{{ item }}"
+                url: "docker://{{ item }}"
+                metadata:
+                  type: container_image
+                  repository: "{{ item.split(':')[0] }}"
+                  tag: "{{ item.split(':')[1] }}"
+      loop: "{{ images_built }}"
+
+- name: Yield to other jobs
+  hosts: localhost
+  tasks:
+    - name: Pause the job
+      zuul_return:
+        data:
+          zuul:
+            pause: true
diff --git a/zuul.d/playbooks/molecule/pre.yml b/zuul.d/playbooks/molecule/pre.yml
index ef5bc95..f9f2213 100644
--- a/zuul.d/playbooks/molecule/pre.yml
+++ b/zuul.d/playbooks/molecule/pre.yml
@@ -22,10 +22,46 @@
     # TODO(mnaser): Drop this when we can use https://github.com/vexxhost/atmosphere/pull/977
     - name: Prefix all images for the job to point to mirror
       ansible.builtin.shell: |
-        sed -i 's/  \(.*\): \(.*\)$/  \1: registry.atmosphere.dev\/\2/' roles/defaults/vars/main.yml
+        sed -i '/registry.atmosphere.dev/!s/  \(.*\): \(.*\)$/  \1: registry.atmosphere.dev\/\2/' roles/defaults/vars/main.yml
       args:
         chdir: "{{ zuul.project.src_dir }}"
 
+    # NOTE(mnaser): This can be removed once the following merges
+    #               https://review.opendev.org/c/zuul/zuul-jobs/+/915025
+    - name: Load "buildset_registry" fact
+      block:
+        - name: Check for results.json
+          stat:
+            path: "{{ zuul.executor.result_data_file }}"
+          register: result_json_stat
+          delegate_to: localhost
+        - name: Load information from zuul_return
+          no_log: true
+          set_fact:
+            buildset_registry: "{{ (lookup('file', zuul.executor.result_data_file) | from_json)['secret_data']['buildset_registry'] }}"
+          when:
+            - buildset_registry is not defined
+            - result_json_stat.stat.exists
+            - result_json_stat.stat.size > 0
+            - "'buildset_registry' in (lookup('file', zuul.executor.result_data_file) | from_json).get('secret_data')"
+
+    - name: Configure buildset registry
+      when: buildset_registry is defined
+      block:
+        - name: Install CA certificate for the registry
+          become: true
+          ansible.builtin.copy:
+            content: "{{ buildset_registry.cert }}"
+            dest: /usr/local/share/ca-certificates/registry.crt
+        - name: Update CA certificates
+          become: true
+          ansible.builtin.shell: update-ca-certificates
+        - name: Replace the registry in image manifest
+          ansible.builtin.replace:
+            path: "{{ zuul.project.src_dir }}/roles/defaults/vars/main.yml"
+            regexp: "registry.atmosphere.dev/library/([^@]*)@sha256:[a-fA-F0-9]{64}"
+            replace: '{{ buildset_registry.host }}:{{ buildset_registry.port }}/library/\1'
+
     # TODO(mnaser): Drop this when we move to PBR
     - name: Add current folder to Git's safe directories
       become: true
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 6f3f0a9..55e21fa 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -15,7 +15,7 @@
 - project:
     check:
       jobs:
-        - atmosphere-build-images
+        - atmosphere-buildset-registry
         - atmosphere-molecule-aio-openvswitch
         - atmosphere-molecule-aio-ovn
         - atmosphere-molecule-csi-local-path-provisioner