[stable/2023.1] Improve CI reliability (#1411)

Signed-off-by: Mohammed Naser <mnaser@vexxhost.com>
Co-authored-by: Mohammed Naser <mnaser@vexxhost.com>
diff --git a/build/lint-jobs.py b/build/lint-jobs.py
new file mode 100644
index 0000000..3edfe12
--- /dev/null
+++ b/build/lint-jobs.py
@@ -0,0 +1,42 @@
+import sys
+import glob
+import yaml
+
+
+def main():
+    passed = True
+
+    for file in glob.glob("zuul.d/container-images/*.yaml"):
+        with open(file, "r") as file:
+            configs = yaml.safe_load(file)
+
+        for config in configs:
+            if "job" in config:
+                job = config["job"]
+
+                # Check if build or upload jobs are missing 'atmosphere-buildset-registry' dependency
+                if (
+                    "build-container-image-" in job["name"]
+                    or "upload-container-image-" in job["name"]
+                ):
+                    deps = job.get("dependencies", [])
+                    if not any(
+                        dep.get("name") == "atmosphere-buildset-registry"
+                        for dep in deps
+                    ):
+                        print(
+                            f"Job '{job['name']}' is missing 'atmosphere-buildset-registry' dependency."
+                        )
+                        passed = False
+
+    if passed:
+        print(
+            "All build and upload jobs have 'atmosphere-buildset-registry' dependency."
+        )
+    else:
+        print("Jobs missing 'atmosphere-buildset-registry' dependency.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/roles/barbican/tasks/main.yml b/roles/barbican/tasks/main.yml
index a88ebca..f6f4aaf 100644
--- a/roles/barbican/tasks/main.yml
+++ b/roles/barbican/tasks/main.yml
@@ -55,3 +55,6 @@
   register: _octavia_implied_role_create
   changed_when: _octavia_implied_role_create.rc == 0
   failed_when: _octavia_implied_role_create.rc != 0 and 'Duplicate entry.' not in _octavia_implied_role_create.stderr
+  retries: 10
+  delay: 1
+  until: _octavia_implied_role_create.rc == 0 or 'Duplicate entry.' in _octavia_implied_role_create.stderr
diff --git a/roles/kube_prometheus_stack/tasks/main.yml b/roles/kube_prometheus_stack/tasks/main.yml
index 056d74e..a44ba15 100644
--- a/roles/kube_prometheus_stack/tasks/main.yml
+++ b/roles/kube_prometheus_stack/tasks/main.yml
@@ -12,6 +12,18 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+- name: Wait until Keycloak service is ready
+  kubernetes.core.k8s_info:
+    api_version: apps/v1
+    kind: StatefulSet
+    name: keycloak
+    namespace: auth-system
+  register: kube_prometheus_stack_keycloak_service
+  retries: 120
+  delay: 5
+  until:
+    - kube_prometheus_stack_keycloak_service.resources[0].status.replicas == kube_prometheus_stack_keycloak_service.resources[0].status.readyReplicas # noqa: yaml[line-length]
+
 - name: Create Keycloak realm
   no_log: true
   run_once: true
diff --git a/roles/octavia/tasks/generate_resources.yml b/roles/octavia/tasks/generate_resources.yml
index af7417d..5960aa1 100644
--- a/roles/octavia/tasks/generate_resources.yml
+++ b/roles/octavia/tasks/generate_resources.yml
@@ -95,6 +95,11 @@
   environment:
     OS_CLOUD: atmosphere
   loop: "{{ groups['controllers'] }}"
+  register: _set_binding_for_ports
+  retries: 10
+  delay: 1
+  until: _set_binding_for_ports.rc == 0
+  failed_when: _set_binding_for_ports.rc != 0
 
 - name: Get health manager networking ports
   openstack.cloud.port_info:
diff --git a/roles/octavia/tasks/main.yml b/roles/octavia/tasks/main.yml
index 603e154..d48a53f 100644
--- a/roles/octavia/tasks/main.yml
+++ b/roles/octavia/tasks/main.yml
@@ -126,6 +126,9 @@
   register: _octavia_implied_role_create
   changed_when: _octavia_implied_role_create.rc == 0
   failed_when: _octavia_implied_role_create.rc != 0 and 'Duplicate entry.' not in _octavia_implied_role_create.stderr
+  retries: 10
+  delay: 1
+  until: _octavia_implied_role_create.rc == 0 or 'Duplicate entry.' in _octavia_implied_role_create.stderr
 
 - name: Create Ingress
   ansible.builtin.include_role:
diff --git a/tox.ini b/tox.ini
index 87b669f..10c5db5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,6 +50,7 @@
 commands =
   pre-commit run --all-files --show-diff-on-failure
   python3 {toxinidir}/build/sync-charts.py --check
+  python3 {toxinidir}/build/lint-jobs.py
 
 [testenv:molecule-keycloak]
 commands =
diff --git a/zuul.d/container-images/cluster-api-provider-openstack.yaml b/zuul.d/container-images/cluster-api-provider-openstack.yaml
index a5b5b62..8469a0e 100644
--- a/zuul.d/container-images/cluster-api-provider-openstack.yaml
+++ b/zuul.d/container-images/cluster-api-provider-openstack.yaml
@@ -45,6 +45,9 @@
 - job:
     name: atmosphere-upload-container-image-cluster-api-provider-openstack
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files
 
diff --git a/zuul.d/container-images/kubernetes-entrypoint.yaml b/zuul.d/container-images/kubernetes-entrypoint.yaml
index 0d87c06..26fe4bf 100644
--- a/zuul.d/container-images/kubernetes-entrypoint.yaml
+++ b/zuul.d/container-images/kubernetes-entrypoint.yaml
@@ -45,6 +45,9 @@
 - job:
     name: atmosphere-upload-container-image-kubernetes-entrypoint
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files
 
diff --git a/zuul.d/container-images/openvswitch.yaml b/zuul.d/container-images/openvswitch.yaml
index e167d53..9e6a4ad 100644
--- a/zuul.d/container-images/openvswitch.yaml
+++ b/zuul.d/container-images/openvswitch.yaml
@@ -45,6 +45,9 @@
 - job:
     name: atmosphere-upload-container-image-openvswitch
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files
 
diff --git a/zuul.d/container-images/python-openstackclient.yaml b/zuul.d/container-images/python-openstackclient.yaml
index fcd4a99..f6c4566 100644
--- a/zuul.d/container-images/python-openstackclient.yaml
+++ b/zuul.d/container-images/python-openstackclient.yaml
@@ -60,6 +60,8 @@
     name: atmosphere-upload-container-image-python-openstackclient
     parent: atmosphere-upload-container-image
     dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
       - name: atmosphere-upload-container-image-ubuntu
         soft: true
       - name: atmosphere-upload-container-image-ubuntu-cloud-archive
diff --git a/zuul.d/container-images/ubuntu.yaml b/zuul.d/container-images/ubuntu.yaml
index 28270cd..9c3f633 100644
--- a/zuul.d/container-images/ubuntu.yaml
+++ b/zuul.d/container-images/ubuntu.yaml
@@ -47,6 +47,9 @@
 - job:
     name: atmosphere-upload-container-image-ubuntu
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files