chore: add upgrade job (#1384)

Relate-to: #1068
Relate-to: #1282
Depends-On: #1408
Depends-On: #1409

Reviewed-by: Mohammed Naser <mnaser@vexxhost.com>
Reviewed-by: Rico Lin <ricolin@ricolky.com>
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 9fab7da..f7d3027 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -84,3 +84,24 @@
     voting: false
     vars:
       tox_envlist: molecule-aio-ovn
+
+- job:
+    name: atmosphere-molecule-aio-upgrade-base
+    run: zuul.d/playbooks/upgrade/run-upgrade.yml
+    abstract: true
+    timeout: 10800
+    parent: atmosphere-molecule-aio-full
+    vars:
+      from_branch: "stable/2024.1"
+
+- job:
+    name: atmosphere-molecule-aio-upgrade-openvswitch
+    parent: atmosphere-molecule-aio-upgrade-base
+    vars:
+      network_backend: "openvswitch"
+
+- job:
+    name: atmosphere-molecule-aio-upgrade-ovn
+    parent: atmosphere-molecule-aio-upgrade-base
+    vars:
+      network_backend: "ovn"
diff --git a/zuul.d/playbooks/upgrade/run-upgrade.yml b/zuul.d/playbooks/upgrade/run-upgrade.yml
new file mode 100644
index 0000000..79d0c4b
--- /dev/null
+++ b/zuul.d/playbooks/upgrade/run-upgrade.yml
@@ -0,0 +1,59 @@
+# Copyright (c) 2024 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Upgrade test
+  hosts: all
+  become: true
+  tasks:
+    - name: Checkout to "{{ from_branch }}" branch
+      become: true
+      ansible.builtin.shell: |
+        sudo apt-get purge -y snapd
+        git stash
+        git checkout "{{ from_branch }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Run molecule with exist branch
+      become: true
+      ansible.builtin.shell: |
+        pip install --ignore-installed poetry
+        poetry install
+        pip install -r test-requirements.txt
+        poetry run molecule converge -s aio
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        ATMOSPHERE_NETWORK_BACKEND: "{{ network_backend }}"
+
+    - name: Checkout to "{{ zuul.branch }}" branch
+      become: true
+      ansible.builtin.shell: |
+        git restore .
+        git checkout "{{ zuul.branch }}"
+        git stash pop
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Run molecule to upgrade
+      become: true
+      ansible.builtin.shell: |
+        poetry install
+        pip install -r test-requirements.txt
+        poetry run molecule converge -s aio
+        poetry run molecule verify -s aio
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        ATMOSPHERE_NETWORK_BACKEND: "{{ network_backend }}"
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index d83fbfc..cd2e55c 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -151,3 +151,9 @@
     release:
       jobs:
         - atmosphere-publish-collection
+    periodic:
+      jobs:
+        - atmosphere-molecule-aio-upgrade-openvswitch:
+            dependencies: *molecule_check_dependencies
+        - atmosphere-molecule-aio-upgrade-ovn:
+            dependencies: *molecule_check_dependencies