test: add aio tests (#318)
* test: add aio tests
* ci: fix runs-on node
* ci: add tmate
* Tmate after converge
* Run destroy always
* Disable ipv6 in runner
* Fix permission for ipv6 disable
* disable ipv6 permanently
* Fix sysctl configuration
* Fix typo
* ci: test disable_ipv6
* ci: use focal node
* ci: add aio inside gha
* wip: aio
* ci: fix become for prepare
* ci: add ceph bootstrap
* ci: add ceph backend
* ci: fix ceph + use bigger node
* ci: fix ci for aio
* ci: increase number of cores
* Set virt_type as qemu
* Fix nic name in auto bridge add
* Disable auto bridge add in molecule aio
* ci: run full suite with eth1
* ci: fix molecule command
* ci: clean-up more values
* ci: use correct ip
* ci: use correct keepalived interface
* Fix glance_images
* fix ceph_mon_config_overrides keys and set mon_max_pg_per_osd as 500
* Enable fact gathering in tempest playbook
* Use other address for ceph_public_network and reduce image upload
- 10.0.0.0/22 is confused with cilium network. So mon ip is catched from cilium host nic in idempotence.
* fix(ceph-provisioners): stop depending on ceph-csi-rbd
* fix: allow for hci label adding
* fix: add ceph scenario
* ci: enable nested virt
* ci: final cleanup
---------
Co-authored-by: okozachenko1203 <okozachenko1203@users.noreply.github.com>
diff --git a/molecule/ceph/cleanup.yml b/molecule/ceph/cleanup.yml
new file mode 100644
index 0000000..f9359ab
--- /dev/null
+++ b/molecule/ceph/cleanup.yml
@@ -0,0 +1,39 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- ansible.builtin.import_playbook: vexxhost.ceph.destroy_fake_devices
+
+- hosts: localhost
+ connection: local
+ gather_facts: false
+ no_log: "{{ molecule_no_log }}"
+ vars:
+ workspace_path: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}"
+ tasks:
+ - name: Capture var files to delete
+ find:
+ paths:
+ - "{{ workspace_path }}/group_vars"
+ - "{{ workspace_path }}/host_vars"
+ file_type: file
+ recurse: true
+ excludes:
+ - "molecule.yml"
+ register: _var_files
+
+ - name: Delete var files
+ file:
+ path: "{{ item.path }}"
+ state: absent
+ with_items: "{{ _var_files['files'] }}"
diff --git a/molecule/ceph/converge.yml b/molecule/ceph/converge.yml
new file mode 100644
index 0000000..fc75f20
--- /dev/null
+++ b/molecule/ceph/converge.yml
@@ -0,0 +1,65 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- ansible.builtin.import_playbook: vexxhost.ceph.site
+- ansible.builtin.import_playbook: vexxhost.atmosphere.kubernetes
+
+# NOTE(mnaser): When using Docker with custom networks, it will use 127.0.0.11
+# as the DNS server which trips up the CoreDNS "loop" plugin.
+- name: Switch CoreDNS to use CloudFlare DNS
+ hosts: controllers[0]
+ become: true
+ tasks:
+ - name: Update CoreDNS ConfigMap
+ kubernetes.core.k8s:
+ state: present
+ definition:
+ apiVersion: v1
+ kind: ConfigMap
+ metadata:
+ name: coredns
+ namespace: kube-system
+ data:
+ Corefile: |
+ .:53 {
+ errors
+ health {
+ lameduck 5s
+ }
+ ready
+ kubernetes cluster.local in-addr.arpa ip6.arpa {
+ pods insecure
+ fallthrough in-addr.arpa ip6.arpa
+ ttl 30
+ }
+ prometheus :9153
+ forward . 1.1.1.1 {
+ max_concurrent 1000
+ }
+ cache 30
+ loop
+ reload
+ loadbalance
+ }
+ notify:
+ - Rollout CoreDNS
+ - Wait for CoreDNS to be ready
+ handlers:
+ - name: Rollout CoreDNS
+ command: kubectl -n kube-system rollout restart deploy/coredns
+ - name: Wait for CoreDNS to be ready
+ command: kubectl -n kube-system rollout status deploy/coredns
+
+- ansible.builtin.import_playbook: vexxhost.atmosphere.csi
+- ansible.builtin.import_playbook: vexxhost.atmosphere.openstack
diff --git a/molecule/ceph/group_vars/.gitkeep b/molecule/ceph/group_vars/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/molecule/ceph/group_vars/.gitkeep
diff --git a/molecule/ceph/group_vars/all/molecule.yml b/molecule/ceph/group_vars/all/molecule.yml
new file mode 100644
index 0000000..8bed0fc
--- /dev/null
+++ b/molecule/ceph/group_vars/all/molecule.yml
@@ -0,0 +1,115 @@
+ceph_conf_overrides:
+ - section: global
+ option: osd crush chooseleaf type
+ value: 0
+ - section: mon
+ option: auth allow insecure global id reclaim
+ value: false
+
+kubernetes_keepalived_interface: eth0
+
+cilium_helm_values:
+ operator:
+ replicas: 1
+
+csi_driver: local-path-provisioner
+
+cluster_issuer_type: self-signed
+
+ingress_nginx_helm_values:
+ controller:
+ config:
+ worker-processes: 2
+
+percona_xtradb_cluster_spec:
+ allowUnsafeConfigurations: true
+ pxc:
+ size: 1
+ haproxy:
+ size: 1
+
+keystone_helm_values:
+ pod:
+ replicas:
+ api: 1
+
+barbican_helm_values:
+ pod:
+ replicas:
+ api: 1
+
+rook_ceph_cluster_radosgw_spec:
+ metadataPool:
+ failureDomain: osd
+ dataPool:
+ failureDomain: osd
+ gateway:
+ instances: 1
+
+glance_helm_values:
+ conf:
+ glance:
+ DEFAULT:
+ workers: 2
+ pod:
+ replicas:
+ api: 1
+glance_images:
+ - name: cirros
+ url: http://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img
+ min_disk: 1
+ disk_format: raw
+ container_format: bare
+ is_public: true
+
+cinder_helm_values:
+ pod:
+ replicas:
+ api: 1
+ scheduler: 1
+
+placement_helm_values:
+ pod:
+ replicas:
+ api: 1
+
+coredns_helm_values:
+ replicaCount: 1
+
+nova_helm_values:
+ conf:
+ nova:
+ DEFAULT:
+ osapi_compute_workers: 2
+ metadata_workers: 2
+ conductor:
+ workers: 2
+ scheduler:
+ workers: 2
+ pod:
+ replicas:
+ api_metadata: 1
+ osapi: 1
+ conductor: 1
+ scheduler: 1
+ novncproxy: 1
+ spiceproxy: 1
+
+neutron_helm_values:
+ conf:
+ auto_bridge_add:
+ br-ex: eth1
+ neutron:
+ DEFAULT:
+ api_workers: 2
+ rpc_workers: 2
+ metadata_workers: 2
+ pod:
+ replicas:
+ server: 1
+
+tempest_helm_values:
+ conf:
+ tempest:
+ service_available:
+ horizon: false
diff --git a/molecule/ceph/host_vars/.gitkeep b/molecule/ceph/host_vars/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/molecule/ceph/host_vars/.gitkeep
diff --git a/molecule/ceph/molecule.yml b/molecule/ceph/molecule.yml
new file mode 100644
index 0000000..2cd63c6
--- /dev/null
+++ b/molecule/ceph/molecule.yml
@@ -0,0 +1,84 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+dependency:
+ name: galaxy
+driver:
+ name: docker
+platforms:
+ - name: instance
+ image: geerlingguy/docker-${MOLECULE_DISTRO:-ubuntu2004}-ansible:latest
+ command: ${MOLECULE_DOCKER_COMMAND:-""}
+ privileged: true
+ cgroupns_mode: host
+ pre_build_image: true
+ purge_networks: true
+ dns_servers:
+ - 1.1.1.1
+ docker_networks:
+ - name: mgmt
+ ipam_config:
+ - subnet: 10.96.240.0/24
+ gateway: 10.96.240.1
+ - name: public
+ ipam_config:
+ - subnet: 10.96.250.0/24
+ gateway: 10.96.250.1
+ networks:
+ - name: mgmt
+ - name: public
+ security_opts:
+ - apparmor=unconfined
+ volumes:
+ - /dev:/dev
+ - /lib/modules:/lib/modules:ro
+ - /sys/fs/cgroup:/sys/fs/cgroup:rw
+ - /usr/src:/usr/src:ro
+ groups:
+ - controllers
+ - cephs
+ - computes
+provisioner:
+ name: ansible
+ config_options:
+ connection:
+ pipelining: true
+ tags:
+ skip: >-
+ sysctl,
+ ethtool,
+ node-feature-discovery,
+ kube-prometheus-stack,
+ loki,
+ vector,
+ prometheus-ethtool-exporter,
+ ipmi-exporter,
+ prometheus-pushgateway,
+ lpfc,
+ senlin,
+ designate,
+ heat,
+ octavia,
+ magnum,
+ manila,
+ horizon,
+ openstack-exporter
+ options:
+ inventory: "${MOLECULE_EPHEMERAL_DIRECTORY}/workspace"
+ inventory:
+ links:
+ host_vars: "${MOLECULE_SCENARIO_DIRECTORY}/host_vars"
+ group_vars: "${MOLECULE_SCENARIO_DIRECTORY}/group_vars"
+verifier:
+ name: ansible
diff --git a/molecule/ceph/prepare.yml b/molecule/ceph/prepare.yml
new file mode 100644
index 0000000..f3337d9
--- /dev/null
+++ b/molecule/ceph/prepare.yml
@@ -0,0 +1,75 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- import_playbook: vexxhost.atmosphere.generate_workspace
+ vars:
+ workspace_path: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}"
+ domain_name: "{{ '{{' }} ansible_default_ipv4['address'].replace('.', '-') {{ '}}' }}.{{ lookup('env', 'ATMOSPHERE_DNS_SUFFIX_NAME') | default('nip.io', True) }}"
+
+- name: Prepare
+ hosts: all
+ become: true
+ pre_tasks:
+ - name: Wait for systemd to complete initialization
+ ansible.builtin.command: systemctl is-system-running
+ register: systemctl_status
+ until: >
+ 'running' in systemctl_status.stdout or
+ 'degraded' in systemctl_status.stdout
+ retries: 30
+ delay: 5
+ changed_when: false
+ failed_when: systemctl_status.rc > 1
+ tasks:
+ - name: Refresh cache & install "iptables"
+ ansible.builtin.package:
+ name: iptables
+ update_cache: true
+
+ # NOTE(mnaser): The base image installs Ansible using `pip` which breaks
+ # the system Python, we uninstall all Python packages.
+ - name: Fix Python installation
+ block:
+ - name: Get all Python packages
+ ansible.builtin.command: pip freeze
+ register: pip_freeze
+
+ - name: Uninstall all Python packages
+ ansible.builtin.pip:
+ name: "{{ pip_freeze.stdout_lines }}"
+ state: absent
+
+- ansible.builtin.import_playbook: vexxhost.ceph.create_fake_devices
+
+- hosts: controllers
+ become: true
+ tasks:
+ - name: Overwrite existing osds.yml file
+ delegate_to: localhost
+ ansible.builtin.copy:
+ dest: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/group_vars/cephs/osds.yml"
+ content: |
+ ceph_osd_devices:
+ - "/dev/ceph-{{ inventory_hostname_short }}-osd0/data"
+ - "/dev/ceph-{{ inventory_hostname_short }}-osd1/data"
+ - "/dev/ceph-{{ inventory_hostname_short }}-osd2/data"
+
+ - name: Set masquerade rule
+ become: yes
+ ansible.builtin.iptables:
+ table: nat
+ chain: POSTROUTING
+ source: 10.96.250.0/24
+ out_interface: "{{ ansible_default_ipv4.interface }}"
+ jump: MASQUERADE
diff --git a/molecule/ceph/verify.yml b/molecule/ceph/verify.yml
new file mode 100644
index 0000000..b88e217
--- /dev/null
+++ b/molecule/ceph/verify.yml
@@ -0,0 +1,15 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- import_playbook: vexxhost.atmosphere.tempest
diff --git a/molecule/default/group_vars/all/molecule.yml b/molecule/default/group_vars/all/molecule.yml
index db8ff71..65ad6dd 100644
--- a/molecule/default/group_vars/all/molecule.yml
+++ b/molecule/default/group_vars/all/molecule.yml
@@ -1,5 +1,7 @@
cluster_issuer_type: self-signed
+kubernetes_keepalived_interface: ens3
+
glance_images:
- name: cirros
url: http://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img