Added Ceph deployment
Change-Id: If9d6de15dc083346b90b6946571369bcf94849ea
diff --git a/playbooks/ceph.yml b/playbooks/ceph.yml
new file mode 100644
index 0000000..8bc3a33
--- /dev/null
+++ b/playbooks/ceph.yml
@@ -0,0 +1,37 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Setup Ceph repository
+ hosts: cephs
+ become: true
+ roles:
+ - ceph_repository
+
+- name: Deploy Ceph monitors
+ hosts: ceph_mons
+ become: true
+ roles:
+ - ceph_mon
+
+- name: Deploy Ceph managers
+ hosts: ceph_mgrs
+ become: true
+ roles:
+ - ceph_mgr
+
+- name: Deploy Ceph OSDs
+ hosts: ceph_osds
+ become: true
+ roles:
+ - ceph_osd
diff --git a/playbooks/site.yml b/playbooks/site.yml
index c6e311a..2b745d4 100644
--- a/playbooks/site.yml
+++ b/playbooks/site.yml
@@ -12,4 +12,5 @@
# License for the specific language governing permissions and limitations
# under the License.
+- import_playbook: vexxhost.atmosphere.ceph
- import_playbook: vexxhost.atmosphere.kubernetes
diff --git a/roles/ceph_mgr/tasks/main.yml b/roles/ceph_mgr/tasks/main.yml
new file mode 100644
index 0000000..879928b
--- /dev/null
+++ b/roles/ceph_mgr/tasks/main.yml
@@ -0,0 +1,42 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: install packages
+ ansible.builtin.apt:
+ name: ["ceph-mgr"]
+ install_recommends: false
+
+- name: create manager folder
+ ansible.builtin.file:
+ path: "/var/lib/ceph/mgr/ceph-{{ inventory_hostname_short }}"
+ state: directory
+
+- name: create mgr keyring
+ ansible.builtin.shell: |
+ ceph auth get-or-create mgr.{{ inventory_hostname_short }} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-{{ inventory_hostname_short }}/keyring
+ args:
+ creates: "/var/lib/ceph/mgr/ceph-{{ inventory_hostname_short }}/keyring"
+
+- name: ensure permissions are fixed
+ ansible.builtin.file:
+ path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+ owner: ceph
+ group: ceph
+ recurse: yes
+
+- name: enable and start service
+ ansible.builtin.service:
+ name: "ceph-mgr@{{ inventory_hostname_short }}"
+ state: started
+ enabled: yes
diff --git a/roles/ceph_mon/defaults/main.yml b/roles/ceph_mon/defaults/main.yml
new file mode 100644
index 0000000..d8f595b
--- /dev/null
+++ b/roles/ceph_mon/defaults/main.yml
@@ -0,0 +1,16 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ceph_mon_group: ceph_mons
+ceph_mon_cluster_network: "{{ ceph_mon_public_network }}"
diff --git a/roles/ceph_mon/tasks/bootstrap-ceph.yml b/roles/ceph_mon/tasks/bootstrap-ceph.yml
new file mode 100644
index 0000000..15c81cc
--- /dev/null
+++ b/roles/ceph_mon/tasks/bootstrap-ceph.yml
@@ -0,0 +1,80 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: create monitor keyring
+ ansible.builtin.shell: |
+ ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+ args:
+ creates: /tmp/ceph.mon.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create admin keyring
+ ansible.builtin.shell: |
+ ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+ args:
+ creates: /etc/ceph/ceph.client.admin.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create bootstrap-osd keyring
+ ansible.builtin.shell: |
+ ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+ args:
+ creates: /var/lib/ceph/bootstrap-osd/ceph.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: add admin keyring to monitor
+ ansible.builtin.shell: |
+ ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: add bootstrap-osd keyring to monitor
+ ansible.builtin.shell: |
+ ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create monmap
+ ansible.builtin.shell: |
+ monmaptool --create --add {{ inventory_hostname_short }} {{ ceph_mon_ip_address }} --fsid {{ ceph_mon_fsid }} /tmp/monmap
+ args:
+ creates: /tmp/monmap
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create monitor folder
+ ansible.builtin.file:
+ path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+ state: directory
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: configure mon initial members
+ community.general.ini_file:
+ path: /etc/ceph/ceph.conf
+ section: global
+ option: mon initial members
+ value: "{{ inventory_hostname_short }}"
+
+- name: start monitor
+ ansible.builtin.include_tasks: start-monitor.yml
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: set bootstrap node
+ ansible.builtin.set_fact:
+ _ceph_mon_bootstrap_node: "{{ groups[ceph_mon_group][0] }}"
diff --git a/roles/ceph_mon/tasks/main.yml b/roles/ceph_mon/tasks/main.yml
new file mode 100644
index 0000000..d4117a0
--- /dev/null
+++ b/roles/ceph_mon/tasks/main.yml
@@ -0,0 +1,89 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: install packages
+ ansible.builtin.apt:
+ name: ["ceph-mon"]
+ install_recommends: false
+
+- name: set ceph monitor ip address
+ set_fact:
+ ceph_mon_ip_address: "{{ ansible_default_ipv4.address }}"
+
+- name: generate basic configuration file
+ community.general.ini_file:
+ path: /etc/ceph/ceph.conf
+ section: global
+ option: "{{ item.option }}"
+ value: "{{ item.value }}"
+ loop:
+ - option: fsid
+ value: "{{ ceph_mon_fsid }}"
+ - option: mon host
+ value: "{{ groups[ceph_mon_group] | map('extract', hostvars, ['ceph_mon_ip_address']) | join(',') }}"
+ - option: public network
+ value: "{{ ceph_mon_public_network }}"
+ - option: cluster network
+ value: "{{ ceph_mon_cluster_network }}"
+
+- name: check if any node is bootstrapped
+ ansible.builtin.stat:
+ path: "/var/lib/ceph/mon/ceph-{{ hostvars[item]['inventory_hostname_short'] }}/store.db"
+ register: _ceph_mon_stat
+ loop: "{{ groups[ceph_mon_group] }}"
+ delegate_to: "{{ item }}"
+
+- name: select pre-existing bootstrap node if exists
+ ansible.builtin.set_fact:
+ _ceph_mon_bootstrap_node: "{{ _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | map(attribute='item') | first }}"
+ when:
+ - _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | length > 0
+
+- name: bootstrap cluster
+ ansible.builtin.include_tasks: bootstrap-ceph.yml
+ when:
+ - _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | length == 0
+
+- name: grab admin keyring
+ delegate_to: "{{ _ceph_mon_bootstrap_node }}"
+ ansible.builtin.slurp:
+ src: /etc/ceph/ceph.client.admin.keyring
+ register: _ceph_mon_admin_keyring
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: upload client.admin keyring
+ ansible.builtin.copy:
+ content: "{{ _ceph_mon_admin_keyring['content'] | b64decode }}"
+ dest: /etc/ceph/ceph.client.admin.keyring
+ mode: 0600
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: get monitor keyring
+ ansible.builtin.shell: ceph auth get mon. -o /tmp/ceph.mon.keyring
+ changed_when: false
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: get monmap keyring
+ ansible.builtin.shell: ceph mon getmap -o /tmp/monmap
+ changed_when: false
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: start monitor
+ ansible.builtin.include_tasks: start-monitor.yml
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: enable msgr2
+ ansible.builtin.shell: ceph mon enable-msgr2
+ changed_when: false
+ when: inventory_hostname == _ceph_mon_bootstrap_node
diff --git a/roles/ceph_mon/tasks/start-monitor.yml b/roles/ceph_mon/tasks/start-monitor.yml
new file mode 100644
index 0000000..c25b9da
--- /dev/null
+++ b/roles/ceph_mon/tasks/start-monitor.yml
@@ -0,0 +1,43 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: mkfs monitor
+ ansible.builtin.shell: |
+ ceph-mon --mkfs -i {{ inventory_hostname_short }} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+ args:
+ creates: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}/store.db"
+
+- name: ensure permissions are fixed
+ ansible.builtin.file:
+ path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+ owner: ceph
+ group: ceph
+ recurse: yes
+
+# NOTE(mnaser): https://bugs.launchpad.net/ubuntu/+source/ceph/+bug/1917414/comments/30
+- name: workaround for aarch64 systems
+ community.general.ini_file:
+ path: /lib/systemd/system/ceph-mon@.service
+ section: Service
+ option: MemoryDenyWriteExecute
+ value: false
+ register: _ceph_aarch64_fix
+ when: ansible_architecture == 'aarch64'
+
+- name: enable and start service
+ ansible.builtin.service:
+ name: "ceph-mon@{{ inventory_hostname_short }}"
+ state: started
+ enabled: yes
+ daemon_reload: "{{ _ceph_aarch64_fix.changed }}"
diff --git a/roles/ceph_osd/defaults/main.yml b/roles/ceph_osd/defaults/main.yml
new file mode 100644
index 0000000..0c6f323
--- /dev/null
+++ b/roles/ceph_osd/defaults/main.yml
@@ -0,0 +1,15 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ceph_osd_mons_group: ceph_mons
diff --git a/roles/ceph_osd/tasks/main.yml b/roles/ceph_osd/tasks/main.yml
new file mode 100644
index 0000000..271d880
--- /dev/null
+++ b/roles/ceph_osd/tasks/main.yml
@@ -0,0 +1,87 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: install packages
+ ansible.builtin.apt:
+ name: ["udev", "ceph-osd"]
+ install_recommends: false
+
+- name: grab ceph fsid from monitors
+ delegate_to: "{{ groups[ceph_osd_mons_group][0] }}"
+ register: _ceph_fsid
+ changed_when: false
+ ansible.builtin.shell: ceph fsid
+
+- name: collect facts for all monitors
+ delegate_to: "{{ item }}"
+ delegate_facts: true
+ ansible.builtin.setup:
+ gather_subset: network
+ loop: "{{ groups[ceph_osd_mons_group] }}"
+ when: inventory_hostname == ansible_play_hosts_all[0]
+
+- name: generate basic configuration file
+ community.general.ini_file:
+ path: /etc/ceph/ceph.conf
+ section: global
+ option: "{{ item.option }}"
+ value: "{{ item.value }}"
+ loop:
+ - option: fsid
+ value: "{{ _ceph_fsid.stdout | trim }}"
+ - option: mon host
+ value: "{{ groups[ceph_osd_mons_group] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | join(',') }}"
+
+- name: grab bootstrap-osd from monitors
+ delegate_to: "{{ groups[ceph_osd_mons_group][0] }}"
+ register: _ceph_bootstrap_osd_keyring
+ changed_when: false
+ ansible.builtin.shell: ceph auth get client.bootstrap-osd
+
+- name: install bootstrap-osd keyring
+ ansible.builtin.copy:
+ content: "{{ _ceph_bootstrap_osd_keyring.stdout }}\n"
+ dest: /var/lib/ceph/bootstrap-osd/ceph.keyring
+
+- name: workaround to allow usage of loop devices
+ ansible.builtin.replace:
+ path: /usr/lib/python3/dist-packages/ceph_volume/util/disk.py
+ regexp: "'mpath']"
+ replace: "'mpath', 'loop']"
+ when: molecule | default(false)
+
+# NOTE(mnaser): https://bugs.launchpad.net/ubuntu/+source/ceph/+bug/1917414/comments/30
+- name: workaround for aarch64 systems
+ community.general.ini_file:
+ path: /lib/systemd/system/ceph-osd@.service
+ section: Service
+ option: MemoryDenyWriteExecute
+ value: false
+ register: _ceph_aarch64_fix
+ when: ansible_architecture == 'aarch64'
+
+- name: reload systemd
+ ansible.builtin.service:
+ daemon_reload: "{{ _ceph_aarch64_fix.changed }}"
+
+- name: get which devices don't contain osds
+ register: _ceph_osd_check
+ failed_when: false
+ changed_when: false
+ ansible.builtin.command: /usr/sbin/ceph-volume lvm list {{ item }}
+ loop: "{{ ceph_osd_devices }}"
+
+- name: create osds for volumes which are not setup
+ ansible.builtin.command: /usr/sbin/ceph-volume lvm create --data {{ item }}
+ loop: "{{ _ceph_osd_check.results | selectattr('rc', 'equalto', 1) | map(attribute='item') }}"
diff --git a/roles/ceph_repository/defaults/main.yml b/roles/ceph_repository/defaults/main.yml
new file mode 100644
index 0000000..a898c6f
--- /dev/null
+++ b/roles/ceph_repository/defaults/main.yml
@@ -0,0 +1,17 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ceph_repository_apt_key: "https://download.ceph.com/keys/release.asc"
+ceph_repository_url: https://download.ceph.com/debian-pacific/
+ceph_repository_version: 16.2.7
diff --git a/roles/ceph_repository/tasks/main.yml b/roles/ceph_repository/tasks/main.yml
new file mode 100644
index 0000000..20e1685
--- /dev/null
+++ b/roles/ceph_repository/tasks/main.yml
@@ -0,0 +1,29 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: add apt key
+ ansible.builtin.apt_key:
+ url: "{{ ceph_repository_apt_key }}"
+ state: present
+
+- name: configure version pinning
+ ansible.builtin.template:
+ src: apt-preferences.j2
+ dest: /etc/apt/preferences.d/ceph
+ mode: 0644
+
+- name: add apt repository
+ ansible.builtin.apt_repository:
+ repo: "deb {{ ceph_repository_url }} {{ ansible_distribution_release }} main"
+ state: present
diff --git a/roles/ceph_repository/templates/apt-preferences.j2 b/roles/ceph_repository/templates/apt-preferences.j2
new file mode 100644
index 0000000..91d6b11
--- /dev/null
+++ b/roles/ceph_repository/templates/apt-preferences.j2
@@ -0,0 +1,11 @@
+Package: ceph-mon
+Pin: version {{ ceph_repository_version }}-*
+Pin-Priority: 1000
+
+Package: ceph-mgr
+Pin: version {{ ceph_repository_version }}-*
+Pin-Priority: 1000
+
+Package: ceph-osd
+Pin: version {{ ceph_repository_version }}-*
+Pin-Priority: 1000
\ No newline at end of file