Added Ceph deployment

Change-Id: If9d6de15dc083346b90b6946571369bcf94849ea
diff --git a/playbooks/ceph.yml b/playbooks/ceph.yml
new file mode 100644
index 0000000..8bc3a33
--- /dev/null
+++ b/playbooks/ceph.yml
@@ -0,0 +1,37 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Setup Ceph repository
+  hosts: cephs
+  become: true
+  roles:
+    - ceph_repository
+
+- name: Deploy Ceph monitors
+  hosts: ceph_mons
+  become: true
+  roles:
+    - ceph_mon
+
+- name: Deploy Ceph managers
+  hosts: ceph_mgrs
+  become: true
+  roles:
+    - ceph_mgr
+
+- name: Deploy Ceph OSDs
+  hosts: ceph_osds
+  become: true
+  roles:
+    - ceph_osd
diff --git a/playbooks/site.yml b/playbooks/site.yml
index c6e311a..2b745d4 100644
--- a/playbooks/site.yml
+++ b/playbooks/site.yml
@@ -12,4 +12,5 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+- import_playbook: vexxhost.atmosphere.ceph
 - import_playbook: vexxhost.atmosphere.kubernetes
diff --git a/roles/ceph_mgr/tasks/main.yml b/roles/ceph_mgr/tasks/main.yml
new file mode 100644
index 0000000..879928b
--- /dev/null
+++ b/roles/ceph_mgr/tasks/main.yml
@@ -0,0 +1,42 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: install packages
+  ansible.builtin.apt:
+    name: ["ceph-mgr"]
+    install_recommends: false
+
+- name: create manager folder
+  ansible.builtin.file:
+    path: "/var/lib/ceph/mgr/ceph-{{ inventory_hostname_short }}"
+    state: directory
+
+- name: create mgr keyring
+  ansible.builtin.shell: |
+    ceph auth get-or-create mgr.{{ inventory_hostname_short }} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-{{ inventory_hostname_short }}/keyring
+  args:
+    creates: "/var/lib/ceph/mgr/ceph-{{ inventory_hostname_short }}/keyring"
+
+- name: ensure permissions are fixed
+  ansible.builtin.file:
+    path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+    owner: ceph
+    group: ceph
+    recurse: yes
+
+- name: enable and start service
+  ansible.builtin.service:
+    name: "ceph-mgr@{{ inventory_hostname_short }}"
+    state: started
+    enabled: yes
diff --git a/roles/ceph_mon/defaults/main.yml b/roles/ceph_mon/defaults/main.yml
new file mode 100644
index 0000000..d8f595b
--- /dev/null
+++ b/roles/ceph_mon/defaults/main.yml
@@ -0,0 +1,16 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ceph_mon_group: ceph_mons
+ceph_mon_cluster_network: "{{ ceph_mon_public_network }}"
diff --git a/roles/ceph_mon/tasks/bootstrap-ceph.yml b/roles/ceph_mon/tasks/bootstrap-ceph.yml
new file mode 100644
index 0000000..15c81cc
--- /dev/null
+++ b/roles/ceph_mon/tasks/bootstrap-ceph.yml
@@ -0,0 +1,80 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: create monitor keyring
+  ansible.builtin.shell: |
+    ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+  args:
+    creates: /tmp/ceph.mon.keyring
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create admin keyring
+  ansible.builtin.shell: |
+    ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+  args:
+    creates: /etc/ceph/ceph.client.admin.keyring
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create bootstrap-osd keyring
+  ansible.builtin.shell: |
+    ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+  args:
+    creates: /var/lib/ceph/bootstrap-osd/ceph.keyring
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: add admin keyring to monitor
+  ansible.builtin.shell: |
+    ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: add bootstrap-osd keyring to monitor
+  ansible.builtin.shell: |
+    ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create monmap
+  ansible.builtin.shell: |
+    monmaptool --create --add {{ inventory_hostname_short }} {{ ceph_mon_ip_address }} --fsid {{ ceph_mon_fsid }} /tmp/monmap
+  args:
+    creates: /tmp/monmap
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create monitor folder
+  ansible.builtin.file:
+    path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+    state: directory
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: configure mon initial members
+  community.general.ini_file:
+    path: /etc/ceph/ceph.conf
+    section: global
+    option: mon initial members
+    value: "{{ inventory_hostname_short }}"
+
+- name: start monitor
+  ansible.builtin.include_tasks: start-monitor.yml
+  when:
+    - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: set bootstrap node
+  ansible.builtin.set_fact:
+    _ceph_mon_bootstrap_node: "{{ groups[ceph_mon_group][0] }}"
diff --git a/roles/ceph_mon/tasks/main.yml b/roles/ceph_mon/tasks/main.yml
new file mode 100644
index 0000000..d4117a0
--- /dev/null
+++ b/roles/ceph_mon/tasks/main.yml
@@ -0,0 +1,89 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: install packages
+  ansible.builtin.apt:
+    name: ["ceph-mon"]
+    install_recommends: false
+
+- name: set ceph monitor ip address
+  set_fact:
+    ceph_mon_ip_address: "{{ ansible_default_ipv4.address }}"
+
+- name: generate basic configuration file
+  community.general.ini_file:
+    path: /etc/ceph/ceph.conf
+    section: global
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+  loop:
+    - option: fsid
+      value: "{{ ceph_mon_fsid }}"
+    - option: mon host
+      value: "{{ groups[ceph_mon_group] | map('extract', hostvars, ['ceph_mon_ip_address']) | join(',') }}"
+    - option: public network
+      value: "{{ ceph_mon_public_network }}"
+    - option: cluster network
+      value: "{{ ceph_mon_cluster_network }}"
+
+- name: check if any node is bootstrapped
+  ansible.builtin.stat:
+    path: "/var/lib/ceph/mon/ceph-{{ hostvars[item]['inventory_hostname_short'] }}/store.db"
+  register: _ceph_mon_stat
+  loop: "{{ groups[ceph_mon_group] }}"
+  delegate_to: "{{ item }}"
+
+- name: select pre-existing bootstrap node if exists
+  ansible.builtin.set_fact:
+    _ceph_mon_bootstrap_node: "{{ _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | map(attribute='item') | first }}"
+  when:
+    - _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | length > 0
+
+- name: bootstrap cluster
+  ansible.builtin.include_tasks: bootstrap-ceph.yml
+  when:
+    - _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | length == 0
+
+- name: grab admin keyring
+  delegate_to: "{{ _ceph_mon_bootstrap_node }}"
+  ansible.builtin.slurp:
+    src: /etc/ceph/ceph.client.admin.keyring
+  register: _ceph_mon_admin_keyring
+  when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: upload client.admin keyring
+  ansible.builtin.copy:
+    content: "{{ _ceph_mon_admin_keyring['content'] | b64decode }}"
+    dest: /etc/ceph/ceph.client.admin.keyring
+    mode: 0600
+  when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: get monitor keyring
+  ansible.builtin.shell: ceph auth get mon. -o /tmp/ceph.mon.keyring
+  changed_when: false
+  when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: get monmap keyring
+  ansible.builtin.shell: ceph mon getmap -o /tmp/monmap
+  changed_when: false
+  when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: start monitor
+  ansible.builtin.include_tasks: start-monitor.yml
+  when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: enable msgr2
+  ansible.builtin.shell: ceph mon enable-msgr2
+  changed_when: false
+  when: inventory_hostname == _ceph_mon_bootstrap_node
diff --git a/roles/ceph_mon/tasks/start-monitor.yml b/roles/ceph_mon/tasks/start-monitor.yml
new file mode 100644
index 0000000..c25b9da
--- /dev/null
+++ b/roles/ceph_mon/tasks/start-monitor.yml
@@ -0,0 +1,43 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: mkfs monitor
+  ansible.builtin.shell: |
+    ceph-mon --mkfs -i {{ inventory_hostname_short }} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+  args:
+    creates: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}/store.db"
+
+- name: ensure permissions are fixed
+  ansible.builtin.file:
+    path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+    owner: ceph
+    group: ceph
+    recurse: yes
+
+# NOTE(mnaser): https://bugs.launchpad.net/ubuntu/+source/ceph/+bug/1917414/comments/30
+- name: workaround for aarch64 systems
+  community.general.ini_file:
+    path: /lib/systemd/system/ceph-mon@.service
+    section: Service
+    option: MemoryDenyWriteExecute
+    value: false
+  register: _ceph_aarch64_fix
+  when: ansible_architecture == 'aarch64'
+
+- name: enable and start service
+  ansible.builtin.service:
+    name: "ceph-mon@{{ inventory_hostname_short }}"
+    state: started
+    enabled: yes
+    daemon_reload: "{{ _ceph_aarch64_fix.changed }}"
diff --git a/roles/ceph_osd/defaults/main.yml b/roles/ceph_osd/defaults/main.yml
new file mode 100644
index 0000000..0c6f323
--- /dev/null
+++ b/roles/ceph_osd/defaults/main.yml
@@ -0,0 +1,15 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ceph_osd_mons_group: ceph_mons
diff --git a/roles/ceph_osd/tasks/main.yml b/roles/ceph_osd/tasks/main.yml
new file mode 100644
index 0000000..271d880
--- /dev/null
+++ b/roles/ceph_osd/tasks/main.yml
@@ -0,0 +1,87 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: install packages
+  ansible.builtin.apt:
+    name: ["udev", "ceph-osd"]
+    install_recommends: false
+
+- name: grab ceph fsid from monitors
+  delegate_to: "{{ groups[ceph_osd_mons_group][0] }}"
+  register: _ceph_fsid
+  changed_when: false
+  ansible.builtin.shell: ceph fsid
+
+- name: collect facts for all monitors
+  delegate_to: "{{ item }}"
+  delegate_facts: true
+  ansible.builtin.setup:
+    gather_subset: network
+  loop: "{{ groups[ceph_osd_mons_group] }}"
+  when: inventory_hostname == ansible_play_hosts_all[0]
+
+- name: generate basic configuration file
+  community.general.ini_file:
+    path: /etc/ceph/ceph.conf
+    section: global
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+  loop:
+    - option: fsid
+      value: "{{ _ceph_fsid.stdout | trim }}"
+    - option: mon host
+      value: "{{ groups[ceph_osd_mons_group] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | join(',') }}"
+
+- name: grab bootstrap-osd from monitors
+  delegate_to: "{{ groups[ceph_osd_mons_group][0] }}"
+  register: _ceph_bootstrap_osd_keyring
+  changed_when: false
+  ansible.builtin.shell: ceph auth get client.bootstrap-osd
+
+- name: install bootstrap-osd keyring
+  ansible.builtin.copy:
+    content: "{{ _ceph_bootstrap_osd_keyring.stdout }}\n"
+    dest: /var/lib/ceph/bootstrap-osd/ceph.keyring
+
+- name: workaround to allow usage of loop devices
+  ansible.builtin.replace:
+    path: /usr/lib/python3/dist-packages/ceph_volume/util/disk.py
+    regexp: "'mpath']"
+    replace: "'mpath', 'loop']"
+  when: molecule | default(false)
+
+# NOTE(mnaser): https://bugs.launchpad.net/ubuntu/+source/ceph/+bug/1917414/comments/30
+- name: workaround for aarch64 systems
+  community.general.ini_file:
+    path: /lib/systemd/system/ceph-osd@.service
+    section: Service
+    option: MemoryDenyWriteExecute
+    value: false
+  register: _ceph_aarch64_fix
+  when: ansible_architecture == 'aarch64'
+
+- name: reload systemd
+  ansible.builtin.service:
+    daemon_reload: "{{ _ceph_aarch64_fix.changed }}"
+
+- name: get which devices don't contain osds
+  register: _ceph_osd_check
+  failed_when: false
+  changed_when: false
+  ansible.builtin.command: /usr/sbin/ceph-volume lvm list {{ item }}
+  loop: "{{ ceph_osd_devices }}"
+
+- name: create osds for volumes which are not setup
+  ansible.builtin.command: /usr/sbin/ceph-volume lvm create --data {{ item }}
+  loop: "{{ _ceph_osd_check.results | selectattr('rc', 'equalto', 1) | map(attribute='item') }}"
diff --git a/roles/ceph_repository/defaults/main.yml b/roles/ceph_repository/defaults/main.yml
new file mode 100644
index 0000000..a898c6f
--- /dev/null
+++ b/roles/ceph_repository/defaults/main.yml
@@ -0,0 +1,17 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ceph_repository_apt_key: "https://download.ceph.com/keys/release.asc"
+ceph_repository_url: https://download.ceph.com/debian-pacific/
+ceph_repository_version: 16.2.7
diff --git a/roles/ceph_repository/tasks/main.yml b/roles/ceph_repository/tasks/main.yml
new file mode 100644
index 0000000..20e1685
--- /dev/null
+++ b/roles/ceph_repository/tasks/main.yml
@@ -0,0 +1,29 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: add apt key
+  ansible.builtin.apt_key:
+    url: "{{ ceph_repository_apt_key }}"
+    state: present
+
+- name: configure version pinning
+  ansible.builtin.template:
+    src: apt-preferences.j2
+    dest: /etc/apt/preferences.d/ceph
+    mode: 0644
+
+- name: add apt repository
+  ansible.builtin.apt_repository:
+    repo: "deb {{ ceph_repository_url }} {{ ansible_distribution_release }} main"
+    state: present
diff --git a/roles/ceph_repository/templates/apt-preferences.j2 b/roles/ceph_repository/templates/apt-preferences.j2
new file mode 100644
index 0000000..91d6b11
--- /dev/null
+++ b/roles/ceph_repository/templates/apt-preferences.j2
@@ -0,0 +1,11 @@
+Package: ceph-mon
+Pin: version {{ ceph_repository_version }}-*
+Pin-Priority: 1000
+
+Package: ceph-mgr
+Pin: version {{ ceph_repository_version }}-*
+Pin-Priority: 1000
+
+Package: ceph-osd
+Pin: version {{ ceph_repository_version }}-*
+Pin-Priority: 1000
\ No newline at end of file