Added Ceph deployment
Change-Id: If9d6de15dc083346b90b6946571369bcf94849ea
diff --git a/roles/ceph_mon/defaults/main.yml b/roles/ceph_mon/defaults/main.yml
new file mode 100644
index 0000000..d8f595b
--- /dev/null
+++ b/roles/ceph_mon/defaults/main.yml
@@ -0,0 +1,16 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+ceph_mon_group: ceph_mons
+ceph_mon_cluster_network: "{{ ceph_mon_public_network }}"
diff --git a/roles/ceph_mon/tasks/bootstrap-ceph.yml b/roles/ceph_mon/tasks/bootstrap-ceph.yml
new file mode 100644
index 0000000..15c81cc
--- /dev/null
+++ b/roles/ceph_mon/tasks/bootstrap-ceph.yml
@@ -0,0 +1,80 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: create monitor keyring
+ ansible.builtin.shell: |
+ ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+ args:
+ creates: /tmp/ceph.mon.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create admin keyring
+ ansible.builtin.shell: |
+ ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+ args:
+ creates: /etc/ceph/ceph.client.admin.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create bootstrap-osd keyring
+ ansible.builtin.shell: |
+ ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+ args:
+ creates: /var/lib/ceph/bootstrap-osd/ceph.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: add admin keyring to monitor
+ ansible.builtin.shell: |
+ ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: add bootstrap-osd keyring to monitor
+ ansible.builtin.shell: |
+ ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create monmap
+ ansible.builtin.shell: |
+ monmaptool --create --add {{ inventory_hostname_short }} {{ ceph_mon_ip_address }} --fsid {{ ceph_mon_fsid }} /tmp/monmap
+ args:
+ creates: /tmp/monmap
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: create monitor folder
+ ansible.builtin.file:
+ path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+ state: directory
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: configure mon initial members
+ community.general.ini_file:
+ path: /etc/ceph/ceph.conf
+ section: global
+ option: mon initial members
+ value: "{{ inventory_hostname_short }}"
+
+- name: start monitor
+ ansible.builtin.include_tasks: start-monitor.yml
+ when:
+ - inventory_hostname == groups[ceph_mon_group][0]
+
+- name: set bootstrap node
+ ansible.builtin.set_fact:
+ _ceph_mon_bootstrap_node: "{{ groups[ceph_mon_group][0] }}"
diff --git a/roles/ceph_mon/tasks/main.yml b/roles/ceph_mon/tasks/main.yml
new file mode 100644
index 0000000..d4117a0
--- /dev/null
+++ b/roles/ceph_mon/tasks/main.yml
@@ -0,0 +1,89 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: install packages
+ ansible.builtin.apt:
+ name: ["ceph-mon"]
+ install_recommends: false
+
+- name: set ceph monitor ip address
+ set_fact:
+ ceph_mon_ip_address: "{{ ansible_default_ipv4.address }}"
+
+- name: generate basic configuration file
+ community.general.ini_file:
+ path: /etc/ceph/ceph.conf
+ section: global
+ option: "{{ item.option }}"
+ value: "{{ item.value }}"
+ loop:
+ - option: fsid
+ value: "{{ ceph_mon_fsid }}"
+ - option: mon host
+ value: "{{ groups[ceph_mon_group] | map('extract', hostvars, ['ceph_mon_ip_address']) | join(',') }}"
+ - option: public network
+ value: "{{ ceph_mon_public_network }}"
+ - option: cluster network
+ value: "{{ ceph_mon_cluster_network }}"
+
+- name: check if any node is bootstrapped
+ ansible.builtin.stat:
+ path: "/var/lib/ceph/mon/ceph-{{ hostvars[item]['inventory_hostname_short'] }}/store.db"
+ register: _ceph_mon_stat
+ loop: "{{ groups[ceph_mon_group] }}"
+ delegate_to: "{{ item }}"
+
+- name: select pre-existing bootstrap node if exists
+ ansible.builtin.set_fact:
+ _ceph_mon_bootstrap_node: "{{ _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | map(attribute='item') | first }}"
+ when:
+ - _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | length > 0
+
+- name: bootstrap cluster
+ ansible.builtin.include_tasks: bootstrap-ceph.yml
+ when:
+ - _ceph_mon_stat.results | selectattr('stat.exists', 'equalto', true) | length == 0
+
+- name: grab admin keyring
+ delegate_to: "{{ _ceph_mon_bootstrap_node }}"
+ ansible.builtin.slurp:
+ src: /etc/ceph/ceph.client.admin.keyring
+ register: _ceph_mon_admin_keyring
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: upload client.admin keyring
+ ansible.builtin.copy:
+ content: "{{ _ceph_mon_admin_keyring['content'] | b64decode }}"
+ dest: /etc/ceph/ceph.client.admin.keyring
+ mode: 0600
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: get monitor keyring
+ ansible.builtin.shell: ceph auth get mon. -o /tmp/ceph.mon.keyring
+ changed_when: false
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: get monmap keyring
+ ansible.builtin.shell: ceph mon getmap -o /tmp/monmap
+ changed_when: false
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: start monitor
+ ansible.builtin.include_tasks: start-monitor.yml
+ when: inventory_hostname != _ceph_mon_bootstrap_node
+
+- name: enable msgr2
+ ansible.builtin.shell: ceph mon enable-msgr2
+ changed_when: false
+ when: inventory_hostname == _ceph_mon_bootstrap_node
diff --git a/roles/ceph_mon/tasks/start-monitor.yml b/roles/ceph_mon/tasks/start-monitor.yml
new file mode 100644
index 0000000..c25b9da
--- /dev/null
+++ b/roles/ceph_mon/tasks/start-monitor.yml
@@ -0,0 +1,43 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: mkfs monitor
+ ansible.builtin.shell: |
+ ceph-mon --mkfs -i {{ inventory_hostname_short }} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
+ args:
+ creates: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}/store.db"
+
+- name: ensure permissions are fixed
+ ansible.builtin.file:
+ path: "/var/lib/ceph/mon/ceph-{{ inventory_hostname_short }}"
+ owner: ceph
+ group: ceph
+ recurse: yes
+
+# NOTE(mnaser): https://bugs.launchpad.net/ubuntu/+source/ceph/+bug/1917414/comments/30
+- name: workaround for aarch64 systems
+ community.general.ini_file:
+ path: /lib/systemd/system/ceph-mon@.service
+ section: Service
+ option: MemoryDenyWriteExecute
+ value: false
+ register: _ceph_aarch64_fix
+ when: ansible_architecture == 'aarch64'
+
+- name: enable and start service
+ ansible.builtin.service:
+ name: "ceph-mon@{{ inventory_hostname_short }}"
+ state: started
+ enabled: yes
+ daemon_reload: "{{ _ceph_aarch64_fix.changed }}"