Add bootstrapping of Ceph

roles/pvc/tasks/bootstrap_ceph.yml

@@ -0,0 +1,156 @@
+---
+- name: add IP addresses to cluster interfaces
+  command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+
+- name: add IP addresses to storage interfaces
+  command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+
+- block:
+    - name: generate cluster FSID
+      command: uuidgen
+      register: fsid_raw
+      delegate_to: localhost
+
+    - set_fact:
+        fsid: "{{ fsid_raw.stdout }}"
+
+    - name: initialize bootstrap ceph.conf
+      file:
+        dest: /etc/ceph/ceph.conf
+        state: touch
+
+    - name: set fsid in bootstrap ceph.conf
+      lineinfile:
+        dest: /etc/ceph/ceph.conf
+        line: "fsid = {{ fsid }}"
+        state: present
+
+    - name: set mon initial members in bootstrap ceph.conf
+      lineinfile:
+        dest: /etc/ceph/ceph.conf
+        line: "mon initial members = {% for host in pvc_nodes %}{{ host.hostname }}{% if not loop.last %},{% endif %}{% endfor %}"
+        state: present
+
+    - name: set mon hosts in bootstrap ceph.conf
+      lineinfile:
+        dest: /etc/ceph/ceph.conf
+        line: "mon host = {% for host in pvc_nodes %}{{ host.storage_ip }}{% if not loop.last %},{% endif %}{% endfor %}"
+        state: present
+
+    - name: create temporary directory
+      file:
+        dest: /tmp/ceph-bootstrap
+        state: directory
+
+    - name: create mon keyring
+      command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+    
+    - name: create client admin keyring
+      command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+
+    - name: add client admin keyring to mon keyring
+      command: ceph-authtool /tmp/ceph-bootstrap/ceph.mon.keyring --import-keyring /tmp/ceph-bootstrap/ceph.client.admin.keyring
+
+    - name: create OSD bootstrap keyring
+      command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.osd.bootstrap.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd'
+
+    - name: add OSD bootstrap keyring to mon keyring
+      command: ceph-authtool /tmp/ceph-bootstrap/ceph.mon.keyring --import-keyring /tmp/ceph-bootstrap/ceph.osd.bootstrap.keyring
+
+    - name: create monmap
+      command: monmaptool --create --fsid {{ fsid }} /tmp/ceph-bootstrap/monmap
+
+    - name: add monitors to monmap
+      command: monmaptool --add {{ item.hostname }} {{ item.storage_ip }} --fsid {{ fsid }} /tmp/ceph-bootstrap/monmap
+      with_items:
+        - "{{ pvc_nodes }}"
+
+    - name: copy initial ceph.conf to the boostrap directory
+      copy:
+        src: /etc/ceph/ceph.conf
+        dest: /tmp/ceph-bootstrap/ceph.conf
+        remote_src: yes
+
+    - name: add additional configuration lines to ceph.conf
+      lineinfile:
+        dest: /tmp/ceph-bootstrap/ceph.conf
+        line: "{{ item }}"
+        state: present
+      with_items:
+        - "public network = {{ pvc_cluster_subnet }}"
+        - "cluster network = {{ pvc_storage_subnet }}"
+        - "auth cluster required = cephx"
+        - "auth service required = cephx"
+        - "auth client required = cephx"
+        - "osd journal size = 2"
+        - "osd pool default size = 3"
+        - "osd pool default min size = 2"
+        - "osd pool default pg num = 512"
+        - "osd pool default pgp num = 512"
+        - "osd crush chooseleaf type = 1"
+
+    - name: collect bootstrapped Ceph files into the role
+      fetch:
+        src: /tmp/ceph-bootstrap/{{ item }}
+        dest: roles/pvc/files/ceph/
+        flat: yes
+      with_items:
+        - ceph.conf
+        - ceph.mon.keyring
+        - ceph.client.admin.keyring
+        - ceph.osd.bootstrap.keyring
+        - monmap
+
+    - name: remove the temporary bootstrap directory
+      file:
+        dest: /tmp/ceph-bootstrap
+        state: absent
+        force: yes
+  run_once: true
+
+- name: deploy out configurations to all nodes
+  copy:
+    src: ceph/{{ item }}
+    dest: /etc/ceph/{{ item }}
+    owner: ceph
+    group: ceph
+    mode: 0640
+  with_items:
+    - ceph.conf
+    - ceph.mon.keyring
+    - ceph.client.admin.keyring
+    - ceph.osd.bootstrap.keyring
+    - monmap
+
+- name: create monitor data directory
+  file:
+    dest: sudo mkdir /var/lib/ceph/mon/ceph-{{ ansible_hostname }}
+    state: directory
+    owner: ceph
+    group: ceph
+    mode: 0750
+
+- name: populate monitor with map and keys
+  command: ceph-mon --mkfs -i {{ ansible_hostname }} --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring
+  become_user: ceph
+
+- name: touch monitor done file
+  file:
+    dest: /var/lib/ceph/mon/ceph-{{ ansible_hostname }}/done
+    state: touch
+
+- name: start the monitor daemon
+  service:
+    name: ceph-mon@{{ ansible_hostname }}
+    state: started
+    enabled: yes

roles/pvc/tasks/ceph.yml

@@ -1,4 +1,10 @@
 ---
+- name: initial deployment check
+  shell: "echo 'bootstrapped' > /etc/ceph-install"
+  register: newhost
+  args:
+    creates: "/etc/ceph-install"
+
 - name: install packages
   apt:
     name:
@@ -33,10 +39,5 @@
     dest: /etc/ceph
     state: directory
 
-- name: install ceph cluster configurations
-  template:
-    src: ceph/{{ item }}.j2
-    dest: /etc/ceph/{{ item }}
-  with_items:
-    - ceph.conf
-    - ceph.client.admin.keyring
+- include: bootstrap_ceph.yml
+  when: newhost.changed