pvc-ansible/roles/pvc/tasks/bootstrap_ceph.yml

---
- name: add IP addresses to cluster interfaces
  command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
  delegate_to: "{{ item.hostname }}"
  run_once: yes
  ignore_errors: yes
  with_items:
    - "{{ pvc_nodes }}"

- name: add IP addresses to storage interfaces
  command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
  delegate_to: "{{ item.hostname }}"
  run_once: yes
  ignore_errors: yes
  with_items:
    - "{{ pvc_nodes }}"

- block:
    - name: generate cluster FSID
      command: uuidgen
      register: fsid_raw
      delegate_to: localhost

    - set_fact:
        fsid: "{{ fsid_raw.stdout }}"

    - name: initialize bootstrap ceph.conf
      file:
        dest: /etc/ceph/ceph.conf
        state: touch

    - name: set fsid in bootstrap ceph.conf
      lineinfile:
        dest: /etc/ceph/ceph.conf
        line: "fsid = {{ fsid }}"
        state: present

    - name: set mon initial members in bootstrap ceph.conf
      lineinfile:
        dest: /etc/ceph/ceph.conf
        line: "mon initial members = {% for host in pvc_nodes %}{{ host.hostname }}{% if not loop.last %},{% endif %}{% endfor %}"
        state: present

    - name: set mon hosts in bootstrap ceph.conf
      lineinfile:
        dest: /etc/ceph/ceph.conf
        line: "mon host = {% for host in pvc_nodes %}{{ host.storage_ip }}{% if not loop.last %},{% endif %}{% endfor %}"
        state: present

    - name: create temporary directory
      file:
        dest: /tmp/ceph-bootstrap
        state: directory

    - name: create mon keyring
      command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
    
    - name: create client admin keyring
      command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'

    - name: add client admin keyring to mon keyring
      command: ceph-authtool /tmp/ceph-bootstrap/ceph.mon.keyring --import-keyring /tmp/ceph-bootstrap/ceph.client.admin.keyring

    - name: create OSD bootstrap keyring
      command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.osd.bootstrap.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd'

    - name: add OSD bootstrap keyring to mon keyring
      command: ceph-authtool /tmp/ceph-bootstrap/ceph.mon.keyring --import-keyring /tmp/ceph-bootstrap/ceph.osd.bootstrap.keyring

    - name: create monmap
      command: monmaptool --create --fsid {{ fsid }} /tmp/ceph-bootstrap/monmap

    - name: add monitors to monmap
      command: monmaptool --add {{ item.hostname }} {{ item.storage_ip }} --fsid {{ fsid }} /tmp/ceph-bootstrap/monmap
      with_items:
        - "{{ pvc_nodes }}"

    - name: copy initial ceph.conf to the boostrap directory
      copy:
        src: /etc/ceph/ceph.conf
        dest: /tmp/ceph-bootstrap/ceph.conf
        remote_src: yes

    - name: add additional configuration lines to ceph.conf
      lineinfile:
        dest: /tmp/ceph-bootstrap/ceph.conf
        line: "{{ item }}"
        state: present
      with_items:
        - "public network = {{ pvc_cluster_subnet }}"
        - "cluster network = {{ pvc_storage_subnet }}"
        - "auth cluster required = cephx"
        - "auth service required = cephx"
        - "auth client required = cephx"
        - "osd journal size = 2"
        - "osd pool default size = 3"
        - "osd pool default min size = 2"
        - "osd pool default pg num = 512"
        - "osd pool default pgp num = 512"
        - "osd crush chooseleaf type = 1"

    - name: collect bootstrapped Ceph files into the role
      fetch:
        src: /tmp/ceph-bootstrap/{{ item }}
        dest: roles/pvc/files/ceph/
        flat: yes
      with_items:
        - ceph.conf
        - ceph.mon.keyring
        - ceph.client.admin.keyring
        - ceph.osd.bootstrap.keyring
        - monmap

    - name: remove the temporary bootstrap directory
      file:
        dest: /tmp/ceph-bootstrap
        state: absent
        force: yes
  run_once: true

- name: deploy out configurations to all nodes
  copy:
    src: ceph/{{ item }}
    dest: /etc/ceph/{{ item }}
    owner: ceph
    group: ceph
    mode: 0640
  with_items:
    - ceph.conf
    - ceph.mon.keyring
    - ceph.client.admin.keyring
    - ceph.osd.bootstrap.keyring
    - monmap

- name: create monitor data directory
  file:
    dest: sudo mkdir /var/lib/ceph/mon/ceph-{{ ansible_hostname }}
    state: directory
    owner: ceph
    group: ceph
    mode: 0750

- name: populate monitor with map and keys
  command: ceph-mon --mkfs -i {{ ansible_hostname }} --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring
  become_user: ceph

- name: touch monitor done file
  file:
    dest: /var/lib/ceph/mon/ceph-{{ ansible_hostname }}/done
    state: touch

- name: start the monitor daemon
  service:
    name: ceph-mon@{{ ansible_hostname }}
    state: started
    enabled: yes

- name: create Libvirt keyring
  command: ceph auth get-or-create client.libvirt mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=pvc*'
  run_once: yes