pvc-ansible/roles/pvc/tasks/ceph/main.yml

---
- name: install packages
  apt:
    name:
      - ceph-osd
      - ceph-mds
      - ceph-mon
      - ceph-mgr
      - radosgw
    state: latest

- name: add admin users to ceph groups
  user:
    name: "{{ item.name }}"
    groups: ceph
    append: yes
  with_items: "{{ admin_users }}"

- name: install sysctl tweaks
  template:
    src: ceph/sysctl.conf.j2
    dest: /etc/sysctl.d/pvc-ceph.conf

- name: activate sysctl tweaks
  command: sysctl -p /etc/sysctl.d/pvc-ceph.conf

- name: install user limits overrides
  template:
    src: ceph/limits.conf.j2
    dest: /etc/security/limits.d/99-pvc-ceph.conf

- name: install ceph default config
  template:
    src: ceph/default.conf.j2
    dest: /etc/default/ceph

- name: create ceph configuration directory
  file:
    dest: /etc/ceph
    state: directory

- include: ceph/bootstrap.yml
  when: bootstrap is defined and bootstrap
  run_once: yes

- name: install configurations
  copy:
    src: ceph/{{ cluster_group }}/ceph/{{ item }}
    dest: /etc/ceph/{{ item }}
    owner: ceph
    group: ceph
    mode: 0640
  notify:
    - restart ceph-mon
    - restart ceph-mgr
  with_items:
    - ceph.conf
    - ceph.mon.keyring
    - ceph.client.admin.keyring
    - ceph.osd.bootstrap.keyring
    - monmap

- name: create monitor and manager data directories
  file:
    dest: /var/lib/ceph/{{ item }}/ceph-{{ ansible_hostname }}
    state: directory
    owner: ceph
    group: ceph
    mode: 0750
  with_items:
    - mon
    - mgr
  when: newhost is defined and newhost

- name: populate monitor with map and keys
  command: ceph-mon --cluster ceph --mkfs -i {{ ansible_hostname }} --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring
  become_user: ceph
  when: newhost is defined and newhost

- name: touch monitor and manager done files
  file:
    dest: /var/lib/ceph/{{ item }}/ceph-{{ ansible_hostname }}/done
    state: touch
  become_user: ceph
  with_items:
    - mon
    - mgr
  when: newhost is defined and newhost

- name: start monitor daemon
  service:
    name: "{{ item }}"
    state: started
  with_items:
    - ceph-mon@{{ ansible_hostname }}
  when: newhost is defined and newhost

- name: set msgr2 enabled
  command: ceph mon enable-msgr2
  run_once: yes
  retries: 6
  delay: 5
  register: result
  until: result.rc == 0

- name: set insecure_global_id_reclaim disabled
  command: ceph config set mon auth_allow_insecure_global_id_reclaim false 
  run_once: yes
  ignore_errors: yes

- name: create mgr auth keyring
  command: ceph auth get-or-create mgr.{{ ansible_hostname }} mon 'allow profile mgr' osd 'allow *' mds 'allow *' --out-file /var/lib/ceph/mgr/ceph-{{ ansible_hostname }}/keyring
  become_user: ceph
  args:
    creates: /var/lib/ceph/mgr/ceph-{{ ansible_hostname }}/keyring

- name: install OSD bootstrap keyring
  file:
    dest: /var/lib/ceph/bootstrap-osd/ceph.keyring
    src: /etc/ceph/ceph.osd.bootstrap.keyring
    state: link

- name: start but disable daemons
  service:
    name: "{{ item }}"
    state: started
    enabled: no
  with_items:
    - ceph-mon@{{ ansible_hostname }}
    - ceph-mgr@{{ ansible_hostname }}

# Single-node cluster ruleset
- name: remove default CRUSH replicated_rule ruleset
  command: ceph osd crush rule rm replicated_rule
  when: "{{ pvc_nodes | length }} == 1"

- name: add single-node CRUSH replicated_rule ruleset
  command: ceph osd crush rule create-replicated replicated_rule default osd
  when: "{{ pvc_nodes | length }} == 1"

# System OSD CPU shielding activation
- block:
    - name: install packages
      apt:
        name:
          - cpuset
          - numactl
        state: latest

    - name: install ceph-osd-cpuset controller config
      template:
        src: ceph/ceph-osd-cpuset-enable.j2
        dest: /etc/default/ceph-osd-cpuset

    - name: install ceph-osd-cpuset script
      template:
        src: ceph/ceph-osd-cpuset.j2
        dest: /usr/local/sbin/ceph-osd-cpuset
        mode: 0755

    - name: install ceph-osd-cpuset service unit
      template:
        src: ceph/ceph-osd-cpuset.service.j2
        dest: /etc/systemd/system/ceph-osd-cpuset.service
      register: systemd_file_cpuset

    - name: create ceph-osd override dropin directory
      file:
        dest: /etc/systemd/system/ceph-osd@.service.d
        state: directory

    - name: install ceph-osd override dropin
      template:
        src: ceph/ceph-osd-cpuset.conf.j2
        dest: /etc/systemd/system/ceph-osd@.service.d/cpuset.conf
      register: systemd_file_osd

    - name: reload systemd to apply previous changes
      command: "systemctl daemon-reload"
      when: systemd_file_cpuset.changed or systemd_file_osd.changed

    - name: enable ceph-osd-cpuset service
      service:
        name: ceph-osd-cpuset
        enabled: yes

    - debug:
        msg: "NOTICE: Any cpuset configs have NOT been applied to the running system. This node must be rebooted to apply these changes."
  tags: pvc-ceph-cpuset
  when:
    - pvc_shield_osds_enable is defined

- meta: flush_handlers