From 20595c7027a9efb606cb2932bf3a5a3b66b9a373 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Tue, 11 Jun 2019 11:17:56 -0400 Subject: [PATCH] Add bootstrapping of Ceph --- roles/pvc/tasks/bootstrap_ceph.yml | 156 +++++++++++++++++++++++++++++ roles/pvc/tasks/ceph.yml | 15 +-- 2 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 roles/pvc/tasks/bootstrap_ceph.yml diff --git a/roles/pvc/tasks/bootstrap_ceph.yml b/roles/pvc/tasks/bootstrap_ceph.yml new file mode 100644 index 0000000..004b73c --- /dev/null +++ b/roles/pvc/tasks/bootstrap_ceph.yml @@ -0,0 +1,156 @@ +--- +- name: add IP addresses to cluster interfaces + command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }} + delegate_to: "{{ item.hostname }}" + run_once: yes + ignore_errors: yes + with_items: + - "{{ pvc_nodes }}" + +- name: add IP addresses to storage interfaces + command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }} + delegate_to: "{{ item.hostname }}" + run_once: yes + ignore_errors: yes + with_items: + - "{{ pvc_nodes }}" + +- block: + - name: generate cluster FSID + command: uuidgen + register: fsid_raw + delegate_to: localhost + + - set_fact: + fsid: "{{ fsid_raw.stdout }}" + + - name: initialize bootstrap ceph.conf + file: + dest: /etc/ceph/ceph.conf + state: touch + + - name: set fsid in bootstrap ceph.conf + lineinfile: + dest: /etc/ceph/ceph.conf + line: "fsid = {{ fsid }}" + state: present + + - name: set mon initial members in bootstrap ceph.conf + lineinfile: + dest: /etc/ceph/ceph.conf + line: "mon initial members = {% for host in pvc_nodes %}{{ host.hostname }}{% if not loop.last %},{% endif %}{% endfor %}" + state: present + + - name: set mon hosts in bootstrap ceph.conf + lineinfile: + dest: /etc/ceph/ceph.conf + line: "mon host = {% for host in pvc_nodes %}{{ host.storage_ip }}{% if not loop.last %},{% endif %}{% endfor %}" + state: present + + - name: create temporary directory + file: + dest: /tmp/ceph-bootstrap + state: directory + + - name: create mon keyring + command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *' + + - name: create client admin keyring + command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' + + - name: add client admin keyring to mon keyring + command: ceph-authtool /tmp/ceph-bootstrap/ceph.mon.keyring --import-keyring /tmp/ceph-bootstrap/ceph.client.admin.keyring + + - name: create OSD bootstrap keyring + command: ceph-authtool --create-keyring /tmp/ceph-bootstrap/ceph.osd.bootstrap.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' + + - name: add OSD bootstrap keyring to mon keyring + command: ceph-authtool /tmp/ceph-bootstrap/ceph.mon.keyring --import-keyring /tmp/ceph-bootstrap/ceph.osd.bootstrap.keyring + + - name: create monmap + command: monmaptool --create --fsid {{ fsid }} /tmp/ceph-bootstrap/monmap + + - name: add monitors to monmap + command: monmaptool --add {{ item.hostname }} {{ item.storage_ip }} --fsid {{ fsid }} /tmp/ceph-bootstrap/monmap + with_items: + - "{{ pvc_nodes }}" + + - name: copy initial ceph.conf to the boostrap directory + copy: + src: /etc/ceph/ceph.conf + dest: /tmp/ceph-bootstrap/ceph.conf + remote_src: yes + + - name: add additional configuration lines to ceph.conf + lineinfile: + dest: /tmp/ceph-bootstrap/ceph.conf + line: "{{ item }}" + state: present + with_items: + - "public network = {{ pvc_cluster_subnet }}" + - "cluster network = {{ pvc_storage_subnet }}" + - "auth cluster required = cephx" + - "auth service required = cephx" + - "auth client required = cephx" + - "osd journal size = 2" + - "osd pool default size = 3" + - "osd pool default min size = 2" + - "osd pool default pg num = 512" + - "osd pool default pgp num = 512" + - "osd crush chooseleaf type = 1" + + - name: collect bootstrapped Ceph files into the role + fetch: + src: /tmp/ceph-bootstrap/{{ item }} + dest: roles/pvc/files/ceph/ + flat: yes + with_items: + - ceph.conf + - ceph.mon.keyring + - ceph.client.admin.keyring + - ceph.osd.bootstrap.keyring + - monmap + + - name: remove the temporary bootstrap directory + file: + dest: /tmp/ceph-bootstrap + state: absent + force: yes + run_once: true + +- name: deploy out configurations to all nodes + copy: + src: ceph/{{ item }} + dest: /etc/ceph/{{ item }} + owner: ceph + group: ceph + mode: 0640 + with_items: + - ceph.conf + - ceph.mon.keyring + - ceph.client.admin.keyring + - ceph.osd.bootstrap.keyring + - monmap + +- name: create monitor data directory + file: + dest: sudo mkdir /var/lib/ceph/mon/ceph-{{ ansible_hostname }} + state: directory + owner: ceph + group: ceph + mode: 0750 + +- name: populate monitor with map and keys + command: ceph-mon --mkfs -i {{ ansible_hostname }} --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring + become_user: ceph + +- name: touch monitor done file + file: + dest: /var/lib/ceph/mon/ceph-{{ ansible_hostname }}/done + state: touch + +- name: start the monitor daemon + service: + name: ceph-mon@{{ ansible_hostname }} + state: started + enabled: yes diff --git a/roles/pvc/tasks/ceph.yml b/roles/pvc/tasks/ceph.yml index 23d79c4..81858eb 100644 --- a/roles/pvc/tasks/ceph.yml +++ b/roles/pvc/tasks/ceph.yml @@ -1,4 +1,10 @@ --- +- name: initial deployment check + shell: "echo 'bootstrapped' > /etc/ceph-install" + register: newhost + args: + creates: "/etc/ceph-install" + - name: install packages apt: name: @@ -33,10 +39,5 @@ dest: /etc/ceph state: directory -- name: install ceph cluster configurations - template: - src: ceph/{{ item }}.j2 - dest: /etc/ceph/{{ item }} - with_items: - - ceph.conf - - ceph.client.admin.keyring +- include: bootstrap_ceph.yml + when: newhost.changed