From 01763088baf731d25fa96f5840edac48db9fb7fc Mon Sep 17 00:00:00 2001
From: "Joshua M. Boniface" <joshua@boniface.me>
Date: Wed, 12 Jun 2019 10:33:48 -0400
Subject: [PATCH] Reorganize some bootstrap tasks

---
 roles/pvc/tasks/add_cluster_ips.yml    | 16 ++++++++++++++++
 roles/pvc/tasks/bootstrap_ceph.yml     | 23 ++++++++---------------
 roles/pvc/tasks/ceph.yml               |  3 +++
 roles/pvc/tasks/pvc.yml                |  7 ++++++-
 roles/pvc/tasks/remove_cluster_ips.yml | 17 +++++++++++++++++
 5 files changed, 50 insertions(+), 16 deletions(-)
 create mode 100644 roles/pvc/tasks/add_cluster_ips.yml
 create mode 100644 roles/pvc/tasks/remove_cluster_ips.yml

diff --git a/roles/pvc/tasks/add_cluster_ips.yml b/roles/pvc/tasks/add_cluster_ips.yml
new file mode 100644
index 0000000..bd52341
--- /dev/null
+++ b/roles/pvc/tasks/add_cluster_ips.yml
@@ -0,0 +1,16 @@
+---
+- name: add IP addresses to cluster interfaces
+  command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+
+- name: add IP addresses to storage interfaces
+  command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
diff --git a/roles/pvc/tasks/bootstrap_ceph.yml b/roles/pvc/tasks/bootstrap_ceph.yml
index 824d552..778b5e7 100644
--- a/roles/pvc/tasks/bootstrap_ceph.yml
+++ b/roles/pvc/tasks/bootstrap_ceph.yml
@@ -1,19 +1,12 @@
 ---
-- name: add IP addresses to cluster interfaces
-  command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
+- name: stop the monitor daemon
+  service:
+    name: ceph-mon@{{ ansible_hostname }}
+    state: stopped
   ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
 
-- name: add IP addresses to storage interfaces
-  command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
-  ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
+- name: reset any systemd failures
+  command: systemctl reset-failed
 
 - block:
     - name: generate cluster FSID
@@ -102,7 +95,7 @@
     - name: collect bootstrapped Ceph files into the role
       fetch:
         src: /tmp/ceph-bootstrap/{{ item }}
-        dest: roles/pvc/files/ceph/
+        dest: roles/pvc/files/ceph/{{ hostvars[inventory_hostname].group_names[0] }}/ceph/
         flat: yes
       with_items:
         - ceph.conf
@@ -120,7 +113,7 @@
 
 - name: deploy out configurations to all nodes
   copy:
-    src: ceph/{{ item }}
+    src: ceph/{{ hostvars[inventory_hostname].group_names[0] }}/ceph/{{ item }}
     dest: /etc/ceph/{{ item }}
     owner: ceph
     group: ceph
diff --git a/roles/pvc/tasks/ceph.yml b/roles/pvc/tasks/ceph.yml
index 81858eb..51cb34d 100644
--- a/roles/pvc/tasks/ceph.yml
+++ b/roles/pvc/tasks/ceph.yml
@@ -39,5 +39,8 @@
     dest: /etc/ceph
     state: directory
 
+- include: add_cluster_ips.yml
+  when: newhost.changed
+
 - include: bootstrap_ceph.yml
   when: newhost.changed
diff --git a/roles/pvc/tasks/pvc.yml b/roles/pvc/tasks/pvc.yml
index 26c0585..0f110d0 100644
--- a/roles/pvc/tasks/pvc.yml
+++ b/roles/pvc/tasks/pvc.yml
@@ -19,10 +19,12 @@
   shell: "/usr/share/zookeeper/bin/zkCli.sh stat /nodes 2>&1 | grep -q 'Node does not exist'"
   register: cluster_init
   failed_when: no
+  run_once: yes
 
 - name: bootstrap a fresh cluster
   shell: /usr/bin/pvc init
-  when: cluster_init.rc == 0 and ansible_local.host_id == 1
+  when: cluster_init.rc == 0
+  run_once: yes
 
 - name: stop and disable unneccessary services
   service:
@@ -32,6 +34,9 @@
   with_items:
     - pdns.service
 
+- include: remove_cluster_ips.yml
+  when: newhost.changed
+
 - name: start and enable services
   service:
     name: "{{ item }}"
diff --git a/roles/pvc/tasks/remove_cluster_ips.yml b/roles/pvc/tasks/remove_cluster_ips.yml
new file mode 100644
index 0000000..1bfad41
--- /dev/null
+++ b/roles/pvc/tasks/remove_cluster_ips.yml
@@ -0,0 +1,17 @@
+---
+- name: remove IP addresses from cluster interfaces before starting PVC
+  command: ip address delete {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+
+- name: remove IP addresses from storage interfaces before starting PVC
+  command: ip address delete {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+