Reorganize some bootstrap tasks

roles/pvc/tasks/add_cluster_ips.yml

@@ -0,0 +1,16 @@
+---
+- name: add IP addresses to cluster interfaces
+  command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+
+- name: add IP addresses to storage interfaces
+  command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"

roles/pvc/tasks/bootstrap_ceph.yml

@@ -1,19 +1,12 @@
 ---
-- name: add IP addresses to cluster interfaces
-  command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
+- name: stop the monitor daemon
+  service:
+    name: ceph-mon@{{ ansible_hostname }}
+    state: stopped
   ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
 
-- name: add IP addresses to storage interfaces
-  command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
-  ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
+- name: reset any systemd failures
+  command: systemctl reset-failed
 
 - block:
     - name: generate cluster FSID
@@ -102,7 +95,7 @@
     - name: collect bootstrapped Ceph files into the role
       fetch:
         src: /tmp/ceph-bootstrap/{{ item }}
-        dest: roles/pvc/files/ceph/
+        dest: roles/pvc/files/ceph/{{ hostvars[inventory_hostname].group_names[0] }}/ceph/
         flat: yes
       with_items:
         - ceph.conf
@@ -120,7 +113,7 @@
 
 - name: deploy out configurations to all nodes
   copy:
-    src: ceph/{{ item }}
+    src: ceph/{{ hostvars[inventory_hostname].group_names[0] }}/ceph/{{ item }}
     dest: /etc/ceph/{{ item }}
     owner: ceph
     group: ceph

roles/pvc/tasks/ceph.yml

@@ -39,5 +39,8 @@
     dest: /etc/ceph
     state: directory
 
+- include: add_cluster_ips.yml
+  when: newhost.changed
+
 - include: bootstrap_ceph.yml
   when: newhost.changed

roles/pvc/tasks/pvc.yml

@@ -19,10 +19,12 @@
   shell: "/usr/share/zookeeper/bin/zkCli.sh stat /nodes 2>&1 | grep -q 'Node does not exist'"
   register: cluster_init
   failed_when: no
+  run_once: yes
 
 - name: bootstrap a fresh cluster
   shell: /usr/bin/pvc init
-  when: cluster_init.rc == 0 and ansible_local.host_id == 1
+  when: cluster_init.rc == 0
+  run_once: yes
 
 - name: stop and disable unneccessary services
   service:
@@ -32,6 +34,9 @@
   with_items:
     - pdns.service
 
+- include: remove_cluster_ips.yml
+  when: newhost.changed
+
 - name: start and enable services
   service:
     name: "{{ item }}"

roles/pvc/tasks/remove_cluster_ips.yml

@@ -0,0 +1,17 @@
+---
+- name: remove IP addresses from cluster interfaces before starting PVC
+  command: ip address delete {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+
+- name: remove IP addresses from storage interfaces before starting PVC
+  command: ip address delete {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
+  delegate_to: "{{ item.hostname }}"
+  run_once: yes
+  ignore_errors: yes
+  with_items:
+    - "{{ pvc_nodes }}"
+