From a19d9c77ad1cbc9456d022970e234dd781d52ffc Mon Sep 17 00:00:00 2001
From: "Joshua M. Boniface" <joshua@boniface.me>
Date: Fri, 1 Sep 2023 15:42:20 -0400
Subject: [PATCH] Clean up some tasks during bootstrap; parallel PVC

---
 roles/pvc/tasks/ceph/main.yml                 | 14 +++++++---
 roles/pvc/tasks/common/add_cluster_ips.yml    | 26 +++++++++++--------
 roles/pvc/tasks/common/remove_cluster_ips.yml | 17 ------------
 roles/pvc/tasks/main.yml                      | 18 +++++--------
 4 files changed, 32 insertions(+), 43 deletions(-)
 delete mode 100644 roles/pvc/tasks/common/remove_cluster_ips.yml

diff --git a/roles/pvc/tasks/ceph/main.yml b/roles/pvc/tasks/ceph/main.yml
index c405b2e..c98c3bc 100644
--- a/roles/pvc/tasks/ceph/main.yml
+++ b/roles/pvc/tasks/ceph/main.yml
@@ -51,13 +51,16 @@
     - ceph.osd.bootstrap.keyring
     - monmap
 
-- name: create monitor data directory
+- name: create monitor and manager data directories
   file:
-    dest: /var/lib/ceph/mon/ceph-{{ ansible_hostname }}
+    dest: /var/lib/ceph/{{ item }}/ceph-{{ ansible_hostname }}
     state: directory
     owner: ceph
     group: ceph
     mode: 0750
+  with_items:
+    - mon
+    - mgr
   when: newhost is defined and newhost
 
 - name: populate monitor with map and keys
@@ -65,11 +68,14 @@
   become_user: ceph
   when: newhost is defined and newhost
 
-- name: touch monitor done file
+- name: touch monitor and manager done files
   file:
-    dest: /var/lib/ceph/mon/ceph-{{ ansible_hostname }}/done
+    dest: /var/lib/ceph/{{ item }}/ceph-{{ ansible_hostname }}/done
     state: touch
   become_user: ceph
+  with_items:
+    - mon
+    - mgr
   when: newhost is defined and newhost
 
 - name: start and enable daemons
diff --git a/roles/pvc/tasks/common/add_cluster_ips.yml b/roles/pvc/tasks/common/add_cluster_ips.yml
index bd52341..28b3f8e 100644
--- a/roles/pvc/tasks/common/add_cluster_ips.yml
+++ b/roles/pvc/tasks/common/add_cluster_ips.yml
@@ -1,16 +1,20 @@
 ---
-- name: add IP addresses to cluster interfaces
-  command: ip address add {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
+- name: create cluster bridge interface
+  command: brctl addbr brcluster
+  ignore_errors: yes
+
+- name: add underlying interface to bridge
+  command: brctl addif brcluster {{ pvc_cluster_device }}
+  ignore_errors: yes
+
+- name: set brcluster up
+  command: ip link set brcluster up
+  ignore_errors: yes
+
+- name: add IP addresses to cluster interfaces
+  command: ip address add {% for node in pvc_nodes if node.hostname == ansible_hostname %}{{ node.cluster_ip }}/{{ node.cluster_cidr }}{% endfor %} dev brcluster
   ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
 
 - name: add IP addresses to storage interfaces
-  command: ip address add {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
+  command: ip address add {% for node in pvc_nodes if node.hostname == ansible_hostname %}{{ node.storage_ip }}/{{ node.storage_cidr }}{% endfor %} dev {{ pvc_storage_device }}
   ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
diff --git a/roles/pvc/tasks/common/remove_cluster_ips.yml b/roles/pvc/tasks/common/remove_cluster_ips.yml
deleted file mode 100644
index 1bfad41..0000000
--- a/roles/pvc/tasks/common/remove_cluster_ips.yml
+++ /dev/null
@@ -1,17 +0,0 @@
----
-- name: remove IP addresses from cluster interfaces before starting PVC
-  command: ip address delete {{ item.cluster_ip }}/{{ item.cluster_cidr }} dev {{ pvc_cluster_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
-  ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
-
-- name: remove IP addresses from storage interfaces before starting PVC
-  command: ip address delete {{ item.storage_ip }}/{{ item.storage_cidr }} dev {{ pvc_storage_device }}
-  delegate_to: "{{ item.hostname }}"
-  run_once: yes
-  ignore_errors: yes
-  with_items:
-    - "{{ pvc_nodes }}"
-
diff --git a/roles/pvc/tasks/main.yml b/roles/pvc/tasks/main.yml
index cc5e3a3..286c2ba 100644
--- a/roles/pvc/tasks/main.yml
+++ b/roles/pvc/tasks/main.yml
@@ -41,21 +41,17 @@
 - include: frr/main.yml
   tags: pvc-frr
 
-# Install PVC sequentially
+# Install PVC daemon
 - include: pvc/main.yml
   tags: pvc-daemon
-  run_once: true
-  delegate_to: "{{ play_host }}"
-  with_items: "{{ play_hosts }}"
-  loop_control:
-    loop_var: play_host
-
-- include: common/remove_cluster_ips.yml
-  when: newhost is defined and newhost
-  tags: always
+#  run_once: true
+#  delegate_to: "{{ play_host }}"
+#  with_items: "{{ play_hosts }}"
+#  loop_control:
+#    loop_var: play_host
 
 - name: restart server on first install
-  shell: 'sleep 1 && shutdown -r now "Ansible updates triggered"'
+  shell: 'sleep 3 && shutdown -r now "Ansible updates triggered"'
   async: 1
   poll: 0
   ignore_errors: yes