oneshot/reboot-pvc-cluster.yml

@@ -160,12 +160,6 @@
   become_user: root
   gather_facts: yes
   tasks:
-    - name: wait 15 seconds for system to stabilize
-      pause:
-        seconds: 15
-      become: no
-      connection: local
-
     - name: unset PVC maintenance mode
       command: pvc cluster maintenance off
       run_once: yes

oneshot/update-pvc-cluster.yml

@@ -244,12 +244,6 @@
   become_user: root
   gather_facts: yes
   tasks:
-    - name: wait 15 seconds for system to stabilize
-      pause:
-        seconds: 15
-      become: no
-      connection: local
-
     - name: unset PVC maintenance mode
       command: pvc cluster maintenance off
       run_once: yes

oneshot/update-pvc-daemons.yml

@@ -145,12 +145,6 @@
         - pvcapid
       run_once: yes
 
-    - name: wait 15 seconds for system to stabilize
-      pause:
-        seconds: 15
-      become: no
-      connection: local
-
     - name: unset PVC maintenance mode on first node
       command: pvc cluster maintenance off
       run_once: yes

oneshot/upgrade-pvc-cluster_deb11.yml

@@ -0,0 +1,229 @@
+---
+- hosts: all
+  remote_user: deploy
+  become: yes
+  become_user: root
+  gather_facts: yes
+  serial: 1
+  tasks:
+    - name: set PVC maintenance mode
+      command: pvc cluster maintenance on
+
+    - name: secondary node
+      command: "pvc node secondary {{ ansible_hostname }}"
+      ignore_errors: yes
+  
+    - name: wait 30 seconds for system to stabilize
+      pause:
+        seconds: 30
+      become: no
+      connection: local
+  
+    - name: flush node
+      command: "pvc node flush {{ ansible_hostname }} --wait"
+  
+    - name: ensure VMs are migrated away
+      shell: "virsh list | grep running | wc -l"
+      register: virshcount
+      failed_when: virshcount.stdout != "0"
+      until: virshcount.stdout == "0"
+      retries: 60
+      delay: 10
+
+    - name: make sure all VMs have migrated
+      shell: "pvc node info {{ ansible_hostname }} | grep '^Domain State' | awk '{ print $NF }'"
+      register: pvcflush
+      failed_when: pvcflush.stdout != 'flushed'
+      until: pvcflush.stdout == 'flushed'
+      retries: 60
+      delay: 10
+
+    - name: wait 15 seconds for system to stabilize
+      pause:
+        seconds: 15
+      become: no
+      connection: local
+
+    - name: stop PVC daemon cleanly
+      service:
+        name: pvcnoded
+        state: stopped
+
+    - name: stop Zookeeper daemon cleanly
+      service:
+        name: zookeeper
+        state: stopped
+
+    - name: wait 15 seconds for system to stabilize
+      pause:
+        seconds: 15
+      become: no
+      connection: local
+
+    - name: set OSD noout
+      command: pvc storage osd set noout
+
+    - name: get running OSD services
+      shell: "systemctl | awk '{ print $1 }' | grep 'ceph-osd@[0-9]*.service'"
+      ignore_errors: yes
+      register: osd_services
+
+    - name: stop Ceph OSD daemons cleanly
+      service:
+        name: "{{ item }}"
+        state: stopped
+      ignore_errors: yes
+      with_items: "{{ osd_services.stdout_lines }}"
+
+    - name: stop Ceph Monitor daemon cleanly
+      service:
+        name: "ceph-mon@{{ ansible_hostname }}"
+        state: stopped
+      ignore_errors: yes
+
+    - name: stop Ceph Manager daemon cleanly
+      service:
+        name: "ceph-mgr@{{ ansible_hostname }}"
+        state: stopped
+      ignore_errors: yes
+
+    - name: wait 30 seconds for system to stabilize
+      pause:
+        seconds: 30
+      become: no
+      connection: local
+
+    - name: remove possible obsolete cset configuration
+      file:
+        dest: /etc/systemd/system/ceph-osd@.service.d
+        state: absent
+
+    - name: replace sources.list entries will bullseye
+      replace:
+        dest: "{{ item }}"
+        regexp: "buster"
+        replace: "bullseye"
+      with_items:
+        - /etc/apt/sources.list
+
+    - name: remove security entry
+      lineinfile:
+        dest: /etc/apt/sources.list
+        regexp: "security.debian.org"
+        state: absent
+
+    - name: update apt cache
+      apt:
+        update_cache: yes
+
+    - name: install python-is-python3
+      apt:
+        name: python-is-python3
+        state: latest
+
+    - name: apt dist upgrade and cleanup
+      apt:
+        update_cache: yes
+        autoremove: yes
+        autoclean: yes
+        upgrade: dist
+
+    - name: clean up obsolete kernels
+      command: /usr/local/sbin/kernel-cleanup.sh
+
+    - name: clean up obsolete packages
+      command: /usr/local/sbin/dpkg-cleanup.sh
+
+    - name: clean apt archives
+      file:
+        dest: /var/cache/apt/archives
+        state: absent
+
+    - name: regather facts
+      setup:
+
+    - name: include base role
+      import_role:
+        name: base
+
+    - name: include pvc role
+      import_role:
+        name: pvc
+
+    - name: apt full upgrade and cleanup
+      apt:
+        update_cache: yes
+        autoremove: yes
+        autoclean: yes
+        upgrade: full
+
+    - name: remove obsolete database directories
+      file:
+        dest: "{{ item }}"
+        state: absent
+      with_items:
+        - "/etc/postgresql/13"
+        - "/var/lib/postgresql/13"
+
+    - name: restart system
+      reboot:
+        post_reboot_delay: 15
+        reboot_timeout: 1800
+
+    - name: make sure all OSDs are active
+      shell: "ceph osd stat | grep 'osds:' | awk '{ if ( $1 == $3 ) { print \"OK\" } else { print \"NOK\" } }'"
+      register: osdstat
+      failed_when: osdstat.stdout == "NOK"
+      until: osdstat.stdout == "OK"
+      retries: 60
+      delay: 10
+
+    - name: make sure all PGs have recovered
+      shell: "ceph health | grep -wo 'Degraded data redundancy'"
+      register: cephhealth
+      failed_when: cephhealth.stdout == "Degraded data redundancy'"
+      until: cephhealth.stdout == ""
+      retries: 60
+      delay: 10
+
+    - name: unset OSD noout
+      command: pvc storage osd unset noout
+
+    - name: unflush node
+      command: "pvc node ready {{ ansible_hostname }} --wait"
+ 
+    - name: make sure all VMs have returned
+      shell: "pvc node info {{ ansible_hostname }} | grep '^Domain State' | awk '{ print $NF }'"
+      register: pvcunflush
+      failed_when: pvcunflush.stdout != 'ready'
+      until: pvcunflush.stdout == 'ready'
+      retries: 60
+      delay: 10
+
+    - name: wait 30 seconds for system to stabilize
+      pause:
+        seconds: 30
+      become: no
+      connection: local
+
+    - name: reset any systemd failures
+      command: systemctl reset-failed
+
+    - name: wait 30 seconds for system to stabilize
+      pause:
+        seconds: 30
+      become: no
+      connection: local
+
+    - name: unset PVC maintenance mode
+      command: pvc cluster maintenance off
+
+- hosts: all
+  remote_user: deploy
+  become: yes
+  become_user: root
+  gather_facts: yes
+  tasks:
+    - name: disable insecure global id reclaim in Ceph
+      command: ceph config set mon auth_allow_insecure_global_id_reclaim false
+      run_once: yes

oneshot/upgrade-pvc-cluster_deb12.yml

@@ -544,22 +544,10 @@
       delegate_to: "{{ item }}"
       loop: "{{ ansible_play_hosts }}"
 
-    - name: wait 30 seconds for system to stabilize
-      pause:
-        seconds: 30
-      become: no
-      connection: local
-
     - name: set first node as primary coordinator
       command: "pvc node primary --wait {{ ansible_play_hosts[0].split('.')[0] }}"
       run_once: yes
       delegate_to: "{{ ansible_play_hosts[0] }}"
 
-    - name: wait 15 seconds for system to stabilize
-      pause:
-        seconds: 15
-      become: no
-      connection: local
-
     - name: unset PVC maintenance mode
       command: pvc cluster maintenance off