pvc-ansible/oneshot/upgrade-pvc-cluster_deb12.yml

---
# Play 1: Sanity check, initial configuration upgrade, information gathering, patroni freeze
- hosts: all
  remote_user: deploy
  become: yes
  become_user: root
  gather_facts: yes
  vars:
    minimum_pvc_version: 0.9.69
  tasks:
    - name: check that cluster is on the minimum PVC version or newer
      shell: "dpkg --compare-versions $(pvc --quiet cluster status -f json | jq '.pvc_version' | tr -d '\"') ge {{ minimum_pvc_version }}"
      run_once: yes

    - name: set PVC maintenance mode
      command: pvc cluster maintenance on
      ignore_errors: yes

    - name: include base role
      import_role:
        name: base

    - name: include pvc role
      import_role:
        name: pvc

    - name: get current postgresql directory
      shell: "find /usr/lib/postgresql -mindepth 1 -maxdepth 1 -type d -name '[0-9][0-9]' | sort -n | tail -1"
      register: old_postgres_dir_output

    - name: set old_postgres_bin_dir fact
      set_fact:
        old_postgres_bin_dir: "{{ old_postgres_dir_output.stdout.strip() }}/bin"

    - name: get current patroni leader node
      shell: "patronictl -c /etc/patroni/config.yml list --format json | jq '.[] | select(.Role == \"Leader\") | .Member' | tr -d '\"'"
      register: patroni_leader_output

    - name: set patroni_leader fact
      set_fact:
        patroni_leader: "{% for node in ansible_play_hosts if node.split('.')[0].strip() == patroni_leader_output.stdout.strip() %}{{ node }}{% endfor %}"

    - name: set patroni_followers fact
      set_fact:
        patroni_followers: "{{ ansible_play_hosts | difference([ patroni_leader ]) }}"

    - debug:
        var: patroni_leader
    - debug:
        var: patroni_followers

    - name: fail out if patroni_leader is empty
      command: "echo {{ patroni_leader }}"
      register: check_output
      failed_when: check_output.stdout == ""

    - name: stop and mask patroni service on followers to perform database upgrade (later)
      service:
        name: patroni
        state: stopped
        masked: yes
      run_once: yes
      delegate_to: "{{ item }}"
      loop: "{{ patroni_followers }}"

    - name: stop and mask patroni service on leader to perform database upgrade (later)
      service:
        name: patroni
        state: stopped
        masked: yes
      run_once: yes
      delegate_to: "{{ patroni_leader }}"

# Play 2: Per-node upgrade to Debian 12
- hosts: all
  remote_user: deploy
  become: yes
  become_user: root
  gather_facts: yes
  serial: 1
  tasks:
    - name: set Debian details (with ansible_distribution_*)
      set_fact:
        debian_version: "{{ ansible_distribution_major_version }}"
        debian_codename: "{{ ansible_distribution_release }}"
      when: ansible_distribution_release is defined

    - name: set Debian details (with ansible_lsb)
      set_fact:
        debian_version: "{{ ansible_lsb.major_release }}"
        debian_codename: "{{ ansible_lsb.codename }}"
      when: ansible_lsb.codename is defined

    - name: secondary node
      command: "pvc node secondary {{ ansible_hostname }}"
      ignore_errors: yes

    - name: wait 30 seconds for system to stabilize
      pause:
        seconds: 30
      become: no
      connection: local

    - name: flush node
      command: "pvc node flush {{ ansible_hostname }} --wait"
      ignore_errors: yes

    - name: ensure VMs are migrated away
      shell: "virsh list | grep running | wc -l"
      register: virshcount
      failed_when: virshcount.stdout != "0"
      until: virshcount.stdout == "0"
      retries: 60
      delay: 10

    - name: make sure all VMs have migrated
      shell: "pvc node info {{ ansible_hostname }} | grep '^Domain State' | awk '{ print $NF }'"
      register: pvcflush
      failed_when: pvcflush.stdout != 'flushed'
      until: pvcflush.stdout == 'flushed'
      retries: 60
      delay: 10

    - name: wait 15 seconds for system to stabilize
      pause:
        seconds: 15
      become: no
      connection: local

    - name: stop PVC daemon cleanly
      service:
        name: pvcnoded
        state: stopped

    - name: stop Zookeeper daemon cleanly
      service:
        name: zookeeper
        state: stopped

    - name: wait 15 seconds for system to stabilize
      pause:
        seconds: 15
      become: no
      connection: local

    - name: set OSD noout
      command: pvc storage osd set noout
      ignore_errors: yes

    - name: get running OSD services
      shell: "systemctl | awk '{ print $1 }' | grep 'ceph-osd@[0-9]*.service'"
      ignore_errors: yes
      register: osd_services

    - name: stop Ceph OSD daemons cleanly
      service:
        name: "{{ item }}"
        state: stopped
      ignore_errors: yes
      with_items: "{{ osd_services.stdout_lines }}"

    - name: stop Ceph Monitor daemon cleanly
      service:
        name: "ceph-mon@{{ ansible_hostname }}"
        state: stopped
      ignore_errors: yes

    - name: stop Ceph Manager daemon cleanly
      service:
        name: "ceph-mgr@{{ ansible_hostname }}"
        state: stopped
      ignore_errors: yes

    - name: wait 30 seconds for system to stabilize
      pause:
        seconds: 30
      become: no
      connection: local

    - name: remove possible obsolete cset configuration
      file:
        dest: "/etc/systemd/system/{{ item }}"
        state: absent
      with_items:
        - ceph-osd@.service.d
        - ceph-osd-cpuset.service

    - name: replace sources.list entries will bookworm
      replace:
        dest: "{{ item }}"
        regexp: "{{ debian_codename }}"
        replace: "bookworm"
      with_items:
        - /etc/apt/sources.list

    - name: replace sources.list non-free with non-free-firmware
      replace:
        dest: "{{ item }}"
        regexp: "non-free$"
        replace: "non-free-firmware"
      with_items:
        - /etc/apt/sources.list

    - name: remove security entry if on Debian 10
      lineinfile:
        dest: /etc/apt/sources.list
        regexp: "security.debian.org"
        state: absent
      when: debian_version|int < 11

    - name: link python to python3 on Debian 10
      file:
        state: link
        src: python3
        dest: /usr/bin/python
        force: yes
      when: debian_version|int < 11

    - name: update apt cache
      apt:
        update_cache: yes
      register: apt_res
      retries: 5
      until: apt_res is success

    # This seems insane, but works around a fatal error upgrading from d10 to d12
    - name: perform initial upgrade on Debian 10
      block:
        - name: install script to perform safe d10->d12 upgrade
          copy:
            dest: /tmp/upgrade-d10.sh
            mode: 0755
            content: |
              #!/usr/bin/env bash
              exec 1>/tmp/upgrade-d10.log
              exec 2>&1
              set -o xtrace
              export DEBIAN_FRONTEND=noninteractive
              echo "Forcibly update libcrypt1 and libc6 to avoid conflicts"
              apt-get --download-only install --yes libcrypt1
              dpkg --force-all --install /var/cache/apt/archives/{libcrypt1,libpam0g,libc6}*.deb
              echo "Fix broken packages"
              apt --fix-broken --option Dpkg::Options::="--force-confold" install --yes
              echo "Upgrade core apt packages (forcing confold)"
              apt-get --no-install-recommends --option Dpkg::Options::="--force-confold" install --yes apt dpkg dpkg-dev base-files zstd
              echo "Upgrade ca-certificates (forcing confold)"
              apt-get --no-install-recommends --option Dpkg::Options::="--force-confold" install --yes ca-certificates
              echo "Upgrade sudo (forcing confold)"
              apt-get --no-install-recommends --option Dpkg::Options::="--force-confold" install --yes sudo
              echo "Upgrade core PVC packages (forcing confold)"
              apt-get --no-install-recommends --option Dpkg::Options::="--force-confold" install --yes ceph-osd ceph-mds ceph-mon ceph-mgr radosgw zookeeper zookeeper-bin patroni libvirt-daemon-system frr
              echo "Perform main upgrade (forcing confnew)"
              apt-get --no-install-recommends --option Dpkg::Options::="--force-confnew" upgrade --yes
              echo "Perform main dist-upgrade (forcing confnew)"
              apt-get --no-install-recommends --option Dpkg::Options::="--force-confnew" dist-upgrade --yes
              echo "Restart SSH so Ansible can continue"
              systemctl restart ssh

        - name: run script to perform safe d10->d12 upgrade (will take a long time)
          shell: /tmp/upgrade-d10.sh

        - name: install python-is-python3
          apt:
            name: python-is-python3
            state: latest

      when: debian_version|int < 11

    - name: apt upgrade
      apt:
        upgrade: safe
      register: apt_res
      retries: 5
      until: apt_res is success

    - name: apt dist upgrade and cleanup
      apt:
        autoremove: yes
        autoclean: yes
        upgrade: dist
      register: apt_res
      retries: 5
      until: apt_res is success

    - name: clean up obsolete kernels
      command: /usr/local/sbin/kernel-cleanup.sh

    - name: clean up obsolete packages
      command: /usr/local/sbin/dpkg-cleanup.sh

    - name: clean apt archives
      file:
        dest: /var/cache/apt/archives
        state: absent

    - name: regather facts
      setup:

    - name: include base role
      import_role:
        name: base

    - name: include pvc role
      import_role:
        name: pvc

    - name: apt full upgrade and cleanup
      apt:
        update_cache: yes
        autoremove: yes
        autoclean: yes
        upgrade: full
      register: apt_res
      retries: 5
      until: apt_res is success

    - name: remove obsolete database directories
      file:
        dest: "{{ item }}"
        state: absent
      with_items:
        - "/etc/postgresql/13"
        - "/var/lib/postgresql/13"

    - name: restart system
      reboot:
        post_reboot_delay: 15
        reboot_timeout: 1800

    - name: make sure all OSDs are active
      shell: "ceph osd stat | grep 'osds:' | awk '{ if ( $1 == $3 ) { print \"OK\" } else { print \"NOK\" } }'"
      register: osdstat
      failed_when: osdstat.stdout == "NOK"
      until: osdstat.stdout == "OK"
      retries: 60
      delay: 10

    - name: make sure all PGs have recovered
      shell: "ceph health | grep -wo 'Degraded data redundancy'"
      register: cephhealth
      failed_when: cephhealth.stdout == "Degraded data redundancy'"
      until: cephhealth.stdout == ""
      retries: 60
      delay: 10

    - name: unset OSD noout
      command: pvc storage osd unset noout

    - name: wait 30 seconds for system to stabilize
      pause:
        seconds: 30
      become: no
      connection: local

    - name: unflush node
      command: "pvc node ready {{ ansible_hostname }} --wait"

    - name: make sure all VMs have returned
      shell: "pvc node info {{ ansible_hostname }} | grep '^Domain State' | awk '{ print $NF }'"
      register: pvcunflush
      failed_when: pvcunflush.stdout != 'ready'
      until: pvcunflush.stdout == 'ready'
      retries: 60
      delay: 10

    - name: wait 30 seconds for system to stabilize
      pause:
        seconds: 30
      become: no
      connection: local

    - name: reset any systemd failures
      command: systemctl reset-failed

    - name: wait 30 seconds for system to stabilize
      pause:
        seconds: 30
      become: no
      connection: local

# Play 3: Ceph upgrades for Debian 12
- hosts: all
  remote_user: deploy
  become: yes
  become_user: root
  gather_facts: yes
  tasks:
    - name: backup CRUSH map
      command: ceph osd getcrushmap -o /srv/backups/backup-crushmap-deb12-upgrade

    - block:
        - name: disable insecure global id reclaim in Ceph
          command: ceph config set mon auth_allow_insecure_global_id_reclaim false

        - name: disable default pg autoscaler
          command: ceph config set global osd_pool_default_pg_autoscale_mode off

        - name: get pools
          shell: "ceph osd lspools | awk '{ $1=\"\"; print $0 }' | tr -d ' '"
          register: pool_output

        - name: set pools fact
          set_fact:
            ceph_pools: "{{ pool_output.stdout.split('\n') | list }}"

        - name: disable pg autoscaler on each pool
          command: "ceph osd pool set {{ item }} pg_autoscale_mode off"
          loop: "{{ ceph_pools }}"

        - name: set OSDs to require pacific
          command: ceph osd require-osd-release pacific

        - name: update CRUSH map
          command: ceph osd crush set-all-straw-buckets-to-straw2

        - name: set OSDs to quick fsck
          command: ceph config set osd bluestore_fsck_quick_fix_on_mount true
      run_once: yes

    - name: restart ceph-mgr
      service:
        name: "ceph-mgr@{{ ansible_hostname }}"
        state: restarted

# Play 4: Ceph OSD restart (serial per-node)
- hosts: all
  remote_user: deploy
  become: yes
  become_user: root
  gather_facts: yes
  serial: 1
  tasks:
    - name: restart all OSDs on node
      shell: "systemctl restart --all ceph-osd@*"

    - name: make sure all OSDs are active
      shell: "ceph osd stat | grep 'osds:' | awk '{ if ( $1 == $3 ) { print \"OK\" } else { print \"NOK\" } }'"
      register: osdstat
      failed_when: osdstat.stdout == "NOK"
      until: osdstat.stdout == "OK"
      retries: 60
      delay: 10

    - name: make sure all PGs have recovered
      shell: "ceph health | grep -wo 'Degraded data redundancy'"
      register: cephhealth
      failed_when: cephhealth.stdout == "Degraded data redundancy'"
      until: cephhealth.stdout == ""
      retries: 60
      delay: 10

# Play 4: Patroni upgrades for Debian 12
- hosts: all
  remote_user: deploy
  become: yes
  become_user: root
  gather_facts: yes
  vars:
    new_postgres_version: 15
  tasks:
    - name: stop patroni service on followers to perform database upgrade
      service:
        name: patroni
        state: stopped
      run_once: yes
      delegate_to: "{{ item }}"
      loop: "{{ patroni_followers }}"

    - name: stop patroni service on leader to perform database upgrade
      service:
        name: patroni
        state: stopped
      run_once: yes
      delegate_to: "{{ patroni_leader }}"

    - block:
        - name: initialize new postgres database
          shell:
            cmd: "sudo -u postgres /usr/lib/postgresql/{{ new_postgres_version }}/bin/initdb -D /var/lib/postgresql/{{ new_postgres_version }}/pvc"
            chdir: "/var/lib/postgresql"

        - name: enable data checksums in new database
          shell:
            cmd: "sudo -u postgres /usr/lib/postgresql/{{ new_postgres_version }}/bin/pg_checksums --enable /var/lib/postgresql/{{ new_postgres_version }}/pvc"
            chdir: "/var/lib/postgresql"

        - name: run postgres upgrade
          shell:
            cmd: "sudo -u postgres /usr/lib/postgresql/{{ new_postgres_version }}/bin/pg_upgrade -b {{ old_postgres_bin_dir }} -d /var/lib/postgresql/patroni/pvc -D /var/lib/postgresql/{{ new_postgres_version }}/pvc"
            chdir: "/var/lib/postgresql"

        - name: move old postgres database out of the way
          shell:
            cmd: "sudo -u postgres mv /var/lib/postgresql/patroni/pvc /var/lib/postgresql/patroni/pvc.old"
            chdir: "/var/lib/postgresql"

        - name: move new postgres database into place
          shell:
            cmd: "sudo -u postgres mv /var/lib/postgresql/{{ new_postgres_version }}/pvc /var/lib/postgresql/patroni/pvc"
            chdir: "/var/lib/postgresql"

        - name: ensure recovery.conf is absent
          file:
            dest: /var/lib/postgresql/patroni/pvc/recovery.conf
            state: absent

        - name: delete patroni cluster znodes
          shell: "/usr/share/zookeeper/bin/zkCli.sh -server {{ ansible_hostname }}:2181 deleteall /patroni/pvc"

        - name: start patroni service on leader
          service:
            name: patroni
            state: started
            masked: no
      run_once: yes
      delegate_to: "{{ patroni_leader }}"

    - name: remove old data directory on patroni followers
      file:
        dest: /var/lib/postgresql/patroni/pvc
        state: absent
      run_once: yes
      delegate_to: "{{ item }}"
      loop: "{{ patroni_followers }}"

    - name: start patroni service on followers
      service:
        name: patroni
        state: started
        masked: no
      run_once: yes
      delegate_to: "{{ item }}"
      loop: "{{ patroni_followers }}"

    - name: wait 30 seconds for system to stabilize
      pause:
        seconds: 30
      become: no
      connection: local

    - name: restart PVC daemons on all nodes
      shell: "systemctl restart pvchealthd && systemctl restart pvcworkerd && systemctl restart pvcnoded && sleep 30"
      run_once: yes
      delegate_to: "{{ item }}"
      loop: "{{ ansible_play_hosts }}"

    - name: wait 30 seconds for system to stabilize
      pause:
        seconds: 30
      become: no
      connection: local

    - name: set first node as primary coordinator
      command: "pvc node primary --wait {{ ansible_play_hosts[0].split('.')[0] }}"
      run_once: yes
      delegate_to: "{{ ansible_play_hosts[0] }}"

    - name: wait 15 seconds for system to stabilize
      pause:
        seconds: 15
      become: no
      connection: local

    - name: unset PVC maintenance mode
      command: pvc cluster maintenance off