From dd451c70c3c642ad956ca22b27b8f60e5137dd1d Mon Sep 17 00:00:00 2001
From: "Joshua M. Boniface" <joshua@boniface.me>
Date: Fri, 15 Nov 2024 01:51:04 -0500
Subject: [PATCH] Add automirror support to Ansible

---
 group_vars/default/pvc.yml                    | 82 +++++++++++++++++++
 roles/pvc/defaults/main.yml                   | 30 +++++++
 roles/pvc/tasks/automirror/disable.yml        | 24 ++++++
 roles/pvc/tasks/automirror/enable.yml         | 23 ++++++
 roles/pvc/tasks/automirror/main.yml           |  7 ++
 roles/pvc/tasks/main.yml                      |  5 ++
 .../templates/autobackup/autobackup.yaml.j2   | 24 ------
 .../automirror/pvc-automirror.service.j2      |  9 ++
 .../automirror/pvc-automirror.timer.j2        |  9 ++
 roles/pvc/templates/pvc/pvc.conf.j2           | 22 ++++-
 10 files changed, 210 insertions(+), 25 deletions(-)
 create mode 100644 roles/pvc/tasks/automirror/disable.yml
 create mode 100644 roles/pvc/tasks/automirror/enable.yml
 create mode 100644 roles/pvc/tasks/automirror/main.yml
 delete mode 100644 roles/pvc/templates/autobackup/autobackup.yaml.j2
 create mode 100644 roles/pvc/templates/automirror/pvc-automirror.service.j2
 create mode 100644 roles/pvc/templates/automirror/pvc-automirror.timer.j2

diff --git a/group_vars/default/pvc.yml b/group_vars/default/pvc.yml
index dbaca1f..7f3cd9a 100644
--- a/group_vars/default/pvc.yml
+++ b/group_vars/default/pvc.yml
@@ -300,6 +300,88 @@ pvc_autobackup:
       # This example shows an fusermount3 unmount (e.g. for SSHFS) leveraging the backup_root_path variable
       # - "/usr/bin/fusermount3 -u {backup_root_path}"
 
+# PVC VM automirrors
+# > PVC supports automirrors, which can perform automatic snapshot-level VM mirrors of selected
+#   virtual machines based on tags. The mirrors are fully managed on a consistent schedule, and
+#   include both full and incremental varieties.
+# > To solve the shared storage issue and ensure mirrors are taken off-cluster, automaticmounting
+#   of remote filesystems is supported by automirror.
+pvc_automirror:
+  # Enable or disable automirror
+  # > If disabled, no timers or "/etc/pvc/automirror.yaml" configuration will be installed, and any
+  #   existing timers or configuration will be REMOVED on each run (even if manually created).
+  # > Since automirror is an integrated PVC CLI feature, the command will always be available regardless
+  #   of this setting, but without this option enabled, the lack of a "/etc/pvc/automirror.yaml" will
+  #   prevent its use.
+  enabled: no
+  # List of possible remote clusters to mirror to
+  destinations:
+    # The name of the cluster, used in tags (e.g. 'automirror:cluster2')
+    cluster2:
+      # The destination address, either an IP or an FQDN the destination API is reachable at
+      address: pvc.cluster2.mydomain.tld
+      # The destination port (usually 7370)
+      port: 7370
+      # The API prefix (usually '/api/v1') without a trailing slash
+      prefix: "/api/v1"
+      # The API key of the destination
+      key: 00000000-0000-0000-0000-000000000000
+      # Whether or not to use SSL for the connection
+      ssl: yes
+      # Whether or not to verify SSL for the connection
+      verify_ssl: yes
+      # Storage pool for VMs on the destination
+      pool: vms
+  # The default destination to send to, for VMs tagged without an explicit cluster
+  # > This is required even if there is only one destination!
+  default_destination: cluster2
+  # Set the VM tag(s) which will be selected for automirror
+  # > Automirror selects VMs based on their tags. If a VM has a tag present in this list, it will be
+  #   selected for automirror at runtime; if not it will be ignored.
+  # > Usually, the tag "automirror" here is sufficient; the administrator should then add this tag
+  #   to any VM(s) they want to use automirrors. However, any tag may be specified to keep the tag list
+  #   cleaner and more focused, should the administrator choose to.
+  # > A cluster can be explicitly set by suffixing `:clustername` from the cluster list to the tag.
+  # > If a VM has multiple `:clustername` tags, it will be mirrored to all of them sequentially
+  #   using the same source snapshot.
+  tags:
+    - automirror
+  # Automirror scheduling
+  schedule:
+    # Mirrors are performed at regular intervals via a systemd timer
+    # > This default schedule performs a mirror every 4 hours starting at midnight
+    # > These options use a systemd timer date string; see "man systemd.time" for details
+    time:  "*-*-* 00/4:00:00"
+    # The retention count specifies how many mirrors should be kept, based on the destination count
+    # for a given VM and cluster pairing to prevent failed mirrors from being cleaned up on the source
+    # > Retention cleanup is run after each full mirror, and thus, that mirror is counted in this number
+    # > For example, a value of 7 means that there will always be at least 7 mirrors on the remote side.
+    #   When a new full mirror is taken, the oldest (i.e. 8rd) mirror is removed.
+    # > Thus, this schedule combined with this retention will ensure there's always 24 hours of mirrors.
+    retention: 7
+  # Set reporting options for automirrors
+  # NOTE: By default, pvc-ansible installs a local Postfix MTA and Postfix sendmail to send emails
+  # This may not be what you want! If you want an alternate sendmail MTA (e.g. msmtp) you must install it
+  # yourself in a custom role!
+  reporting:
+    # Enable or disable email reporting; if disabled ("no"), no reports are ever sent
+    enabled: no
+    # Email a report to these addresses; at least one MUST be specified if enabled
+    emails:
+      - myuser@domain.tld
+      - otheruser@domain.tld
+    # Email a report on the specified job results
+    # > These options are like this for clarity. Functionally, only "error" changes anything:
+    #   * If yes & yes, all results send a report.
+    #   * If yes & no, all results send a report.
+    #   * If no & yes, only errors send a report.
+    #   * If no & no, no reports are ever sent; functionally equivalent to setting enabled:no above.
+    report_on:
+      # Report on a successful job (all snapshots were sent successfully)
+      success: no
+      # Report on an error (at least one snapshot was not sent successfully)
+      error: yes
+
 # Configuration file networks
 # > Taken from base.yml's configuration; DO NOT MODIFY THIS SECTION.
 pvc_upstream_device: "{{ networks['upstream']['device'] }}"
diff --git a/roles/pvc/defaults/main.yml b/roles/pvc/defaults/main.yml
index 593a480..3470aec 100644
--- a/roles/pvc/defaults/main.yml
+++ b/roles/pvc/defaults/main.yml
@@ -91,6 +91,36 @@ pvc_autobackup:
       # Example: Unmount the {backup_root_path}
       # - "/usr/bin/umount {backup_root_path}"
 
+# PVC VM automirrors
+# This is uncommented but disabled so this is not installed by default; enable it in your per-cluster configs
+# Automirror allows the sending of VM snapshots automatically to an external cluster.
+# These values are default; ensure you modify them in your own group_vars to match your system!
+pvc_automirror:
+  enabled: no
+  destinations:
+    cluster2:
+      address: pvc.cluster2.mydomain.tld
+      port: 7370
+      prefix: "/api/v1"
+      key: 00000000-0000-0000-0000-000000000000
+      ssl: yes
+      verify_ssl: yes
+      pool: vms
+  default_destination: cluster2
+  tags:
+    - automirror
+  schedule:
+    time:  "*-*-* 00/4:00:00"
+    retention: 7
+  reporting:
+    enabled: no
+    emails:
+      - myuser@domain.tld
+      - otheruser@domain.tld
+    report_on:
+      success: no
+      error: yes
+
 # Coordinators & Nodes list
 pvc_nodes:
   - hostname: "pvc1"                # The full ansible inventory hostname of the node
diff --git a/roles/pvc/tasks/automirror/disable.yml b/roles/pvc/tasks/automirror/disable.yml
new file mode 100644
index 0000000..0b52194
--- /dev/null
+++ b/roles/pvc/tasks/automirror/disable.yml
@@ -0,0 +1,24 @@
+---
+
+- name: disable timer units
+  systemd:
+    name: "{{ item }}"
+    state: stopped
+    enabled: false
+  loop:
+    - pvc-automirror.timer
+  ignore_errors: yes
+
+- name: remove automirror configurations
+  file:
+    dest: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/systemd/system/pvc-automirror.timer
+    - /etc/systemd/system/pvc-automirror.service
+  register: systemd
+  ignore_errors: yes
+
+- name: reload systemd to apply changes
+  command: systemctl daemon-reload
+  when: systemd.changed
diff --git a/roles/pvc/tasks/automirror/enable.yml b/roles/pvc/tasks/automirror/enable.yml
new file mode 100644
index 0000000..68d7ffc
--- /dev/null
+++ b/roles/pvc/tasks/automirror/enable.yml
@@ -0,0 +1,23 @@
+---
+
+- name: install automirror systemd units
+  template:
+    src: "automirror/pvc-automirror.{{ item }}.j2"
+    dest: "/etc/systemd/system/pvc-automirror.{{ item }}"
+  loop:
+    - timer
+    - service
+  register: systemd
+
+- name: reload systemd to apply changes
+  command: systemctl daemon-reload
+  when: systemd.changed
+
+- name: enable timer units
+  systemd:
+    name: "{{ item }}"
+    state: started
+    enabled: true
+  loop:
+    - pvc-automirror.timer
+
diff --git a/roles/pvc/tasks/automirror/main.yml b/roles/pvc/tasks/automirror/main.yml
new file mode 100644
index 0000000..9b06d1b
--- /dev/null
+++ b/roles/pvc/tasks/automirror/main.yml
@@ -0,0 +1,7 @@
+---
+
+- include: enable.yml
+  when: pvc_automirror.enabled
+
+- include: disable.yml
+  when: not pvc_automirror.enabled
diff --git a/roles/pvc/tasks/main.yml b/roles/pvc/tasks/main.yml
index 77017cd..db956cc 100644
--- a/roles/pvc/tasks/main.yml
+++ b/roles/pvc/tasks/main.yml
@@ -65,6 +65,11 @@
   tags: pvc-autobackup
   when: pvc_autobackup is defined
 
+# Install PVC automirror
+- include: automirror/main.yml
+  tags: pvc-automirror
+  when: pvc_automirror is defined
+
 # Install CPU tuning
 - include: cputuning/main.yml
   tags: pvc-cputuning
diff --git a/roles/pvc/templates/autobackup/autobackup.yaml.j2 b/roles/pvc/templates/autobackup/autobackup.yaml.j2
deleted file mode 100644
index 943a9ca..0000000
--- a/roles/pvc/templates/autobackup/autobackup.yaml.j2
+++ /dev/null
@@ -1,24 +0,0 @@
----
-# PVC Autobackup configuration
-# {{ ansible_managed }}
-
-autobackup:
-  backup_root_path: {{ pvc_autobackup.backup_root_path }}
-  backup_root_suffix: {{ pvc_autobackup.backup_root_suffix }}
-  backup_tags:
-{% for tag in pvc_autobackup.backup_tags %}
-    - {{ tag }}
-{% endfor %}
-  backup_schedule:
-    full_interval: {{ pvc_autobackup.schedule.full_interval }}
-    full_retention: {{ pvc_autobackup.schedule.full_retention }}
-  auto_mount:
-    enabled: {{ pvc_autobackup.auto_mount.enabled }}
-    mount_cmds:
-{% for cmd in pvc_autobackup.auto_mount.mount_cmds %}
-      - "{{ cmd }}"
-{% endfor %}
-    unmount_cmds:
-{% for cmd in pvc_autobackup.auto_mount.unmount_cmds %}
-      - "{{ cmd }}"
-{% endfor %}
diff --git a/roles/pvc/templates/automirror/pvc-automirror.service.j2 b/roles/pvc/templates/automirror/pvc-automirror.service.j2
new file mode 100644
index 0000000..a23c3b4
--- /dev/null
+++ b/roles/pvc/templates/automirror/pvc-automirror.service.j2
@@ -0,0 +1,9 @@
+[Unit]
+Description=[Cron] PVC VM automirror
+
+[Service]
+Type=oneshot
+IgnoreSIGPIPE=false
+KillMode=process
+ExecCondition=/usr/bin/pvc --quiet node is-primary
+ExecStart=/usr/bin/pvc --quiet vm automirror --cron {% if pvc_automirror.reporting.enabled and (pvc_automirror.reporting.report_on.error or pvc_automirror.reporting.report_on.success) %}--email-report {{ pvc_automirror.reporting.emails|join(',') }}{% endif %} {% if pvc_automirror.reporting.enabled and (pvc_automirror.reporting.report_on.error and not pvc_automirror.reporting.report_on.success) %}--email-errors-only{% endif %}
diff --git a/roles/pvc/templates/automirror/pvc-automirror.timer.j2 b/roles/pvc/templates/automirror/pvc-automirror.timer.j2
new file mode 100644
index 0000000..49dd32d
--- /dev/null
+++ b/roles/pvc/templates/automirror/pvc-automirror.timer.j2
@@ -0,0 +1,9 @@
+[Unit]
+Description=[Timer] PVC VM automirror
+
+[Timer]
+Unit=pvc-automirror.service
+OnCalendar={{ pvc_automirror.schedule.time }}
+
+[Install]
+WantedBy=pvc.target
diff --git a/roles/pvc/templates/pvc/pvc.conf.j2 b/roles/pvc/templates/pvc/pvc.conf.j2
index 04ff610..fb9518a 100644
--- a/roles/pvc/templates/pvc/pvc.conf.j2
+++ b/roles/pvc/templates/pvc/pvc.conf.j2
@@ -174,7 +174,7 @@ autobackup:
     full_interval: {{ pvc_autobackup.schedule.full_interval }}
     full_retention: {{ pvc_autobackup.schedule.full_retention }}
   auto_mount:
-    enabled: {{ pvc_autobackup.auto_mount.enabled }}
+    enabled: {{ 'yes' if pvc_autobackup.auto_mount.enabled else 'no' }}
     mount_cmds:
 {% for cmd in pvc_autobackup.auto_mount.mount_cmds %}
       - "{{ cmd }}"
@@ -184,6 +184,26 @@ autobackup:
       - "{{ cmd }}"
 {% endfor %} 
 {% endif %}
+automirror:
+{% if pvc_automirror is defined and pvc_automirror.enabled is defined and pvc_automirror.enabled %}
+  destinations:
+{% for destination in pvc_automirror.destinations %}
+    {{ destination }}:
+      address: {{ pvc_automirror.destinations[destination].address }}
+      port: {{ pvc_automirror.destinations[destination].port }}
+      prefix: {{ pvc_automirror.destinations[destination].prefix }}
+      key: {{ pvc_automirror.destinations[destination].key }}
+      ssl: {{ 'yes' if pvc_automirror.destinations[destination].ssl else 'no' }}
+      verify_ssl: {{ 'yes' if pvc_automirror.destinations[destination].verify_ssl else 'no' }}
+      pool: {{ pvc_automirror.destinations[destination].pool }}
+{% endfor %}
+  default_destination: {{ pvc_automirror.default_destination }}
+  mirror_tags:
+{% for tag in pvc_automirror.tags %}
+    - {{ tag }}
+{% endfor %}
+  keep_snapshots: {{ pvc_automirror.schedule.retention }}
+{% endif %}
 
 # VIM modeline, requires "set modeline" in your VIMRC
 # vim: expandtab shiftwidth=2 tabstop=2 filetype=yaml