Add enhancements to autobackup

1. Add a cron mode to avoid exit(1) during cronjobs/timers
2. Revamp the remote_mount settings into auto_mount
   This removes a lot of unnecessary complexity while giving the
   administrator more flexibility in what they want to execute to mount
   a filesystem and how. The naming reflects the goal but the possible
   commands are arbitrary.

client-cli/autobackup.sample.yaml

@@ -31,117 +31,22 @@ autobackup:
                        # > Should usually be at least 2 when using incrementals (full_interval > 1) to
                        #   avoid there being too few backups after cleanup from a new full backup
 
-  # Remote mount settings for backup root path
+  # Automatic mount settings
-  # If remote mount support is disabled, it is up to the administrator to that the backup root path is
+  # These settings permit running an arbitrary set of commands, ideally a "mount" command or similar, to
-  # created and a valid destination filesystem is mounted on it
+  # ensure that a remote filesystem is mounted on the backup root path
-  remote_mount:
+  # While the examples here show absolute paths, that is not required; they will run with the $PATH of the
-    enabled: no  # Enable automatic remote mount/unmount support
+  # executing environment (either the "pvc" command on a CLI or a cron/systemd timer)
-    type: sshfs  # Set the type of remote mount; optional if remote_mount is disabled
+  # A "{backup_root_path}" f-string/str.format type variable MAY be present in any cmds string to represent
-                 # > Supported values are: sshfs, nfs, cifs (i.e. SMB), cephfs, and s3fs
+  # the above configured root backup path, which is interpolated at runtime
-                 # > WARNING: s3fs has serious known bugs that we don't work around; avoid it if possible
+  # If multiple commands are given, they will be executed in the order given; if no commands are given,
-
+  # nothing is executed, but the keys MUST be present
-  # Remote mount configurations, per-type; you only need to specify the type(s) you plan to use, but all
+  auto_mount:
-  # are given here for completeness as examples
+    enabled: no  # Enable automatic mount/unmount support
-  # > NOTE: This key (and all children) are optional if remote mounting is not enabled
+    # These commands are executed at the start of the backup run and should mount a filesystem
-  remote_mount_config:
+    mount_cmds:
-
+      # This example shows an NFS mount leveraging the backup_root_path variable
-    # SSHFS specific options
+      - "/usr/sbin/mount.nfs -o nfsvers=3 10.0.0.10:/backups {backup_root_path}"
-    # > NOTE: This SSHFS implementation does not support password authentication; keys MUST be used
+    # These commands are executed at the end of the backup run and should unmount a filesystem
-    sshfs:
+    unmount_cmds:
-      # Remote username
+      # This example shows a generic umount leveraging the backup_root_path variable
-      user: username
+      - "/usr/bin/umount {backup_root_path}"
-      # Remote hostname
-      host: hostname
-      # Remote path
-      path: /srv/vm_backups
-      # Required command to check for or error
-      command: /usr/bin/sshfs
-      # Options to pass to the mount command (joined, each requires "-o"!)
-      # See the command manual page for more options
-      options:
-        - "-o IdentityFile=/srv/pvc_autobackup.id_ed25519" # Identity (SSH key) file, required!
-        - "-o port=22" # Port number
-        - "-o reconnect" # Enable reconnection
-        - "-o default_permissions" # Enable local permission checking
-        - "-o compression=no" # Disable compression; testing shows that compression slows things
-                              # down a fair bit (1m40.435s vs 0m22.253s for 750MB on 10GbE net)
-        - "-o sshfs_sync" # Enable sync; ensures consistent writes with an acceptable performance
-                          # overhead (0m22.253s vs 0m17.453s for 750GB on 10GbE net)
-      # Mount command, populated at import time
-      mount_cmd: "{command} {sshfs_user}@{sshfs_host}:{sshfs_path} {backup_root_path} {sshfs_options}"
-      # Unmount command, populated at import time
-      unmount_cmd: "fusermount3 -u {backup_root_path}"
-
-    # NFS specific options
-    nfs:
-      # Remote hostname
-      host: hostname
-      # Remote path
-      path: /srv/vm_backups
-      # Required command to check for or error
-      command: /usr/sbin/mount.nfs
-      # Options to pass to the mount command (joined and passed to "-o")
-      # See the command manual page for more options
-      options:
-        - "nfsvers=3" # Use a specific NFS version
-      # Mount command, populated at import time
-      mount_cmd: "{command} -o {nfs_options} {nfs_host}:{nfs_path} {backup_root_path}"
-      # Unmount command, populated at import time
-      unmount_cmd: "umount {backup_root_path}"
-
-    # CIFS specific options
-    cifs:
-      # Remote hostname
-      host: hostname
-      # Remote path be sure to include the leading '/'!)
-      path: /srv/vm_backups
-      # Required command to check for or error
-      command: /usr/sbin/mount.cifs
-      # Options to pass to the mount command (joined and passed to "-o")
-      # See the command manual page for more options
-      options:
-        - "credentials=/srv/backup_vms.cifs_credentials" # Specify a credentials file
-        - "guest" # Use guest access, alternate to above
-      # Mount command, populated at import time
-      mount_cmd: "{command} -o {cifs_options} //{cifs_host}{cifs_path} {backup_root_path}"
-      # Unmount command, populated at import time
-      unmount_cmd: "umount {backup_root_path}"
-
-    # CephFS specific options
-    cephfs:
-      # Monitor address/hostname list
-      monitors:
-        - mon1
-      # CephFS path; at least "/" is always required
-      path: "/mysubdir"
-      # Required command to check for or error
-      command: /usr/sbin/mount.ceph
-      # Options to pass to mount command (joined and passed to "-o")
-      # See the command manual page for more options
-      options:
-        - "secretfile=/srv/backup_vms.cephfs_secret" # Specify a cephx secret file
-        - "conf=/srv/backup_vms.ceph.conf" # Specify a nonstandard ceph.conf file
-      # Mount command, populated at import time
-      mount_cmd: "{command} {cephfs_monitors}:{cephfs_path} {backup_root_path} -o {cephfs_options}"
-      # Unmount command, populated at import time
-      unmount_cmd: "umount {backup_root_path}"
-
-    # S3FS specific options
-    s3fs:
-      # S3 bucket
-      bucket: mybucket
-      # S3 bucket (sub)path, including leading ':' if used!
-      # Leave empty for no (sub)path
-      path: ":/mypath"
-      # Required command to check for or error
-      command: /usr/bin/s3fs
-      # Options to pass to the mount command (joined, each requires "-o"!)
-      # See the command manual page for more options
-      options:
-        - "-o passwd_file=/srv/backup_vms.s3fs_credentials" # Specify a password file
-        - "-o host=https://s3.amazonaws.com" # Specify an alternate host
-        - "-o endpoint=us-east-1" # Specify an alternate endpoint/region
-      # Mount command, populated at import time
-      mount_cmd: "{command} {s2fs_bucket}{s3fs_path} {backup_root_path} {s3fs_options}"
-      # Unmount command, populated at import time
-      unmount_cmd: "fusermount3 -u {backup_root_path}"

client-cli/pvc/cli/cli.py

@@ -1755,7 +1755,7 @@ def cli_vm_autobackup(autobackup_cfgfile, force_full_flag, cron_flag):
     functions with an internal rentention and cleanup system as well as determination of full vs. incremental
     backups at different intervals. VMs are selected based on configured VM tags. The destination storage
     may either be local, or provided by a remote filesystem which is automatically mounted and unmounted during
-    the backup run.
+    the backup run via a set of configured commands before and after the backup run.
 
     NOTE: This command performs its tasks in a local context. It MUST be run from the cluster's active primary
     coordinator using the "local" connection only; if either is not correct, the command will error.

client-cli/pvc/cli/helpers.py

@@ -258,118 +258,25 @@ def get_autobackup_config(CLI_CONFIG, cfgfile):
         config["backup_root_suffix"] = backup_config["backup_root_suffix"]
         config["backup_tags"] = backup_config["backup_tags"]
         config["backup_schedule"] = backup_config["backup_schedule"]
-        config["remote_mount_enabled"] = backup_config["remote_mount"]["enabled"]
+        config["auto_mount_enabled"] = backup_config["auto_mount"]["enabled"]
-        if config["remote_mount_enabled"]:
+        if config["auto_mount_enabled"]:
-            config["remote_mount_type"] = backup_config["remote_mount"]["type"]
+            config["mount_cmds"] = list()
-        else:
+            _mount_cmds = backup_config["auto_mount"]["mount_cmds"]
-            config["remote_mount_type"] = None
+            for _mount_cmd in _mount_cmds:
+                if "{backup_root_path}" in _mount_cmd:
+                    _mount_cmd = _mount_cmd.format(
+                        backup_root_path=backup_config["backup_root_path"]
+                    )
+                config["mount_cmds"].append(_mount_cmd)
 
-        if config["remote_mount_type"] == "sshfs":
+            config["unmount_cmds"] = list()
-            config["check_command"] = backup_config["remote_mount_config"]["sshfs"][
+            _unmount_cmds = backup_config["auto_mount"]["unmount_cmds"]
-                "command"
+            for _unmount_cmd in _unmount_cmds:
-            ]
+                if "{backup_root_path}" in _unmount_cmd:
-            config["remote_mount_cmd"] = backup_config["remote_mount_config"]["sshfs"][
+                    _unmount_cmd = _unmount_cmd.format(
-                "mount_cmd"
+                        backup_root_path=backup_config["backup_root_path"]
-            ].format(
+                    )
-                command=backup_config["remote_mount_config"]["sshfs"]["command"],
+                config["unmount_cmds"].append(_unmount_cmd)
-                sshfs_user=backup_config["remote_mount_config"]["sshfs"]["user"],
-                sshfs_host=backup_config["remote_mount_config"]["sshfs"]["host"],
-                sshfs_path=backup_config["remote_mount_config"]["sshfs"]["path"],
-                sshfs_options=" ".join(
-                    backup_config["remote_mount_config"]["sshfs"]["options"]
-                ),
-                backup_root_path=backup_config["backup_root_path"],
-            )
-            config["remote_unmount_cmd"] = backup_config["remote_mount_config"][
-                "sshfs"
-            ]["unmount_cmd"].format(
-                backup_root_path=backup_config["backup_root_path"],
-            )
-        elif config["remote_mount_type"] == "nfs":
-            config["check_command"] = backup_config["remote_mount_config"]["nfs"][
-                "command"
-            ]
-            config["remote_mount_cmd"] = backup_config["remote_mount_config"]["nfs"][
-                "mount_cmd"
-            ].format(
-                command=backup_config["remote_mount_config"]["nfs"]["command"],
-                nfs_host=backup_config["remote_mount_config"]["nfs"]["host"],
-                nfs_path=backup_config["remote_mount_config"]["nfs"]["path"],
-                nfs_options=",".join(
-                    backup_config["remote_mount_config"]["nfs"]["options"]
-                ),
-                backup_root_path=backup_config["backup_root_path"],
-            )
-            config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["nfs"][
-                "unmount_cmd"
-            ].format(
-                backup_root_path=backup_config["backup_root_path"],
-            )
-        elif config["remote_mount_type"] == "cifs":
-            config["check_command"] = backup_config["remote_mount_config"]["cifs"][
-                "command"
-            ]
-            config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cifs"][
-                "mount_cmd"
-            ].format(
-                command=backup_config["remote_mount_config"]["cifs"]["command"],
-                cifs_host=backup_config["remote_mount_config"]["cifs"]["host"],
-                cifs_path=backup_config["remote_mount_config"]["cifs"]["path"],
-                cifs_options=",".join(
-                    backup_config["remote_mount_config"]["cifs"]["options"]
-                ),
-                backup_root_path=backup_config["backup_root_path"],
-            )
-            config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["cifs"][
-                "unmount_cmd"
-            ].format(
-                backup_root_path=backup_config["backup_root_path"],
-            )
-        elif config["remote_mount_type"] == "s3fs":
-            config["check_command"] = backup_config["remote_mount_config"]["s3fs"][
-                "command"
-            ]
-            config["remote_mount_cmd"] = backup_config["remote_mount_config"]["s3fs"][
-                "mount_cmd"
-            ].format(
-                command=backup_config["remote_mount_config"]["s3fs"]["command"],
-                s2fs_bucket=backup_config["remote_mount_config"]["s3fs"]["bucket"],
-                s2fs_path=backup_config["remote_mount_config"]["s3fs"]["path"],
-                s3fs_options=" ".join(
-                    backup_config["remote_mount_config"]["s3fs"]["options"]
-                ),
-                backup_root_path=backup_config["backup_root_path"],
-            )
-            config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["s3fs"][
-                "unmount_cmd"
-            ].format(
-                backup_root_path=backup_config["backup_root_path"],
-            )
-        elif config["remote_mount_type"] == "cephfs":
-            config["check_command"] = backup_config["remote_mount_config"]["cephfs"][
-                "command"
-            ]
-            config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cephfs"][
-                "mount_cmd"
-            ].format(
-                command=backup_config["remote_mount_config"]["cephfs"]["command"],
-                cephfs_monitors=",".join(
-                    backup_config["remote_mount_config"]["cephfs"]["monitors"]
-                ),
-                cephfs_path=backup_config["remote_mount_config"]["cephfs"]["path"],
-                cephfs_options=",".join(
-                    backup_config["remote_mount_config"]["cephfs"]["options"]
-                ),
-                backup_root_path=backup_config["backup_root_path"],
-            )
-            config["remote_unmount_cmd"] = backup_config["remote_mount_config"][
-                "cephfs"
-            ]["unmount_cmd"].format(
-                backup_root_path=backup_config["backup_root_path"],
-            )
-        else:
-            config["remote_mount_cmd"] = None
-            config["remote_unmount_cmd"] = None
 
     except FileNotFoundError:
         echo(CLI_CONFIG, "ERROR: Specified backup configuration does not exist!")
@@ -382,7 +289,10 @@ def get_autobackup_config(CLI_CONFIG, cfgfile):
 
 
 def vm_autobackup(
-    CLI_CONFIG, autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME, force_full_flag=False
+    CLI_CONFIG,
+    autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME,
+    force_full_flag=False,
+    cron_flag=False,
 ):
     """
     Perform automatic backups of VMs based on an external config file.
@@ -393,15 +303,22 @@ def vm_autobackup(
     CLI_CONFIG["connection"] = "local"
     retcode, retdata = pvc.lib.node.node_info(CLI_CONFIG, DEFAULT_NODE_HOSTNAME)
     if not retcode or retdata.get("coordinator_state") != "primary":
-        echo(
+        if cron_flag:
-            CLI_CONFIG,
+            echo(
-            f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.",
+                CLI_CONFIG,
-        )
+                "Current host is not the primary coordinator of the local cluster and running in cron mode. Exiting cleanly.",
-        echo(
+            )
-            CLI_CONFIG,
+            exit(0)
-            "Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.",
+        else:
-        )
+            echo(
-        exit(1)
+                CLI_CONFIG,
+                f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.",
+            )
+            echo(
+                CLI_CONFIG,
+                "Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.",
+            )
+            exit(1)
 
     # Ensure we're running as root, or show a warning & confirmation
     if getuser() != "root":
@@ -463,38 +380,34 @@ def vm_autobackup(
     echo(CLI_CONFIG, "  {}".format("\n  ".join(vm_list_rows)), stderr=True)
     echo(CLI_CONFIG, "", stderr=True)
 
-    if autobackup_config["remote_mount_cmd"] is not None:
+    if autobackup_config["auto_mount_enabled"]:
-        # Validate that the mount command is valid
+        # Execute each mount_cmds command in sequence
-        if not path.exists(autobackup_config["check_command"]):
+        for cmd in autobackup_config["mount_cmds"]:
             echo(
                 CLI_CONFIG,
-                f"ERROR: Failed to find required command {autobackup_config['check_command']}; ensure it is installed.",
+                f"Executing mount command '{cmd.split()[0]}'... ",
+                newline=False,
             )
-            exit(1)
+            tstart = datetime.now()
-
+            ret = run(
-        # Try to mount the remote mount
+                cmd.split(),
-        echo(
+                stdout=PIPE,
-            CLI_CONFIG,
+                stderr=PIPE,
-            f"Mounting remote {autobackup_config['remote_mount_type']} filesystem on {autobackup_config['backup_root_path']}... ",
-            newline=False,
-        )
-        tstart = datetime.now()
-        ret = run(
-            autobackup_config["remote_mount_cmd"].split(),
-            stdout=PIPE,
-            stderr=PIPE,
-        )
-        tend = datetime.now()
-        ttot = tend - tstart
-        if ret.returncode != 0:
-            echo(
-                CLI_CONFIG,
-                f"failed. [{ttot.seconds}s]",
             )
-            echo(CLI_CONFIG, f"Exiting; command reports: {ret.stderr.decode().strip()}")
+            tend = datetime.now()
-            exit(1)
+            ttot = tend - tstart
-        else:
+            if ret.returncode != 0:
-            echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
+                echo(
+                    CLI_CONFIG,
+                    f"failed. [{ttot.seconds}s]",
+                )
+                echo(
+                    CLI_CONFIG,
+                    f"Exiting; command reports: {ret.stderr.decode().strip()}",
+                )
+                exit(1)
+            else:
+                echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
 
     # For each VM, perform the backup
     for vm in backup_vms:
@@ -625,26 +538,30 @@ def vm_autobackup(
         with open(autobackup_state_file, "w") as fh:
             jdump(state_data, fh)
 
-    # Try to unmount the remote mount
+    if autobackup_config["auto_mount_enabled"]:
-    if autobackup_config["remote_unmount_cmd"] is not None:
+        # Execute each unmount_cmds command in sequence
-        echo(
+        for cmd in autobackup_config["unmount_cmds"]:
-            CLI_CONFIG,
-            f"Unmounting remote {autobackup_config['remote_mount_type']} filesystem from {autobackup_config['backup_root_path']}... ",
-            newline=False,
-        )
-        tstart = datetime.now()
-        ret = run(
-            autobackup_config["remote_unmount_cmd"].split(),
-            stdout=PIPE,
-            stderr=PIPE,
-        )
-        tend = datetime.now()
-        ttot = tend - tstart
-        if ret.returncode != 0:
-            echo(CLI_CONFIG, f"failed. [{ttot.seconds}s]")
             echo(
                 CLI_CONFIG,
-                f"Continuing; command reports: {ret.stderr.decode().strip()}",
+                f"Executing unmount command '{cmd.split()[0]}'... ",
+                newline=False,
             )
-        else:
+            tstart = datetime.now()
-            echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
+            ret = run(
+                cmd.split(),
+                stdout=PIPE,
+                stderr=PIPE,
+            )
+            tend = datetime.now()
+            ttot = tend - tstart
+            if ret.returncode != 0:
+                echo(
+                    CLI_CONFIG,
+                    f"failed. [{ttot.seconds}s]",
+                )
+                echo(
+                    CLI_CONFIG,
+                    f"Continuing; command reports: {ret.stderr.decode().strip()}",
+                )
+            else:
+                echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")