From 2fccbcda8924f72a5a14b69089075ec544e19608 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Fri, 27 Oct 2023 01:47:10 -0400 Subject: [PATCH] Add enhancements to autobackup 1. Add a cron mode to avoid exit(1) during cronjobs/timers 2. Revamp the remote_mount settings into auto_mount This removes a lot of unnecessary complexity while giving the administrator more flexibility in what they want to execute to mount a filesystem and how. The naming reflects the goal but the possible commands are arbitrary. --- client-cli/autobackup.sample.yaml | 133 +++------------- client-cli/pvc/cli/cli.py | 2 +- client-cli/pvc/cli/helpers.py | 255 ++++++++++-------------------- 3 files changed, 106 insertions(+), 284 deletions(-) diff --git a/client-cli/autobackup.sample.yaml b/client-cli/autobackup.sample.yaml index 50e6d8e0..250e7ed5 100644 --- a/client-cli/autobackup.sample.yaml +++ b/client-cli/autobackup.sample.yaml @@ -31,117 +31,22 @@ autobackup: # > Should usually be at least 2 when using incrementals (full_interval > 1) to # avoid there being too few backups after cleanup from a new full backup - # Remote mount settings for backup root path - # If remote mount support is disabled, it is up to the administrator to that the backup root path is - # created and a valid destination filesystem is mounted on it - remote_mount: - enabled: no # Enable automatic remote mount/unmount support - type: sshfs # Set the type of remote mount; optional if remote_mount is disabled - # > Supported values are: sshfs, nfs, cifs (i.e. SMB), cephfs, and s3fs - # > WARNING: s3fs has serious known bugs that we don't work around; avoid it if possible - - # Remote mount configurations, per-type; you only need to specify the type(s) you plan to use, but all - # are given here for completeness as examples - # > NOTE: This key (and all children) are optional if remote mounting is not enabled - remote_mount_config: - - # SSHFS specific options - # > NOTE: This SSHFS implementation does not support password authentication; keys MUST be used - sshfs: - # Remote username - user: username - # Remote hostname - host: hostname - # Remote path - path: /srv/vm_backups - # Required command to check for or error - command: /usr/bin/sshfs - # Options to pass to the mount command (joined, each requires "-o"!) - # See the command manual page for more options - options: - - "-o IdentityFile=/srv/pvc_autobackup.id_ed25519" # Identity (SSH key) file, required! - - "-o port=22" # Port number - - "-o reconnect" # Enable reconnection - - "-o default_permissions" # Enable local permission checking - - "-o compression=no" # Disable compression; testing shows that compression slows things - # down a fair bit (1m40.435s vs 0m22.253s for 750MB on 10GbE net) - - "-o sshfs_sync" # Enable sync; ensures consistent writes with an acceptable performance - # overhead (0m22.253s vs 0m17.453s for 750GB on 10GbE net) - # Mount command, populated at import time - mount_cmd: "{command} {sshfs_user}@{sshfs_host}:{sshfs_path} {backup_root_path} {sshfs_options}" - # Unmount command, populated at import time - unmount_cmd: "fusermount3 -u {backup_root_path}" - - # NFS specific options - nfs: - # Remote hostname - host: hostname - # Remote path - path: /srv/vm_backups - # Required command to check for or error - command: /usr/sbin/mount.nfs - # Options to pass to the mount command (joined and passed to "-o") - # See the command manual page for more options - options: - - "nfsvers=3" # Use a specific NFS version - # Mount command, populated at import time - mount_cmd: "{command} -o {nfs_options} {nfs_host}:{nfs_path} {backup_root_path}" - # Unmount command, populated at import time - unmount_cmd: "umount {backup_root_path}" - - # CIFS specific options - cifs: - # Remote hostname - host: hostname - # Remote path be sure to include the leading '/'!) - path: /srv/vm_backups - # Required command to check for or error - command: /usr/sbin/mount.cifs - # Options to pass to the mount command (joined and passed to "-o") - # See the command manual page for more options - options: - - "credentials=/srv/backup_vms.cifs_credentials" # Specify a credentials file - - "guest" # Use guest access, alternate to above - # Mount command, populated at import time - mount_cmd: "{command} -o {cifs_options} //{cifs_host}{cifs_path} {backup_root_path}" - # Unmount command, populated at import time - unmount_cmd: "umount {backup_root_path}" - - # CephFS specific options - cephfs: - # Monitor address/hostname list - monitors: - - mon1 - # CephFS path; at least "/" is always required - path: "/mysubdir" - # Required command to check for or error - command: /usr/sbin/mount.ceph - # Options to pass to mount command (joined and passed to "-o") - # See the command manual page for more options - options: - - "secretfile=/srv/backup_vms.cephfs_secret" # Specify a cephx secret file - - "conf=/srv/backup_vms.ceph.conf" # Specify a nonstandard ceph.conf file - # Mount command, populated at import time - mount_cmd: "{command} {cephfs_monitors}:{cephfs_path} {backup_root_path} -o {cephfs_options}" - # Unmount command, populated at import time - unmount_cmd: "umount {backup_root_path}" - - # S3FS specific options - s3fs: - # S3 bucket - bucket: mybucket - # S3 bucket (sub)path, including leading ':' if used! - # Leave empty for no (sub)path - path: ":/mypath" - # Required command to check for or error - command: /usr/bin/s3fs - # Options to pass to the mount command (joined, each requires "-o"!) - # See the command manual page for more options - options: - - "-o passwd_file=/srv/backup_vms.s3fs_credentials" # Specify a password file - - "-o host=https://s3.amazonaws.com" # Specify an alternate host - - "-o endpoint=us-east-1" # Specify an alternate endpoint/region - # Mount command, populated at import time - mount_cmd: "{command} {s2fs_bucket}{s3fs_path} {backup_root_path} {s3fs_options}" - # Unmount command, populated at import time - unmount_cmd: "fusermount3 -u {backup_root_path}" + # Automatic mount settings + # These settings permit running an arbitrary set of commands, ideally a "mount" command or similar, to + # ensure that a remote filesystem is mounted on the backup root path + # While the examples here show absolute paths, that is not required; they will run with the $PATH of the + # executing environment (either the "pvc" command on a CLI or a cron/systemd timer) + # A "{backup_root_path}" f-string/str.format type variable MAY be present in any cmds string to represent + # the above configured root backup path, which is interpolated at runtime + # If multiple commands are given, they will be executed in the order given; if no commands are given, + # nothing is executed, but the keys MUST be present + auto_mount: + enabled: no # Enable automatic mount/unmount support + # These commands are executed at the start of the backup run and should mount a filesystem + mount_cmds: + # This example shows an NFS mount leveraging the backup_root_path variable + - "/usr/sbin/mount.nfs -o nfsvers=3 10.0.0.10:/backups {backup_root_path}" + # These commands are executed at the end of the backup run and should unmount a filesystem + unmount_cmds: + # This example shows a generic umount leveraging the backup_root_path variable + - "/usr/bin/umount {backup_root_path}" diff --git a/client-cli/pvc/cli/cli.py b/client-cli/pvc/cli/cli.py index 067bdd75..faefc5f9 100644 --- a/client-cli/pvc/cli/cli.py +++ b/client-cli/pvc/cli/cli.py @@ -1755,7 +1755,7 @@ def cli_vm_autobackup(autobackup_cfgfile, force_full_flag, cron_flag): functions with an internal rentention and cleanup system as well as determination of full vs. incremental backups at different intervals. VMs are selected based on configured VM tags. The destination storage may either be local, or provided by a remote filesystem which is automatically mounted and unmounted during - the backup run. + the backup run via a set of configured commands before and after the backup run. NOTE: This command performs its tasks in a local context. It MUST be run from the cluster's active primary coordinator using the "local" connection only; if either is not correct, the command will error. diff --git a/client-cli/pvc/cli/helpers.py b/client-cli/pvc/cli/helpers.py index 70f0a538..04157d54 100644 --- a/client-cli/pvc/cli/helpers.py +++ b/client-cli/pvc/cli/helpers.py @@ -258,118 +258,25 @@ def get_autobackup_config(CLI_CONFIG, cfgfile): config["backup_root_suffix"] = backup_config["backup_root_suffix"] config["backup_tags"] = backup_config["backup_tags"] config["backup_schedule"] = backup_config["backup_schedule"] - config["remote_mount_enabled"] = backup_config["remote_mount"]["enabled"] - if config["remote_mount_enabled"]: - config["remote_mount_type"] = backup_config["remote_mount"]["type"] - else: - config["remote_mount_type"] = None + config["auto_mount_enabled"] = backup_config["auto_mount"]["enabled"] + if config["auto_mount_enabled"]: + config["mount_cmds"] = list() + _mount_cmds = backup_config["auto_mount"]["mount_cmds"] + for _mount_cmd in _mount_cmds: + if "{backup_root_path}" in _mount_cmd: + _mount_cmd = _mount_cmd.format( + backup_root_path=backup_config["backup_root_path"] + ) + config["mount_cmds"].append(_mount_cmd) - if config["remote_mount_type"] == "sshfs": - config["check_command"] = backup_config["remote_mount_config"]["sshfs"][ - "command" - ] - config["remote_mount_cmd"] = backup_config["remote_mount_config"]["sshfs"][ - "mount_cmd" - ].format( - command=backup_config["remote_mount_config"]["sshfs"]["command"], - sshfs_user=backup_config["remote_mount_config"]["sshfs"]["user"], - sshfs_host=backup_config["remote_mount_config"]["sshfs"]["host"], - sshfs_path=backup_config["remote_mount_config"]["sshfs"]["path"], - sshfs_options=" ".join( - backup_config["remote_mount_config"]["sshfs"]["options"] - ), - backup_root_path=backup_config["backup_root_path"], - ) - config["remote_unmount_cmd"] = backup_config["remote_mount_config"][ - "sshfs" - ]["unmount_cmd"].format( - backup_root_path=backup_config["backup_root_path"], - ) - elif config["remote_mount_type"] == "nfs": - config["check_command"] = backup_config["remote_mount_config"]["nfs"][ - "command" - ] - config["remote_mount_cmd"] = backup_config["remote_mount_config"]["nfs"][ - "mount_cmd" - ].format( - command=backup_config["remote_mount_config"]["nfs"]["command"], - nfs_host=backup_config["remote_mount_config"]["nfs"]["host"], - nfs_path=backup_config["remote_mount_config"]["nfs"]["path"], - nfs_options=",".join( - backup_config["remote_mount_config"]["nfs"]["options"] - ), - backup_root_path=backup_config["backup_root_path"], - ) - config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["nfs"][ - "unmount_cmd" - ].format( - backup_root_path=backup_config["backup_root_path"], - ) - elif config["remote_mount_type"] == "cifs": - config["check_command"] = backup_config["remote_mount_config"]["cifs"][ - "command" - ] - config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cifs"][ - "mount_cmd" - ].format( - command=backup_config["remote_mount_config"]["cifs"]["command"], - cifs_host=backup_config["remote_mount_config"]["cifs"]["host"], - cifs_path=backup_config["remote_mount_config"]["cifs"]["path"], - cifs_options=",".join( - backup_config["remote_mount_config"]["cifs"]["options"] - ), - backup_root_path=backup_config["backup_root_path"], - ) - config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["cifs"][ - "unmount_cmd" - ].format( - backup_root_path=backup_config["backup_root_path"], - ) - elif config["remote_mount_type"] == "s3fs": - config["check_command"] = backup_config["remote_mount_config"]["s3fs"][ - "command" - ] - config["remote_mount_cmd"] = backup_config["remote_mount_config"]["s3fs"][ - "mount_cmd" - ].format( - command=backup_config["remote_mount_config"]["s3fs"]["command"], - s2fs_bucket=backup_config["remote_mount_config"]["s3fs"]["bucket"], - s2fs_path=backup_config["remote_mount_config"]["s3fs"]["path"], - s3fs_options=" ".join( - backup_config["remote_mount_config"]["s3fs"]["options"] - ), - backup_root_path=backup_config["backup_root_path"], - ) - config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["s3fs"][ - "unmount_cmd" - ].format( - backup_root_path=backup_config["backup_root_path"], - ) - elif config["remote_mount_type"] == "cephfs": - config["check_command"] = backup_config["remote_mount_config"]["cephfs"][ - "command" - ] - config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cephfs"][ - "mount_cmd" - ].format( - command=backup_config["remote_mount_config"]["cephfs"]["command"], - cephfs_monitors=",".join( - backup_config["remote_mount_config"]["cephfs"]["monitors"] - ), - cephfs_path=backup_config["remote_mount_config"]["cephfs"]["path"], - cephfs_options=",".join( - backup_config["remote_mount_config"]["cephfs"]["options"] - ), - backup_root_path=backup_config["backup_root_path"], - ) - config["remote_unmount_cmd"] = backup_config["remote_mount_config"][ - "cephfs" - ]["unmount_cmd"].format( - backup_root_path=backup_config["backup_root_path"], - ) - else: - config["remote_mount_cmd"] = None - config["remote_unmount_cmd"] = None + config["unmount_cmds"] = list() + _unmount_cmds = backup_config["auto_mount"]["unmount_cmds"] + for _unmount_cmd in _unmount_cmds: + if "{backup_root_path}" in _unmount_cmd: + _unmount_cmd = _unmount_cmd.format( + backup_root_path=backup_config["backup_root_path"] + ) + config["unmount_cmds"].append(_unmount_cmd) except FileNotFoundError: echo(CLI_CONFIG, "ERROR: Specified backup configuration does not exist!") @@ -382,7 +289,10 @@ def get_autobackup_config(CLI_CONFIG, cfgfile): def vm_autobackup( - CLI_CONFIG, autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME, force_full_flag=False + CLI_CONFIG, + autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME, + force_full_flag=False, + cron_flag=False, ): """ Perform automatic backups of VMs based on an external config file. @@ -393,15 +303,22 @@ def vm_autobackup( CLI_CONFIG["connection"] = "local" retcode, retdata = pvc.lib.node.node_info(CLI_CONFIG, DEFAULT_NODE_HOSTNAME) if not retcode or retdata.get("coordinator_state") != "primary": - echo( - CLI_CONFIG, - f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.", - ) - echo( - CLI_CONFIG, - "Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.", - ) - exit(1) + if cron_flag: + echo( + CLI_CONFIG, + "Current host is not the primary coordinator of the local cluster and running in cron mode. Exiting cleanly.", + ) + exit(0) + else: + echo( + CLI_CONFIG, + f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.", + ) + echo( + CLI_CONFIG, + "Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.", + ) + exit(1) # Ensure we're running as root, or show a warning & confirmation if getuser() != "root": @@ -463,38 +380,34 @@ def vm_autobackup( echo(CLI_CONFIG, " {}".format("\n ".join(vm_list_rows)), stderr=True) echo(CLI_CONFIG, "", stderr=True) - if autobackup_config["remote_mount_cmd"] is not None: - # Validate that the mount command is valid - if not path.exists(autobackup_config["check_command"]): + if autobackup_config["auto_mount_enabled"]: + # Execute each mount_cmds command in sequence + for cmd in autobackup_config["mount_cmds"]: echo( CLI_CONFIG, - f"ERROR: Failed to find required command {autobackup_config['check_command']}; ensure it is installed.", + f"Executing mount command '{cmd.split()[0]}'... ", + newline=False, ) - exit(1) - - # Try to mount the remote mount - echo( - CLI_CONFIG, - f"Mounting remote {autobackup_config['remote_mount_type']} filesystem on {autobackup_config['backup_root_path']}... ", - newline=False, - ) - tstart = datetime.now() - ret = run( - autobackup_config["remote_mount_cmd"].split(), - stdout=PIPE, - stderr=PIPE, - ) - tend = datetime.now() - ttot = tend - tstart - if ret.returncode != 0: - echo( - CLI_CONFIG, - f"failed. [{ttot.seconds}s]", + tstart = datetime.now() + ret = run( + cmd.split(), + stdout=PIPE, + stderr=PIPE, ) - echo(CLI_CONFIG, f"Exiting; command reports: {ret.stderr.decode().strip()}") - exit(1) - else: - echo(CLI_CONFIG, f"done. [{ttot.seconds}s]") + tend = datetime.now() + ttot = tend - tstart + if ret.returncode != 0: + echo( + CLI_CONFIG, + f"failed. [{ttot.seconds}s]", + ) + echo( + CLI_CONFIG, + f"Exiting; command reports: {ret.stderr.decode().strip()}", + ) + exit(1) + else: + echo(CLI_CONFIG, f"done. [{ttot.seconds}s]") # For each VM, perform the backup for vm in backup_vms: @@ -625,26 +538,30 @@ def vm_autobackup( with open(autobackup_state_file, "w") as fh: jdump(state_data, fh) - # Try to unmount the remote mount - if autobackup_config["remote_unmount_cmd"] is not None: - echo( - CLI_CONFIG, - f"Unmounting remote {autobackup_config['remote_mount_type']} filesystem from {autobackup_config['backup_root_path']}... ", - newline=False, - ) - tstart = datetime.now() - ret = run( - autobackup_config["remote_unmount_cmd"].split(), - stdout=PIPE, - stderr=PIPE, - ) - tend = datetime.now() - ttot = tend - tstart - if ret.returncode != 0: - echo(CLI_CONFIG, f"failed. [{ttot.seconds}s]") + if autobackup_config["auto_mount_enabled"]: + # Execute each unmount_cmds command in sequence + for cmd in autobackup_config["unmount_cmds"]: echo( CLI_CONFIG, - f"Continuing; command reports: {ret.stderr.decode().strip()}", + f"Executing unmount command '{cmd.split()[0]}'... ", + newline=False, ) - else: - echo(CLI_CONFIG, f"done. [{ttot.seconds}s]") + tstart = datetime.now() + ret = run( + cmd.split(), + stdout=PIPE, + stderr=PIPE, + ) + tend = datetime.now() + ttot = tend - tstart + if ret.returncode != 0: + echo( + CLI_CONFIG, + f"failed. [{ttot.seconds}s]", + ) + echo( + CLI_CONFIG, + f"Continuing; command reports: {ret.stderr.decode().strip()}", + ) + else: + echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")