Add enhancements to autobackup

1. Add a cron mode to avoid exit(1) during cronjobs/timers
2. Revamp the remote_mount settings into auto_mount
   This removes a lot of unnecessary complexity while giving the
   administrator more flexibility in what they want to execute to mount
   a filesystem and how. The naming reflects the goal but the possible
   commands are arbitrary.
This commit is contained in:
Joshua Boniface 2023-10-27 01:47:10 -04:00
parent 6ad51ea4bb
commit 2fccbcda89
3 changed files with 106 additions and 284 deletions

View File

@ -31,117 +31,22 @@ autobackup:
# > Should usually be at least 2 when using incrementals (full_interval > 1) to
# avoid there being too few backups after cleanup from a new full backup
# Remote mount settings for backup root path
# If remote mount support is disabled, it is up to the administrator to that the backup root path is
# created and a valid destination filesystem is mounted on it
remote_mount:
enabled: no # Enable automatic remote mount/unmount support
type: sshfs # Set the type of remote mount; optional if remote_mount is disabled
# > Supported values are: sshfs, nfs, cifs (i.e. SMB), cephfs, and s3fs
# > WARNING: s3fs has serious known bugs that we don't work around; avoid it if possible
# Remote mount configurations, per-type; you only need to specify the type(s) you plan to use, but all
# are given here for completeness as examples
# > NOTE: This key (and all children) are optional if remote mounting is not enabled
remote_mount_config:
# SSHFS specific options
# > NOTE: This SSHFS implementation does not support password authentication; keys MUST be used
sshfs:
# Remote username
user: username
# Remote hostname
host: hostname
# Remote path
path: /srv/vm_backups
# Required command to check for or error
command: /usr/bin/sshfs
# Options to pass to the mount command (joined, each requires "-o"!)
# See the command manual page for more options
options:
- "-o IdentityFile=/srv/pvc_autobackup.id_ed25519" # Identity (SSH key) file, required!
- "-o port=22" # Port number
- "-o reconnect" # Enable reconnection
- "-o default_permissions" # Enable local permission checking
- "-o compression=no" # Disable compression; testing shows that compression slows things
# down a fair bit (1m40.435s vs 0m22.253s for 750MB on 10GbE net)
- "-o sshfs_sync" # Enable sync; ensures consistent writes with an acceptable performance
# overhead (0m22.253s vs 0m17.453s for 750GB on 10GbE net)
# Mount command, populated at import time
mount_cmd: "{command} {sshfs_user}@{sshfs_host}:{sshfs_path} {backup_root_path} {sshfs_options}"
# Unmount command, populated at import time
unmount_cmd: "fusermount3 -u {backup_root_path}"
# NFS specific options
nfs:
# Remote hostname
host: hostname
# Remote path
path: /srv/vm_backups
# Required command to check for or error
command: /usr/sbin/mount.nfs
# Options to pass to the mount command (joined and passed to "-o")
# See the command manual page for more options
options:
- "nfsvers=3" # Use a specific NFS version
# Mount command, populated at import time
mount_cmd: "{command} -o {nfs_options} {nfs_host}:{nfs_path} {backup_root_path}"
# Unmount command, populated at import time
unmount_cmd: "umount {backup_root_path}"
# CIFS specific options
cifs:
# Remote hostname
host: hostname
# Remote path be sure to include the leading '/'!)
path: /srv/vm_backups
# Required command to check for or error
command: /usr/sbin/mount.cifs
# Options to pass to the mount command (joined and passed to "-o")
# See the command manual page for more options
options:
- "credentials=/srv/backup_vms.cifs_credentials" # Specify a credentials file
- "guest" # Use guest access, alternate to above
# Mount command, populated at import time
mount_cmd: "{command} -o {cifs_options} //{cifs_host}{cifs_path} {backup_root_path}"
# Unmount command, populated at import time
unmount_cmd: "umount {backup_root_path}"
# CephFS specific options
cephfs:
# Monitor address/hostname list
monitors:
- mon1
# CephFS path; at least "/" is always required
path: "/mysubdir"
# Required command to check for or error
command: /usr/sbin/mount.ceph
# Options to pass to mount command (joined and passed to "-o")
# See the command manual page for more options
options:
- "secretfile=/srv/backup_vms.cephfs_secret" # Specify a cephx secret file
- "conf=/srv/backup_vms.ceph.conf" # Specify a nonstandard ceph.conf file
# Mount command, populated at import time
mount_cmd: "{command} {cephfs_monitors}:{cephfs_path} {backup_root_path} -o {cephfs_options}"
# Unmount command, populated at import time
unmount_cmd: "umount {backup_root_path}"
# S3FS specific options
s3fs:
# S3 bucket
bucket: mybucket
# S3 bucket (sub)path, including leading ':' if used!
# Leave empty for no (sub)path
path: ":/mypath"
# Required command to check for or error
command: /usr/bin/s3fs
# Options to pass to the mount command (joined, each requires "-o"!)
# See the command manual page for more options
options:
- "-o passwd_file=/srv/backup_vms.s3fs_credentials" # Specify a password file
- "-o host=https://s3.amazonaws.com" # Specify an alternate host
- "-o endpoint=us-east-1" # Specify an alternate endpoint/region
# Mount command, populated at import time
mount_cmd: "{command} {s2fs_bucket}{s3fs_path} {backup_root_path} {s3fs_options}"
# Unmount command, populated at import time
unmount_cmd: "fusermount3 -u {backup_root_path}"
# Automatic mount settings
# These settings permit running an arbitrary set of commands, ideally a "mount" command or similar, to
# ensure that a remote filesystem is mounted on the backup root path
# While the examples here show absolute paths, that is not required; they will run with the $PATH of the
# executing environment (either the "pvc" command on a CLI or a cron/systemd timer)
# A "{backup_root_path}" f-string/str.format type variable MAY be present in any cmds string to represent
# the above configured root backup path, which is interpolated at runtime
# If multiple commands are given, they will be executed in the order given; if no commands are given,
# nothing is executed, but the keys MUST be present
auto_mount:
enabled: no # Enable automatic mount/unmount support
# These commands are executed at the start of the backup run and should mount a filesystem
mount_cmds:
# This example shows an NFS mount leveraging the backup_root_path variable
- "/usr/sbin/mount.nfs -o nfsvers=3 10.0.0.10:/backups {backup_root_path}"
# These commands are executed at the end of the backup run and should unmount a filesystem
unmount_cmds:
# This example shows a generic umount leveraging the backup_root_path variable
- "/usr/bin/umount {backup_root_path}"

View File

@ -1755,7 +1755,7 @@ def cli_vm_autobackup(autobackup_cfgfile, force_full_flag, cron_flag):
functions with an internal rentention and cleanup system as well as determination of full vs. incremental
backups at different intervals. VMs are selected based on configured VM tags. The destination storage
may either be local, or provided by a remote filesystem which is automatically mounted and unmounted during
the backup run.
the backup run via a set of configured commands before and after the backup run.
NOTE: This command performs its tasks in a local context. It MUST be run from the cluster's active primary
coordinator using the "local" connection only; if either is not correct, the command will error.

View File

@ -258,118 +258,25 @@ def get_autobackup_config(CLI_CONFIG, cfgfile):
config["backup_root_suffix"] = backup_config["backup_root_suffix"]
config["backup_tags"] = backup_config["backup_tags"]
config["backup_schedule"] = backup_config["backup_schedule"]
config["remote_mount_enabled"] = backup_config["remote_mount"]["enabled"]
if config["remote_mount_enabled"]:
config["remote_mount_type"] = backup_config["remote_mount"]["type"]
else:
config["remote_mount_type"] = None
config["auto_mount_enabled"] = backup_config["auto_mount"]["enabled"]
if config["auto_mount_enabled"]:
config["mount_cmds"] = list()
_mount_cmds = backup_config["auto_mount"]["mount_cmds"]
for _mount_cmd in _mount_cmds:
if "{backup_root_path}" in _mount_cmd:
_mount_cmd = _mount_cmd.format(
backup_root_path=backup_config["backup_root_path"]
)
config["mount_cmds"].append(_mount_cmd)
if config["remote_mount_type"] == "sshfs":
config["check_command"] = backup_config["remote_mount_config"]["sshfs"][
"command"
]
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["sshfs"][
"mount_cmd"
].format(
command=backup_config["remote_mount_config"]["sshfs"]["command"],
sshfs_user=backup_config["remote_mount_config"]["sshfs"]["user"],
sshfs_host=backup_config["remote_mount_config"]["sshfs"]["host"],
sshfs_path=backup_config["remote_mount_config"]["sshfs"]["path"],
sshfs_options=" ".join(
backup_config["remote_mount_config"]["sshfs"]["options"]
),
backup_root_path=backup_config["backup_root_path"],
)
config["remote_unmount_cmd"] = backup_config["remote_mount_config"][
"sshfs"
]["unmount_cmd"].format(
backup_root_path=backup_config["backup_root_path"],
)
elif config["remote_mount_type"] == "nfs":
config["check_command"] = backup_config["remote_mount_config"]["nfs"][
"command"
]
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["nfs"][
"mount_cmd"
].format(
command=backup_config["remote_mount_config"]["nfs"]["command"],
nfs_host=backup_config["remote_mount_config"]["nfs"]["host"],
nfs_path=backup_config["remote_mount_config"]["nfs"]["path"],
nfs_options=",".join(
backup_config["remote_mount_config"]["nfs"]["options"]
),
backup_root_path=backup_config["backup_root_path"],
)
config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["nfs"][
"unmount_cmd"
].format(
backup_root_path=backup_config["backup_root_path"],
)
elif config["remote_mount_type"] == "cifs":
config["check_command"] = backup_config["remote_mount_config"]["cifs"][
"command"
]
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cifs"][
"mount_cmd"
].format(
command=backup_config["remote_mount_config"]["cifs"]["command"],
cifs_host=backup_config["remote_mount_config"]["cifs"]["host"],
cifs_path=backup_config["remote_mount_config"]["cifs"]["path"],
cifs_options=",".join(
backup_config["remote_mount_config"]["cifs"]["options"]
),
backup_root_path=backup_config["backup_root_path"],
)
config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["cifs"][
"unmount_cmd"
].format(
backup_root_path=backup_config["backup_root_path"],
)
elif config["remote_mount_type"] == "s3fs":
config["check_command"] = backup_config["remote_mount_config"]["s3fs"][
"command"
]
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["s3fs"][
"mount_cmd"
].format(
command=backup_config["remote_mount_config"]["s3fs"]["command"],
s2fs_bucket=backup_config["remote_mount_config"]["s3fs"]["bucket"],
s2fs_path=backup_config["remote_mount_config"]["s3fs"]["path"],
s3fs_options=" ".join(
backup_config["remote_mount_config"]["s3fs"]["options"]
),
backup_root_path=backup_config["backup_root_path"],
)
config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["s3fs"][
"unmount_cmd"
].format(
backup_root_path=backup_config["backup_root_path"],
)
elif config["remote_mount_type"] == "cephfs":
config["check_command"] = backup_config["remote_mount_config"]["cephfs"][
"command"
]
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cephfs"][
"mount_cmd"
].format(
command=backup_config["remote_mount_config"]["cephfs"]["command"],
cephfs_monitors=",".join(
backup_config["remote_mount_config"]["cephfs"]["monitors"]
),
cephfs_path=backup_config["remote_mount_config"]["cephfs"]["path"],
cephfs_options=",".join(
backup_config["remote_mount_config"]["cephfs"]["options"]
),
backup_root_path=backup_config["backup_root_path"],
)
config["remote_unmount_cmd"] = backup_config["remote_mount_config"][
"cephfs"
]["unmount_cmd"].format(
backup_root_path=backup_config["backup_root_path"],
)
else:
config["remote_mount_cmd"] = None
config["remote_unmount_cmd"] = None
config["unmount_cmds"] = list()
_unmount_cmds = backup_config["auto_mount"]["unmount_cmds"]
for _unmount_cmd in _unmount_cmds:
if "{backup_root_path}" in _unmount_cmd:
_unmount_cmd = _unmount_cmd.format(
backup_root_path=backup_config["backup_root_path"]
)
config["unmount_cmds"].append(_unmount_cmd)
except FileNotFoundError:
echo(CLI_CONFIG, "ERROR: Specified backup configuration does not exist!")
@ -382,7 +289,10 @@ def get_autobackup_config(CLI_CONFIG, cfgfile):
def vm_autobackup(
CLI_CONFIG, autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME, force_full_flag=False
CLI_CONFIG,
autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME,
force_full_flag=False,
cron_flag=False,
):
"""
Perform automatic backups of VMs based on an external config file.
@ -393,15 +303,22 @@ def vm_autobackup(
CLI_CONFIG["connection"] = "local"
retcode, retdata = pvc.lib.node.node_info(CLI_CONFIG, DEFAULT_NODE_HOSTNAME)
if not retcode or retdata.get("coordinator_state") != "primary":
echo(
CLI_CONFIG,
f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.",
)
echo(
CLI_CONFIG,
"Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.",
)
exit(1)
if cron_flag:
echo(
CLI_CONFIG,
"Current host is not the primary coordinator of the local cluster and running in cron mode. Exiting cleanly.",
)
exit(0)
else:
echo(
CLI_CONFIG,
f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.",
)
echo(
CLI_CONFIG,
"Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.",
)
exit(1)
# Ensure we're running as root, or show a warning & confirmation
if getuser() != "root":
@ -463,38 +380,34 @@ def vm_autobackup(
echo(CLI_CONFIG, " {}".format("\n ".join(vm_list_rows)), stderr=True)
echo(CLI_CONFIG, "", stderr=True)
if autobackup_config["remote_mount_cmd"] is not None:
# Validate that the mount command is valid
if not path.exists(autobackup_config["check_command"]):
if autobackup_config["auto_mount_enabled"]:
# Execute each mount_cmds command in sequence
for cmd in autobackup_config["mount_cmds"]:
echo(
CLI_CONFIG,
f"ERROR: Failed to find required command {autobackup_config['check_command']}; ensure it is installed.",
f"Executing mount command '{cmd.split()[0]}'... ",
newline=False,
)
exit(1)
# Try to mount the remote mount
echo(
CLI_CONFIG,
f"Mounting remote {autobackup_config['remote_mount_type']} filesystem on {autobackup_config['backup_root_path']}... ",
newline=False,
)
tstart = datetime.now()
ret = run(
autobackup_config["remote_mount_cmd"].split(),
stdout=PIPE,
stderr=PIPE,
)
tend = datetime.now()
ttot = tend - tstart
if ret.returncode != 0:
echo(
CLI_CONFIG,
f"failed. [{ttot.seconds}s]",
tstart = datetime.now()
ret = run(
cmd.split(),
stdout=PIPE,
stderr=PIPE,
)
echo(CLI_CONFIG, f"Exiting; command reports: {ret.stderr.decode().strip()}")
exit(1)
else:
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
tend = datetime.now()
ttot = tend - tstart
if ret.returncode != 0:
echo(
CLI_CONFIG,
f"failed. [{ttot.seconds}s]",
)
echo(
CLI_CONFIG,
f"Exiting; command reports: {ret.stderr.decode().strip()}",
)
exit(1)
else:
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
# For each VM, perform the backup
for vm in backup_vms:
@ -625,26 +538,30 @@ def vm_autobackup(
with open(autobackup_state_file, "w") as fh:
jdump(state_data, fh)
# Try to unmount the remote mount
if autobackup_config["remote_unmount_cmd"] is not None:
echo(
CLI_CONFIG,
f"Unmounting remote {autobackup_config['remote_mount_type']} filesystem from {autobackup_config['backup_root_path']}... ",
newline=False,
)
tstart = datetime.now()
ret = run(
autobackup_config["remote_unmount_cmd"].split(),
stdout=PIPE,
stderr=PIPE,
)
tend = datetime.now()
ttot = tend - tstart
if ret.returncode != 0:
echo(CLI_CONFIG, f"failed. [{ttot.seconds}s]")
if autobackup_config["auto_mount_enabled"]:
# Execute each unmount_cmds command in sequence
for cmd in autobackup_config["unmount_cmds"]:
echo(
CLI_CONFIG,
f"Continuing; command reports: {ret.stderr.decode().strip()}",
f"Executing unmount command '{cmd.split()[0]}'... ",
newline=False,
)
else:
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
tstart = datetime.now()
ret = run(
cmd.split(),
stdout=PIPE,
stderr=PIPE,
)
tend = datetime.now()
ttot = tend - tstart
if ret.returncode != 0:
echo(
CLI_CONFIG,
f"failed. [{ttot.seconds}s]",
)
echo(
CLI_CONFIG,
f"Continuing; command reports: {ret.stderr.decode().strip()}",
)
else:
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")