Add enhancements to autobackup
1. Add a cron mode to avoid exit(1) during cronjobs/timers 2. Revamp the remote_mount settings into auto_mount This removes a lot of unnecessary complexity while giving the administrator more flexibility in what they want to execute to mount a filesystem and how. The naming reflects the goal but the possible commands are arbitrary.
This commit is contained in:
parent
6ad51ea4bb
commit
2fccbcda89
|
|
@ -31,117 +31,22 @@ autobackup:
|
|||
# > Should usually be at least 2 when using incrementals (full_interval > 1) to
|
||||
# avoid there being too few backups after cleanup from a new full backup
|
||||
|
||||
# Remote mount settings for backup root path
|
||||
# If remote mount support is disabled, it is up to the administrator to that the backup root path is
|
||||
# created and a valid destination filesystem is mounted on it
|
||||
remote_mount:
|
||||
enabled: no # Enable automatic remote mount/unmount support
|
||||
type: sshfs # Set the type of remote mount; optional if remote_mount is disabled
|
||||
# > Supported values are: sshfs, nfs, cifs (i.e. SMB), cephfs, and s3fs
|
||||
# > WARNING: s3fs has serious known bugs that we don't work around; avoid it if possible
|
||||
|
||||
# Remote mount configurations, per-type; you only need to specify the type(s) you plan to use, but all
|
||||
# are given here for completeness as examples
|
||||
# > NOTE: This key (and all children) are optional if remote mounting is not enabled
|
||||
remote_mount_config:
|
||||
|
||||
# SSHFS specific options
|
||||
# > NOTE: This SSHFS implementation does not support password authentication; keys MUST be used
|
||||
sshfs:
|
||||
# Remote username
|
||||
user: username
|
||||
# Remote hostname
|
||||
host: hostname
|
||||
# Remote path
|
||||
path: /srv/vm_backups
|
||||
# Required command to check for or error
|
||||
command: /usr/bin/sshfs
|
||||
# Options to pass to the mount command (joined, each requires "-o"!)
|
||||
# See the command manual page for more options
|
||||
options:
|
||||
- "-o IdentityFile=/srv/pvc_autobackup.id_ed25519" # Identity (SSH key) file, required!
|
||||
- "-o port=22" # Port number
|
||||
- "-o reconnect" # Enable reconnection
|
||||
- "-o default_permissions" # Enable local permission checking
|
||||
- "-o compression=no" # Disable compression; testing shows that compression slows things
|
||||
# down a fair bit (1m40.435s vs 0m22.253s for 750MB on 10GbE net)
|
||||
- "-o sshfs_sync" # Enable sync; ensures consistent writes with an acceptable performance
|
||||
# overhead (0m22.253s vs 0m17.453s for 750GB on 10GbE net)
|
||||
# Mount command, populated at import time
|
||||
mount_cmd: "{command} {sshfs_user}@{sshfs_host}:{sshfs_path} {backup_root_path} {sshfs_options}"
|
||||
# Unmount command, populated at import time
|
||||
unmount_cmd: "fusermount3 -u {backup_root_path}"
|
||||
|
||||
# NFS specific options
|
||||
nfs:
|
||||
# Remote hostname
|
||||
host: hostname
|
||||
# Remote path
|
||||
path: /srv/vm_backups
|
||||
# Required command to check for or error
|
||||
command: /usr/sbin/mount.nfs
|
||||
# Options to pass to the mount command (joined and passed to "-o")
|
||||
# See the command manual page for more options
|
||||
options:
|
||||
- "nfsvers=3" # Use a specific NFS version
|
||||
# Mount command, populated at import time
|
||||
mount_cmd: "{command} -o {nfs_options} {nfs_host}:{nfs_path} {backup_root_path}"
|
||||
# Unmount command, populated at import time
|
||||
unmount_cmd: "umount {backup_root_path}"
|
||||
|
||||
# CIFS specific options
|
||||
cifs:
|
||||
# Remote hostname
|
||||
host: hostname
|
||||
# Remote path be sure to include the leading '/'!)
|
||||
path: /srv/vm_backups
|
||||
# Required command to check for or error
|
||||
command: /usr/sbin/mount.cifs
|
||||
# Options to pass to the mount command (joined and passed to "-o")
|
||||
# See the command manual page for more options
|
||||
options:
|
||||
- "credentials=/srv/backup_vms.cifs_credentials" # Specify a credentials file
|
||||
- "guest" # Use guest access, alternate to above
|
||||
# Mount command, populated at import time
|
||||
mount_cmd: "{command} -o {cifs_options} //{cifs_host}{cifs_path} {backup_root_path}"
|
||||
# Unmount command, populated at import time
|
||||
unmount_cmd: "umount {backup_root_path}"
|
||||
|
||||
# CephFS specific options
|
||||
cephfs:
|
||||
# Monitor address/hostname list
|
||||
monitors:
|
||||
- mon1
|
||||
# CephFS path; at least "/" is always required
|
||||
path: "/mysubdir"
|
||||
# Required command to check for or error
|
||||
command: /usr/sbin/mount.ceph
|
||||
# Options to pass to mount command (joined and passed to "-o")
|
||||
# See the command manual page for more options
|
||||
options:
|
||||
- "secretfile=/srv/backup_vms.cephfs_secret" # Specify a cephx secret file
|
||||
- "conf=/srv/backup_vms.ceph.conf" # Specify a nonstandard ceph.conf file
|
||||
# Mount command, populated at import time
|
||||
mount_cmd: "{command} {cephfs_monitors}:{cephfs_path} {backup_root_path} -o {cephfs_options}"
|
||||
# Unmount command, populated at import time
|
||||
unmount_cmd: "umount {backup_root_path}"
|
||||
|
||||
# S3FS specific options
|
||||
s3fs:
|
||||
# S3 bucket
|
||||
bucket: mybucket
|
||||
# S3 bucket (sub)path, including leading ':' if used!
|
||||
# Leave empty for no (sub)path
|
||||
path: ":/mypath"
|
||||
# Required command to check for or error
|
||||
command: /usr/bin/s3fs
|
||||
# Options to pass to the mount command (joined, each requires "-o"!)
|
||||
# See the command manual page for more options
|
||||
options:
|
||||
- "-o passwd_file=/srv/backup_vms.s3fs_credentials" # Specify a password file
|
||||
- "-o host=https://s3.amazonaws.com" # Specify an alternate host
|
||||
- "-o endpoint=us-east-1" # Specify an alternate endpoint/region
|
||||
# Mount command, populated at import time
|
||||
mount_cmd: "{command} {s2fs_bucket}{s3fs_path} {backup_root_path} {s3fs_options}"
|
||||
# Unmount command, populated at import time
|
||||
unmount_cmd: "fusermount3 -u {backup_root_path}"
|
||||
# Automatic mount settings
|
||||
# These settings permit running an arbitrary set of commands, ideally a "mount" command or similar, to
|
||||
# ensure that a remote filesystem is mounted on the backup root path
|
||||
# While the examples here show absolute paths, that is not required; they will run with the $PATH of the
|
||||
# executing environment (either the "pvc" command on a CLI or a cron/systemd timer)
|
||||
# A "{backup_root_path}" f-string/str.format type variable MAY be present in any cmds string to represent
|
||||
# the above configured root backup path, which is interpolated at runtime
|
||||
# If multiple commands are given, they will be executed in the order given; if no commands are given,
|
||||
# nothing is executed, but the keys MUST be present
|
||||
auto_mount:
|
||||
enabled: no # Enable automatic mount/unmount support
|
||||
# These commands are executed at the start of the backup run and should mount a filesystem
|
||||
mount_cmds:
|
||||
# This example shows an NFS mount leveraging the backup_root_path variable
|
||||
- "/usr/sbin/mount.nfs -o nfsvers=3 10.0.0.10:/backups {backup_root_path}"
|
||||
# These commands are executed at the end of the backup run and should unmount a filesystem
|
||||
unmount_cmds:
|
||||
# This example shows a generic umount leveraging the backup_root_path variable
|
||||
- "/usr/bin/umount {backup_root_path}"
|
||||
|
|
|
|||
|
|
@ -1755,7 +1755,7 @@ def cli_vm_autobackup(autobackup_cfgfile, force_full_flag, cron_flag):
|
|||
functions with an internal rentention and cleanup system as well as determination of full vs. incremental
|
||||
backups at different intervals. VMs are selected based on configured VM tags. The destination storage
|
||||
may either be local, or provided by a remote filesystem which is automatically mounted and unmounted during
|
||||
the backup run.
|
||||
the backup run via a set of configured commands before and after the backup run.
|
||||
|
||||
NOTE: This command performs its tasks in a local context. It MUST be run from the cluster's active primary
|
||||
coordinator using the "local" connection only; if either is not correct, the command will error.
|
||||
|
|
|
|||
|
|
@ -258,118 +258,25 @@ def get_autobackup_config(CLI_CONFIG, cfgfile):
|
|||
config["backup_root_suffix"] = backup_config["backup_root_suffix"]
|
||||
config["backup_tags"] = backup_config["backup_tags"]
|
||||
config["backup_schedule"] = backup_config["backup_schedule"]
|
||||
config["remote_mount_enabled"] = backup_config["remote_mount"]["enabled"]
|
||||
if config["remote_mount_enabled"]:
|
||||
config["remote_mount_type"] = backup_config["remote_mount"]["type"]
|
||||
else:
|
||||
config["remote_mount_type"] = None
|
||||
config["auto_mount_enabled"] = backup_config["auto_mount"]["enabled"]
|
||||
if config["auto_mount_enabled"]:
|
||||
config["mount_cmds"] = list()
|
||||
_mount_cmds = backup_config["auto_mount"]["mount_cmds"]
|
||||
for _mount_cmd in _mount_cmds:
|
||||
if "{backup_root_path}" in _mount_cmd:
|
||||
_mount_cmd = _mount_cmd.format(
|
||||
backup_root_path=backup_config["backup_root_path"]
|
||||
)
|
||||
config["mount_cmds"].append(_mount_cmd)
|
||||
|
||||
if config["remote_mount_type"] == "sshfs":
|
||||
config["check_command"] = backup_config["remote_mount_config"]["sshfs"][
|
||||
"command"
|
||||
]
|
||||
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["sshfs"][
|
||||
"mount_cmd"
|
||||
].format(
|
||||
command=backup_config["remote_mount_config"]["sshfs"]["command"],
|
||||
sshfs_user=backup_config["remote_mount_config"]["sshfs"]["user"],
|
||||
sshfs_host=backup_config["remote_mount_config"]["sshfs"]["host"],
|
||||
sshfs_path=backup_config["remote_mount_config"]["sshfs"]["path"],
|
||||
sshfs_options=" ".join(
|
||||
backup_config["remote_mount_config"]["sshfs"]["options"]
|
||||
),
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
config["remote_unmount_cmd"] = backup_config["remote_mount_config"][
|
||||
"sshfs"
|
||||
]["unmount_cmd"].format(
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
elif config["remote_mount_type"] == "nfs":
|
||||
config["check_command"] = backup_config["remote_mount_config"]["nfs"][
|
||||
"command"
|
||||
]
|
||||
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["nfs"][
|
||||
"mount_cmd"
|
||||
].format(
|
||||
command=backup_config["remote_mount_config"]["nfs"]["command"],
|
||||
nfs_host=backup_config["remote_mount_config"]["nfs"]["host"],
|
||||
nfs_path=backup_config["remote_mount_config"]["nfs"]["path"],
|
||||
nfs_options=",".join(
|
||||
backup_config["remote_mount_config"]["nfs"]["options"]
|
||||
),
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["nfs"][
|
||||
"unmount_cmd"
|
||||
].format(
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
elif config["remote_mount_type"] == "cifs":
|
||||
config["check_command"] = backup_config["remote_mount_config"]["cifs"][
|
||||
"command"
|
||||
]
|
||||
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cifs"][
|
||||
"mount_cmd"
|
||||
].format(
|
||||
command=backup_config["remote_mount_config"]["cifs"]["command"],
|
||||
cifs_host=backup_config["remote_mount_config"]["cifs"]["host"],
|
||||
cifs_path=backup_config["remote_mount_config"]["cifs"]["path"],
|
||||
cifs_options=",".join(
|
||||
backup_config["remote_mount_config"]["cifs"]["options"]
|
||||
),
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["cifs"][
|
||||
"unmount_cmd"
|
||||
].format(
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
elif config["remote_mount_type"] == "s3fs":
|
||||
config["check_command"] = backup_config["remote_mount_config"]["s3fs"][
|
||||
"command"
|
||||
]
|
||||
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["s3fs"][
|
||||
"mount_cmd"
|
||||
].format(
|
||||
command=backup_config["remote_mount_config"]["s3fs"]["command"],
|
||||
s2fs_bucket=backup_config["remote_mount_config"]["s3fs"]["bucket"],
|
||||
s2fs_path=backup_config["remote_mount_config"]["s3fs"]["path"],
|
||||
s3fs_options=" ".join(
|
||||
backup_config["remote_mount_config"]["s3fs"]["options"]
|
||||
),
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
config["remote_unmount_cmd"] = backup_config["remote_mount_config"]["s3fs"][
|
||||
"unmount_cmd"
|
||||
].format(
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
elif config["remote_mount_type"] == "cephfs":
|
||||
config["check_command"] = backup_config["remote_mount_config"]["cephfs"][
|
||||
"command"
|
||||
]
|
||||
config["remote_mount_cmd"] = backup_config["remote_mount_config"]["cephfs"][
|
||||
"mount_cmd"
|
||||
].format(
|
||||
command=backup_config["remote_mount_config"]["cephfs"]["command"],
|
||||
cephfs_monitors=",".join(
|
||||
backup_config["remote_mount_config"]["cephfs"]["monitors"]
|
||||
),
|
||||
cephfs_path=backup_config["remote_mount_config"]["cephfs"]["path"],
|
||||
cephfs_options=",".join(
|
||||
backup_config["remote_mount_config"]["cephfs"]["options"]
|
||||
),
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
config["remote_unmount_cmd"] = backup_config["remote_mount_config"][
|
||||
"cephfs"
|
||||
]["unmount_cmd"].format(
|
||||
backup_root_path=backup_config["backup_root_path"],
|
||||
)
|
||||
else:
|
||||
config["remote_mount_cmd"] = None
|
||||
config["remote_unmount_cmd"] = None
|
||||
config["unmount_cmds"] = list()
|
||||
_unmount_cmds = backup_config["auto_mount"]["unmount_cmds"]
|
||||
for _unmount_cmd in _unmount_cmds:
|
||||
if "{backup_root_path}" in _unmount_cmd:
|
||||
_unmount_cmd = _unmount_cmd.format(
|
||||
backup_root_path=backup_config["backup_root_path"]
|
||||
)
|
||||
config["unmount_cmds"].append(_unmount_cmd)
|
||||
|
||||
except FileNotFoundError:
|
||||
echo(CLI_CONFIG, "ERROR: Specified backup configuration does not exist!")
|
||||
|
|
@ -382,7 +289,10 @@ def get_autobackup_config(CLI_CONFIG, cfgfile):
|
|||
|
||||
|
||||
def vm_autobackup(
|
||||
CLI_CONFIG, autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME, force_full_flag=False
|
||||
CLI_CONFIG,
|
||||
autobackup_cfgfile=DEFAULT_AUTOBACKUP_FILENAME,
|
||||
force_full_flag=False,
|
||||
cron_flag=False,
|
||||
):
|
||||
"""
|
||||
Perform automatic backups of VMs based on an external config file.
|
||||
|
|
@ -393,15 +303,22 @@ def vm_autobackup(
|
|||
CLI_CONFIG["connection"] = "local"
|
||||
retcode, retdata = pvc.lib.node.node_info(CLI_CONFIG, DEFAULT_NODE_HOSTNAME)
|
||||
if not retcode or retdata.get("coordinator_state") != "primary":
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.",
|
||||
)
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
"Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.",
|
||||
)
|
||||
exit(1)
|
||||
if cron_flag:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
"Current host is not the primary coordinator of the local cluster and running in cron mode. Exiting cleanly.",
|
||||
)
|
||||
exit(0)
|
||||
else:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"ERROR: Current host is not the primary coordinator of the local cluster; got connection '{real_connection}', host '{DEFAULT_NODE_HOSTNAME}'.",
|
||||
)
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
"Autobackup MUST be run from the cluster active primary coordinator using the 'local' connection. See '-h'/'--help' for details.",
|
||||
)
|
||||
exit(1)
|
||||
|
||||
# Ensure we're running as root, or show a warning & confirmation
|
||||
if getuser() != "root":
|
||||
|
|
@ -463,38 +380,34 @@ def vm_autobackup(
|
|||
echo(CLI_CONFIG, " {}".format("\n ".join(vm_list_rows)), stderr=True)
|
||||
echo(CLI_CONFIG, "", stderr=True)
|
||||
|
||||
if autobackup_config["remote_mount_cmd"] is not None:
|
||||
# Validate that the mount command is valid
|
||||
if not path.exists(autobackup_config["check_command"]):
|
||||
if autobackup_config["auto_mount_enabled"]:
|
||||
# Execute each mount_cmds command in sequence
|
||||
for cmd in autobackup_config["mount_cmds"]:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"ERROR: Failed to find required command {autobackup_config['check_command']}; ensure it is installed.",
|
||||
f"Executing mount command '{cmd.split()[0]}'... ",
|
||||
newline=False,
|
||||
)
|
||||
exit(1)
|
||||
|
||||
# Try to mount the remote mount
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"Mounting remote {autobackup_config['remote_mount_type']} filesystem on {autobackup_config['backup_root_path']}... ",
|
||||
newline=False,
|
||||
)
|
||||
tstart = datetime.now()
|
||||
ret = run(
|
||||
autobackup_config["remote_mount_cmd"].split(),
|
||||
stdout=PIPE,
|
||||
stderr=PIPE,
|
||||
)
|
||||
tend = datetime.now()
|
||||
ttot = tend - tstart
|
||||
if ret.returncode != 0:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"failed. [{ttot.seconds}s]",
|
||||
tstart = datetime.now()
|
||||
ret = run(
|
||||
cmd.split(),
|
||||
stdout=PIPE,
|
||||
stderr=PIPE,
|
||||
)
|
||||
echo(CLI_CONFIG, f"Exiting; command reports: {ret.stderr.decode().strip()}")
|
||||
exit(1)
|
||||
else:
|
||||
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
|
||||
tend = datetime.now()
|
||||
ttot = tend - tstart
|
||||
if ret.returncode != 0:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"failed. [{ttot.seconds}s]",
|
||||
)
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"Exiting; command reports: {ret.stderr.decode().strip()}",
|
||||
)
|
||||
exit(1)
|
||||
else:
|
||||
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
|
||||
|
||||
# For each VM, perform the backup
|
||||
for vm in backup_vms:
|
||||
|
|
@ -625,26 +538,30 @@ def vm_autobackup(
|
|||
with open(autobackup_state_file, "w") as fh:
|
||||
jdump(state_data, fh)
|
||||
|
||||
# Try to unmount the remote mount
|
||||
if autobackup_config["remote_unmount_cmd"] is not None:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"Unmounting remote {autobackup_config['remote_mount_type']} filesystem from {autobackup_config['backup_root_path']}... ",
|
||||
newline=False,
|
||||
)
|
||||
tstart = datetime.now()
|
||||
ret = run(
|
||||
autobackup_config["remote_unmount_cmd"].split(),
|
||||
stdout=PIPE,
|
||||
stderr=PIPE,
|
||||
)
|
||||
tend = datetime.now()
|
||||
ttot = tend - tstart
|
||||
if ret.returncode != 0:
|
||||
echo(CLI_CONFIG, f"failed. [{ttot.seconds}s]")
|
||||
if autobackup_config["auto_mount_enabled"]:
|
||||
# Execute each unmount_cmds command in sequence
|
||||
for cmd in autobackup_config["unmount_cmds"]:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"Continuing; command reports: {ret.stderr.decode().strip()}",
|
||||
f"Executing unmount command '{cmd.split()[0]}'... ",
|
||||
newline=False,
|
||||
)
|
||||
else:
|
||||
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
|
||||
tstart = datetime.now()
|
||||
ret = run(
|
||||
cmd.split(),
|
||||
stdout=PIPE,
|
||||
stderr=PIPE,
|
||||
)
|
||||
tend = datetime.now()
|
||||
ttot = tend - tstart
|
||||
if ret.returncode != 0:
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"failed. [{ttot.seconds}s]",
|
||||
)
|
||||
echo(
|
||||
CLI_CONFIG,
|
||||
f"Continuing; command reports: {ret.stderr.decode().strip()}",
|
||||
)
|
||||
else:
|
||||
echo(CLI_CONFIG, f"done. [{ttot.seconds}s]")
|
||||
|
|
|
|||
Loading…
Reference in New Issue