From 63d0a85e29a1748074c1fcb85564ae64ea20821a Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Tue, 24 Oct 2023 01:08:36 -0400 Subject: [PATCH] Add backup deletion command --- api-daemon/pvcapid/flaskapi.py | 55 +++++++++++++++++ api-daemon/pvcapid/helper.py | 26 ++++++++ client-cli/pvc/cli/cli.py | 57 ++++++++++++++++-- client-cli/pvc/lib/vm.py | 20 +++++++ daemon-common/vm.py | 105 +++++++++++++++++++++++++++++---- 5 files changed, 248 insertions(+), 15 deletions(-) diff --git a/api-daemon/pvcapid/flaskapi.py b/api-daemon/pvcapid/flaskapi.py index fd532800..f4e6dcd0 100755 --- a/api-daemon/pvcapid/flaskapi.py +++ b/api-daemon/pvcapid/flaskapi.py @@ -2360,6 +2360,61 @@ class API_VM_Backup(Resource): vm, target_path, incremental_parent, retain_snapshot ) + @RequestParser( + [ + { + "name": "target_path", + "required": True, + "helptext": "A local filesystem path on the primary coordinator must be specified", + }, + { + "name": "backup_datestring", + "required": True, + "helptext": "A backup datestring must be specified", + }, + ] + ) + @Authenticator + def delete(self, vm, reqargs): + """ + Remove a backup of {vm}, including snapshots, from a local primary coordinator filesystem path + --- + tags: + - vm + parameters: + - in: query + name: target_path + type: string + required: true + description: A local filesystem path on the primary coordinator where the backup is stored + - in: query + name: backup_datestring + type: string + required: true + description: The backup datestring identifier (e.g. 20230102030405) + responses: + 200: + description: OK + schema: + type: object + id: Message + 400: + description: Execution error + schema: + type: object + id: Message + 404: + description: Not found + schema: + type: object + id: Message + """ + target_path = reqargs.get("target_path", None) + backup_datestring = reqargs.get("backup_datestring", None) + return api_helper.vm_remove_backup( + vm, target_path, backup_datestring + ) + api.add_resource(API_VM_Backup, "/vm//backup") diff --git a/api-daemon/pvcapid/helper.py b/api-daemon/pvcapid/helper.py index bc6c9726..24656f15 100755 --- a/api-daemon/pvcapid/helper.py +++ b/api-daemon/pvcapid/helper.py @@ -498,6 +498,32 @@ def vm_backup( return output, retcode +@ZKConnection(config) +def vm_remove_backup( + zkhandler, + domain, + source_path, + datestring, +): + """ + Remove a VM backup from snapshots and a local (primary coordinator) filesystem path. + """ + retflag, retdata = pvc_vm.remove_backup( + zkhandler, + domain, + source_path, + datestring, + ) + + if retflag: + retcode = 200 + else: + retcode = 400 + + output = {"message": retdata.replace('"', "'")} + return output, retcode + + @ZKConnection(config) def vm_restore( zkhandler, diff --git a/client-cli/pvc/cli/cli.py b/client-cli/pvc/cli/cli.py index 45296184..93d6647c 100644 --- a/client-cli/pvc/cli/cli.py +++ b/client-cli/pvc/cli/cli.py @@ -1593,7 +1593,22 @@ def cli_vm_flush_locks(domain): ############################################################################### # > pvc vm backup ############################################################################### -@click.command(name="backup", short_help="Create a backup of a virtual machine.") +@click.group( + name="backup", + short_help="Manage backups for PVC VMs.", + context_settings=CONTEXT_SETTINGS, +) +def cli_vm_backup(): + """ + Manage backups of VMs in a PVC cluster. + """ + pass + + +############################################################################### +# > pvc vm backup create +############################################################################### +@click.command(name="create", short_help="Create a backup of a virtual machine.") @connection_req @click.argument("domain") @click.argument("target_path") @@ -1612,7 +1627,7 @@ def cli_vm_flush_locks(domain): default=False, help="Retain volume snapshot for future incremental use (full only).", ) -def cli_vm_backup(domain, target_path, incremental_parent, retain_snapshot): +def cli_vm_backup_create(domain, target_path, incremental_parent, retain_snapshot): """ Create a backup of virtual machine DOMAIN to TARGET_PATH on the cluster primary coordinator. DOMAIN may be a UUID or name. @@ -1643,7 +1658,7 @@ def cli_vm_backup(domain, target_path, incremental_parent, retain_snapshot): ############################################################################### -# > pvc vm restore +# > pvc vm backup restore ############################################################################### @click.command(name="restore", short_help="Restore a backup of a virtual machine.") @connection_req @@ -1658,7 +1673,7 @@ def cli_vm_backup(domain, target_path, incremental_parent, retain_snapshot): default=True, help="Retain or remove restored (parent, if incremental) snapshot.", ) -def cli_vm_restore(domain, backup_datestring, target_path, retain_snapshot): +def cli_vm_backup_restore(domain, backup_datestring, target_path, retain_snapshot): """ Restore the backup BACKUP_DATESTRING of virtual machine DOMAIN stored in TARGET_PATH on the cluster primary coordinator. DOMAIN may be a UUID or name. @@ -1688,6 +1703,36 @@ def cli_vm_restore(domain, backup_datestring, target_path, retain_snapshot): finish(retcode, retmsg) +############################################################################### +# > pvc vm backup remove +############################################################################### +@click.command(name="remove", short_help="Remove a backup of a virtual machine.") +@connection_req +@click.argument("domain") +@click.argument("backup_datestring") +@click.argument("target_path") +def cli_vm_backup_remove(domain, backup_datestring, target_path): + """ + Remove the backup BACKUP_DATESTRING, including snapshots, of virtual machine DOMAIN stored in TARGET_PATH on the cluster primary coordinator. DOMAIN may be a UUID or name. + + WARNING: Removing an incremental parent will invalidate any existing incremental backups based on that backup. + """ + + echo( + CLI_CONFIG, + f"Removing backup {backup_datestring} of VM '{domain}'... ", + newline=False, + ) + retcode, retmsg = pvc.lib.vm.vm_remove_backup( + CLI_CONFIG, domain, target_path, backup_datestring + ) + if retcode: + echo(CLI_CONFIG, "done.") + else: + echo(CLI_CONFIG, "failed.") + finish(retcode, retmsg) + + ############################################################################### # > pvc vm tag ############################################################################### @@ -5757,8 +5802,10 @@ cli_vm.add_command(cli_vm_move) cli_vm.add_command(cli_vm_migrate) cli_vm.add_command(cli_vm_unmigrate) cli_vm.add_command(cli_vm_flush_locks) +cli_vm_backup.add_command(cli_vm_backup_create) +cli_vm_backup.add_command(cli_vm_backup_restore) +cli_vm_backup.add_command(cli_vm_backup_remove) cli_vm.add_command(cli_vm_backup) -cli_vm.add_command(cli_vm_restore) cli_vm_tag.add_command(cli_vm_tag_get) cli_vm_tag.add_command(cli_vm_tag_add) cli_vm_tag.add_command(cli_vm_tag_remove) diff --git a/client-cli/pvc/lib/vm.py b/client-cli/pvc/lib/vm.py index 09ab6628..459a01b7 100644 --- a/client-cli/pvc/lib/vm.py +++ b/client-cli/pvc/lib/vm.py @@ -454,6 +454,26 @@ def vm_backup(config, vm, target_path, incremental_parent=None, retain_snapshot= return True, response.json().get("message", "") +def vm_remove_backup(config, vm, target_path, backup_datestring): + """ + Remove a backup of {vm}, including snapshots, from a local primary coordinator filesystem path + + API endpoint: DELETE /vm/{vm}/backup + API arguments: target_path={target_path}, backup_datestring={backup_datestring} + API schema: {"message":"{data}"} + """ + params = { + "target_path": target_path, + "backup_datestring": backup_datestring, + } + response = call_api(config, "delete", "/vm/{vm}/backup".format(vm=vm), params=params) + + if response.status_code != 200: + return False, response.json().get("message", "") + else: + return True, response.json().get("message", "") + + def vm_restore(config, vm, target_path, backup_datestring, retain_snapshot=False): """ Restore a backup of {vm} and its volumes from a local primary coordinator filesystem path diff --git a/daemon-common/vm.py b/daemon-common/vm.py index 32abbf8b..75140c0f 100644 --- a/daemon-common/vm.py +++ b/daemon-common/vm.py @@ -25,13 +25,15 @@ import os.path import lxml.objectify import lxml.etree -from distutils.util import strtobool -from uuid import UUID from concurrent.futures import ThreadPoolExecutor from datetime import datetime -from socket import gethostname +from distutils.util import strtobool from json import dump as jdump from json import load as jload +from os import remove +from shutil import rmtree +from socket import gethostname +from uuid import UUID import daemon_lib.common as common @@ -1322,7 +1324,7 @@ def backup_vm( if not dom_uuid: return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain) - # Validate that the target path exists + # Validate that the target path is valid if not re.match(r"^/", target_path): return ( False, @@ -1486,7 +1488,7 @@ def backup_vm( retlines = list() if is_snapshot_remove_failed: - retlines.append(f"WARNING: Failed to remove snapshot as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}") + retlines.append(f"WARNING: Failed to remove snapshot(s) as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}") myhostname = gethostname().split(".")[0] if retain_snapshot: @@ -1497,8 +1499,87 @@ def backup_vm( return True, '\n'.join(retlines) -def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False): +def remove_backup(zkhandler, domain, source_path, datestring): + tstart = time.time() + # 0. Validation + # Validate that VM exists in cluster + dom_uuid = getDomainUUID(zkhandler, domain) + if not dom_uuid: + return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain) + + # Validate that the source path is valid + if not re.match(r"^/", source_path): + return ( + False, + f"ERROR: Source path {source_path} is not a valid absolute path on the primary coordinator!", + ) + + # Ensure that source_path (on this node) exists + if not os.path.isdir(source_path): + return False, f"ERROR: Source path {source_path} does not exist!" + + # Ensure that domain path (on this node) exists + backup_source_path = f"{source_path}/{domain}" + if not os.path.isdir(backup_source_path): + return False, f"ERROR: Source VM path {backup_source_path} does not exist!" + + # Ensure that the archives are present + backup_source_pvcbackup_file = f"{backup_source_path}/{domain}.{datestring}.pvcbackup" + if not os.path.isfile(backup_source_pvcbackup_file): + return False, "ERROR: The specified source backup files do not exist!" + + backup_source_pvcdisks_path = f"{backup_source_path}/{domain}.{datestring}.pvcdisks" + if not os.path.isdir(backup_source_pvcdisks_path): + return False, "ERROR: The specified source backup files do not exist!" + + # 1. Read the backup file and get VM details + try: + with open(backup_source_pvcbackup_file) as fh: + backup_source_details = jload(fh) + except Exception as e: + return False, f"ERROR: Failed to read source backup details: {e}" + + # 2. Remove snapshots + is_snapshot_remove_failed = False + which_snapshot_remove_failed = list() + msg_snapshot_remove_failed = list() + for volume_file, _ in backup_source_details.get('backup_files'): + pool, volume, _ = volume_file.split('/')[-1].split('.') + snapshot = f"backup_{datestring}" + retcode, retmsg = ceph.remove_snapshot(zkhandler, pool, volume, snapshot) + if not retcode: + is_snapshot_remove_failed = True + which_snapshot_remove_failed.append(f"{pool}/{volume}") + msg_snapshot_remove_failed.append(retmsg) + + # 3. Remove files + is_files_remove_failed = False + msg_files_remove_failed = None + try: + remove(backup_source_pvcbackup_file) + rmtree(backup_source_pvcdisks_path) + except Exception as e: + is_files_remove_failed = True + msg_files_remove_failed = e + + tend = time.time() + ttot = round(tend - tstart, 2) + retlines = list() + + if is_snapshot_remove_failed: + retlines.append(f"WARNING: Failed to remove snapshot(s) as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}") + + if is_files_remove_failed: + retlines.append(f"WARNING: Failed to remove backup file(s) from {source_path}: {msg_files_remove_failed}") + + myhostname = gethostname().split(".")[0] + retlines.append(f"Removed VM backup {datestring} for '{domain}' from '{myhostname}:{source_path}' in {ttot}s.") + + return True, '\n'.join(retlines) + + +def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False): tstart = time.time() # 0. Validations @@ -1510,7 +1591,7 @@ def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False f'ERROR: VM "{domain}" already exists in the cluster! Remove or rename it before restoring a backup.', ) - # Validate that the target path exists + # Validate that the source path is valid if not re.match(r"^/", source_path): return ( False, @@ -1626,12 +1707,16 @@ def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False f"rbd snap rm {pool}/{volume}@backup_{incremental_parent}" ) if retcode: - return False, f"ERROR: Failed to remove imported image snapshot for {parent_volume_file}: {stderr}" + is_snapshot_remove_failed = True + which_snapshot_remove_failed.append(f"{pool}/{volume}") + msg_snapshot_remove_failed.append(retmsg) retcode, stdout, stderr = common.run_os_command( f"rbd snap rm {pool}/{volume}@backup_{datestring}" ) if retcode: - return False, f"ERROR: Failed to remove imported image snapshot for {volume_file}: {stderr}" + is_snapshot_remove_failed = True + which_snapshot_remove_failed.append(f"{pool}/{volume}") + msg_snapshot_remove_failed.append(retmsg) else: for volume_file, volume_size in backup_source_details.get('backup_files'): @@ -1681,7 +1766,7 @@ def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False retlines = list() if is_snapshot_remove_failed: - retlines.append(f"WARNING: Failed to remove parent snapshot as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}") + retlines.append(f"WARNING: Failed to remove hanging snapshot(s) as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}") myhostname = gethostname().split(".")[0] retlines.append(f"Successfully restored VM backup {datestring} for '{domain}' from '{myhostname}:{source_path}' in {ttot}s.")