Add backup deletion command

api-daemon/pvcapid/flaskapi.py

@@ -2360,6 +2360,61 @@ class API_VM_Backup(Resource):
             vm, target_path, incremental_parent, retain_snapshot
         )
 
+    @RequestParser(
+        [
+            {
+                "name": "target_path",
+                "required": True,
+                "helptext": "A local filesystem path on the primary coordinator must be specified",
+            },
+            {
+                "name": "backup_datestring",
+                "required": True,
+                "helptext": "A backup datestring must be specified",
+            },
+        ]
+    )
+    @Authenticator
+    def delete(self, vm, reqargs):
+        """
+        Remove a backup of {vm}, including snapshots, from a local primary coordinator filesystem path
+        ---
+        tags:
+          - vm
+        parameters:
+          - in: query
+            name: target_path
+            type: string
+            required: true
+            description: A local filesystem path on the primary coordinator where the backup is stored
+          - in: query
+            name: backup_datestring
+            type: string
+            required: true
+            description: The backup datestring identifier (e.g. 20230102030405)
+        responses:
+          200:
+            description: OK
+            schema:
+              type: object
+              id: Message
+          400:
+            description: Execution error
+            schema:
+              type: object
+              id: Message
+          404:
+            description: Not found
+            schema:
+              type: object
+              id: Message
+        """
+        target_path = reqargs.get("target_path", None)
+        backup_datestring = reqargs.get("backup_datestring", None)
+        return api_helper.vm_remove_backup(
+            vm, target_path, backup_datestring
+        )
+
 
 api.add_resource(API_VM_Backup, "/vm/<vm>/backup")
 

api-daemon/pvcapid/helper.py

@@ -498,6 +498,32 @@ def vm_backup(
     return output, retcode
 
 
+@ZKConnection(config)
+def vm_remove_backup(
+    zkhandler,
+    domain,
+    source_path,
+    datestring,
+):
+    """
+    Remove a VM backup from snapshots and a local (primary coordinator) filesystem path.
+    """
+    retflag, retdata = pvc_vm.remove_backup(
+        zkhandler,
+        domain,
+        source_path,
+        datestring,
+    )
+
+    if retflag:
+        retcode = 200
+    else:
+        retcode = 400
+
+    output = {"message": retdata.replace('"', "'")}
+    return output, retcode
+
+
 @ZKConnection(config)
 def vm_restore(
     zkhandler,

client-cli/pvc/cli/cli.py

@@ -1593,7 +1593,22 @@ def cli_vm_flush_locks(domain):
 ###############################################################################
 # > pvc vm backup
 ###############################################################################
-@click.command(name="backup", short_help="Create a backup of a virtual machine.")
+@click.group(
+    name="backup",
+    short_help="Manage backups for PVC VMs.",
+    context_settings=CONTEXT_SETTINGS,
+)
+def cli_vm_backup():
+    """
+    Manage backups of VMs in a PVC cluster.
+    """
+    pass
+
+
+###############################################################################
+# > pvc vm backup create
+###############################################################################
+@click.command(name="create", short_help="Create a backup of a virtual machine.")
 @connection_req
 @click.argument("domain")
 @click.argument("target_path")
@@ -1612,7 +1627,7 @@ def cli_vm_flush_locks(domain):
     default=False,
     help="Retain volume snapshot for future incremental use (full only).",
 )
-def cli_vm_backup(domain, target_path, incremental_parent, retain_snapshot):
+def cli_vm_backup_create(domain, target_path, incremental_parent, retain_snapshot):
     """
     Create a backup of virtual machine DOMAIN to TARGET_PATH on the cluster primary coordinator. DOMAIN may be a UUID or name.
 
@@ -1643,7 +1658,7 @@ def cli_vm_backup(domain, target_path, incremental_parent, retain_snapshot):
 
 
 ###############################################################################
-# > pvc vm restore
+# > pvc vm backup restore
 ###############################################################################
 @click.command(name="restore", short_help="Restore a backup of a virtual machine.")
 @connection_req
@@ -1658,7 +1673,7 @@ def cli_vm_backup(domain, target_path, incremental_parent, retain_snapshot):
     default=True,
     help="Retain or remove restored (parent, if incremental) snapshot.",
 )
-def cli_vm_restore(domain, backup_datestring, target_path, retain_snapshot):
+def cli_vm_backup_restore(domain, backup_datestring, target_path, retain_snapshot):
     """
     Restore the backup BACKUP_DATESTRING of virtual machine DOMAIN stored in TARGET_PATH on the cluster primary coordinator. DOMAIN may be a UUID or name.
 
@@ -1688,6 +1703,36 @@ def cli_vm_restore(domain, backup_datestring, target_path, retain_snapshot):
     finish(retcode, retmsg)
 
 
+###############################################################################
+# > pvc vm backup remove
+###############################################################################
+@click.command(name="remove", short_help="Remove a backup of a virtual machine.")
+@connection_req
+@click.argument("domain")
+@click.argument("backup_datestring")
+@click.argument("target_path")
+def cli_vm_backup_remove(domain, backup_datestring, target_path):
+    """
+    Remove the backup BACKUP_DATESTRING, including snapshots, of virtual machine DOMAIN stored in TARGET_PATH on the cluster primary coordinator. DOMAIN may be a UUID or name.
+
+    WARNING: Removing an incremental parent will invalidate any existing incremental backups based on that backup.
+    """
+
+    echo(
+        CLI_CONFIG,
+        f"Removing backup {backup_datestring} of VM '{domain}'... ",
+        newline=False,
+    )
+    retcode, retmsg = pvc.lib.vm.vm_remove_backup(
+        CLI_CONFIG, domain, target_path, backup_datestring
+    )
+    if retcode:
+        echo(CLI_CONFIG, "done.")
+    else:
+        echo(CLI_CONFIG, "failed.")
+    finish(retcode, retmsg)
+
+
 ###############################################################################
 # > pvc vm tag
 ###############################################################################
@@ -5757,8 +5802,10 @@ cli_vm.add_command(cli_vm_move)
 cli_vm.add_command(cli_vm_migrate)
 cli_vm.add_command(cli_vm_unmigrate)
 cli_vm.add_command(cli_vm_flush_locks)
+cli_vm_backup.add_command(cli_vm_backup_create)
+cli_vm_backup.add_command(cli_vm_backup_restore)
+cli_vm_backup.add_command(cli_vm_backup_remove)
 cli_vm.add_command(cli_vm_backup)
-cli_vm.add_command(cli_vm_restore)
 cli_vm_tag.add_command(cli_vm_tag_get)
 cli_vm_tag.add_command(cli_vm_tag_add)
 cli_vm_tag.add_command(cli_vm_tag_remove)

client-cli/pvc/lib/vm.py

@@ -454,6 +454,26 @@ def vm_backup(config, vm, target_path, incremental_parent=None, retain_snapshot=
         return True, response.json().get("message", "")
 
 
+def vm_remove_backup(config, vm, target_path, backup_datestring):
+    """
+    Remove a backup of {vm}, including snapshots, from a local primary coordinator filesystem path
+
+    API endpoint: DELETE /vm/{vm}/backup
+    API arguments: target_path={target_path}, backup_datestring={backup_datestring}
+    API schema: {"message":"{data}"}
+    """
+    params = {
+        "target_path": target_path,
+        "backup_datestring": backup_datestring,
+    }
+    response = call_api(config, "delete", "/vm/{vm}/backup".format(vm=vm), params=params)
+
+    if response.status_code != 200:
+        return False, response.json().get("message", "")
+    else:
+        return True, response.json().get("message", "")
+
+
 def vm_restore(config, vm, target_path, backup_datestring, retain_snapshot=False):
     """
     Restore a backup of {vm} and its volumes from a local primary coordinator filesystem path

daemon-common/vm.py

@@ -25,13 +25,15 @@ import os.path
 import lxml.objectify
 import lxml.etree
 
-from distutils.util import strtobool
-from uuid import UUID
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
-from socket import gethostname
+from distutils.util import strtobool
 from json import dump as jdump
 from json import load as jload
+from os import remove
+from shutil import rmtree
+from socket import gethostname
+from uuid import UUID
 
 import daemon_lib.common as common
 
@@ -1322,7 +1324,7 @@ def backup_vm(
     if not dom_uuid:
         return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
 
-    # Validate that the target path exists
+    # Validate that the target path is valid
     if not re.match(r"^/", target_path):
         return (
             False,
@@ -1486,7 +1488,7 @@ def backup_vm(
     retlines = list()
 
     if is_snapshot_remove_failed:
-        retlines.append(f"WARNING: Failed to remove snapshot as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}")
+        retlines.append(f"WARNING: Failed to remove snapshot(s) as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}")
 
     myhostname = gethostname().split(".")[0]
     if retain_snapshot:
@@ -1497,8 +1499,87 @@ def backup_vm(
     return True, '\n'.join(retlines)
 
 
-def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False):
+def remove_backup(zkhandler, domain, source_path, datestring):
+    tstart = time.time()
 
+    # 0. Validation
+    # Validate that VM exists in cluster
+    dom_uuid = getDomainUUID(zkhandler, domain)
+    if not dom_uuid:
+        return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
+
+    # Validate that the source path is valid
+    if not re.match(r"^/", source_path):
+        return (
+            False,
+            f"ERROR: Source path {source_path} is not a valid absolute path on the primary coordinator!",
+        )
+
+    # Ensure that source_path (on this node) exists
+    if not os.path.isdir(source_path):
+        return False, f"ERROR: Source path {source_path} does not exist!"
+
+    # Ensure that domain path (on this node) exists
+    backup_source_path = f"{source_path}/{domain}"
+    if not os.path.isdir(backup_source_path):
+        return False, f"ERROR: Source VM path {backup_source_path} does not exist!"
+
+    # Ensure that the archives are present
+    backup_source_pvcbackup_file = f"{backup_source_path}/{domain}.{datestring}.pvcbackup"
+    if not os.path.isfile(backup_source_pvcbackup_file):
+        return False, "ERROR: The specified source backup files do not exist!"
+
+    backup_source_pvcdisks_path = f"{backup_source_path}/{domain}.{datestring}.pvcdisks"
+    if not os.path.isdir(backup_source_pvcdisks_path):
+        return False, "ERROR: The specified source backup files do not exist!"
+
+    # 1. Read the backup file and get VM details
+    try:
+        with open(backup_source_pvcbackup_file) as fh:
+            backup_source_details = jload(fh)
+    except Exception as e:
+        return False, f"ERROR: Failed to read source backup details: {e}"
+
+    # 2. Remove snapshots
+    is_snapshot_remove_failed = False
+    which_snapshot_remove_failed = list()
+    msg_snapshot_remove_failed = list()
+    for volume_file, _ in backup_source_details.get('backup_files'):
+        pool, volume, _ = volume_file.split('/')[-1].split('.')
+        snapshot = f"backup_{datestring}"
+        retcode, retmsg = ceph.remove_snapshot(zkhandler, pool, volume, snapshot)
+        if not retcode:
+            is_snapshot_remove_failed = True
+            which_snapshot_remove_failed.append(f"{pool}/{volume}")
+            msg_snapshot_remove_failed.append(retmsg)
+
+    # 3. Remove files
+    is_files_remove_failed = False
+    msg_files_remove_failed = None
+    try:
+        remove(backup_source_pvcbackup_file)
+        rmtree(backup_source_pvcdisks_path)
+    except Exception as e:
+        is_files_remove_failed = True
+        msg_files_remove_failed = e
+
+    tend = time.time()
+    ttot = round(tend - tstart, 2)
+    retlines = list()
+
+    if is_snapshot_remove_failed:
+        retlines.append(f"WARNING: Failed to remove snapshot(s) as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}")
+
+    if is_files_remove_failed:
+        retlines.append(f"WARNING: Failed to remove backup file(s) from {source_path}: {msg_files_remove_failed}")
+
+    myhostname = gethostname().split(".")[0]
+    retlines.append(f"Removed VM backup {datestring} for '{domain}' from '{myhostname}:{source_path}' in {ttot}s.")
+
+    return True, '\n'.join(retlines)
+
+
+def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False):
     tstart = time.time()
 
     # 0. Validations
@@ -1510,7 +1591,7 @@ def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False
             f'ERROR: VM "{domain}" already exists in the cluster! Remove or rename it before restoring a backup.',
         )
 
-    # Validate that the target path exists
+    # Validate that the source path is valid
     if not re.match(r"^/", source_path):
         return (
             False,
@@ -1626,12 +1707,16 @@ def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False
                     f"rbd snap rm {pool}/{volume}@backup_{incremental_parent}"
                 )
                 if retcode:
-                    return False, f"ERROR: Failed to remove imported image snapshot for {parent_volume_file}: {stderr}"
+                    is_snapshot_remove_failed = True
+                    which_snapshot_remove_failed.append(f"{pool}/{volume}")
+                    msg_snapshot_remove_failed.append(retmsg)
             retcode, stdout, stderr = common.run_os_command(
                 f"rbd snap rm {pool}/{volume}@backup_{datestring}"
             )
             if retcode:
-                return False, f"ERROR: Failed to remove imported image snapshot for {volume_file}: {stderr}"
+                is_snapshot_remove_failed = True
+                which_snapshot_remove_failed.append(f"{pool}/{volume}")
+                msg_snapshot_remove_failed.append(retmsg)
 
     else:
         for volume_file, volume_size in backup_source_details.get('backup_files'):
@@ -1681,7 +1766,7 @@ def restore_vm(zkhandler, domain, source_path, datestring, retain_snapshot=False
     retlines = list()
 
     if is_snapshot_remove_failed:
-        retlines.append(f"WARNING: Failed to remove parent snapshot as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}")
+        retlines.append(f"WARNING: Failed to remove hanging snapshot(s) as requested for volume(s) {', '.join(which_snapshot_remove_failed)}: {', '.join(msg_snapshot_remove_failed)}")
 
     myhostname = gethostname().split(".")[0]
     retlines.append(f"Successfully restored VM backup {datestring} for '{domain}' from '{myhostname}:{source_path}' in {ttot}s.")