Add mass ack/delete of faults

api-daemon/pvcapid/flaskapi.py

@@ -701,6 +701,44 @@ class API_Faults(Resource):
         """
         return api_helper.fault_list(sort_key=reqargs.get("sort_key", "last_reported"))
 
+    @Authenticator
+    def put(self):
+        """
+        Acknowledge all cluster faults
+        ---
+        tags:
+          - faults
+        responses:
+          200:
+            description: OK
+            schema:
+              type: object
+              properties:
+                message:
+                  type: string
+                  description: A text message
+        """
+        return api_helper.fault_acknowledge_all()
+
+    @Authenticator
+    def delete(self):
+        """
+        Delete all cluster faults
+        ---
+        tags:
+          - faults
+        responses:
+          200:
+            description: OK
+            schema:
+              type: object
+              properties:
+                message:
+                  type: string
+                  description: A text message
+        """
+        return api_helper.fault_delete_all()
+
 
 api.add_resource(API_Faults, "/faults")
 

api-daemon/pvcapid/helper.py

@@ -148,7 +148,25 @@ def fault_acknowledge(zkhandler, fault_id):
     """
     Acknowledge a fault of FAULT_ID.
     """
-    retflag, retdata = pvc_faults.acknowledge(zkhandler, fault_id)
+    retflag, retdata = pvc_faults.acknowledge(zkhandler, fault_id=fault_id)
+
+    if retflag:
+        retcode = 200
+    else:
+        retcode = 404
+
+    retdata = {"message": retdata}
+
+    return retdata, retcode
+
+
+@pvc_common.Profiler(config)
+@ZKConnection(config)
+def fault_acknowledge_all(zkhandler):
+    """
+    Acknowledge all faults.
+    """
+    retflag, retdata = pvc_faults.acknowledge(zkhandler)
 
     if retflag:
         retcode = 200
@@ -166,7 +184,25 @@ def fault_delete(zkhandler, fault_id):
     """
     Delete a fault of FAULT_ID.
     """
-    retflag, retdata = pvc_faults.delete(zkhandler, fault_id)
+    retflag, retdata = pvc_faults.delete(zkhandler, fault_id=fault_id)
+
+    if retflag:
+        retcode = 200
+    else:
+        retcode = 404
+
+    retdata = {"message": retdata}
+
+    return retdata, retcode
+
+
+@pvc_common.Profiler(config)
+@ZKConnection(config)
+def fault_delete_all(zkhandler):
+    """
+    Delete all faults.
+    """
+    retflag, retdata = pvc_faults.delete(zkhandler)
 
     if retflag:
         retcode = 200

client-cli/pvc/cli/cli.py

@@ -549,6 +549,24 @@ def cli_cluster_fault_acknowledge(fault_id):
     finish(retcode, retdata)
 
 
+###############################################################################
+# > pvc cluster fault ack-all
+###############################################################################
+@click.command(
+    name="ack-all",
+    short_help="Acknowledge all cluster faults.",
+)
+@confirm_opt("Acknowledge all current cluster faults")
+@connection_req
+def cli_cluster_fault_acknowledge_all():
+    """
+    Acknowledge all cluster faults.
+    """
+
+    retcode, retdata = pvc.lib.faults.acknowledge_all(CLI_CONFIG)
+    finish(retcode, retdata)
+
+
 ###############################################################################
 # > pvc cluster fault delete
 ###############################################################################
@@ -567,6 +585,24 @@ def cli_cluster_fault_delete(fault_id):
     finish(retcode, retdata)
 
 
+###############################################################################
+# > pvc cluster fault delete-all
+###############################################################################
+@click.command(
+    name="delete-all",
+    short_help="Delete all cluster faults.",
+)
+@confirm_opt("Delete all current cluster faults")
+@connection_req
+def cli_cluster_fault_delete_all():
+    """
+    Delete all cluster faults.
+    """
+
+    retcode, retdata = pvc.lib.faults.delete_all(CLI_CONFIG)
+    finish(retcode, retdata)
+
+
 ###############################################################################
 # > pvc cluster maintenance
 ###############################################################################
@@ -6258,7 +6294,9 @@ cli_cluster.add_command(cli_cluster_restore)
 cli_cluster.add_command(cli_cluster_status)
 cli_cluster_fault.add_command(cli_cluster_fault_list)
 cli_cluster_fault.add_command(cli_cluster_fault_acknowledge)
+cli_cluster_fault.add_command(cli_cluster_fault_acknowledge_all)
 cli_cluster_fault.add_command(cli_cluster_fault_delete)
+cli_cluster_fault.add_command(cli_cluster_fault_delete_all)
 cli_cluster.add_command(cli_cluster_fault)
 cli_cluster_maintenance.add_command(cli_cluster_maintenance_on)
 cli_cluster_maintenance.add_command(cli_cluster_maintenance_off)

client-cli/pvc/lib/faults.py

@@ -55,7 +55,22 @@ def acknowledge(config, fault_id):
     """
     response = call_api(config, "put", f"/faults/{fault_id}")
 
-    print(response.json())
+    if response.status_code == 200:
+        return True, response.json().get("message", "")
+    else:
+        return False, response.json().get("message", "")
+
+
+def acknowledge_all(config):
+    """
+    Acknowledge all PVC faults
+
+    API endpoint: PUT /api/v1/faults
+    API arguments:
+    API schema: {json_message}
+    """
+    response = call_api(config, "put", "/faults")
+
     if response.status_code == 200:
         return True, response.json().get("message", "")
     else:
@@ -76,3 +91,19 @@ def delete(config, fault_id):
         return True, response.json().get("message", "")
     else:
         return False, response.json().get("message", "")
+
+
+def delete_all(config):
+    """
+    Delete all PVC faults
+
+    API endpoint: DELETE /api/v1/faults
+    API arguments:
+    API schema: {json_message}
+    """
+    response = call_api(config, "delete", "/faults")
+
+    if response.status_code == 200:
+        return True, response.json().get("message", "")
+    else:
+        return False, response.json().get("message", "")

daemon-common/faults.py

@@ -152,34 +152,54 @@ def get_list(zkhandler, limit=None, sort_key="last_reported"):
     return True, all_faults
 
 
-def acknowledge(zkhandler, fault_id):
+def acknowledge(zkhandler, fault_id=None):
     """
-    Acknowledge a fault
+    Acknowledge a fault or all faults
     """
-    fault = getFault(zkhandler, fault_id)
+    if fault_id is None:
+        faults = getAllFaults(zkhandler)
+    else:
+        fault = getFault(zkhandler, fault_id)
 
-    if fault is None:
+        if fault is None:
-        return False, f"No fault with ID {fault_id} found"
+            return False, f"No fault with ID {fault_id} found"
 
-    zkhandler.write(
+        faults = [fault]
-        [
+
-            (("faults.ack_time", fault_id), str(datetime.now()).split(".")[0]),
+    for fault in faults:
-            (("faults.status", fault_id), "ack"),
+        # Don't reacknowledge already-acknowledged faults
-        ]
+        if fault["status"] != "ack":
+            zkhandler.write(
+                [
+                    (("faults.ack_time", fault_id), str(datetime.now()).split(".")[0]),
+                    (("faults.status", fault_id), "ack"),
+                ]
+            )
+
+    return (
+        True,
+        f"Successfully acknowledged fault(s) {', '.join([fault['id'] for fault in faults])}",
     )
 
-    return True, f"Successfully acknowledged fault {fault_id}"
 
-
+def delete(zkhandler, fault_id=None):
-def delete(zkhandler, fault_id):
     """
-    Delete a fault
+    Delete a fault or all faults
     """
-    fault = getFault(zkhandler, fault_id)
+    if fault_id is None:
+        faults = getAllFaults(zkhandler)
+    else:
+        fault = getFault(zkhandler, fault_id)
 
-    if fault is None:
+        if fault is None:
-        return False, f"No fault with ID {fault_id} found"
+            return False, f"No fault with ID {fault_id} found"
 
-    zkhandler.delete(("faults.id", fault_id), recursive=True)
+        faults = [fault]
 
-    return True, f"Successfully deleted fault {fault_id}"
+    for fault in faults:
+        zkhandler.delete(("faults.id", fault_id), recursive=True)
+
+    return (
+        True,
+        f"Successfully deleted fault(s) {', '.join([fault['id'] for fault in faults])}",
+    )