Make zkhandler accept failures more robustly

Most of these would silently fail if there was e.g. an issue with the ZK
connection. Instead, encase things in try blocks and handle the
exceptions in a more graceful way, returning None or False if
applicable. Except for locks, which should retry 5 times before
aborting.

node-daemon/pvcnoded/zkhandler.py

@@ -24,22 +24,34 @@ import uuid
 
 # Child list function
 def listchildren(zk_conn, key):
+    try:
         children = zk_conn.get_children(key)
         return children
+    except Exception:
+        return None
 
 # Key deletion function
 def deletekey(zk_conn, key, recursive=True):
+    try:
         zk_conn.delete(key, recursive=recursive)
+        return True
+    except Exception:
+        return False
 
 # Data read function
 def readdata(zk_conn, key):
+    try:
         data_raw = zk_conn.get(key)
         data = data_raw[0].decode('utf8')
         meta = data_raw[1]
         return data
+    except Exception:
+        return None
 
 # Data write function
 def writedata(zk_conn, kv):
+    # Commit the transaction
+    try:
         # Start up a transaction
         zk_transaction = zk_conn.transaction()
 
@@ -71,8 +83,6 @@ def writedata(zk_conn, kv):
                     print('Zookeeper key "{}" does not match expected version'.format(key))
                     return False
 
-    # Commit the transaction
-    try:
         zk_transaction.commit()
         return True
     except Exception:
@@ -84,6 +94,7 @@ def renamekey(zk_conn, kv):
     # support either the recursive delete or recursive create operations that
     # we need. Why? No explanation in the docs that I can find.
 
+    try:
         # Proceed one KV pair at a time
         for key in sorted(kv):
             old_name = key
@@ -118,20 +129,54 @@ def renamekey(zk_conn, kv):
             # Remove recursively the old key
             zk_conn.delete(old_name, recursive=True)
 
+        return True
+    except Exception:
+        return False
+
 # Write lock function
 def writelock(zk_conn, key):
+    count = 1
+    while True:
+        try:
             lock_id = str(uuid.uuid1())
             lock = zk_conn.WriteLock('{}'.format(key), lock_id)
+            break
+        except Exception:
+            count += 1
+            if count > 5:
+                break
+            else:
+                continue
     return lock
 
 # Read lock function
 def readlock(zk_conn, key):
+    count = 1
+    while True:
+        try:
             lock_id = str(uuid.uuid1())
             lock = zk_conn.ReadLock('{}'.format(key), lock_id)
+            break
+        except Exception:
+            count += 1
+            if count > 5:
+                break
+            else:
+                continue
     return lock
 
 # Exclusive lock function
 def exclusivelock(zk_conn, key):
+    count = 1
+    while True:
+        try:
             lock_id = str(uuid.uuid1())
             lock = zk_conn.Lock('{}'.format(key), lock_id)
+            break
+        except Exception:
+            count += 1
+            if count > 5:
+                break
+            else:
+                continue
     return lock