Add ceph df output to pool data

Allows additional information visible in the `ceph df` command,
including pool free space and used percentage.

client-common/ceph.py

@@ -131,6 +131,10 @@ def format_ops_fromhuman(datahuman):
     dataops = datasize * ops_unit_matrix[dataunit]
     return '{}'.format(dataops)
 
+def format_pct_tohuman(datapct):
+    datahuman = "{0:.1f}".format(float(datapct * 100.0))
+    return datahuman
+
 #
 # Status functions
 #
@@ -748,7 +752,9 @@ def format_list_pool(pool_list):
 
     pool_name_length = 5
     pool_id_length = 3
-    pool_size_length = 5
+    pool_used_length = 5
+    pool_usedpct_length = 5
+    pool_free_length = 5
     pool_num_objects_length = 6
     pool_num_clones_length = 7
     pool_num_copies_length = 7
@@ -760,14 +766,18 @@ def format_list_pool(pool_list):
 
     for pool_information in pool_list:
         # Deal with the size to human readable
-        for datatype in 'size_bytes', 'write_bytes', 'read_bytes':
+        for datatype in ['free_bytes', 'used_bytes', 'write_bytes', 'read_bytes']:
             databytes = pool_information['stats'][datatype]
             databytes_formatted = format_bytes_tohuman(int(databytes))
             pool_information['stats'][datatype] = databytes_formatted
-        for datatype in 'write_ops', 'read_ops':
+        for datatype in ['write_ops', 'read_ops']:
             dataops = pool_information['stats'][datatype]
             dataops_formatted = format_ops_tohuman(int(dataops))
             pool_information['stats'][datatype] = dataops_formatted
+        for datatype in ['used_percent']:
+            datapct = pool_information['stats'][datatype]
+            datapct_formatted = format_pct_tohuman(float(datapct))
+            pool_information['stats'][datatype] = datapct_formatted
 
         # Set the Pool name length
         _pool_name_length = len(pool_information['name']) + 1
@@ -779,10 +789,20 @@ def format_list_pool(pool_list):
         if _pool_id_length > pool_id_length:
             pool_id_length = _pool_id_length
 
-        # Set the size and length
+        # Set the used and length
-        _pool_size_length = len(str(pool_information['stats']['size_bytes'])) + 1
+        _pool_used_length = len(str(pool_information['stats']['used_bytes'])) + 1
-        if _pool_size_length > pool_size_length:
+        if _pool_used_length > pool_used_length:
-            pool_size_length = _pool_size_length
+            pool_used_length = _pool_used_length
+
+        # Set the usedpct and length
+        _pool_usedpct_length = len(str(pool_information['stats']['used_percent'])) + 1
+        if _pool_usedpct_length > pool_usedpct_length:
+            pool_usedpct_length = _pool_usedpct_length
+
+        # Set the free and length
+        _pool_free_length = len(str(pool_information['stats']['free_bytes'])) + 1
+        if _pool_free_length > pool_free_length:
+            pool_free_length = _pool_free_length
 
         # Set the num_objects and length
         _pool_num_objects_length = len(str(pool_information['stats']['num_objects'])) + 1
@@ -825,7 +845,9 @@ def format_list_pool(pool_list):
     pool_list_output.append('{bold}\
 {pool_id: <{pool_id_length}} \
 {pool_name: <{pool_name_length}} \
-{pool_size: <{pool_size_length}} \
+{pool_used: <{pool_used_length}} \
+{pool_usedpct: <{pool_usedpct_length}} \
+{pool_free: <{pool_free_length}} \
 Obj: {pool_objects: <{pool_objects_length}} \
 {pool_clones: <{pool_clones_length}} \
 {pool_copies: <{pool_copies_length}} \
@@ -839,7 +861,9 @@ Wr: {pool_write_ops: <{pool_write_ops_length}} \
             end_bold=ansiprint.end(),
             pool_id_length=pool_id_length,
             pool_name_length=pool_name_length,
-            pool_size_length=pool_size_length,
+            pool_used_length=pool_used_length,
+            pool_usedpct_length=pool_usedpct_length,
+            pool_free_length=pool_free_length,
             pool_objects_length=pool_num_objects_length,
             pool_clones_length=pool_num_clones_length,
             pool_copies_length=pool_num_copies_length,
@@ -850,7 +874,9 @@ Wr: {pool_write_ops: <{pool_write_ops_length}} \
             pool_read_data_length=pool_read_data_length,
             pool_id='ID',
             pool_name='Name',
-            pool_size='Used',
+            pool_used='Used',
+            pool_usedpct='%',
+            pool_free='Free',
             pool_objects='Count',
             pool_clones='Clones',
             pool_copies='Copies',
@@ -867,7 +893,9 @@ Wr: {pool_write_ops: <{pool_write_ops_length}} \
         pool_list_output.append('{bold}\
 {pool_id: <{pool_id_length}} \
 {pool_name: <{pool_name_length}} \
-{pool_size: <{pool_size_length}} \
+{pool_used: <{pool_used_length}} \
+{pool_usedpct: <{pool_usedpct_length}} \
+{pool_free: <{pool_free_length}} \
      {pool_objects: <{pool_objects_length}} \
 {pool_clones: <{pool_clones_length}} \
 {pool_copies: <{pool_copies_length}} \
@@ -881,7 +909,9 @@ Wr: {pool_write_ops: <{pool_write_ops_length}} \
                 end_bold='',
                 pool_id_length=pool_id_length,
                 pool_name_length=pool_name_length,
-                pool_size_length=pool_size_length,
+                pool_used_length=pool_used_length,
+                pool_usedpct_length=pool_usedpct_length,
+                pool_free_length=pool_free_length,
                 pool_objects_length=pool_num_objects_length,
                 pool_clones_length=pool_num_clones_length,
                 pool_copies_length=pool_num_copies_length,
@@ -892,7 +922,9 @@ Wr: {pool_write_ops: <{pool_write_ops_length}} \
                 pool_read_data_length=pool_read_data_length,
                 pool_id=pool_information['stats']['id'],
                 pool_name=pool_information['name'],
-                pool_size=pool_information['stats']['size_bytes'],
+                pool_used=pool_information['stats']['used_bytes'],
+                pool_usedpct=pool_information['stats']['used_percent'],
+                pool_free=pool_information['stats']['free_bytes'],
                 pool_objects=pool_information['stats']['num_objects'],
                 pool_clones=pool_information['stats']['num_object_clones'],
                 pool_copies=pool_information['stats']['num_object_copies'],

node-daemon/pvcd/Daemon.py

@@ -1009,21 +1009,42 @@ def update_zookeeper():
         if this_node.router_state == 'primary':
             if debug:
                 print("Set pool information in zookeeper (primary only)")
+
             # Get pool info
-            pool_df = dict()
+            retcode, stdout, stderr = common.run_os_command('ceph df --format json', timeout=1)
+            try:
+                ceph_pool_df_raw = json.loads(stdout)['pools']
+            except json.decoder.JSONDecodeError:
+                logger.out('Failed to obtain Pool data (ceph df)', state='w')
+                ceph_pool_df_raw = []
+
             retcode, stdout, stderr = common.run_os_command('rados df --format json', timeout=1)
             try:
-                pool_df_raw = json.loads(stdout)['pools']
+                rados_pool_df_raw = json.loads(stdout)['pools']
             except json.decoder.JSONDecodeError:
-                logger.out('Failed to obtain Pool data', state='w')
+                logger.out('Failed to obtain Pool data (rados df)', state='w')
-                pool_df_raw = []
+                rados_pool_df_raw = []
 
-            for pool in pool_df_raw:
+            pool_count = len(ceph_pool_df_raw)
-                pool_df.update({
+            for pool_idx in range(0, pool_count - 1):
-                    str(pool['name']): {
+                try:
+                    # Combine all the data for this pool
+                    ceph_pool_df = ceph_pool_df_raw[pool_idx]
+                    rados_pool_df = rados_pool_df_raw[pool_idx]
+                    pool = ceph_pool_df
+                    pool.update(rados_pool_df)
+
+                    # Ignore any pools that aren't in our pool list
+                    if pool['name'] not in pool_list:
+                        continue
+
+                    # Assemble a useful data structure
+                    pool_df = {
                         'id': pool['id'],
-                        'size_bytes': pool['size_bytes'],
+                        'free_bytes': pool['stats']['max_avail'],
-                        'num_objects': pool['num_objects'],
+                        'used_bytes': pool['stats']['bytes_used'],
+                        'used_percent': pool['stats']['percent_used'],
+                        'num_objects': pool['stats']['objects'],
                         'num_object_clones': pool['num_object_clones'],
                         'num_object_copies': pool['num_object_copies'],
                         'num_objects_missing_on_primary': pool['num_objects_missing_on_primary'],
@@ -1034,17 +1055,14 @@ def update_zookeeper():
                         'write_ops': pool['write_ops'],
                         'write_bytes': pool['write_bytes']
                     }
-                })
                     
-            # Trigger updates for each pool on this node
+                    # Write the pool data to Zookeeper
-            for pool in pool_list:
-                try:
-                    stats = json.dumps(pool_df[pool])
                     zkhandler.writedata(zk_conn, {
-                        '/ceph/pools/{}/stats'.format(pool): str(stats)
+                        '/ceph/pools/{}/stats'.format(pool['name']): str(json.dumps(pool_df))
                     })
-                except KeyError:
+                except Exception as e:
                     # One or more of the status commands timed out, just continue
+                    logger.out('Failed to format and send pool data', state='w')
                     pass
 
         # Only grab OSD stats if there are OSDs to grab (otherwise `ceph osd df` hangs)