Add device class tiers to Ceph pools

Allows specifying a particular device class ("tier") for a given pool,
for instance SSD-only or NVMe-only. This is implemented with Crush
rules on the Ceph side, and via an additional new key in the pool
Zookeeper schema which is defaulted to "default".

api-daemon/pvcapid/helper.py

@@ -1403,11 +1403,11 @@ def ceph_pool_list(zkhandler, limit=None, is_fuzzy=True):
 
 
 @ZKConnection(config)
-def ceph_pool_add(zkhandler, name, pgs, replcfg):
+def ceph_pool_add(zkhandler, name, pgs, replcfg, tier=None):
     """
     Add a Ceph RBD pool to the PVC Ceph storage cluster.
     """
-    retflag, retdata = pvc_ceph.add_pool(zkhandler, name, pgs, replcfg)
+    retflag, retdata = pvc_ceph.add_pool(zkhandler, name, pgs, replcfg, tier)
 
     if retflag:
         retcode = 200