From 65df807b091b231b5dc87ad9c5cd434586224d64 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Thu, 23 Sep 2021 23:31:58 -0400 Subject: [PATCH] Add support for configurable OSD DB ratios The default of 0.05 (5%) is likely ideal in the initial implementation, but allow this to be set explicitly for maximum flexibility in space-constrained or performance-critical use-cases. --- api-daemon/pvcapid/flaskapi.py | 7 +++++++ api-daemon/pvcapid/helper.py | 4 ++-- client-cli/pvc/cli_lib/ceph.py | 7 ++++--- client-cli/pvc/pvc.py | 11 +++++++++-- daemon-common/ceph.py | 4 ++-- docs/manuals/swagger.json | 7 +++++++ node-daemon/pvcnoded/objects/CephInstance.py | 17 +++++++++-------- 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/api-daemon/pvcapid/flaskapi.py b/api-daemon/pvcapid/flaskapi.py index 56b91733..e7ec69e9 100755 --- a/api-daemon/pvcapid/flaskapi.py +++ b/api-daemon/pvcapid/flaskapi.py @@ -3751,6 +3751,7 @@ class API_Storage_Ceph_OSD_Root(Resource): {'name': 'device', 'required': True, 'helptext': "A valid device must be specified."}, {'name': 'weight', 'required': True, 'helptext': "An OSD weight must be specified."}, {'name': 'ext_db', 'required': False, 'helptext': "Whether to use an external OSD DB LV device."}, + {'name': 'ext_db_ratio', 'required': False, 'helptext': "Decimal size ratio of the external OSD DB LV device."}, ]) @Authenticator def post(self, reqargs): @@ -3781,6 +3782,11 @@ class API_Storage_Ceph_OSD_Root(Resource): type: boolean required: false description: Whether to use an external OSD DB LV device + - in: query + name: ext_db_ratio + type: float + required: false + description: Decimal ratio of total OSD size for the external OSD DB LV device, default 0.05 (5%) responses: 200: description: OK @@ -3798,6 +3804,7 @@ class API_Storage_Ceph_OSD_Root(Resource): reqargs.get('device', None), reqargs.get('weight', None), reqargs.get('ext_db', False), + float(reqargs.get('ext_db_ratio', 0.05)), ) diff --git a/api-daemon/pvcapid/helper.py b/api-daemon/pvcapid/helper.py index 43cc4f46..3a38d6c1 100755 --- a/api-daemon/pvcapid/helper.py +++ b/api-daemon/pvcapid/helper.py @@ -1292,11 +1292,11 @@ def ceph_osd_db_vg_add(zkhandler, node, device): @ZKConnection(config) -def ceph_osd_add(zkhandler, node, device, weight, ext_db_flag=False): +def ceph_osd_add(zkhandler, node, device, weight, ext_db_flag=False, ext_db_ratio=0.05): """ Add a Ceph OSD to the PVC Ceph storage cluster. """ - retflag, retdata = pvc_ceph.add_osd(zkhandler, node, device, weight, ext_db_flag) + retflag, retdata = pvc_ceph.add_osd(zkhandler, node, device, weight, ext_db_flag, ext_db_ratio) if retflag: retcode = 200 diff --git a/client-cli/pvc/cli_lib/ceph.py b/client-cli/pvc/cli_lib/ceph.py index 0d4a63ef..f5b3273d 100644 --- a/client-cli/pvc/cli_lib/ceph.py +++ b/client-cli/pvc/cli_lib/ceph.py @@ -222,19 +222,20 @@ def ceph_osd_list(config, limit): return False, response.json().get('message', '') -def ceph_osd_add(config, node, device, weight, ext_db_flag): +def ceph_osd_add(config, node, device, weight, ext_db_flag, ext_db_ratio): """ Add new Ceph OSD API endpoint: POST /api/v1/storage/ceph/osd - API arguments: node={node}, device={device}, weight={weight}, ext_db={ext_db_flag} + API arguments: node={node}, device={device}, weight={weight}, ext_db={ext_db_flag}, ext_db_ratio={ext_db_ratio} API schema: {"message":"{data}"} """ params = { 'node': node, 'device': device, 'weight': weight, - 'ext_db': ext_db_flag + 'ext_db': ext_db_flag, + 'ext_db_ratio': ext_db_ratio } response = call_api(config, 'post', '/storage/ceph/osd', params=params) diff --git a/client-cli/pvc/pvc.py b/client-cli/pvc/pvc.py index 5389b97e..2b55751c 100755 --- a/client-cli/pvc/pvc.py +++ b/client-cli/pvc/pvc.py @@ -2635,17 +2635,24 @@ def ceph_osd_create_db_vg(node, device, confirm_flag): is_flag=True, default=False, help='Use an external database logical volume for this OSD.' ) +@click.option( + '-r', '--ext-db-ratio', 'ext_db_ratio', + default=0.05, show_default=True, type=float, + help='Decimal ratio of the external database logical volume to the OSD size.' +) @click.option( '-y', '--yes', 'confirm_flag', is_flag=True, default=False, help='Confirm the creation' ) @cluster_req -def ceph_osd_add(node, device, weight, ext_db_flag, confirm_flag): +def ceph_osd_add(node, device, weight, ext_db_flag, ext_db_ratio, confirm_flag): """ Add a new Ceph OSD on node NODE with block device DEVICE. If '--ext-db' is specified, the existing OSD database volume group on NODE will be used; it must exist first or OSD creation will fail. See the 'pvc storage osd create-db-vg' command for more details. + + The default '--ext-db-ratio' of 0.05 (5%) is sufficient for most RBD workloads and OSD sizes, though this can be adjusted based on the sizes of the OSD(s) and the underlying database device. Ceph documentation recommends at least 0.02 (2%) for RBD use-cases, and higher values may improve WAL performance under write-heavy workloads with fewer OSDs per node. """ if not confirm_flag and not config['unsafe']: try: @@ -2653,7 +2660,7 @@ def ceph_osd_add(node, device, weight, ext_db_flag, confirm_flag): except Exception: exit(0) - retcode, retmsg = pvc_ceph.ceph_osd_add(config, node, device, weight, ext_db_flag) + retcode, retmsg = pvc_ceph.ceph_osd_add(config, node, device, weight, ext_db_flag, ext_db_ratio) cleanup(retcode, retmsg) diff --git a/daemon-common/ceph.py b/daemon-common/ceph.py index b9a788dd..79c9d3df 100644 --- a/daemon-common/ceph.py +++ b/daemon-common/ceph.py @@ -236,7 +236,7 @@ def add_osd_db_vg(zkhandler, node, device): # OSD addition and removal uses the /cmd/ceph pipe # These actions must occur on the specific node they reference -def add_osd(zkhandler, node, device, weight, ext_db_flag=False): +def add_osd(zkhandler, node, device, weight, ext_db_flag=False, ext_db_ratio=0.05): # Verify the target node exists if not common.verifyNode(zkhandler, node): return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node) @@ -247,7 +247,7 @@ def add_osd(zkhandler, node, device, weight, ext_db_flag=False): return False, 'ERROR: Block device "{}" on node "{}" is used by OSD "{}"'.format(device, node, block_osd) # Tell the cluster to create a new OSD for the host - add_osd_string = 'osd_add {},{},{},{}'.format(node, device, weight, ext_db_flag) + add_osd_string = 'osd_add {},{},{},{},{}'.format(node, device, weight, ext_db_flag, ext_db_ratio) zkhandler.write([ ('base.cmd.ceph', add_osd_string) ]) diff --git a/docs/manuals/swagger.json b/docs/manuals/swagger.json index 3aebb0b5..1275b129 100644 --- a/docs/manuals/swagger.json +++ b/docs/manuals/swagger.json @@ -5026,6 +5026,13 @@ "name": "ext_db", "required": false, "type": "boolean" + }, + { + "description": "Decimal ratio of total OSD size for the external OSD DB LV device, default 0.05 (5%)", + "in": "query", + "name": "ext_db_ratio", + "required": false, + "type": "float" } ], "responses": { diff --git a/node-daemon/pvcnoded/objects/CephInstance.py b/node-daemon/pvcnoded/objects/CephInstance.py index cf825617..5831e4ff 100644 --- a/node-daemon/pvcnoded/objects/CephInstance.py +++ b/node-daemon/pvcnoded/objects/CephInstance.py @@ -68,7 +68,7 @@ class CephOSDInstance(object): self.stats = json.loads(data) @staticmethod - def add_osd(zkhandler, logger, node, device, weight, ext_db_flag=False): + def add_osd(zkhandler, logger, node, device, weight, ext_db_flag=False, ext_db_ratio=0.05): # We are ready to create a new OSD on this node logger.out('Creating new OSD disk on block device {}'.format(device), state='i') try: @@ -104,7 +104,7 @@ class CephOSDInstance(object): if ext_db_flag: _, osd_size_bytes, _ = common.run_os_command('blockdev --getsize64 {}'.format(device)) osd_size_bytes = int(osd_size_bytes) - result = CephOSDInstance.create_osd_db_lv(zkhandler, logger, osd_id, osd_size_bytes) + result = CephOSDInstance.create_osd_db_lv(zkhandler, logger, osd_id, ext_db_ratio, osd_size_bytes) if not result: raise db_device = "osd-db/osd-{}".format(osd_id) @@ -359,7 +359,7 @@ class CephOSDInstance(object): return False @staticmethod - def create_osd_db_lv(zkhandler, logger, osd_id, osd_size_bytes): + def create_osd_db_lv(zkhandler, logger, osd_id, ext_db_ratio, osd_size_bytes): logger.out('Creating new OSD database logical volume for OSD ID {}'.format(osd_id), state='i') try: # 0. Check if an existsing logical volume exists @@ -370,11 +370,11 @@ class CephOSDInstance(object): logger.out('Ceph OSD database LV "osd-db/osd{}" already exists'.format(osd_id), state='e') return False - # 1. Determine LV sizing (5% of OSD size, in MB) - osd_db_size = int(osd_size_bytes * 0.05 / 1024 / 1024) + # 1. Determine LV sizing + osd_db_size = int(osd_size_bytes * ext_db_ratio / 1024 / 1024) # 2. Create the LV - logger.out('Creating LV "osd-db/osd-{}"'.format(osd_id), state='i') + logger.out('Creating DB LV "osd-db/osd-{}" of {}M ({} * {})'.format(osd_id, osd_db_size, osd_size_bytes, ext_db_ratio), state='i') retcode, stdout, stderr = common.run_os_command( 'lvcreate --yes --name osd-{} --size {} osd-db'.format(osd_id, osd_db_size) ) @@ -489,14 +489,15 @@ def ceph_command(zkhandler, logger, this_node, data, d_osd): # Adding a new OSD if command == 'osd_add': - node, device, weight, ext_db_flag = args.split(',') + node, device, weight, ext_db_flag, ext_db_ratio = args.split(',') ext_db_flag = bool(strtobool(ext_db_flag)) + ext_db_ratio = float(ext_db_ratio) if node == this_node.name: # Lock the command queue zk_lock = zkhandler.writelock('base.cmd.ceph') with zk_lock: # Add the OSD - result = CephOSDInstance.add_osd(zkhandler, logger, node, device, weight, ext_db_flag) + result = CephOSDInstance.add_osd(zkhandler, logger, node, device, weight, ext_db_flag, ext_db_ratio) # Command succeeded if result: # Update the command queue