Add separate OSD DB device support
Adds in three parts: 1. Create an API endpoint to create OSD DB volume groups on a device. Passed through to the node via the same command pipeline as creating/removing OSDs, and creates a volume group with a fixed name (osd-db). 2. Adds API support for specifying whether or not to use this DB volume group when creating a new OSD via the "ext_db" flag. Naming and sizing is fixed for simplicity and based on Ceph recommendations (5% of OSD size). The Zookeeper schema tracks the block device to use during removal. 3. Adds CLI support for the new and modified API endpoints, as well as displaying the block device and DB block device in the OSD list. While I debated supporting adding a DB device to an existing OSD, in practice this ended up being a very complex operation involving stopping the OSD and setting some options, so this is not supported; this can be specified during OSD creation only. Closes #142
This commit is contained in:
parent
df277edf1c
commit
adc8a5a3bc
|
|
@ -3599,6 +3599,52 @@ class API_Storage_Ceph_Option(Resource):
|
|||
api.add_resource(API_Storage_Ceph_Option, '/storage/ceph/option')
|
||||
|
||||
|
||||
# /storage/ceph/osddb
|
||||
class API_Storage_Ceph_OSDDB_Root(Resource):
|
||||
@RequestParser([
|
||||
{'name': 'node', 'required': True, 'helptext': "A valid node must be specified."},
|
||||
{'name': 'device', 'required': True, 'helptext': "A valid device must be specified."},
|
||||
])
|
||||
@Authenticator
|
||||
def post(self, reqargs):
|
||||
"""
|
||||
Add a Ceph OSD database volume group to the cluster
|
||||
Note: This task may take up to 30s to complete and return
|
||||
---
|
||||
tags:
|
||||
- storage / ceph
|
||||
parameters:
|
||||
- in: query
|
||||
name: node
|
||||
type: string
|
||||
required: true
|
||||
description: The PVC node to create the OSD DB volume group on
|
||||
- in: query
|
||||
name: device
|
||||
type: string
|
||||
required: true
|
||||
description: The block device (e.g. "/dev/sdb", "/dev/disk/by-path/...", etc.) to create the OSD DB volume group on
|
||||
responses:
|
||||
200:
|
||||
description: OK
|
||||
schema:
|
||||
type: object
|
||||
id: Message
|
||||
400:
|
||||
description: Bad request
|
||||
schema:
|
||||
type: object
|
||||
id: Message
|
||||
"""
|
||||
return api_helper.ceph_osd_db_vg_add(
|
||||
reqargs.get('node', None),
|
||||
reqargs.get('device', None)
|
||||
)
|
||||
|
||||
|
||||
api.add_resource(API_Storage_Ceph_OSDDB_Root, '/storage/ceph/osddb')
|
||||
|
||||
|
||||
# /storage/ceph/osd
|
||||
class API_Storage_Ceph_OSD_Root(Resource):
|
||||
@RequestParser([
|
||||
|
|
@ -3619,6 +3665,12 @@ class API_Storage_Ceph_OSD_Root(Resource):
|
|||
id:
|
||||
type: string (containing integer)
|
||||
description: The Ceph ID of the OSD
|
||||
device:
|
||||
type: string
|
||||
description: The OSD data block device
|
||||
db_device:
|
||||
type: string
|
||||
description: The OSD database/WAL block device (logical volume); empty if not applicable
|
||||
stats:
|
||||
type: object
|
||||
properties:
|
||||
|
|
@ -3698,6 +3750,7 @@ class API_Storage_Ceph_OSD_Root(Resource):
|
|||
{'name': 'node', 'required': True, 'helptext': "A valid node must be specified."},
|
||||
{'name': 'device', 'required': True, 'helptext': "A valid device must be specified."},
|
||||
{'name': 'weight', 'required': True, 'helptext': "An OSD weight must be specified."},
|
||||
{'name': 'ext_db', 'required': False, 'helptext': "Whether to use an external OSD DB LV device."},
|
||||
])
|
||||
@Authenticator
|
||||
def post(self, reqargs):
|
||||
|
|
@ -3723,6 +3776,11 @@ class API_Storage_Ceph_OSD_Root(Resource):
|
|||
type: number
|
||||
required: true
|
||||
description: The Ceph CRUSH weight for the OSD
|
||||
- in: query
|
||||
name: ext_db
|
||||
type: boolean
|
||||
required: false
|
||||
description: Whether to use an external OSD DB LV device
|
||||
responses:
|
||||
200:
|
||||
description: OK
|
||||
|
|
@ -3738,7 +3796,8 @@ class API_Storage_Ceph_OSD_Root(Resource):
|
|||
return api_helper.ceph_osd_add(
|
||||
reqargs.get('node', None),
|
||||
reqargs.get('device', None),
|
||||
reqargs.get('weight', None)
|
||||
reqargs.get('weight', None),
|
||||
reqargs.get('ext_db', False),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1274,11 +1274,29 @@ def ceph_osd_state(zkhandler, osd):
|
|||
|
||||
|
||||
@ZKConnection(config)
|
||||
def ceph_osd_add(zkhandler, node, device, weight):
|
||||
def ceph_osd_db_vg_add(zkhandler, node, device):
|
||||
"""
|
||||
Add a Ceph OSD database VG to the PVC Ceph storage cluster.
|
||||
"""
|
||||
retflag, retdata = pvc_ceph.add_osd_db_vg(zkhandler, node, device)
|
||||
|
||||
if retflag:
|
||||
retcode = 200
|
||||
else:
|
||||
retcode = 400
|
||||
|
||||
output = {
|
||||
'message': retdata.replace('\"', '\'')
|
||||
}
|
||||
return output, retcode
|
||||
|
||||
|
||||
@ZKConnection(config)
|
||||
def ceph_osd_add(zkhandler, node, device, weight, ext_db_flag=False):
|
||||
"""
|
||||
Add a Ceph OSD to the PVC Ceph storage cluster.
|
||||
"""
|
||||
retflag, retdata = pvc_ceph.add_osd(zkhandler, node, device, weight)
|
||||
retflag, retdata = pvc_ceph.add_osd(zkhandler, node, device, weight, ext_db_flag)
|
||||
|
||||
if retflag:
|
||||
retcode = 200
|
||||
|
|
|
|||
|
|
@ -149,6 +149,31 @@ def format_raw_output(status_data):
|
|||
return '\n'.join(ainformation)
|
||||
|
||||
|
||||
#
|
||||
# OSD DB VG functions
|
||||
#
|
||||
def ceph_osd_db_vg_add(config, node, device):
|
||||
"""
|
||||
Add new Ceph OSD database volume group
|
||||
|
||||
API endpoint: POST /api/v1/storage/ceph/osddb
|
||||
API arguments: node={node}, device={device}
|
||||
API schema: {"message":"{data}"}
|
||||
"""
|
||||
params = {
|
||||
'node': node,
|
||||
'device': device
|
||||
}
|
||||
response = call_api(config, 'post', '/storage/ceph/osddb', params=params)
|
||||
|
||||
if response.status_code == 200:
|
||||
retstatus = True
|
||||
else:
|
||||
retstatus = False
|
||||
|
||||
return retstatus, response.json().get('message', '')
|
||||
|
||||
|
||||
#
|
||||
# OSD functions
|
||||
#
|
||||
|
|
@ -197,18 +222,19 @@ def ceph_osd_list(config, limit):
|
|||
return False, response.json().get('message', '')
|
||||
|
||||
|
||||
def ceph_osd_add(config, node, device, weight):
|
||||
def ceph_osd_add(config, node, device, weight, ext_db_flag):
|
||||
"""
|
||||
Add new Ceph OSD
|
||||
|
||||
API endpoint: POST /api/v1/storage/ceph/osd
|
||||
API arguments: node={node}, device={device}, weight={weight}
|
||||
API arguments: node={node}, device={device}, weight={weight}, ext_db={ext_db_flag}
|
||||
API schema: {"message":"{data}"}
|
||||
"""
|
||||
params = {
|
||||
'node': node,
|
||||
'device': device,
|
||||
'weight': weight
|
||||
'weight': weight,
|
||||
'ext_db': ext_db_flag
|
||||
}
|
||||
response = call_api(config, 'post', '/storage/ceph/osd', params=params)
|
||||
|
||||
|
|
@ -312,13 +338,15 @@ def format_list_osd(osd_list):
|
|||
osd_list_output = []
|
||||
|
||||
osd_id_length = 3
|
||||
osd_node_length = 5
|
||||
osd_device_length = 6
|
||||
osd_db_device_length = 9
|
||||
osd_up_length = 4
|
||||
osd_in_length = 4
|
||||
osd_size_length = 5
|
||||
osd_weight_length = 3
|
||||
osd_reweight_length = 5
|
||||
osd_pgs_length = 4
|
||||
osd_node_length = 5
|
||||
osd_used_length = 5
|
||||
osd_free_length = 6
|
||||
osd_util_length = 6
|
||||
|
|
@ -358,10 +386,21 @@ def format_list_osd(osd_list):
|
|||
if _osd_id_length > osd_id_length:
|
||||
osd_id_length = _osd_id_length
|
||||
|
||||
# Set the OSD node length
|
||||
_osd_node_length = len(osd_information['stats']['node']) + 1
|
||||
if _osd_node_length > osd_node_length:
|
||||
osd_node_length = _osd_node_length
|
||||
|
||||
# Set the OSD device length
|
||||
_osd_device_length = len(osd_information['device']) + 1
|
||||
if _osd_device_length > osd_device_length:
|
||||
osd_device_length = _osd_device_length
|
||||
|
||||
# Set the OSD db_device length
|
||||
_osd_db_device_length = len(osd_information['db_device']) + 1
|
||||
if _osd_db_device_length > osd_db_device_length:
|
||||
osd_db_device_length = _osd_db_device_length
|
||||
|
||||
# Set the size and length
|
||||
_osd_size_length = len(str(osd_information['stats']['size'])) + 1
|
||||
if _osd_size_length > osd_size_length:
|
||||
|
|
@ -422,12 +461,12 @@ def format_list_osd(osd_list):
|
|||
osd_list_output.append('{bold}{osd_header: <{osd_header_length}} {state_header: <{state_header_length}} {details_header: <{details_header_length}} {read_header: <{read_header_length}} {write_header: <{write_header_length}}{end_bold}'.format(
|
||||
bold=ansiprint.bold(),
|
||||
end_bold=ansiprint.end(),
|
||||
osd_header_length=osd_id_length + osd_node_length + 1,
|
||||
osd_header_length=osd_id_length + osd_node_length + osd_device_length + osd_db_device_length + 3,
|
||||
state_header_length=osd_up_length + osd_in_length + 1,
|
||||
details_header_length=osd_size_length + osd_pgs_length + osd_weight_length + osd_reweight_length + osd_used_length + osd_free_length + osd_util_length + osd_var_length + 7,
|
||||
read_header_length=osd_rdops_length + osd_rddata_length + 1,
|
||||
write_header_length=osd_wrops_length + osd_wrdata_length + 1,
|
||||
osd_header='OSDs ' + ''.join(['-' for _ in range(5, osd_id_length + osd_node_length)]),
|
||||
osd_header='OSDs ' + ''.join(['-' for _ in range(5, osd_id_length + osd_node_length + osd_device_length + osd_db_device_length + 2)]),
|
||||
state_header='State ' + ''.join(['-' for _ in range(6, osd_up_length + osd_in_length)]),
|
||||
details_header='Details ' + ''.join(['-' for _ in range(8, osd_size_length + osd_pgs_length + osd_weight_length + osd_reweight_length + osd_used_length + osd_free_length + osd_util_length + osd_var_length + 6)]),
|
||||
read_header='Read ' + ''.join(['-' for _ in range(5, osd_rdops_length + osd_rddata_length)]),
|
||||
|
|
@ -437,6 +476,8 @@ def format_list_osd(osd_list):
|
|||
osd_list_output.append('{bold}\
|
||||
{osd_id: <{osd_id_length}} \
|
||||
{osd_node: <{osd_node_length}} \
|
||||
{osd_device: <{osd_device_length}} \
|
||||
{osd_db_device: <{osd_db_device_length}} \
|
||||
{osd_up: <{osd_up_length}} \
|
||||
{osd_in: <{osd_in_length}} \
|
||||
{osd_size: <{osd_size_length}} \
|
||||
|
|
@ -456,6 +497,8 @@ def format_list_osd(osd_list):
|
|||
end_bold=ansiprint.end(),
|
||||
osd_id_length=osd_id_length,
|
||||
osd_node_length=osd_node_length,
|
||||
osd_device_length=osd_device_length,
|
||||
osd_db_device_length=osd_db_device_length,
|
||||
osd_up_length=osd_up_length,
|
||||
osd_in_length=osd_in_length,
|
||||
osd_size_length=osd_size_length,
|
||||
|
|
@ -472,6 +515,8 @@ def format_list_osd(osd_list):
|
|||
osd_rddata_length=osd_rddata_length,
|
||||
osd_id='ID',
|
||||
osd_node='Node',
|
||||
osd_device='Block',
|
||||
osd_db_device='DB Block',
|
||||
osd_up='Up',
|
||||
osd_in='In',
|
||||
osd_size='Size',
|
||||
|
|
@ -500,10 +545,16 @@ def format_list_osd(osd_list):
|
|||
osd_util = round(osd_information['stats']['utilization'], 2)
|
||||
osd_var = round(osd_information['stats']['var'], 2)
|
||||
|
||||
osd_db_device = osd_information['db_device']
|
||||
if not osd_db_device:
|
||||
osd_db_device = 'N/A'
|
||||
|
||||
# Format the output header
|
||||
osd_list_output.append('{bold}\
|
||||
{osd_id: <{osd_id_length}} \
|
||||
{osd_node: <{osd_node_length}} \
|
||||
{osd_device: <{osd_device_length}} \
|
||||
{osd_db_device: <{osd_db_device_length}} \
|
||||
{osd_up_colour}{osd_up: <{osd_up_length}}{end_colour} \
|
||||
{osd_in_colour}{osd_in: <{osd_in_length}}{end_colour} \
|
||||
{osd_size: <{osd_size_length}} \
|
||||
|
|
@ -524,6 +575,8 @@ def format_list_osd(osd_list):
|
|||
end_colour=ansiprint.end(),
|
||||
osd_id_length=osd_id_length,
|
||||
osd_node_length=osd_node_length,
|
||||
osd_device_length=osd_device_length,
|
||||
osd_db_device_length=osd_db_device_length,
|
||||
osd_up_length=osd_up_length,
|
||||
osd_in_length=osd_in_length,
|
||||
osd_size_length=osd_size_length,
|
||||
|
|
@ -540,6 +593,8 @@ def format_list_osd(osd_list):
|
|||
osd_rddata_length=osd_rddata_length,
|
||||
osd_id=osd_information['id'],
|
||||
osd_node=osd_information['stats']['node'],
|
||||
osd_device=osd_information['device'],
|
||||
osd_db_device=osd_db_device,
|
||||
osd_up_colour=osd_up_colour,
|
||||
osd_up=osd_up_flag,
|
||||
osd_in_colour=osd_in_colour,
|
||||
|
|
|
|||
|
|
@ -2583,6 +2583,38 @@ def ceph_osd():
|
|||
pass
|
||||
|
||||
|
||||
###############################################################################
|
||||
# pvc storage osd create-db-vg
|
||||
###############################################################################
|
||||
@click.command(name='create-db-vg', short_help='Create new OSD database volume group.')
|
||||
@click.argument(
|
||||
'node'
|
||||
)
|
||||
@click.argument(
|
||||
'device'
|
||||
)
|
||||
@click.option(
|
||||
'-y', '--yes', 'confirm_flag',
|
||||
is_flag=True, default=False,
|
||||
help='Confirm the creation'
|
||||
)
|
||||
@cluster_req
|
||||
def ceph_osd_create_db_vg(node, device, confirm_flag):
|
||||
"""
|
||||
Create a new Ceph OSD database volume group on node NODE with block device DEVICE.
|
||||
|
||||
This volume group will be used for Ceph OSD database functionality if the '--ext-db' flag is passed to newly-created OSDs during 'pvc storage osd add'. DEVICE should be an extremely fast SSD device (NVMe, Intel Optane, etc.) which is significantly faster than the normal OSD disks and with very high write endurance. Only one OSD database volume group on a single physical device is supported per node, so it must be fast and large enough to act as an effective OSD database device for all OSDs on the node; the database volume for each OSD is fixed to 5% of the OSD's size. Attempting to add additional database volume groups after the first will fail.
|
||||
"""
|
||||
if not confirm_flag and not config['unsafe']:
|
||||
try:
|
||||
click.confirm('Destroy all data and create a new OSD database volume group on {}:{}'.format(node, device), prompt_suffix='? ', abort=True)
|
||||
except Exception:
|
||||
exit(0)
|
||||
|
||||
retcode, retmsg = pvc_ceph.ceph_osd_db_vg_add(config, node, device)
|
||||
cleanup(retcode, retmsg)
|
||||
|
||||
|
||||
###############################################################################
|
||||
# pvc storage osd add
|
||||
###############################################################################
|
||||
|
|
@ -2598,15 +2630,22 @@ def ceph_osd():
|
|||
default=1.0, show_default=True,
|
||||
help='Weight of the OSD within the CRUSH map.'
|
||||
)
|
||||
@click.option(
|
||||
'-d', '--ext-db', 'ext_db_flag',
|
||||
is_flag=True, default=False,
|
||||
help='Use an external database logical volume for this OSD.'
|
||||
)
|
||||
@click.option(
|
||||
'-y', '--yes', 'confirm_flag',
|
||||
is_flag=True, default=False,
|
||||
help='Confirm the removal'
|
||||
help='Confirm the creation'
|
||||
)
|
||||
@cluster_req
|
||||
def ceph_osd_add(node, device, weight, confirm_flag):
|
||||
def ceph_osd_add(node, device, weight, ext_db_flag, confirm_flag):
|
||||
"""
|
||||
Add a new Ceph OSD on node NODE with block device DEVICE.
|
||||
|
||||
If '--ext-db' is specified, the existing OSD database volume group on NODE will be used; it must exist first or OSD creation will fail. See the 'pvc storage osd create-db-vg' command for more details.
|
||||
"""
|
||||
if not confirm_flag and not config['unsafe']:
|
||||
try:
|
||||
|
|
@ -2614,7 +2653,7 @@ def ceph_osd_add(node, device, weight, confirm_flag):
|
|||
except Exception:
|
||||
exit(0)
|
||||
|
||||
retcode, retmsg = pvc_ceph.ceph_osd_add(config, node, device, weight)
|
||||
retcode, retmsg = pvc_ceph.ceph_osd_add(config, node, device, weight, ext_db_flag)
|
||||
cleanup(retcode, retmsg)
|
||||
|
||||
|
||||
|
|
@ -4856,6 +4895,7 @@ ceph_benchmark.add_command(ceph_benchmark_run)
|
|||
ceph_benchmark.add_command(ceph_benchmark_info)
|
||||
ceph_benchmark.add_command(ceph_benchmark_list)
|
||||
|
||||
ceph_osd.add_command(ceph_osd_create_db_vg)
|
||||
ceph_osd.add_command(ceph_osd_add)
|
||||
ceph_osd.add_command(ceph_osd_remove)
|
||||
ceph_osd.add_command(ceph_osd_in)
|
||||
|
|
|
|||
|
|
@ -180,20 +180,63 @@ def getClusterOSDList(zkhandler):
|
|||
|
||||
|
||||
def getOSDInformation(zkhandler, osd_id):
|
||||
# Get the devices
|
||||
osd_device = zkhandler.read(('osd.device', osd_id))
|
||||
osd_db_device = zkhandler.read(('osd.db_device', osd_id))
|
||||
# Parse the stats data
|
||||
osd_stats_raw = zkhandler.read(('osd.stats', osd_id))
|
||||
osd_stats = dict(json.loads(osd_stats_raw))
|
||||
|
||||
osd_information = {
|
||||
'id': osd_id,
|
||||
'stats': osd_stats
|
||||
'device': osd_device,
|
||||
'db_device': osd_db_device,
|
||||
'stats': osd_stats,
|
||||
}
|
||||
return osd_information
|
||||
|
||||
|
||||
# OSD DB VG actions use the /cmd/ceph pipe
|
||||
# These actions must occur on the specific node they reference
|
||||
def add_osd_db_vg(zkhandler, node, device):
|
||||
# Verify the target node exists
|
||||
if not common.verifyNode(zkhandler, node):
|
||||
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
|
||||
|
||||
# Tell the cluster to create a new OSD for the host
|
||||
add_osd_db_vg_string = 'db_vg_add {},{}'.format(node, device)
|
||||
zkhandler.write([
|
||||
('base.cmd.ceph', add_osd_db_vg_string)
|
||||
])
|
||||
# Wait 1/2 second for the cluster to get the message and start working
|
||||
time.sleep(0.5)
|
||||
# Acquire a read lock, so we get the return exclusively
|
||||
with zkhandler.readlock('base.cmd.ceph'):
|
||||
try:
|
||||
result = zkhandler.read('base.cmd.ceph').split()[0]
|
||||
if result == 'success-db_vg_add':
|
||||
message = 'Created new OSD database VG at "{}" on node "{}".'.format(device, node)
|
||||
success = True
|
||||
else:
|
||||
message = 'ERROR: Failed to create new OSD database VG; check node logs for details.'
|
||||
success = False
|
||||
except Exception:
|
||||
message = 'ERROR: Command ignored by node.'
|
||||
success = False
|
||||
|
||||
# Acquire a write lock to ensure things go smoothly
|
||||
with zkhandler.writelock('base.cmd.ceph'):
|
||||
time.sleep(0.5)
|
||||
zkhandler.write([
|
||||
('base.cmd.ceph', '')
|
||||
])
|
||||
|
||||
return success, message
|
||||
|
||||
|
||||
# OSD addition and removal uses the /cmd/ceph pipe
|
||||
# These actions must occur on the specific node they reference
|
||||
def add_osd(zkhandler, node, device, weight):
|
||||
def add_osd(zkhandler, node, device, weight, ext_db_flag=False):
|
||||
# Verify the target node exists
|
||||
if not common.verifyNode(zkhandler, node):
|
||||
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
|
||||
|
|
@ -204,7 +247,7 @@ def add_osd(zkhandler, node, device, weight):
|
|||
return False, 'ERROR: Block device "{}" on node "{}" is used by OSD "{}"'.format(device, node, block_osd)
|
||||
|
||||
# Tell the cluster to create a new OSD for the host
|
||||
add_osd_string = 'osd_add {},{},{}'.format(node, device, weight)
|
||||
add_osd_string = 'osd_add {},{},{},{}'.format(node, device, weight, ext_db_flag)
|
||||
zkhandler.write([
|
||||
('base.cmd.ceph', add_osd_string)
|
||||
])
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
{"version": "5", "root": "", "base": {"root": "", "schema": "/schema", "schema.version": "/schema/version", "config": "/config", "config.maintenance": "/config/maintenance", "config.primary_node": "/config/primary_node", "config.primary_node.sync_lock": "/config/primary_node/sync_lock", "config.upstream_ip": "/config/upstream_ip", "config.migration_target_selector": "/config/migration_target_selector", "cmd": "/cmd", "cmd.node": "/cmd/nodes", "cmd.domain": "/cmd/domains", "cmd.ceph": "/cmd/ceph", "logs": "/logs", "node": "/nodes", "domain": "/domains", "network": "/networks", "storage": "/ceph", "storage.util": "/ceph/util", "osd": "/ceph/osds", "pool": "/ceph/pools", "volume": "/ceph/volumes", "snapshot": "/ceph/snapshots"}, "logs": {"node": "", "messages": "/messages"}, "node": {"name": "", "keepalive": "/keepalive", "mode": "/daemonmode", "data.active_schema": "/activeschema", "data.latest_schema": "/latestschema", "data.static": "/staticdata", "data.pvc_version": "/pvcversion", "running_domains": "/runningdomains", "count.provisioned_domains": "/domainscount", "count.networks": "/networkscount", "state.daemon": "/daemonstate", "state.router": "/routerstate", "state.domain": "/domainstate", "cpu.load": "/cpuload", "vcpu.allocated": "/vcpualloc", "memory.total": "/memtotal", "memory.used": "/memused", "memory.free": "/memfree", "memory.allocated": "/memalloc", "memory.provisioned": "/memprov", "ipmi.hostname": "/ipmihostname", "ipmi.username": "/ipmiusername", "ipmi.password": "/ipmipassword", "sriov": "/sriov", "sriov.pf": "/sriov/pf", "sriov.vf": "/sriov/vf"}, "sriov_pf": {"phy": "", "mtu": "/mtu", "vfcount": "/vfcount"}, "sriov_vf": {"phy": "", "pf": "/pf", "mtu": "/mtu", "mac": "/mac", "phy_mac": "/phy_mac", "config": "/config", "config.vlan_id": "/config/vlan_id", "config.vlan_qos": "/config/vlan_qos", "config.tx_rate_min": "/config/tx_rate_min", "config.tx_rate_max": "/config/tx_rate_max", "config.spoof_check": "/config/spoof_check", "config.link_state": "/config/link_state", "config.trust": "/config/trust", "config.query_rss": "/config/query_rss", "pci": "/pci", "pci.domain": "/pci/domain", "pci.bus": "/pci/bus", "pci.slot": "/pci/slot", "pci.function": "/pci/function", "used": "/used", "used_by": "/used_by"}, "domain": {"name": "", "xml": "/xml", "state": "/state", "profile": "/profile", "stats": "/stats", "node": "/node", "last_node": "/lastnode", "failed_reason": "/failedreason", "storage.volumes": "/rbdlist", "console.log": "/consolelog", "console.vnc": "/vnc", "meta.autostart": "/node_autostart", "meta.migrate_method": "/migration_method", "meta.node_selector": "/node_selector", "meta.node_limit": "/node_limit", "meta.tags": "/tags", "migrate.sync_lock": "/migrate_sync_lock"}, "tag": {"name": "", "type": "/type", "protected": "/protected"}, "network": {"vni": "", "type": "/nettype", "rule": "/firewall_rules", "rule.in": "/firewall_rules/in", "rule.out": "/firewall_rules/out", "nameservers": "/name_servers", "domain": "/domain", "reservation": "/dhcp4_reservations", "lease": "/dhcp4_leases", "ip4.gateway": "/ip4_gateway", "ip4.network": "/ip4_network", "ip4.dhcp": "/dhcp4_flag", "ip4.dhcp_start": "/dhcp4_start", "ip4.dhcp_end": "/dhcp4_end", "ip6.gateway": "/ip6_gateway", "ip6.network": "/ip6_network", "ip6.dhcp": "/dhcp6_flag"}, "reservation": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname"}, "lease": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname", "expiry": "/expiry", "client_id": "/clientid"}, "rule": {"description": "", "rule": "/rule", "order": "/order"}, "osd": {"id": "", "node": "/node", "device": "/device", "db_device": "/db_device", "stats": "/stats"}, "pool": {"name": "", "pgs": "/pgs", "stats": "/stats"}, "volume": {"name": "", "stats": "/stats"}, "snapshot": {"name": "", "stats": "/stats"}}
|
||||
|
|
@ -466,7 +466,7 @@ class ZKHandler(object):
|
|||
#
|
||||
class ZKSchema(object):
|
||||
# Current version
|
||||
_version = 4
|
||||
_version = 5
|
||||
|
||||
# Root for doing nested keys
|
||||
_schema_root = ''
|
||||
|
|
@ -636,6 +636,7 @@ class ZKSchema(object):
|
|||
'id': '', # The root key
|
||||
'node': '/node',
|
||||
'device': '/device',
|
||||
'db_device': '/db_device',
|
||||
'stats': '/stats'
|
||||
},
|
||||
# The schema of an individual pool entry (/ceph/pools/{pool_name})
|
||||
|
|
|
|||
|
|
@ -520,6 +520,14 @@
|
|||
},
|
||||
"osd": {
|
||||
"properties": {
|
||||
"db_device": {
|
||||
"description": "The OSD database/WAL block device (logical volume); empty if not applicable",
|
||||
"type": "string"
|
||||
},
|
||||
"device": {
|
||||
"description": "The OSD data block device",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"description": "The Ceph ID of the OSD",
|
||||
"type": "string (containing integer)"
|
||||
|
|
@ -5011,6 +5019,13 @@
|
|||
"name": "weight",
|
||||
"required": true,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"description": "Whether to use an external OSD DB LV device",
|
||||
"in": "query",
|
||||
"name": "ext_db",
|
||||
"required": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
|
|
@ -5133,6 +5148,45 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/storage/ceph/osddb": {
|
||||
"post": {
|
||||
"description": "Note: This task may take up to 30s to complete and return",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "The PVC node to create the OSD DB volume group on",
|
||||
"in": "query",
|
||||
"name": "node",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "The block device (e.g. \"/dev/sdb\", \"/dev/disk/by-path/...\", etc.) to create the OSD DB volume group on",
|
||||
"in": "query",
|
||||
"name": "device",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Message"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Message"
|
||||
}
|
||||
}
|
||||
},
|
||||
"summary": "Add a Ceph OSD database volume group to the cluster",
|
||||
"tags": [
|
||||
"storage / ceph"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/storage/ceph/pool": {
|
||||
"get": {
|
||||
"description": "",
|
||||
|
|
@ -6248,6 +6302,68 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/vm/{vm}/device": {
|
||||
"delete": {
|
||||
"description": "",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "The raw Libvirt XML definition of the device to detach",
|
||||
"in": "query",
|
||||
"name": "xml",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Message"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Message"
|
||||
}
|
||||
}
|
||||
},
|
||||
"summary": "Hot-detach device XML to {vm}",
|
||||
"tags": [
|
||||
"vm"
|
||||
]
|
||||
},
|
||||
"post": {
|
||||
"description": "",
|
||||
"parameters": [
|
||||
{
|
||||
"description": "The raw Libvirt XML definition of the device to attach",
|
||||
"in": "query",
|
||||
"name": "xml",
|
||||
"required": true,
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Message"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/Message"
|
||||
}
|
||||
}
|
||||
},
|
||||
"summary": "Hot-attach device XML to {vm}",
|
||||
"tags": [
|
||||
"vm"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/api/v1/vm/{vm}/locks": {
|
||||
"post": {
|
||||
"description": "",
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ import psutil
|
|||
|
||||
import daemon_lib.common as common
|
||||
|
||||
from distutils.util import strtobool
|
||||
|
||||
|
||||
class CephOSDInstance(object):
|
||||
def __init__(self, zkhandler, this_node, osd_id):
|
||||
|
|
@ -66,7 +68,7 @@ class CephOSDInstance(object):
|
|||
self.stats = json.loads(data)
|
||||
|
||||
@staticmethod
|
||||
def add_osd(zkhandler, logger, node, device, weight):
|
||||
def add_osd(zkhandler, logger, node, device, weight, ext_db_flag=False):
|
||||
# We are ready to create a new OSD on this node
|
||||
logger.out('Creating new OSD disk on block device {}'.format(device), state='i')
|
||||
try:
|
||||
|
|
@ -96,12 +98,26 @@ class CephOSDInstance(object):
|
|||
print(stderr)
|
||||
raise
|
||||
|
||||
# 3b. Create the OSD for real
|
||||
dev_flags = "--data {}".format(device)
|
||||
|
||||
# 3b. Prepare the logical volume if ext_db_flag
|
||||
if ext_db_flag:
|
||||
_, osd_size_bytes, _ = common.run_os_command('blockdev --getsize64 {}'.format(device))
|
||||
osd_size_bytes = int(osd_size_bytes)
|
||||
result = CephOSDInstance.create_osd_db_lv(zkhandler, logger, osd_id, osd_size_bytes)
|
||||
if not result:
|
||||
raise
|
||||
db_device = "osd-db/osd-{}".format(osd_id)
|
||||
dev_flags += " --block.db {}".format(db_device)
|
||||
else:
|
||||
db_device = ""
|
||||
|
||||
# 3c. Create the OSD for real
|
||||
logger.out('Preparing LVM for new OSD disk with ID {} on {}'.format(osd_id, device), state='i')
|
||||
retcode, stdout, stderr = common.run_os_command(
|
||||
'ceph-volume lvm prepare --bluestore --data {device}'.format(
|
||||
'ceph-volume lvm prepare --bluestore {devices}'.format(
|
||||
osdid=osd_id,
|
||||
device=device
|
||||
devices=dev_flags
|
||||
)
|
||||
)
|
||||
if retcode:
|
||||
|
|
@ -177,6 +193,7 @@ class CephOSDInstance(object):
|
|||
(('osd', osd_id), ''),
|
||||
(('osd.node', osd_id), node),
|
||||
(('osd.device', osd_id), device),
|
||||
(('osd.db_device', osd_id), db_device),
|
||||
(('osd.stats', osd_id), '{}'),
|
||||
])
|
||||
|
||||
|
|
@ -270,7 +287,13 @@ class CephOSDInstance(object):
|
|||
print(stderr)
|
||||
raise
|
||||
|
||||
# 7. Delete OSD from ZK
|
||||
# 7. Remove the DB device
|
||||
if zkhandler.exists(('osd.db_device', osd_id)):
|
||||
db_device = zkhandler.read(('osd.db_device', osd_id))
|
||||
logger.out('Removing OSD DB logical volume "{}"'.format(db_device), state='i')
|
||||
retcode, stdout, stderr = common.run_os_command('lvremove --yes --force {}'.format(db_device))
|
||||
|
||||
# 8. Delete OSD from ZK
|
||||
logger.out('Deleting OSD disk with ID {} from Zookeeper'.format(osd_id), state='i')
|
||||
zkhandler.delete(('osd', osd_id), recursive=True)
|
||||
|
||||
|
|
@ -282,6 +305,93 @@ class CephOSDInstance(object):
|
|||
logger.out('Failed to purge OSD disk with ID {}: {}'.format(osd_id, e), state='e')
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def add_db_vg(zkhandler, logger, device):
|
||||
logger.out('Creating new OSD database volume group on block device {}'.format(device), state='i')
|
||||
try:
|
||||
# 0. Check if an existsing volume group exists
|
||||
retcode, stdout, stderr = common.run_os_command(
|
||||
'vgdisplay osd-db'
|
||||
)
|
||||
if retcode != 5:
|
||||
logger.out('Ceph OSD database VG "osd-db" already exists', state='e')
|
||||
return False
|
||||
|
||||
# 1. Create an empty partition table
|
||||
logger.out('Creating empty partiton table on block device {}'.format(device), state='i')
|
||||
retcode, stdout, stderr = common.run_os_command(
|
||||
'echo -e "o\ny\nn\n\n\n\n8e00\nw\ny\n" | sudo gdisk {}'.format(device)
|
||||
)
|
||||
if retcode:
|
||||
print('gdisk partitioning')
|
||||
print(stdout)
|
||||
print(stderr)
|
||||
raise
|
||||
|
||||
# 2. Create the PV
|
||||
logger.out('Creating PV on block device {}1'.format(device), state='i')
|
||||
retcode, stdout, stderr = common.run_os_command(
|
||||
'pvcreate --force {}1'.format(device)
|
||||
)
|
||||
if retcode:
|
||||
print('pv creation')
|
||||
print(stdout)
|
||||
print(stderr)
|
||||
raise
|
||||
|
||||
# 2. Create the VG (named 'osd-db')
|
||||
logger.out('Creating VG "osd-db" on block device {}1'.format(device), state='i')
|
||||
retcode, stdout, stderr = common.run_os_command(
|
||||
'vgcreate --force osd-db {}1'.format(device)
|
||||
)
|
||||
if retcode:
|
||||
print('vg creation')
|
||||
print(stdout)
|
||||
print(stderr)
|
||||
raise
|
||||
|
||||
# Log it
|
||||
logger.out('Created new OSD database volume group on block device {}'.format(device), state='o')
|
||||
return True
|
||||
except Exception as e:
|
||||
# Log it
|
||||
logger.out('Failed to create OSD database volume group: {}'.format(e), state='e')
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def create_osd_db_lv(zkhandler, logger, osd_id, osd_size_bytes):
|
||||
logger.out('Creating new OSD database logical volume for OSD ID {}'.format(osd_id), state='i')
|
||||
try:
|
||||
# 0. Check if an existsing logical volume exists
|
||||
retcode, stdout, stderr = common.run_os_command(
|
||||
'lvdisplay osd-db/osd{}'.format(osd_id)
|
||||
)
|
||||
if retcode != 5:
|
||||
logger.out('Ceph OSD database LV "osd-db/osd{}" already exists'.format(osd_id), state='e')
|
||||
return False
|
||||
|
||||
# 1. Determine LV sizing (5% of OSD size, in MB)
|
||||
osd_db_size = int(osd_size_bytes * 0.05 / 1024 / 1024)
|
||||
|
||||
# 2. Create the LV
|
||||
logger.out('Creating LV "osd-db/osd-{}"'.format(osd_id), state='i')
|
||||
retcode, stdout, stderr = common.run_os_command(
|
||||
'lvcreate --yes --name osd-{} --size {} osd-db'.format(osd_id, osd_db_size)
|
||||
)
|
||||
if retcode:
|
||||
print('db lv creation')
|
||||
print(stdout)
|
||||
print(stderr)
|
||||
raise
|
||||
|
||||
# Log it
|
||||
logger.out('Created new OSD database logical volume "osd-db/osd-{}"'.format(osd_id), state='o')
|
||||
return True
|
||||
except Exception as e:
|
||||
# Log it
|
||||
logger.out('Failed to create OSD database logical volume: {}'.format(e), state='e')
|
||||
return False
|
||||
|
||||
|
||||
class CephPoolInstance(object):
|
||||
def __init__(self, zkhandler, this_node, name):
|
||||
|
|
@ -379,13 +489,14 @@ def ceph_command(zkhandler, logger, this_node, data, d_osd):
|
|||
|
||||
# Adding a new OSD
|
||||
if command == 'osd_add':
|
||||
node, device, weight = args.split(',')
|
||||
node, device, weight, ext_db_flag = args.split(',')
|
||||
ext_db_flag = bool(strtobool(ext_db_flag))
|
||||
if node == this_node.name:
|
||||
# Lock the command queue
|
||||
zk_lock = zkhandler.writelock('base.cmd.ceph')
|
||||
with zk_lock:
|
||||
# Add the OSD
|
||||
result = CephOSDInstance.add_osd(zkhandler, logger, node, device, weight)
|
||||
result = CephOSDInstance.add_osd(zkhandler, logger, node, device, weight, ext_db_flag)
|
||||
# Command succeeded
|
||||
if result:
|
||||
# Update the command queue
|
||||
|
|
@ -426,3 +537,27 @@ def ceph_command(zkhandler, logger, this_node, data, d_osd):
|
|||
])
|
||||
# Wait 1 seconds before we free the lock, to ensure the client hits the lock
|
||||
time.sleep(1)
|
||||
|
||||
# Adding a new DB VG
|
||||
elif command == 'db_vg_add':
|
||||
node, device = args.split(',')
|
||||
if node == this_node.name:
|
||||
# Lock the command queue
|
||||
zk_lock = zkhandler.writelock('base.cmd.ceph')
|
||||
with zk_lock:
|
||||
# Add the VG
|
||||
result = CephOSDInstance.add_db_vg(zkhandler, logger, device)
|
||||
# Command succeeded
|
||||
if result:
|
||||
# Update the command queue
|
||||
zkhandler.write([
|
||||
('base.cmd.ceph', 'success-{}'.format(data))
|
||||
])
|
||||
# Command failed
|
||||
else:
|
||||
# Update the command queue
|
||||
zkhandler.write([
|
||||
('base.cmd.ceph', 'failure={}'.format(data))
|
||||
])
|
||||
# Wait 1 seconds before we free the lock, to ensure the client hits the lock
|
||||
time.sleep(1)
|
||||
|
|
|
|||
Loading…
Reference in New Issue