From e4198559113ef7917f4aaede0ed7a83b341c1814 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Sun, 9 Feb 2020 19:43:07 -0500 Subject: [PATCH] Support converting types during upload Allow the user to specify other, non-raw files and upload them, performing a conversion with qemu-img convert and a temporary block device as a shim (since qemu-img can't use FIFOs). Also ensures that the target volume exists before proceeding. Addresses #68 --- api-daemon/pvcapid/flaskapi.py | 21 ++++- api-daemon/pvcapid/helper.py | 163 ++++++++++++++++++++++++++------- daemon-common/common.py | 3 +- docs/manuals/swagger.json | 17 +++- 4 files changed, 166 insertions(+), 38 deletions(-) diff --git a/api-daemon/pvcapid/flaskapi.py b/api-daemon/pvcapid/flaskapi.py index cc1820ab..53829075 100755 --- a/api-daemon/pvcapid/flaskapi.py +++ b/api-daemon/pvcapid/flaskapi.py @@ -3336,8 +3336,11 @@ api.add_resource(API_Storage_Ceph_Volume_Element_Clone, '/storage/ceph/volume/

//upload class API_Storage_Ceph_Volume_Element_Upload(Resource): + @RequestParser([ + { 'name': 'image_format' } + ]) @Authenticator - def post(self, pool, volume): + def post(self, pool, volume, reqargs): """ Upload a disk image to Ceph volume {volume} in pool {pool} --- @@ -3345,6 +3348,18 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource): - storage / ceph parameters: - in: query + name: image_format + type: string + required: true + description: The type of source image file + enum: + - raw + - vmdk + - qcow2 + - qed + - vdi + - vpc + - in: formdata name: file type: binary required: true @@ -3374,10 +3389,12 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource): image_data = data.get('file', None) if not image_data: return {'message':'An image file contents must be specified'}, 400 + return api_helper.ceph_volume_upload( pool, volume, - image_data + image_data, + reqargs.get('image_format', None) ) api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume///upload') diff --git a/api-daemon/pvcapid/helper.py b/api-daemon/pvcapid/helper.py index 05107793..d39a43e8 100755 --- a/api-daemon/pvcapid/helper.py +++ b/api-daemon/pvcapid/helper.py @@ -1327,48 +1327,143 @@ def ceph_volume_remove(pool, name): } return output, retcode -def ceph_volume_upload(pool, volume, data): +def ceph_volume_upload(pool, volume, data, img_type): """ Upload a raw file via HTTP post to a PVC Ceph volume """ - # Map the target blockdev - zk_conn = pvc_common.startZKConnection(config['coordinators']) - retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume) - pvc_common.stopZKConnection(zk_conn) - if not retflag: + # Determine the image conversion options + if img_type not in ['raw', 'vmdk', 'qcow2', 'qed', 'vdi', 'vpc']: output = { - 'message': retdata.replace('\"', '\'') - } - retcode = 400 - return output, retcode - blockdev = retdata - - output = { - 'message': "Wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool) - } - retcode = 200 - - # Save the data to the blockdev - try: - data.save(blockdev) - except: - output = { - 'message': "ERROR: Failed to write image file to volume." - } - retcode = 400 - - # Unmap the target blockdev - zk_conn = pvc_common.startZKConnection(config['coordinators']) - retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume) - pvc_common.stopZKConnection(zk_conn) - if not retflag: - output = { - 'message': retdata.replace('\"', '\'') + "message": "ERROR: Image type '{}' is not valid.".format(img_type) } retcode = 400 return output, retcode - return output, retcode + # Get the size of the target block device + zk_conn = pvc_common.startZKConnection(config['coordinators']) + retcode, retdata = pvc_ceph.get_list_volume(zk_conn, pool, volume, is_fuzzy=False) + pvc_common.stopZKConnection(zk_conn) + # If there's no target, return failure + if not retcode or len(retdata) < 1: + output = { + "message": "ERROR: Target volume '{}' does not exist in pool '{}'.".format(volume, pool) + } + retcode = 400 + return output, retcode + dev_size = retdata[0]['stats']['size'] + + def cleanup_maps_and_volumes(): + zk_conn = pvc_common.startZKConnection(config['coordinators']) + # Unmap the target blockdev + retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume) + # Unmap the temporary blockdev + retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, "{}_tmp".format(volume)) + # Remove the temporary blockdev + retflag, retdata = pvc_ceph.remove_volume(zk_conn, pool, "{}_tmp".format(volume)) + pvc_common.stopZKConnection(zk_conn) + + # Create a temporary block device to store non-raw images + if img_type == 'raw': + # Map the target blockdev + zk_conn = pvc_common.startZKConnection(config['coordinators']) + retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume) + pvc_common.stopZKConnection(zk_conn) + if not retflag: + output = { + 'message': retdata.replace('\"', '\'') + } + retcode = 400 + cleanup_maps_and_volumes() + return output, retcode + dest_blockdev = retdata + + # Save the data to the blockdev directly + try: + data.save(dest_blockdev) + except: + output = { + 'message': "ERROR: Failed to write image file to volume." + } + retcode = 400 + cleanup_maps_and_volumes() + return output, retcode + + output = { + 'message': "Wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool) + } + retcode = 200 + cleanup_maps_and_volumes() + return output, retcode + + # Write the image directly to the blockdev + else: + # Create a temporary blockdev + zk_conn = pvc_common.startZKConnection(config['coordinators']) + retflag, retdata = pvc_ceph.add_volume(zk_conn, pool, "{}_tmp".format(volume), dev_size) + pvc_common.stopZKConnection(zk_conn) + if not retflag: + output = { + 'message': retdata.replace('\"', '\'') + } + retcode = 400 + cleanup_maps_and_volumes() + return output, retcode + + # Map the temporary target blockdev + zk_conn = pvc_common.startZKConnection(config['coordinators']) + retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, "{}_tmp".format(volume)) + pvc_common.stopZKConnection(zk_conn) + if not retflag: + output = { + 'message': retdata.replace('\"', '\'') + } + retcode = 400 + cleanup_maps_and_volumes() + return output, retcode + temp_blockdev = retdata + + # Map the target blockdev + zk_conn = pvc_common.startZKConnection(config['coordinators']) + retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume) + pvc_common.stopZKConnection(zk_conn) + if not retflag: + output = { + 'message': retdata.replace('\"', '\'') + } + retcode = 400 + cleanup_maps_and_volumes() + return output, retcode + dest_blockdev = retdata + + # Save the data to the temporary blockdev directly + try: + data.save(temp_blockdev) + except: + output = { + 'message': "ERROR: Failed to write image file to temporary volume." + } + retcode = 400 + cleanup_maps_and_volumes() + return output, retcode + + # Convert from the temporary to destination format on the blockdevs + retcode, stdout, stderr = pvc_common.run_os_command( + 'qemu-img convert -C -f {} -O raw {} {}'.format(img_type, temp_blockdev, dest_blockdev) + ) + if retcode: + output = { + 'message': "ERROR: Failed to convert image format from '{}' to 'raw': {}".format(img_type, stderr) + } + retcode = 400 + cleanup_maps_and_volumes() + return output, retcode + + output = { + 'message': "Converted and wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool) + } + retcode = 200 + cleanup_maps_and_volumes() + return output, retcode def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True): """ diff --git a/daemon-common/common.py b/daemon-common/common.py index c665961f..cbdb19cf 100644 --- a/daemon-common/common.py +++ b/daemon-common/common.py @@ -38,11 +38,12 @@ import daemon_lib.zkhandler as zkhandler # # Run a local OS command via shell # -def run_os_command(command_string, background=False, environment=None, timeout=None): +def run_os_command(command_string, background=False, environment=None, timeout=None, shell=False): command = shlex.split(command_string) try: command_output = subprocess.run( command, + shell=shell, env=environment, timeout=timeout, stdout=subprocess.PIPE, diff --git a/docs/manuals/swagger.json b/docs/manuals/swagger.json index d79709d0..475782a6 100644 --- a/docs/manuals/swagger.json +++ b/docs/manuals/swagger.json @@ -4696,8 +4696,23 @@ "description": "", "parameters": [ { - "description": "The raw binary contents of the file", + "description": "The type of source image file", + "enum": [ + "raw", + "vmdk", + "qcow2", + "qed", + "vdi", + "vpc" + ], "in": "query", + "name": "image_format", + "required": true, + "type": "string" + }, + { + "description": "The raw binary contents of the file", + "in": "formdata", "name": "file", "required": true, "type": "binary"