From 35e27f79ef9faa24c8b42d6446370410df8ec399 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Fri, 29 Sep 2023 16:13:08 -0400 Subject: [PATCH] Fix uploading of non-raw image files Adds a new API query parameter to define the file size, which is then used for the temporary image. This is required for, at least VMDK, files to work properly in qemu-img convert. --- api-daemon/pvcapid/flaskapi.py | 17 +++++++++++++++-- api-daemon/pvcapid/helper.py | 28 +++++++++++++++++++++++++--- client-cli/pvc/cli/cli.py | 6 +++--- client-cli/pvc/lib/storage.py | 8 +++++++- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/api-daemon/pvcapid/flaskapi.py b/api-daemon/pvcapid/flaskapi.py index 718c46cc..ecb97b16 100755 --- a/api-daemon/pvcapid/flaskapi.py +++ b/api-daemon/pvcapid/flaskapi.py @@ -5088,7 +5088,12 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource): "required": True, "location": ["args"], "helptext": "A source image format must be specified.", - } + }, + { + "name": "file_size", + "required": False, + "location": ["args"], + }, ] ) @Authenticator @@ -5113,6 +5118,11 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource): - qed - vdi - vpc + - in: query + name: file_size + type: integer + required: false + description: The size of the image file, if {image_format} is not "raw" responses: 200: description: OK @@ -5131,7 +5141,10 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource): id: Message """ return api_helper.ceph_volume_upload( - pool, volume, reqargs.get("image_format", None) + pool, + volume, + reqargs.get("image_format", None), + reqargs.get("file_size", None), ) diff --git a/api-daemon/pvcapid/helper.py b/api-daemon/pvcapid/helper.py index b50e1cbf..6afebba1 100755 --- a/api-daemon/pvcapid/helper.py +++ b/api-daemon/pvcapid/helper.py @@ -1584,7 +1584,7 @@ def ceph_volume_remove(zkhandler, pool, name): @ZKConnection(config) -def ceph_volume_upload(zkhandler, pool, volume, img_type): +def ceph_volume_upload(zkhandler, pool, volume, img_type, file_size=None): """ Upload a raw file via HTTP post to a PVC Ceph volume """ @@ -1605,7 +1605,17 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type): } retcode = 400 return output, retcode - dev_size = retdata[0]["stats"]["size"] + + try: + dev_size = retdata[0]["stats"]["size"] + except Exception: + output = { + "message": "Target volume '{}' does not exist in pool '{}'.".format( + volume, pool + ) + } + retcode = 400 + return output, retcode def cleanup_maps_and_volumes(): # Unmap the target blockdev @@ -1621,6 +1631,13 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type): # Create a temporary block device to store non-raw images if img_type == "raw": + if file_size != dev_size: + output = { + "message": f"Image file size {file_size} does not match volume size {dev_size}" + } + retcode = 400 + return output, retcode + # Map the target blockdev retflag, retdata = pvc_ceph.map_volume(zkhandler, pool, volume) if not retflag: @@ -1661,9 +1678,14 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type): # Write the image directly to the blockdev else: + if file_size is None: + output = {"message": "A file size must be specified"} + retcode = 400 + return output, retcode + # Create a temporary blockdev retflag, retdata = pvc_ceph.add_volume( - zkhandler, pool, "{}_tmp".format(volume), dev_size + zkhandler, pool, "{}_tmp".format(volume), file_size ) if not retflag: output = {"message": retdata.replace('"', "'")} diff --git a/client-cli/pvc/cli/cli.py b/client-cli/pvc/cli/cli.py index 94641935..ada319c6 100644 --- a/client-cli/pvc/cli/cli.py +++ b/client-cli/pvc/cli/cli.py @@ -3598,7 +3598,7 @@ def cli_storage_volume_upload(pool, name, image_format, image_file): If the image format is "raw", the image is uploaded directly to the target volume without modification. Otherwise, it will be converted into raw format by "qemu-img convert" on the remote side before writing using a temporary volume. The image format must be a valid format recognized by "qemu-img", such as "vmdk" or "qcow2". """ - if not os.path.exists(image_file): + if not path.exists(image_file): echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(image_file)) exit(1) @@ -4910,13 +4910,13 @@ def cli_provisioner_ova_upload(name, filename, pool): Storage templates, provisioning scripts, and arguments for OVA-type profiles will be ignored and should not be set. """ - if not os.path.exists(filename): + if not path.exists(filename): echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(filename)) exit(1) params = dict() params["pool"] = pool - params["ova_size"] = os.path.getsize(filename) + params["ova_size"] = path.getsize(filename) retcode, retdata = pvc.lib.provisioner.ova_upload( CLI_CONFIG, name, filename, params diff --git a/client-cli/pvc/lib/storage.py b/client-cli/pvc/lib/storage.py index 476d8173..4b83747c 100644 --- a/client-cli/pvc/lib/storage.py +++ b/client-cli/pvc/lib/storage.py @@ -21,6 +21,7 @@ import math +from os import path from json import loads from requests_toolbelt.multipart.encoder import ( MultipartEncoder, @@ -1209,6 +1210,11 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file): """ import click + if image_format != "raw": + file_size = path.getsize(image_file) + else: + file_size = None + bar = UploadProgressBar( image_file, end_message="Parsing file on remote side...", end_nl=False ) @@ -1220,7 +1226,7 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file): upload_monitor = MultipartEncoderMonitor(upload_data, bar.update) headers = {"Content-Type": upload_monitor.content_type} - params = {"image_format": image_format} + params = {"image_format": image_format, "file_size": file_size} response = call_api( config,