Fix uploading of non-raw image files

Adds a new API query parameter to define the file size, which is then
used for the temporary image. This is required for, at least VMDK, files
to work properly in qemu-img convert.

api-daemon/pvcapid/flaskapi.py

@@ -5088,7 +5088,12 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
                 "required": True,
                 "location": ["args"],
                 "helptext": "A source image format must be specified.",
-            }
+            },
+            {
+                "name": "file_size",
+                "required": False,
+                "location": ["args"],
+            },
         ]
     )
     @Authenticator
@@ -5113,6 +5118,11 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
               - qed
               - vdi
               - vpc
+          - in: query
+            name: file_size
+            type: integer
+            required: false
+            description: The size of the image file, if {image_format} is not "raw"
         responses:
           200:
             description: OK
@@ -5131,7 +5141,10 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
               id: Message
         """
         return api_helper.ceph_volume_upload(
-            pool, volume, reqargs.get("image_format", None)
+            pool,
+            volume,
+            reqargs.get("image_format", None),
+            reqargs.get("file_size", None),
         )
 
 

api-daemon/pvcapid/helper.py

@@ -1584,7 +1584,7 @@ def ceph_volume_remove(zkhandler, pool, name):
 
 
 @ZKConnection(config)
-def ceph_volume_upload(zkhandler, pool, volume, img_type):
+def ceph_volume_upload(zkhandler, pool, volume, img_type, file_size=None):
     """
     Upload a raw file via HTTP post to a PVC Ceph volume
     """
@@ -1605,7 +1605,17 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
         }
         retcode = 400
         return output, retcode
+
+    try:
         dev_size = retdata[0]["stats"]["size"]
+    except Exception:
+        output = {
+            "message": "Target volume '{}' does not exist in pool '{}'.".format(
+                volume, pool
+            )
+        }
+        retcode = 400
+        return output, retcode
 
     def cleanup_maps_and_volumes():
         # Unmap the target blockdev
@@ -1621,6 +1631,13 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
 
     # Create a temporary block device to store non-raw images
     if img_type == "raw":
+        if file_size != dev_size:
+            output = {
+                "message": f"Image file size {file_size} does not match volume size {dev_size}"
+            }
+            retcode = 400
+            return output, retcode
+
         # Map the target blockdev
         retflag, retdata = pvc_ceph.map_volume(zkhandler, pool, volume)
         if not retflag:
@@ -1661,9 +1678,14 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
 
     # Write the image directly to the blockdev
     else:
+        if file_size is None:
+            output = {"message": "A file size must be specified"}
+            retcode = 400
+            return output, retcode
+
         # Create a temporary blockdev
         retflag, retdata = pvc_ceph.add_volume(
-            zkhandler, pool, "{}_tmp".format(volume), dev_size
+            zkhandler, pool, "{}_tmp".format(volume), file_size
         )
         if not retflag:
             output = {"message": retdata.replace('"', "'")}

client-cli/pvc/cli/cli.py

@@ -3598,7 +3598,7 @@ def cli_storage_volume_upload(pool, name, image_format, image_file):
     If the image format is "raw", the image is uploaded directly to the target volume without modification. Otherwise, it will be converted into raw format by "qemu-img convert" on the remote side before writing using a temporary volume. The image format must be a valid format recognized by "qemu-img", such as "vmdk" or "qcow2".
     """
 
-    if not os.path.exists(image_file):
+    if not path.exists(image_file):
         echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(image_file))
         exit(1)
 
@@ -4910,13 +4910,13 @@ def cli_provisioner_ova_upload(name, filename, pool):
     Storage templates, provisioning scripts, and arguments for OVA-type profiles will be ignored and should not be set.
     """
 
-    if not os.path.exists(filename):
+    if not path.exists(filename):
         echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(filename))
         exit(1)
 
     params = dict()
     params["pool"] = pool
-    params["ova_size"] = os.path.getsize(filename)
+    params["ova_size"] = path.getsize(filename)
 
     retcode, retdata = pvc.lib.provisioner.ova_upload(
         CLI_CONFIG, name, filename, params

client-cli/pvc/lib/storage.py

@@ -21,6 +21,7 @@
 
 import math
 
+from os import path
 from json import loads
 from requests_toolbelt.multipart.encoder import (
     MultipartEncoder,
@@ -1209,6 +1210,11 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     """
     import click
 
+    if image_format != "raw":
+        file_size = path.getsize(image_file)
+    else:
+        file_size = None
+
     bar = UploadProgressBar(
         image_file, end_message="Parsing file on remote side...", end_nl=False
     )
@@ -1220,7 +1226,7 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     upload_monitor = MultipartEncoderMonitor(upload_data, bar.update)
 
     headers = {"Content-Type": upload_monitor.content_type}
-    params = {"image_format": image_format}
+    params = {"image_format": image_format, "file_size": file_size}
 
     response = call_api(
         config,

gen-api-doc

@@ -14,7 +14,7 @@ sys.path.append('api-daemon')
 
 import pvcapid.flaskapi as pvc_api
 
-swagger_file = "docs/manuals/swagger.json"
+swagger_file = "swagger.json"
 swagger_data = swagger(pvc_api.app)
 swagger_data['info']['version'] = "1.0"
 swagger_data['info']['title'] = "PVC Client and Provisioner API"
@@ -22,3 +22,5 @@ swagger_data['host'] = "pvc.local:7370"
 
 with open(swagger_file, 'w') as fd:
     fd.write(json.dumps(swagger_data, sort_keys=True, indent=4))
+
+print(f"Swagger file output to {swagger_file}; add it to the PVC documentation repo.")