Fix uploading of non-raw image files

Adds a new API query parameter to define the file size, which is then
used for the temporary image. This is required for, at least VMDK, files
to work properly in qemu-img convert.

.version

@@ -1 +1 @@
-0.9.76
+0.9.77

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## PVC Changelog
 
+###### [v0.9.77](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.77)
+
+  * [Client CLI] Fixes a bug from a bad library import
+
 ###### [v0.9.76](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.76)
 
   * [API, Client CLI] Corrects some missing node states for fencing in status output

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from ssl import SSLContext, TLSVersion
 from distutils.util import strtobool as dustrtobool
 
 # Daemon version
-version = "0.9.76"
+version = "0.9.77"
 
 # API version
 API_VERSION = 1.0

api-daemon/pvcapid/flaskapi.py

@@ -5088,7 +5088,12 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
                 "required": True,
                 "location": ["args"],
                 "helptext": "A source image format must be specified.",
-            }
+            },
+            {
+                "name": "file_size",
+                "required": False,
+                "location": ["args"],
+            },
         ]
     )
     @Authenticator
@@ -5113,6 +5118,11 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
               - qed
               - vdi
               - vpc
+          - in: query
+            name: file_size
+            type: integer
+            required: false
+            description: The size of the image file, if {image_format} is not "raw"
         responses:
           200:
             description: OK
@@ -5131,7 +5141,10 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
               id: Message
         """
         return api_helper.ceph_volume_upload(
-            pool, volume, reqargs.get("image_format", None)
+            pool,
+            volume,
+            reqargs.get("image_format", None),
+            reqargs.get("file_size", None),
         )
 
 

api-daemon/pvcapid/helper.py

@@ -1584,7 +1584,7 @@ def ceph_volume_remove(zkhandler, pool, name):
 
 
 @ZKConnection(config)
-def ceph_volume_upload(zkhandler, pool, volume, img_type):
+def ceph_volume_upload(zkhandler, pool, volume, img_type, file_size=None):
     """
     Upload a raw file via HTTP post to a PVC Ceph volume
     """
@@ -1605,7 +1605,17 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
         }
         retcode = 400
         return output, retcode
-    dev_size = retdata[0]["stats"]["size"]
+
+    try:
+        dev_size = retdata[0]["stats"]["size"]
+    except Exception:
+        output = {
+            "message": "Target volume '{}' does not exist in pool '{}'.".format(
+                volume, pool
+            )
+        }
+        retcode = 400
+        return output, retcode
 
     def cleanup_maps_and_volumes():
         # Unmap the target blockdev
@@ -1621,6 +1631,13 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
 
     # Create a temporary block device to store non-raw images
     if img_type == "raw":
+        if file_size != dev_size:
+            output = {
+                "message": f"Image file size {file_size} does not match volume size {dev_size}"
+            }
+            retcode = 400
+            return output, retcode
+
         # Map the target blockdev
         retflag, retdata = pvc_ceph.map_volume(zkhandler, pool, volume)
         if not retflag:
@@ -1661,9 +1678,14 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
 
     # Write the image directly to the blockdev
     else:
+        if file_size is None:
+            output = {"message": "A file size must be specified"}
+            retcode = 400
+            return output, retcode
+
         # Create a temporary blockdev
         retflag, retdata = pvc_ceph.add_volume(
-            zkhandler, pool, "{}_tmp".format(volume), dev_size
+            zkhandler, pool, "{}_tmp".format(volume), file_size
         )
         if not retflag:
             output = {"message": retdata.replace('"', "'")}

client-cli/pvc/cli/cli.py

@@ -1188,10 +1188,8 @@ def cli_vm_modify(
 
     # Grab the current config
     current_vm_cfg_raw = vm_information.get("xml")
-    xml_data = etree.fromstring(current_vm_cfg_raw)
-    current_vm_cfgfile = (
-        etree.tostring(xml_data, pretty_print=True).decode("utf8").strip()
-    )
+    xml_data = fromstring(current_vm_cfg_raw)
+    current_vm_cfgfile = tostring(xml_data, pretty_print=True).decode("utf8").strip()
 
     if editor is True:
         new_vm_cfgfile = click.edit(
@@ -3600,7 +3598,7 @@ def cli_storage_volume_upload(pool, name, image_format, image_file):
     If the image format is "raw", the image is uploaded directly to the target volume without modification. Otherwise, it will be converted into raw format by "qemu-img convert" on the remote side before writing using a temporary volume. The image format must be a valid format recognized by "qemu-img", such as "vmdk" or "qcow2".
     """
 
-    if not os.path.exists(image_file):
+    if not path.exists(image_file):
         echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(image_file))
         exit(1)
 
@@ -4912,13 +4910,13 @@ def cli_provisioner_ova_upload(name, filename, pool):
     Storage templates, provisioning scripts, and arguments for OVA-type profiles will be ignored and should not be set.
     """
 
-    if not os.path.exists(filename):
+    if not path.exists(filename):
         echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(filename))
         exit(1)
 
     params = dict()
     params["pool"] = pool
-    params["ova_size"] = os.path.getsize(filename)
+    params["ova_size"] = path.getsize(filename)
 
     retcode, retdata = pvc.lib.provisioner.ova_upload(
         CLI_CONFIG, name, filename, params

client-cli/pvc/lib/storage.py

@@ -21,6 +21,7 @@
 
 import math
 
+from os import path
 from json import loads
 from requests_toolbelt.multipart.encoder import (
     MultipartEncoder,
@@ -1209,6 +1210,11 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     """
     import click
 
+    if image_format != "raw":
+        file_size = path.getsize(image_file)
+    else:
+        file_size = None
+
     bar = UploadProgressBar(
         image_file, end_message="Parsing file on remote side...", end_nl=False
     )
@@ -1220,7 +1226,7 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     upload_monitor = MultipartEncoderMonitor(upload_data, bar.update)
 
     headers = {"Content-Type": upload_monitor.content_type}
-    params = {"image_format": image_format}
+    params = {"image_format": image_format, "file_size": file_size}
 
     response = call_api(
         config,

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.76",
+    version="0.9.77",
     packages=["pvc.cli", "pvc.lib"],
     install_requires=[
         "Click",

debian/changelog

@@ -1,3 +1,9 @@
+pvc (0.9.77-0) unstable; urgency=high
+
+  * [Client CLI] Fixes a bug from a bad library import
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 19 Sep 2023 11:05:55 -0400
+
 pvc (0.9.76-0) unstable; urgency=high
 
   * [API, Client CLI] Corrects some missing node states for fencing in status output

gen-api-doc

@@ -14,7 +14,7 @@ sys.path.append('api-daemon')
 
 import pvcapid.flaskapi as pvc_api
 
-swagger_file = "docs/manuals/swagger.json"
+swagger_file = "swagger.json"
 swagger_data = swagger(pvc_api.app)
 swagger_data['info']['version'] = "1.0"
 swagger_data['info']['title'] = "PVC Client and Provisioner API"
@@ -22,3 +22,5 @@ swagger_data['host'] = "pvc.local:7370"
 
 with open(swagger_file, 'w') as fd:
     fd.write(json.dumps(swagger_data, sort_keys=True, indent=4))
+
+print(f"Swagger file output to {swagger_file}; add it to the PVC documentation repo.")

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.76"
+version = "0.9.77"
 
 
 ##########################################################