Support converting types during upload

Allow the user to specify other, non-raw files and upload them,
performing a conversion with qemu-img convert and a temporary block
device as a shim (since qemu-img can't use FIFOs).

Also ensures that the target volume exists before proceeding.

Addresses #68

api-daemon/pvcapid/flaskapi.py

@@ -3336,8 +3336,11 @@ api.add_resource(API_Storage_Ceph_Volume_Element_Clone, '/storage/ceph/volume/<p
 
 # /storage/ceph/volume/<pool>/<volume>/upload
 class API_Storage_Ceph_Volume_Element_Upload(Resource):
+    @RequestParser([
+        { 'name': 'image_format' }
+    ])
     @Authenticator
-    def post(self, pool, volume):
+    def post(self, pool, volume, reqargs):
         """
         Upload a disk image to Ceph volume {volume} in pool {pool}
         ---
@@ -3345,6 +3348,18 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
           - storage / ceph
         parameters:
           - in: query
+            name: image_format
+            type: string
+            required: true
+            description: The type of source image file
+            enum:
+              - raw
+              - vmdk
+              - qcow2
+              - qed
+              - vdi
+              - vpc
+          - in: formdata
             name: file
             type: binary
             required: true
@@ -3374,10 +3389,12 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
         image_data = data.get('file', None)
         if not image_data:
             return {'message':'An image file contents must be specified'}, 400
+
         return api_helper.ceph_volume_upload(
             pool,
             volume,
-            image_data
+            image_data,
+            reqargs.get('image_format', None)
         )
 api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume/<pool>/<volume>/upload')
 

api-daemon/pvcapid/helper.py

@@ -1327,10 +1327,43 @@ def ceph_volume_remove(pool, name):
     }
     return output, retcode
 
-def ceph_volume_upload(pool, volume, data):
+def ceph_volume_upload(pool, volume, data, img_type):
     """
     Upload a raw file via HTTP post to a PVC Ceph volume
     """
+    # Determine the image conversion options
+    if img_type not in ['raw', 'vmdk', 'qcow2', 'qed', 'vdi', 'vpc']:
+        output = {
+            "message": "ERROR: Image type '{}' is not valid.".format(img_type)
+        }
+        retcode = 400
+        return output, retcode
+
+    # Get the size of the target block device
+    zk_conn = pvc_common.startZKConnection(config['coordinators'])
+    retcode, retdata = pvc_ceph.get_list_volume(zk_conn, pool, volume, is_fuzzy=False)
+    pvc_common.stopZKConnection(zk_conn)
+    # If there's no target, return failure
+    if not retcode or len(retdata) < 1:
+        output = {
+            "message": "ERROR: Target volume '{}' does not exist in pool '{}'.".format(volume, pool)
+        }
+        retcode = 400
+        return output, retcode
+    dev_size = retdata[0]['stats']['size']
+
+    def cleanup_maps_and_volumes():
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        # Unmap the target blockdev
+        retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume)
+        # Unmap the temporary blockdev
+        retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, "{}_tmp".format(volume))
+        # Remove the temporary blockdev
+        retflag, retdata = pvc_ceph.remove_volume(zk_conn, pool, "{}_tmp".format(volume))
+        pvc_common.stopZKConnection(zk_conn)
+
+    # Create a temporary block device to store non-raw images
+    if img_type == 'raw':
         # Map the target blockdev
         zk_conn = pvc_common.startZKConnection(config['coordinators'])
         retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume)
@@ -1340,34 +1373,96 @@ def ceph_volume_upload(pool, volume, data):
                 'message': retdata.replace('\"', '\'')
             }
             retcode = 400
+            cleanup_maps_and_volumes()
             return output, retcode
-    blockdev = retdata
+        dest_blockdev = retdata
 
-    output = {
-        'message': "Wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
-    }
-    retcode = 200
-
-    # Save the data to the blockdev
+        # Save the data to the blockdev directly
         try:
-        data.save(blockdev)
+            data.save(dest_blockdev)
         except:
             output = {
                 'message': "ERROR: Failed to write image file to volume."
             }
             retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
 
-    # Unmap the target blockdev
+        output = {
+            'message': "Wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
+        }
+        retcode = 200
+        cleanup_maps_and_volumes()
+        return output, retcode
+
+    # Write the image directly to the blockdev
+    else:
+        # Create a temporary blockdev
         zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume)
+        retflag, retdata = pvc_ceph.add_volume(zk_conn, pool, "{}_tmp".format(volume), dev_size)
         pvc_common.stopZKConnection(zk_conn)
         if not retflag:
             output = {
                 'message': retdata.replace('\"', '\'')
             }
             retcode = 400
+            cleanup_maps_and_volumes()
             return output, retcode
 
+        # Map the temporary target blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, "{}_tmp".format(volume))
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+        temp_blockdev = retdata
+
+        # Map the target blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume)
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+        dest_blockdev = retdata
+
+        # Save the data to the temporary blockdev directly
+        try:
+            data.save(temp_blockdev)
+        except:
+            output = {
+                'message': "ERROR: Failed to write image file to temporary volume."
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+
+        # Convert from the temporary to destination format on the blockdevs
+        retcode, stdout, stderr = pvc_common.run_os_command(
+            'qemu-img convert -C -f {} -O raw {} {}'.format(img_type, temp_blockdev, dest_blockdev)
+        )
+        if retcode:
+            output = {
+                'message': "ERROR: Failed to convert image format from '{}' to 'raw': {}".format(img_type, stderr)
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+
+        output = {
+            'message': "Converted and wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
+        }
+        retcode = 200
+        cleanup_maps_and_volumes()
         return output, retcode
 
 def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True):

daemon-common/common.py

@@ -38,11 +38,12 @@ import daemon_lib.zkhandler as zkhandler
 #
 # Run a local OS command via shell
 #
-def run_os_command(command_string, background=False, environment=None, timeout=None):
+def run_os_command(command_string, background=False, environment=None, timeout=None, shell=False):
     command = shlex.split(command_string)
     try:
         command_output = subprocess.run(
             command,
+            shell=shell,
             env=environment,
             timeout=timeout,
             stdout=subprocess.PIPE,

docs/manuals/swagger.json

@@ -4696,8 +4696,23 @@
                 "description": "",
                 "parameters": [
                     {
-                        "description": "The raw binary contents of the file",
+                        "description": "The type of source image file",
+                        "enum": [
+                            "raw",
+                            "vmdk",
+                            "qcow2",
+                            "qed",
+                            "vdi",
+                            "vpc"
+                        ],
                         "in": "query",
+                        "name": "image_format",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "description": "The raw binary contents of the file",
+                        "in": "formdata",
                         "name": "file",
                         "required": true,
                         "type": "binary"