Improve handling of large file uploads

By default, Werkzeug would require the entire file (be it an OVA or
image file) to be uploaded and saved to a temporary, fake file under
`/tmp`, before any further processing could occur. This blocked most of
the execution of these functions until the upload was completed.

This entirely defeated the purpose of what I was trying to do, which was
to save the uploads directly to the temporary blockdev in each case,
thus avoiding any sort of memory or (host) disk usage.

The solution is two-fold:

  1. First, ensure that the `location='args'` value is set in
  RequestParser; without this, the `files` portion would be parsed
  during the argument parsing, which was the original source of this
  blocking behaviour.

  2. Instead of the convoluted request handling that was being done
  originally here, instead entirely defer the parsing of the `files`
  arguments until the point in the code where they are ready to be
  saved. Then, using an override stream_factory that simply opens the
  temporary blockdev, the upload can commence while being written
  directly out to it, rather than using `/tmp` space.

This does alter the error handling slightly; it is impossible to check
if the argument was passed until this point in the code, so it may take
longer to fail if the API consumer does not specify a file as they
should. This is a minor trade-off and I would expect my API consumers to
be sane here.

api-daemon/pvcapid/flaskapi.py

@@ -166,7 +166,8 @@ class RequestParser(object):
                     required=reqarg.get('required', False),
                     action=reqarg.get('action', None),
                     choices=reqarg.get('choices', ()),
-                    help=reqarg.get('helptext', None)
+                    help=reqarg.get('helptext', None),
+                    location='args'
                 )
             reqargs = parser.parse_args()
             kwargs['reqargs'] = reqargs
@@ -3694,19 +3695,9 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
               type: object
               id: Message
         """
-        from flask_restful import reqparse
-        from werkzeug.datastructures import FileStorage
-        parser = reqparse.RequestParser()
-        parser.add_argument('file', type=FileStorage, location='files')
-        data = parser.parse_args()
-        image_data = data.get('file', None)
-        if not image_data:
-            return { 'message': 'An image file contents must be specified.' }, 400
-
         return api_helper.ceph_volume_upload(
             pool,
             volume,
-            image_data,
             reqargs.get('image_format', None)
         )
 api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume/<pool>/<volume>/upload')
@@ -5668,17 +5659,7 @@ class API_Provisioner_OVA_Root(Resource):
               type: object
               id: Message
         """
-        from flask_restful import reqparse
-        from werkzeug.datastructures import FileStorage
-        parser = reqparse.RequestParser()
-        parser.add_argument('file', type=FileStorage, location='files')
-        data = parser.parse_args()
-        ova_data = data.get('file', None)
-        if not ova_data:
-            return { 'message': 'An OVA file contents must be specified.' }, 400
-
         return api_ova.upload_ova(
-            ova_data,
             reqargs.get('pool', None),
             reqargs.get('name', None),
             reqargs.get('ova_size', None),
@@ -5746,17 +5727,7 @@ class API_Provisioner_OVA_Element(Resource):
               type: object
               id: Message
         """
-        from flask_restful import reqparse
-        from werkzeug.datastructures import FileStorage
-        parser = reqparse.RequestParser()
-        parser.add_argument('file', type=FileStorage, location='files')
-        data = parser.parse_args()
-        ova_data = data.get('file', None)
-        if not ova_data:
-            return { 'message': 'An OVA file contents must be specified.' }, 400
-
         return api_ova.upload_ova(
-            ova_data,
             reqargs.get('pool', None),
             ova,
             reqargs.get('ova_size', None),

api-daemon/pvcapid/helper.py

@@ -26,6 +26,8 @@ import lxml.etree as etree
 
 from distutils.util import strtobool as dustrtobool
 
+from werkzeug.formparser import parse_form_data
+
 import daemon_lib.common as pvc_common
 import daemon_lib.cluster as pvc_cluster
 import daemon_lib.node as pvc_node
@@ -1337,7 +1339,7 @@ def ceph_volume_remove(pool, name):
     }
     return output, retcode
 
-def ceph_volume_upload(pool, volume, data, img_type):
+def ceph_volume_upload(pool, volume, img_type):
     """
     Upload a raw file via HTTP post to a PVC Ceph volume
     """
@@ -1447,10 +1449,16 @@ def ceph_volume_upload(pool, volume, data, img_type):
 
         # Save the data to the temporary blockdev directly
         try:
-            data.save(temp_blockdev)
+            # This sets up a custom stream_factory that writes directly into the ova_blockdev,
+            # rather than the standard stream_factory which writes to a temporary file waiting
+            # on a save() call. This will break if the API ever uploaded multiple files, but
+            # this is an acceptable workaround.
+            def ova_stream_factory(total_content_length, filename, content_type, content_length=None):
+                return open(temp_blockdev, 'wb')
+            parse_form_data(flask.request.environ, stream_factory=ova_stream_factory)
         except:
             output = {
-                'message': "Failed to write image file to temporary volume."
+                'message': "Failed to upload or write image file to temporary volume."
             }
             retcode = 400
             cleanup_maps_and_volumes()

api-daemon/pvcapid/ova.py

@@ -35,6 +35,8 @@ import subprocess
 
 import lxml.etree
 
+from werkzeug.formparser import parse_form_data
+
 import daemon_lib.common as pvc_common
 import daemon_lib.node as pvc_node
 import daemon_lib.vm as pvc_vm
@@ -162,7 +164,7 @@ def delete_ova(name):
     close_database(conn, cur)
     return retmsg, retcode
 
-def upload_ova(ova_data, pool, name, ova_size):
+def upload_ova(pool, name, ova_size):
     ova_archive = None
 
     # Cleanup function
@@ -224,10 +226,16 @@ def upload_ova(ova_data, pool, name, ova_size):
 
     # Save the OVA data to the temporary blockdev directly
     try:
-        ova_data.save(ova_blockdev)
+        # This sets up a custom stream_factory that writes directly into the ova_blockdev,
+        # rather than the standard stream_factory which writes to a temporary file waiting
+        # on a save() call. This will break if the API ever uploaded multiple files, but
+        # this is an acceptable workaround.
+        def ova_stream_factory(total_content_length, filename, content_type, content_length=None):
+            return open(ova_blockdev, 'wb')
+        parse_form_data(flask.request.environ, stream_factory=ova_stream_factory)
     except:
         output = {
-            'message': "Failed to write OVA file to temporary volume."
+            'message': "Failed to upload or write OVA file to temporary volume."
         }
         retcode = 400
         cleanup_ova_maps_and_volumes()