Improve handling of large file uploads
By default, Werkzeug would require the entire file (be it an OVA or image file) to be uploaded and saved to a temporary, fake file under `/tmp`, before any further processing could occur. This blocked most of the execution of these functions until the upload was completed. This entirely defeated the purpose of what I was trying to do, which was to save the uploads directly to the temporary blockdev in each case, thus avoiding any sort of memory or (host) disk usage. The solution is two-fold: 1. First, ensure that the `location='args'` value is set in RequestParser; without this, the `files` portion would be parsed during the argument parsing, which was the original source of this blocking behaviour. 2. Instead of the convoluted request handling that was being done originally here, instead entirely defer the parsing of the `files` arguments until the point in the code where they are ready to be saved. Then, using an override stream_factory that simply opens the temporary blockdev, the upload can commence while being written directly out to it, rather than using `/tmp` space. This does alter the error handling slightly; it is impossible to check if the argument was passed until this point in the code, so it may take longer to fail if the API consumer does not specify a file as they should. This is a minor trade-off and I would expect my API consumers to be sane here.
This commit is contained in:
parent
7a27503f1b
commit
ffaa4c033f
|
@ -166,7 +166,8 @@ class RequestParser(object):
|
||||||
required=reqarg.get('required', False),
|
required=reqarg.get('required', False),
|
||||||
action=reqarg.get('action', None),
|
action=reqarg.get('action', None),
|
||||||
choices=reqarg.get('choices', ()),
|
choices=reqarg.get('choices', ()),
|
||||||
help=reqarg.get('helptext', None)
|
help=reqarg.get('helptext', None),
|
||||||
|
location='args'
|
||||||
)
|
)
|
||||||
reqargs = parser.parse_args()
|
reqargs = parser.parse_args()
|
||||||
kwargs['reqargs'] = reqargs
|
kwargs['reqargs'] = reqargs
|
||||||
|
@ -3694,19 +3695,9 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
|
||||||
type: object
|
type: object
|
||||||
id: Message
|
id: Message
|
||||||
"""
|
"""
|
||||||
from flask_restful import reqparse
|
|
||||||
from werkzeug.datastructures import FileStorage
|
|
||||||
parser = reqparse.RequestParser()
|
|
||||||
parser.add_argument('file', type=FileStorage, location='files')
|
|
||||||
data = parser.parse_args()
|
|
||||||
image_data = data.get('file', None)
|
|
||||||
if not image_data:
|
|
||||||
return { 'message': 'An image file contents must be specified.' }, 400
|
|
||||||
|
|
||||||
return api_helper.ceph_volume_upload(
|
return api_helper.ceph_volume_upload(
|
||||||
pool,
|
pool,
|
||||||
volume,
|
volume,
|
||||||
image_data,
|
|
||||||
reqargs.get('image_format', None)
|
reqargs.get('image_format', None)
|
||||||
)
|
)
|
||||||
api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume/<pool>/<volume>/upload')
|
api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume/<pool>/<volume>/upload')
|
||||||
|
@ -5668,17 +5659,7 @@ class API_Provisioner_OVA_Root(Resource):
|
||||||
type: object
|
type: object
|
||||||
id: Message
|
id: Message
|
||||||
"""
|
"""
|
||||||
from flask_restful import reqparse
|
|
||||||
from werkzeug.datastructures import FileStorage
|
|
||||||
parser = reqparse.RequestParser()
|
|
||||||
parser.add_argument('file', type=FileStorage, location='files')
|
|
||||||
data = parser.parse_args()
|
|
||||||
ova_data = data.get('file', None)
|
|
||||||
if not ova_data:
|
|
||||||
return { 'message': 'An OVA file contents must be specified.' }, 400
|
|
||||||
|
|
||||||
return api_ova.upload_ova(
|
return api_ova.upload_ova(
|
||||||
ova_data,
|
|
||||||
reqargs.get('pool', None),
|
reqargs.get('pool', None),
|
||||||
reqargs.get('name', None),
|
reqargs.get('name', None),
|
||||||
reqargs.get('ova_size', None),
|
reqargs.get('ova_size', None),
|
||||||
|
@ -5746,17 +5727,7 @@ class API_Provisioner_OVA_Element(Resource):
|
||||||
type: object
|
type: object
|
||||||
id: Message
|
id: Message
|
||||||
"""
|
"""
|
||||||
from flask_restful import reqparse
|
|
||||||
from werkzeug.datastructures import FileStorage
|
|
||||||
parser = reqparse.RequestParser()
|
|
||||||
parser.add_argument('file', type=FileStorage, location='files')
|
|
||||||
data = parser.parse_args()
|
|
||||||
ova_data = data.get('file', None)
|
|
||||||
if not ova_data:
|
|
||||||
return { 'message': 'An OVA file contents must be specified.' }, 400
|
|
||||||
|
|
||||||
return api_ova.upload_ova(
|
return api_ova.upload_ova(
|
||||||
ova_data,
|
|
||||||
reqargs.get('pool', None),
|
reqargs.get('pool', None),
|
||||||
ova,
|
ova,
|
||||||
reqargs.get('ova_size', None),
|
reqargs.get('ova_size', None),
|
||||||
|
|
|
@ -26,6 +26,8 @@ import lxml.etree as etree
|
||||||
|
|
||||||
from distutils.util import strtobool as dustrtobool
|
from distutils.util import strtobool as dustrtobool
|
||||||
|
|
||||||
|
from werkzeug.formparser import parse_form_data
|
||||||
|
|
||||||
import daemon_lib.common as pvc_common
|
import daemon_lib.common as pvc_common
|
||||||
import daemon_lib.cluster as pvc_cluster
|
import daemon_lib.cluster as pvc_cluster
|
||||||
import daemon_lib.node as pvc_node
|
import daemon_lib.node as pvc_node
|
||||||
|
@ -1337,7 +1339,7 @@ def ceph_volume_remove(pool, name):
|
||||||
}
|
}
|
||||||
return output, retcode
|
return output, retcode
|
||||||
|
|
||||||
def ceph_volume_upload(pool, volume, data, img_type):
|
def ceph_volume_upload(pool, volume, img_type):
|
||||||
"""
|
"""
|
||||||
Upload a raw file via HTTP post to a PVC Ceph volume
|
Upload a raw file via HTTP post to a PVC Ceph volume
|
||||||
"""
|
"""
|
||||||
|
@ -1447,10 +1449,16 @@ def ceph_volume_upload(pool, volume, data, img_type):
|
||||||
|
|
||||||
# Save the data to the temporary blockdev directly
|
# Save the data to the temporary blockdev directly
|
||||||
try:
|
try:
|
||||||
data.save(temp_blockdev)
|
# This sets up a custom stream_factory that writes directly into the ova_blockdev,
|
||||||
|
# rather than the standard stream_factory which writes to a temporary file waiting
|
||||||
|
# on a save() call. This will break if the API ever uploaded multiple files, but
|
||||||
|
# this is an acceptable workaround.
|
||||||
|
def ova_stream_factory(total_content_length, filename, content_type, content_length=None):
|
||||||
|
return open(temp_blockdev, 'wb')
|
||||||
|
parse_form_data(flask.request.environ, stream_factory=ova_stream_factory)
|
||||||
except:
|
except:
|
||||||
output = {
|
output = {
|
||||||
'message': "Failed to write image file to temporary volume."
|
'message': "Failed to upload or write image file to temporary volume."
|
||||||
}
|
}
|
||||||
retcode = 400
|
retcode = 400
|
||||||
cleanup_maps_and_volumes()
|
cleanup_maps_and_volumes()
|
||||||
|
|
|
@ -35,6 +35,8 @@ import subprocess
|
||||||
|
|
||||||
import lxml.etree
|
import lxml.etree
|
||||||
|
|
||||||
|
from werkzeug.formparser import parse_form_data
|
||||||
|
|
||||||
import daemon_lib.common as pvc_common
|
import daemon_lib.common as pvc_common
|
||||||
import daemon_lib.node as pvc_node
|
import daemon_lib.node as pvc_node
|
||||||
import daemon_lib.vm as pvc_vm
|
import daemon_lib.vm as pvc_vm
|
||||||
|
@ -162,7 +164,7 @@ def delete_ova(name):
|
||||||
close_database(conn, cur)
|
close_database(conn, cur)
|
||||||
return retmsg, retcode
|
return retmsg, retcode
|
||||||
|
|
||||||
def upload_ova(ova_data, pool, name, ova_size):
|
def upload_ova(pool, name, ova_size):
|
||||||
ova_archive = None
|
ova_archive = None
|
||||||
|
|
||||||
# Cleanup function
|
# Cleanup function
|
||||||
|
@ -224,10 +226,16 @@ def upload_ova(ova_data, pool, name, ova_size):
|
||||||
|
|
||||||
# Save the OVA data to the temporary blockdev directly
|
# Save the OVA data to the temporary blockdev directly
|
||||||
try:
|
try:
|
||||||
ova_data.save(ova_blockdev)
|
# This sets up a custom stream_factory that writes directly into the ova_blockdev,
|
||||||
|
# rather than the standard stream_factory which writes to a temporary file waiting
|
||||||
|
# on a save() call. This will break if the API ever uploaded multiple files, but
|
||||||
|
# this is an acceptable workaround.
|
||||||
|
def ova_stream_factory(total_content_length, filename, content_type, content_length=None):
|
||||||
|
return open(ova_blockdev, 'wb')
|
||||||
|
parse_form_data(flask.request.environ, stream_factory=ova_stream_factory)
|
||||||
except:
|
except:
|
||||||
output = {
|
output = {
|
||||||
'message': "Failed to write OVA file to temporary volume."
|
'message': "Failed to upload or write OVA file to temporary volume."
|
||||||
}
|
}
|
||||||
retcode = 400
|
retcode = 400
|
||||||
cleanup_ova_maps_and_volumes()
|
cleanup_ova_maps_and_volumes()
|
||||||
|
|
Loading…
Reference in New Issue