Support converting types during upload

Allow the user to specify other, non-raw files and upload them,
performing a conversion with qemu-img convert and a temporary block
device as a shim (since qemu-img can't use FIFOs).

Also ensures that the target volume exists before proceeding.

Addresses #68
This commit is contained in:
Joshua Boniface 2020-02-09 19:43:07 -05:00
parent 49e5ce1176
commit e419855911
4 changed files with 166 additions and 38 deletions

View File

@ -3336,8 +3336,11 @@ api.add_resource(API_Storage_Ceph_Volume_Element_Clone, '/storage/ceph/volume/<p
# /storage/ceph/volume/<pool>/<volume>/upload # /storage/ceph/volume/<pool>/<volume>/upload
class API_Storage_Ceph_Volume_Element_Upload(Resource): class API_Storage_Ceph_Volume_Element_Upload(Resource):
@RequestParser([
{ 'name': 'image_format' }
])
@Authenticator @Authenticator
def post(self, pool, volume): def post(self, pool, volume, reqargs):
""" """
Upload a disk image to Ceph volume {volume} in pool {pool} Upload a disk image to Ceph volume {volume} in pool {pool}
--- ---
@ -3345,6 +3348,18 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
- storage / ceph - storage / ceph
parameters: parameters:
- in: query - in: query
name: image_format
type: string
required: true
description: The type of source image file
enum:
- raw
- vmdk
- qcow2
- qed
- vdi
- vpc
- in: formdata
name: file name: file
type: binary type: binary
required: true required: true
@ -3374,10 +3389,12 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
image_data = data.get('file', None) image_data = data.get('file', None)
if not image_data: if not image_data:
return {'message':'An image file contents must be specified'}, 400 return {'message':'An image file contents must be specified'}, 400
return api_helper.ceph_volume_upload( return api_helper.ceph_volume_upload(
pool, pool,
volume, volume,
image_data image_data,
reqargs.get('image_format', None)
) )
api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume/<pool>/<volume>/upload') api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume/<pool>/<volume>/upload')

View File

@ -1327,10 +1327,43 @@ def ceph_volume_remove(pool, name):
} }
return output, retcode return output, retcode
def ceph_volume_upload(pool, volume, data): def ceph_volume_upload(pool, volume, data, img_type):
""" """
Upload a raw file via HTTP post to a PVC Ceph volume Upload a raw file via HTTP post to a PVC Ceph volume
""" """
# Determine the image conversion options
if img_type not in ['raw', 'vmdk', 'qcow2', 'qed', 'vdi', 'vpc']:
output = {
"message": "ERROR: Image type '{}' is not valid.".format(img_type)
}
retcode = 400
return output, retcode
# Get the size of the target block device
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retcode, retdata = pvc_ceph.get_list_volume(zk_conn, pool, volume, is_fuzzy=False)
pvc_common.stopZKConnection(zk_conn)
# If there's no target, return failure
if not retcode or len(retdata) < 1:
output = {
"message": "ERROR: Target volume '{}' does not exist in pool '{}'.".format(volume, pool)
}
retcode = 400
return output, retcode
dev_size = retdata[0]['stats']['size']
def cleanup_maps_and_volumes():
zk_conn = pvc_common.startZKConnection(config['coordinators'])
# Unmap the target blockdev
retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume)
# Unmap the temporary blockdev
retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, "{}_tmp".format(volume))
# Remove the temporary blockdev
retflag, retdata = pvc_ceph.remove_volume(zk_conn, pool, "{}_tmp".format(volume))
pvc_common.stopZKConnection(zk_conn)
# Create a temporary block device to store non-raw images
if img_type == 'raw':
# Map the target blockdev # Map the target blockdev
zk_conn = pvc_common.startZKConnection(config['coordinators']) zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume) retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume)
@ -1340,34 +1373,96 @@ def ceph_volume_upload(pool, volume, data):
'message': retdata.replace('\"', '\'') 'message': retdata.replace('\"', '\'')
} }
retcode = 400 retcode = 400
cleanup_maps_and_volumes()
return output, retcode return output, retcode
blockdev = retdata dest_blockdev = retdata
output = { # Save the data to the blockdev directly
'message': "Wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
}
retcode = 200
# Save the data to the blockdev
try: try:
data.save(blockdev) data.save(dest_blockdev)
except: except:
output = { output = {
'message': "ERROR: Failed to write image file to volume." 'message': "ERROR: Failed to write image file to volume."
} }
retcode = 400 retcode = 400
cleanup_maps_and_volumes()
return output, retcode
# Unmap the target blockdev output = {
'message': "Wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
}
retcode = 200
cleanup_maps_and_volumes()
return output, retcode
# Write the image directly to the blockdev
else:
# Create a temporary blockdev
zk_conn = pvc_common.startZKConnection(config['coordinators']) zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume) retflag, retdata = pvc_ceph.add_volume(zk_conn, pool, "{}_tmp".format(volume), dev_size)
pvc_common.stopZKConnection(zk_conn) pvc_common.stopZKConnection(zk_conn)
if not retflag: if not retflag:
output = { output = {
'message': retdata.replace('\"', '\'') 'message': retdata.replace('\"', '\'')
} }
retcode = 400 retcode = 400
cleanup_maps_and_volumes()
return output, retcode return output, retcode
# Map the temporary target blockdev
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, "{}_tmp".format(volume))
pvc_common.stopZKConnection(zk_conn)
if not retflag:
output = {
'message': retdata.replace('\"', '\'')
}
retcode = 400
cleanup_maps_and_volumes()
return output, retcode
temp_blockdev = retdata
# Map the target blockdev
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume)
pvc_common.stopZKConnection(zk_conn)
if not retflag:
output = {
'message': retdata.replace('\"', '\'')
}
retcode = 400
cleanup_maps_and_volumes()
return output, retcode
dest_blockdev = retdata
# Save the data to the temporary blockdev directly
try:
data.save(temp_blockdev)
except:
output = {
'message': "ERROR: Failed to write image file to temporary volume."
}
retcode = 400
cleanup_maps_and_volumes()
return output, retcode
# Convert from the temporary to destination format on the blockdevs
retcode, stdout, stderr = pvc_common.run_os_command(
'qemu-img convert -C -f {} -O raw {} {}'.format(img_type, temp_blockdev, dest_blockdev)
)
if retcode:
output = {
'message': "ERROR: Failed to convert image format from '{}' to 'raw': {}".format(img_type, stderr)
}
retcode = 400
cleanup_maps_and_volumes()
return output, retcode
output = {
'message': "Converted and wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
}
retcode = 200
cleanup_maps_and_volumes()
return output, retcode return output, retcode
def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True): def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True):

View File

@ -38,11 +38,12 @@ import daemon_lib.zkhandler as zkhandler
# #
# Run a local OS command via shell # Run a local OS command via shell
# #
def run_os_command(command_string, background=False, environment=None, timeout=None): def run_os_command(command_string, background=False, environment=None, timeout=None, shell=False):
command = shlex.split(command_string) command = shlex.split(command_string)
try: try:
command_output = subprocess.run( command_output = subprocess.run(
command, command,
shell=shell,
env=environment, env=environment,
timeout=timeout, timeout=timeout,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,

View File

@ -4696,8 +4696,23 @@
"description": "", "description": "",
"parameters": [ "parameters": [
{ {
"description": "The raw binary contents of the file", "description": "The type of source image file",
"enum": [
"raw",
"vmdk",
"qcow2",
"qed",
"vdi",
"vpc"
],
"in": "query", "in": "query",
"name": "image_format",
"required": true,
"type": "string"
},
{
"description": "The raw binary contents of the file",
"in": "formdata",
"name": "file", "name": "file",
"required": true, "required": true,
"type": "binary" "type": "binary"