Implement progress bars for file uploads

Provide pretty status bars to indicate upload progress for tasks that
perform large file uploads to the API ('provisioner ova upload' and
'storage volume upload') so the administrator can gauge progress and
estimated time to completion.
This commit is contained in:
Joshua Boniface 2020-02-20 22:40:49 -05:00
parent 56a9e48163
commit 9d5f50f82a
3 changed files with 102 additions and 13 deletions

View File

@ -25,8 +25,10 @@ import json
import time
import math
from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
import cli_lib.ansiprint as ansiprint
from cli_lib.common import call_api
from cli_lib.common import UploadProgressBar, call_api
#
# Supplemental functions
@ -863,13 +865,25 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
API arguments: image_format={image_format}
API schema: {"message":"{data}"}
"""
import click
bar = UploadProgressBar(image_file, end_message="Parsing file on remote side...", end_nl=False)
upload_data = MultipartEncoder(
fields={ 'file': ('filename', open(image_file, 'rb'), 'text/plain')}
)
upload_monitor = MultipartEncoderMonitor(upload_data, bar.update)
headers = {
"Content-Type": upload_monitor.content_type
}
params = {
'image_format': image_format
}
files = {
'file': open(image_file,'rb')
}
response = call_api(config, 'post', '/storage/ceph/volume/{}/{}/upload'.format(pool, volume), params=params, files=files)
response = call_api(config, 'post', '/storage/ceph/volume/{}/{}/upload'.format(pool, volume), headers=headers, params=params, data=upload_monitor)
click.echo("done.")
click.echo()
if response.status_code == 200:
retstatus = True

View File

@ -20,9 +20,72 @@
#
###############################################################################
import os
import io
import math
import time
import requests
import click
def format_bytes(size_bytes):
byte_unit_matrix = {
'B': 1,
'K': 1024,
'M': 1024*1024,
'G': 1024*1024*1024,
'T': 1024*1024*1024*1024,
'P': 1024*1024*1024*1024*1024
}
human_bytes = '0B'
for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get):
formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit]))
if formatted_bytes < 10000:
human_bytes = '{}{}'.format(formatted_bytes, unit)
break
return human_bytes
class UploadProgressBar(object):
def __init__(self, filename, end_message='', end_nl=True):
file_size = os.path.getsize(filename)
file_size_human = format_bytes(file_size)
click.echo("Uploading file (total size {})...".format(file_size_human))
self.length = file_size
self.time_last = int(round(time.time() * 1000)) - 1000
self.bytes_last = 0
self.bytes_diff = 0
self.is_end = False
self.end_message = end_message
self.end_nl = end_nl
if not self.end_nl:
self.end_suffix = ' '
else:
self.end_suffix = ''
self.bar = click.progressbar(length=self.length, show_eta=True)
def update(self, monitor):
bytes_cur = monitor.bytes_read
self.bytes_diff += bytes_cur - self.bytes_last
if self.bytes_last == bytes_cur:
self.is_end = True
self.bytes_last = bytes_cur
time_cur = int(round(time.time() * 1000))
if (time_cur - 1000) > self.time_last:
self.time_last = time_cur
self.bar.update(self.bytes_diff)
self.bytes_diff = 0
if self.is_end:
self.bar.update(self.bytes_diff)
self.bytes_diff = 0
click.echo()
click.echo()
if self.end_message:
click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
class ErrorResponse(requests.Response):
def __init__(self, json_data, status_code):
self.json_data = json_data
@ -31,7 +94,7 @@ class ErrorResponse(requests.Response):
def json(self):
return self.json_data
def call_api(config, operation, request_uri, params=None, data=None, files=None):
def call_api(config, operation, request_uri, headers={}, params=None, data=None, files=None):
# Craft the URI
uri = '{}://{}{}{}'.format(
config['api_scheme'],
@ -42,9 +105,7 @@ def call_api(config, operation, request_uri, params=None, data=None, files=None)
# Craft the authentication header if required
if config['api_key']:
headers = {'X-Api-Key': config['api_key']}
else:
headers = None
headers['X-Api-Key'] = config['api_key']
# Determine the request type and hit the API
try:

View File

@ -25,8 +25,10 @@ import re
import subprocess
import ast
from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
import cli_lib.ansiprint as ansiprint
from cli_lib.common import call_api
from cli_lib.common import UploadProgressBar, call_api
#
# Primary functions
@ -399,10 +401,22 @@ def ova_upload(config, name, ova_file, params):
API arguments: pool={pool}, ova_size={ova_size}
API schema: {"message":"{data}"}
"""
files = {
'file': open(ova_file,'rb')
import click
bar = UploadProgressBar(ova_file, end_message="Parsing file on remote side...", end_nl=False)
upload_data = MultipartEncoder(
fields={ 'file': ('filename', open(ova_file, 'rb'), 'text/plain')}
)
upload_monitor = MultipartEncoderMonitor(upload_data, bar.update)
headers = {
"Content-Type": upload_monitor.content_type
}
response = call_api(config, 'post', '/provisioner/ova/{}'.format(name), params=params, files=files)
response = call_api(config, 'post', '/provisioner/ova/{}'.format(name), headers=headers, params=params, data=upload_monitor)
click.echo("done.")
click.echo()
if response.status_code == 200:
retstatus = True