From 9d5f50f82a59bc8d620ae92c783619c1f0a66dd6 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Thu, 20 Feb 2020 22:40:49 -0500 Subject: [PATCH] Implement progress bars for file uploads Provide pretty status bars to indicate upload progress for tasks that perform large file uploads to the API ('provisioner ova upload' and 'storage volume upload') so the administrator can gauge progress and estimated time to completion. --- client-cli/cli_lib/ceph.py | 24 ++++++++--- client-cli/cli_lib/common.py | 69 +++++++++++++++++++++++++++++-- client-cli/cli_lib/provisioner.py | 22 ++++++++-- 3 files changed, 102 insertions(+), 13 deletions(-) diff --git a/client-cli/cli_lib/ceph.py b/client-cli/cli_lib/ceph.py index f04bc54b..09ba8ec1 100644 --- a/client-cli/cli_lib/ceph.py +++ b/client-cli/cli_lib/ceph.py @@ -25,8 +25,10 @@ import json import time import math +from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor + import cli_lib.ansiprint as ansiprint -from cli_lib.common import call_api +from cli_lib.common import UploadProgressBar, call_api # # Supplemental functions @@ -863,13 +865,25 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file): API arguments: image_format={image_format} API schema: {"message":"{data}"} """ + import click + + bar = UploadProgressBar(image_file, end_message="Parsing file on remote side...", end_nl=False) + upload_data = MultipartEncoder( + fields={ 'file': ('filename', open(image_file, 'rb'), 'text/plain')} + ) + upload_monitor = MultipartEncoderMonitor(upload_data, bar.update) + + headers = { + "Content-Type": upload_monitor.content_type + } params = { 'image_format': image_format } - files = { - 'file': open(image_file,'rb') - } - response = call_api(config, 'post', '/storage/ceph/volume/{}/{}/upload'.format(pool, volume), params=params, files=files) + + response = call_api(config, 'post', '/storage/ceph/volume/{}/{}/upload'.format(pool, volume), headers=headers, params=params, data=upload_monitor) + + click.echo("done.") + click.echo() if response.status_code == 200: retstatus = True diff --git a/client-cli/cli_lib/common.py b/client-cli/cli_lib/common.py index d5541a31..54e4e1cd 100644 --- a/client-cli/cli_lib/common.py +++ b/client-cli/cli_lib/common.py @@ -20,9 +20,72 @@ # ############################################################################### +import os +import io +import math +import time import requests import click +def format_bytes(size_bytes): + byte_unit_matrix = { + 'B': 1, + 'K': 1024, + 'M': 1024*1024, + 'G': 1024*1024*1024, + 'T': 1024*1024*1024*1024, + 'P': 1024*1024*1024*1024*1024 + } + human_bytes = '0B' + for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get): + formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit])) + if formatted_bytes < 10000: + human_bytes = '{}{}'.format(formatted_bytes, unit) + break + return human_bytes + +class UploadProgressBar(object): + def __init__(self, filename, end_message='', end_nl=True): + file_size = os.path.getsize(filename) + file_size_human = format_bytes(file_size) + click.echo("Uploading file (total size {})...".format(file_size_human)) + + self.length = file_size + self.time_last = int(round(time.time() * 1000)) - 1000 + self.bytes_last = 0 + self.bytes_diff = 0 + self.is_end = False + + self.end_message = end_message + self.end_nl = end_nl + if not self.end_nl: + self.end_suffix = ' ' + else: + self.end_suffix = '' + + self.bar = click.progressbar(length=self.length, show_eta=True) + + def update(self, monitor): + bytes_cur = monitor.bytes_read + self.bytes_diff += bytes_cur - self.bytes_last + if self.bytes_last == bytes_cur: + self.is_end = True + self.bytes_last = bytes_cur + + time_cur = int(round(time.time() * 1000)) + if (time_cur - 1000) > self.time_last: + self.time_last = time_cur + self.bar.update(self.bytes_diff) + self.bytes_diff = 0 + + if self.is_end: + self.bar.update(self.bytes_diff) + self.bytes_diff = 0 + click.echo() + click.echo() + if self.end_message: + click.echo(self.end_message + self.end_suffix, nl=self.end_nl) + class ErrorResponse(requests.Response): def __init__(self, json_data, status_code): self.json_data = json_data @@ -31,7 +94,7 @@ class ErrorResponse(requests.Response): def json(self): return self.json_data -def call_api(config, operation, request_uri, params=None, data=None, files=None): +def call_api(config, operation, request_uri, headers={}, params=None, data=None, files=None): # Craft the URI uri = '{}://{}{}{}'.format( config['api_scheme'], @@ -42,9 +105,7 @@ def call_api(config, operation, request_uri, params=None, data=None, files=None) # Craft the authentication header if required if config['api_key']: - headers = {'X-Api-Key': config['api_key']} - else: - headers = None + headers['X-Api-Key'] = config['api_key'] # Determine the request type and hit the API try: diff --git a/client-cli/cli_lib/provisioner.py b/client-cli/cli_lib/provisioner.py index 7ffa32fb..eb4001d9 100644 --- a/client-cli/cli_lib/provisioner.py +++ b/client-cli/cli_lib/provisioner.py @@ -25,8 +25,10 @@ import re import subprocess import ast +from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor + import cli_lib.ansiprint as ansiprint -from cli_lib.common import call_api +from cli_lib.common import UploadProgressBar, call_api # # Primary functions @@ -399,10 +401,22 @@ def ova_upload(config, name, ova_file, params): API arguments: pool={pool}, ova_size={ova_size} API schema: {"message":"{data}"} """ - files = { - 'file': open(ova_file,'rb') + import click + + bar = UploadProgressBar(ova_file, end_message="Parsing file on remote side...", end_nl=False) + upload_data = MultipartEncoder( + fields={ 'file': ('filename', open(ova_file, 'rb'), 'text/plain')} + ) + upload_monitor = MultipartEncoderMonitor(upload_data, bar.update) + + headers = { + "Content-Type": upload_monitor.content_type } - response = call_api(config, 'post', '/provisioner/ova/{}'.format(name), params=params, files=files) + + response = call_api(config, 'post', '/provisioner/ova/{}'.format(name), headers=headers, params=params, data=upload_monitor) + + click.echo("done.") + click.echo() if response.status_code == 200: retstatus = True