Implement progress bars for file uploads

Provide pretty status bars to indicate upload progress for tasks that
perform large file uploads to the API ('provisioner ova upload' and
'storage volume upload') so the administrator can gauge progress and
estimated time to completion.

client-cli/cli_lib/ceph.py

@@ -25,8 +25,10 @@ import json
 import time
 import math
 
+from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
+
 import cli_lib.ansiprint as ansiprint
-from cli_lib.common import call_api
+from cli_lib.common import UploadProgressBar, call_api
 
 #
 # Supplemental functions
@@ -863,13 +865,25 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     API arguments: image_format={image_format}
     API schema: {"message":"{data}"}
     """
+    import click
+
+    bar = UploadProgressBar(image_file, end_message="Parsing file on remote side...", end_nl=False)
+    upload_data = MultipartEncoder(
+        fields={ 'file': ('filename', open(image_file, 'rb'), 'text/plain')}
+    )
+    upload_monitor = MultipartEncoderMonitor(upload_data, bar.update)
+
+    headers = {
+        "Content-Type": upload_monitor.content_type
+    }
     params = {
         'image_format': image_format
     }
-    files = {
+
-        'file': open(image_file,'rb')
+    response = call_api(config, 'post', '/storage/ceph/volume/{}/{}/upload'.format(pool, volume), headers=headers, params=params, data=upload_monitor)
-    }
+
-    response = call_api(config, 'post', '/storage/ceph/volume/{}/{}/upload'.format(pool, volume), params=params, files=files)
+    click.echo("done.")
+    click.echo()
 
     if response.status_code == 200:
         retstatus = True

client-cli/cli_lib/common.py

@@ -20,9 +20,72 @@
 #
 ###############################################################################
 
+import os
+import io
+import math
+import time
 import requests
 import click
 
+def format_bytes(size_bytes):
+    byte_unit_matrix = {
+        'B': 1,
+        'K': 1024,
+        'M': 1024*1024,
+        'G': 1024*1024*1024,
+        'T': 1024*1024*1024*1024,
+        'P': 1024*1024*1024*1024*1024
+    }
+    human_bytes = '0B'
+    for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get):
+        formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit]))
+        if formatted_bytes < 10000:
+            human_bytes = '{}{}'.format(formatted_bytes, unit)
+            break
+    return human_bytes
+
+class UploadProgressBar(object):
+    def __init__(self, filename, end_message='', end_nl=True):
+        file_size = os.path.getsize(filename)
+        file_size_human = format_bytes(file_size)
+        click.echo("Uploading file (total size {})...".format(file_size_human))
+
+        self.length = file_size
+        self.time_last = int(round(time.time() * 1000)) - 1000
+        self.bytes_last = 0
+        self.bytes_diff = 0
+        self.is_end = False
+
+        self.end_message = end_message
+        self.end_nl = end_nl
+        if not self.end_nl:
+            self.end_suffix = ' '
+        else:
+            self.end_suffix = ''
+
+        self.bar = click.progressbar(length=self.length, show_eta=True)
+
+    def update(self, monitor):
+        bytes_cur = monitor.bytes_read
+        self.bytes_diff += bytes_cur - self.bytes_last
+        if self.bytes_last == bytes_cur:
+            self.is_end = True
+        self.bytes_last = bytes_cur
+
+        time_cur = int(round(time.time() * 1000))
+        if (time_cur - 1000) > self.time_last:
+            self.time_last = time_cur
+            self.bar.update(self.bytes_diff)
+            self.bytes_diff = 0
+
+        if self.is_end:
+            self.bar.update(self.bytes_diff)
+            self.bytes_diff = 0
+            click.echo()
+            click.echo()
+            if self.end_message:
+                click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
+
 class ErrorResponse(requests.Response):
     def __init__(self, json_data, status_code):
         self.json_data = json_data
@@ -31,7 +94,7 @@ class ErrorResponse(requests.Response):
     def json(self):
         return self.json_data
 
-def call_api(config, operation, request_uri, params=None, data=None, files=None):
+def call_api(config, operation, request_uri, headers={}, params=None, data=None, files=None):
     # Craft the URI
     uri = '{}://{}{}{}'.format(
         config['api_scheme'],
@@ -42,9 +105,7 @@ def call_api(config, operation, request_uri, params=None, data=None, files=None)
 
     # Craft the authentication header if required
     if config['api_key']:
-        headers = {'X-Api-Key': config['api_key']}
+        headers['X-Api-Key'] = config['api_key']
-    else:
-        headers = None
 
     # Determine the request type and hit the API
     try:

client-cli/cli_lib/provisioner.py

@@ -25,8 +25,10 @@ import re
 import subprocess
 import ast
 
+from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
+
 import cli_lib.ansiprint as ansiprint
-from cli_lib.common import call_api
+from cli_lib.common import UploadProgressBar, call_api
 
 #
 # Primary functions
@@ -399,10 +401,22 @@ def ova_upload(config, name, ova_file, params):
     API arguments: pool={pool}, ova_size={ova_size}
     API schema: {"message":"{data}"}
     """
-    files = {
+    import click
-        'file': open(ova_file,'rb')
+
+    bar = UploadProgressBar(ova_file, end_message="Parsing file on remote side...", end_nl=False)
+    upload_data = MultipartEncoder(
+        fields={ 'file': ('filename', open(ova_file, 'rb'), 'text/plain')}
+    )
+    upload_monitor = MultipartEncoderMonitor(upload_data, bar.update)
+
+    headers = {
+        "Content-Type": upload_monitor.content_type
     }
-    response = call_api(config, 'post', '/provisioner/ova/{}'.format(name), params=params, files=files)
+
+    response = call_api(config, 'post', '/provisioner/ova/{}'.format(name), headers=headers, params=params, data=upload_monitor)
+
+    click.echo("done.")
+    click.echo()
 
     if response.status_code == 200:
         retstatus = True