parallelvirtualcluster/pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 115 Wiki Activity

Improve loading times in client

Browse Source
This commit is contained in:
Joshua Boniface
2025-03-13 00:49:45 -04:00
parent 768d435445
commit aa401a12c8
3 changed files with 376 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
client-cli/pvc/cli/cli.py
View File

@@ -19,6 +19,15 @@
#
###############################################################################
# Import only the essential parts of click for CLI definition
from click import group, command, option, argument, pass_context
# Import minimal required modules
from pvc.lib.common import call_api
# Use lazy imports for modules not needed during initial CLI parsing
from pvc.lib.lazy_imports import yaml, click_advanced
from colorama import Fore
from difflib import unified_diff
from functools import wraps

444
client-cli/pvc/lib/common.py
View File

@@ -23,10 +23,336 @@ from ast import literal_eval
from click import echo, progressbar
from math import ceil
from os.path import getsize
from requests import get, post, put, patch, delete, Response
from requests.exceptions import ConnectionError
from time import time
from urllib3 import disable_warnings
import json
import socket
import ssl
# Define a Response class to mimic requests.Response
class Response:
def __init__(self, status_code, headers, content):
self.status_code = status_code
self.headers = headers
self.content = content
self._json = None
def json(self):
if self._json is None:
try:
self._json = json.loads(self.content.decode('utf-8'))
except json.JSONDecodeError:
self._json = {}
return self._json
# Define ConnectionError to mimic requests.exceptions.ConnectionError
class ConnectionError(Exception):
pass
class ErrorResponse(Response):
def __init__(self, json_data, status_code, headers):
self.status_code = status_code
self.headers = headers
self._json = json_data
self.content = json.dumps(json_data).encode('utf-8') if json_data else b''
def json(self):
return self._json
def _encode_params(params):
"""Simple URL parameter encoder"""
if not params:
return ''
parts = []
for key, value in params.items():
if isinstance(value, bool):
value = str(value).lower()
elif value is None:
value = ''
else:
value = str(value)
parts.append(f"{key}={value}")
return '?' + '&'.join(parts)
def call_api(
config,
operation,
request_uri,
headers={},
params=None,
data=None,
files=None,
):
"""
Make an API call to the PVC API using native Python libraries.
"""
# Set the connect timeout to 2 seconds
timeout = 2.05
# Craft the URI
uri = "{}://{}{}{}".format(
config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
)
# Parse the URI without using urllib
if '://' in uri:
scheme, rest = uri.split('://', 1)
else:
scheme = 'http'
rest = uri
if '/' in rest:
netloc, path = rest.split('/', 1)
path = '/' + path
else:
netloc = rest
path = '/'
# Extract host and port
if ':' in netloc:
host, port_str = netloc.split(':', 1)
port = int(port_str)
else:
host = netloc
port = 443 if scheme == 'https' else 80
# Craft the authentication header if required
if config["api_key"]:
headers["X-Api-Key"] = config["api_key"]
# Add content type if not present
if "Content-Type" not in headers and data is not None and files is None:
headers["Content-Type"] = "application/json"
# Prepare query string
query_string = _encode_params(params)
# Prepare path with query string
full_path = path + query_string
# Prepare body
body = None
if data is not None and files is None:
if isinstance(data, dict):
body = json.dumps(data).encode('utf-8')
else:
body = data.encode('utf-8') if isinstance(data, str) else data
# Handle file uploads (multipart/form-data)
if files:
boundary = '----WebKitFormBoundary' + str(int(time()))
headers['Content-Type'] = f'multipart/form-data; boundary={boundary}'
body = b''
# Add form fields
if data:
for key, value in data.items():
body += f'--{boundary}\r\n'.encode()
body += f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode()
body += f'{value}\r\n'.encode()
# Add files
for key, file_tuple in files.items():
filename, fileobj, content_type = file_tuple
body += f'--{boundary}\r\n'.encode()
body += f'Content-Disposition: form-data; name="{key}"; filename="{filename}"\r\n'.encode()
body += f'Content-Type: {content_type}\r\n\r\n'.encode()
body += fileobj.read()
body += b'\r\n'
body += f'--{boundary}--\r\n'.encode()
# Determine the request type and hit the API
response = None
try:
# Special handling for GET with retries
if operation == "get":
retry_on_code = [429, 500, 502, 503, 504]
for i in range(3):
failed = False
try:
# Create socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
# Handle SSL for HTTPS
if scheme == 'https':
context = ssl.create_default_context()
if not config["verify_ssl"]:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
s = context.wrap_socket(s, server_hostname=host)
# Connect
s.connect((host, port))
# Build request
request = f"{operation.upper()} {full_path} HTTP/1.1\r\n"
request += f"Host: {host}\r\n"
for key, value in headers.items():
request += f"{key}: {value}\r\n"
if body:
request += f"Content-Length: {len(body)}\r\n"
request += "Connection: close\r\n\r\n"
# Send request
s.sendall(request.encode('utf-8'))
if body:
s.sendall(body)
# Read response
response_data = b""
while True:
try:
chunk = s.recv(4096)
if not chunk:
break
response_data += chunk
except socket.timeout:
# If we've received some data but timed out, that's okay
if response_data:
break
else:
raise
# Parse response
if b'\r\n\r\n' in response_data:
header_end = response_data.find(b'\r\n\r\n')
headers_raw = response_data[:header_end].decode('utf-8')
body_raw = response_data[header_end + 4:]
# Parse status code
status_line = headers_raw.split('\r\n')[0]
status_code = int(status_line.split(' ')[1])
# Parse headers
headers_dict = {}
for line in headers_raw.split('\r\n')[1:]:
if not line:
continue
if ':' in line:
key, value = line.split(':', 1)
headers_dict[key.strip()] = value.strip()
# Create response object
response = Response(status_code, headers_dict, body_raw)
if response.status_code in retry_on_code:
failed = True
s.close()
continue
break
else:
failed = True
s.close()
continue
except Exception as e:
failed = True
if 's' in locals():
s.close()
continue
finally:
if 's' in locals():
s.close()
if failed:
error = f"Code {response.status_code}" if response else "Timeout"
raise ConnectionError(f"Failed to connect after 3 tries ({error})")
else:
# Create socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
try:
# Handle SSL for HTTPS
if scheme == 'https':
context = ssl.create_default_context()
if not config["verify_ssl"]:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
s = context.wrap_socket(s, server_hostname=host)
# Connect
s.connect((host, port))
# Build request
request = f"{operation.upper()} {full_path} HTTP/1.1\r\n"
request += f"Host: {host}\r\n"
for key, value in headers.items():
request += f"{key}: {value}\r\n"
if body:
request += f"Content-Length: {len(body)}\r\n"
request += "Connection: close\r\n\r\n"
# Send request
s.sendall(request.encode('utf-8'))
if body:
s.sendall(body)
# Read response
response_data = b""
while True:
try:
chunk = s.recv(4096)
if not chunk:
break
response_data += chunk
except socket.timeout:
# If we've received some data but timed out, that's okay
if response_data:
break
else:
raise
# Parse response
if b'\r\n\r\n' in response_data:
header_end = response_data.find(b'\r\n\r\n')
headers_raw = response_data[:header_end].decode('utf-8')
body_raw = response_data[header_end + 4:]
# Parse status code
status_line = headers_raw.split('\r\n')[0]
status_code = int(status_line.split(' ')[1])
# Parse headers
headers_dict = {}
for line in headers_raw.split('\r\n')[1:]:
if not line:
continue
if ':' in line:
key, value = line.split(':', 1)
headers_dict[key.strip()] = value.strip()
# Create response object
response = Response(status_code, headers_dict, body_raw)
else:
raise Exception("Invalid HTTP response")
finally:
s.close()
except Exception as e:
message = f"Failed to connect to the API: {e}"
code = getattr(response, 'status_code', 504)
response = ErrorResponse({"message": message}, code, None)
# Display debug output
if config["debug"]:
echo("API endpoint: {}".format(uri), err=True)
echo("Response code: {}".format(response.status_code), err=True)
echo("Response headers: {}".format(response.headers), err=True)
echo(err=True)
# Return the response object
return response
def format_bytes(size_bytes):
@@ -139,118 +465,6 @@ class UploadProgressBar(object):
echo(self.end_message + self.end_suffix, nl=self.end_nl)
class ErrorResponse(Response):
def __init__(self, json_data, status_code, headers):
self.json_data = json_data
self.status_code = status_code
self.headers = headers
def json(self):
return self.json_data
def call_api(
config,
operation,
request_uri,
headers={},
params=None,
data=None,
files=None,
):
# Set the connect timeout to 2 seconds but extremely long (48 hour) data timeout
timeout = (2.05, 172800)
# Craft the URI
uri = "{}://{}{}{}".format(
config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
)
# Craft the authentication header if required
if config["api_key"]:
headers["X-Api-Key"] = config["api_key"]
# Determine the request type and hit the API
disable_warnings()
try:
response = None
if operation == "get":
retry_on_code = [429, 500, 502, 503, 504]
for i in range(3):
failed = False
try:
response = get(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
verify=config["verify_ssl"],
)
if response.status_code in retry_on_code:
failed = True
continue
break
except ConnectionError:
failed = True
continue
if failed:
error = f"Code {response.status_code}" if response else "Timeout"
raise ConnectionError(f"Failed to connect after 3 tries ({error})")
if operation == "post":
response = post(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
files=files,
verify=config["verify_ssl"],
)
if operation == "put":
response = put(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
files=files,
verify=config["verify_ssl"],
)
if operation == "patch":
response = patch(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
verify=config["verify_ssl"],
)
if operation == "delete":
response = patch, delete(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
verify=config["verify_ssl"],
)
except Exception as e:
message = "Failed to connect to the API: {}".format(e)
code = response.status_code if response else 504
response = ErrorResponse({"message": message}, code, None)
# Display debug output
if config["debug"]:
echo("API endpoint: {}".format(uri), err=True)
echo("Response code: {}".format(response.status_code), err=True)
echo("Response headers: {}".format(response.headers), err=True)
echo(err=True)
# Return the response object
return response
def get_wait_retdata(response, wait_flag):
if response.status_code == 202:
retvalue = True

38
client-cli/pvc/lib/lazy_imports.py Normal file
View File

@@ -0,0 +1,38 @@
#!/usr/bin/env python3
# lazy_imports.py - Lazy module importer library
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2024 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
class LazyModule:
"""
A proxy for a module that is loaded only when actually used
"""
def __init__(self, name):
self.name = name
self._module = None
def __getattr__(self, attr):
if self._module is None:
import importlib
self._module = importlib.import_module(self.name)
return getattr(self._module, attr)
# Create lazy module proxies
yaml = LazyModule('yaml')
click_advanced = LazyModule('click') # For advanced click features not used at startup