From dd631f1ae4ccc364d8a62bda7355dd6eff1d433d Mon Sep 17 00:00:00 2001 From: Joshua Boniface Date: Thu, 13 Mar 2025 00:10:26 -0400 Subject: [PATCH] Massively simplify imports and API client --- client-cli/pvc/cli/cli.py | 9 + client-cli/pvc/lib/common.py | 354 +++++++++++++++++++---------- client-cli/pvc/lib/lazy_imports.py | 21 ++ 3 files changed, 269 insertions(+), 115 deletions(-) create mode 100644 client-cli/pvc/lib/lazy_imports.py diff --git a/client-cli/pvc/cli/cli.py b/client-cli/pvc/cli/cli.py index 181171e2..6175d927 100644 --- a/client-cli/pvc/cli/cli.py +++ b/client-cli/pvc/cli/cli.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 +# Import only the essential parts of click for CLI definition +from click import group, command, option, argument, pass_context + +# Import minimal required modules +from pvc.lib.common import call_api + +# Use lazy imports for modules not needed during initial CLI parsing +from pvc.lib.lazy_imports import yaml, click_advanced + # cli.py - PVC Click CLI main library # Part of the Parallel Virtual Cluster (PVC) system # diff --git a/client-cli/pvc/lib/common.py b/client-cli/pvc/lib/common.py index a918a1e7..62d99449 100644 --- a/client-cli/pvc/lib/common.py +++ b/client-cli/pvc/lib/common.py @@ -23,10 +23,246 @@ from ast import literal_eval from click import echo, progressbar from math import ceil from os.path import getsize -from requests import get, post, put, patch, delete, Response -from requests.exceptions import ConnectionError from time import time -from urllib3 import disable_warnings +import json + +# Define a minimal HTTP client implementation that doesn't rely on http.client +# This avoids importing the heavy http module tree +def _make_http_request(host, port, is_https, method, path, headers, body, timeout, verify_ssl): + """ + Ultra-lightweight HTTP client implementation using raw sockets + """ + import socket + + if is_https: + # Only import ssl when needed for HTTPS + import ssl + + context = ssl.create_default_context() + if not verify_ssl: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + + try: + if is_https: + s = context.wrap_socket(s, server_hostname=host) + + s.connect((host, port)) + + # Construct HTTP request + request = f"{method} {path} HTTP/1.1\r\n" + request += f"Host: {host}\r\n" + + for key, value in headers.items(): + request += f"{key}: {value}\r\n" + + if body: + request += f"Content-Length: {len(body)}\r\n" + + request += "\r\n" + + # Send request + s.sendall(request.encode('utf-8')) + + if body: + s.sendall(body) + + # Read response + response_data = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + response_data += chunk + + # Parse response + response_parts = response_data.split(b'\r\n\r\n', 1) + headers_raw = response_parts[0].decode('utf-8') + body_raw = response_parts[1] if len(response_parts) > 1 else b'' + + # Parse status code + status_line = headers_raw.split('\r\n')[0] + status_code = int(status_line.split(' ')[1]) + + # Parse headers + headers_dict = {} + for line in headers_raw.split('\r\n')[1:]: + if not line: + continue + key, value = line.split(':', 1) + headers_dict[key.strip()] = value.strip() + + return status_code, headers_dict, body_raw + + finally: + s.close() + + +class ErrorResponse: + def __init__(self, json_data, status_code, headers): + self.status_code = status_code + self.headers = headers + self._json_data = json_data + self.content = json.dumps(json_data).encode('utf-8') if json_data else b'' + + def json(self): + return self._json_data + + +def call_api( + config, + operation, + request_uri, + headers={}, + params=None, + data=None, + files=None, +): + # Set the connect timeout to 2 seconds + timeout = 2.05 + + # Craft the URI + uri = "{}://{}{}{}".format( + config["api_scheme"], config["api_host"], config["api_prefix"], request_uri + ) + + # Parse the URI without using urllib.parse + scheme, rest = uri.split('://', 1) + netloc, path_and_query = rest.split('/', 1) + path_and_query = '/' + path_and_query + + # Extract host and port + if ':' in netloc: + host, port_str = netloc.split(':', 1) + port = int(port_str) + else: + host = netloc + port = 443 if scheme == 'https' else 80 + + # Craft the authentication header if required + if config["api_key"]: + headers["X-Api-Key"] = config["api_key"] + + # Add content type if not present + if "Content-Type" not in headers and data is not None and files is None: + headers["Content-Type"] = "application/json" + + # Prepare query string + query_string = "" + if params: + # Simple URL encoding without urllib.parse + query_parts = [] + for key, value in params.items(): + if isinstance(value, bool): + value = str(value).lower() + elif value is None: + value = '' + query_parts.append(f"{key}={value}") + query_string = "?" + "&".join(query_parts) + + # Prepare path with query string + full_path = path_and_query + query_string + + # Prepare body + body = None + if data is not None and files is None: + if isinstance(data, dict): + body = json.dumps(data).encode('utf-8') + else: + body = data.encode('utf-8') if isinstance(data, str) else data + + # Handle file uploads (multipart/form-data) + if files: + boundary = '----WebKitFormBoundary' + str(int(time())) + headers['Content-Type'] = f'multipart/form-data; boundary={boundary}' + + body = b'' + # Add form fields + if data: + for key, value in data.items(): + body += f'--{boundary}\r\n'.encode() + body += f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode() + body += f'{value}\r\n'.encode() + + # Add files + for key, file_tuple in files.items(): + filename, fileobj, content_type = file_tuple + body += f'--{boundary}\r\n'.encode() + body += f'Content-Disposition: form-data; name="{key}"; filename="{filename}"\r\n'.encode() + body += f'Content-Type: {content_type}\r\n\r\n'.encode() + body += fileobj.read() + body += b'\r\n' + + body += f'--{boundary}--\r\n'.encode() + + # Determine the request type and hit the API + response = None + + try: + # Special handling for GET with retries + if operation == "get": + retry_on_code = [429, 500, 502, 503, 504] + for i in range(3): + failed = False + try: + status_code, response_headers, response_data = _make_http_request( + host, port, scheme == 'https', operation.upper(), + full_path, headers, body, timeout, config["verify_ssl"] + ) + + if status_code in retry_on_code: + failed = True + continue + break + except Exception: + failed = True + continue + + if failed: + error = f"Code {status_code}" if 'status_code' in locals() else "Timeout" + raise Exception(f"Failed to connect after 3 tries ({error})") + else: + # Handle other HTTP methods + status_code, response_headers, response_data = _make_http_request( + host, port, scheme == 'https', operation.upper(), + full_path, headers, body, timeout, config["verify_ssl"] + ) + + # Create a Response-like object to match the requests library interface + class CustomResponse: + def __init__(self, status_code, headers, content): + self.status_code = status_code + self.headers = headers + self.content = content + self._json = None + + def json(self): + if self._json is None: + try: + self._json = json.loads(self.content.decode('utf-8')) + except json.JSONDecodeError: + self._json = {} + return self._json + + response = CustomResponse(status_code, response_headers, response_data) + + except Exception as e: + message = "Failed to connect to the API: {}".format(e) + code = getattr(response, 'status_code', 504) + response = ErrorResponse({"message": message}, code, None) + + # Display debug output + if config["debug"]: + echo("API endpoint: {}".format(uri), err=True) + echo("Response code: {}".format(response.status_code), err=True) + echo("Response headers: {}".format(response.headers), err=True) + echo(err=True) + + # Return the response object + return response def format_bytes(size_bytes): @@ -139,118 +375,6 @@ class UploadProgressBar(object): echo(self.end_message + self.end_suffix, nl=self.end_nl) -class ErrorResponse(Response): - def __init__(self, json_data, status_code, headers): - self.json_data = json_data - self.status_code = status_code - self.headers = headers - - def json(self): - return self.json_data - - -def call_api( - config, - operation, - request_uri, - headers={}, - params=None, - data=None, - files=None, -): - # Set the connect timeout to 2 seconds but extremely long (48 hour) data timeout - timeout = (2.05, 172800) - - # Craft the URI - uri = "{}://{}{}{}".format( - config["api_scheme"], config["api_host"], config["api_prefix"], request_uri - ) - - # Craft the authentication header if required - if config["api_key"]: - headers["X-Api-Key"] = config["api_key"] - - # Determine the request type and hit the API - disable_warnings() - try: - response = None - if operation == "get": - retry_on_code = [429, 500, 502, 503, 504] - for i in range(3): - failed = False - try: - response = get( - uri, - timeout=timeout, - headers=headers, - params=params, - data=data, - verify=config["verify_ssl"], - ) - if response.status_code in retry_on_code: - failed = True - continue - break - except ConnectionError: - failed = True - continue - if failed: - error = f"Code {response.status_code}" if response else "Timeout" - raise ConnectionError(f"Failed to connect after 3 tries ({error})") - if operation == "post": - response = post( - uri, - timeout=timeout, - headers=headers, - params=params, - data=data, - files=files, - verify=config["verify_ssl"], - ) - if operation == "put": - response = put( - uri, - timeout=timeout, - headers=headers, - params=params, - data=data, - files=files, - verify=config["verify_ssl"], - ) - if operation == "patch": - response = patch( - uri, - timeout=timeout, - headers=headers, - params=params, - data=data, - verify=config["verify_ssl"], - ) - if operation == "delete": - response = patch, delete( - uri, - timeout=timeout, - headers=headers, - params=params, - data=data, - verify=config["verify_ssl"], - ) - except Exception as e: - message = "Failed to connect to the API: {}".format(e) - code = response.status_code if response else 504 - response = ErrorResponse({"message": message}, code, None) - - # Display debug output - if config["debug"]: - echo("API endpoint: {}".format(uri), err=True) - echo("Response code: {}".format(response.status_code), err=True) - echo("Response headers: {}".format(response.headers), err=True) - echo(err=True) - - # Return the response object - return response - - def get_wait_retdata(response, wait_flag): if response.status_code == 202: retvalue = True diff --git a/client-cli/pvc/lib/lazy_imports.py b/client-cli/pvc/lib/lazy_imports.py new file mode 100644 index 00000000..edd3819e --- /dev/null +++ b/client-cli/pvc/lib/lazy_imports.py @@ -0,0 +1,21 @@ +""" +Lazy import mechanism for PVC CLI to reduce startup time +""" + +class LazyModule: + """ + A proxy for a module that is loaded only when actually used + """ + def __init__(self, name): + self.name = name + self._module = None + + def __getattr__(self, attr): + if self._module is None: + import importlib + self._module = importlib.import_module(self.name) + return getattr(self._module, attr) + +# Create lazy module proxies +yaml = LazyModule('yaml') +click_advanced = LazyModule('click') # For advanced click features not used at startup \ No newline at end of file