Move to lazy imports and custom API client

client-cli/pvc/lib/common.py

@@ -24,335 +24,10 @@ from click import echo, progressbar
 from math import ceil
 from os.path import getsize
 from time import time
-import json
 import socket
+import json
 import ssl
-
+import base64
-# Define a Response class to mimic requests.Response
-class Response:
-    def __init__(self, status_code, headers, content):
-        self.status_code = status_code
-        self.headers = headers
-        self.content = content
-        self._json = None
-
-    def json(self):
-        if self._json is None:
-            try:
-                self._json = json.loads(self.content.decode('utf-8'))
-            except json.JSONDecodeError:
-                self._json = {}
-        return self._json
-
-# Define ConnectionError to mimic requests.exceptions.ConnectionError
-class ConnectionError(Exception):
-    pass
-
-class ErrorResponse(Response):
-    def __init__(self, json_data, status_code, headers):
-        self.status_code = status_code
-        self.headers = headers
-        self._json = json_data
-        self.content = json.dumps(json_data).encode('utf-8') if json_data else b''
-
-    def json(self):
-        return self._json
-
-def _encode_params(params):
-    """Simple URL parameter encoder"""
-    if not params:
-        return ''
-
-    parts = []
-    for key, value in params.items():
-        if isinstance(value, bool):
-            value = str(value).lower()
-        elif value is None:
-            value = ''
-        else:
-            value = str(value)
-        parts.append(f"{key}={value}")
-
-    return '?' + '&'.join(parts)
-
-def call_api(
-    config,
-    operation,
-    request_uri,
-    headers={},
-    params=None,
-    data=None,
-    files=None,
-):
-    """
-    Make an API call to the PVC API using native Python libraries.
-    """
-    # Set the connect timeout to 2 seconds
-    timeout = 2.05
-
-    # Craft the URI
-    uri = "{}://{}{}{}".format(
-        config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
-    )
-
-    # Parse the URI without using urllib
-    if '://' in uri:
-        scheme, rest = uri.split('://', 1)
-    else:
-        scheme = 'http'
-        rest = uri
-
-    if '/' in rest:
-        netloc, path = rest.split('/', 1)
-        path = '/' + path
-    else:
-        netloc = rest
-        path = '/'
-
-    # Extract host and port
-    if ':' in netloc:
-        host, port_str = netloc.split(':', 1)
-        port = int(port_str)
-    else:
-        host = netloc
-        port = 443 if scheme == 'https' else 80
-
-    # Craft the authentication header if required
-    if config["api_key"]:
-        headers["X-Api-Key"] = config["api_key"]
-
-    # Add content type if not present
-    if "Content-Type" not in headers and data is not None and files is None:
-        headers["Content-Type"] = "application/json"
-
-    # Prepare query string
-    query_string = _encode_params(params)
-
-    # Prepare path with query string
-    full_path = path + query_string
-
-    # Prepare body
-    body = None
-    if data is not None and files is None:
-        if isinstance(data, dict):
-            body = json.dumps(data).encode('utf-8')
-        else:
-            body = data.encode('utf-8') if isinstance(data, str) else data
-
-    # Handle file uploads (multipart/form-data)
-    if files:
-        boundary = '----WebKitFormBoundary' + str(int(time()))
-        headers['Content-Type'] = f'multipart/form-data; boundary={boundary}'
-
-        body = b''
-        # Add form fields
-        if data:
-            for key, value in data.items():
-                body += f'--{boundary}\r\n'.encode()
-                body += f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode()
-                body += f'{value}\r\n'.encode()
-
-        # Add files
-        for key, file_tuple in files.items():
-            filename, fileobj, content_type = file_tuple
-            body += f'--{boundary}\r\n'.encode()
-            body += f'Content-Disposition: form-data; name="{key}"; filename="{filename}"\r\n'.encode()
-            body += f'Content-Type: {content_type}\r\n\r\n'.encode()
-            body += fileobj.read()
-            body += b'\r\n'
-
-        body += f'--{boundary}--\r\n'.encode()
-
-    # Determine the request type and hit the API
-    response = None
-
-    try:
-        # Special handling for GET with retries
-        if operation == "get":
-            retry_on_code = [429, 500, 502, 503, 504]
-            for i in range(3):
-                failed = False
-                try:
-                    # Create socket
-                    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                    s.settimeout(timeout)
-
-                    # Handle SSL for HTTPS
-                    if scheme == 'https':
-                        context = ssl.create_default_context()
-                        if not config["verify_ssl"]:
-                            context.check_hostname = False
-                            context.verify_mode = ssl.CERT_NONE
-                        s = context.wrap_socket(s, server_hostname=host)
-
-                    # Connect
-                    s.connect((host, port))
-
-                    # Build request
-                    request = f"{operation.upper()} {full_path} HTTP/1.1\r\n"
-                    request += f"Host: {host}\r\n"
-
-                    for key, value in headers.items():
-                        request += f"{key}: {value}\r\n"
-
-                    if body:
-                        request += f"Content-Length: {len(body)}\r\n"
-
-                    request += "Connection: close\r\n\r\n"
-
-                    # Send request
-                    s.sendall(request.encode('utf-8'))
-                    if body:
-                        s.sendall(body)
-
-                    # Read response
-                    response_data = b""
-                    while True:
-                        try:
-                            chunk = s.recv(4096)
-                            if not chunk:
-                                break
-                            response_data += chunk
-                        except socket.timeout:
-                            # If we've received some data but timed out, that's okay
-                            if response_data:
-                                break
-                            else:
-                                raise
-
-                    # Parse response
-                    if b'\r\n\r\n' in response_data:
-                        header_end = response_data.find(b'\r\n\r\n')
-                        headers_raw = response_data[:header_end].decode('utf-8')
-                        body_raw = response_data[header_end + 4:]
-
-                        # Parse status code
-                        status_line = headers_raw.split('\r\n')[0]
-                        status_code = int(status_line.split(' ')[1])
-
-                        # Parse headers
-                        headers_dict = {}
-                        for line in headers_raw.split('\r\n')[1:]:
-                            if not line:
-                                continue
-                            if ':' in line:
-                                key, value = line.split(':', 1)
-                                headers_dict[key.strip()] = value.strip()
-
-                        # Create response object
-                        response = Response(status_code, headers_dict, body_raw)
-
-                        if response.status_code in retry_on_code:
-                            failed = True
-                            s.close()
-                            continue
-                        break
-                    else:
-                        failed = True
-                        s.close()
-                        continue
-
-                except Exception as e:
-                    failed = True
-                    if 's' in locals():
-                        s.close()
-                    continue
-                finally:
-                    if 's' in locals():
-                        s.close()
-
-            if failed:
-                error = f"Code {response.status_code}" if response else "Timeout"
-                raise ConnectionError(f"Failed to connect after 3 tries ({error})")
-        else:
-            # Create socket
-            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            s.settimeout(timeout)
-
-            try:
-                # Handle SSL for HTTPS
-                if scheme == 'https':
-                    context = ssl.create_default_context()
-                    if not config["verify_ssl"]:
-                        context.check_hostname = False
-                        context.verify_mode = ssl.CERT_NONE
-                    s = context.wrap_socket(s, server_hostname=host)
-
-                # Connect
-                s.connect((host, port))
-
-                # Build request
-                request = f"{operation.upper()} {full_path} HTTP/1.1\r\n"
-                request += f"Host: {host}\r\n"
-
-                for key, value in headers.items():
-                    request += f"{key}: {value}\r\n"
-
-                if body:
-                    request += f"Content-Length: {len(body)}\r\n"
-
-                request += "Connection: close\r\n\r\n"
-
-                # Send request
-                s.sendall(request.encode('utf-8'))
-                if body:
-                    s.sendall(body)
-
-                # Read response
-                response_data = b""
-                while True:
-                    try:
-                        chunk = s.recv(4096)
-                        if not chunk:
-                            break
-                        response_data += chunk
-                    except socket.timeout:
-                        # If we've received some data but timed out, that's okay
-                        if response_data:
-                            break
-                        else:
-                            raise
-
-                # Parse response
-                if b'\r\n\r\n' in response_data:
-                    header_end = response_data.find(b'\r\n\r\n')
-                    headers_raw = response_data[:header_end].decode('utf-8')
-                    body_raw = response_data[header_end + 4:]
-
-                    # Parse status code
-                    status_line = headers_raw.split('\r\n')[0]
-                    status_code = int(status_line.split(' ')[1])
-
-                    # Parse headers
-                    headers_dict = {}
-                    for line in headers_raw.split('\r\n')[1:]:
-                        if not line:
-                            continue
-                        if ':' in line:
-                            key, value = line.split(':', 1)
-                            headers_dict[key.strip()] = value.strip()
-
-                    # Create response object
-                    response = Response(status_code, headers_dict, body_raw)
-                else:
-                    raise Exception("Invalid HTTP response")
-            finally:
-                s.close()
-
-    except Exception as e:
-        message = f"Failed to connect to the API: {e}"
-        code = getattr(response, 'status_code', 504)
-        response = ErrorResponse({"message": message}, code, None)
-
-    # Display debug output
-    if config["debug"]:
-        echo("API endpoint: {}".format(uri), err=True)
-        echo("Response code: {}".format(response.status_code), err=True)
-        echo("Response headers: {}".format(response.headers), err=True)
-        echo(err=True)
-
-    # Return the response object
-    return response
 
 
 def format_bytes(size_bytes):
@@ -465,6 +140,294 @@ class UploadProgressBar(object):
                 echo(self.end_message + self.end_suffix, nl=self.end_nl)
 
 
+class Response:
+    """Minimal Response class to replace requests.Response"""
+    def __init__(self, status_code, headers, content):
+        self.status_code = status_code
+        self.headers = headers
+        self.content = content
+        self._json = None
+    
+    def json(self):
+        if self._json is None:
+            try:
+                self._json = json.loads(self.content.decode('utf-8'))
+            except json.JSONDecodeError:
+                self._json = {}
+        return self._json
+
+
+class ConnectionError(Exception):
+    """Simple ConnectionError class to replace requests.exceptions.ConnectionError"""
+    pass
+
+
+class ErrorResponse(Response):
+    def __init__(self, json_data, status_code, headers):
+        self.status_code = status_code
+        self.headers = headers
+        self._json = json_data
+        self.content = json.dumps(json_data).encode('utf-8') if json_data else b''
+    
+    def json(self):
+        return self._json
+
+
+def _parse_url(url):
+    """Simple URL parser without using urllib"""
+    if '://' in url:
+        scheme, rest = url.split('://', 1)
+    else:
+        scheme = 'http'
+        rest = url
+    
+    if '/' in rest:
+        host_port, path = rest.split('/', 1)
+        path = '/' + path
+    else:
+        host_port = rest
+        path = '/'
+    
+    if ':' in host_port:
+        host, port_str = host_port.split(':', 1)
+        port = int(port_str)
+    else:
+        host = host_port
+        port = 443 if scheme == 'https' else 80
+    
+    return scheme, host, port, path
+
+
+def _encode_params(params):
+    """Simple URL parameter encoder"""
+    if not params:
+        return ''
+    
+    parts = []
+    for key, value in params.items():
+        if isinstance(value, bool):
+            value = str(value).lower()
+        elif value is None:
+            value = ''
+        else:
+            value = str(value)
+        parts.append(f"{key}={value}")
+    
+    return '?' + '&'.join(parts)
+
+
+def _make_request(method, url, headers=None, params=None, data=None, files=None, timeout=5, verify=True):
+    """Simple HTTP client using sockets"""
+    headers = headers or {}
+    scheme, host, port, path = _parse_url(url)
+    
+    # Add query parameters
+    path += _encode_params(params)
+    
+    # Prepare body
+    body = None
+    if data is not None and files is None:
+        if isinstance(data, dict):
+            body = json.dumps(data).encode('utf-8')
+            if 'Content-Type' not in headers:
+                headers['Content-Type'] = 'application/json'
+        else:
+            body = data.encode('utf-8') if isinstance(data, str) else data
+    
+    # Handle file uploads
+    if files:
+        boundary = f'----WebKitFormBoundary{int(time())}'
+        headers['Content-Type'] = f'multipart/form-data; boundary={boundary}'
+        
+        body = b''
+        # Add form fields
+        if data:
+            for key, value in data.items():
+                body += f'--{boundary}\r\n'.encode()
+                body += f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode()
+                body += f'{value}\r\n'.encode()
+        
+        # Add files
+        for key, file_tuple in files.items():
+            filename, fileobj, content_type = file_tuple
+            body += f'--{boundary}\r\n'.encode()
+            body += f'Content-Disposition: form-data; name="{key}"; filename="{filename}"\r\n'.encode()
+            body += f'Content-Type: {content_type}\r\n\r\n'.encode()
+            body += fileobj.read()
+            body += b'\r\n'
+        
+        body += f'--{boundary}--\r\n'.encode()
+    
+    # Create socket
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.settimeout(timeout)
+    
+    try:
+        # Handle SSL for HTTPS
+        if scheme == 'https':
+            context = ssl.create_default_context()
+            if not verify:
+                context.check_hostname = False
+                context.verify_mode = ssl.CERT_NONE
+            s = context.wrap_socket(s, server_hostname=host)
+        
+        # Connect
+        s.connect((host, port))
+        
+        # Build request
+        request = f"{method} {path} HTTP/1.1\r\n"
+        request += f"Host: {host}\r\n"
+        
+        for key, value in headers.items():
+            request += f"{key}: {value}\r\n"
+        
+        if body:
+            request += f"Content-Length: {len(body)}\r\n"
+        
+        request += "Connection: close\r\n\r\n"
+        
+        # Send request
+        s.sendall(request.encode('utf-8'))
+        if body:
+            s.sendall(body)
+        
+        # Read response
+        response_data = b""
+        while True:
+            chunk = s.recv(4096)
+            if not chunk:
+                break
+            response_data += chunk
+        
+        # Parse response
+        header_end = response_data.find(b'\r\n\r\n')
+        headers_raw = response_data[:header_end].decode('utf-8')
+        body_raw = response_data[header_end + 4:]
+        
+        # Parse status code
+        status_line = headers_raw.split('\r\n')[0]
+        status_code = int(status_line.split(' ')[1])
+        
+        # Parse headers
+        headers_dict = {}
+        for line in headers_raw.split('\r\n')[1:]:
+            if not line:
+                continue
+            key, value = line.split(':', 1)
+            headers_dict[key.strip()] = value.strip()
+        
+        return Response(status_code, headers_dict, body_raw)
+    
+    finally:
+        s.close()
+
+
+def call_api(
+    config,
+    operation,
+    request_uri,
+    headers={},
+    params=None,
+    data=None,
+    files=None,
+):
+    # Set the connect timeout to 2 seconds but extremely long (48 hour) data timeout
+    timeout = 2.05
+
+    # Craft the URI
+    uri = "{}://{}{}{}".format(
+        config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
+    )
+
+    # Craft the authentication header if required
+    if config["api_key"]:
+        headers["X-Api-Key"] = config["api_key"]
+
+    # Determine the request type and hit the API
+    try:
+        response = None
+        if operation == "get":
+            retry_on_code = [429, 500, 502, 503, 504]
+            for i in range(3):
+                failed = False
+                try:
+                    response = _make_request(
+                        "GET", 
+                        uri, 
+                        headers=headers, 
+                        params=params, 
+                        data=data, 
+                        timeout=timeout, 
+                        verify=config["verify_ssl"]
+                    )
+                    if response.status_code in retry_on_code:
+                        failed = True
+                        continue
+                    break
+                except Exception:
+                    failed = True
+                    continue
+            if failed:
+                error = f"Code {response.status_code}" if response else "Timeout"
+                raise ConnectionError(f"Failed to connect after 3 tries ({error})")
+        elif operation == "post":
+            response = _make_request(
+                "POST", 
+                uri, 
+                headers=headers, 
+                params=params, 
+                data=data, 
+                files=files, 
+                timeout=timeout, 
+                verify=config["verify_ssl"]
+            )
+        elif operation == "put":
+            response = _make_request(
+                "PUT", 
+                uri, 
+                headers=headers, 
+                params=params, 
+                data=data, 
+                files=files, 
+                timeout=timeout, 
+                verify=config["verify_ssl"]
+            )
+        elif operation == "patch":
+            response = _make_request(
+                "PATCH", 
+                uri, 
+                headers=headers, 
+                params=params, 
+                data=data, 
+                timeout=timeout, 
+                verify=config["verify_ssl"]
+            )
+        elif operation == "delete":
+            response = _make_request(
+                "DELETE", 
+                uri, 
+                headers=headers, 
+                params=params, 
+                data=data, 
+                timeout=timeout, 
+                verify=config["verify_ssl"]
+            )
+    except Exception as e:
+        message = "Failed to connect to the API: {}".format(e)
+        code = response.status_code if response else 504
+        response = ErrorResponse({"message": message}, code, None)
+
+    # Display debug output
+    if config["debug"]:
+        echo("API endpoint: {}".format(uri), err=True)
+        echo("Response code: {}".format(response.status_code), err=True)
+        echo("Response headers: {}".format(response.headers), err=True)
+        echo(err=True)
+
+    # Return the response object
+    return response
+
+
 def get_wait_retdata(response, wait_flag):
     if response.status_code == 202:
         retvalue = True

client-cli/pvc/lib/lazy_imports.py

@@ -9,13 +9,9 @@ class LazyModule:
     def __init__(self, name):
         self.name = name
         self._module = None
-
+    
     def __getattr__(self, attr):
         if self._module is None:
             import importlib
             self._module = importlib.import_module(self.name)
-        return getattr(self._module, attr)
+        return getattr(self._module, attr) 
-
-# Create lazy module proxies
-yaml = LazyModule('yaml')
-click_advanced = LazyModule('click')  # For advanced click features not used at startup

debian/control

@@ -5,7 +5,6 @@ Maintainer: Joshua Boniface <joshua@boniface.me>
 Standards-Version: 3.9.8
 Homepage: https://www.boniface.me
 X-Python3-Version: >= 3.7
-Build-Depends: pybuild-plugin-pyproject, dh-python
 
 Package: pvc-daemon-node
 Architecture: all

debian/rules

@@ -14,7 +14,7 @@ override_dh_python3:
 
 override_dh_auto_clean:
 	find $(CURDIR) -name "__pycache__" -o -name ".pybuild" -exec rm -fr {} + || true
-	rm -r $(CURDIR)/client-cli/build || true
+	rm -r $(CURDIR)/client-cli/build
 
 # If you need to rebuild the Sphinx documentation
 # Add spinxdoc to the dh --with line