Optimize CLI client dependency loading

The CLI client was quite heavy in loading in a lot of libraries and
modules during runtime, which slowed it down quite a bit, especially on
slower systems.

This commit makes several major changes to help improve the situation.

1. Don't use pkg_resources to get our version, just hardcode it.

2. Reimplement our entire API call to use a custom http.client-based
system that prevents importing any unnecessary libraries (with a
custom User-Agent too).

3. Implement a lazy-loading method for some of the heavier modules, so
that they are only loaded if absolutely necessary.

bump-version

@@ -22,7 +22,8 @@ sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," node-d
 sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," health-daemon/pvchealthd/Daemon.py
 sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," worker-daemon/pvcworkerd/Daemon.py
 sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," api-daemon/pvcapid/Daemon.py
-sed -i "s,version=\"${current_version}\",version=\"${new_version}\"," client-cli/setup.py
+sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," client-cli/pyproject.toml
+sed -i "s,VERSION = \"${current_version}\",VERSION = \"${new_version}\"," client-cli/pvc/cli/cli.py
 echo ${new_version} > .version
 
 changelog_tmpdir=$( mktemp -d )

client-cli/pvc/cli/cli.py

@@ -1,14 +1,5 @@
 #!/usr/bin/env python3
 
-# Import only the essential parts of click for CLI definition
-from click import group, command, option, argument, pass_context
-
-# Import minimal required modules
-from pvc.lib.common import call_api
-
-# Use lazy imports for modules not needed during initial CLI parsing
-from pvc.lib.lazy_imports import yaml, click_advanced
-
 # cli.py - PVC Click CLI main library
 # Part of the Parallel Virtual Cluster (PVC) system
 #
@@ -28,6 +19,17 @@ from pvc.lib.lazy_imports import yaml, click_advanced
 #
 ###############################################################################
 
+VERSION = "0.9.107"
+
+# Import only the essential parts of click for CLI definition
+from click import group, command, option, argument, pass_context
+
+# Import minimal required modules
+from pvc.lib.common import call_api
+
+# Use lazy imports for modules not needed during initial CLI parsing
+from pvc.lib.lazy_imports import yaml, click_advanced
+
 from colorama import Fore
 from difflib import unified_diff
 from functools import wraps
@@ -106,10 +108,7 @@ def version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
 
-    from pkg_resources import get_distribution
+    echo(CLI_CONFIG, f"Parallel Virtual Cluster CLI client version {VERSION}")
-
-    version = get_distribution("pvc").version
-    echo(CLI_CONFIG, f"Parallel Virtual Cluster CLI client version {version}")
     ctx.exit()
 
 

client-cli/pvc/lib/common.py

@@ -35,15 +35,25 @@ class Response:
         self.headers = headers
         self.content = content
         self._json = None
+        self.text = content.decode('utf-8', errors='replace') if content else ''
+        self.ok = 200 <= status_code < 300
+        self.url = None  # Will be set by call_api
+        self.reason = None  # HTTP reason phrase
+        self.encoding = 'utf-8'
+        self.elapsed = None  # Time elapsed
+        self.request = None  # Original request
 
     def json(self):
         if self._json is None:
-            try:
+            # Don't catch JSONDecodeError - let it propagate like requests does
             self._json = json.loads(self.content.decode('utf-8'))
-            except json.JSONDecodeError:
-                self._json = {}
         return self._json
 
+    def raise_for_status(self):
+        """Raises HTTPError if the status code is 4XX/5XX"""
+        if 400 <= self.status_code < 600:
+            raise Exception(f"HTTP Error {self.status_code}")
+
 # Define ConnectionError to mimic requests.exceptions.ConnectionError
 class ConnectionError(Exception):
     pass
@@ -69,6 +79,11 @@ def _encode_params(params):
             value = str(value).lower()
         elif value is None:
             value = ''
+        elif isinstance(value, (list, tuple)):
+            # Handle lists and tuples by creating multiple parameters with the same name
+            for item in value:
+                parts.append(f"{key}={item}")
+            continue  # Skip the normal append since we've already added the parts
         else:
             value = str(value)
         parts.append(f"{key}={value}")
@@ -87,8 +102,16 @@ def call_api(
     """
     Make an API call to the PVC API using native Python libraries.
     """
-    # Set the connect timeout to 2 seconds
+    # Set timeouts - fast connection timeout, longer read timeout
-    timeout = 2.05
+    connect_timeout = 2.05  # Connection timeout in seconds
+    read_timeout = 172800.0 # Read timeout in seconds (much longer to allow for slow operations)
+
+    # Import VERSION from cli.py - this is a lightweight import since cli.py is already loaded
+    from pvc.cli.cli import VERSION
+
+    # Set User-Agent header if not already set
+    if "User-Agent" not in headers:
+        headers["User-Agent"] = f"pvc-client-cli/{VERSION}"
 
     # Craft the URI
     uri = "{}://{}{}{}".format(
@@ -163,9 +186,7 @@ def call_api(
 
         body += f'--{boundary}--\r\n'.encode()
 
-    # Determine the request type and hit the API
+    # Use http.client instead of raw sockets for better HTTP protocol handling
-    response = None
-
     try:
         # Special handling for GET with retries
         if operation == "get":
@@ -173,175 +194,103 @@ def call_api(
             for i in range(3):
                 failed = False
                 try:
-                    # Create socket
+                    # Create the appropriate connection with separate timeouts
-                    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                    s.settimeout(timeout)
-
-                    # Handle SSL for HTTPS
                     if scheme == 'https':
-                        context = ssl.create_default_context()
+                        import ssl
+                        context = None
                         if not config["verify_ssl"]:
-                            context.check_hostname = False
+                            context = ssl._create_unverified_context()
-                            context.verify_mode = ssl.CERT_NONE
-                        s = context.wrap_socket(s, server_hostname=host)
 
-                    # Connect
+                        import http.client
-                    s.connect((host, port))
+                        conn = http.client.HTTPSConnection(
-
+                            host, 
-                    # Build request
+                            port=port,
-                    request = f"{operation.upper()} {full_path} HTTP/1.1\r\n"
+                            timeout=connect_timeout,  # This is the connection timeout
-                    request += f"Host: {host}\r\n"
+                            context=context
-
+                        )
-                    for key, value in headers.items():
-                        request += f"{key}: {value}\r\n"
-
-                    if body:
-                        request += f"Content-Length: {len(body)}\r\n"
-
-                    request += "Connection: close\r\n\r\n"
-
-                    # Send request
-                    s.sendall(request.encode('utf-8'))
-                    if body:
-                        s.sendall(body)
-
-                    # Read response
-                    response_data = b""
-                    while True:
-                        try:
-                            chunk = s.recv(4096)
-                            if not chunk:
-                                break
-                            response_data += chunk
-                        except socket.timeout:
-                            # If we've received some data but timed out, that's okay
-                            if response_data:
-                                break
                     else:
-                                raise
+                        import http.client
+                        conn = http.client.HTTPConnection(
+                            host,
+                            port=port,
+                            timeout=connect_timeout  # This is the connection timeout
+                        )
 
-                    # Parse response
+                    # Make the request
-                    if b'\r\n\r\n' in response_data:
+                    conn.request(operation.upper(), full_path, body=body, headers=headers)
-                        header_end = response_data.find(b'\r\n\r\n')
-                        headers_raw = response_data[:header_end].decode('utf-8')
-                        body_raw = response_data[header_end + 4:]
 
-                        # Parse status code
+                    # Set a longer timeout for reading the response
-                        status_line = headers_raw.split('\r\n')[0]
+                    conn.sock.settimeout(read_timeout)
-                        status_code = int(status_line.split(' ')[1])
 
-                        # Parse headers
+                    http_response = conn.getresponse()
-                        headers_dict = {}
-                        for line in headers_raw.split('\r\n')[1:]:
-                            if not line:
-                                continue
-                            if ':' in line:
-                                key, value = line.split(':', 1)
-                                headers_dict[key.strip()] = value.strip()
 
-                        # Create response object
+                    # Read response data
-                        response = Response(status_code, headers_dict, body_raw)
+                    status_code = http_response.status
+                    response_headers = dict(http_response.getheaders())
+                    response_data = http_response.read()
+
+                    conn.close()
+
+                    # Create a Response object
+                    response = Response(status_code, response_headers, response_data)
 
                     if response.status_code in retry_on_code:
                         failed = True
-                            s.close()
                         continue
                     break
-                    else:
-                        failed = True
-                        s.close()
-                        continue
-
                 except Exception as e:
                     failed = True
-                    if 's' in locals():
+                    if 'conn' in locals():
-                        s.close()
+                        conn.close()
                     continue
-                finally:
-                    if 's' in locals():
-                        s.close()
 
             if failed:
-                error = f"Code {response.status_code}" if response else "Timeout"
+                error = f"Code {response.status_code}" if 'response' in locals() else "Timeout"
                 raise ConnectionError(f"Failed to connect after 3 tries ({error})")
         else:
-            # Create socket
+            # Handle other HTTP methods
-            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            s.settimeout(timeout)
-
-            try:
-                # Handle SSL for HTTPS
             if scheme == 'https':
-                    context = ssl.create_default_context()
+                import ssl
+                context = None
                 if not config["verify_ssl"]:
-                        context.check_hostname = False
+                    context = ssl._create_unverified_context()
-                        context.verify_mode = ssl.CERT_NONE
-                    s = context.wrap_socket(s, server_hostname=host)
 
-                # Connect
+                import http.client
-                s.connect((host, port))
+                conn = http.client.HTTPSConnection(
-
+                    host, 
-                # Build request
+                    port=port,
-                request = f"{operation.upper()} {full_path} HTTP/1.1\r\n"
+                    timeout=connect_timeout,  # This is the connection timeout
-                request += f"Host: {host}\r\n"
+                    context=context
-
+                )
-                for key, value in headers.items():
-                    request += f"{key}: {value}\r\n"
-
-                if body:
-                    request += f"Content-Length: {len(body)}\r\n"
-
-                request += "Connection: close\r\n\r\n"
-
-                # Send request
-                s.sendall(request.encode('utf-8'))
-                if body:
-                    s.sendall(body)
-
-                # Read response
-                response_data = b""
-                while True:
-                    try:
-                        chunk = s.recv(4096)
-                        if not chunk:
-                            break
-                        response_data += chunk
-                    except socket.timeout:
-                        # If we've received some data but timed out, that's okay
-                        if response_data:
-                            break
             else:
-                            raise
+                import http.client
+                conn = http.client.HTTPConnection(
+                    host,
+                    port=port,
+                    timeout=connect_timeout  # This is the connection timeout
+                )
 
-                # Parse response
+            # Make the request
-                if b'\r\n\r\n' in response_data:
+            conn.request(operation.upper(), full_path, body=body, headers=headers)
-                    header_end = response_data.find(b'\r\n\r\n')
-                    headers_raw = response_data[:header_end].decode('utf-8')
-                    body_raw = response_data[header_end + 4:]
 
-                    # Parse status code
+            # Set a longer timeout for reading the response
-                    status_line = headers_raw.split('\r\n')[0]
+            conn.sock.settimeout(read_timeout)
-                    status_code = int(status_line.split(' ')[1])
 
-                    # Parse headers
+            http_response = conn.getresponse()
-                    headers_dict = {}
-                    for line in headers_raw.split('\r\n')[1:]:
-                        if not line:
-                            continue
-                        if ':' in line:
-                            key, value = line.split(':', 1)
-                            headers_dict[key.strip()] = value.strip()
 
-                    # Create response object
+            # Read response data
-                    response = Response(status_code, headers_dict, body_raw)
+            status_code = http_response.status
-                else:
+            response_headers = dict(http_response.getheaders())
-                    raise Exception("Invalid HTTP response")
+            response_data = http_response.read()
-            finally:
+
-                s.close()
+            conn.close()
+
+            # Create a Response object
+            response = Response(status_code, response_headers, response_data)
 
     except Exception as e:
         message = f"Failed to connect to the API: {e}"
-        code = getattr(response, 'status_code', 504)
+        code = getattr(response, 'status_code', 504) if 'response' in locals() else 504
         response = ErrorResponse({"message": message}, code, None)
 
     # Display debug output
@@ -351,7 +300,7 @@ def call_api(
         echo("Response headers: {}".format(response.headers), err=True)
         echo(err=True)
 
-    # Return the response object
+    # Always return the response object - no special handling
     return response
 
 

client-cli/pvc/lib/lazy_imports.py

@@ -1,6 +1,23 @@
-"""
+#!/usr/bin/env python3
-Lazy import mechanism for PVC CLI to reduce startup time
+
-"""
+# lazy_imports.py - Lazy module importer library
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2024 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
 
 class LazyModule:
     """

debian/control

@@ -49,7 +49,7 @@ Description: Parallel Virtual Cluster common libraries
 
 Package: pvc-client-cli
 Architecture: all
-Depends: python3-requests, python3-requests-toolbelt, python3-yaml, python3-lxml, python3-click
+Depends: python3-requests-toolbelt, python3-yaml, python3-lxml, python3-click
 Description: Parallel Virtual Cluster CLI client
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .