Revert "Switch to ZK+PG over Redis for Celery queue"

This reverts commit 54215bab6c.

api-daemon/pvcapid.sample.yaml

@@ -45,8 +45,8 @@ pvc:
     provisioner:
         # database: Backend database configuration
         database:
-            # host: PostgreSQL hostname, the primary coordinator floating IP in the cluster network
+            # host: PostgreSQL hostname, usually 'localhost'
-            host: pvchvP
+            host: localhost
             # port: PostgreSQL port, invariably '5432'
             port: 5432
             # name: PostgreSQL database name, invariably 'pvcapi'
@@ -55,6 +55,14 @@ pvc:
             user: pvcapi
             # pass: PostgreSQL user password, randomly generated
             pass: pvcapi
+        # queue: Celery backend queue using the PVC Zookeeper cluster
+        queue:
+            # host: Redis hostname, usually 'localhost'
+            host: localhost
+            # port: Redis port, invariably '6279'
+            port: 6379
+            # path: Redis queue path, invariably '/0'
+            path: /0
         # ceph_cluster: Information about the Ceph storage cluster
         ceph_cluster:
             # storage_hosts: The list of hosts that the Ceph monitors are valid on; if empty (the default),

api-daemon/pvcapid/Daemon.py

@@ -90,6 +90,9 @@ try:
         "database_name": o_config["pvc"]["provisioner"]["database"]["name"],
         "database_user": o_config["pvc"]["provisioner"]["database"]["user"],
         "database_password": o_config["pvc"]["provisioner"]["database"]["pass"],
+        "queue_host": o_config["pvc"]["provisioner"]["queue"]["host"],
+        "queue_port": o_config["pvc"]["provisioner"]["queue"]["port"],
+        "queue_path": o_config["pvc"]["provisioner"]["queue"]["path"],
         "storage_hosts": o_config["pvc"]["provisioner"]["ceph_cluster"][
             "storage_hosts"
         ],

api-daemon/pvcapid/flaskapi.py

@@ -43,23 +43,11 @@ from flask_sqlalchemy import SQLAlchemy
 
 # Create Flask app and set config values
 app = flask.Flask(__name__)
-
+app.config["CELERY_BROKER_URL"] = "redis://{}:{}{}".format(
-# Set up Celery queue
+    config["queue_host"], config["queue_port"], config["queue_path"]
-# Default Zookeeper port; not configurable
-queue_port = 2181
-# Default Zookeeper path, not configurable
-queue_path = "/apibroker"
-# Join the coordinator hostnames with the queue port, semicolon separated for Celery
-queue_hostport_pairs = ";".join([f"{h}:{queue_port}" for h in config["coordinators"]])
-app.config["CELERY_BROKER_URL"] = "zookeeper://{}{}".format(
-    queue_hostport_pairs, queue_path
 )
-app.config["CELERY_RESULT_BACKEND"] = "db+postgresql://{}:{}@{}:{}/{}".format(
+app.config["CELERY_RESULT_BACKEND"] = "redis://{}:{}{}".format(
-    config["database_user"],
+    config["queue_host"], config["queue_port"], config["queue_path"]
-    config["database_password"],
-    config["database_host"],
-    config["database_port"],
-    config["database_name"],
 )
 
 # Set up Celery queues
@@ -111,13 +99,11 @@ app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql://{}:{}@{}:{}/{}".format(
     config["database_name"],
 )
 
-# Set up debugging
 if config["debug"]:
     app.config["DEBUG"] = True
 else:
     app.config["DEBUG"] = False
 
-# Set up authentication
 if config["auth_enabled"]:
     app.config["SECRET_KEY"] = config["auth_secret_key"]
 

debian/control

@@ -16,7 +16,7 @@ Description: Parallel Virtual Cluster node daemon (Python 3)
 
 Package: pvc-daemon-api
 Architecture: all
-Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-celery, python-celery-common, python3-distutils, python3-lxml, python3-flask-migrate, fio
+Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, fio
 Description: Parallel Virtual Cluster API daemon (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .