parallelvirtualcluster/pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 115 Wiki Activity

Move config parser into daemon_lib

Browse Source 
And reformat/add config values for API.
This commit is contained in:
Joshua Boniface
2023-11-29 23:35:12 -05:00
parent 4df5fdbca6
commit 03a738f878
11 changed files with 109 additions and 902 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
api-daemon/pvcapid/Daemon.py
View File

@ -19,13 +19,13 @@
#
###############################################################################
import os
import yaml
from ssl import SSLContext, TLSVersion
from distutils.util import strtobool as dustrtobool
import daemon_lib.config as cfg
# Daemon version
version = "0.9.82"
@ -53,160 +53,13 @@ def strtobool(stringv):
# Configuration Parsing
##########################################################
# Parse the configuration file
config_file = None
try:
_config_file = "/etc/pvc/pvcapid.yaml"
if not os.path.exists(_config_file):
raise
config_file = _config_file
config_type = "legacy"
except Exception:
pass
try:
_config_file = os.environ["PVC_CONFIG_FILE"]
if not os.path.exists(_config_file):
raise
config_file = _config_file
config_type = "current"
except Exception:
pass
if not config_file:
print(
'Error: The "PVC_CONFIG_FILE" environment variable must be set before starting pvcapid.'
)
exit(1)
# Get our configuration
config = cfg.get_configuration()
config["daemon_name"] = "pvcapid"
config["daemon_version"] = version
def load_configuration_file(config_file):
print('Loading configuration from file "{}"'.format(config_file))
# Read in the config
try:
with open(config_file, "r") as cfgfile:
o_config = yaml.load(cfgfile, Loader=yaml.BaseLoader)
except Exception as e:
print("ERROR: Failed to parse configuration file: {}".format(e))
exit(1)
return o_config
def get_configuration_current(config_file):
o_config = load_configuration_file(config_file)
try:
# Create the config object
config = {
"debug": strtobool(o_config["logging"].get("debug_logging", "False")),
"all_nodes": o_config["cluster"]["all_nodes"],
"coordinators": o_config["cluster"]["coordinator_nodes"],
"listen_address": o_config["api"]["listen"]["address"],
"listen_port": int(o_config["api"]["listen"]["port"]),
"auth_enabled": strtobool(
o_config["api"]["authentication"].get("enabled", "False")
),
"auth_secret_key": o_config["api"]["authentication"]["secret_key"],
"auth_source": o_config["api"]["authentication"]["source"],
"ssl_enabled": strtobool(o_config["api"]["ssl"].get("enabled", "False")),
"ssl_cert_file": o_config["api"]["ssl"]["certificate"],
"ssl_key_file": o_config["api"]["ssl"]["private_key"],
"database_port": o_config["database"]["postgres"]["port"],
"database_host": o_config["database"]["postgres"]["hostname"],
"database_name": o_config["database"]["postgres"]["credentials"]["api"][
"database"
],
"database_user": o_config["database"]["postgres"]["credentials"]["api"][
"username"
],
"database_password": o_config["database"]["postgres"]["credentials"]["api"][
"password"
],
"queue_port": o_config["database"]["keydb"]["port"],
"queue_host": o_config["database"]["keydb"]["hostname"],
"queue_path": o_config["database"]["keydb"]["path"],
"storage_domain": o_config["cluster"]["networks"]["storage"]["domain"],
"storage_hosts": o_config["ceph"].get("monitor_hosts", None),
"ceph_monitor_port": o_config["ceph"]["monitor_port"],
"ceph_storage_secret_uuid": o_config["ceph"]["secret_uuid"],
}
# Use coordinators as storage hosts if not explicitly specified
if not config["storage_hosts"] or len(config["storage_hosts"]) < 1:
config["storage_hosts"] = config["coordinators"]
# Set up our token list if specified
if config["auth_source"] == "token":
config["auth_tokens"] = o_config["api"]["token"]
else:
if config["auth_enabled"]:
print(
"WARNING: No authentication method provided; disabling authentication."
)
config["auth_enabled"] = False
except Exception as e:
print(f"ERROR: Failed to load configuration: {e}")
exit(1)
return config
def get_configuration_legacy(config_file):
o_config = load_configuration_file(config_file)
try:
# Create the config object
config = {
"debug": strtobool(o_config["pvc"]["debug"]),
"coordinators": o_config["pvc"]["coordinators"],
"listen_address": o_config["pvc"]["api"]["listen_address"],
"listen_port": int(o_config["pvc"]["api"]["listen_port"]),
"auth_enabled": strtobool(
o_config["pvc"]["api"]["authentication"]["enabled"]
),
"auth_secret_key": o_config["pvc"]["api"]["authentication"]["secret_key"],
"auth_tokens": o_config["pvc"]["api"]["authentication"]["tokens"],
"ssl_enabled": strtobool(o_config["pvc"]["api"]["ssl"]["enabled"]),
"ssl_key_file": o_config["pvc"]["api"]["ssl"]["key_file"],
"ssl_cert_file": o_config["pvc"]["api"]["ssl"]["cert_file"],
"database_host": o_config["pvc"]["provisioner"]["database"]["host"],
"database_port": int(o_config["pvc"]["provisioner"]["database"]["port"]),
"database_name": o_config["pvc"]["provisioner"]["database"]["name"],
"database_user": o_config["pvc"]["provisioner"]["database"]["user"],
"database_password": o_config["pvc"]["provisioner"]["database"]["pass"],
"queue_host": o_config["pvc"]["provisioner"]["queue"]["host"],
"queue_port": o_config["pvc"]["provisioner"]["queue"]["port"],
"queue_path": o_config["pvc"]["provisioner"]["queue"]["path"],
"storage_hosts": o_config["pvc"]["provisioner"]["ceph_cluster"][
"storage_hosts"
],
"storage_domain": o_config["pvc"]["provisioner"]["ceph_cluster"][
"storage_domain"
],
"ceph_monitor_port": o_config["pvc"]["provisioner"]["ceph_cluster"][
"ceph_monitor_port"
],
"ceph_storage_secret_uuid": o_config["pvc"]["provisioner"]["ceph_cluster"][
"ceph_storage_secret_uuid"
],
}
# Use coordinators as storage hosts if not explicitly specified
if not config["storage_hosts"]:
config["storage_hosts"] = config["coordinators"]
except Exception as e:
print("ERROR: Failed to load configuration: {}".format(e))
exit(1)
return config
if config_type == "legacy":
config = get_configuration_legacy(config_file)
else:
config = get_configuration_current(config_file)
##########################################################
# Entrypoint
##########################################################
@ -215,11 +68,13 @@ else:
def entrypoint():
import pvcapid.flaskapi as pvc_api # noqa: E402
if config["ssl_enabled"]:
if config["api_ssl_enabled"]:
context = SSLContext()
context.minimum_version = TLSVersion.TLSv1
context.get_ca_certs()
context.load_cert_chain(config["ssl_cert_file"], keyfile=config["ssl_key_file"])
context.load_cert_chain(
config["api_ssl_cert_file"], keyfile=config["api_ssl_key_file"]
)
else:
context = None
@ -238,18 +93,18 @@ def entrypoint():
print("| API version: v{0: <46} |".format(API_VERSION))
print(
"| Listen: {0: <52} |".format(
"{}:{}".format(config["listen_address"], config["listen_port"])
"{}:{}".format(config["api_listen_address"], config["api_listen_port"])
)
)
print("| SSL: {0: <55} |".format(str(config["ssl_enabled"])))
print("| Authentication: {0: <44} |".format(str(config["auth_enabled"])))
print("| SSL: {0: <55} |".format(str(config["api_ssl_enabled"])))
print("| Authentication: {0: <44} |".format(str(config["api_auth_enabled"])))
print("|--------------------------------------------------------------|")
print("")
pvc_api.celery_startup()
pvc_api.app.run(
config["listen_address"],
config["listen_port"],
config["api_listen_address"],
config["api_listen_port"],
threaded=True,
ssl_context=context,
)

10
api-daemon/pvcapid/benchmark.py
View File

@ -135,11 +135,11 @@ def cleanup(job_name, db_conn=None, db_cur=None, zkhandler=None):
# Database connections
def open_database(config):
conn = psycopg2.connect(
host=config["database_host"],
port=config["database_port"],
dbname=config["database_name"],
user=config["database_user"],
password=config["database_password"],
host=config["api_postgresql_host"],
port=config["api_postgresql_port"],
dbname=config["api_postgresql_name"],
user=config["api_postgresql_user"],
password=config["api_postgresql_password"],
)
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
return conn, cur

26
api-daemon/pvcapid/flaskapi.py
View File

@ -61,11 +61,11 @@ app = flask.Flask(__name__)
# Set up SQLAlchemy backend
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql://{}:{}@{}:{}/{}".format(
config["database_user"],
config["database_password"],
config["database_host"],
config["database_port"],
config["database_name"],
config["api_postgresql_user"],
config["api_postgresql_password"],
config["api_postgresql_host"],
config["api_postgresql_port"],
config["api_postgresql_dbname"],
)
if config["debug"]:
@ -73,8 +73,8 @@ if config["debug"]:
else:
app.config["DEBUG"] = False
if config["auth_enabled"]:
app.config["SECRET_KEY"] = config["auth_secret_key"]
if config["api_auth_enabled"]:
app.config["SECRET_KEY"] = config["api_auth_secret_key"]
# Create SQLAlchemy database
db = SQLAlchemy(app)
@ -133,7 +133,7 @@ def run_celery_task(task_def, **kwargs):
# Create celery definition
celery_task_uri = "redis://{}:{}{}".format(
config["queue_host"], config["queue_port"], config["queue_path"]
config["keydb_host"], config["keydb_port"], config["keydb_path"]
)
celery = Celery(
app.name,
@ -199,7 +199,7 @@ def Authenticator(function):
@wraps(function)
def authenticate(*args, **kwargs):
# No authentication required
if not config["auth_enabled"]:
if not config["api_auth_enabled"]:
return function(*args, **kwargs)
# Session-based authentication
if "token" in flask.session:
@ -208,7 +208,7 @@ def Authenticator(function):
if "X-Api-Key" in flask.request.headers:
if any(
token
for token in config["auth_tokens"]
for token in config["api_auth_tokens"]
if flask.request.headers.get("X-Api-Key") == token.get("token")
):
return function(*args, **kwargs)
@ -469,12 +469,12 @@ class API_Login(Resource):
type: object
id: Message
"""
if not config["auth_enabled"]:
if not config["api_auth_enabled"]:
return flask.redirect(Api.url_for(api, API_Root))
if any(
token
for token in config["auth_tokens"]
for token in config["api_auth_tokens"]
if flask.request.values["token"] in token["token"]
):
flask.session["token"] = flask.request.form["token"]
@ -503,7 +503,7 @@ class API_Logout(Resource):
302:
description: Authentication disabled
"""
if not config["auth_enabled"]:
if not config["api_auth_enabled"]:
return flask.redirect(Api.url_for(api, API_Root))
flask.session.pop("token", None)

10
api-daemon/pvcapid/ova.py
View File

@ -48,11 +48,11 @@ import pvcapid.provisioner as provisioner
# Database connections
def open_database(config):
conn = psycopg2.connect(
host=config["database_host"],
port=config["database_port"],
dbname=config["database_name"],
user=config["database_user"],
password=config["database_password"],
host=config["api_postgresql_host"],
port=config["api_postgresql_port"],
dbname=config["api_postgresql_name"],
user=config["api_postgresql_user"],
password=config["api_postgresql_password"],
)
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
return conn, cur

10
api-daemon/pvcapid/provisioner.py
View File

@ -63,11 +63,11 @@ class ProvisioningError(Exception):
# Database connections
def open_database(config):
conn = psycopg2.connect(
host=config["database_host"],
port=config["database_port"],
dbname=config["database_name"],
user=config["database_user"],
password=config["database_password"],
host=config["api_postgresql_host"],
port=config["api_postgresql_port"],
dbname=config["api_postgresql_dbname"],
user=config["api_postgresql_user"],
password=config["api_postgresql_password"],
)
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
return conn, cur

16
api-daemon/pvcapid/vmbuilder.py
View File

@ -167,11 +167,11 @@ def chroot(destination):
def open_db(config):
try:
conn = psycopg2.connect(
host=config["database_host"],
port=config["database_port"],
dbname=config["database_name"],
user=config["database_user"],
password=config["database_password"],
host=config["api_postgresql_host"],
port=config["api_postgresql_port"],
dbname=config["api_postgresql_name"],
user=config["api_postgresql_user"],
password=config["api_postgresql_password"],
)
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
except Exception:
@ -326,9 +326,9 @@ def create_vm(
vm_data["system_architecture"] = stdout.strip()
monitor_list = list()
coordinator_names = config["storage_hosts"]
for coordinator in coordinator_names:
monitor_list.append("{}.{}".format(coordinator, config["storage_domain"]))
monitor_names = config["storage_hosts"]
for monitor in monitor_names:
monitor_list.append("{}.{}".format(monitor, config["storage_domain"]))
vm_data["ceph_monitor_list"] = monitor_list
vm_data["ceph_monitor_port"] = config["ceph_monitor_port"]
vm_data["ceph_monitor_secret"] = config["ceph_storage_secret_uuid"]