From a5763c9d253c097652c24238bb1b6ac1cb73c7ce Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Thu, 11 Jan 2024 00:06:11 -0500 Subject: [PATCH] Fix possible race condition applying schemas Found an instance where two of these fired too close together, and caused a fatal error. Use a write lock, and then catch the schema.apply function in case it fails anyways. --- node-daemon/pvcnoded/Daemon.py | 4 +++- node-daemon/pvcnoded/util/zookeeper.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/node-daemon/pvcnoded/Daemon.py b/node-daemon/pvcnoded/Daemon.py index d5778738..48fb9927 100644 --- a/node-daemon/pvcnoded/Daemon.py +++ b/node-daemon/pvcnoded/Daemon.py @@ -197,7 +197,9 @@ def entrypoint(): os.execv(sys.argv[0], sys.argv) # Validate the schema - pvcnoded.util.zookeeper.validate_schema(logger, zkhandler) + with zkhandler.writelock("base.schema.version"): + sleep(0.5) + pvcnoded.util.zookeeper.validate_schema(logger, zkhandler) # Define a cleanup function def cleanup(failure=False): diff --git a/node-daemon/pvcnoded/util/zookeeper.py b/node-daemon/pvcnoded/util/zookeeper.py index 17e896ef..5afe04d9 100644 --- a/node-daemon/pvcnoded/util/zookeeper.py +++ b/node-daemon/pvcnoded/util/zookeeper.py @@ -94,7 +94,10 @@ def validate_schema(logger, zkhandler): # Validate our schema against the active version if not zkhandler.schema.validate(zkhandler, logger): logger.out("Found schema violations, applying", state="i") - zkhandler.schema.apply(zkhandler) + try: + zkhandler.schema.apply(zkhandler) + except Exception as e: + logger.out(f"Failed to apply schema updates: {e}", state="w") else: logger.out("Schema successfully validated", state="o")