2023-02-22 13:08:54 -05:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
# psql.py - PVC Monitoring example plugin for Postgres/Patroni
|
|
|
|
# Part of the Parallel Virtual Cluster (PVC) system
|
|
|
|
#
|
2023-12-29 11:16:59 -05:00
|
|
|
# Copyright (C) 2018-2024 Joshua M. Boniface <joshua@boniface.me>
|
2023-02-22 13:08:54 -05:00
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
# the Free Software Foundation, version 3.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
#
|
|
|
|
###############################################################################
|
|
|
|
|
|
|
|
# This script provides an example of a PVC monitoring plugin script. It will create
|
|
|
|
# a simple plugin to check the Patroni PostgreSQL instance on the node for operation.
|
|
|
|
|
|
|
|
# This script can thus be used as an example or reference implementation of a
|
|
|
|
# PVC monitoring pluginscript and expanded upon as required.
|
|
|
|
|
|
|
|
# A monitoring plugin script must implement the class "MonitoringPluginScript" which
|
|
|
|
# extends "MonitoringPlugin", providing the 3 functions indicated. Detailed explanation
|
|
|
|
# of the role of each function is provided in context of the example; see the other
|
|
|
|
# examples for more potential uses.
|
|
|
|
|
|
|
|
# WARNING:
|
|
|
|
#
|
|
|
|
# This script will run in the context of the node daemon keepalives as root.
|
|
|
|
# DO NOT install untrusted, unvetted plugins under any circumstances.
|
|
|
|
|
|
|
|
|
|
|
|
# This import is always required here, as MonitoringPlugin is used by the
|
|
|
|
# MonitoringPluginScript class
|
2023-11-29 15:36:49 -05:00
|
|
|
from pvchealthd.objects.MonitoringInstance import MonitoringPlugin
|
2023-02-22 13:08:54 -05:00
|
|
|
|
|
|
|
|
|
|
|
# A monitoring plugin script must always expose its nice name, which must be identical to
|
|
|
|
# the file name
|
|
|
|
PLUGIN_NAME = "psql"
|
|
|
|
|
|
|
|
|
|
|
|
# The MonitoringPluginScript class must be named as such, and extend MonitoringPlugin.
|
|
|
|
class MonitoringPluginScript(MonitoringPlugin):
|
|
|
|
def setup(self):
|
|
|
|
"""
|
|
|
|
setup(): Perform special setup steps during node daemon startup
|
|
|
|
|
|
|
|
This step is optional and should be used sparingly.
|
|
|
|
"""
|
|
|
|
|
2023-12-07 15:07:59 -05:00
|
|
|
# Prepare the last coordinator state
|
|
|
|
self.last_coordinator_state = None
|
2023-02-22 13:08:54 -05:00
|
|
|
|
2023-09-15 16:27:41 -04:00
|
|
|
def run(self, coordinator_state=None):
|
2023-02-22 13:08:54 -05:00
|
|
|
"""
|
|
|
|
run(): Perform the check actions and return a PluginResult object
|
2023-09-15 16:27:41 -04:00
|
|
|
|
2023-09-15 16:51:04 -04:00
|
|
|
The {coordinator_state} can be used to check if this is a "primary" coordinator, "secondary" coordinator, or "client" (non-coordinator)
|
2023-02-22 13:08:54 -05:00
|
|
|
"""
|
|
|
|
|
|
|
|
# Run any imports first
|
|
|
|
from psycopg2 import connect
|
2023-12-07 11:14:16 -05:00
|
|
|
from json import loads as jloads
|
|
|
|
from daemon_lib.common import run_os_command
|
2023-02-22 13:08:54 -05:00
|
|
|
|
2023-12-01 03:58:21 -05:00
|
|
|
conn_api = None
|
|
|
|
cur_api = None
|
2023-02-22 13:08:54 -05:00
|
|
|
conn_dns = None
|
|
|
|
cur_dns = None
|
|
|
|
|
|
|
|
# Set the health delta to 0 (no change)
|
|
|
|
health_delta = 0
|
|
|
|
# Craft a message that can be used by the clients
|
|
|
|
message = "Successfully connected to PostgreSQL databases on localhost"
|
|
|
|
|
2023-12-07 11:14:16 -05:00
|
|
|
# Check the API database
|
2023-02-22 13:08:54 -05:00
|
|
|
try:
|
2023-12-01 03:58:21 -05:00
|
|
|
conn_api = connect(
|
2023-02-22 13:45:21 -05:00
|
|
|
host=self.this_node.name,
|
2023-12-01 03:58:21 -05:00
|
|
|
port=self.config["api_postgresql_port"],
|
|
|
|
dbname=self.config["api_postgresql_dbname"],
|
|
|
|
user=self.config["api_postgresql_user"],
|
|
|
|
password=self.config["api_postgresql_password"],
|
2023-02-22 13:08:54 -05:00
|
|
|
)
|
2023-12-01 03:58:21 -05:00
|
|
|
cur_api = conn_api.cursor()
|
|
|
|
cur_api.execute("""SELECT * FROM alembic_version""")
|
|
|
|
data = cur_api.fetchone()
|
2023-02-22 13:08:54 -05:00
|
|
|
except Exception as e:
|
|
|
|
health_delta = 50
|
|
|
|
err = str(e).split('\n')[0]
|
2023-12-01 03:58:21 -05:00
|
|
|
message = f"Failed to connect to PostgreSQL database {self.config['api_postgresql_dbname']}: {err}"
|
2023-02-22 13:08:54 -05:00
|
|
|
finally:
|
2023-12-01 03:58:21 -05:00
|
|
|
if cur_api is not None:
|
|
|
|
cur_api.close()
|
|
|
|
if conn_api is not None:
|
|
|
|
conn_api.close()
|
2023-02-22 13:08:54 -05:00
|
|
|
|
2023-12-07 11:14:16 -05:00
|
|
|
# Check for Patroni status
|
|
|
|
_, stdout, _ = run_os_command("patronictl --config-file /etc/patroni/config.yml list --format json")
|
|
|
|
patronictl_status = jloads(stdout)
|
|
|
|
this_node_patronictl_status = [p for p in patronictl_status if p["Member"] == self.this_node.name][0]
|
2023-12-07 21:36:05 -05:00
|
|
|
self.logger.out(f"{this_node_patronictl_status}, last node state: {self.last_coordinator_state}, current node state: {coordinator_state}", state="d")
|
2023-12-07 11:14:16 -05:00
|
|
|
|
2023-12-07 15:07:59 -05:00
|
|
|
# Invalid state, nothing returned; this is a fault
|
2023-12-07 11:14:16 -05:00
|
|
|
if health_delta == 0 and not this_node_patronictl_status:
|
|
|
|
health_delta = 10
|
|
|
|
message = "Unable to determine Patroni PostgreSQL node state"
|
2023-12-07 15:07:59 -05:00
|
|
|
# We want to check for a non-running Patroni, but not during or immediately after a coordinator
|
|
|
|
# transition. So we wait until 2 runs with the same coordinator state have been completed.
|
|
|
|
elif health_delta == 0 and self.last_coordinator_state == coordinator_state and this_node_patronictl_status["State"] != "running":
|
2023-12-07 11:14:16 -05:00
|
|
|
health_delta = 10
|
2023-12-07 11:19:26 -05:00
|
|
|
message = "Patroni PostgreSQL state is not running"
|
2023-12-07 11:14:16 -05:00
|
|
|
|
|
|
|
# Handle some exceptional cases
|
|
|
|
if health_delta > 0:
|
2023-12-07 11:19:26 -05:00
|
|
|
if coordinator_state in ["takeover", "relinquish"]:
|
2023-12-07 11:23:25 -05:00
|
|
|
# This scenario occurrs if this plugin run catches a node transitioning from primary to
|
|
|
|
# secondary coordinator. We can ignore it.
|
2023-12-07 11:14:16 -05:00
|
|
|
health_delta = 0
|
|
|
|
message = "Patroni PostgreSQL error reported but currently transitioning coordinator state; ignoring."
|
2023-02-22 13:08:54 -05:00
|
|
|
|
|
|
|
# Set the health delta in our local PluginResult object
|
|
|
|
self.plugin_result.set_health_delta(health_delta)
|
|
|
|
|
|
|
|
# Set the message in our local PluginResult object
|
|
|
|
self.plugin_result.set_message(message)
|
2023-12-07 11:14:16 -05:00
|
|
|
|
2023-12-07 15:07:59 -05:00
|
|
|
# Update the last coordinator state
|
|
|
|
self.last_coordinator_state = coordinator_state
|
|
|
|
|
2023-02-22 13:08:54 -05:00
|
|
|
# Return our local PluginResult object
|
|
|
|
return self.plugin_result
|
|
|
|
|
|
|
|
def cleanup(self):
|
|
|
|
"""
|
|
|
|
cleanup(): Perform special cleanup steps during node daemon termination
|
|
|
|
|
|
|
|
This step is optional and should be used sparingly.
|
|
|
|
"""
|
|
|
|
|
|
|
|
pass
|