Add cluster backup + restore functionality

Adds cluster backup (JSON dump) and restore functions for use in
disaster recovery.

Further, adds additional confirmation to the initialization (as well as
restore) endpoints to avoid accidental triggering, and also groups the
init, backup, and restore commands in the CLI into a new "task"
subsection.
This commit is contained in:
Joshua Boniface 2020-11-24 02:39:06 -05:00
parent 3f2c7293d1
commit 8f705c9cc2
4 changed files with 266 additions and 6 deletions

View File

@ -333,14 +333,23 @@ api.add_resource(API_Logout, '/logout')
# /initialize
class API_Initialize(Resource):
@RequestParser([
{'name': 'yes-i-really-mean-it', 'required': True, 'helptext': "Initialization is destructive; please confirm with the argument 'yes-i-really-mean-it'."}
])
@Authenticator
def post(self):
def post(self, reqargs):
"""
Initialize a new PVC cluster
Note: Normally used only once during cluster bootstrap; checks for the existence of the "/primary_node" key before proceeding and returns 400 if found
---
tags:
- root
parameters:
- in: query
name: yes-i-really-mean-it
type: string
required: true
description: A confirmation string to ensure that the API consumer really means it
responses:
200:
description: OK
@ -363,6 +372,82 @@ class API_Initialize(Resource):
api.add_resource(API_Initialize, '/initialize')
# /backup
class API_Backup(Resource):
@Authenticator
def get(self):
"""
Back up the Zookeeper data of a cluster in JSON format
---
tags:
- root
responses:
200:
description: OK
schema:
type: object
id: Cluster Data
400:
description: Bad request
"""
return api_helper.backup_cluster()
api.add_resource(API_Backup, '/backup')
# /restore
class API_Restore(Resource):
@RequestParser([
{'name': 'yes-i-really-mean-it', 'required': True, 'helptext': "Restore is destructive; please confirm with the argument 'yes-i-really-mean-it'."},
{'name': 'cluster_data', 'required': True, 'helptext': "A cluster JSON backup must be provided."}
])
@Authenticator
def post(self, reqargs):
"""
Restore a backup over the cluster; destroys the existing data
---
tags:
- root
parameters:
- in: query
name: yes-i-really-mean-it
type: string
required: true
description: A confirmation string to ensure that the API consumer really means it
- in: query
name: cluster_data
type: string
required: true
description: The raw JSON cluster backup data
responses:
200:
description: OK
schema:
type: object
id: Message
400:
description: Bad request
schema:
type: object
id: Message
500:
description: Restore error or code failure
schema:
type: object
id: Message
"""
try:
cluster_data = reqargs.get('cluster_data')
except Exception as e:
return {"message": "Failed to load JSON backup: {}.".format(e)}, 400
return api_helper.restore_cluster(cluster_data)
api.add_resource(API_Restore, '/restore')
# /status
class API_Status(Resource):
@Authenticator

View File

@ -21,6 +21,7 @@
###############################################################################
import flask
import json
import lxml.etree as etree
from distutils.util import strtobool as dustrtobool
@ -49,7 +50,7 @@ def strtobool(stringv):
#
# Initialization function
# Cluster base functions
#
def initialize_cluster():
# Open a Zookeeper connection
@ -86,6 +87,66 @@ def initialize_cluster():
return True
def backup_cluster():
# Open a zookeeper connection
zk_conn = pvc_common.startZKConnection(config['coordinators'])
# Dictionary of values to come
cluster_data = dict()
def get_data(path):
data_raw = zk_conn.get(path)
if data_raw:
data = data_raw[0].decode('utf8')
children = zk_conn.get_children(path)
cluster_data[path] = data
if children:
if path == '/':
child_prefix = '/'
else:
child_prefix = path + '/'
for child in children:
if child_prefix + child == '/zookeeper':
# We must skip the built-in /zookeeper tree
continue
get_data(child_prefix + child)
get_data('/')
return cluster_data, 200
def restore_cluster(cluster_data_raw):
# Open a zookeeper connection
zk_conn = pvc_common.startZKConnection(config['coordinators'])
# Open a single transaction (restore is atomic)
zk_transaction = zk_conn.transaction()
try:
cluster_data = json.loads(cluster_data_raw)
except Exception as e:
return {"message": "Failed to parse JSON data: {}.".format(e)}, 400
for key in cluster_data:
data = cluster_data[key]
if zk_conn.exists(key):
zk_transaction.set_data(key, str(data).encode('utf8'))
else:
zk_transaction.create(key, str(data).encode('utf8'))
try:
zk_transaction.commit()
return {'message': 'Restore completed successfully.'}, 200
except Exception as e:
raise
return {'message': 'Restore failed: {}.'.format(e)}, 500
#
# Cluster functions
#

View File

@ -31,10 +31,55 @@ def initialize(config):
Initialize the PVC cluster
API endpoint: GET /api/v1/initialize
API arguments: yes-i-really-mean-it
API schema: {json_data_object}
"""
params = {
'yes-i-really-mean-it': 'yes'
}
response = call_api(config, 'post', '/initialize', params=params)
if response.status_code == 200:
retstatus = True
else:
retstatus = False
return retstatus, response.json().get('message', '')
def backup(config):
"""
Get a JSON backup of the cluster
API endpoint: GET /api/v1/backup
API arguments:
API schema: {json_data_object}
"""
response = call_api(config, 'post', '/initialize')
response = call_api(config, 'get', '/backup')
if response.status_code == 200:
return True, response.json()
else:
return False, response.json().get('message', '')
def restore(config, cluster_data):
"""
Restore a JSON backup to the cluster
API endpoint: POST /api/v1/restore
API arguments: yes-i-really-mean-it
API schema: {json_data_object}
"""
cluster_data_json = json.dumps(cluster_data)
params = {
'yes-i-really-mean-it': 'yes'
}
data = {
'cluster_data': cluster_data_json
}
response = call_api(config, 'post', '/restore', params=params, data=data)
if response.status_code == 200:
retstatus = True

View File

@ -4078,6 +4078,71 @@ def status_cluster(oformat):
cleanup(retcode, retdata)
###############################################################################
# pvc task
###############################################################################
@click.group(name='task', short_help='Perform PVC cluster tasks.', context_settings=CONTEXT_SETTINGS)
def cli_task():
"""
Perform administrative tasks against the PVC cluster.
"""
pass
###############################################################################
# pvc task backup
###############################################################################
@click.command(name='backup', short_help='Create JSON backup of cluster.')
@click.option(
'-f', '--file', 'filename',
default=None, type=click.File(),
help='Write backup data to this file.'
)
@cluster_req
def task_backup(filename):
"""
Create a JSON-format backup of the cluster Zookeeper database.
"""
retcode, retdata = pvc_cluster.backup(config)
if filename:
with open(filename, 'wb') as fh:
fh.write(retdata)
retdata = 'Data written to {}'.format(filename)
cleanup(retcode, retdata)
###############################################################################
# pvc task restore
###############################################################################
@click.command(name='restore', short_help='Restore JSON backup to cluster.')
@click.option(
'-f', '--file', 'filename',
required=True, default=None, type=click.File(),
help='Read backup data from this file.'
)
@click.option(
'-y', '--yes', 'confirm_flag',
is_flag=True, default=False,
help='Confirm the restore'
)
@cluster_req
def task_restore(filename, confirm_flag):
"""
Restore the JSON backup data from a file to the cluster.
"""
if not confirm_flag:
try:
click.confirm('Replace all existing cluster data from coordinators with backup file "{}"'.format(filename.name), prompt_suffix='? ', abort=True)
except Exception:
exit(0)
cluster_data = json.loads(filename.read())
retcode, retmsg = pvc_cluster.restore(config, cluster_data)
cleanup(retcode, retmsg)
###############################################################################
# pvc init
###############################################################################
@ -4085,10 +4150,10 @@ def status_cluster(oformat):
@click.option(
'-y', '--yes', 'confirm_flag',
is_flag=True, default=False,
help='Confirm the removal'
help='Confirm the initialization'
)
@cluster_req
def init_cluster(confirm_flag):
def task_init(confirm_flag):
"""
Perform initialization of a new PVC cluster.
"""
@ -4323,6 +4388,10 @@ cli_provisioner.add_command(provisioner_status)
cli_maintenance.add_command(maintenance_on)
cli_maintenance.add_command(maintenance_off)
cli_task.add_command(task_backup)
cli_task.add_command(task_restore)
cli_task.add_command(task_init)
cli.add_command(cli_cluster)
cli.add_command(cli_node)
cli.add_command(cli_vm)
@ -4330,8 +4399,8 @@ cli.add_command(cli_network)
cli.add_command(cli_storage)
cli.add_command(cli_provisioner)
cli.add_command(cli_maintenance)
cli.add_command(cli_task)
cli.add_command(status_cluster)
cli.add_command(init_cluster)
#