Add cluster backup + restore functionality
Adds cluster backup (JSON dump) and restore functions for use in disaster recovery. Further, adds additional confirmation to the initialization (as well as restore) endpoints to avoid accidental triggering, and also groups the init, backup, and restore commands in the CLI into a new "task" subsection.
This commit is contained in:
parent
3f2c7293d1
commit
8f705c9cc2
|
|
@ -333,14 +333,23 @@ api.add_resource(API_Logout, '/logout')
|
|||
|
||||
# /initialize
|
||||
class API_Initialize(Resource):
|
||||
@RequestParser([
|
||||
{'name': 'yes-i-really-mean-it', 'required': True, 'helptext': "Initialization is destructive; please confirm with the argument 'yes-i-really-mean-it'."}
|
||||
])
|
||||
@Authenticator
|
||||
def post(self):
|
||||
def post(self, reqargs):
|
||||
"""
|
||||
Initialize a new PVC cluster
|
||||
Note: Normally used only once during cluster bootstrap; checks for the existence of the "/primary_node" key before proceeding and returns 400 if found
|
||||
---
|
||||
tags:
|
||||
- root
|
||||
parameters:
|
||||
- in: query
|
||||
name: yes-i-really-mean-it
|
||||
type: string
|
||||
required: true
|
||||
description: A confirmation string to ensure that the API consumer really means it
|
||||
responses:
|
||||
200:
|
||||
description: OK
|
||||
|
|
@ -363,6 +372,82 @@ class API_Initialize(Resource):
|
|||
api.add_resource(API_Initialize, '/initialize')
|
||||
|
||||
|
||||
# /backup
|
||||
class API_Backup(Resource):
|
||||
@Authenticator
|
||||
def get(self):
|
||||
"""
|
||||
Back up the Zookeeper data of a cluster in JSON format
|
||||
---
|
||||
tags:
|
||||
- root
|
||||
responses:
|
||||
200:
|
||||
description: OK
|
||||
schema:
|
||||
type: object
|
||||
id: Cluster Data
|
||||
400:
|
||||
description: Bad request
|
||||
"""
|
||||
return api_helper.backup_cluster()
|
||||
|
||||
|
||||
api.add_resource(API_Backup, '/backup')
|
||||
|
||||
|
||||
# /restore
|
||||
class API_Restore(Resource):
|
||||
@RequestParser([
|
||||
{'name': 'yes-i-really-mean-it', 'required': True, 'helptext': "Restore is destructive; please confirm with the argument 'yes-i-really-mean-it'."},
|
||||
{'name': 'cluster_data', 'required': True, 'helptext': "A cluster JSON backup must be provided."}
|
||||
])
|
||||
@Authenticator
|
||||
def post(self, reqargs):
|
||||
"""
|
||||
Restore a backup over the cluster; destroys the existing data
|
||||
---
|
||||
tags:
|
||||
- root
|
||||
parameters:
|
||||
- in: query
|
||||
name: yes-i-really-mean-it
|
||||
type: string
|
||||
required: true
|
||||
description: A confirmation string to ensure that the API consumer really means it
|
||||
- in: query
|
||||
name: cluster_data
|
||||
type: string
|
||||
required: true
|
||||
description: The raw JSON cluster backup data
|
||||
responses:
|
||||
200:
|
||||
description: OK
|
||||
schema:
|
||||
type: object
|
||||
id: Message
|
||||
400:
|
||||
description: Bad request
|
||||
schema:
|
||||
type: object
|
||||
id: Message
|
||||
500:
|
||||
description: Restore error or code failure
|
||||
schema:
|
||||
type: object
|
||||
id: Message
|
||||
"""
|
||||
try:
|
||||
cluster_data = reqargs.get('cluster_data')
|
||||
except Exception as e:
|
||||
return {"message": "Failed to load JSON backup: {}.".format(e)}, 400
|
||||
|
||||
return api_helper.restore_cluster(cluster_data)
|
||||
|
||||
|
||||
api.add_resource(API_Restore, '/restore')
|
||||
|
||||
|
||||
# /status
|
||||
class API_Status(Resource):
|
||||
@Authenticator
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
###############################################################################
|
||||
|
||||
import flask
|
||||
import json
|
||||
import lxml.etree as etree
|
||||
|
||||
from distutils.util import strtobool as dustrtobool
|
||||
|
|
@ -49,7 +50,7 @@ def strtobool(stringv):
|
|||
|
||||
|
||||
#
|
||||
# Initialization function
|
||||
# Cluster base functions
|
||||
#
|
||||
def initialize_cluster():
|
||||
# Open a Zookeeper connection
|
||||
|
|
@ -86,6 +87,66 @@ def initialize_cluster():
|
|||
return True
|
||||
|
||||
|
||||
def backup_cluster():
|
||||
# Open a zookeeper connection
|
||||
zk_conn = pvc_common.startZKConnection(config['coordinators'])
|
||||
|
||||
# Dictionary of values to come
|
||||
cluster_data = dict()
|
||||
|
||||
def get_data(path):
|
||||
data_raw = zk_conn.get(path)
|
||||
if data_raw:
|
||||
data = data_raw[0].decode('utf8')
|
||||
children = zk_conn.get_children(path)
|
||||
|
||||
cluster_data[path] = data
|
||||
|
||||
if children:
|
||||
if path == '/':
|
||||
child_prefix = '/'
|
||||
else:
|
||||
child_prefix = path + '/'
|
||||
|
||||
for child in children:
|
||||
if child_prefix + child == '/zookeeper':
|
||||
# We must skip the built-in /zookeeper tree
|
||||
continue
|
||||
get_data(child_prefix + child)
|
||||
|
||||
get_data('/')
|
||||
|
||||
return cluster_data, 200
|
||||
|
||||
|
||||
def restore_cluster(cluster_data_raw):
|
||||
# Open a zookeeper connection
|
||||
zk_conn = pvc_common.startZKConnection(config['coordinators'])
|
||||
|
||||
# Open a single transaction (restore is atomic)
|
||||
zk_transaction = zk_conn.transaction()
|
||||
|
||||
try:
|
||||
cluster_data = json.loads(cluster_data_raw)
|
||||
except Exception as e:
|
||||
return {"message": "Failed to parse JSON data: {}.".format(e)}, 400
|
||||
|
||||
for key in cluster_data:
|
||||
data = cluster_data[key]
|
||||
|
||||
if zk_conn.exists(key):
|
||||
zk_transaction.set_data(key, str(data).encode('utf8'))
|
||||
else:
|
||||
zk_transaction.create(key, str(data).encode('utf8'))
|
||||
|
||||
try:
|
||||
zk_transaction.commit()
|
||||
return {'message': 'Restore completed successfully.'}, 200
|
||||
except Exception as e:
|
||||
raise
|
||||
return {'message': 'Restore failed: {}.'.format(e)}, 500
|
||||
|
||||
|
||||
#
|
||||
# Cluster functions
|
||||
#
|
||||
|
|
|
|||
|
|
@ -31,10 +31,55 @@ def initialize(config):
|
|||
Initialize the PVC cluster
|
||||
|
||||
API endpoint: GET /api/v1/initialize
|
||||
API arguments: yes-i-really-mean-it
|
||||
API schema: {json_data_object}
|
||||
"""
|
||||
params = {
|
||||
'yes-i-really-mean-it': 'yes'
|
||||
}
|
||||
response = call_api(config, 'post', '/initialize', params=params)
|
||||
|
||||
if response.status_code == 200:
|
||||
retstatus = True
|
||||
else:
|
||||
retstatus = False
|
||||
|
||||
return retstatus, response.json().get('message', '')
|
||||
|
||||
|
||||
def backup(config):
|
||||
"""
|
||||
Get a JSON backup of the cluster
|
||||
|
||||
API endpoint: GET /api/v1/backup
|
||||
API arguments:
|
||||
API schema: {json_data_object}
|
||||
"""
|
||||
response = call_api(config, 'post', '/initialize')
|
||||
response = call_api(config, 'get', '/backup')
|
||||
|
||||
if response.status_code == 200:
|
||||
return True, response.json()
|
||||
else:
|
||||
return False, response.json().get('message', '')
|
||||
|
||||
|
||||
def restore(config, cluster_data):
|
||||
"""
|
||||
Restore a JSON backup to the cluster
|
||||
|
||||
API endpoint: POST /api/v1/restore
|
||||
API arguments: yes-i-really-mean-it
|
||||
API schema: {json_data_object}
|
||||
"""
|
||||
cluster_data_json = json.dumps(cluster_data)
|
||||
|
||||
params = {
|
||||
'yes-i-really-mean-it': 'yes'
|
||||
}
|
||||
data = {
|
||||
'cluster_data': cluster_data_json
|
||||
}
|
||||
response = call_api(config, 'post', '/restore', params=params, data=data)
|
||||
|
||||
if response.status_code == 200:
|
||||
retstatus = True
|
||||
|
|
|
|||
|
|
@ -4078,6 +4078,71 @@ def status_cluster(oformat):
|
|||
cleanup(retcode, retdata)
|
||||
|
||||
|
||||
###############################################################################
|
||||
# pvc task
|
||||
###############################################################################
|
||||
@click.group(name='task', short_help='Perform PVC cluster tasks.', context_settings=CONTEXT_SETTINGS)
|
||||
def cli_task():
|
||||
"""
|
||||
Perform administrative tasks against the PVC cluster.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
###############################################################################
|
||||
# pvc task backup
|
||||
###############################################################################
|
||||
@click.command(name='backup', short_help='Create JSON backup of cluster.')
|
||||
@click.option(
|
||||
'-f', '--file', 'filename',
|
||||
default=None, type=click.File(),
|
||||
help='Write backup data to this file.'
|
||||
)
|
||||
@cluster_req
|
||||
def task_backup(filename):
|
||||
"""
|
||||
Create a JSON-format backup of the cluster Zookeeper database.
|
||||
"""
|
||||
|
||||
retcode, retdata = pvc_cluster.backup(config)
|
||||
if filename:
|
||||
with open(filename, 'wb') as fh:
|
||||
fh.write(retdata)
|
||||
retdata = 'Data written to {}'.format(filename)
|
||||
cleanup(retcode, retdata)
|
||||
|
||||
|
||||
###############################################################################
|
||||
# pvc task restore
|
||||
###############################################################################
|
||||
@click.command(name='restore', short_help='Restore JSON backup to cluster.')
|
||||
@click.option(
|
||||
'-f', '--file', 'filename',
|
||||
required=True, default=None, type=click.File(),
|
||||
help='Read backup data from this file.'
|
||||
)
|
||||
@click.option(
|
||||
'-y', '--yes', 'confirm_flag',
|
||||
is_flag=True, default=False,
|
||||
help='Confirm the restore'
|
||||
)
|
||||
@cluster_req
|
||||
def task_restore(filename, confirm_flag):
|
||||
"""
|
||||
Restore the JSON backup data from a file to the cluster.
|
||||
"""
|
||||
|
||||
if not confirm_flag:
|
||||
try:
|
||||
click.confirm('Replace all existing cluster data from coordinators with backup file "{}"'.format(filename.name), prompt_suffix='? ', abort=True)
|
||||
except Exception:
|
||||
exit(0)
|
||||
|
||||
cluster_data = json.loads(filename.read())
|
||||
retcode, retmsg = pvc_cluster.restore(config, cluster_data)
|
||||
cleanup(retcode, retmsg)
|
||||
|
||||
|
||||
###############################################################################
|
||||
# pvc init
|
||||
###############################################################################
|
||||
|
|
@ -4085,10 +4150,10 @@ def status_cluster(oformat):
|
|||
@click.option(
|
||||
'-y', '--yes', 'confirm_flag',
|
||||
is_flag=True, default=False,
|
||||
help='Confirm the removal'
|
||||
help='Confirm the initialization'
|
||||
)
|
||||
@cluster_req
|
||||
def init_cluster(confirm_flag):
|
||||
def task_init(confirm_flag):
|
||||
"""
|
||||
Perform initialization of a new PVC cluster.
|
||||
"""
|
||||
|
|
@ -4323,6 +4388,10 @@ cli_provisioner.add_command(provisioner_status)
|
|||
cli_maintenance.add_command(maintenance_on)
|
||||
cli_maintenance.add_command(maintenance_off)
|
||||
|
||||
cli_task.add_command(task_backup)
|
||||
cli_task.add_command(task_restore)
|
||||
cli_task.add_command(task_init)
|
||||
|
||||
cli.add_command(cli_cluster)
|
||||
cli.add_command(cli_node)
|
||||
cli.add_command(cli_vm)
|
||||
|
|
@ -4330,8 +4399,8 @@ cli.add_command(cli_network)
|
|||
cli.add_command(cli_storage)
|
||||
cli.add_command(cli_provisioner)
|
||||
cli.add_command(cli_maintenance)
|
||||
cli.add_command(cli_task)
|
||||
cli.add_command(status_cluster)
|
||||
cli.add_command(init_cluster)
|
||||
|
||||
|
||||
#
|
||||
|
|
|
|||
Loading…
Reference in New Issue