diff --git a/api-daemon/pvcapid/flaskapi.py b/api-daemon/pvcapid/flaskapi.py index 7e5fe26a..4896c3dc 100755 --- a/api-daemon/pvcapid/flaskapi.py +++ b/api-daemon/pvcapid/flaskapi.py @@ -333,14 +333,23 @@ api.add_resource(API_Logout, '/logout') # /initialize class API_Initialize(Resource): + @RequestParser([ + {'name': 'yes-i-really-mean-it', 'required': True, 'helptext': "Initialization is destructive; please confirm with the argument 'yes-i-really-mean-it'."} + ]) @Authenticator - def post(self): + def post(self, reqargs): """ Initialize a new PVC cluster Note: Normally used only once during cluster bootstrap; checks for the existence of the "/primary_node" key before proceeding and returns 400 if found --- tags: - root + parameters: + - in: query + name: yes-i-really-mean-it + type: string + required: true + description: A confirmation string to ensure that the API consumer really means it responses: 200: description: OK @@ -363,6 +372,82 @@ class API_Initialize(Resource): api.add_resource(API_Initialize, '/initialize') +# /backup +class API_Backup(Resource): + @Authenticator + def get(self): + """ + Back up the Zookeeper data of a cluster in JSON format + --- + tags: + - root + responses: + 200: + description: OK + schema: + type: object + id: Cluster Data + 400: + description: Bad request + """ + return api_helper.backup_cluster() + + +api.add_resource(API_Backup, '/backup') + + +# /restore +class API_Restore(Resource): + @RequestParser([ + {'name': 'yes-i-really-mean-it', 'required': True, 'helptext': "Restore is destructive; please confirm with the argument 'yes-i-really-mean-it'."}, + {'name': 'cluster_data', 'required': True, 'helptext': "A cluster JSON backup must be provided."} + ]) + @Authenticator + def post(self, reqargs): + """ + Restore a backup over the cluster; destroys the existing data + --- + tags: + - root + parameters: + - in: query + name: yes-i-really-mean-it + type: string + required: true + description: A confirmation string to ensure that the API consumer really means it + - in: query + name: cluster_data + type: string + required: true + description: The raw JSON cluster backup data + responses: + 200: + description: OK + schema: + type: object + id: Message + 400: + description: Bad request + schema: + type: object + id: Message + 500: + description: Restore error or code failure + schema: + type: object + id: Message + """ + try: + cluster_data = reqargs.get('cluster_data') + except Exception as e: + return {"message": "Failed to load JSON backup: {}.".format(e)}, 400 + + return api_helper.restore_cluster(cluster_data) + + +api.add_resource(API_Restore, '/restore') + + # /status class API_Status(Resource): @Authenticator diff --git a/api-daemon/pvcapid/helper.py b/api-daemon/pvcapid/helper.py index 8fb6594a..a8444d0c 100755 --- a/api-daemon/pvcapid/helper.py +++ b/api-daemon/pvcapid/helper.py @@ -21,6 +21,7 @@ ############################################################################### import flask +import json import lxml.etree as etree from distutils.util import strtobool as dustrtobool @@ -49,7 +50,7 @@ def strtobool(stringv): # -# Initialization function +# Cluster base functions # def initialize_cluster(): # Open a Zookeeper connection @@ -86,6 +87,66 @@ def initialize_cluster(): return True +def backup_cluster(): + # Open a zookeeper connection + zk_conn = pvc_common.startZKConnection(config['coordinators']) + + # Dictionary of values to come + cluster_data = dict() + + def get_data(path): + data_raw = zk_conn.get(path) + if data_raw: + data = data_raw[0].decode('utf8') + children = zk_conn.get_children(path) + + cluster_data[path] = data + + if children: + if path == '/': + child_prefix = '/' + else: + child_prefix = path + '/' + + for child in children: + if child_prefix + child == '/zookeeper': + # We must skip the built-in /zookeeper tree + continue + get_data(child_prefix + child) + + get_data('/') + + return cluster_data, 200 + + +def restore_cluster(cluster_data_raw): + # Open a zookeeper connection + zk_conn = pvc_common.startZKConnection(config['coordinators']) + + # Open a single transaction (restore is atomic) + zk_transaction = zk_conn.transaction() + + try: + cluster_data = json.loads(cluster_data_raw) + except Exception as e: + return {"message": "Failed to parse JSON data: {}.".format(e)}, 400 + + for key in cluster_data: + data = cluster_data[key] + + if zk_conn.exists(key): + zk_transaction.set_data(key, str(data).encode('utf8')) + else: + zk_transaction.create(key, str(data).encode('utf8')) + + try: + zk_transaction.commit() + return {'message': 'Restore completed successfully.'}, 200 + except Exception as e: + raise + return {'message': 'Restore failed: {}.'.format(e)}, 500 + + # # Cluster functions # diff --git a/client-cli/cli_lib/cluster.py b/client-cli/cli_lib/cluster.py index ad21f627..4b196861 100644 --- a/client-cli/cli_lib/cluster.py +++ b/client-cli/cli_lib/cluster.py @@ -31,10 +31,55 @@ def initialize(config): Initialize the PVC cluster API endpoint: GET /api/v1/initialize + API arguments: yes-i-really-mean-it + API schema: {json_data_object} + """ + params = { + 'yes-i-really-mean-it': 'yes' + } + response = call_api(config, 'post', '/initialize', params=params) + + if response.status_code == 200: + retstatus = True + else: + retstatus = False + + return retstatus, response.json().get('message', '') + + +def backup(config): + """ + Get a JSON backup of the cluster + + API endpoint: GET /api/v1/backup API arguments: API schema: {json_data_object} """ - response = call_api(config, 'post', '/initialize') + response = call_api(config, 'get', '/backup') + + if response.status_code == 200: + return True, response.json() + else: + return False, response.json().get('message', '') + + +def restore(config, cluster_data): + """ + Restore a JSON backup to the cluster + + API endpoint: POST /api/v1/restore + API arguments: yes-i-really-mean-it + API schema: {json_data_object} + """ + cluster_data_json = json.dumps(cluster_data) + + params = { + 'yes-i-really-mean-it': 'yes' + } + data = { + 'cluster_data': cluster_data_json + } + response = call_api(config, 'post', '/restore', params=params, data=data) if response.status_code == 200: retstatus = True diff --git a/client-cli/pvc.py b/client-cli/pvc.py index 57c92854..40571d25 100755 --- a/client-cli/pvc.py +++ b/client-cli/pvc.py @@ -4078,6 +4078,71 @@ def status_cluster(oformat): cleanup(retcode, retdata) +############################################################################### +# pvc task +############################################################################### +@click.group(name='task', short_help='Perform PVC cluster tasks.', context_settings=CONTEXT_SETTINGS) +def cli_task(): + """ + Perform administrative tasks against the PVC cluster. + """ + pass + + +############################################################################### +# pvc task backup +############################################################################### +@click.command(name='backup', short_help='Create JSON backup of cluster.') +@click.option( + '-f', '--file', 'filename', + default=None, type=click.File(), + help='Write backup data to this file.' +) +@cluster_req +def task_backup(filename): + """ + Create a JSON-format backup of the cluster Zookeeper database. + """ + + retcode, retdata = pvc_cluster.backup(config) + if filename: + with open(filename, 'wb') as fh: + fh.write(retdata) + retdata = 'Data written to {}'.format(filename) + cleanup(retcode, retdata) + + +############################################################################### +# pvc task restore +############################################################################### +@click.command(name='restore', short_help='Restore JSON backup to cluster.') +@click.option( + '-f', '--file', 'filename', + required=True, default=None, type=click.File(), + help='Read backup data from this file.' +) +@click.option( + '-y', '--yes', 'confirm_flag', + is_flag=True, default=False, + help='Confirm the restore' +) +@cluster_req +def task_restore(filename, confirm_flag): + """ + Restore the JSON backup data from a file to the cluster. + """ + + if not confirm_flag: + try: + click.confirm('Replace all existing cluster data from coordinators with backup file "{}"'.format(filename.name), prompt_suffix='? ', abort=True) + except Exception: + exit(0) + + cluster_data = json.loads(filename.read()) + retcode, retmsg = pvc_cluster.restore(config, cluster_data) + cleanup(retcode, retmsg) + + ############################################################################### # pvc init ############################################################################### @@ -4085,10 +4150,10 @@ def status_cluster(oformat): @click.option( '-y', '--yes', 'confirm_flag', is_flag=True, default=False, - help='Confirm the removal' + help='Confirm the initialization' ) @cluster_req -def init_cluster(confirm_flag): +def task_init(confirm_flag): """ Perform initialization of a new PVC cluster. """ @@ -4323,6 +4388,10 @@ cli_provisioner.add_command(provisioner_status) cli_maintenance.add_command(maintenance_on) cli_maintenance.add_command(maintenance_off) +cli_task.add_command(task_backup) +cli_task.add_command(task_restore) +cli_task.add_command(task_init) + cli.add_command(cli_cluster) cli.add_command(cli_node) cli.add_command(cli_vm) @@ -4330,8 +4399,8 @@ cli.add_command(cli_network) cli.add_command(cli_storage) cli.add_command(cli_provisioner) cli.add_command(cli_maintenance) +cli.add_command(cli_task) cli.add_command(status_cluster) -cli.add_command(init_cluster) #