parallelvirtualcluster/pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 115 Wiki Activity

Rebuild API using Flask-RESTful and Swagger docs

Browse Source
This commit is contained in:
Joshua Boniface
2019-12-23 20:43:20 -05:00
parent 91cb542e37
commit a6683d5b17
12 changed files with 10065 additions and 1981 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
client-api/api_lib/pvcapi_helper.py
View File

@@ -23,21 +23,57 @@
import flask
import json
from distutils.util import strtobool
import client_lib.common as pvc_common
import client_lib.node as pvc_node
import client_lib.vm as pvc_vm
import client_lib.network as pvc_network
import client_lib.ceph as pvc_ceph
#
# Initialization function
#
def initialize_cluster():
# Open a Zookeeper connection
zk_conn = pvc_common.startZKConnection(config['coordinators'])
# Abort if we've initialized the cluster before
if zk_conn.exists('/primary_node'):
return False
# Create the root keys
transaction = zk_conn.transaction()
transaction.create('/primary_node', 'none'.encode('ascii'))
transaction.create('/upstream_ip', 'none'.encode('ascii'))
transaction.create('/nodes', ''.encode('ascii'))
transaction.create('/domains', ''.encode('ascii'))
transaction.create('/networks', ''.encode('ascii'))
transaction.create('/ceph', ''.encode('ascii'))
transaction.create('/ceph/osds', ''.encode('ascii'))
transaction.create('/ceph/pools', ''.encode('ascii'))
transaction.create('/ceph/volumes', ''.encode('ascii'))
transaction.create('/ceph/snapshots', ''.encode('ascii'))
transaction.create('/cmd', ''.encode('ascii'))
transaction.create('/cmd/domains', ''.encode('ascii'))
transaction.create('/cmd/ceph', ''.encode('ascii'))
transaction.create('/locks', ''.encode('ascii'))
transaction.create('/locks/flush_lock', ''.encode('ascii'))
transaction.create('/locks/primary_node', ''.encode('ascii'))
transaction.commit()
# Close the Zookeeper connection
pvc_common.stopZKConnection(zk_conn)
#
# Node functions
#
def node_list(limit=None):
def node_list(limit=None, is_fuzzy=True):
"""
Return a list of nodes with limit LIMIT.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_node.get_list(zk_conn, limit)
retflag, retdata = pvc_node.get_list(zk_conn, limit, is_fuzzy=is_fuzzy)
if retflag:
if retdata:
retcode = 200
@@ -50,7 +86,12 @@ def node_list(limit=None):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
return retdata, retcode
def node_daemon_state(node):
"""
@@ -74,7 +115,7 @@ def node_daemon_state(node):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def node_coordinator_state(node):
"""
@@ -98,7 +139,7 @@ def node_coordinator_state(node):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def node_domain_state(node):
"""
@@ -122,7 +163,7 @@ def node_domain_state(node):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def node_secondary(node):
"""
@@ -139,7 +180,7 @@ def node_secondary(node):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def node_primary(node):
"""
@@ -156,7 +197,7 @@ def node_primary(node):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def node_flush(node):
"""
@@ -173,7 +214,7 @@ def node_flush(node):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def node_ready(node):
"""
@@ -190,7 +231,7 @@ def node_ready(node):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
#
# VM functions
@@ -209,12 +250,17 @@ def vm_state(vm):
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_vm.get_list(zk_conn, None, None, vm, is_fuzzy=False)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
retdata = {
'name': vm,
'state': retdata[0]['state']
'state': retdata['state']
}
else:
retcode = 404
@@ -225,7 +271,7 @@ def vm_state(vm):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def vm_node(vm):
"""
@@ -233,13 +279,18 @@ def vm_node(vm):
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_vm.get_list(zk_conn, None, None, vm, is_fuzzy=False)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
retdata = {
'name': vm,
'node': retdata[0]['node'],
'last_node': retdata[0]['last_node']
'node': retdata['node'],
'last_node': retdata['last_node']
}
else:
retcode = 404
@@ -250,7 +301,7 @@ def vm_node(vm):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def vm_list(node=None, state=None, limit=None, is_fuzzy=True):
"""
@@ -258,6 +309,11 @@ def vm_list(node=None, state=None, limit=None, is_fuzzy=True):
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_vm.get_list(zk_conn, node, state, limit, is_fuzzy)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
@@ -270,7 +326,8 @@ def vm_list(node=None, state=None, limit=None, is_fuzzy=True):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def vm_define(xml, node, limit, selector, autostart):
"""
@@ -287,14 +344,45 @@ def vm_define(xml, node, limit, selector, autostart):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_meta(vm, limit, selector, autostart):
def get_vm_meta(vm):
"""
Get metadata of a VM.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_vm.get_list(zk_conn, None, None, vm, is_fuzzy=False)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
retdata = {
'name': vm,
'node_limit': retdata['node_limit'],
'node_selector': retdata['node_selector'],
'node_autostart': retdata['node_autostart']
}
else:
retcode = 404
retdata = {
'message': 'VM not found.'
}
else:
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return retdata, retcode
def update_vm_meta(vm, limit, selector, autostart):
"""
Update metadata of a VM.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_vm.modify_vm_metadata(zk_conn, vm. limit, selector, autostart)
retflag, retdata = pvc_vm.modify_vm_metadata(zk_conn, vm. limit, selector, strtobool(autostart))
if retflag:
retcode = 200
else:
@@ -304,7 +392,7 @@ def vm_meta(vm, limit, selector, autostart):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_modify(name, restart, xml):
"""
@@ -321,7 +409,7 @@ def vm_modify(name, restart, xml):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_undefine(name):
"""
@@ -338,7 +426,7 @@ def vm_undefine(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_remove(name):
"""
@@ -355,7 +443,7 @@ def vm_remove(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_start(name):
"""
@@ -372,7 +460,7 @@ def vm_start(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_restart(name):
"""
@@ -389,7 +477,7 @@ def vm_restart(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_shutdown(name):
"""
@@ -406,7 +494,7 @@ def vm_shutdown(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_stop(name):
"""
@@ -423,7 +511,7 @@ def vm_stop(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_move(name, node):
"""
@@ -440,7 +528,7 @@ def vm_move(name, node):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_migrate(name, node, flag_force):
"""
@@ -457,7 +545,7 @@ def vm_migrate(name, node, flag_force):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_unmigrate(name):
"""
@@ -474,14 +562,23 @@ def vm_unmigrate(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def vm_flush_locks(name):
def vm_flush_locks(vm):
"""
Flush locks of a (stopped) VM.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_vm.flush_locks(zk_conn, name)
retflag, retdata = pvc_vm.get_list(zk_conn, None, None, vm, is_fuzzy=False)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retdata['state'] not in ['stop', 'disable']:
return {"message":"VM must be stopped to flush locks"}, 400
retflag, retdata = pvc_vm.flush_locks(zk_conn, vm)
if retflag:
retcode = 200
else:
@@ -491,17 +588,22 @@ def vm_flush_locks(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
#
# Network functions
#
def net_list(limit=None):
def net_list(limit=None, is_fuzzy=True):
"""
Return a list of client networks with limit LIMIT.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_network.get_list(zk_conn, limit)
retflag, retdata = pvc_network.get_list(zk_conn, limit, is_fuzzy)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
@@ -514,7 +616,7 @@ def net_list(limit=None):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def net_add(vni, description, nettype, domain, name_servers,
ip4_network, ip4_gateway, ip6_network, ip6_gateway,
@@ -535,7 +637,7 @@ def net_add(vni, description, nettype, domain, name_servers,
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def net_modify(vni, description, domain, name_servers,
ip4_network, ip4_gateway,
@@ -557,7 +659,7 @@ def net_modify(vni, description, domain, name_servers,
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def net_remove(network):
"""
@@ -574,7 +676,7 @@ def net_remove(network):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def net_dhcp_list(network, limit=None, static=False):
"""
@@ -594,7 +696,7 @@ def net_dhcp_list(network, limit=None, static=False):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def net_dhcp_add(network, ipaddress, macaddress, hostname):
"""
@@ -611,7 +713,7 @@ def net_dhcp_add(network, ipaddress, macaddress, hostname):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def net_dhcp_remove(network, macaddress):
"""
@@ -628,14 +730,14 @@ def net_dhcp_remove(network, macaddress):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def net_acl_list(network, limit=None, direction=None):
def net_acl_list(network, limit=None, direction=None, is_fuzzy=True):
"""
Return a list of network ACLs in network NETWORK with limit LIMIT.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_network.get_list_acl(zk_conn, network, limit, direction)
retflag, retdata = pvc_network.get_list_acl(zk_conn, network, limit, direction, is_fuzzy=True)
if retflag:
if retdata:
retcode = 200
@@ -647,11 +749,12 @@ def net_acl_list(network, limit=None, direction=None):
else:
retcode = 400
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
pvc_common.stopZKConnection(zk_conn)
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return retdata, retcode
def net_acl_add(network, direction, description, rule, order):
"""
@@ -668,14 +771,14 @@ def net_acl_add(network, direction, description, rule, order):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def net_acl_remove(network, direction, description):
def net_acl_remove(network, description):
"""
Remove an ACL from a virtual client network.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_network.remove_acl(zk_conn, network, description, direction)
retflag, retdata = pvc_network.remove_acl(zk_conn, network, description)
if retflag:
retcode = 200
else:
@@ -685,7 +788,7 @@ def net_acl_remove(network, direction, description):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
#
# Ceph functions
@@ -702,7 +805,7 @@ def ceph_status():
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def ceph_radosdf():
"""
@@ -716,7 +819,7 @@ def ceph_radosdf():
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def ceph_osd_list(limit=None):
"""
@@ -736,7 +839,7 @@ def ceph_osd_list(limit=None):
else:
retcode = 400
return flask.jsonify(retdata), retcode
return retdata, retcode
def ceph_osd_state(osd):
zk_conn = pvc_common.startZKConnection(config['coordinators'])
@@ -756,7 +859,7 @@ def ceph_osd_state(osd):
in_state = retdata[0]['stats']['in']
up_state = retdata[0]['stats']['up']
return flask.jsonify({ "id": osd, "in": in_state, "up": up_state }), retcode
return { "id": osd, "in": in_state, "up": up_state }, retcode
def ceph_osd_add(node, device, weight):
"""
@@ -773,7 +876,7 @@ def ceph_osd_add(node, device, weight):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_osd_remove(osd_id):
"""
@@ -790,7 +893,7 @@ def ceph_osd_remove(osd_id):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_osd_in(osd_id):
"""
@@ -807,7 +910,7 @@ def ceph_osd_in(osd_id):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_osd_out(osd_id):
"""
@@ -824,7 +927,7 @@ def ceph_osd_out(osd_id):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_osd_set(option):
"""
@@ -841,7 +944,7 @@ def ceph_osd_set(option):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_osd_unset(option):
"""
@@ -858,14 +961,19 @@ def ceph_osd_unset(option):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_pool_list(limit=None):
def ceph_pool_list(limit=None, is_fuzzy=True):
"""
Get the list of RBD pools in the Ceph storage cluster.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.get_list_pool(zk_conn, limit)
retflag, retdata = pvc_ceph.get_list_pool(zk_conn, limit, is_fuzzy)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
@@ -878,7 +986,7 @@ def ceph_pool_list(limit=None):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def ceph_pool_add(name, pgs, replcfg):
"""
@@ -895,7 +1003,7 @@ def ceph_pool_add(name, pgs, replcfg):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_pool_remove(name):
"""
@@ -912,14 +1020,19 @@ def ceph_pool_remove(name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_list(pool=None, limit=None):
def ceph_volume_list(pool=None, limit=None, is_fuzzy=True):
"""
Get the list of RBD volumes in the Ceph storage cluster.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.get_list_volume(zk_conn, pool, limit)
retflag, retdata = pvc_ceph.get_list_volume(zk_conn, pool, limit, is_fuzzy)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
@@ -932,7 +1045,7 @@ def ceph_volume_list(pool=None, limit=None):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def ceph_volume_add(pool, name, size):
"""
@@ -949,7 +1062,7 @@ def ceph_volume_add(pool, name, size):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_clone(pool, name, source_volume):
"""
@@ -966,7 +1079,7 @@ def ceph_volume_clone(pool, name, source_volume):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_resize(pool, name, size):
"""
@@ -983,7 +1096,7 @@ def ceph_volume_resize(pool, name, size):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_rename(pool, name, new_name):
"""
@@ -1000,7 +1113,7 @@ def ceph_volume_rename(pool, name, new_name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_remove(pool, name):
"""
@@ -1017,14 +1130,19 @@ def ceph_volume_remove(pool, name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_snapshot_list(pool=None, volume=None, limit=None):
def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True):
"""
Get the list of RBD volume snapshots in the Ceph storage cluster.
"""
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.get_list_snapshot(zk_conn, pool, volume, limit)
retflag, retdata = pvc_ceph.get_list_snapshot(zk_conn, pool, volume, limit, is_fuzzy)
# If this is a single element, strip it out of the list
if isinstance(retdata, list) and len(retdata) == 1:
retdata = retdata[0]
if retflag:
if retdata:
retcode = 200
@@ -1037,13 +1155,12 @@ def ceph_volume_snapshot_list(pool=None, volume=None, limit=None):
retcode = 400
pvc_common.stopZKConnection(zk_conn)
return flask.jsonify(retdata), retcode
return retdata, retcode
def ceph_volume_snapshot_add(pool, volume, name):
"""
Add a Ceph RBD volume snapshot to the PVC Ceph storage cluster.
"""
return '', 200
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.add_snapshot(zk_conn, pool, volume, name)
if retflag:
@@ -1055,7 +1172,7 @@ def ceph_volume_snapshot_add(pool, volume, name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_snapshot_rename(pool, volume, name, new_name):
"""
@@ -1072,7 +1189,7 @@ def ceph_volume_snapshot_rename(pool, volume, name, new_name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode
def ceph_volume_snapshot_remove(pool, volume, name):
"""
@@ -1089,5 +1206,5 @@ def ceph_volume_snapshot_remove(pool, volume, name):
output = {
'message': retdata.replace('\"', '\'')
}
return flask.jsonify(output), retcode
return output, retcode

32
client-api/api_lib/pvcapi_provisioner.py
View File

@@ -110,7 +110,7 @@ def list_template(limit, table, is_fuzzy=True):
if table == 'network_template':
for template_id, template_data in enumerate(data):
# Fetch list of VNIs from network table
query = "SELECT vni FROM network WHERE network_template = %s;"
query = "SELECT * FROM network WHERE network_template = %s;"
args = (template_data['id'],)
cur.execute(query, args)
vnis = cur.fetchall()
@@ -119,13 +119,18 @@ def list_template(limit, table, is_fuzzy=True):
if table == 'storage_template':
for template_id, template_data in enumerate(data):
# Fetch list of VNIs from network table
query = "SELECT * FROM storage WHERE storage_template = %s;"
query = 'SELECT * FROM storage WHERE storage_template = %s'
args = (template_data['id'],)
cur.execute(query, args)
disks = cur.fetchall()
data[template_id]['disks'] = disks
close_database(conn, cur)
# Strip outer list if only one element
if isinstance(data, list) and len(data) == 1:
data = data[0]
return data
def list_template_system(limit, is_fuzzy=True):
@@ -183,14 +188,14 @@ def template_list(limit):
#
# Template Create functions
#
def create_template_system(name, vcpu_count, vram_mb, serial=False, vnc=False, vnc_bind=None, node_limit=None, node_selector=None, start_with_node=False):
def create_template_system(name, vcpu_count, vram_mb, serial=False, vnc=False, vnc_bind=None, node_limit=None, node_selector=None, node_autostart=False):
if list_template_system(name, is_fuzzy=False):
retmsg = { "message": "The system template {} already exists".format(name) }
retcode = 400
return flask.jsonify(retmsg), retcode
query = "INSERT INTO system_template (name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, start_with_node) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);"
args = (name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, start_with_node)
query = "INSERT INTO system_template (name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, node_autostart) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);"
args = (name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, node_autostart)
conn, cur = open_database(config)
try:
@@ -606,7 +611,7 @@ def delete_script(name):
#
def list_profile(limit, is_fuzzy=True):
if limit:
if is_fuzzy:
if not is_fuzzy:
# Handle fuzzy vs. non-fuzzy limits
if not re.match('\^.*', limit):
limit = '%' + limit
@@ -629,6 +634,7 @@ def list_profile(limit, is_fuzzy=True):
data = list()
for profile in orig_data:
profile_data = dict()
profile_data['id'] = profile['id']
profile_data['name'] = profile['name']
# Parse the name of each subelement
for etype in 'system_template', 'network_template', 'storage_template', 'userdata_template', 'script':
@@ -811,38 +817,38 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True):
vm_data = dict()
# Get the profile information
query = "SELECT system_template, network_template, storage_template, script, arguments FROM profile WHERE name = %s"
query = "SELECT * FROM profile WHERE name = %s"
args = (vm_profile,)
db_cur.execute(query, args)
profile_data = db_cur.fetchone()
vm_data['script_arguments'] = profile_data['arguments'].split('|')
# Get the system details
query = 'SELECT vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, start_with_node FROM system_template WHERE id = %s'
query = 'SELECT * FROM system_template WHERE id = %s'
args = (profile_data['system_template'],)
db_cur.execute(query, args)
vm_data['system_details'] = db_cur.fetchone()
# Get the MAC template
query = 'SELECT mac_template FROM network_template WHERE id = %s'
query = 'SELECT * FROM network_template WHERE id = %s'
args = (profile_data['network_template'],)
db_cur.execute(query, args)
vm_data['mac_template'] = db_cur.fetchone()['mac_template']
# Get the networks
query = 'SELECT vni FROM network WHERE network_template = %s'
query = 'SELECT * FROM network WHERE network_template = %s'
args = (profile_data['network_template'],)
db_cur.execute(query, args)
vm_data['networks'] = db_cur.fetchall()
# Get the storage volumes
query = 'SELECT pool, disk_id, disk_size_gb, mountpoint, filesystem, filesystem_args FROM storage WHERE storage_template = %s'
query = 'SELECT * FROM storage WHERE storage_template = %s'
args = (profile_data['storage_template'],)
db_cur.execute(query, args)
vm_data['volumes'] = db_cur.fetchall()
# Get the script
query = 'SELECT script FROM script WHERE id = %s'
query = 'SELECT * FROM script WHERE id = %s'
args = (profile_data['script'],)
db_cur.execute(query, args)
vm_data['script'] = db_cur.fetchone()['script']
@@ -1216,7 +1222,7 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True):
print("Defining and starting VM on cluster")
if define_vm:
retcode, retmsg = pvc_vm.define_vm(zk_conn, vm_schema, target_node, vm_data['system_details']['node_limit'].split(','), vm_data['system_details']['node_selector'], vm_data['system_details']['start_with_node'], vm_profile)
retcode, retmsg = pvc_vm.define_vm(zk_conn, vm_schema, target_node, vm_data['system_details']['node_limit'].split(','), vm_data['system_details']['node_selector'], vm_data['system_details']['node_autostart'], vm_profile)
print(retmsg)
if start_vm:

21
client-api/gen-doc.py Executable file
View File

@@ -0,0 +1,21 @@
#!/usr/bin/env python3
# gen-doc.py - Generate a Swagger JSON document for the API
# Part of the Parallel Virtual Cluster (PVC) system
from flask_swagger import swagger
import sys
import json
sys.path.append(',')
pvc_api = __import__('pvc-api')
swagger_file = "swagger.json"
swagger_data = swagger(pvc_api.app)
swagger_data['info']['version'] = "1.0"
swagger_data['info']['title'] = "PVC Client and Provisioner API"
with open(swagger_file, 'w') as fd:
fd.write(json.dumps(swagger_data, sort_keys=True, indent=4))

2
client-api/provisioner/schema.sql
View File

@@ -1,6 +1,6 @@
create database pvcprov with owner = pvcprov connection limit = -1;
\c pvcprov
create table system_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, vcpu_count INT NOT NULL, vram_mb INT NOT NULL, serial BOOL NOT NULL, vnc BOOL NOT NULL, vnc_bind TEXT, node_limit TEXT, node_selector TEXT, start_with_node BOOL NOT NULL);
create table system_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, vcpu_count INT NOT NULL, vram_mb INT NOT NULL, serial BOOL NOT NULL, vnc BOOL NOT NULL, vnc_bind TEXT, node_limit TEXT, node_selector TEXT, node_autostart BOOL NOT NULL);
create table network_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, mac_template TEXT);
create table network (id SERIAL PRIMARY KEY, network_template INT REFERENCES network_template(id), vni INT NOT NULL);
create table storage_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE);

6750
client-api/pvc-api.py
View File

File diff suppressed because it is too large Load Diff

6
client-api/pvc-api.sample.yaml
View File

@@ -12,9 +12,9 @@ pvc:
debug: True
# coordinators: The list of cluster coordinator hostnames
coordinators:
- pvc-hv1
- pvc-hv2
- pvc-hv3
- hv1
- hv2
- hv3
# api: Configuration of the API listener
api:
# listen_address: IP address(es) to listen on; use 0.0.0.0 for all interfaces

13
client-api/swagger.html Normal file
View File

@@ -0,0 +1,13 @@
<!DOCTYPE html>
<html>
<head>
<title>PVC Client API Documentation</title>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style> body { margin: 0; padding: 0; } </style>
</head>
<body>
<redoc spec-url='./swagger.json' hide-loading></redoc>
<script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"> </script>
</body>
</html>

4750
client-api/swagger.json Normal file
View File

File diff suppressed because it is too large Load Diff

36
client-common/ceph.py
View File

@@ -728,13 +728,10 @@ def get_list_pool(zk_conn, limit, is_fuzzy=True):
pool_list = []
full_pool_list = zkhandler.listchildren(zk_conn, '/ceph/pools')
if is_fuzzy and limit:
# Implicitly assume fuzzy limits
if not re.match('\^.*', limit):
limit = '.*' + limit
if not re.match('.*\$', limit):
limit = limit + '.*'
if limit:
if not is_fuzzy:
limit = '^' + limit + '$'
for pool in full_pool_list:
if limit:
try:
@@ -1121,23 +1118,26 @@ def remove_volume(zk_conn, pool, name):
def get_list_volume(zk_conn, pool, limit, is_fuzzy=True):
volume_list = []
if pool and not verifyPool(zk_conn, name):
if pool and not verifyPool(zk_conn, pool):
return False, 'ERROR: No pool with name "{}" is present in the cluster.'.format(name)
full_volume_list = getCephVolumes(zk_conn, pool)
if is_fuzzy and limit:
# Implicitly assume fuzzy limits
if not re.match('\^.*', limit):
limit = '.*' + limit
if not re.match('.*\$', limit):
limit = limit + '.*'
if limit:
if not is_fuzzy:
limit = '^' + limit + '$'
else:
# Implicitly assume fuzzy limits
if not re.match('\^.*', limit):
limit = '.*' + limit
if not re.match('.*\$', limit):
limit = limit + '.*'
for volume in full_volume_list:
pool_name, volume_name = volume.split('/')
if limit:
try:
if re.match(limit, volume):
if re.match(limit, volume_name):
volume_list.append(getVolumeInformation(zk_conn, pool_name, volume_name))
except Exception as e:
return False, 'Regex Error: {}'.format(e)
@@ -1387,12 +1387,12 @@ def get_list_snapshot(zk_conn, pool, volume, limit, is_fuzzy=True):
pool_name, volume_name = volume.split('/')
if limit:
try:
if re.match(limit, snapshot):
snapshot_list.append(snapshot)
if re.match(limit, snapshot_name):
snapshot_list.append({'pool': pool_name, 'volume': volume_name, 'snapshot': snapshot_name})
except Exception as e:
return False, 'Regex Error: {}'.format(e)
else:
snapshot_list.append(snapshot)
snapshot_list.append({'pool': pool_name, 'volume': volume_name, 'snapshot': snapshot_name})
return True, snapshot_list

8
client-common/common.py
View File

@@ -25,6 +25,8 @@ import lxml
import math
import kazoo.client
from distutils.util import strtobool
import client_lib.zkhandler as zkhandler
###############################################################################
@@ -216,11 +218,11 @@ def getInformationFromXML(zk_conn, uuid):
'failed_reason': domain_failedreason,
'node_limit': domain_node_limit,
'node_selector': domain_node_selector,
'node_autostart': domain_node_autostart,
'node_autostart': bool(strtobool(domain_node_autostart)),
'description': domain_description,
'profile': domain_profile,
'memory': domain_memory,
'vcpu': domain_vcpu,
'memory': int(domain_memory),
'vcpu': int(domain_vcpu),
'vcpu_topology': domain_vcputopo,
'networks': domain_networks,
'type': domain_type,

16
client-common/node.py
View File

@@ -47,17 +47,17 @@ def getNodeInformation(zk_conn, node_name):
node_coordinator_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node_name))
node_domain_state = zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node_name))
node_static_data = zkhandler.readdata(zk_conn, '/nodes/{}/staticdata'.format(node_name)).split()
node_cpu_count = node_static_data[0]
node_cpu_count = int(node_static_data[0])
node_kernel = node_static_data[1]
node_os = node_static_data[2]
node_arch = node_static_data[3]
node_vcpu_allocated = zkhandler.readdata(zk_conn, 'nodes/{}/vcpualloc'.format(node_name))
node_vcpu_allocated = int(zkhandler.readdata(zk_conn, 'nodes/{}/vcpualloc'.format(node_name)))
node_mem_total = int(zkhandler.readdata(zk_conn, '/nodes/{}/memtotal'.format(node_name)))
node_mem_allocated = int(zkhandler.readdata(zk_conn, '/nodes/{}/memalloc'.format(node_name)))
node_mem_used = int(zkhandler.readdata(zk_conn, '/nodes/{}/memused'.format(node_name)))
node_mem_free = int(zkhandler.readdata(zk_conn, '/nodes/{}/memfree'.format(node_name)))
node_load = zkhandler.readdata(zk_conn, '/nodes/{}/cpuload'.format(node_name))
node_domains_count = zkhandler.readdata(zk_conn, '/nodes/{}/domainscount'.format(node_name))
node_load = float(zkhandler.readdata(zk_conn, '/nodes/{}/cpuload'.format(node_name)))
node_domains_count = int(zkhandler.readdata(zk_conn, '/nodes/{}/domainscount'.format(node_name)))
node_running_domains = zkhandler.readdata(zk_conn, '/nodes/{}/runningdomains'.format(node_name)).split()
# Construct a data structure to represent the data
@@ -200,12 +200,8 @@ def get_list(zk_conn, limit, is_fuzzy=True):
for node in full_node_list:
if limit:
try:
if is_fuzzy:
# Implcitly assume fuzzy limits
if not re.match('\^.*', limit):
limit = '.*' + limit
if not re.match('.*\$', limit):
limit = limit + '.*'
if not is_fuzzy:
limit = '^' + limit + '$'
if re.match(limit, node):
node_list.append(getNodeInformation(zk_conn, node))

131
docs/manuals/api.md
View File

@@ -838,16 +838,137 @@ Remove a Ceph RBD volume snapshot `<snapshot>` of Ceph RBD volume `<volume>` on
## Provisioner API endpoint documentation
### Node endpoints
### General endpoints
These endpoints manage PVC node state and operation.
#### `/api/v1/node`
#### `/api/v1/provisioner/template`
* Methods: `GET`
###### `GET`
* Manadatory values: N/A
* Optional values: `limit`
Return a JSON document containing information about all available templates. If `limit` is specified, return a JSON document containing information about any templates with names matching`limit` as a fuzzy regex; `^` or `$` can be used for force exact start/end matches.
### System Template endpoints
These endpoints manage PVC system templates.
#### `/api/v1/provisioner/template/system`
* Methods: `GET`, `POST`
###### `GET`
* Mandatory values: N/A
* Optional values: `limit`
Return a JSON document containing information about all cluster nodes. If `limit` is specified, return a JSON document containing information about cluster nodes with names matching `limit` as fuzzy regex.
Return a JSON document containing information about all available system templates. If `limit` is specified, return a JSON document containing information about any system templates with names matching`limit` as a fuzzy regex; `^` or `$` can be used for force exact start/end matches.
###### `POST`
* Mandatory values: `name`, `vcpus`, `vram`, `serial`, `vnc`
* Optional values: `vnc_bind`, `node_limit`, `node_selector`, `start_with_node`
Add a new system template `name` with `vcpus` vCPUs, `vram` MB of RAM, and with `serial` and/or `vnc` enabled (`true`) or disabled (`false`).
If `vnc=true`, the `vnc_bind` option specifies which IP address(es) for VNC to listen on. Common values would include `127.0.0.1` (the default), `0.0.0.0` for all interfaces, or a specific IP such as the cluster upstream floating IP address.
If `node_limit` is specified and is a valid list of PVC nodes in CSV format, the PVC VM metadata to limit the VM to the specified nodes will be set.
If `node_selector` is specified, the PVC VM metadata to specify the node selector for the VM will be set. Valid `node_selector` values are: `mem`: the node with the least allocated VM memory; `vcpus`: the node with the least allocated VM vCPUs; `load`: the node with the least current load average; `vms`: the node with the least number of provisioned VMs.
If `start_with_node` is specified, the PVC VM metadata to specify the autostart-with-node will be set for the first boot. This is most useful if the VM will not be started by default and is set to run on a stopped or flushed node.
#### `/api/v1/provisioner/template/system/<name>`
* Methods: `GET`, `POST`, `PUT`, `DELETE`
###### `GET`
* Mandatory values: N/A
* Optional values: N/A
Return a JSON document containing information about system template `<name>`. The output is identical to `/api/v1/provisioner/template/system?limit=<name>` without fuzzy regex matching.
###### `POST`
* Mandatory values: `vcpus`, `vram`, `serial`, `vnc`
* Optional values: `vnc_bind`, `node_limit`, `node_selector`, `start_with_node`
Add a new system template `name` with `vcpus` vCPUs, `vram` MB of RAM, and with `serial` and/or `vnc` enabled (`true`) or disabled (`false`).
If `vnc=true`, the `vnc_bind` option specifies which IP address(es) for VNC to listen on. Common values would include `127.0.0.1` (the default), `0.0.0.0` for all interfaces, or a specific IP such as the cluster upstream floating IP address.
If `node_limit` is specified and is a valid list of PVC nodes in CSV format, the PVC VM metadata to limit the VM to the specified nodes will be set.
If `node_selector` is specified, the PVC VM metadata to specify the node selector for the VM will be set. Valid `node_selector` values are: `mem`: the node with the least allocated VM memory; `vcpus`: the node with the least allocated VM vCPUs; `load`: the node with the least current load average; `vms`: the node with the least number of provisioned VMs.
If `start_with_node` is specified, the PVC VM metadata to specify the autostart-with-node will be set for the first boot. This is most useful if the VM will not be started by default and is set to run on a stopped or flushed node.
###### `PUT`
* Mandatory values: N/A
* Optional values: `vcpus`, `vram`, `serial`, `vnc`, `vnc_bind`, `node_limit`, `node_selector`, `start_with_node`
Modify an existing system template `name` by updating the specified value(s).
###### `DELETE`
* Mandatory values: N/A
* Optional values: N/A
Delete the system template `name`. Note that you cannot delete a template which is a member of a profile; it must first be removed from the profile, or the profile deleted.
### Network Template endpoints
These endpoints manage PVC network templates.
#### `/api/v1/provisioner/template/network`
* Methods: `GET`, `POST`
###### `GET`
* Mandatory values: N/A
* Optional values: `limit`
Return a JSON document containing information about all available network templates. If `limit` is specified, return a JSON document containing information about any network templates with names matching`limit` as a fuzzy regex; `^` or `$` can be used for force exact start/end matches.
###### `POST`
* Mandatory values:
* Optional values:
Add a new network template `name`.
#### `/api/v1/provisioner/template/network/<name>`
* Methods: `GET`, `POST`, `DELETE`
###### `GET`
* Mandatory values: N/A
* Optional values: N/A
Return a JSON document containing information about network template `<name>`. The output is identical to `/api/v1/provisioner/template/network?limit=<name>` without fuzzy regex matching.
###### `POST`
* Mandatory values:
* Optional values:
Add a new network template `name`.
###### `DELETE`
* Mandatory values: N/A
* Optional values: N/A
Delete the network template `name`. Note that you cannot delete a template which is a member of a profile; it must first be removed from the profile, or the profile deleted.
#### `/api/v1/provisioner/template/network/<name>/net`
* Methods: `GET`, `POST`
###### `GET`
* Mandatory values: N/A
* Optional values: N/A
Return a JSON document containing information about network template `<name>`. The output is identical to `/api/v1/provisioner/template/network?limit=<name>` without fuzzy regex matching.
###### `POST`
* Mandatory values:
* Optional values:
Add a new network template `name`.
###### `DELETE`
* Mandatory values: N/A
* Optional values: N/A
Delete the network template `name`. Note that you cannot delete a template which is a member of a profile; it must first be removed from the profile, or the profile deleted.