parallelvirtualcluster
/
pvc
1
0
Fork 0
Code Issues 15 Pull Requests Projects Releases 106 Wiki Activity

Merge branch 'the-great-linting' 

Complete linting of the project to standard flak8 styling.
This commit is contained in:
Joshua Boniface 2020-11-07 15:37:39 -05:00
parent 601ab1a181 34ce3bf3f8
commit c7a289e9bb
47 changed files with 2255 additions and 3100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.hooks/pre-commit Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env bash
pushd $( git rev-parse --show-toplevel ) &>/dev/null
ex=0
# Linting
./lint
if [[ $? -ne 0 ]]; then
echo "Aborting commit due to linting errors."
ex=1
fi
echo
popd &>/dev/null
exit $ex

3
api-daemon/migrations/env.py
View File

@ -2,6 +2,7 @@ from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
from flask import current_app
import logging
# this is the Alembic Config object, which provides
@ -17,7 +18,6 @@ logger = logging.getLogger('alembic.env')
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from flask import current_app
config.set_main_option('sqlalchemy.url',
current_app.config.get('SQLALCHEMY_DATABASE_URI'))
target_metadata = current_app.extensions['migrate'].db.metadata
@ -81,6 +81,7 @@ def run_migrations_online():
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:

6
api-daemon/provisioner/examples/debootstrap_script.py
View File

@ -50,15 +50,15 @@ def install(**kwargs):
# failures of these gracefully, should administrators forget to specify them.
try:
deb_release = kwargs['deb_release']
except:
except Exception:
deb_release = "stable"
try:
deb_mirror = kwargs['deb_mirror']
except:
except Exception:
deb_mirror = "http://ftp.debian.org/debian"
try:
deb_packages = kwargs['deb_packages'].split(',')
except:
except Exception:
deb_packages = ["linux-image-amd64", "grub-pc", "cloud-init", "python3-cffi-backend", "wget"]
# We need to know our root disk

2
api-daemon/provisioner/examples/dummy_script.py
View File

@ -30,8 +30,6 @@
# This script will run under root privileges as the provisioner does. Be careful
# with that.
import os
# Installation function - performs a debootstrap install of a Debian system
# Note that the only arguments are keyword arguments.
def install(**kwargs):

4
api-daemon/pvcapid-manage.py
View File

@ -20,11 +20,11 @@
#
###############################################################################
import os
from flask_migrate import Migrate, MigrateCommand
from flask_script import Manager
from pvcapid.flaskapi import app, db, config
from pvcapid.flaskapi import app, db
from pvcapid.models import * # noqa F401,F403
migrate = Migrate(app, db)
manager = Manager(app)

2
api-daemon/pvcapid.py
View File

@ -20,4 +20,4 @@
#
###############################################################################
import pvcapid.Daemon
import pvcapid.Daemon # noqa: F401

26
api-daemon/pvcapid/benchmark.py
View File

@ -20,25 +20,16 @@
#
###############################################################################
import flask
import json
import psycopg2
import psycopg2.extras
import os
import re
import time
import shlex
import subprocess
from distutils.util import strtobool as dustrtobool
import daemon_lib.common as pvc_common
import daemon_lib.node as pvc_node
import daemon_lib.ceph as pvc_ceph
import pvcapid.libvirt_schema as libvirt_schema
config = None # Set in this namespace by flaskapi
from pvcapid.ova import list_ova
def strtobool(stringv):
if stringv is None:
@ -47,9 +38,10 @@ def strtobool(stringv):
return bool(stringv)
try:
return bool(dustrtobool(stringv))
except:
except Exception:
return False
#
# Exceptions (used by Celery tasks)
#
@ -76,6 +68,7 @@ class BenchmarkError(Exception):
# Common functions
#
# Database connections
def open_database(config):
conn = psycopg2.connect(
@ -88,12 +81,14 @@ def open_database(config):
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
return conn, cur
def close_database(conn, cur, failed=False):
if not failed:
conn.commit()
cur.close()
conn.close()
def list_benchmarks(job=None):
if job is not None:
query = "SELECT * FROM {} WHERE job = %s;".format('storage_benchmarks')
@ -119,6 +114,7 @@ def list_benchmarks(job=None):
else:
return {'message': 'No benchmark found.'}, 404
def run_benchmark(self, pool):
# Runtime imports
import time
@ -134,17 +130,16 @@ def run_benchmark(self, pool):
# Phase 0 - connect to databases
try:
db_conn, db_cur = open_database(config)
except:
except Exception:
print('FATAL - failed to connect to Postgres')
raise Exception
try:
zk_conn = pvc_common.startZKConnection(config['coordinators'])
except:
except Exception:
print('FATAL - failed to connect to Zookeeper')
raise Exception
print("Storing running status for job '{}' in database".format(cur_time))
try:
query = "INSERT INTO storage_benchmarks (job, result) VALUES (%s, %s);"
@ -246,8 +241,7 @@ def run_benchmark(self, pool):
volume=volume,
test=test,
bs=test_matrix[test]['bs'],
rw=test_matrix[test]['rw']
)
rw=test_matrix[test]['rw'])
retcode, stdout, stderr = pvc_common.run_os_command(fio_cmd)
if retcode:

226
api-daemon/pvcapid/flaskapi.py
View File

@ -20,22 +20,14 @@
#
###############################################################################
import json
import yaml
import os
import gevent.pywsgi
import flask
from distutils.util import strtobool as dustrtobool
from functools import wraps
from flask_restful import Resource, Api, reqparse, abort
from celery import Celery
from celery.task.control import inspect
import pvcapid.helper as api_helper
import pvcapid.provisioner as api_provisioner
@ -46,6 +38,7 @@ from flask_sqlalchemy import SQLAlchemy
API_VERSION = 1.0
def strtobool(stringv):
if stringv is None:
return False
@ -53,13 +46,14 @@ def strtobool(stringv):
return bool(stringv)
try:
return bool(dustrtobool(stringv))
except:
except Exception:
return False
# Parse the configuration file
try:
pvc_config_file = os.environ['PVC_CONFIG_FILE']
except:
except Exception:
print('Error: The "PVC_CONFIG_FILE" environment variable must be set before starting pvcapid.')
exit(1)
@ -133,9 +127,6 @@ if config['auth_enabled']:
# Create SQLAlchemy database
db = SQLAlchemy(app)
# Import database models
from pvcapid.models import *
# Create Flask blueprint
blueprint = flask.Blueprint('api', __name__, url_prefix='/api/v1')
@ -147,6 +138,7 @@ app.register_blueprint(blueprint)
celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
#
# Custom decorators
#
@ -155,9 +147,11 @@ celery.conf.update(app.config)
class RequestParser(object):
def __init__(self, reqargs):
self.reqargs = reqargs
def __call__(self, function):
if not callable(function):
return
@wraps(function)
def wrapped_function(*args, **kwargs):
parser = reqparse.RequestParser()
@ -176,6 +170,7 @@ class RequestParser(object):
return function(*args, **kwargs)
return wrapped_function
# Authentication decorator function
def Authenticator(function):
@wraps(function)
@ -204,6 +199,7 @@ def Authenticator(function):
def create_vm(self, vm_name, profile_name, define_vm=True, start_vm=True, script_run_args=[]):
return api_provisioner.create_vm(self, vm_name, profile_name, define_vm=define_vm, start_vm=start_vm, script_run_args=script_run_args)
@celery.task(bind=True)
def run_benchmark(self, pool):
return api_benchmark.run_benchmark(self, pool)
@ -234,8 +230,11 @@ class API_Root(Resource):
example: "PVC API version 1.0"
"""
return {"message": "PVC API version {}".format(API_VERSION)}
api.add_resource(API_Root, '/')
# /doc - NOTE: Until flask_swagger is packaged for Debian this must be disabled
# class API_Doc(Resource):
# def get(self):
@ -253,8 +252,11 @@ api.add_resource(API_Root, '/')
# swagger_data['info']['title'] = "PVC Client and Provisioner API"
# swagger_data['host'] = "{}:{}".format(config['listen_address'], config['listen_port'])
# return swagger_data
#
#
# api.add_resource(API_Doc, '/doc')
# /login
class API_Login(Resource):
def post(self):
@ -294,8 +296,11 @@ class API_Login(Resource):
return {"message": "Authentication successful"}, 200
else:
{"message": "Authentication failed"}, 401
api.add_resource(API_Login, '/login')
# /logout
class API_Logout(Resource):
def post(self):
@ -318,8 +323,11 @@ class API_Logout(Resource):
flask.session.pop('token', None)
return {"message": "Deauthentication successful"}, 200
api.add_resource(API_Logout, '/logout')
# /initialize
class API_Initialize(Resource):
@Authenticator
@ -347,8 +355,11 @@ class API_Initialize(Resource):
return {"message": "Successfully initialized a new PVC cluster"}, 200
else:
return {"message": "PVC cluster already initialized"}, 400
api.add_resource(API_Initialize, '/initialize')
# /status
class API_Status(Resource):
@Authenticator
@ -458,6 +469,7 @@ class API_Status(Resource):
"""
return api_helper.cluster_maintenance(reqargs.get('state', 'false'))
api.add_resource(API_Status, '/status')
@ -581,8 +593,11 @@ class API_Node_Root(Resource):
coordinator_state=reqargs.get('coordinator_state', None),
domain_state=reqargs.get('domain_state', None)
)
api.add_resource(API_Node_Root, '/node')
# /node/<node>
class API_Node_Element(Resource):
@Authenticator
@ -604,8 +619,11 @@ class API_Node_Element(Resource):
id: Message
"""
return api_helper.node_list(node, is_fuzzy=False)
api.add_resource(API_Node_Element, '/node/<node>')
# /node/<node>/daemon-state
class API_Node_DaemonState(Resource):
@Authenticator
@ -635,8 +653,11 @@ class API_Node_DaemonState(Resource):
id: Message
"""
return api_helper.node_daemon_state(node)
api.add_resource(API_Node_DaemonState, '/node/<node>/daemon-state')
# /node/<node>/coordinator-state
class API_Node_CoordinatorState(Resource):
@Authenticator
@ -703,8 +724,11 @@ class API_Node_CoordinatorState(Resource):
if reqargs['state'] == 'secondary':
return api_helper.node_secondary(node)
abort(400)
api.add_resource(API_Node_CoordinatorState, '/node/<node>/coordinator-state')
# /node/<node>/domain-state
class API_Node_DomainState(Resource):
@Authenticator
@ -776,6 +800,8 @@ class API_Node_DomainState(Resource):
if reqargs['state'] == 'ready':
return api_helper.node_ready(node, bool(strtobool(reqargs.get('wait', 'false'))))
abort(400)
api.add_resource(API_Node_DomainState, '/node/<node>/domain-state')
@ -1111,8 +1137,11 @@ class API_VM_Root(Resource):
bool(strtobool(reqargs.get('autostart', 'false'))),
reqargs.get('migration_method', 'none')
)
api.add_resource(API_VM_Root, '/vm')
# /vm/<vm>
class API_VM_Element(Resource):
@Authenticator
@ -1291,8 +1320,11 @@ class API_VM_Element(Resource):
return api_helper.vm_remove(vm)
else:
return api_helper.vm_undefine(vm)
api.add_resource(API_VM_Element, '/vm/<vm>')
# /vm/<vm>/meta
class API_VM_Metadata(Resource):
@Authenticator
@ -1404,8 +1436,11 @@ class API_VM_Metadata(Resource):
reqargs.get('profile', None),
reqargs.get('migration_method', None)
)
api.add_resource(API_VM_Metadata, '/vm/<vm>/meta')
# /vm/<vm</state
class API_VM_State(Resource):
@Authenticator
@ -1489,8 +1524,11 @@ class API_VM_State(Resource):
if state == 'disable':
return api_helper.vm_disable(vm)
abort(400)
api.add_resource(API_VM_State, '/vm/<vm>/state')
# /vm/<vm>/node
class API_VM_Node(Resource):
@Authenticator
@ -1589,8 +1627,11 @@ class API_VM_Node(Resource):
if action == 'unmigrate':
return api_helper.vm_unmigrate(vm, wait, force_live)
abort(400)
api.add_resource(API_VM_Node, '/vm/<vm>/node')
# /vm/<vm>/locks
class API_VM_Locks(Resource):
@Authenticator
@ -1613,8 +1654,11 @@ class API_VM_Locks(Resource):
id: Message
"""
return api_helper.vm_flush_locks(vm)
api.add_resource(API_VM_Locks, '/vm/<vm>/locks')
# /vm/<vm</console
class API_VM_Console(Resource):
@RequestParser([
@ -1656,6 +1700,8 @@ class API_VM_Console(Resource):
vm,
reqargs.get('lines', None)
)
api.add_resource(API_VM_Console, '/vm/<vm>/console')
@ -1854,8 +1900,11 @@ class API_Network_Root(Resource):
reqargs.get('dhcp4_start', None),
reqargs.get('dhcp4_end', None),
)
api.add_resource(API_Network_Root, '/network')
# /network/<vni>
class API_Network_Element(Resource):
@Authenticator
@ -2100,8 +2149,11 @@ class API_Network_Element(Resource):
id: Message
"""
return api_helper.net_remove(vni)
api.add_resource(API_Network_Element, '/network/<vni>')
# /network/<vni>/lease
class API_Network_Lease_Root(Resource):
@RequestParser([
@ -2219,8 +2271,11 @@ class API_Network_Lease_Root(Resource):
reqargs.get('macaddress', None),
reqargs.get('hostname', None)
)
api.add_resource(API_Network_Lease_Root, '/network/<vni>/lease')
# /network/<vni>/lease/{mac}
class API_Network_Lease_Element(Resource):
@Authenticator
@ -2276,7 +2331,7 @@ class API_Network_Lease_Element(Resource):
{'name': 'hostname'}
])
@Authenticator
def post(self, vni, mac):
def post(self, vni, mac, reqargs):
"""
Create a new static DHCP lease {mac} in network {vni}
---
@ -2345,8 +2400,11 @@ class API_Network_Lease_Element(Resource):
vni,
mac
)
api.add_resource(API_Network_Lease_Element, '/network/<vni>/lease/<mac>')
# /network/<vni>/acl
class API_Network_ACL_Root(Resource):
@RequestParser([
@ -2467,8 +2525,11 @@ class API_Network_ACL_Root(Resource):
reqargs.get('rule', None),
reqargs.get('order', None)
)
api.add_resource(API_Network_ACL_Root, '/network/<vni>/acl')
# /network/<vni>/acl/<description>
class API_Network_ACL_Element(Resource):
@Authenticator
@ -2574,6 +2635,8 @@ class API_Network_ACL_Element(Resource):
vni,
description
)
api.add_resource(API_Network_ACL_Element, '/network/<vni>/acl/<description>')
@ -2590,15 +2653,21 @@ class API_Storage_Root(Resource):
@Authenticator
def get(self):
pass
api.add_resource(API_Storage_Root, '/storage')
# /storage/ceph
class API_Storage_Ceph_Root(Resource):
@Authenticator
def get(self):
pass
api.add_resource(API_Storage_Ceph_Root, '/storage/ceph')
# /storage/ceph/status
class API_Storage_Ceph_Status(Resource):
@Authenticator
@ -2625,8 +2694,11 @@ class API_Storage_Ceph_Status(Resource):
description: The raw output data
"""
return api_helper.ceph_status()
api.add_resource(API_Storage_Ceph_Status, '/storage/ceph/status')
# /storage/ceph/utilization
class API_Storage_Ceph_Utilization(Resource):
@Authenticator
@ -2653,8 +2725,11 @@ class API_Storage_Ceph_Utilization(Resource):
description: The raw output data
"""
return api_helper.ceph_util()
api.add_resource(API_Storage_Ceph_Utilization, '/storage/ceph/utilization')
# /storage/ceph/benchmark
class API_Storage_Ceph_Benchmark(Resource):
@RequestParser([
@ -2811,8 +2886,11 @@ class API_Storage_Ceph_Benchmark(Resource):
reqargs.get('pool', None)
)
return {"task_id": task.id}, 202, {'Location': Api.url_for(api, API_Storage_Ceph_Benchmark, task_id=task.id)}
api.add_resource(API_Storage_Ceph_Benchmark, '/storage/ceph/benchmark')
# /storage/ceph/option
class API_Storage_Ceph_Option(Resource):
@RequestParser([
@ -2857,8 +2935,11 @@ class API_Storage_Ceph_Option(Resource):
if reqargs.get('action') == 'unset':
return api_helper.ceph_osd_unset(reqargs.get('option'))
abort(400)
api.add_resource(API_Storage_Ceph_Option, '/storage/ceph/option')
# /storage/ceph/osd
class API_Storage_Ceph_OSD_Root(Resource):
@RequestParser([
@ -3000,8 +3081,11 @@ class API_Storage_Ceph_OSD_Root(Resource):
reqargs.get('device', None),
reqargs.get('weight', None)
)
api.add_resource(API_Storage_Ceph_OSD_Root, '/storage/ceph/osd')
# /storage/ceph/osd/<osdid>
class API_Storage_Ceph_OSD_Element(Resource):
@Authenticator
@ -3059,8 +3143,11 @@ class API_Storage_Ceph_OSD_Element(Resource):
return api_helper.ceph_osd_remove(
osdid
)
api.add_resource(API_Storage_Ceph_OSD_Element, '/storage/ceph/osd/<osdid>')
# /storage/ceph/osd/<osdid>/state
class API_Storage_Ceph_OSD_State(Resource):
@Authenticator
@ -3116,8 +3203,11 @@ class API_Storage_Ceph_OSD_State(Resource):
osdid
)
abort(400)
api.add_resource(API_Storage_Ceph_OSD_State, '/storage/ceph/osd/<osdid>/state')
# /storage/ceph/pool
class API_Storage_Ceph_Pool_Root(Resource):
@RequestParser([
@ -3246,8 +3336,11 @@ class API_Storage_Ceph_Pool_Root(Resource):
reqargs.get('pgs', None),
reqargs.get('replcfg', None)
)
api.add_resource(API_Storage_Ceph_Pool_Root, '/storage/ceph/pool')
# /storage/ceph/pool/<pool>
class API_Storage_Ceph_Pool_Element(Resource):
@Authenticator
@ -3268,7 +3361,7 @@ class API_Storage_Ceph_Pool_Element(Resource):
type: object
id: Message
"""
return api_helper,ceph_pool_list(
return api_helper, api_helper.ceph_pool_list(
pool,
is_fuzzy=False
)
@ -3278,7 +3371,7 @@ class API_Storage_Ceph_Pool_Element(Resource):
{'name': 'replcfg', 'required': True, 'helpmsg': "A valid replication configuration must be specified."}
])
@Authenticator
def post(self, pool):
def post(self, pool, reqargs):
"""
Create a new Ceph pool {pool}
---
@ -3355,8 +3448,11 @@ class API_Storage_Ceph_Pool_Element(Resource):
return api_helper.ceph_pool_remove(
pool
)
api.add_resource(API_Storage_Ceph_Pool_Element, '/storage/ceph/pool/<pool>')
# /storage/ceph/volume
class API_Storage_Ceph_Volume_Root(Resource):
@RequestParser([
@ -3504,8 +3600,11 @@ class API_Storage_Ceph_Volume_Root(Resource):
reqargs.get('volume', None),
reqargs.get('size', None)
)
api.add_resource(API_Storage_Ceph_Volume_Root, '/storage/ceph/volume')
# /storage/ceph/volume/<pool>/<volume>
class API_Storage_Ceph_Volume_Element(Resource):
@Authenticator
@ -3651,8 +3750,11 @@ class API_Storage_Ceph_Volume_Element(Resource):
pool,
volume
)
api.add_resource(API_Storage_Ceph_Volume_Element, '/storage/ceph/volume/<pool>/<volume>')
# /storage/ceph/volume/<pool>/<volume>/clone
class API_Storage_Ceph_Volume_Element_Clone(Resource):
@RequestParser([
@ -3693,8 +3795,11 @@ class API_Storage_Ceph_Volume_Element_Clone(Resource):
reqargs.get('new_volume', None),
volume
)
api.add_resource(API_Storage_Ceph_Volume_Element_Clone, '/storage/ceph/volume/<pool>/<volume>/clone')
# /storage/ceph/volume/<pool>/<volume>/upload
class API_Storage_Ceph_Volume_Element_Upload(Resource):
@RequestParser([
@ -3744,8 +3849,11 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
volume,
reqargs.get('image_format', None)
)
api.add_resource(API_Storage_Ceph_Volume_Element_Upload, '/storage/ceph/volume/<pool>/<volume>/upload')
# /storage/ceph/snapshot
class API_Storage_Ceph_Snapshot_Root(Resource):
@RequestParser([
@ -3849,8 +3957,11 @@ class API_Storage_Ceph_Snapshot_Root(Resource):
reqargs.get('volume', None),
reqargs.get('snapshot', None)
)
api.add_resource(API_Storage_Ceph_Snapshot_Root, '/storage/ceph/snapshot')
# /storage/ceph/snapshot/<pool>/<volume>/<snapshot>
class API_Storage_Ceph_Snapshot_Element(Resource):
@Authenticator
@ -3989,6 +4100,8 @@ class API_Storage_Ceph_Snapshot_Element(Resource):
volume,
snapshot
)
api.add_resource(API_Storage_Ceph_Snapshot_Element, '/storage/ceph/snapshot/<pool>/<volume>/<snapshot>')
@ -4004,8 +4117,11 @@ class API_Provisioner_Root(Resource):
Unused endpoint
"""
abort(404)
api.add_resource(API_Provisioner_Root, '/provisioner')
# /provisioner/template
class API_Provisioner_Template_Root(Resource):
@RequestParser([
@ -4050,8 +4166,11 @@ class API_Provisioner_Template_Root(Resource):
return api_provisioner.template_list(
reqargs.get('limit', None)
)
api.add_resource(API_Provisioner_Template_Root, '/provisioner/template')
# /provisioner/template/system
class API_Provisioner_Template_System_Root(Resource):
@RequestParser([
@ -4205,11 +4324,11 @@ class API_Provisioner_Template_System_Root(Resource):
# Validate arguments
try:
vcpus = int(reqargs.get('vcpus'))
except:
except Exception:
return {"message": "A vcpus value must be an integer"}, 400
try:
vram = int(reqargs.get('vram'))
except:
except Exception:
return {"message": "A vram value must be an integer"}, 400
# Cast boolean arguments
if bool(strtobool(reqargs.get('serial', 'false'))):
@ -4239,8 +4358,11 @@ class API_Provisioner_Template_System_Root(Resource):
node_autostart,
reqargs.get('migration_method', None),
)
api.add_resource(API_Provisioner_Template_System_Root, '/provisioner/template/system')
# /provisioner/template/system/<template>
class API_Provisioner_Template_System_Element(Resource):
@Authenticator
@ -4345,11 +4467,11 @@ class API_Provisioner_Template_System_Element(Resource):
# Validate arguments
try:
vcpus = int(reqargs.get('vcpus'))
except:
except Exception:
return {"message": "A vcpus value must be an integer"}, 400
try:
vram = int(reqargs.get('vram'))
except:
except Exception:
return {"message": "A vram value must be an integer"}, 400
# Cast boolean arguments
if bool(strtobool(reqargs.get('serial', False))):
@ -4482,8 +4604,11 @@ class API_Provisioner_Template_System_Element(Resource):
return api_provisioner.delete_template_system(
template
)
api.add_resource(API_Provisioner_Template_System_Element, '/provisioner/template/system/<template>')
# /provisioner/template/network
class API_Provisioner_Template_Network_Root(Resource):
@RequestParser([
@ -4583,8 +4708,11 @@ class API_Provisioner_Template_Network_Root(Resource):
reqargs.get('name', None),
reqargs.get('mac_template', None)
)
api.add_resource(API_Provisioner_Template_Network_Root, '/provisioner/template/network')
# /provisioner/template/network/<template>
class API_Provisioner_Template_Network_Element(Resource):
@Authenticator
@ -4665,8 +4793,11 @@ class API_Provisioner_Template_Network_Element(Resource):
return api_provisioner.delete_template_network(
template
)
api.add_resource(API_Provisioner_Template_Network_Element, '/provisioner/template/network/<template>')
# /provisioner/template/network/<template>/net
class API_Provisioner_Template_Network_Net_Root(Resource):
@Authenticator
@ -4702,7 +4833,7 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
{'name': 'vni', 'required': True, 'helpmsg': "A valid VNI must be specified."}
])
@Authenticator
def post(self, reqargs):
def post(self, template, reqargs):
"""
Create a new network in network template {template}
---
@ -4730,8 +4861,11 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
template,
reqargs.get('vni', None)
)
api.add_resource(API_Provisioner_Template_Network_Net_Root, '/provisioner/template/network/<template>/net')
# /provisioner/template/network/<template>/net/<vni>
class API_Provisioner_Template_Network_Net_Element(Resource):
@Authenticator
@ -4808,8 +4942,11 @@ class API_Provisioner_Template_Network_Net_Element(Resource):
template,
vni
)
api.add_resource(API_Provisioner_Template_Network_Net_Element, '/provisioner/template/network/<template>/net/<vni>')
# /provisioner/template/storage
class API_Provisioner_Template_Storage_Root(Resource):
@RequestParser([
@ -4916,8 +5053,11 @@ class API_Provisioner_Template_Storage_Root(Resource):
return api_provisioner.create_template_storage(
reqargs.get('name', None)
)
api.add_resource(API_Provisioner_Template_Storage_Root, '/provisioner/template/storage')
# /provisioner/template/storage/<template>
class API_Provisioner_Template_Storage_Element(Resource):
@Authenticator
@ -4988,8 +5128,11 @@ class API_Provisioner_Template_Storage_Element(Resource):
return api_provisioner.delete_template_storage(
template
)
api.add_resource(API_Provisioner_Template_Storage_Element, '/provisioner/template/storage/<template>')
# /provisioner/template/storage/<template>/disk
class API_Provisioner_Template_Storage_Disk_Root(Resource):
@RequestParser([
@ -5098,8 +5241,11 @@ class API_Provisioner_Template_Storage_Disk_Root(Resource):
reqargs.get('filesystem_arg', []),
reqargs.get('mountpoint', None)
)
api.add_resource(API_Provisioner_Template_Storage_Disk_Root, '/provisioner/template/storage/<template>/disk')
# /provisioner/template/storage/<template>/disk/<disk_id>
class API_Provisioner_Template_Storage_Disk_Element(Resource):
@Authenticator
@ -5222,8 +5368,11 @@ class API_Provisioner_Template_Storage_Disk_Element(Resource):
template,
disk_id
)
api.add_resource(API_Provisioner_Template_Storage_Disk_Element, '/provisioner/template/storage/<template>/disk/<disk_id>')
# /provisioner/userdata
class API_Provisioner_Userdata_Root(Resource):
@RequestParser([
@ -5306,8 +5455,11 @@ class API_Provisioner_Userdata_Root(Resource):
reqargs.get('name', None),
reqargs.get('data', None)
)
api.add_resource(API_Provisioner_Userdata_Root, '/provisioner/userdata')
# /provisioner/userdata/<userdata>
class API_Provisioner_Userdata_Element(Resource):
@Authenticator
@ -5421,8 +5573,11 @@ class API_Provisioner_Userdata_Element(Resource):
return api_provisioner.delete_userdata(
userdata
)
api.add_resource(API_Provisioner_Userdata_Element, '/provisioner/userdata/<userdata>')
# /provisioner/script
class API_Provisioner_Script_Root(Resource):
@RequestParser([
@ -5505,8 +5660,11 @@ class API_Provisioner_Script_Root(Resource):
reqargs.get('name', None),
reqargs.get('data', None)
)
api.add_resource(API_Provisioner_Script_Root, '/provisioner/script')
# /provisioner/script/<script>
class API_Provisioner_Script_Element(Resource):
@Authenticator
@ -5536,7 +5694,7 @@ class API_Provisioner_Script_Element(Resource):
{'name': 'data', 'required': True, 'helpmsg': "A script document must be specified."}
])
@Authenticator
def post(self, script):
def post(self, script, reqargs):
"""
Create a new script {script}
---
@ -5620,8 +5778,11 @@ class API_Provisioner_Script_Element(Resource):
return api_provisioner.delete_script(
script
)
api.add_resource(API_Provisioner_Script_Element, '/provisioner/script/<script>')
# /provisioner/profile
class API_Provisioner_OVA_Root(Resource):
@RequestParser([
@ -5731,8 +5892,11 @@ class API_Provisioner_OVA_Root(Resource):
reqargs.get('name', None),
reqargs.get('ova_size', None),
)
api.add_resource(API_Provisioner_OVA_Root, '/provisioner/ova')
# /provisioner/ova/<ova>
class API_Provisioner_OVA_Element(Resource):
@Authenticator
@ -5822,8 +5986,11 @@ class API_Provisioner_OVA_Element(Resource):
return api_ova.delete_ova(
ova
)
api.add_resource(API_Provisioner_OVA_Element, '/provisioner/ova/<ova>')
# /provisioner/profile
class API_Provisioner_Profile_Root(Resource):
@RequestParser([
@ -5974,8 +6141,11 @@ class API_Provisioner_Profile_Root(Resource):
reqargs.get('ova', None),
reqargs.get('arg', [])
)
api.add_resource(API_Provisioner_Profile_Root, '/provisioner/profile')
# /provisioner/profile/<profile>
class API_Provisioner_Profile_Element(Resource):
@Authenticator
@ -6176,8 +6346,11 @@ class API_Provisioner_Profile_Element(Resource):
return api_provisioner.delete_profile(
profile
)
api.add_resource(API_Provisioner_Profile_Element, '/provisioner/profile/<profile>')
# /provisioner/create
class API_Provisioner_Create_Root(Resource):
@RequestParser([
@ -6258,8 +6431,11 @@ class API_Provisioner_Create_Root(Resource):
script_run_args=reqargs.get('arg', []),
)
return {"task_id": task.id}, 202, {'Location': Api.url_for(api, API_Provisioner_Status_Element, task_id=task.id)}
api.add_resource(API_Provisioner_Create_Root, '/provisioner/create')
# /provisioner/status
class API_Provisioner_Status_Root(Resource):
@Authenticator
@ -6292,8 +6468,11 @@ class API_Provisioner_Status_Root(Resource):
'reserved': queue.reserved()
}
return response
api.add_resource(API_Provisioner_Status_Root, '/provisioner/status')
# /provisioner/status/<task_id>
class API_Provisioner_Status_Element(Resource):
@Authenticator
@ -6352,5 +6531,6 @@ class API_Provisioner_Status_Element(Resource):
'status': str(task.info)
}
return response
api.add_resource(API_Provisioner_Status_Element, '/provisioner/status/<task_id>')
api.add_resource(API_Provisioner_Status_Element, '/provisioner/status/<task_id>')

86
api-daemon/pvcapid/helper.py
View File

@ -21,7 +21,6 @@
###############################################################################
import flask
import json
import lxml.etree as etree
from distutils.util import strtobool as dustrtobool
@ -35,6 +34,9 @@ import daemon_lib.vm as pvc_vm
import daemon_lib.network as pvc_network
import daemon_lib.ceph as pvc_ceph
config = None # Set in this namespace by flaskapi
def strtobool(stringv):
if stringv is None:
return False
@ -42,9 +44,10 @@ def strtobool(stringv):
return bool(stringv)
try:
return bool(dustrtobool(stringv))
except:
except Exception:
return False
#
# Initialization function
#
@ -82,6 +85,7 @@ def initialize_cluster():
return True
#
# Cluster functions
#
@ -95,6 +99,7 @@ def cluster_status():
return retdata, 200
def cluster_maintenance(maint_state='false'):
"""
Set the cluster in or out of maintenance state
@ -113,6 +118,7 @@ def cluster_maintenance(maint_state='false'):
return retdata, retcode
#
# Node functions
#
@ -144,6 +150,7 @@ def node_list(limit=None, daemon_state=None, coordinator_state=None, domain_stat
return retdata, retcode
def node_daemon_state(node):
"""
Return the daemon state of node NODE.
@ -172,6 +179,7 @@ def node_daemon_state(node):
return retdata, retcode
def node_coordinator_state(node):
"""
Return the coordinator state of node NODE.
@ -200,6 +208,7 @@ def node_coordinator_state(node):
return retdata, retcode
def node_domain_state(node):
"""
Return the domain state of node NODE.
@ -225,6 +234,7 @@ def node_domain_state(node):
return retdata, retcode
def node_secondary(node):
"""
Take NODE out of primary router mode.
@ -243,6 +253,7 @@ def node_secondary(node):
}
return output, retcode
def node_primary(node):
"""
Set NODE to primary router mode.
@ -261,6 +272,7 @@ def node_primary(node):
}
return output, retcode
def node_flush(node, wait):
"""
Flush NODE of running VMs.
@ -279,6 +291,7 @@ def node_flush(node, wait):
}
return output, retcode
def node_ready(node, wait):
"""
Restore NODE to active service.
@ -297,6 +310,7 @@ def node_ready(node, wait):
}
return output, retcode
#
# VM functions
#
@ -310,6 +324,7 @@ def vm_is_migrated(vm):
return retdata
def vm_state(vm):
"""
Return the state of virtual machine VM.
@ -342,6 +357,7 @@ def vm_state(vm):
return retdata, retcode
def vm_node(vm):
"""
Return the current node of virtual machine VM.
@ -375,6 +391,7 @@ def vm_node(vm):
return retdata, retcode
def vm_console(vm, lines=None):
"""
Return the current console log for VM.
@ -403,6 +420,7 @@ def vm_console(vm, lines=None):
return retdata, retcode
def vm_list(node=None, state=None, limit=None, is_fuzzy=True):
"""
Return a list of VMs with limit LIMIT.
@ -431,6 +449,7 @@ def vm_list(node=None, state=None, limit=None, is_fuzzy=True):
return retdata, retcode
def vm_define(xml, node, limit, selector, autostart, migration_method):
"""
Define a VM from Libvirt XML in the PVC cluster.
@ -456,6 +475,7 @@ def vm_define(xml, node, limit, selector, autostart, migration_method):
}
return output, retcode
def get_vm_meta(vm):
"""
Get metadata of a VM.
@ -491,6 +511,7 @@ def get_vm_meta(vm):
return retdata, retcode
def update_vm_meta(vm, limit, selector, autostart, provisioner_profile, migration_method):
"""
Update metadata of a VM.
@ -499,7 +520,7 @@ def update_vm_meta(vm, limit, selector, autostart, provisioner_profile, migratio
if autostart is not None:
try:
autostart = bool(strtobool(autostart))
except:
except Exception:
autostart = False
retflag, retdata = pvc_vm.modify_vm_metadata(zk_conn, vm, limit, selector, autostart, provisioner_profile, migration_method)
pvc_common.stopZKConnection(zk_conn)
@ -514,6 +535,7 @@ def update_vm_meta(vm, limit, selector, autostart, provisioner_profile, migratio
}
return output, retcode
def vm_modify(name, restart, xml):
"""
Modify a VM Libvirt XML in the PVC cluster.
@ -538,6 +560,7 @@ def vm_modify(name, restart, xml):
}
return output, retcode
def vm_undefine(name):
"""
Undefine a VM from the PVC cluster.
@ -556,6 +579,7 @@ def vm_undefine(name):
}
return output, retcode
def vm_remove(name):
"""
Remove a VM from the PVC cluster.
@ -574,6 +598,7 @@ def vm_remove(name):
}
return output, retcode
def vm_start(name):
"""
Start a VM in the PVC cluster.
@ -592,6 +617,7 @@ def vm_start(name):
}
return output, retcode
def vm_restart(name, wait):
"""
Restart a VM in the PVC cluster.
@ -610,6 +636,7 @@ def vm_restart(name, wait):
}
return output, retcode
def vm_shutdown(name, wait):
"""
Shutdown a VM in the PVC cluster.
@ -628,6 +655,7 @@ def vm_shutdown(name, wait):
}
return output, retcode
def vm_stop(name):
"""
Forcibly stop a VM in the PVC cluster.
@ -646,6 +674,7 @@ def vm_stop(name):
}
return output, retcode
def vm_disable(name):
"""
Disable a (stopped) VM in the PVC cluster.
@ -664,6 +693,7 @@ def vm_disable(name):
}
return output, retcode
def vm_move(name, node, wait, force_live):
"""
Move a VM to another node.
@ -682,6 +712,7 @@ def vm_move(name, node, wait, force_live):
}
return output, retcode
def vm_migrate(name, node, flag_force, wait, force_live):
"""
Temporarily migrate a VM to another node.
@ -700,6 +731,7 @@ def vm_migrate(name, node, flag_force, wait, force_live):
}
return output, retcode
def vm_unmigrate(name, wait, force_live):
"""
Unmigrate a migrated VM.
@ -718,6 +750,7 @@ def vm_unmigrate(name, wait, force_live):
}
return output, retcode
def vm_flush_locks(vm):
"""
Flush locks of a (stopped) VM.
@ -747,6 +780,7 @@ def vm_flush_locks(vm):
}
return output, retcode
#
# Network functions
#
@ -778,6 +812,7 @@ def net_list(limit=None, is_fuzzy=True):
return retdata, retcode
def net_add(vni, description, nettype, domain, name_servers,
ip4_network, ip4_gateway, ip6_network, ip6_gateway,
dhcp4_flag, dhcp4_start, dhcp4_end):
@ -802,6 +837,7 @@ def net_add(vni, description, nettype, domain, name_servers,
}
return output, retcode
def net_modify(vni, description, domain, name_servers,
ip4_network, ip4_gateway,
ip6_network, ip6_gateway,
@ -827,6 +863,7 @@ def net_modify(vni, description, domain, name_servers,
}
return output, retcode
def net_remove(network):
"""
Remove a virtual client network from the PVC cluster.
@ -845,6 +882,7 @@ def net_remove(network):
}
return output, retcode
def net_dhcp_list(network, limit=None, static=False):
"""
Return a list of DHCP leases in network NETWORK with limit LIMIT.
@ -869,6 +907,7 @@ def net_dhcp_list(network, limit=None, static=False):
return retdata, retcode
def net_dhcp_add(network, ipaddress, macaddress, hostname):
"""
Add a static DHCP lease to a virtual client network.
@ -887,6 +926,7 @@ def net_dhcp_add(network, ipaddress, macaddress, hostname):
}
return output, retcode
def net_dhcp_remove(network, macaddress):
"""
Remove a static DHCP lease from a virtual client network.
@ -905,6 +945,7 @@ def net_dhcp_remove(network, macaddress):
}
return output, retcode
def net_acl_list(network, limit=None, direction=None, is_fuzzy=True):
"""
Return a list of network ACLs in network NETWORK with limit LIMIT.
@ -933,6 +974,7 @@ def net_acl_list(network, limit=None, direction=None, is_fuzzy=True):
return retdata, retcode
def net_acl_add(network, direction, description, rule, order):
"""
Add an ACL to a virtual client network.
@ -951,6 +993,7 @@ def net_acl_add(network, direction, description, rule, order):
}
return output, retcode
def net_acl_remove(network, description):
"""
Remove an ACL from a virtual client network.
@ -969,6 +1012,7 @@ def net_acl_remove(network, description):
}
return output, retcode
#
# Ceph functions
#
@ -987,6 +1031,7 @@ def ceph_status():
return retdata, retcode
def ceph_util():
"""
Get the current Ceph cluster utilization.
@ -1002,6 +1047,7 @@ def ceph_util():
return retdata, retcode
def ceph_osd_list(limit=None):
"""
Get the list of OSDs in the Ceph storage cluster.
@ -1026,6 +1072,7 @@ def ceph_osd_list(limit=None):
return retdata, retcode
def ceph_osd_state(osd):
zk_conn = pvc_common.startZKConnection(config['coordinators'])
retflag, retdata = pvc_ceph.get_list_osd(zk_conn, osd)
@ -1050,6 +1097,7 @@ def ceph_osd_state(osd):
return {"id": osd, "in": in_state, "up": up_state}, retcode
def ceph_osd_add(node, device, weight):
"""
Add a Ceph OSD to the PVC Ceph storage cluster.
@ -1068,6 +1116,7 @@ def ceph_osd_add(node, device, weight):
}
return output, retcode
def ceph_osd_remove(osd_id):
"""
Remove a Ceph OSD from the PVC Ceph storage cluster.
@ -1086,6 +1135,7 @@ def ceph_osd_remove(osd_id):
}
return output, retcode
def ceph_osd_in(osd_id):
"""
Set in a Ceph OSD in the PVC Ceph storage cluster.
@ -1104,6 +1154,7 @@ def ceph_osd_in(osd_id):
}
return output, retcode
def ceph_osd_out(osd_id):
"""
Set out a Ceph OSD in the PVC Ceph storage cluster.
@ -1122,6 +1173,7 @@ def ceph_osd_out(osd_id):
}
return output, retcode
def ceph_osd_set(option):
"""
Set options on a Ceph OSD in the PVC Ceph storage cluster.
@ -1140,6 +1192,7 @@ def ceph_osd_set(option):
}
return output, retcode
def ceph_osd_unset(option):
"""
Unset options on a Ceph OSD in the PVC Ceph storage cluster.
@ -1158,6 +1211,7 @@ def ceph_osd_unset(option):
}
return output, retcode
def ceph_pool_list(limit=None, is_fuzzy=True):
"""
Get the list of RBD pools in the Ceph storage cluster.
@ -1186,6 +1240,7 @@ def ceph_pool_list(limit=None, is_fuzzy=True):
return retdata, retcode
def ceph_pool_add(name, pgs, replcfg):
"""
Add a Ceph RBD pool to the PVC Ceph storage cluster.
@ -1204,6 +1259,7 @@ def ceph_pool_add(name, pgs, replcfg):
}
return output, retcode
def ceph_pool_remove(name):
"""
Remove a Ceph RBD pool to the PVC Ceph storage cluster.
@ -1222,6 +1278,7 @@ def ceph_pool_remove(name):
}
return output, retcode
def ceph_volume_list(pool=None, limit=None, is_fuzzy=True):
"""
Get the list of RBD volumes in the Ceph storage cluster.
@ -1250,6 +1307,7 @@ def ceph_volume_list(pool=None, limit=None, is_fuzzy=True):
return retdata, retcode
def ceph_volume_add(pool, name, size):
"""
Add a Ceph RBD volume to the PVC Ceph storage cluster.
@ -1268,6 +1326,7 @@ def ceph_volume_add(pool, name, size):
}
return output, retcode
def ceph_volume_clone(pool, name, source_volume):
"""
Clone a Ceph RBD volume to a new volume on the PVC Ceph storage cluster.
@ -1286,6 +1345,7 @@ def ceph_volume_clone(pool, name, source_volume):
}
return output, retcode
def ceph_volume_resize(pool, name, size):
"""
Resize an existing Ceph RBD volume in the PVC Ceph storage cluster.
@ -1304,6 +1364,7 @@ def ceph_volume_resize(pool, name, size):
}
return output, retcode
def ceph_volume_rename(pool, name, new_name):
"""
Rename a Ceph RBD volume in the PVC Ceph storage cluster.
@ -1322,6 +1383,7 @@ def ceph_volume_rename(pool, name, new_name):
}
return output, retcode
def ceph_volume_remove(pool, name):
"""
Remove a Ceph RBD volume to the PVC Ceph storage cluster.
@ -1340,6 +1402,7 @@ def ceph_volume_remove(pool, name):
}
return output, retcode
def ceph_volume_upload(pool, volume, img_type):
"""
Upload a raw file via HTTP post to a PVC Ceph volume
@ -1392,8 +1455,14 @@ def ceph_volume_upload(pool, volume, img_type):
# Save the data to the blockdev directly
try:
data.save(dest_blockdev)
except:
# This sets up a custom stream_factory that writes directly into the ova_blockdev,
# rather than the standard stream_factory which writes to a temporary file waiting
# on a save() call. This will break if the API ever uploaded multiple files, but
# this is an acceptable workaround.
def image_stream_factory(total_content_length, filename, content_type, content_length=None):
return open(dest_blockdev, 'wb')
parse_form_data(flask.request.environ, stream_factory=image_stream_factory)
except Exception:
output = {
'message': "Failed to write image file to volume."
}
@ -1457,7 +1526,7 @@ def ceph_volume_upload(pool, volume, img_type):
def ova_stream_factory(total_content_length, filename, content_type, content_length=None):
return open(temp_blockdev, 'wb')
parse_form_data(flask.request.environ, stream_factory=ova_stream_factory)
except:
except Exception:
output = {
'message': "Failed to upload or write image file to temporary volume."
}
@ -1484,6 +1553,7 @@ def ceph_volume_upload(pool, volume, img_type):
cleanup_maps_and_volumes()
return output, retcode
def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True):
"""
Get the list of RBD volume snapshots in the Ceph storage cluster.
@ -1512,6 +1582,7 @@ def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True)
return retdata, retcode
def ceph_volume_snapshot_add(pool, volume, name):
"""
Add a Ceph RBD volume snapshot to the PVC Ceph storage cluster.
@ -1530,6 +1601,7 @@ def ceph_volume_snapshot_add(pool, volume, name):
}
return output, retcode
def ceph_volume_snapshot_rename(pool, volume, name, new_name):
"""
Rename a Ceph RBD volume snapshot in the PVC Ceph storage cluster.
@ -1548,6 +1620,7 @@ def ceph_volume_snapshot_rename(pool, volume, name, new_name):
}
return output, retcode
def ceph_volume_snapshot_remove(pool, volume, name):
"""
Remove a Ceph RBD volume snapshot from the PVC Ceph storage cluster.
@ -1565,4 +1638,3 @@ def ceph_volume_snapshot_remove(pool, volume, name):
'message': retdata.replace('\"', '\'')
}
return output, retcode

13
api-daemon/pvcapid/models.py
View File

@ -20,7 +20,8 @@
#
###############################################################################
from pvcapid.flaskapi import app, db
from pvcapid.flaskapi import db
class DBSystemTemplate(db.Model):
__tablename__ = 'system_template'
@ -54,6 +55,7 @@ class DBSystemTemplate(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBNetworkTemplate(db.Model):
__tablename__ = 'network_template'
@ -70,6 +72,7 @@ class DBNetworkTemplate(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBNetworkElement(db.Model):
__tablename__ = 'network'
@ -84,6 +87,7 @@ class DBNetworkElement(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBStorageTemplate(db.Model):
__tablename__ = 'storage_template'
@ -98,6 +102,7 @@ class DBStorageTemplate(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBStorageElement(db.Model):
__tablename__ = 'storage'
@ -124,6 +129,7 @@ class DBStorageElement(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBUserdata(db.Model):
__tablename__ = 'userdata'
@ -138,6 +144,7 @@ class DBUserdata(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBScript(db.Model):
__tablename__ = 'script'
@ -152,6 +159,7 @@ class DBScript(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBOva(db.Model):
__tablename__ = 'ova'
@ -166,6 +174,7 @@ class DBOva(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBOvaVolume(db.Model):
__tablename__ = 'ova_volume'
@ -188,6 +197,7 @@ class DBOvaVolume(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBProfile(db.Model):
__tablename__ = 'profile'
@ -216,6 +226,7 @@ class DBProfile(db.Model):
def __repr__(self):
return '<id {}>'.format(self.id)
class DBStorageBenchmarks(db.Model):
__tablename__ = 'storage_benchmarks'

34
api-daemon/pvcapid/ova.py
View File

@ -21,31 +21,24 @@
###############################################################################
import flask
import json
import psycopg2
import psycopg2.extras
import os
import re
import time
import math
import tarfile
import shutil
import shlex
import subprocess
import lxml.etree
from werkzeug.formparser import parse_form_data
import daemon_lib.common as pvc_common
import daemon_lib.node as pvc_node
import daemon_lib.vm as pvc_vm
import daemon_lib.network as pvc_network
import daemon_lib.ceph as pvc_ceph
import pvcapid.libvirt_schema as libvirt_schema
import pvcapid.provisioner as provisioner
config = None # Set in this namespace by flaskapi
#
# Common functions
#
@ -62,12 +55,14 @@ def open_database(config):
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
return conn, cur
def close_database(conn, cur, failed=False):
if not failed:
conn.commit()
cur.close()
conn.close()
#
# OVA functions
#
@ -75,11 +70,11 @@ def list_ova(limit, is_fuzzy=True):
if limit:
if is_fuzzy:
# Handle fuzzy vs. non-fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '%' + limit
else:
limit = limit[1:]
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '%'
else:
limit = limit[:-1]
@ -115,6 +110,7 @@ def list_ova(limit, is_fuzzy=True):
else:
return {'message': 'No OVAs found.'}, 404
def delete_ova(name):
ova_data, retcode = list_ova(name, is_fuzzy=False)
if retcode != 200:
@ -164,6 +160,7 @@ def delete_ova(name):
close_database(conn, cur)
return retmsg, retcode
def upload_ova(pool, name, ova_size):
ova_archive = None
@ -233,7 +230,7 @@ def upload_ova(pool, name, ova_size):
def ova_stream_factory(total_content_length, filename, content_type, content_length=None):
return open(ova_blockdev, 'wb')
parse_form_data(flask.request.environ, stream_factory=ova_stream_factory)
except:
except Exception:
output = {
'message': "Failed to upload or write OVA file to temporary volume."
}
@ -255,7 +252,7 @@ def upload_ova(pool, name, ova_size):
return output, retcode
# Parse through the members list and extract the OVF file
for element in set(x for x in members if re.match('.*\.ovf$', x.name)):
for element in set(x for x in members if re.match('.*[.]ovf$', x.name)):
ovf_file = ova_archive.extractfile(element)
# Parse the OVF file to get our VM details
@ -273,12 +270,10 @@ def upload_ova(pool, name, ova_size):
disk_identifier = "sd{}".format(chr(ord('a') + idx))
volume = "ova_{}_{}".format(name, disk_identifier)
dev_src = disk.get('src')
dev_type = dev_src.split('.')[-1]
dev_size_raw = ova_archive.getmember(dev_src).size
vm_volume_size = disk.get('capacity')
# Normalize the dev size to bytes
dev_size_bytes = int(pvc_ceph.format_bytes_fromhuman(dev_size_raw)[:-1])
dev_size = pvc_ceph.format_bytes_fromhuman(dev_size_raw)
def cleanup_img_maps():
@ -321,15 +316,13 @@ def upload_ova(pool, name, ova_size):
# Open the temporary blockdev and seek to byte 0
blk_file = open(temp_blockdev, 'wb')
blk_file.seek(0)
# Write the contents of vmdk_file into blk_file
bytes_written = blk_file.write(vmdk_file.read())
# Close blk_file (and flush the buffers)
blk_file.close()
# Close vmdk_file
vmdk_file.close()
# Perform an OS-level sync
pvc_common.run_os_command('sync')
except:
except Exception:
output = {
'message': "Failed to write image file '{}' to temporary volume.".format(disk.get('src'))
}
@ -419,6 +412,7 @@ def upload_ova(pool, name, ova_size):
retcode = 200
return output, retcode
#
# OVF parser
#
@ -493,7 +487,7 @@ class OVFParser(object):
for item in hardware_list:
try:
item_type = self.RASD_TYPE[item.find("{{{rasd}}}ResourceType".format(rasd=self.RASD_SCHEMA)).text]
except:
except Exception:
continue
quantity = item.find("{{{rasd}}}VirtualQuantity".format(rasd=self.RASD_SCHEMA))
if quantity is None:

85
api-daemon/pvcapid/provisioner.py
View File

@ -20,15 +20,10 @@
#
###############################################################################
import flask
import json
import psycopg2
import psycopg2.extras
import os
import re
import time
import shlex
import subprocess
from distutils.util import strtobool as dustrtobool
@ -42,6 +37,9 @@ import pvcapid.libvirt_schema as libvirt_schema
from pvcapid.ova import list_ova
config = None # Set in this namespace by flaskapi
def strtobool(stringv):
if stringv is None:
return False
@ -49,9 +47,10 @@ def strtobool(stringv):
return bool(stringv)
try:
return bool(dustrtobool(stringv))
except:
except Exception:
return False
#
# Exceptions (used by Celery tasks)
#
@ -61,18 +60,21 @@ class ValidationError(Exception):
"""
pass
class ClusterError(Exception):
"""
An exception that results from the PVC cluster being out of alignment with the action.
"""
pass
class ProvisioningError(Exception):
"""
An exception that results from a failure of a provisioning command.
"""
pass
#
# Common functions
#
@ -89,12 +91,14 @@ def open_database(config):
cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
return conn, cur
def close_database(conn, cur, failed=False):
if not failed:
conn.commit()
cur.close()
conn.close()
#
# Template List functions
#
@ -102,11 +106,11 @@ def list_template(limit, table, is_fuzzy=True):
if limit:
if is_fuzzy:
# Handle fuzzy vs. non-fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '%' + limit
else:
limit = limit[1:]
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '%'
else:
limit = limit[:-1]
@ -146,6 +150,7 @@ def list_template(limit, table, is_fuzzy=True):
return data
def list_template_system(limit, is_fuzzy=True):
"""
Obtain a list of system templates.
@ -156,6 +161,7 @@ def list_template_system(limit, is_fuzzy=True):
else:
return {'message': 'No system templates found.'}, 404
def list_template_network(limit, is_fuzzy=True):
"""
Obtain a list of network templates.
@ -166,6 +172,7 @@ def list_template_network(limit, is_fuzzy=True):
else:
return {'message': 'No network templates found.'}, 404
def list_template_network_vnis(name):
"""
Obtain a list of network template VNIs.
@ -177,6 +184,7 @@ def list_template_network_vnis(name):
else:
return {'message': 'No network template networks found.'}, 404
def list_template_storage(limit, is_fuzzy=True):
"""
Obtain a list of storage templates.
@ -187,6 +195,7 @@ def list_template_storage(limit, is_fuzzy=True):
else:
return {'message': 'No storage templates found.'}, 404
def list_template_storage_disks(name):
"""
Obtain a list of storage template disks.
@ -198,6 +207,7 @@ def list_template_storage_disks(name):
else:
return {'message': 'No storage template disks found.'}, 404
def template_list(limit):
system_templates, code = list_template_system(limit)
if code != 200:
@ -211,6 +221,7 @@ def template_list(limit):
return {"system_templates": system_templates, "network_templates": network_templates, "storage_templates": storage_templates}
#
# Template Create functions
#
@ -234,6 +245,7 @@ def create_template_system(name, vcpu_count, vram_mb, serial=False, vnc=False, v
close_database(conn, cur)
return retmsg, retcode
def create_template_network(name, mac_template=None):
if list_template_network(name, is_fuzzy=False)[-1] != 404:
retmsg = {'message': 'The network template "{}" already exists.'.format(name)}
@ -253,6 +265,7 @@ def create_template_network(name, mac_template=None):
close_database(conn, cur)
return retmsg, retcode
def create_template_network_element(name, vni):
if list_template_network(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The network template "{}" does not exist.'.format(name)}
@ -288,6 +301,7 @@ def create_template_network_element(name, vni):
close_database(conn, cur)
return retmsg, retcode
def create_template_storage(name):
if list_template_storage(name, is_fuzzy=False)[-1] != 404:
retmsg = {'message': 'The storage template "{}" already exists.'.format(name)}
@ -307,6 +321,7 @@ def create_template_storage(name):
close_database(conn, cur)
return retmsg, retcode
def create_template_storage_element(name, disk_id, pool, source_volume=None, disk_size_gb=None, filesystem=None, filesystem_args=[], mountpoint=None):
if list_template_storage(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The storage template "{}" does not exist.'.format(name)}
@ -356,6 +371,7 @@ def create_template_storage_element(name, disk_id, pool, source_volume=None, dis
close_database(conn, cur)
return retmsg, retcode
#
# Template Modify functions
#
@ -370,7 +386,7 @@ def modify_template_system(name, vcpu_count=None, vram_mb=None, serial=None, vnc
if vcpu_count is not None:
try:
vcpu_count = int(vcpu_count)
except:
except Exception:
retmsg = {'message': 'The vcpus value must be an integer.'}
retcode = 400
return retmsg, retcode
@ -379,7 +395,7 @@ def modify_template_system(name, vcpu_count=None, vram_mb=None, serial=None, vnc
if vram_mb is not None:
try:
vram_mb = int(vram_mb)
except:
except Exception:
retmsg = {'message': 'The vram value must be an integer.'}
retcode = 400
return retmsg, retcode
@ -388,7 +404,7 @@ def modify_template_system(name, vcpu_count=None, vram_mb=None, serial=None, vnc
if serial is not None:
try:
serial = bool(strtobool(serial))
except:
except Exception:
retmsg = {'message': 'The serial value must be a boolean.'}
retcode = 400
return retmsg, retcode
@ -397,7 +413,7 @@ def modify_template_system(name, vcpu_count=None, vram_mb=None, serial=None, vnc
if vnc is not None:
try:
vnc = bool(strtobool(vnc))
except:
except Exception:
retmsg = {'message': 'The vnc value must be a boolean.'}
retcode = 400
return retmsg, retcode
@ -415,7 +431,7 @@ def modify_template_system(name, vcpu_count=None, vram_mb=None, serial=None, vnc
if node_autostart is not None:
try:
node_autostart = bool(strtobool(node_autostart))
except:
except Exception:
retmsg = {'message': 'The node_autostart value must be a boolean.'}
retcode = 400
fields.append({'field': 'node_autostart', 'data': node_autostart})
@ -437,6 +453,7 @@ def modify_template_system(name, vcpu_count=None, vram_mb=None, serial=None, vnc
close_database(conn, cur)
return retmsg, retcode
#
# Template Delete functions
#
@ -459,6 +476,7 @@ def delete_template_system(name):
close_database(conn, cur)
return retmsg, retcode
def delete_template_network(name):
if list_template_network(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The network template "{}" does not exist.'.format(name)}
@ -485,6 +503,7 @@ def delete_template_network(name):
close_database(conn, cur)
return retmsg, retcode
def delete_template_network_element(name, vni):
if list_template_network(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The network template "{}" does not exist.'.format(name)}
@ -518,6 +537,7 @@ def delete_template_network_element(name, vni):
close_database(conn, cur)
return retmsg, retcode
def delete_template_storage(name):
if list_template_storage(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The storage template "{}" does not exist.'.format(name)}
@ -544,6 +564,7 @@ def delete_template_storage(name):
close_database(conn, cur)
return retmsg, retcode
def delete_template_storage_element(name, disk_id):
if list_template_storage(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The storage template "{}" does not exist.'.format(name)}
@ -577,6 +598,7 @@ def delete_template_storage_element(name, disk_id):
close_database(conn, cur)
return retmsg, retcode
#
# Userdata functions
#
@ -584,11 +606,11 @@ def list_userdata(limit, is_fuzzy=True):
if limit:
if is_fuzzy:
# Handle fuzzy vs. non-fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '%' + limit
else:
limit = limit[1:]
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '%'
else:
limit = limit[:-1]
@ -608,6 +630,7 @@ def list_userdata(limit, is_fuzzy=True):
else:
return {'message': 'No userdata documents found.'}, 404
def create_userdata(name, userdata):
if list_userdata(name, is_fuzzy=False)[-1] != 404:
retmsg = {'message': 'The userdata document "{}" already exists.'.format(name)}
@ -627,6 +650,7 @@ def create_userdata(name, userdata):
close_database(conn, cur)
return retmsg, retcode
def update_userdata(name, userdata):
if list_userdata(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The userdata "{}" does not exist.'.format(name)}
@ -649,6 +673,7 @@ def update_userdata(name, userdata):
close_database(conn, cur)
return retmsg, retcode
def delete_userdata(name):
if list_userdata(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The userdata "{}" does not exist.'.format(name)}
@ -668,6 +693,7 @@ def delete_userdata(name):
close_database(conn, cur)
return retmsg, retcode
#
# Script functions
#
@ -675,11 +701,11 @@ def list_script(limit, is_fuzzy=True):
if limit:
if is_fuzzy:
# Handle fuzzy vs. non-fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '%' + limit
else:
limit = limit[1:]
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '%'
else:
limit = limit[:-1]
@ -699,6 +725,7 @@ def list_script(limit, is_fuzzy=True):
else:
return {'message': 'No scripts found.'}, 404
def create_script(name, script):
if list_script(name, is_fuzzy=False)[-1] != 404:
retmsg = {'message': 'The script "{}" already exists.'.format(name)}
@ -718,6 +745,7 @@ def create_script(name, script):
close_database(conn, cur)
return retmsg, retcode
def update_script(name, script):
if list_script(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The script "{}" does not exist.'.format(name)}
@ -740,6 +768,7 @@ def update_script(name, script):
close_database(conn, cur)
return retmsg, retcode
def delete_script(name):
if list_script(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The script "{}" does not exist.'.format(name)}
@ -759,6 +788,7 @@ def delete_script(name):
close_database(conn, cur)
return retmsg, retcode
#
# Profile functions
#
@ -766,11 +796,11 @@ def list_profile(limit, is_fuzzy=True):
if limit:
if is_fuzzy:
# Handle fuzzy vs. non-fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '%' + limit
else:
limit = limit[1:]
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '%'
else:
limit = limit[:-1]
@ -797,7 +827,7 @@ def list_profile(limit, is_fuzzy=True):
cur.execute(query, args)
try:
name = cur.fetchone()['name']
except Exception as e:
except Exception:
name = "N/A"
profile_data[etype] = name
# Split the arguments back into a list
@ -810,6 +840,7 @@ def list_profile(limit, is_fuzzy=True):
else:
return {'message': 'No profiles found.'}, 404
def create_profile(name, profile_type, system_template, network_template, storage_template, userdata=None, script=None, ova=None, arguments=None):
if list_profile(name, is_fuzzy=False)[-1] != 404:
retmsg = {'message': 'The profile "{}" already exists.'.format(name)}
@ -899,6 +930,7 @@ def create_profile(name, profile_type, system_template, network_template, storag
close_database(conn, cur)
return retmsg, retcode
def modify_profile(name, profile_type, system_template, network_template, storage_template, userdata, script, ova, arguments=None):
if list_profile(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The profile "{}" does not exist.'.format(name)}
@ -1007,6 +1039,7 @@ def modify_profile(name, profile_type, system_template, network_template, storag
close_database(conn, cur)
return retmsg, retcode
def delete_profile(name):
if list_profile(name, is_fuzzy=False)[-1] != 200:
retmsg = {'message': 'The profile "{}" does not exist.'.format(name)}
@ -1026,6 +1059,7 @@ def delete_profile(name):
close_database(conn, cur)
return retmsg, retcode
#
# Main VM provisioning function - executed by the Celery worker
#
@ -1044,13 +1078,13 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
# Phase 0 - connect to databases
try:
db_conn, db_cur = open_database(config)
except:
except Exception:
print('FATAL - failed to connect to Postgres')
raise Exception
try:
zk_conn = pvc_common.startZKConnection(config['coordinators'])
except:
except Exception:
print('FATAL - failed to connect to Zookeeper')
raise Exception
@ -1182,7 +1216,7 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
cluster_networks, _discard = pvc_network.getClusterNetworkList(zk_conn)
for network in vm_data['networks']:
vni = str(network['vni'])
if not vni in cluster_networks:
if vni not in cluster_networks:
raise ClusterError('The network VNI "{}" is not present on the cluster.'.format(vni))
print("All configured networks for VM are valid")
@ -1212,7 +1246,7 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
pool_information = pvc_ceph.getPoolInformation(zk_conn, pool)
if not pool_information:
raise
except:
except Exception:
raise ClusterError('Pool "{}" is not present on the cluster.'.format(pool))
pool_free_space_gb = int(pool_information['stats']['free_bytes'] / 1024 / 1024 / 1024)
pool_vm_usage_gb = int(pools[pool])
@ -1266,7 +1300,7 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
loader.exec_module(installer_script)
# Verify that the install() function is valid
if not "install" in dir(installer_script):
if "install" not in dir(installer_script):
raise ProvisioningError("Specified script does not contain an install() function.")
print("Provisioning script imported successfully")
@ -1634,4 +1668,3 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
pvc_common.stopZKConnection(zk_conn)
return {'status': 'VM "{}" with profile "{}" has been provisioned and started successfully'.format(vm_name, vm_profile), 'current': 10, 'total': 10}

16
client-cli/cli_lib/ansiprint.py
View File

@ -22,24 +22,40 @@
import datetime
# ANSII colours for output
def red():
return '\033[91m'
def blue():
return '\033[94m'
def cyan():
return '\033[96m'
def green():
return '\033[92m'
def yellow():
return '\033[93m'
def purple():
return '\033[95m'
def bold():
return '\033[1m'
def end():
return '\033[0m'
# Print function
def echo(message, prefix, state):
# Get the date

105
client-cli/cli_lib/ceph.py
View File

@ -20,9 +20,7 @@
#
###############################################################################
import re
import json
import time
import math
from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
@ -34,7 +32,7 @@ from cli_lib.common import UploadProgressBar, call_api
# Supplemental functions
#
# Format byte sizes to/from human-readable units
# Matrix of human-to-byte values
byte_unit_matrix = {
'B': 1,
'K': 1024,
@ -43,6 +41,19 @@ byte_unit_matrix = {
'T': 1024 * 1024 * 1024 * 1024,
'P': 1024 * 1024 * 1024 * 1024 * 1024
}
# Matrix of human-to-metric values
ops_unit_matrix = {
'': 1,
'K': 1000,
'M': 1000 * 1000,
'G': 1000 * 1000 * 1000,
'T': 1000 * 1000 * 1000 * 1000,
'P': 1000 * 1000 * 1000 * 1000 * 1000
}
# Format byte sizes to/from human-readable units
def format_bytes_tohuman(databytes):
datahuman = ''
for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get, reverse=True):
@ -57,6 +68,7 @@ def format_bytes_tohuman(databytes):
return datahuman
def format_bytes_fromhuman(datahuman):
# Trim off human-readable character
dataunit = datahuman[-1]
@ -64,15 +76,8 @@ def format_bytes_fromhuman(datahuman):
databytes = datasize * byte_unit_matrix[dataunit]
return '{}B'.format(databytes)
# Format ops sizes to/from human-readable units
ops_unit_matrix = {
'': 1,
'K': 1000,
'M': 1000*1000,
'G': 1000*1000*1000,
'T': 1000*1000*1000*1000,
'P': 1000*1000*1000*1000*1000
}
def format_ops_tohuman(dataops):
datahuman = ''
for unit in sorted(ops_unit_matrix, key=ops_unit_matrix.get, reverse=True):
@ -87,6 +92,7 @@ def format_ops_tohuman(dataops):
return datahuman
def format_ops_fromhuman(datahuman):
# Trim off human-readable character
dataunit = datahuman[-1]
@ -94,10 +100,12 @@ def format_ops_fromhuman(datahuman):
dataops = datasize * ops_unit_matrix[dataunit]
return '{}'.format(dataops)
def format_pct_tohuman(datapct):
datahuman = "{0:.1f}".format(float(datapct * 100.0))
return datahuman
#
# Status functions
#
@ -116,6 +124,7 @@ def ceph_status(config):
else:
return False, response.json().get('message', '')
def ceph_util(config):
"""
Get utilization of the Ceph cluster
@ -131,6 +140,7 @@ def ceph_util(config):
else:
return False, response.json().get('message', '')
def format_raw_output(status_data):
ainformation = list()
ainformation.append('{bold}Ceph cluster {stype} (primary node {end}{blue}{primary}{end}{bold}){end}\n'.format(bold=ansiprint.bold(), end=ansiprint.end(), blue=ansiprint.blue(), stype=status_data['type'], primary=status_data['primary_node']))
@ -139,6 +149,7 @@ def format_raw_output(status_data):
return '\n'.join(ainformation)
#
# OSD functions
#
@ -157,6 +168,7 @@ def ceph_osd_info(config, osd):
else:
return False, response.json().get('message', '')
def ceph_osd_list(config, limit):
"""
Get list information about Ceph OSDs (limited by {limit})
@ -176,6 +188,7 @@ def ceph_osd_list(config, limit):
else:
return False, response.json().get('message', '')
def ceph_osd_add(config, node, device, weight):
"""
Add new Ceph OSD
@ -198,6 +211,7 @@ def ceph_osd_add(config, node, device, weight):
return retstatus, response.json().get('message', '')
def ceph_osd_remove(config, osdid):
"""
Remove Ceph OSD
@ -218,6 +232,7 @@ def ceph_osd_remove(config, osdid):
return retstatus, response.json().get('message', '')
def ceph_osd_state(config, osdid, state):
"""
Set state of Ceph OSD
@ -238,6 +253,7 @@ def ceph_osd_state(config, osdid, state):
return retstatus, response.json().get('message', '')
def ceph_osd_option(config, option, action):
"""
Set cluster option of Ceph OSDs
@ -259,6 +275,7 @@ def ceph_osd_option(config, option, action):
return retstatus, response.json().get('message', '')
def getOutputColoursOSD(osd_information):
# Set the UP status
if osd_information['stats']['up'] == 1:
@ -278,6 +295,7 @@ def getOutputColoursOSD(osd_information):
return osd_up_flag, osd_up_colour, osd_in_flag, osd_in_colour
def format_list_osd(osd_list):
# Handle empty list
if not osd_list:
@ -447,8 +465,7 @@ Wr: {osd_wrops: <{osd_wrops_length}} \
osd_wrops='OPS',
osd_wrdata='Data',
osd_rdops='OPS',
osd_rddata='Data'
)
osd_rddata='Data')
)
for osd_information in sorted(osd_list, key=lambda x: int(x['id'])):
@ -518,8 +535,7 @@ Wr: {osd_wrops: <{osd_wrops_length}} \
osd_wrops=osd_information['stats']['wr_ops'],
osd_wrdata=osd_information['stats']['wr_data'],
osd_rdops=osd_information['stats']['rd_ops'],
osd_rddata=osd_information['stats']['rd_data']
)
osd_rddata=osd_information['stats']['rd_data'])
)
return '\n'.join(osd_list_output)
@ -543,6 +559,7 @@ def ceph_pool_info(config, pool):
else:
return False, response.json().get('message', '')
def ceph_pool_list(config, limit):
"""
Get list information about Ceph OSDs (limited by {limit})
@ -562,6 +579,7 @@ def ceph_pool_list(config, limit):
else:
return False, response.json().get('message', '')
def ceph_pool_add(config, pool, pgs, replcfg):
"""
Add new Ceph OSD
@ -584,6 +602,7 @@ def ceph_pool_add(config, pool, pgs, replcfg):
return retstatus, response.json().get('message', '')
def ceph_pool_remove(config, pool):
"""
Remove Ceph OSD
@ -604,6 +623,7 @@ def ceph_pool_remove(config, pool):
return retstatus, response.json().get('message', '')
def format_list_pool(pool_list):
# Handle empty list
if not pool_list:
@ -748,8 +768,7 @@ Wr: {pool_write_ops: <{pool_write_ops_length}} \
pool_write_ops='OPS',
pool_write_data='Data',
pool_read_ops='OPS',
pool_read_data='Data'
)
pool_read_data='Data')
)
for pool_information in sorted(pool_list, key=lambda x: int(x['stats']['id'])):
@ -796,8 +815,7 @@ Wr: {pool_write_ops: <{pool_write_ops_length}} \
pool_write_ops=pool_information['stats']['write_ops'],
pool_write_data=pool_information['stats']['write_bytes'],
pool_read_ops=pool_information['stats']['read_ops'],
pool_read_data=pool_information['stats']['read_bytes']
)
pool_read_data=pool_information['stats']['read_bytes'])
)
return '\n'.join(pool_list_output)
@ -821,6 +839,7 @@ def ceph_volume_info(config, pool, volume):
else:
return False, response.json().get('message', '')
def ceph_volume_list(config, limit, pool):
"""
Get list information about Ceph volumes (limited by {limit} and by {pool})
@ -842,6 +861,7 @@ def ceph_volume_list(config, limit, pool):
else:
return False, response.json().get('message', '')
def ceph_volume_add(config, pool, volume, size):
"""
Add new Ceph volume
@ -864,6 +884,7 @@ def ceph_volume_add(config, pool, volume, size):
return retstatus, response.json().get('message', '')
def ceph_volume_upload(config, pool, volume, image_format, image_file):
"""
Upload a disk image to a Ceph volume
@ -899,6 +920,7 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
return retstatus, response.json().get('message', '')
def ceph_volume_remove(config, pool, volume):
"""
Remove Ceph volume
@ -916,6 +938,7 @@ def ceph_volume_remove(config, pool, volume):
return retstatus, response.json().get('message', '')
def ceph_volume_modify(config, pool, volume, new_name=None, new_size=None):
"""
Modify Ceph volume
@ -940,6 +963,7 @@ def ceph_volume_modify(config, pool, volume, new_name=None, new_size=None):
return retstatus, response.json().get('message', '')
def ceph_volume_clone(config, pool, volume, new_volume):
"""
Clone Ceph volume
@ -960,6 +984,7 @@ def ceph_volume_clone(config, pool, volume, new_volume):
return retstatus, response.json().get('message', '')
def format_list_volume(volume_list):
# Handle empty list
if not volume_list:
@ -1039,8 +1064,7 @@ def format_list_volume(volume_list):
volume_objects='Objects',
volume_order='Order',
volume_format='Format',
volume_features='Features',
)
volume_features='Features')
)
for volume_information in volume_list:
@ -1068,8 +1092,7 @@ def format_list_volume(volume_list):
volume_objects=volume_information['stats']['objects'],
volume_order=volume_information['stats']['order'],
volume_format=volume_information['stats']['format'],
volume_features=','.join(volume_information['stats']['features']),
)
volume_features=','.join(volume_information['stats']['features']))
)
return '\n'.join(sorted(volume_list_output))
@ -1093,6 +1116,7 @@ def ceph_snapshot_info(config, pool, volume, snapshot):
else:
return False, response.json().get('message', '')
def ceph_snapshot_list(config, limit, volume, pool):
"""
Get list information about Ceph snapshots (limited by {limit}, by {pool}, or by {volume})
@ -1116,6 +1140,7 @@ def ceph_snapshot_list(config, limit, volume, pool):
else:
return False, response.json().get('message', '')
def ceph_snapshot_add(config, pool, volume, snapshot):
"""
Add new Ceph snapshot
@ -1138,6 +1163,7 @@ def ceph_snapshot_add(config, pool, volume, snapshot):
return retstatus, response.json().get('message', '')
def ceph_snapshot_remove(config, pool, volume, snapshot):
"""
Remove Ceph snapshot
@ -1155,6 +1181,7 @@ def ceph_snapshot_remove(config, pool, volume, snapshot):
return retstatus, response.json().get('message', '')
def ceph_snapshot_modify(config, pool, volume, snapshot, new_name=None):
"""
Modify Ceph snapshot
@ -1177,6 +1204,7 @@ def ceph_snapshot_modify(config, pool, volume, snapshot, new_name=None):
return retstatus, response.json().get('message', '')
def format_list_snapshot(snapshot_list):
# Handle empty list
if not snapshot_list:
@ -1224,8 +1252,7 @@ def format_list_snapshot(snapshot_list):
snapshot_pool_length=snapshot_pool_length,
snapshot_name='Name',
snapshot_volume='Volume',
snapshot_pool='Pool',
)
snapshot_pool='Pool')
)
for snapshot_information in snapshot_list:
@ -1244,12 +1271,12 @@ def format_list_snapshot(snapshot_list):
snapshot_pool_length=snapshot_pool_length,
snapshot_name=snapshot_name,
snapshot_volume=snapshot_volume,
snapshot_pool=snapshot_pool,
)
snapshot_pool=snapshot_pool)
)
return '\n'.join(sorted(snapshot_list_output))
#
# Benchmark functions
#
@ -1275,6 +1302,7 @@ def ceph_benchmark_run(config, pool):
return retvalue, retdata
def ceph_benchmark_list(config, job):
"""
View results of one or more previous benchmark runs
@ -1301,10 +1329,10 @@ def ceph_benchmark_list(config, job):
return retvalue, retdata
def format_list_benchmark(config, benchmark_information):
benchmark_list_output = []
benchmark_id_length = 3
benchmark_job_length = 20
benchmark_bandwidth_length = dict()
benchmark_iops_length = dict()
@ -1351,8 +1379,7 @@ def format_list_benchmark(config, benchmark_information):
rand_header_length=benchmark_bandwidth_length['rand_read_4K'] + benchmark_bandwidth_length['rand_write_4K'] + benchmark_iops_length['rand_read_4K'] + benchmark_iops_length['rand_write_4K'] + 2,
benchmark_job='Benchmark Job',
seq_header='Sequential (4M blocks):',
rand_header='Random (4K blocks):'
)
rand_header='Random (4K blocks):')
)
benchmark_list_output.append('{bold}\
@ -1373,8 +1400,7 @@ def format_list_benchmark(config, benchmark_information):
seq_benchmark_bandwidth='R/W Bandwith/s',
seq_benchmark_iops='R/W IOPS',
rand_benchmark_bandwidth='R/W Bandwith/s',
rand_benchmark_iops='R/W IOPS'
)
rand_benchmark_iops='R/W IOPS')
)
for benchmark in benchmark_information:
@ -1398,7 +1424,6 @@ def format_list_benchmark(config, benchmark_information):
rand_benchmark_bandwidth = "{} / {}".format(benchmark_bandwidth['rand_read_4K'], benchmark_bandwidth['rand_write_4K'])
rand_benchmark_iops = "{} / {}".format(benchmark_iops['rand_read_4K'], benchmark_iops['rand_write_4K'])
benchmark_list_output.append('{bold}\
{benchmark_job: <{benchmark_job_length}} \
{seq_benchmark_bandwidth: <{seq_benchmark_bandwidth_length}} \
@ -1417,17 +1442,13 @@ def format_list_benchmark(config, benchmark_information):
seq_benchmark_bandwidth=seq_benchmark_bandwidth,
seq_benchmark_iops=seq_benchmark_iops,
rand_benchmark_bandwidth=rand_benchmark_bandwidth,
rand_benchmark_iops=rand_benchmark_iops
)
rand_benchmark_iops=rand_benchmark_iops)
)
return '\n'.join(benchmark_list_output)
def format_info_benchmark(config, benchmark_information):
# Load information from benchmark output
benchmark_id = benchmark_information[0]['id']
benchmark_job = benchmark_information[0]['job']
def format_info_benchmark(config, benchmark_information):
if benchmark_information[0]['benchmark_result'] == "Running":
return "Benchmark test is still running."
@ -1471,7 +1492,7 @@ def format_info_benchmark(config, benchmark_information):
for element in benchmark_details[test]['bandwidth']:
try:
_element_length = len(format_bytes_tohuman(int(float(benchmark_details[test]['bandwidth'][element]))))
except:
except Exception:
_element_length = len(benchmark_details[test]['bandwidth'][element])
if _element_length > bandwidth_column_length:
bandwidth_column_length = _element_length
@ -1479,7 +1500,7 @@ def format_info_benchmark(config, benchmark_information):
for element in benchmark_details[test]['iops']:
try:
_element_length = len(format_ops_tohuman(int(float(benchmark_details[test]['iops'][element]))))
except:
except Exception:
_element_length = len(benchmark_details[test]['iops'][element])
if _element_length > iops_column_length:
iops_column_length = _element_length
@ -1494,8 +1515,6 @@ def format_info_benchmark(config, benchmark_information):
if _element_length > cpuutil_column_length:
cpuutil_column_length = _element_length
for test in benchmark_details:
ainformation.append('')

12
client-cli/cli_lib/cluster.py
View File

@ -25,6 +25,7 @@ import json
import cli_lib.ansiprint as ansiprint
from cli_lib.common import call_api
def initialize(config):
"""
Initialize the PVC cluster
@ -42,6 +43,7 @@ def initialize(config):
return retstatus, response.json().get('message', '')
def maintenance_mode(config, state):
"""
Enable or disable PVC cluster maintenance mode
@ -62,6 +64,7 @@ def maintenance_mode(config, state):
return retstatus, response.json().get('message', '')
def get_info(config):
"""
Get status of the PVC cluster
@ -77,6 +80,7 @@ def get_info(config):
else:
return False, response.json().get('message', '')
def format_info(cluster_information, oformat):
if oformat == 'json':
return json.dumps(cluster_information)
@ -105,15 +109,11 @@ def format_info(cluster_information, oformat):
ainformation.append('{}Cluster health:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), health_colour, cluster_information['health'], ansiprint.end()))
if cluster_information['health_msg']:
for line in cluster_information['health_msg']:
ainformation.append(
' > {}'.format(line)
)
ainformation.append(' > {}'.format(line))
ainformation.append('{}Storage health:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), storage_health_colour, cluster_information['storage_health'], ansiprint.end()))
if cluster_information['storage_health_msg']:
for line in cluster_information['storage_health_msg']:
ainformation.append(
' > {}'.format(line)
)
ainformation.append(' > {}'.format(line))
ainformation.append('')
ainformation.append('{}Primary node:{} {}'.format(ansiprint.purple(), ansiprint.end(), cluster_information['primary_node']))
ainformation.append('{}Cluster upstream IP:{} {}'.format(ansiprint.purple(), ansiprint.end(), cluster_information['upstream_ip']))

7
client-cli/cli_lib/common.py
View File

@ -21,13 +21,13 @@
###############################################################################
import os
import io
import math
import time
import requests
import click
from urllib3 import disable_warnings
def format_bytes(size_bytes):
byte_unit_matrix = {
'B': 1,
@ -45,6 +45,7 @@ def format_bytes(size_bytes):
break
return human_bytes
def format_metric(integer):
integer_unit_matrix = {
'': 1,
@ -62,6 +63,7 @@ def format_metric(integer):
break
return human_integer
class UploadProgressBar(object):
def __init__(self, filename, end_message='', end_nl=True):
file_size = os.path.getsize(filename)
@ -104,6 +106,7 @@ class UploadProgressBar(object):
if self.end_message:
click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
class ErrorResponse(requests.Response):
def __init__(self, json_data, status_code):
self.json_data = json_data
@ -112,6 +115,7 @@ class ErrorResponse(requests.Response):
def json(self):
return self.json_data
def call_api(config, operation, request_uri, headers={}, params=None, data=None, files=None):
# Craft the URI
uri = '{}://{}{}{}'.format(
@ -183,4 +187,3 @@ def call_api(config, operation, request_uri, headers={}, params=None, data=None,
# Return the response object
return response

50
client-cli/cli_lib/network.py
View File

@ -20,12 +20,11 @@
#
###############################################################################
import difflib
import colorama
import re
import cli_lib.ansiprint as ansiprint
from cli_lib.common import call_api
def isValidMAC(macaddr):
allowed = re.compile(r"""
(
@ -39,6 +38,7 @@ def isValidMAC(macaddr):
else:
return False
def isValidIP(ipaddr):
ip4_blocks = str(ipaddr).split(".")
if len(ip4_blocks) == 4:
@ -52,6 +52,7 @@ def isValidIP(ipaddr):
return True
return False
#
# Primary functions
#
@ -70,6 +71,7 @@ def net_info(config, net):
else:
return False, response.json().get('message', '')
def net_list(config, limit):
"""
Get list information about networks (limited by {limit})
@ -89,6 +91,7 @@ def net_list(config, limit):
else:
return False, response.json().get('message', '')
def net_add(config, vni, description, nettype, domain, name_servers, ip4_network, ip4_gateway, ip6_network, ip6_gateway, dhcp4_flag, dhcp4_start, dhcp4_end):
"""
Add new network
@ -120,6 +123,7 @@ def net_add(config, vni, description, nettype, domain, name_servers, ip4_network
return retstatus, response.json().get('message', '')
def net_modify(config, net, description, domain, name_servers, ip4_network, ip4_gateway, ip6_network, ip6_gateway, dhcp4_flag, dhcp4_start, dhcp4_end):
"""
Modify a network
@ -159,6 +163,7 @@ def net_modify(config, net, description, domain, name_servers, ip4_network, ip4_
return retstatus, response.json().get('message', '')
def net_remove(config, net):
"""
Remove a network
@ -176,6 +181,7 @@ def net_remove(config, net):
return retstatus, response.json().get('message', '')
#
# DHCP lease functions
#
@ -194,6 +200,7 @@ def net_dhcp_info(config, net, mac):
else:
return False, response.json().get('message', '')
def net_dhcp_list(config, net, limit, only_static=False):
"""
Get list information about leases (limited by {limit})
@ -218,6 +225,7 @@ def net_dhcp_list(config, net, limit, only_static=False):
else:
return False, response.json().get('message', '')
def net_dhcp_add(config, net, ipaddr, macaddr, hostname):
"""
Add new network DHCP lease
@ -240,6 +248,7 @@ def net_dhcp_add(config, net, ipaddr, macaddr, hostname):
return retstatus, response.json().get('message', '')
def net_dhcp_remove(config, net, mac):
"""
Remove a network DHCP lease
@ -257,6 +266,7 @@ def net_dhcp_remove(config, net, mac):
return retstatus, response.json().get('message', '')
#
# ACL functions
#
@ -275,6 +285,7 @@ def net_acl_info(config, net, description):
else:
return False, response.json().get('message', '')
def net_acl_list(config, net, limit, direction):
"""
Get list information about ACLs (limited by {limit})
@ -296,6 +307,7 @@ def net_acl_list(config, net, limit, direction):
else:
return False, response.json().get('message', '')
def net_acl_add(config, net, direction, description, rule, order):
"""
Add new network acl
@ -320,7 +332,9 @@ def net_acl_add(config, net, direction, description, rule, order):
return retstatus, response.json().get('message', '')
def net_acl_remove(config, net, description):
"""
Remove a network ACL
@ -362,6 +376,7 @@ def getOutputColours(network_information):
return v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour
def format_info(config, network_information, long_output):
if not network_information:
return "No network found"
@ -420,6 +435,7 @@ def format_info(config, network_information, long_output):
# Join it all together
return '\n'.join(ainformation)
def format_list(config, network_list):
if not network_list:
return "No network found"
@ -481,8 +497,7 @@ def format_list(config, network_list):
net_v6_flag='IPv6',
net_dhcp6_flag='DHCPv6',
net_v4_flag='IPv4',
net_dhcp4_flag='DHCPv4',
)
net_dhcp4_flag='DHCPv4')
)
for network_information in network_list:
@ -497,13 +512,7 @@ def format_list(config, network_list):
else:
v6_flag = 'False'
if network_information['ip4']['dhcp_flag'] == "True":
dhcp4_range = '{} - {}'.format(network_information['ip4']['dhcp_start'], network_information['ip4']['dhcp_end'])
else:
dhcp4_range = 'N/A'
network_list_output.append(
'{bold}\
network_list_output.append('{bold}\
{net_vni: <{net_vni_length}} \
{net_description: <{net_description_length}} \
{net_nettype: <{net_nettype_length}} \
@ -535,12 +544,12 @@ def format_list(config, network_list):
v4_flag_colour=v4_flag_colour,
net_dhcp4_flag=network_information['ip4']['dhcp_flag'],
dhcp4_flag_colour=dhcp4_flag_colour,
colour_off=ansiprint.end()
)
colour_off=ansiprint.end())
)
return '\n'.join(sorted(network_list_output))
def format_list_dhcp(dhcp_lease_list):
dhcp_lease_list_output = []
@ -579,8 +588,7 @@ def format_list_dhcp(dhcp_lease_list):
lease_hostname='Hostname',
lease_ip4_address='IP Address',
lease_mac_address='MAC Address',
lease_timestamp='Timestamp'
)
lease_timestamp='Timestamp')
)
for dhcp_lease_information in dhcp_lease_list:
@ -599,12 +607,12 @@ def format_list_dhcp(dhcp_lease_list):
lease_hostname=str(dhcp_lease_information['hostname']),
lease_ip4_address=str(dhcp_lease_information['ip4_address']),
lease_mac_address=str(dhcp_lease_information['mac_address']),
lease_timestamp=str(dhcp_lease_information['timestamp'])
)
lease_timestamp=str(dhcp_lease_information['timestamp']))
)
return '\n'.join(sorted(dhcp_lease_list_output))
def format_list_acl(acl_list):
# Handle when we get an empty entry
if not acl_list:
@ -650,8 +658,7 @@ def format_list_acl(acl_list):
acl_direction='Direction',
acl_order='Order',
acl_description='Description',
acl_rule='Rule',
)
acl_rule='Rule')
)
for acl_information in acl_list:
@ -670,8 +677,7 @@ def format_list_acl(acl_list):
acl_direction=acl_information['direction'],
acl_order=acl_information['order'],
acl_description=acl_information['description'],
acl_rule=acl_information['rule'],
)
acl_rule=acl_information['rule'])
)
return '\n'.join(sorted(acl_list_output))

7
client-cli/cli_lib/node.py
View File

@ -23,6 +23,7 @@
import cli_lib.ansiprint as ansiprint
from cli_lib.common import call_api
#
# Primary functions
#
@ -46,6 +47,7 @@ def node_coordinator_state(config, node, action):
return retstatus, response.json().get('message', '')
def node_domain_state(config, node, action, wait):
"""
Set node domain state state (flush/ready)
@ -67,6 +69,7 @@ def node_domain_state(config, node, action, wait):
return retstatus, response.json().get('message', '')
def node_info(config, node):
"""
Get information about node
@ -82,6 +85,7 @@ def node_info(config, node):
else:
return False, response.json().get('message', '')
def node_list(config, limit, target_daemon_state, target_coordinator_state, target_domain_state):
"""
Get list information about nodes (limited by {limit})
@ -107,6 +111,7 @@ def node_list(config, limit, target_daemon_state, target_coordinator_state, targ
else:
return False, response.json().get('message', '')
#
# Output display functions
#
@ -148,6 +153,7 @@ def getOutputColours(node_information):
return daemon_state_colour, coordinator_state_colour, domain_state_colour, mem_allocated_colour, mem_provisioned_colour
def format_info(node_information, long_output):
daemon_state_colour, coordinator_state_colour, domain_state_colour, mem_allocated_colour, mem_provisioned_colour = getOutputColours(node_information)
@ -178,6 +184,7 @@ def format_info(node_information, long_output):
ainformation.append('')
return '\n'.join(ainformation)
def format_list(node_list, raw):
# Handle single-element lists
if not isinstance(node_list, list):

71
client-cli/cli_lib/provisioner.py
View File

@ -20,9 +20,6 @@
#
###############################################################################
import time
import re
import subprocess
import ast
from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
@ -30,6 +27,7 @@ from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncod
import cli_lib.ansiprint as ansiprint
from cli_lib.common import UploadProgressBar, call_api
#
# Primary functions
#
@ -48,6 +46,7 @@ def template_info(config, template, template_type):
else:
return False, response.json().get('message', '')
def template_list(config, limit, template_type=None):
"""
Get list information about templates (limited by {limit})
@ -70,6 +69,7 @@ def template_list(config, limit, template_type=None):
else:
return False, response.json().get('message', '')
def template_add(config, params, template_type=None):
"""
Add a new template of {template_type} with {params}
@ -87,6 +87,7 @@ def template_add(config, params, template_type=None):
return retvalue, response.json().get('message', '')
def template_modify(config, params, name, template_type):
"""
Modify an existing template of {template_type} with {params}
@ -104,6 +105,7 @@ def template_modify(config, params, name, template_type):
return retvalue, response.json().get('message', '')
def template_remove(config, name, template_type):
"""
Remove template {name} of {template_type}
@ -121,6 +123,7 @@ def template_remove(config, name, template_type):
return retvalue, response.json().get('message', '')
def template_element_add(config, name, element_id, params, element_type=None, template_type=None):
"""
Add a new template element of {element_type} with {params} to template {name} of {template_type}
@ -138,6 +141,7 @@ def template_element_add(config, name, element_id, params, element_type=None, te
return retvalue, response.json().get('message', '')
def template_element_remove(config, name, element_id, element_type=None, template_type=None):
"""
Remove template element {element_id} of {element_type} from template {name} of {template_type}
@ -155,6 +159,7 @@ def template_element_remove(config, name, element_id, element_type=None, templat
return retvalue, response.json().get('message', '')
def userdata_info(config, userdata):
"""
Get information about userdata
@ -170,6 +175,7 @@ def userdata_info(config, userdata):
else:
return False, response.json().get('message', '')
def userdata_list(config, limit):
"""
Get list information about userdatas (limited by {limit})
@ -189,6 +195,7 @@ def userdata_list(config, limit):
else:
return False, response.json().get('message', '')
def userdata_show(config, name):
"""
Get information about userdata name
@ -204,6 +211,7 @@ def userdata_show(config, name):
else:
return False, response.json().get('message', '')
def userdata_add(config, params):
"""
Add a new userdata with {params}
@ -230,6 +238,7 @@ def userdata_add(config, params):
return retvalue, response.json().get('message', '')
def userdata_modify(config, name, params):
"""
Modify userdata {name} with {params}
@ -255,6 +264,7 @@ def userdata_modify(config, name, params):
return retvalue, response.json().get('message', '')
def userdata_remove(config, name):
"""
Remove userdata {name}
@ -272,6 +282,7 @@ def userdata_remove(config, name):
return retvalue, response.json().get('message', '')
def script_info(config, script):
"""
Get information about script
@ -287,6 +298,7 @@ def script_info(config, script):
else:
return False, response.json().get('message', '')
def script_list(config, limit):
"""
Get list information about scripts (limited by {limit})
@ -306,6 +318,7 @@ def script_list(config, limit):
else:
return False, response.json().get('message', '')
def script_show(config, name):
"""
Get information about script name
@ -321,6 +334,7 @@ def script_show(config, name):
else:
return False, response.json().get('message', '')
def script_add(config, params):
"""
Add a new script with {params}
@ -347,6 +361,7 @@ def script_add(config, params):
return retvalue, response.json().get('message', '')
def script_modify(config, name, params):
"""
Modify script {name} with {params}
@ -372,6 +387,7 @@ def script_modify(config, name, params):
return retvalue, response.json().get('message', '')
def script_remove(config, name):
"""
Remove script {name}
@ -389,6 +405,7 @@ def script_remove(config, name):
return retvalue, response.json().get('message', '')
def ova_info(config, name):
"""
Get information about OVA image {name}
@ -404,6 +421,7 @@ def ova_info(config, name):
else:
return False, response.json().get('message', '')
def ova_list(config, limit):
"""
Get list information about OVA images (limited by {limit})
@ -423,6 +441,7 @@ def ova_list(config, limit):
else:
return False, response.json().get('message', '')
def ova_upload(config, name, ova_file, params):
"""
Upload an OVA image to the cluster
@ -455,6 +474,7 @@ def ova_upload(config, name, ova_file, params):
return retstatus, response.json().get('message', '')
def ova_remove(config, name):
"""
Remove OVA image {name}
@ -472,6 +492,7 @@ def ova_remove(config, name):
return retvalue, response.json().get('message', '')
def profile_info(config, profile):
"""
Get information about profile
@ -487,6 +508,7 @@ def profile_info(config, profile):
else:
return False, response.json().get('message', '')
def profile_list(config, limit):
"""
Get list information about profiles (limited by {limit})
@ -506,6 +528,7 @@ def profile_list(config, limit):
else:
return False, response.json().get('message', '')
def profile_add(config, params):
"""
Add a new profile with {params}
@ -523,6 +546,7 @@ def profile_add(config, params):
return retvalue, response.json().get('message', '')
def profile_modify(config, name, params):
"""
Modify profile {name} with {params}
@ -540,6 +564,7 @@ def profile_modify(config, name, params):
return retvalue, response.json().get('message', '')
def profile_remove(config, name):
"""
Remove profile {name}
@ -557,6 +582,7 @@ def profile_remove(config, name):
return retvalue, response.json().get('message', '')
def vm_create(config, name, profile, wait_flag, define_flag, start_flag, script_args):
"""
Create a new VM named {name} with profile {profile}
@ -587,6 +613,7 @@ def vm_create(config, name, profile, wait_flag, define_flag, start_flag, script_
return retvalue, retdata
def task_status(config, task_id=None, is_watching=False):
"""
Get information about provisioner job {task_id} or all tasks if None
@ -661,6 +688,7 @@ def task_status(config, task_id=None, is_watching=False):
return retvalue, retdata
#
# Format functions
#
@ -703,6 +731,7 @@ def format_list_template(template_data, template_type=None):
return '\n'.join(ainformation)
def format_list_template_system(template_data):
if isinstance(template_data, dict):
template_data = [template_data]
@ -804,13 +833,9 @@ Meta: {template_node_limit: <{template_node_limit_length}} \
template_node_limit='Limit',
template_node_selector='Selector',
template_node_autostart='Autostart',
template_migration_method='Migration'
)
template_migration_method='Migration')
# Keep track of nets we found to be valid to cut down on duplicate API hits
valid_net_list = []
# Format the string (elements)
for template in sorted(template_data, key=lambda i: i.get('name', None)):
template_list_output.append(
'{bold}{template_name: <{template_name_length}} {template_id: <{template_id_length}} \
@ -854,6 +879,7 @@ Meta: {template_node_limit: <{template_node_limit_length}} \
return True, ''
def format_list_template_network(template_template):
if isinstance(template_template, dict):
template_template = [template_template]
@ -904,8 +930,7 @@ def format_list_template_network(template_template):
template_name='Name',
template_id='ID',
template_mac_template='MAC template',
template_networks='Network VNIs'
)
template_networks='Network VNIs')
# Format the string (elements)
for template in sorted(template_template, key=lambda i: i.get('name', None)):
@ -928,6 +953,7 @@ def format_list_template_network(template_template):
return '\n'.join([template_list_output_header] + template_list_output)
def format_list_template_storage(template_template):
if isinstance(template_template, dict):
template_template = [template_template]
@ -1013,8 +1039,7 @@ def format_list_template_storage(template_template):
template_disk_size='Size [GB]',
template_disk_filesystem='Filesystem',
template_disk_fsargs='Arguments',
template_disk_mountpoint='Mountpoint'
)
template_disk_mountpoint='Mountpoint')
# Format the string (elements)
for template in sorted(template_template, key=lambda i: i.get('name', None)):
@ -1063,6 +1088,7 @@ def format_list_template_storage(template_template):
return '\n'.join([template_list_output_header] + template_list_output)
def format_list_userdata(userdata_data, lines=None):
if isinstance(userdata_data, dict):
userdata_data = [userdata_data]
@ -1072,7 +1098,6 @@ def format_list_userdata(userdata_data, lines=None):
# Determine optimal column widths
userdata_name_length = 5
userdata_id_length = 3
userdata_useruserdata_length = 8
for userdata in userdata_data:
# userdata_name column
@ -1093,8 +1118,7 @@ def format_list_userdata(userdata_data, lines=None):
end_bold=ansiprint.end(),
userdata_name='Name',
userdata_id='ID',
userdata_data='Document'
)
userdata_data='Document')
# Format the string (elements)
for data in sorted(userdata_data, key=lambda i: i.get('name', None)):
@ -1138,6 +1162,7 @@ def format_list_userdata(userdata_data, lines=None):
return '\n'.join([userdata_list_output_header] + userdata_list_output)
def format_list_script(script_data, lines=None):
if isinstance(script_data, dict):
script_data = [script_data]
@ -1147,7 +1172,6 @@ def format_list_script(script_data, lines=None):
# Determine optimal column widths
script_name_length = 5
script_id_length = 3
script_script_length = 8
for script in script_data:
# script_name column
@ -1168,8 +1192,7 @@ def format_list_script(script_data, lines=None):
end_bold=ansiprint.end(),
script_name='Name',
script_id='ID',
script_data='Script'
)
script_data='Script')
# Format the string (elements)
for script in sorted(script_data, key=lambda i: i.get('name', None)):
@ -1213,6 +1236,7 @@ def format_list_script(script_data, lines=None):
return '\n'.join([script_list_output_header] + script_list_output)
def format_list_ova(ova_data):
if isinstance(ova_data, dict):
ova_data = [ova_data]
@ -1282,8 +1306,7 @@ def format_list_ova(ova_data):
ova_disk_size='Size [GB]',
ova_disk_pool='Pool',
ova_disk_volume_format='Format',
ova_disk_volume_name='Source Volume',
)
ova_disk_volume_name='Source Volume')
# Format the string (elements)
for ova in sorted(ova_data, key=lambda i: i.get('name', None)):
@ -1326,6 +1349,7 @@ def format_list_ova(ova_data):
return '\n'.join([ova_list_output_header] + ova_list_output)
def format_list_profile(profile_data):
if isinstance(profile_data, dict):
profile_data = [profile_data]
@ -1413,8 +1437,7 @@ Data: {profile_userdata: <{profile_userdata_length}} \
profile_storage_template='Storage',
profile_userdata='Userdata',
profile_script='Script',
profile_arguments='Script Arguments'
)
profile_arguments='Script Arguments')
# Format the string (elements)
for profile in sorted(profile_data, key=lambda i: i.get('name', None)):
@ -1450,6 +1473,7 @@ Data: {profile_userdata: <{profile_userdata_length}} \
return '\n'.join([profile_list_output_header] + profile_list_output)
def format_list_task(task_data):
task_list_output = []
@ -1514,8 +1538,7 @@ VM: {task_vm_name: <{task_vm_name_length}} \
task_vm_name='Name',
task_vm_profile='Profile',
task_vm_define='Define?',
task_vm_start='Start?'
)
task_vm_start='Start?')
# Format the string (elements)
for task in sorted(task_data, key=lambda i: i.get('type', None)):

25
client-cli/cli_lib/vm.py
View File

@ -22,14 +22,11 @@
import time
import re
import subprocess
from collections import deque
import cli_lib.ansiprint as ansiprint
import cli_lib.ceph as ceph
from cli_lib.common import call_api, format_bytes, format_metric
#
# Primary functions
#
@ -57,6 +54,7 @@ def vm_info(config, vm):
else:
return False, response.json().get('message', '')
def vm_list(config, limit, target_node, target_state):
"""
Get list information about VMs (limited by {limit}, {target_node}, or {target_state})
@ -80,6 +78,7 @@ def vm_list(config, limit, target_node, target_state):
else:
return False, response.json().get('message', '')
def vm_define(config, xml, node, node_limit, node_selector, node_autostart, migration_method):
"""
Define a new VM on the cluster
@ -107,6 +106,7 @@ def vm_define(config, xml, node, node_limit, node_selector, node_autostart, migr
return retstatus, response.json().get('message', '')
def vm_modify(config, vm, xml, restart):
"""
Modify the configuration of VM
@ -130,6 +130,7 @@ def vm_modify(config, vm, xml, restart):
return retstatus, response.json().get('message', '')
def vm_metadata(config, vm, node_limit, node_selector, node_autostart, migration_method, provisioner_profile):
"""
Modify PVC metadata of a VM
@ -166,6 +167,7 @@ def vm_metadata(config, vm, node_limit, node_selector, node_autostart, migration
return retstatus, response.json().get('message', '')
def vm_remove(config, vm, delete_disks=False):
"""
Remove a VM
@ -186,6 +188,7 @@ def vm_remove(config, vm, delete_disks=False):
return retstatus, response.json().get('message', '')
def vm_state(config, vm, target_state, wait=False):
"""
Modify the current state of VM
@ -207,6 +210,7 @@ def vm_state(config, vm, target_state, wait=False):
return retstatus, response.json().get('message', '')
def vm_node(config, vm, target_node, action, force=False, wait=False, force_live=False):
"""
Modify the current node of VM via {action}
@ -231,6 +235,7 @@ def vm_node(config, vm, target_node, action, force=False, wait=False, force_live
return retstatus, response.json().get('message', '')
def vm_locks(config, vm):
"""
Flush RBD locks of (stopped) VM
@ -248,6 +253,7 @@ def vm_locks(config, vm):
return retstatus, response.json().get('message', '')
def view_console_log(config, vm, lines=100):
"""
Return console log lines from the API (and display them in a pager in the main CLI)
@ -272,6 +278,7 @@ def view_console_log(config, vm, lines=100):
return True, loglines
def follow_console_log(config, vm, lines=10):
"""
Return and follow console log lines from the API
@ -301,7 +308,7 @@ def follow_console_log(config, vm, lines=10):
try:
response = call_api(config, 'get', '/vm/{vm}/console'.format(vm=vm), params=params)
new_console_log = response.json()['data']
except:
except Exception:
break
# Split the new and old log strings into constitutent lines
old_console_loglines = console_log.split('\n')
@ -327,6 +334,7 @@ def follow_console_log(config, vm, lines=10):
return True, ''
#
# Output display functions
#
@ -344,7 +352,7 @@ def format_info(config, domain_information, long_output):
ainformation.append('{}vCPUs:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['vcpu']))
ainformation.append('{}Topology (S/C/T):{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['vcpu_topology']))
if long_output == True:
if long_output is True:
# Virtualization information
ainformation.append('')
ainformation.append('{}Emulator:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['emulator']))
@ -439,7 +447,7 @@ def format_info(config, domain_information, long_output):
ainformation.append('')
ainformation.append('{}Networks:{} {}'.format(ansiprint.purple(), ansiprint.end(), ', '.join(net_list)))
if long_output == True:
if long_output is True:
# Disk list
ainformation.append('')
name_length = 0
@ -482,6 +490,7 @@ def format_info(config, domain_information, long_output):
ainformation.append('')
return '\n'.join(ainformation)
def format_list(config, vm_list, raw):
# Handle single-element lists
if not isinstance(vm_list, list):
@ -596,7 +605,7 @@ def format_list(config, vm_list, raw):
net_list = []
vm_net_colour = ''
for net_vni in raw_net_list:
if not net_vni in valid_net_list:
if net_vni not in valid_net_list:
response = call_api(config, 'get', '/network/{net}'.format(net=net_vni))
if response.status_code != 200 and net_vni not in ['cluster', 'storage', 'upstream']:
vm_net_colour = ansiprint.red()

9
client-cli/cli_lib/zkhandler.py
View File

@ -20,10 +20,8 @@
#
###############################################################################
import kazoo.client
import uuid
import daemon_lib.ansiprint as ansiprint
# Exists function
def exists(zk_conn, key):
@ -33,22 +31,25 @@ def exists(zk_conn, key):
else:
return False
# Child list function
def listchildren(zk_conn, key):
children = zk_conn.get_children(key)
return children
# Delete key function
def deletekey(zk_conn, key, recursive=True):
zk_conn.delete(key, recursive=recursive)
# Data read function
def readdata(zk_conn, key):
data_raw = zk_conn.get(key)
data = data_raw[0].decode('utf8')
meta = data_raw[1]
return data
# Data write function
def writedata(zk_conn, kv):
# Start up a transaction
@ -87,12 +88,14 @@ def writedata(zk_conn, kv):
except Exception:
return False
# Write lock function
def writelock(zk_conn, key):
lock_id = str(uuid.uuid1())
lock = zk_conn.WriteLock('{}'.format(key), lock_id)
return lock
# Read lock function
def readlock(zk_conn, key):
lock_id = str(uuid.uuid1())

235
client-cli/pvc.py
View File

File diff suppressed because it is too large Load Diff

82
daemon-common/ansiprint.py
View File

@ -1,82 +0,0 @@
#!/usr/bin/env python3
# ansiprint.py - Printing function for formatted messages
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import datetime
# ANSII colours for output
def red():
return '\033[91m'
def blue():
return '\033[94m'
def cyan():
return '\033[96m'
def green():
return '\033[92m'
def yellow():
return '\033[93m'
def purple():
return '\033[95m'
def bold():
return '\033[1m'
def end():
return '\033[0m'
# Print function
def echo(message, prefix, state):
# Get the date
date = '{} - '.format(datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S.%f'))
endc = end()
# Continuation
if state == 'c':
date = ''
colour = ''
prompt = ' '
# OK
elif state == 'o':
colour = green()
prompt = '>>> '
# Error
elif state == 'e':
colour = red()
prompt = '>>> '
# Warning
elif state == 'w':
colour = yellow()
prompt = '>>> '
# Tick
elif state == 't':
colour = purple()
prompt = '>>> '
# Information
elif state == 'i':
colour = blue()
prompt = '>>> '
else:
colour = bold()
prompt = '>>> '
# Append space to prefix
if prefix != '':
prefix = prefix + ' '
print(colour + prompt + endc + date + prefix + message)

708
daemon-common/ceph.py
View File

File diff suppressed because it is too large Load Diff

11
daemon-common/cluster.py
View File

@ -20,12 +20,8 @@
#
###############################################################################
import json
import re
from distutils.util import strtobool
import daemon_lib.ansiprint as ansiprint
import daemon_lib.zkhandler as zkhandler
import daemon_lib.common as common
import daemon_lib.vm as pvc_vm
@ -33,6 +29,7 @@ import daemon_lib.node as pvc_node
import daemon_lib.network as pvc_network
import daemon_lib.ceph as pvc_ceph
def set_maintenance(zk_conn, maint_state):
try:
if maint_state == 'true':
@ -41,14 +38,15 @@ def set_maintenance(zk_conn, maint_state):
else:
zkhandler.writedata(zk_conn, {'/maintenance': 'false'})
return True, 'Successfully set cluster in normal mode'
except:
except Exception:
return False, 'Failed to set cluster maintenance state'
def getClusterInformation(zk_conn):
# Get cluster maintenance state
try:
maint_state = zkhandler.readdata(zk_conn, '/maintenance')
except:
except Exception:
maint_state = 'false'
# List of messages to display to the clients
@ -250,6 +248,7 @@ def getClusterInformation(zk_conn):
return cluster_information
def get_info(zk_conn):
# This is a thin wrapper function for naming purposes
cluster_information = getClusterInformation(zk_conn)

68
daemon-common/common.py
View File

@ -20,9 +20,9 @@
#
###############################################################################
import time
import uuid
import lxml
import math
import shlex
import subprocess
import kazoo.client
@ -36,6 +36,7 @@ import daemon_lib.zkhandler as zkhandler
# Supplemental functions
###############################################################################
#
# Run a local OS command via shell
#
@ -56,14 +57,15 @@ def run_os_command(command_string, background=False, environment=None, timeout=N
try:
stdout = command_output.stdout.decode('ascii')
except:
except Exception:
stdout = ''
try:
stderr = command_output.stderr.decode('ascii')
except:
except Exception:
stderr = ''
return retcode, stdout, stderr
#
# Validate a UUID
#
@ -71,9 +73,10 @@ def validateUUID(dom_uuid):
try:
uuid.UUID(dom_uuid)
return True
except:
except Exception:
return False
#
# Connect and disconnect from Zookeeper
#
@ -89,23 +92,27 @@ def startZKConnection(zk_host):
exit(1)
return zk_conn
def stopZKConnection(zk_conn):
zk_conn.stop()
zk_conn.close()
return 0
#
# Parse a Domain XML object
#
def getDomainXML(zk_conn, dom_uuid):
try:
xml = zkhandler.readdata(zk_conn, '/domains/{}/xml'.format(dom_uuid))
except:
except Exception:
return None
# Parse XML using lxml.objectify
parsed_xml = lxml.objectify.fromstring(xml)
return parsed_xml
#
# Get the main details for a VM object from XML
#
@ -126,11 +133,12 @@ def getDomainMainDetails(parsed_xml):
dvcpu = str(parsed_xml.vcpu)
try:
dvcputopo = '{}/{}/{}'.format(parsed_xml.cpu.topology.attrib.get('sockets'), parsed_xml.cpu.topology.attrib.get('cores'), parsed_xml.cpu.topology.attrib.get('threads'))
except:
except Exception:
dvcputopo = 'N/A'
return duuid, dname, ddescription, dmemory, dvcpu, dvcputopo
#
# Get long-format details
#
@ -143,6 +151,7 @@ def getDomainExtraDetails(parsed_xml):
return dtype, darch, dmachine, dconsole, demulator
#
# Get CPU features
#
@ -151,11 +160,12 @@ def getDomainCPUFeatures(parsed_xml):
try:
for feature in parsed_xml.features.getchildren():
dfeatures.append(feature.tag)
except:
except Exception:
pass
return dfeatures
#
# Get disk devices
#
@ -169,7 +179,7 @@ def getDomainDisks(parsed_xml, stats_data):
disk_stats_list = [x for x in stats_data.get('disk_stats', []) if x.get('name') == disk_attrib.get('name')]
try:
disk_stats = disk_stats_list[0]
except:
except Exception:
disk_stats = {}
if disk_type == 'network':
@ -200,6 +210,7 @@ def getDomainDisks(parsed_xml, stats_data):
return ddisks
#
# Get a list of disk devices
#
@ -211,6 +222,7 @@ def getDomainDiskList(zk_conn, dom_uuid):
return disk_list
#
# Get domain information from XML
#
@ -226,19 +238,19 @@ def getInformationFromXML(zk_conn, uuid):
try:
domain_node_limit = zkhandler.readdata(zk_conn, '/domains/{}/node_limit'.format(uuid))
except:
except Exception:
domain_node_limit = None
try:
domain_node_selector = zkhandler.readdata(zk_conn, '/domains/{}/node_selector'.format(uuid))
except:
except Exception:
domain_node_selector = None
try:
domain_node_autostart = zkhandler.readdata(zk_conn, '/domains/{}/node_autostart'.format(uuid))
except:
except Exception:
domain_node_autostart = None
try:
domain_migration_method = zkhandler.readdata(zk_conn, '/domains/{}/migration_method'.format(uuid))
except:
except Exception:
domain_migration_method = None
if not domain_node_limit:
@ -251,14 +263,14 @@ def getInformationFromXML(zk_conn, uuid):
try:
domain_profile = zkhandler.readdata(zk_conn, '/domains/{}/profile'.format(uuid))
except:
except Exception:
domain_profile = None
parsed_xml = getDomainXML(zk_conn, uuid)
try:
stats_data = loads(zkhandler.readdata(zk_conn, '/domains/{}/stats'.format(uuid)))
except:
except Exception:
stats_data = {}
domain_uuid, domain_name, domain_description, domain_memory, domain_vcpu, domain_vcputopo = getDomainMainDetails(parsed_xml)
@ -308,6 +320,7 @@ def getInformationFromXML(zk_conn, uuid):
return domain_information
#
# Get network devices
#
@ -317,24 +330,24 @@ def getDomainNetworks(parsed_xml, stats_data):
if device.tag == 'interface':
try:
net_type = device.attrib.get('type')
except:
except Exception:
net_type = None
try:
net_mac = device.mac.attrib.get('address')
except:
except Exception:
net_mac = None
try:
net_bridge = device.source.attrib.get(net_type)
except:
except Exception:
net_bridge = None
try:
net_model = device.model.attrib.get('type')
except:
except Exception:
net_model = None
try:
net_stats_list = [x for x in stats_data.get('net_stats', []) if x.get('bridge') == net_bridge]
net_stats = net_stats_list[0]
except:
except Exception:
net_stats = {}
net_rd_bytes = net_stats.get('rd_bytes', 0)
net_rd_packets = net_stats.get('rd_packets', 0)
@ -362,6 +375,7 @@ def getDomainNetworks(parsed_xml, stats_data):
return dnets
#
# Get controller devices
#
@ -379,6 +393,7 @@ def getDomainControllers(parsed_xml):
return dcontrollers
#
# Verify node is valid in cluster
#
@ -388,6 +403,7 @@ def verifyNode(zk_conn, node):
else:
return False
#
# Get the primary coordinator node
#
@ -396,7 +412,7 @@ def getPrimaryNode(zk_conn):
while True:
try:
primary_node = zkhandler.readdata(zk_conn, '/primary_node')
except:
except Exception:
primary_node == 'none'
if primary_node == 'none':
@ -412,6 +428,7 @@ def getPrimaryNode(zk_conn):
return primary_node
#
# Find a migration target
#
@ -421,13 +438,13 @@ def findTargetNode(zk_conn, dom_uuid):
node_limit = zkhandler.readdata(zk_conn, '/domains/{}/node_limit'.format(dom_uuid)).split(',')
if not any(node_limit):
node_limit = None
except:
except Exception:
node_limit = None
# Determine VM search field or use default; set config value if read fails
try:
search_field = zkhandler.readdata(zk_conn, '/domains/{}/node_selector'.format(dom_uuid))
except:
except Exception:
search_field = 'mem'
# Execute the search
@ -443,6 +460,7 @@ def findTargetNode(zk_conn, dom_uuid):
# Nothing was found
return None
# Get the list of valid target nodes
def getNodes(zk_conn, node_limit, dom_uuid):
valid_node_list = []
@ -469,6 +487,7 @@ def getNodes(zk_conn, node_limit, dom_uuid):
return valid_node_list
# via free memory (relative to allocated memory)
def findTargetNodeMem(zk_conn, node_limit, dom_uuid):
most_provfree = 0
@ -488,6 +507,7 @@ def findTargetNodeMem(zk_conn, node_limit, dom_uuid):
return target_node
# via load average
def findTargetNodeLoad(zk_conn, node_limit, dom_uuid):
least_load = 9999.0
@ -503,6 +523,7 @@ def findTargetNodeLoad(zk_conn, node_limit, dom_uuid):
return target_node
# via total vCPUs
def findTargetNodeVCPUs(zk_conn, node_limit, dom_uuid):
least_vcpus = 9999
@ -518,6 +539,7 @@ def findTargetNodeVCPUs(zk_conn, node_limit, dom_uuid):
return target_node
# via total VMs
def findTargetNodeVMs(zk_conn, node_limit, dom_uuid):
least_vms = 9999
@ -533,6 +555,7 @@ def findTargetNodeVMs(zk_conn, node_limit, dom_uuid):
return target_node
# Connect to the primary host and run a command
def runRemoteCommand(node, command, become=False):
import paramiko
@ -560,7 +583,6 @@ def runRemoteCommand(node, command, become=False):
ssh_client = paramiko.client.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(DnssecPolicy())
#ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(node)
stdin, stdout, stderr = ssh_client.exec_command(command)
return stdout.read().decode('ascii').rstrip(), stderr.read().decode('ascii').rstrip()

376
daemon-common/network.py
View File

@ -20,23 +20,12 @@
#
###############################################################################
import os
import socket
import time
import uuid
import re
import tempfile
import subprocess
import difflib
import colorama
import click
import lxml.objectify
import configparser
import kazoo.client
import daemon_lib.ansiprint as ansiprint
from kazoo.exceptions import NoNodeError
import daemon_lib.zkhandler as zkhandler
import daemon_lib.common as common
#
# Cluster search functions
@ -50,6 +39,7 @@ def getClusterNetworkList(zk_conn):
description_list.append(zkhandler.readdata(zk_conn, '/networks/{}'.format(vni)))
return vni_list, description_list
def searchClusterByVNI(zk_conn, vni):
try:
# Get the lists
@ -64,6 +54,7 @@ def searchClusterByVNI(zk_conn, vni):
return description
def searchClusterByDescription(zk_conn, description):
try:
# Get the lists
@ -78,6 +69,7 @@ def searchClusterByDescription(zk_conn, description):
return vni
def getNetworkVNI(zk_conn, network):
# Validate and obtain alternate passed value
if network.isdigit():
@ -89,6 +81,7 @@ def getNetworkVNI(zk_conn, network):
return net_vni
def getNetworkDescription(zk_conn, network):
# Validate and obtain alternate passed value
if network.isdigit():
@ -100,16 +93,19 @@ def getNetworkDescription(zk_conn, network):
return net_description
def getNetworkDHCPLeases(zk_conn, vni):
# Get a list of DHCP leases by listing the children of /networks/<vni>/dhcp4_leases
dhcp4_leases = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp4_leases'.format(vni))
return sorted(dhcp4_leases)
def getNetworkDHCPReservations(zk_conn, vni):
# Get a list of DHCP reservations by listing the children of /networks/<vni>/dhcp4_reservations
dhcp4_reservations = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp4_reservations'.format(vni))
return sorted(dhcp4_reservations)
def getNetworkACLs(zk_conn, vni, _direction):
# Get the (sorted) list of active ACLs
if _direction == 'both':
@ -131,6 +127,7 @@ def getNetworkACLs(zk_conn, vni, _direction):
return full_acl_list
def getNetworkInformation(zk_conn, vni):
description = zkhandler.readdata(zk_conn, '/networks/{}'.format(vni))
nettype = zkhandler.readdata(zk_conn, '/networks/{}/nettype'.format(vni))
@ -167,12 +164,13 @@ def getNetworkInformation(zk_conn, vni):
}
return network_information
def getDHCPLeaseInformation(zk_conn, vni, mac_address):
# Check whether this is a dynamic or static lease
try:
zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}'.format(vni, mac_address))
type_key = 'dhcp4_leases'
except kazoo.exceptions.NoNodeError:
except NoNodeError:
zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_reservations/{}'.format(vni, mac_address))
type_key = 'dhcp4_reservations'
@ -192,6 +190,7 @@ def getDHCPLeaseInformation(zk_conn, vni, mac_address):
}
return lease_information
def getACLInformation(zk_conn, vni, direction, description):
order = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/order'.format(vni, direction, description))
rule = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/rule'.format(vni, direction, description))
@ -205,6 +204,7 @@ def getACLInformation(zk_conn, vni, direction, description):
}
return acl_information
def isValidMAC(macaddr):
allowed = re.compile(r"""
(
@ -218,6 +218,7 @@ def isValidMAC(macaddr):
else:
return False
def isValidIP(ipaddr):
ip4_blocks = str(ipaddr).split(".")
if len(ip4_blocks) == 4:
@ -231,6 +232,7 @@ def isValidIP(ipaddr):
return True
return False
#
# Direct functions
#
@ -284,6 +286,7 @@ def add_network(zk_conn, vni, description, nettype,
return True, 'Network "{}" added successfully!'.format(description)
def modify_network(zk_conn, vni, description=None, domain=None, name_servers=None,
ip4_network=None, ip4_gateway=None, ip6_network=None, ip6_gateway=None,
dhcp4_flag=None, dhcp4_start=None, dhcp4_end=None):
@ -325,6 +328,7 @@ def modify_network(zk_conn, vni, description=None, domain=None, name_servers=Non
return True, 'Network "{}" modified successfully!'.format(vni)
def remove_network(zk_conn, network):
# Validate and obtain alternate passed value
vni = getNetworkVNI(zk_conn, network)
@ -368,6 +372,7 @@ def add_dhcp_reservation(zk_conn, network, ipaddress, macaddress, hostname):
return True, 'DHCP reservation "{}" added successfully!'.format(macaddress)
def remove_dhcp_reservation(zk_conn, network, reservation):
# Validate and obtain standard passed value
net_vni = getNetworkVNI(zk_conn, network)
@ -402,11 +407,12 @@ def remove_dhcp_reservation(zk_conn, network, reservation):
# Remove the entry from zookeeper
try:
zkhandler.deletekey(zk_conn, '/networks/{}/dhcp4_{}/{}'.format(net_vni, lease_type_zk, match_description))
except:
except Exception:
return False, 'ERROR: Failed to write to Zookeeper!'
return True, 'DHCP {} "{}" removed successfully!'.format(lease_type_human, match_description)
def add_acl(zk_conn, network, direction, description, rule, order):
# Validate and obtain standard passed value
net_vni = getNetworkVNI(zk_conn, network)
@ -470,6 +476,7 @@ def add_acl(zk_conn, network, direction, description, rule, order):
return True, 'Firewall rule "{}" added successfully!'.format(description)
def remove_acl(zk_conn, network, description):
# Validate and obtain standard passed value
net_vni = getNetworkVNI(zk_conn, network)
@ -510,6 +517,7 @@ def remove_acl(zk_conn, network, description):
return True, 'Firewall rule "{}" removed successfully!'.format(match_description)
def get_info(zk_conn, network):
# Validate and obtain alternate passed value
net_vni = getNetworkVNI(zk_conn, network)
@ -522,6 +530,7 @@ def get_info(zk_conn, network):
return True, network_information
def get_list(zk_conn, limit, is_fuzzy=True):
net_list = []
full_net_list = zkhandler.listchildren(zk_conn, '/networks')
@ -542,9 +551,9 @@ def get_list(zk_conn, limit, is_fuzzy=True):
else:
net_list.append(getNetworkInformation(zk_conn, net))
#output_string = formatNetworkList(zk_conn, net_list)
return True, net_list
def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
# Validate and obtain alternate passed value
net_vni = getNetworkVNI(zk_conn, network)
@ -555,11 +564,9 @@ def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
if only_static:
full_dhcp_list = getNetworkDHCPReservations(zk_conn, net_vni)
reservations = True
else:
full_dhcp_list = getNetworkDHCPReservations(zk_conn, net_vni)
full_dhcp_list += getNetworkDHCPLeases(zk_conn, net_vni)
reservations = False
if limit:
try:
@ -567,9 +574,9 @@ def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
limit = '^' + limit + '$'
# Implcitly assume fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '.*' + limit
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '.*'
except Exception as e:
return False, 'Regex Error: {}'.format(e)
@ -589,6 +596,7 @@ def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
return True, dhcp_list
def get_list_acl(zk_conn, network, limit, direction, is_fuzzy=True):
# Validate and obtain alternate passed value
net_vni = getNetworkVNI(zk_conn, network)
@ -612,9 +620,9 @@ def get_list_acl(zk_conn, network, limit, direction, is_fuzzy=True):
limit = '^' + limit + '$'
# Implcitly assume fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '.*' + limit
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '.*'
except Exception as e:
return False, 'Regex Error: {}'.format(e)
@ -630,326 +638,4 @@ def get_list_acl(zk_conn, network, limit, direction, is_fuzzy=True):
if valid_acl:
acl_list.append(acl)
#output_string = formatACLList(zk_conn, net_vni, direction, acl_list)
return True, acl_list
# CLI-only functions
def getOutputColours(network_information):
if network_information['ip6']['network'] != "None":
v6_flag_colour = ansiprint.green()
else:
v6_flag_colour = ansiprint.blue()
if network_information['ip4']['network'] != "None":
v4_flag_colour = ansiprint.green()
else:
v4_flag_colour = ansiprint.blue()
if network_information['ip6']['dhcp_flag'] == "True":
dhcp6_flag_colour = ansiprint.green()
else:
dhcp6_flag_colour = ansiprint.blue()
if network_information['ip4']['dhcp_flag'] == "True":
dhcp4_flag_colour = ansiprint.green()
else:
dhcp4_flag_colour = ansiprint.blue()
return v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour
def format_info(network_information, long_output):
if not network_information:
click.echo("No network found")
return
v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour = getOutputColours(network_information)
# Format a nice output: do this line-by-line then concat the elements at the end
ainformation = []
ainformation.append('{}Virtual network information:{}'.format(ansiprint.bold(), ansiprint.end()))
ainformation.append('')
# Basic information
ainformation.append('{}VNI:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['vni']))
ainformation.append('{}Type:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['type']))
ainformation.append('{}Description:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['description']))
if network_information['type'] == 'managed':
ainformation.append('{}Domain:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['domain']))
ainformation.append('{}DNS Servers:{} {}'.format(ansiprint.purple(), ansiprint.end(), ', '.join(network_information['name_servers'])))
if network_information['ip6']['network'] != "None":
ainformation.append('')
ainformation.append('{}IPv6 network:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip6']['network']))
ainformation.append('{}IPv6 gateway:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip6']['gateway']))
ainformation.append('{}DHCPv6 enabled:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), dhcp6_flag_colour, network_information['ip6']['dhcp_flag'], ansiprint.end()))
if network_information['ip4']['network'] != "None":
ainformation.append('')
ainformation.append('{}IPv4 network:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip4']['network']))
ainformation.append('{}IPv4 gateway:{} {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip4']['gateway']))
ainformation.append('{}DHCPv4 enabled:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), dhcp4_flag_colour, network_information['ip4']['dhcp_flag'], ansiprint.end()))
if network_information['ip4']['dhcp_flag'] == "True":
ainformation.append('{}DHCPv4 range:{} {} - {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip4']['dhcp_start'], network_information['ip4']['dhcp_end']))
if long_output:
dhcp4_reservations_list = getNetworkDHCPReservations(zk_conn, vni)
if dhcp4_reservations_list:
ainformation.append('')
ainformation.append('{}Client DHCPv4 reservations:{}'.format(ansiprint.bold(), ansiprint.end()))
ainformation.append('')
# Only show static reservations in the detailed information
dhcp4_reservations_string = formatDHCPLeaseList(zk_conn, vni, dhcp4_reservations_list, reservations=True)
for line in dhcp4_reservations_string.split('\n'):
ainformation.append(line)
firewall_rules = zkhandler.listchildren(zk_conn, '/networks/{}/firewall_rules'.format(vni))
if firewall_rules:
ainformation.append('')
ainformation.append('{}Network firewall rules:{}'.format(ansiprint.bold(), ansiprint.end()))
ainformation.append('')
formatted_firewall_rules = get_list_firewall_rules(zk_conn, vni)
# Join it all together
click.echo('\n'.join(ainformation))
def format_list(network_list):
if not network_list:
click.echo("No network found")
return
network_list_output = []
# Determine optimal column widths
net_vni_length = 5
net_description_length = 12
net_nettype_length = 8
net_domain_length = 6
net_v6_flag_length = 6
net_dhcp6_flag_length = 7
net_v4_flag_length = 6
net_dhcp4_flag_length = 7
for network_information in network_list:
# vni column
_net_vni_length = len(str(network_information['vni'])) + 1
if _net_vni_length > net_vni_length:
net_vni_length = _net_vni_length
# description column
_net_description_length = len(network_information['description']) + 1
if _net_description_length > net_description_length:
net_description_length = _net_description_length
# domain column
_net_domain_length = len(network_information['domain']) + 1
if _net_domain_length > net_domain_length:
net_domain_length = _net_domain_length
# Format the string (header)
network_list_output.append('{bold}\
{net_vni: <{net_vni_length}} \
{net_description: <{net_description_length}} \
{net_nettype: <{net_nettype_length}} \
{net_domain: <{net_domain_length}} \
{net_v6_flag: <{net_v6_flag_length}} \
{net_dhcp6_flag: <{net_dhcp6_flag_length}} \
{net_v4_flag: <{net_v4_flag_length}} \
{net_dhcp4_flag: <{net_dhcp4_flag_length}} \
{end_bold}'.format(
bold=ansiprint.bold(),
end_bold=ansiprint.end(),
net_vni_length=net_vni_length,
net_description_length=net_description_length,
net_nettype_length=net_nettype_length,
net_domain_length=net_domain_length,
net_v6_flag_length=net_v6_flag_length,
net_dhcp6_flag_length=net_dhcp6_flag_length,
net_v4_flag_length=net_v4_flag_length,
net_dhcp4_flag_length=net_dhcp4_flag_length,
net_vni='VNI',
net_description='Description',
net_nettype='Type',
net_domain='Domain',
net_v6_flag='IPv6',
net_dhcp6_flag='DHCPv6',
net_v4_flag='IPv4',
net_dhcp4_flag='DHCPv4',
)
)
for network_information in network_list:
v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour = getOutputColours(network_information)
if network_information['ip4']['network'] != "None":
v4_flag = 'True'
else:
v4_flag = 'False'
if network_information['ip6']['network'] != "None":
v6_flag = 'True'
else:
v6_flag = 'False'
if network_information['ip4']['dhcp_flag'] == "True":
dhcp4_range = '{} - {}'.format(network_information['ip4']['dhcp_start'], network_information['ip4']['dhcp_end'])
else:
dhcp4_range = 'N/A'
network_list_output.append(
'{bold}\
{net_vni: <{net_vni_length}} \
{net_description: <{net_description_length}} \
{net_nettype: <{net_nettype_length}} \
{net_domain: <{net_domain_length}} \
{v6_flag_colour}{net_v6_flag: <{net_v6_flag_length}}{colour_off} \
{dhcp6_flag_colour}{net_dhcp6_flag: <{net_dhcp6_flag_length}}{colour_off} \
{v4_flag_colour}{net_v4_flag: <{net_v4_flag_length}}{colour_off} \
{dhcp4_flag_colour}{net_dhcp4_flag: <{net_dhcp4_flag_length}}{colour_off} \
{end_bold}'.format(
bold='',
end_bold='',
net_vni_length=net_vni_length,
net_description_length=net_description_length,
net_nettype_length=net_nettype_length,
net_domain_length=net_domain_length,
net_v6_flag_length=net_v6_flag_length,
net_dhcp6_flag_length=net_dhcp6_flag_length,
net_v4_flag_length=net_v4_flag_length,
net_dhcp4_flag_length=net_dhcp4_flag_length,
net_vni=network_information['vni'],
net_description=network_information['description'],
net_nettype=network_information['type'],
net_domain=network_information['domain'],
net_v6_flag=v6_flag,
v6_flag_colour=v6_flag_colour,
net_dhcp6_flag=network_information['ip6']['dhcp_flag'],
dhcp6_flag_colour=dhcp6_flag_colour,
net_v4_flag=v4_flag,
v4_flag_colour=v4_flag_colour,
net_dhcp4_flag=network_information['ip4']['dhcp_flag'],
dhcp4_flag_colour=dhcp4_flag_colour,
colour_off=ansiprint.end()
)
)
click.echo('\n'.join(sorted(network_list_output)))
def format_list_dhcp(dhcp_lease_list):
dhcp_lease_list_output = []
# Determine optimal column widths
lease_hostname_length = 9
lease_ip4_address_length = 11
lease_mac_address_length = 13
lease_timestamp_length = 13
for dhcp_lease_information in dhcp_lease_list:
# hostname column
_lease_hostname_length = len(dhcp_lease_information['hostname']) + 1
if _lease_hostname_length > lease_hostname_length:
lease_hostname_length = _lease_hostname_length
# ip4_address column
_lease_ip4_address_length = len(dhcp_lease_information['ip4_address']) + 1
if _lease_ip4_address_length > lease_ip4_address_length:
lease_ip4_address_length = _lease_ip4_address_length
# mac_address column
_lease_mac_address_length = len(dhcp_lease_information['mac_address']) + 1
if _lease_mac_address_length > lease_mac_address_length:
lease_mac_address_length = _lease_mac_address_length
# Format the string (header)
dhcp_lease_list_output.append('{bold}\
{lease_hostname: <{lease_hostname_length}} \
{lease_ip4_address: <{lease_ip4_address_length}} \
{lease_mac_address: <{lease_mac_address_length}} \
{lease_timestamp: <{lease_timestamp_length}} \
{end_bold}'.format(
bold=ansiprint.bold(),
end_bold=ansiprint.end(),
lease_hostname_length=lease_hostname_length,
lease_ip4_address_length=lease_ip4_address_length,
lease_mac_address_length=lease_mac_address_length,
lease_timestamp_length=lease_timestamp_length,
lease_hostname='Hostname',
lease_ip4_address='IP Address',
lease_mac_address='MAC Address',
lease_timestamp='Timestamp'
)
)
for dhcp_lease_information in dhcp_lease_list:
dhcp_lease_list_output.append('{bold}\
{lease_hostname: <{lease_hostname_length}} \
{lease_ip4_address: <{lease_ip4_address_length}} \
{lease_mac_address: <{lease_mac_address_length}} \
{lease_timestamp: <{lease_timestamp_length}} \
{end_bold}'.format(
bold='',
end_bold='',
lease_hostname_length=lease_hostname_length,
lease_ip4_address_length=lease_ip4_address_length,
lease_mac_address_length=lease_mac_address_length,
lease_timestamp_length=12,
lease_hostname=dhcp_lease_information['hostname'],
lease_ip4_address=dhcp_lease_information['ip4_address'],
lease_mac_address=dhcp_lease_information['mac_address'],
lease_timestamp=dhcp_lease_information['timestamp']
)
)
click.echo('\n'.join(sorted(dhcp_lease_list_output)))
def format_list_acl(acl_list):
acl_list_output = []
# Determine optimal column widths
acl_direction_length = 10
acl_order_length = 6
acl_description_length = 12
acl_rule_length = 5
for acl_information in acl_list:
# order column
_acl_order_length = len(str(acl_information['order'])) + 1
if _acl_order_length > acl_order_length:
acl_order_length = _acl_order_length
# description column
_acl_description_length = len(acl_information['description']) + 1
if _acl_description_length > acl_description_length:
acl_description_length = _acl_description_length
# rule column
_acl_rule_length = len(acl_information['rule']) + 1
if _acl_rule_length > acl_rule_length:
acl_rule_length = _acl_rule_length
# Format the string (header)
acl_list_output.append('{bold}\
{acl_direction: <{acl_direction_length}} \
{acl_order: <{acl_order_length}} \
{acl_description: <{acl_description_length}} \
{acl_rule: <{acl_rule_length}} \
{end_bold}'.format(
bold=ansiprint.bold(),
end_bold=ansiprint.end(),
acl_direction_length=acl_direction_length,
acl_order_length=acl_order_length,
acl_description_length=acl_description_length,
acl_rule_length=acl_rule_length,
acl_direction='Direction',
acl_order='Order',
acl_description='Description',
acl_rule='Rule',
)
)
for acl_information in acl_list:
acl_list_output.append('{bold}\
{acl_direction: <{acl_direction_length}} \
{acl_order: <{acl_order_length}} \
{acl_description: <{acl_description_length}} \
{acl_rule: <{acl_rule_length}} \
{end_bold}'.format(
bold='',
end_bold='',
acl_direction_length=acl_direction_length,
acl_order_length=acl_order_length,
acl_description_length=acl_description_length,
acl_rule_length=acl_rule_length,
acl_direction=acl_information['direction'],
acl_order=acl_information['order'],
acl_description=acl_information['description'],
acl_rule=acl_information['rule'],
)
)
click.echo('\n'.join(sorted(acl_list_output)))

232
daemon-common/node.py
View File

@ -20,24 +20,12 @@
#
###############################################################################
import os
import socket
import time
import uuid
import re
import tempfile
import subprocess
import difflib
import colorama
import click
import lxml.objectify
import configparser
import kazoo.client
import daemon_lib.ansiprint as ansiprint
import daemon_lib.zkhandler as zkhandler
import daemon_lib.common as common
import daemon_lib.vm as pvc_vm
def getNodeInformation(zk_conn, node_name):
"""
@ -88,6 +76,7 @@ def getNodeInformation(zk_conn, node_name):
}
return node_information
#
# Direct Functions
#
@ -118,6 +107,7 @@ def secondary_node(zk_conn, node):
return True, retmsg
def primary_node(zk_conn, node):
# Verify node is valid
if not common.verifyNode(zk_conn, node):
@ -145,6 +135,7 @@ def primary_node(zk_conn, node):
return True, retmsg
def flush_node(zk_conn, node, wait=False):
# Verify node is valid
if not common.verifyNode(zk_conn, node):
@ -164,6 +155,7 @@ def flush_node(zk_conn, node, wait=False):
return True, retmsg
def ready_node(zk_conn, node, wait=False):
# Verify node is valid
if not common.verifyNode(zk_conn, node):
@ -183,6 +175,7 @@ def ready_node(zk_conn, node, wait=False):
return True, retmsg
def get_info(zk_conn, node):
# Verify node is valid
if not common.verifyNode(zk_conn, node):
@ -195,6 +188,7 @@ def get_info(zk_conn, node):
return True, node_information
def get_list(zk_conn, limit, daemon_state=None, coordinator_state=None, domain_state=None, is_fuzzy=True):
node_list = []
full_node_list = zkhandler.listchildren(zk_conn, '/nodes')
@ -227,215 +221,3 @@ def get_list(zk_conn, limit, daemon_state=None, coordinator_state=None, domain_s
node_list = limited_node_list
return True, node_list
#
# CLI-specific functions
#
def getOutputColours(node_information):
if node_information['daemon_state'] == 'run':
daemon_state_colour = ansiprint.green()
elif node_information['daemon_state'] == 'stop':
daemon_state_colour = ansiprint.red()
elif node_information['daemon_state'] == 'shutdown':
daemon_state_colour = ansiprint.yellow()
elif node_information['daemon_state'] == 'init':
daemon_state_colour = ansiprint.yellow()
elif node_information['daemon_state'] == 'dead':
daemon_state_colour = ansiprint.red() + ansiprint.bold()
else:
daemon_state_colour = ansiprint.blue()
if node_information['coordinator_state'] == 'primary':
coordinator_state_colour = ansiprint.green()
elif node_information['coordinator_state'] == 'secondary':
coordinator_state_colour = ansiprint.blue()
else:
coordinator_state_colour = ansiprint.cyan()
if node_information['domain_state'] == 'ready':
domain_state_colour = ansiprint.green()
else:
domain_state_colour = ansiprint.blue()
return daemon_state_colour, coordinator_state_colour, domain_state_colour
def format_info(node_information, long_output):
daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
# Format a nice output; do this line-by-line then concat the elements at the end
ainformation = []
# Basic information
ainformation.append('{}Name:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['name']))
ainformation.append('{}Daemon State:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), daemon_state_colour, node_information['daemon_state'], ansiprint.end()))
ainformation.append('{}Coordinator State:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), coordinator_state_colour, node_information['coordinator_state'], ansiprint.end()))
ainformation.append('{}Domain State:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), domain_state_colour, node_information['domain_state'], ansiprint.end()))
ainformation.append('{}Active VM Count:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['domains_count']))
if long_output:
ainformation.append('')
ainformation.append('{}Architecture:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['arch']))
ainformation.append('{}Operating System:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['os']))
ainformation.append('{}Kernel Version:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['kernel']))
ainformation.append('')
ainformation.append('{}Host CPUs:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['total']))
ainformation.append('{}vCPUs:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['allocated']))
ainformation.append('{}Load:{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['load']))
ainformation.append('{}Total RAM (MiB):{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['total']))
ainformation.append('{}Used RAM (MiB):{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['used']))
ainformation.append('{}Free RAM (MiB):{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['free']))
ainformation.append('{}Allocated RAM (MiB):{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['allocated']))
ainformation.append('{}Provisioned RAM (MiB):{} {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['provisioned']))
# Join it all together
information = '\n'.join(ainformation)
click.echo(information)
click.echo('')
def format_list(node_list):
node_list_output = []
# Determine optimal column widths
node_name_length = 5
daemon_state_length = 7
coordinator_state_length = 12
domain_state_length = 8
domains_count_length = 4
cpu_count_length = 6
load_length = 5
mem_total_length = 6
mem_used_length = 5
mem_free_length = 5
mem_alloc_length = 4
mem_prov_length = 4
for node_information in node_list:
# node_name column
_node_name_length = len(node_information['name']) + 1
if _node_name_length > node_name_length:
node_name_length = _node_name_length
# daemon_state column
_daemon_state_length = len(node_information['daemon_state']) + 1
if _daemon_state_length > daemon_state_length:
daemon_state_length = _daemon_state_length
# coordinator_state column
_coordinator_state_length = len(node_information['coordinator_state']) + 1
if _coordinator_state_length > coordinator_state_length:
coordinator_state_length = _coordinator_state_length
# domain_state column
_domain_state_length = len(node_information['domain_state']) + 1
if _domain_state_length > domain_state_length:
domain_state_length = _domain_state_length
# domains_count column
_domains_count_length = len(str(node_information['domains_count'])) + 1
if _domains_count_length > domains_count_length:
domains_count_length = _domains_count_length
# cpu_count column
_cpu_count_length = len(str(node_information['cpu_count'])) + 1
if _cpu_count_length > cpu_count_length:
cpu_count_length = _cpu_count_length
# load column
_load_length = len(str(node_information['load'])) + 1
if _load_length > load_length:
load_length = _load_length
# mem_total column
_mem_total_length = len(str(node_information['memory']['total'])) + 1
if _mem_total_length > mem_total_length:
mem_total_length = _mem_total_length
# mem_used column
_mem_used_length = len(str(node_information['memory']['used'])) + 1
if _mem_used_length > mem_used_length:
mem_used_length = _mem_used_length
# mem_free column
_mem_free_length = len(str(node_information['memory']['free'])) + 1
if _mem_free_length > mem_free_length:
mem_free_length = _mem_free_length
# mem_alloc column
_mem_alloc_length = len(str(node_information['memory']['allocated'])) + 1
if _mem_alloc_length > mem_alloc_length:
mem_alloc_length = _mem_alloc_length
# mem_prov column
_mem_prov_length = len(str(node_information['memory']['provisioned'])) + 1
if _mem_prov_length > mem_prov_length:
mem_prov_length = _mem_prov_length
# Format the string (header)
node_list_output.append(
'{bold}{node_name: <{node_name_length}} \
St: {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
Res: {node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}} {node_mem_provisioned: <{mem_prov_length}}{end_bold}'.format(
node_name_length=node_name_length,
daemon_state_length=daemon_state_length,
coordinator_state_length=coordinator_state_length,
domain_state_length=domain_state_length,
domains_count_length=domains_count_length,
cpu_count_length=cpu_count_length,
load_length=load_length,
mem_total_length=mem_total_length,
mem_used_length=mem_used_length,
mem_free_length=mem_free_length,
mem_alloc_length=mem_alloc_length,
mem_prov_length=mem_prov_length,
bold=ansiprint.bold(),
end_bold=ansiprint.end(),
daemon_state_colour='',
coordinator_state_colour='',
domain_state_colour='',
end_colour='',
node_name='Name',
node_daemon_state='Daemon',
node_coordinator_state='Coordinator',
node_domain_state='Domain',
node_domains_count='VMs',
node_cpu_count='vCPUs',
node_load='Load',
node_mem_total='Total',
node_mem_used='Used',
node_mem_free='Free',
node_mem_allocated='VMs Run',
node_mem_provisioned='VMs Total'
)
)
# Format the string (elements)
for node_information in node_list:
daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
node_list_output.append(
'{bold}{node_name: <{node_name_length}} \
{daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
{node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
{node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}} {node_mem_provisioned: <{mem_prov_length}}{end_bold}'.format(
node_name_length=node_name_length,
daemon_state_length=daemon_state_length,
coordinator_state_length=coordinator_state_length,
domain_state_length=domain_state_length,
domains_count_length=domains_count_length,
cpu_count_length=cpu_count_length,
load_length=load_length,
mem_total_length=mem_total_length,
mem_used_length=mem_used_length,
mem_free_length=mem_free_length,
mem_alloc_length=mem_alloc_length,
mem_prov_length=mem_prov_length,
bold='',
end_bold='',
daemon_state_colour=daemon_state_colour,
coordinator_state_colour=coordinator_state_colour,
domain_state_colour=domain_state_colour,
end_colour=ansiprint.end(),
node_name=node_information['name'],
node_daemon_state=node_information['daemon_state'],
node_coordinator_state=node_information['coordinator_state'],
node_domain_state=node_information['domain_state'],
node_domains_count=node_information['domains_count'],
node_cpu_count=node_information['vcpu']['allocated'],
node_load=node_information['load'],
node_mem_total=node_information['memory']['total'],
node_mem_used=node_information['memory']['used'],
node_mem_free=node_information['memory']['free'],
node_mem_allocated=node_information['memory']['allocated'],
node_mem_provisioned=node_information['memory']['provisioned']
)
)
click.echo('\n'.join(sorted(node_list_output)))

317
daemon-common/vm.py
View File

@ -20,27 +20,16 @@
#
###############################################################################
import os
import socket
import time
import uuid
import re
import subprocess
import difflib
import colorama
import click
import lxml.objectify
import configparser
import kazoo.client
from collections import deque
import daemon_lib.ansiprint as ansiprint
import daemon_lib.zkhandler as zkhandler
import daemon_lib.common as common
import daemon_lib.ceph as ceph
#
# Cluster search functions
#
@ -53,6 +42,7 @@ def getClusterDomainList(zk_conn):
name_list.append(zkhandler.readdata(zk_conn, '/domains/%s' % uuid))
return uuid_list, name_list
def searchClusterByUUID(zk_conn, uuid):
try:
# Get the lists
@ -67,6 +57,7 @@ def searchClusterByUUID(zk_conn, uuid):
return name
def searchClusterByName(zk_conn, name):
try:
# Get the lists
@ -81,6 +72,7 @@ def searchClusterByName(zk_conn, name):
return uuid
def getDomainUUID(zk_conn, domain):
# Validate that VM exists in cluster
if common.validateUUID(domain):
@ -92,6 +84,7 @@ def getDomainUUID(zk_conn, domain):
return dom_uuid
def getDomainName(zk_conn, domain):
# Validate that VM exists in cluster
if common.validateUUID(domain):
@ -103,6 +96,7 @@ def getDomainName(zk_conn, domain):
return dom_name
#
# Direct functions
#
@ -118,6 +112,7 @@ def is_migrated(zk_conn, domain):
else:
return False
def flush_locks(zk_conn, domain):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -145,7 +140,7 @@ def flush_locks(zk_conn, domain):
else:
message = 'ERROR: Failed to flush locks on VM "{}"; check node logs for details.'.format(domain)
success = False
except:
except Exception:
message = 'ERROR: Command ignored by node.'
success = False
@ -157,11 +152,12 @@ def flush_locks(zk_conn, domain):
return success, message
def define_vm(zk_conn, config_data, target_node, node_limit, node_selector, node_autostart, migration_method=None, profile=None, initial_state='stop'):
# Parse the XML data
try:
parsed_xml = lxml.objectify.fromstring(config_data)
except:
except Exception:
return False, 'ERROR: Failed to parse XML data.'
dom_uuid = parsed_xml.uuid.text
dom_name = parsed_xml.name.text
@ -216,6 +212,7 @@ def define_vm(zk_conn, config_data, target_node, node_limit, node_selector, node
return True, 'Added new VM with Name "{}" and UUID "{}" to database.'.format(dom_name, dom_uuid)
def modify_vm_metadata(zk_conn, domain, node_limit, node_selector, node_autostart, provisioner_profile, migration_method):
dom_uuid = getDomainUUID(zk_conn, domain)
if not dom_uuid:
@ -248,6 +245,7 @@ def modify_vm_metadata(zk_conn, domain, node_limit, node_selector, node_autostar
return True, 'Successfully modified PVC metadata of VM "{}".'.format(domain)
def modify_vm(zk_conn, domain, restart, new_vm_config):
dom_uuid = getDomainUUID(zk_conn, domain)
if not dom_uuid:
@ -257,7 +255,7 @@ def modify_vm(zk_conn, domain, restart, new_vm_config):
# Parse and valiate the XML
try:
parsed_xml = lxml.objectify.fromstring(new_vm_config)
except:
except Exception:
return False, 'ERROR: Failed to parse XML data.'
# Obtain the RBD disk list using the common functions
@ -289,6 +287,7 @@ def modify_vm(zk_conn, domain, restart, new_vm_config):
return True, ''
def dump_vm(zk_conn, domain):
dom_uuid = getDomainUUID(zk_conn, domain)
if not dom_uuid:
@ -299,6 +298,7 @@ def dump_vm(zk_conn, domain):
return True, vm_xml
def undefine_vm(zk_conn, domain):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -326,6 +326,7 @@ def undefine_vm(zk_conn, domain):
return True, 'Undefined VM "{}" from the cluster.'.format(domain)
def remove_vm(zk_conn, domain):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -365,6 +366,7 @@ def remove_vm(zk_conn, domain):
return True, 'Removed VM "{}" and disks from the cluster.'.format(domain)
def start_vm(zk_conn, domain):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -379,6 +381,7 @@ def start_vm(zk_conn, domain):
return True, 'Starting VM "{}".'.format(domain)
def restart_vm(zk_conn, domain, wait=False):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -405,6 +408,7 @@ def restart_vm(zk_conn, domain, wait=False):
return True, retmsg
def shutdown_vm(zk_conn, domain, wait=False):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -431,15 +435,13 @@ def shutdown_vm(zk_conn, domain, wait=False):
return True, retmsg
def stop_vm(zk_conn, domain):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
if not dom_uuid:
return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
# Get state and verify we're OK to proceed
current_state = zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid))
# Set the VM to start
lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
lock.acquire()
@ -448,6 +450,7 @@ def stop_vm(zk_conn, domain):
return True, 'Forcibly stopping VM "{}".'.format(domain)
def disable_vm(zk_conn, domain):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -467,6 +470,7 @@ def disable_vm(zk_conn, domain):
return True, 'Marked VM "{}" as disable.'.format(domain)
def move_vm(zk_conn, domain, target_node, wait=False, force_live=False):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -529,6 +533,7 @@ def move_vm(zk_conn, domain, target_node, wait=False, force_live=False):
return True, retmsg
def migrate_vm(zk_conn, domain, target_node, force_migrate, wait=False, force_live=False):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -594,6 +599,7 @@ def migrate_vm(zk_conn, domain, target_node, force_migrate, wait=False, force_li
return True, retmsg
def unmigrate_vm(zk_conn, domain, wait=False, force_live=False):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -634,6 +640,7 @@ def unmigrate_vm(zk_conn, domain, wait=False, force_live=False):
return True, retmsg
def get_console_log(zk_conn, domain, lines=1000):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -649,6 +656,7 @@ def get_console_log(zk_conn, domain, lines=1000):
return True, loglines
def get_info(zk_conn, domain):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zk_conn, domain)
@ -662,6 +670,7 @@ def get_info(zk_conn, domain):
return True, domain_information
def get_list(zk_conn, node, state, limit, is_fuzzy=True):
if node:
# Verify node is valid
@ -670,7 +679,7 @@ def get_list(zk_conn, node, state, limit, is_fuzzy=True):
if state:
valid_states = ['start', 'restart', 'shutdown', 'stop', 'disable', 'fail', 'migrate', 'unmigrate', 'provision']
if not state in valid_states:
if state not in valid_states:
return False, 'VM state "{}" is not valid.'.format(state)
full_vm_list = zkhandler.listchildren(zk_conn, '/domains')
@ -680,9 +689,9 @@ def get_list(zk_conn, node, state, limit, is_fuzzy=True):
if limit and is_fuzzy:
try:
# Implcitly assume fuzzy limits
if not re.match('\^.*', limit):
if not re.match('[^].*', limit):
limit = '.*' + limit
if not re.match('.*\$', limit):
if not re.match('.*[$]', limit):
limit = limit + '.*'
except Exception as e:
return False, 'Regex Error: {}'.format(e)
@ -722,269 +731,3 @@ def get_list(zk_conn, node, state, limit, is_fuzzy=True):
vm_list.append(common.getInformationFromXML(zk_conn, vm))
return True, vm_list
#
# CLI-specific functions
#
def format_info(zk_conn, domain_information, long_output):
# Format a nice output; do this line-by-line then concat the elements at the end
ainformation = []
ainformation.append('{}Virtual machine information:{}'.format(ansiprint.bold(), ansiprint.end()))
ainformation.append('')
# Basic information
ainformation.append('{}UUID:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['uuid']))
ainformation.append('{}Name:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['name']))
ainformation.append('{}Description:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['description']))
ainformation.append('{}Profile:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['profile']))
ainformation.append('{}Memory (M):{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['memory']))
ainformation.append('{}vCPUs:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['vcpu']))
ainformation.append('{}Topology (S/C/T):{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['vcpu_topology']))
if long_output == True:
# Virtualization information
ainformation.append('')
ainformation.append('{}Emulator:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['emulator']))
ainformation.append('{}Type:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['type']))
ainformation.append('{}Arch:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['arch']))
ainformation.append('{}Machine:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['machine']))
ainformation.append('{}Features:{} {}'.format(ansiprint.purple(), ansiprint.end(), ' '.join(domain_information['features'])))
# PVC cluster information
ainformation.append('')
dstate_colour = {
'start': ansiprint.green(),
'restart': ansiprint.yellow(),
'shutdown': ansiprint.yellow(),
'stop': ansiprint.red(),
'disable': ansiprint.blue(),
'fail': ansiprint.red(),
'migrate': ansiprint.blue(),
'unmigrate': ansiprint.blue()
}
ainformation.append('{}State:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), dstate_colour[domain_information['state']], domain_information['state'], ansiprint.end()))
ainformation.append('{}Current Node:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['node']))
if not domain_information['last_node']:
domain_information['last_node'] = "N/A"
ainformation.append('{}Previous Node:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['last_node']))
# Get a failure reason if applicable
if domain_information['failed_reason']:
ainformation.append('')
ainformation.append('{}Failure reason:{} {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['failed_reason']))
if not domain_information['node_selector']:
formatted_node_selector = "False"
else:
formatted_node_selector = domain_information['node_selector']
if not domain_information['node_limit']:
formatted_node_limit = "False"
else:
formatted_node_limit = ', '.join(domain_information['node_limit'])
if not domain_information['node_autostart']:
formatted_node_autostart = "False"
else:
formatted_node_autostart = domain_information['node_autostart']
if not domain_information['migration_method']:
formatted_migration_method = "False"
else:
formatted_migration_method = domain_information['migration_method']
ainformation.append('{}Migration selector:{} {}'.format(ansiprint.purple(), ansiprint.end(), formatted_node_selector))
ainformation.append('{}Node limit:{} {}'.format(ansiprint.purple(), ansiprint.end(), formatted_node_limit))
ainformation.append('{}Autostart:{} {}'.format(ansiprint.purple(), ansiprint.end(), formatted_node_autostart))
ainformation.append('{}Migration Method:{} {}'.format(ansiprint.purple(), ansiprint.end(), formatted_migration_method))
# Network list
net_list = []
for net in domain_information['networks']:
# Split out just the numerical (VNI) part of the brXXXX name
net_vnis = re.findall(r'\d+', net['source'])
if net_vnis:
net_vni = net_vnis[0]
else:
net_vni = re.sub('br', '', net['source'])
net_exists = zkhandler.exists(zk_conn, '/networks/{}'.format(net_vni))
if not net_exists and net_vni != 'cluster':
net_list.append(ansiprint.red() + net_vni + ansiprint.end() + ' [invalid]')
else:
net_list.append(net_vni)
ainformation.append('')
ainformation.append('{}Networks:{} {}'.format(ansiprint.purple(), ansiprint.end(), ', '.join(net_list)))
if long_output == True:
# Disk list
ainformation.append('')
name_length = 0
for disk in domain_information['disks']:
_name_length = len(disk['name']) + 1
if _name_length > name_length:
name_length = _name_length
ainformation.append('{0}Disks:{1} {2}ID Type {3: <{width}} Dev Bus{4}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), 'Name', ansiprint.end(), width=name_length))
for disk in domain_information['disks']:
ainformation.append(' {0: <3} {1: <5} {2: <{width}} {3: <4} {4: <5}'.format(domain_information['disks'].index(disk), disk['type'], disk['name'], disk['dev'], disk['bus'], width=name_length))
ainformation.append('')
ainformation.append('{}Interfaces:{} {}ID Type Source Model MAC{}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), ansiprint.end()))
for net in domain_information['networks']:
ainformation.append(' {0: <3} {1: <8} {2: <10} {3: <8} {4}'.format(domain_information['networks'].index(net), net['type'], net['source'], net['model'], net['mac']))
# Controller list
ainformation.append('')
ainformation.append('{}Controllers:{} {}ID Type Model{}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), ansiprint.end()))
for controller in domain_information['controllers']:
ainformation.append(' {0: <3} {1: <14} {2: <8}'.format(domain_information['controllers'].index(controller), controller['type'], controller['model']))
# Join it all together
information = '\n'.join(ainformation)
click.echo(information)
click.echo('')
def format_list(zk_conn, vm_list, raw):
# Function to strip the "br" off of nets and return a nicer list
def getNiceNetID(domain_information):
# Network list
net_list = []
for net in domain_information['networks']:
# Split out just the numerical (VNI) part of the brXXXX name
net_vnis = re.findall(r'\d+', net['source'])
if net_vnis:
net_vni = net_vnis[0]
else:
net_vni = re.sub('br', '', net['source'])
net_list.append(net_vni)
return net_list
# Handle raw mode since it just lists the names
if raw:
for vm in sorted(item['name'] for item in vm_list):
click.echo(vm)
return True, ''
vm_list_output = []
# Determine optimal column widths
# Dynamic columns: node_name, node, migrated
vm_name_length = 5
vm_uuid_length = 37
vm_state_length = 6
vm_nets_length = 9
vm_ram_length = 8
vm_vcpu_length = 6
vm_node_length = 8
vm_migrated_length = 10
for domain_information in vm_list:
net_list = getNiceNetID(domain_information)
# vm_name column
_vm_name_length = len(domain_information['name']) + 1
if _vm_name_length > vm_name_length:
vm_name_length = _vm_name_length
# vm_state column
_vm_state_length = len(domain_information['state']) + 1
if _vm_state_length > vm_state_length:
vm_state_length = _vm_state_length
# vm_nets column
_vm_nets_length = len(','.join(net_list)) + 1
if _vm_nets_length > vm_nets_length:
vm_nets_length = _vm_nets_length
# vm_node column
_vm_node_length = len(domain_information['node']) + 1
if _vm_node_length > vm_node_length:
vm_node_length = _vm_node_length
# vm_migrated column
_vm_migrated_length = len(domain_information['migrated']) + 1
if _vm_migrated_length > vm_migrated_length:
vm_migrated_length = _vm_migrated_length
# Format the string (header)
vm_list_output.append(
'{bold}{vm_name: <{vm_name_length}} {vm_uuid: <{vm_uuid_length}} \
{vm_state_colour}{vm_state: <{vm_state_length}}{end_colour} \
{vm_networks: <{vm_nets_length}} \
{vm_memory: <{vm_ram_length}} {vm_vcpu: <{vm_vcpu_length}} \
{vm_node: <{vm_node_length}} \
{vm_migrated: <{vm_migrated_length}}{end_bold}'.format(
vm_name_length=vm_name_length,
vm_uuid_length=vm_uuid_length,
vm_state_length=vm_state_length,
vm_nets_length=vm_nets_length,
vm_ram_length=vm_ram_length,
vm_vcpu_length=vm_vcpu_length,
vm_node_length=vm_node_length,
vm_migrated_length=vm_migrated_length,
bold=ansiprint.bold(),
end_bold=ansiprint.end(),
vm_state_colour='',
end_colour='',
vm_name='Name',
vm_uuid='UUID',
vm_state='State',
vm_networks='Networks',
vm_memory='RAM (M)',
vm_vcpu='vCPUs',
vm_node='Node',
vm_migrated='Migrated'
)
)
# Format the string (elements)
for domain_information in vm_list:
if domain_information['state'] == 'start':
vm_state_colour = ansiprint.green()
elif domain_information['state'] == 'restart':
vm_state_colour = ansiprint.yellow()
elif domain_information['state'] == 'shutdown':
vm_state_colour = ansiprint.yellow()
elif domain_information['state'] == 'stop':
vm_state_colour = ansiprint.red()
elif domain_information['state'] == 'fail':
vm_state_colour = ansiprint.red()
else:
vm_state_colour = ansiprint.blue()
# Handle colouring for an invalid network config
raw_net_list = getNiceNetID(domain_information)
net_list = []
vm_net_colour = ''
for net_vni in raw_net_list:
net_exists = zkhandler.exists(zk_conn, '/networks/{}'.format(net_vni))
if not net_exists and net_vni != 'cluster':
vm_net_colour = ansiprint.red()
net_list.append(net_vni)
vm_list_output.append(
'{bold}{vm_name: <{vm_name_length}} {vm_uuid: <{vm_uuid_length}} \
{vm_state_colour}{vm_state: <{vm_state_length}}{end_colour} \
{vm_net_colour}{vm_networks: <{vm_nets_length}}{end_colour} \
{vm_memory: <{vm_ram_length}} {vm_vcpu: <{vm_vcpu_length}} \
{vm_node: <{vm_node_length}} \
{vm_migrated: <{vm_migrated_length}}{end_bold}'.format(
vm_name_length=vm_name_length,
vm_uuid_length=vm_uuid_length,
vm_state_length=vm_state_length,
vm_nets_length=vm_nets_length,
vm_ram_length=vm_ram_length,
vm_vcpu_length=vm_vcpu_length,
vm_node_length=vm_node_length,
vm_migrated_length=vm_migrated_length,
bold='',
end_bold='',
vm_state_colour=vm_state_colour,
end_colour=ansiprint.end(),
vm_name=domain_information['name'],
vm_uuid=domain_information['uuid'],
vm_state=domain_information['state'],
vm_net_colour=vm_net_colour,
vm_networks=','.join(net_list),
vm_memory=domain_information['memory'],
vm_vcpu=domain_information['vcpu'],
vm_node=domain_information['node'],
vm_migrated=domain_information['migrated']
)
)
click.echo('\n'.join(sorted(vm_list_output)))
return True, ''

15
daemon-common/zkhandler.py
View File

@ -20,10 +20,9 @@
#
###############################################################################
import kazoo.client
import time
import uuid
import daemon_lib.ansiprint as ansiprint
# Exists function
def exists(zk_conn, key):
@ -33,15 +32,18 @@ def exists(zk_conn, key):
else:
return False
# Child list function
def listchildren(zk_conn, key):
children = zk_conn.get_children(key)
return children
# Delete key function
def deletekey(zk_conn, key, recursive=True):
zk_conn.delete(key, recursive=recursive)
# Rename key recursive function
def rename_key_element(zk_conn, zk_transaction, source_key, destination_key):
data_raw = zk_conn.get(source_key)
@ -56,6 +58,7 @@ def rename_key_element(zk_conn, zk_transaction, source_key, destination_key):
zk_transaction.delete(source_key)
# Rename key function
def renamekey(zk_conn, kv):
# Start up a transaction
@ -81,13 +84,14 @@ def renamekey(zk_conn, kv):
except Exception:
return False
# Data read function
def readdata(zk_conn, key):
data_raw = zk_conn.get(key)
data = data_raw[0].decode('utf8')
meta = data_raw[1]
return data
# Data write function
def writedata(zk_conn, kv):
# Start up a transaction
@ -126,8 +130,10 @@ def writedata(zk_conn, kv):
except Exception:
return False
# Write lock function
def writelock(zk_conn, key):
count = 1
while True:
try:
lock_id = str(uuid.uuid1())
@ -142,8 +148,10 @@ def writelock(zk_conn, key):
continue
return lock
# Read lock function
def readlock(zk_conn, key):
count = 1
while True:
try:
lock_id = str(uuid.uuid1())
@ -158,6 +166,7 @@ def readlock(zk_conn, key):
continue
return lock
# Exclusive lock function
def exclusivelock(zk_conn, key):
count = 1

15
lint Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
if ! which flake8 &>/dev/null; then
echo "Flake8 is required to lint this project"
exit 1
fi
flake8 \
--ignore=E501 \
--exclude=api-daemon/migrations/versions,api-daemon/provisioner/examples
ret=$?
if [[ $ret -eq 0 ]]; then
echo "No linting issues found!"
fi
exit $ret

2
node-daemon/pvcnoded.py
View File

@ -20,4 +20,4 @@
#
###############################################################################
import pvcnoded.Daemon
import pvcnoded.Daemon # noqa: F401

14
node-daemon/pvcnoded/CephInstance.py
View File

@ -24,10 +24,10 @@ import time
import json
import psutil
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
import pvcnoded.common as common
class CephOSDInstance(object):
def __init__(self, zk_conn, this_node, osd_id):
self.zk_conn = zk_conn
@ -67,6 +67,7 @@ class CephOSDInstance(object):
if data and data != self.stats:
self.stats = json.loads(data)
def add_osd(zk_conn, logger, node, device, weight):
# We are ready to create a new OSD on this node
logger.out('Creating new OSD disk on block device {}'.format(device), state='i')
@ -189,13 +190,14 @@ def add_osd(zk_conn, logger, node, device, weight):
logger.out('Failed to create new OSD disk: {}'.format(e), state='e')
return False
def remove_osd(zk_conn, logger, osd_id, osd_obj):
logger.out('Removing OSD disk {}'.format(osd_id), state='i')
try:
# 1. Verify the OSD is present
retcode, stdout, stderr = common.run_os_command('ceph osd ls')
osd_list = stdout.split('\n')
if not osd_id in osd_list:
if osd_id not in osd_list:
logger.out('Could not find OSD {} in the cluster'.format(osd_id), state='e')
return True
@ -223,7 +225,7 @@ def remove_osd(zk_conn, logger, osd_id, osd_obj):
time.sleep(5)
else:
raise
except:
except Exception:
break
# 3. Stop the OSD process and wait for it to be terminated
@ -282,6 +284,7 @@ def remove_osd(zk_conn, logger, osd_id, osd_obj):
logger.out('Failed to purge OSD disk with ID {}: {}'.format(osd_id, e), state='e')
return False
class CephPoolInstance(object):
def __init__(self, zk_conn, this_node, name):
self.zk_conn = zk_conn
@ -320,6 +323,7 @@ class CephPoolInstance(object):
if data and data != self.stats:
self.stats = json.loads(data)
class CephVolumeInstance(object):
def __init__(self, zk_conn, this_node, pool, name):
self.zk_conn = zk_conn
@ -343,8 +347,9 @@ class CephVolumeInstance(object):
if data and data != self.stats:
self.stats = json.loads(data)
class CephSnapshotInstance(object):
def __init__(self, zk_conn, this_node, name):
def __init__(self, zk_conn, this_node, pool, volume, name):
self.zk_conn = zk_conn
self.this_node = this_node
self.pool = pool
@ -367,6 +372,7 @@ class CephSnapshotInstance(object):
if data and data != self.stats:
self.stats = json.loads(data)
# Primary command function
# This command pipe is only used for OSD adds and removes
def run_command(zk_conn, logger, this_node, data, d_osd):

35
node-daemon/pvcnoded/DNSAggregatorInstance.py
View File

@ -27,10 +27,9 @@ import psycopg2
from threading import Thread, Event
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
import pvcnoded.common as common
class DNSAggregatorInstance(object):
# Initialization function
def __init__(self, zk_conn, config, logger):
@ -67,6 +66,7 @@ class DNSAggregatorInstance(object):
del self.dns_networks[network]
self.dns_axfr_daemon.update_networks(self.dns_networks)
class PowerDNSInstance(object):
# Initialization function
def __init__(self, aggregator):
@ -93,8 +93,7 @@ class PowerDNSInstance(object):
'--disable-syslog=yes', # Log only to stdout (which is then captured)
'--disable-axfr=no', # Allow AXFRs
'--allow-axfr-ips=0.0.0.0/0', # Allow AXFRs to anywhere
'--local-address={},{}'.format(self.vni_ipaddr, self.upstream_ipaddr),
# Listen on floating IPs
'--local-address={},{}'.format(self.vni_ipaddr, self.upstream_ipaddr), # Listen on floating IPs
'--local-port=53', # On port 53
'--log-dns-details=on', # Log details
'--loglevel=3', # Log info
@ -103,19 +102,13 @@ class PowerDNSInstance(object):
'--slave-renotify=yes', # Renotify out for our slaved zones
'--version-string=powerdns', # Set the version string
'--default-soa-name=dns.pvc.local', # Override dnsmasq's invalid name
'--socket-dir={}'.format(self.config['pdns_dynamic_directory']),
# Standard socket directory
'--socket-dir={}'.format(self.config['pdns_dynamic_directory']), # Standard socket directory
'--launch=gpgsql', # Use the PostgreSQL backend
'--gpgsql-host={}'.format(self.config['pdns_postgresql_host']),
# PostgreSQL instance
'--gpgsql-port={}'.format(self.config['pdns_postgresql_port']),
# Default port
'--gpgsql-dbname={}'.format(self.config['pdns_postgresql_dbname']),
# Database name
'--gpgsql-user={}'.format(self.config['pdns_postgresql_user']),
# User name
'--gpgsql-password={}'.format(self.config['pdns_postgresql_password']),
# User password
'--gpgsql-host={}'.format(self.config['pdns_postgresql_host']), # PostgreSQL instance
'--gpgsql-port={}'.format(self.config['pdns_postgresql_port']), # Default port
'--gpgsql-dbname={}'.format(self.config['pdns_postgresql_dbname']), # Database name
'--gpgsql-user={}'.format(self.config['pdns_postgresql_user']), # User name
'--gpgsql-password={}'.format(self.config['pdns_postgresql_password']), # User password
'--gpgsql-dnssec=no', # Do DNSSEC elsewhere
]
# Start the pdns process in a thread
@ -132,7 +125,6 @@ class PowerDNSInstance(object):
state='o'
)
def stop(self):
if self.dns_server_daemon:
self.logger.out(
@ -148,6 +140,7 @@ class PowerDNSInstance(object):
state='o'
)
class DNSNetworkInstance(object):
# Initialization function
def __init__(self, aggregator, network):
@ -160,10 +153,6 @@ class DNSNetworkInstance(object):
# Add a new network to the aggregator database
def add_network(self):
network_domain = self.network.domain
if self.network.ip4_gateway != 'None':
network_gateway = self.network.ip4_gateway
else:
network_gateway = self.network.ip6_gateway
self.logger.out(
'Adding entry for client domain {}'.format(
@ -332,12 +321,10 @@ class AXFRDaemonInstance(object):
while not self.thread_stopper.is_set():
# We do this for each network
for network, instance in self.dns_networks.items():
zone_modified = False
# Set up our SQL cursor
try:
sql_curs = self.sql_conn.cursor()
except:
except Exception:
time.sleep(0.5)
continue

113
node-daemon/pvcnoded/Daemon.py
View File

@ -20,9 +20,6 @@
#
###############################################################################
# Version string for startup output
version = '0.9.1'
import kazoo.client
import libvirt
import sys
@ -56,6 +53,9 @@ import pvcnoded.DNSAggregatorInstance as DNSAggregatorInstance
import pvcnoded.CephInstance as CephInstance
import pvcnoded.MetadataAPIInstance as MetadataAPIInstance
# Version string for startup output
version = '0.9.1'
###############################################################################
# PVCD - node daemon startup program
###############################################################################
@ -74,6 +74,7 @@ import pvcnoded.MetadataAPIInstance as MetadataAPIInstance
# Daemon functions
###############################################################################
# Create timer to update this node in Zookeeper
def startKeepaliveTimer():
# Create our timer object
@ -85,14 +86,16 @@ def startKeepaliveTimer():
node_keepalive()
return update_timer
def stopKeepaliveTimer():
global update_timer
try:
update_timer.shutdown()
logger.out('Stopping keepalive timer', state='s')
except:
except Exception:
pass
###############################################################################
# PHASE 1a - Configuration parsing
###############################################################################
@ -100,13 +103,12 @@ def stopKeepaliveTimer():
# Get the config file variable from the environment
try:
pvcnoded_config_file = os.environ['PVCD_CONFIG_FILE']
except:
except Exception:
print('ERROR: The "PVCD_CONFIG_FILE" environment variable must be set before starting pvcnoded.')
exit(1)
# Set local hostname and domain variables
myfqdn = gethostname()
#myfqdn = 'pvc-hv1.domain.net'
myhostname = myfqdn.split('.', 1)[0]
mydomainname = ''.join(myfqdn.split('.', 1)[1:])
try:
@ -125,6 +127,7 @@ staticdata.append(subprocess.run(['uname', '-r'], stdout=subprocess.PIPE).stdout
staticdata.append(subprocess.run(['uname', '-o'], stdout=subprocess.PIPE).stdout.decode('ascii').strip())
staticdata.append(subprocess.run(['uname', '-m'], stdout=subprocess.PIPE).stdout.decode('ascii').strip())
# Read and parse the config file
def readConfig(pvcnoded_config_file, myhostname):
print('Loading configuration from file "{}"'.format(pvcnoded_config_file))
@ -176,7 +179,7 @@ def readConfig(pvcnoded_config_file, myhostname):
config_debug = {
'debug': o_config['pvc']['debug']
}
except:
except Exception:
config_debug = {
'debug': False
}
@ -223,9 +226,7 @@ def readConfig(pvcnoded_config_file, myhostname):
config = {**config, **config_networking}
# Create the by-id address entries
for net in [ 'vni',
'storage',
'upstream' ]:
for net in ['vni', 'storage', 'upstream']:
address_key = '{}_dev_ip'.format(net)
floating_key = '{}_floating_ip'.format(net)
network_key = '{}_network'.format(net)
@ -233,7 +234,7 @@ def readConfig(pvcnoded_config_file, myhostname):
# Verify the network provided is valid
try:
network = ip_network(config[network_key])
except Exception as e:
except Exception:
print('ERROR: Network address {} for {} is not valid!'.format(config[network_key], network_key))
exit(1)
@ -251,9 +252,9 @@ def readConfig(pvcnoded_config_file, myhostname):
# Set the ipaddr
floating_addr = ip_address(config[floating_key].split('/')[0])
# Verify we're in the network
if not floating_addr in list(network.hosts()):
if floating_addr not in list(network.hosts()):
raise
except Exception as e:
except Exception:
print('ERROR: Floating address {} for {} is not valid!'.format(config[floating_key], floating_key))
exit(1)
@ -271,10 +272,11 @@ def readConfig(pvcnoded_config_file, myhostname):
# Handle an empty ipmi_hostname
if config['ipmi_hostname'] == '':
config['ipmi_hostname'] = myshorthostname + '-lom.' + mydomainname
config['ipmi_hostname'] = myhostname + '-lom.' + mydomainname
return config
# Get the config object from readConfig()
config = readConfig(pvcnoded_config_file, myhostname)
debug = config['debug']
@ -513,6 +515,7 @@ except Exception as e:
logger.out('ERROR: Failed to connect to Zookeeper cluster: {}'.format(e), state='e')
exit(1)
# Handle zookeeper failures
def zk_listener(state):
global zk_conn, update_timer
@ -535,7 +538,7 @@ def zk_listener(state):
_zk_conn = kazoo.client.KazooClient(hosts=config['coordinators'])
try:
_zk_conn.start()
except:
except Exception:
del _zk_conn
continue
@ -545,12 +548,14 @@ def zk_listener(state):
zk_conn.add_listener(zk_listener)
break
zk_conn.add_listener(zk_listener)
###############################################################################
# PHASE 5 - Gracefully handle termination
###############################################################################
# Cleanup function
def cleanup():
global zk_conn, update_timer, d_domain
@ -571,22 +576,21 @@ def cleanup():
if d_domain[domain].getnode() == myhostname:
try:
d_domain[domain].console_log_instance.stop()
except NameError as e:
except NameError:
pass
except AttributeError as e:
except AttributeError:
pass
# Force into secondary coordinator state if needed
try:
if this_node.router_state == 'primary':
is_primary = True
zkhandler.writedata(zk_conn, {
'/primary_node': 'none'
})
logger.out('Waiting for primary migration', state='s')
while this_node.router_state != 'secondary':
time.sleep(0.5)
except:
except Exception:
pass
# Stop keepalive thread
@ -610,21 +614,24 @@ def cleanup():
try:
zk_conn.stop()
zk_conn.close()
except:
except Exception:
pass
logger.out('Terminated pvc daemon', state='s')
sys.exit(0)
# Termination function
def term(signum='', frame=''):
cleanup()
# Hangup (logrotate) function
def hup(signum='', frame=''):
if config['file_logging']:
logger.hup()
# Handle signals gracefully
signal.signal(signal.SIGTERM, term)
signal.signal(signal.SIGINT, term)
@ -796,6 +803,7 @@ else:
dns_aggregator = None
metadata_api = None
# Node objects
@zk_conn.ChildrenWatch('/nodes')
def update_nodes(new_node_list):
@ -803,12 +811,12 @@ def update_nodes(new_node_list):
# Add any missing nodes to the list
for node in new_node_list:
if not node in node_list:
if node not in node_list:
d_node[node] = NodeInstance.NodeInstance(node, myhostname, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator, metadata_api)
# Remove any deleted nodes from the list
for node in node_list:
if not node in new_node_list:
if node not in new_node_list:
# Delete the object
del(d_node[node])
@ -820,18 +828,21 @@ def update_nodes(new_node_list):
for node in d_node:
d_node[node].update_node_list(d_node)
# Alias for our local node (passed to network and domain objects)
this_node = d_node[myhostname]
# Maintenance mode
@zk_conn.DataWatch('/maintenance')
def set_maintenance(_maintenance, stat, event=''):
global maintenance
try:
maintenance = bool(strtobool(_maintenance.decode('ascii')))
except:
except Exception:
maintenance = False
# Primary node
@zk_conn.DataWatch('/primary_node')
def update_primary(new_primary, stat, event=''):
@ -877,6 +888,7 @@ def update_primary(new_primary, stat, event=''):
for node in d_node:
d_node[node].primary_node = new_primary
if enable_networking:
# Network objects
@zk_conn.ChildrenWatch('/networks')
@ -885,13 +897,13 @@ if enable_networking:
# Add any missing networks to the list
for network in new_network_list:
if not network in network_list:
if network not in network_list:
d_network[network] = VXNetworkInstance.VXNetworkInstance(network, zk_conn, config, logger, this_node, dns_aggregator)
if config['daemon_mode'] == 'coordinator' and d_network[network].nettype == 'managed':
try:
dns_aggregator.add_network(d_network[network])
except Exception as e:
logger.out('Failed to create DNS Aggregator for network {}'.format(network), 'w')
logger.out('Failed to create DNS Aggregator for network {}: {}'.format(network, e), 'w')
# Start primary functionality
if this_node.router_state == 'primary' and d_network[network].nettype == 'managed':
d_network[network].createGateways()
@ -899,7 +911,7 @@ if enable_networking:
# Remove any deleted networks from the list
for network in network_list:
if not network in new_network_list:
if network not in new_network_list:
if d_network[network].nettype == 'managed':
# Stop primary functionality
if this_node.router_state == 'primary':
@ -923,7 +935,7 @@ if enable_networking:
if enable_hypervisor:
# VM command pipeline key
@zk_conn.DataWatch('/cmd/domains')
def cmd(data, stat, event=''):
def cmd_domains(data, stat, event=''):
if data:
VMInstance.run_command(zk_conn, logger, this_node, data.decode('ascii'))
@ -934,12 +946,12 @@ if enable_hypervisor:
# Add any missing domains to the list
for domain in new_domain_list:
if not domain in domain_list:
if domain not in domain_list:
d_domain[domain] = VMInstance.VMInstance(domain, zk_conn, config, logger, this_node)
# Remove any deleted domains from the list
for domain in domain_list:
if not domain in new_domain_list:
if domain not in new_domain_list:
# Delete the object
del(d_domain[domain])
@ -954,7 +966,7 @@ if enable_hypervisor:
if enable_storage:
# Ceph command pipeline key
@zk_conn.DataWatch('/cmd/ceph')
def cmd(data, stat, event=''):
def cmd_ceph(data, stat, event=''):
if data:
CephInstance.run_command(zk_conn, logger, this_node, data.decode('ascii'), d_osd)
@ -965,12 +977,12 @@ if enable_storage:
# Add any missing OSDs to the list
for osd in new_osd_list:
if not osd in osd_list:
if osd not in osd_list:
d_osd[osd] = CephInstance.CephOSDInstance(zk_conn, this_node, osd)
# Remove any deleted OSDs from the list
for osd in osd_list:
if not osd in new_osd_list:
if osd not in new_osd_list:
# Delete the object
del(d_osd[osd])
@ -985,14 +997,14 @@ if enable_storage:
# Add any missing Pools to the list
for pool in new_pool_list:
if not pool in pool_list:
if pool not in pool_list:
d_pool[pool] = CephInstance.CephPoolInstance(zk_conn, this_node, pool)
d_volume[pool] = dict()
volume_list[pool] = []
# Remove any deleted Pools from the list
for pool in pool_list:
if not pool in new_pool_list:
if pool not in new_pool_list:
# Delete the object
del(d_pool[pool])
@ -1008,12 +1020,12 @@ if enable_storage:
# Add any missing Volumes to the list
for volume in new_volume_list:
if not volume in volume_list[pool]:
if volume not in volume_list[pool]:
d_volume[pool][volume] = CephInstance.CephVolumeInstance(zk_conn, this_node, pool, volume)
# Remove any deleted Volumes from the list
for volume in volume_list[pool]:
if not volume in new_volume_list:
if volume not in new_volume_list:
# Delete the object
del(d_volume[pool][volume])
@ -1021,6 +1033,7 @@ if enable_storage:
volume_list[pool] = new_volume_list
logger.out('{}Volume list [{pool}]:{} {plist}'.format(fmt_blue, fmt_end, pool=pool, plist=' '.join(volume_list[pool])), state='i')
###############################################################################
# PHASE 9 - Run the daemon
###############################################################################
@ -1150,7 +1163,7 @@ def collect_ceph_stats(queue):
})
except Exception as e:
# One or more of the status commands timed out, just continue
logger.out('Failed to format and send pool data', state='w')
logger.out('Failed to format and send pool data: {}'.format(e), state='w')
pass
# Only grab OSD stats if there are OSDs to grab (otherwise `ceph osd df` hangs)
@ -1296,6 +1309,7 @@ def collect_ceph_stats(queue):
if debug:
logger.out("Thread finished", state='d', prefix='ceph-thread')
# State table for pretty stats
libvirt_vm_states = {
0: "NOSTATE",
@ -1308,6 +1322,7 @@ libvirt_vm_states = {
7: "PMSUSPENDED"
}
# VM stats update function
def collect_vm_stats(queue):
if debug:
@ -1318,7 +1333,7 @@ def collect_vm_stats(queue):
if debug:
logger.out("Connecting to libvirt", state='d', prefix='vm-thread')
lv_conn = libvirt.open(libvirt_name)
if lv_conn == None:
if lv_conn is None:
logger.out('Failed to open connection to "{}"'.format(libvirt_name), state='e')
return
@ -1337,11 +1352,11 @@ def collect_vm_stats(queue):
memprov += instance.getmemory()
vcpualloc += instance.getvcpus()
if instance.getstate() == 'start' and instance.getnode() == this_node.name:
if instance.getdom() != None:
if instance.getdom() is not None:
try:
if instance.getdom().state()[0] != libvirt.VIR_DOMAIN_RUNNING:
raise
except Exception as e:
except Exception:
# Toggle a state "change"
zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(domain): instance.getstate()})
elif instance.getnode() == this_node.name:
@ -1371,7 +1386,7 @@ def collect_vm_stats(queue):
if debug:
try:
logger.out("Failed getting VM information for {}: {}".format(domain.name(), e), state='d', prefix='vm-thread')
except:
except Exception:
pass
continue
@ -1451,6 +1466,7 @@ def collect_vm_stats(queue):
if debug:
logger.out("Thread finished", state='d', prefix='vm-thread')
# Keepalive update function
def node_keepalive():
if debug:
@ -1462,7 +1478,7 @@ def node_keepalive():
try:
if zkhandler.readdata(zk_conn, '/upstream_ip') != config['upstream_floating_ip']:
raise
except:
except Exception:
zkhandler.writedata(zk_conn, {'/upstream_ip': config['upstream_floating_ip']})
# Get past state and update if needed
@ -1517,7 +1533,7 @@ def node_keepalive():
this_node.memalloc = vm_thread_queue.get()
this_node.memprov = vm_thread_queue.get()
this_node.vcpualloc = vm_thread_queue.get()
except:
except Exception:
pass
else:
this_node.domains_count = 0
@ -1530,7 +1546,7 @@ def node_keepalive():
ceph_health_colour = ceph_thread_queue.get()
ceph_health = ceph_thread_queue.get()
osds_this_node = ceph_thread_queue.get()
except:
except Exception:
ceph_health_colour = fmt_cyan
ceph_health = 'UNKNOWN'
osds_this_node = '?'
@ -1552,7 +1568,7 @@ def node_keepalive():
'/nodes/{}/runningdomains'.format(this_node.name): ' '.join(this_node.domain_list),
'/nodes/{}/keepalive'.format(this_node.name): str(keepalive_time)
})
except:
except Exception:
logger.out('Failed to set keepalive data', state='e')
return
@ -1621,11 +1637,9 @@ def node_keepalive():
for node_name in d_node:
try:
node_daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name))
node_domain_state = zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node_name))
node_keepalive = int(zkhandler.readdata(zk_conn, '/nodes/{}/keepalive'.format(node_name)))
except:
except Exception:
node_daemon_state = 'unknown'
node_domain_state = 'unknown'
node_keepalive = 0
# Handle deadtime and fencng if needed
@ -1647,6 +1661,7 @@ def node_keepalive():
if debug:
logger.out("Keepalive finished", state='d', prefix='main-thread')
# Start keepalive thread
update_timer = startKeepaliveTimer()
@ -1654,5 +1669,5 @@ update_timer = startKeepaliveTimer()
while True:
try:
time.sleep(1)
except:
except Exception:
break

10
node-daemon/pvcnoded/MetadataAPIInstance.py
View File

@ -32,6 +32,7 @@ from psycopg2.extras import RealDictCursor
import daemon_lib.vm as pvc_vm
import daemon_lib.network as pvc_network
class MetadataAPIInstance(object):
mdapi = flask.Flask(__name__)
@ -170,26 +171,21 @@ class MetadataAPIInstance(object):
try:
if information.get('ip4_address', None) == source_address:
host_information = information
except:
except Exception:
pass
# Get our real information on the host; now we can start querying about it
client_hostname = host_information.get('hostname', None)
client_macaddr = host_information.get('mac_address', None)
client_ipaddr = host_information.get('ip4_address', None)
# Find the VM with that MAC address - we can't assume that the hostname is actually right
_discard, vm_list = pvc_vm.get_list(self.zk_conn, None, None, None)
vm_name = None
vm_details = dict()
for vm in vm_list:
try:
for network in vm.get('networks'):
if network.get('mac', None) == client_macaddr:
vm_name = vm.get('name')
vm_details = vm
except:
except Exception:
pass
return vm_details

24
node-daemon/pvcnoded/NodeInstance.py
View File

@ -24,10 +24,10 @@ import time
from threading import Thread
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
import pvcnoded.common as common
class NodeInstance(object):
# Initialization function
def __init__(self, name, this_node, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator, metadata_api):
@ -324,12 +324,12 @@ class NodeInstance(object):
self.logger.out('Acquiring write lock for synchronization phase A', state='i')
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase A', state='o')
time.sleep(1) # Time for reader to acquire the lock
time.sleep(1) # Time fir reader to acquire the lock
self.logger.out('Releasing write lock for synchronization phase A', state='i')
zkhandler.writedata(self.zk_conn, {'/locks/primary_node': ''})
lock.release()
self.logger.out('Released write lock for synchronization phase A', state='o')
time.sleep(0.1) # Time for new writer to acquire the lock
time.sleep(0.1) # Time fir new writer to acquire the lock
# Synchronize nodes B (I am reader)
lock = zkhandler.readlock(self.zk_conn, '/locks/primary_node')
@ -345,7 +345,7 @@ class NodeInstance(object):
self.logger.out('Acquiring write lock for synchronization phase C', state='i')
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase C', state='o')
time.sleep(0.5) # Time for reader to acquire the lock
time.sleep(0.5) # Time fir reader to acquire the lock
# 1. Add Upstream floating IP
self.logger.out(
'Creating floating upstream IP {}/{} on interface {}'.format(
@ -366,7 +366,7 @@ class NodeInstance(object):
self.logger.out('Acquiring write lock for synchronization phase D', state='i')
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase D', state='o')
time.sleep(0.2) # Time for reader to acquire the lock
time.sleep(0.2) # Time fir reader to acquire the lock
# 2. Add Cluster floating IP
self.logger.out(
'Creating floating management IP {}/{} on interface {}'.format(
@ -387,7 +387,7 @@ class NodeInstance(object):
self.logger.out('Acquiring write lock for synchronization phase E', state='i')
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase E', state='o')
time.sleep(0.2) # Time for reader to acquire the lock
time.sleep(0.2) # Time fir reader to acquire the lock
# 3. Add Metadata link-local IP
self.logger.out(
'Creating Metadata link-local IP {}/{} on interface {}'.format(
@ -408,7 +408,7 @@ class NodeInstance(object):
self.logger.out('Acquiring write lock for synchronization phase F', state='i')
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase F', state='o')
time.sleep(0.2) # Time for reader to acquire the lock
time.sleep(0.2) # Time fir reader to acquire the lock
# 4. Add gateway IPs
for network in self.d_network:
self.d_network[network].createGateways()
@ -422,7 +422,7 @@ class NodeInstance(object):
self.logger.out('Acquiring write lock for synchronization phase G', state='i')
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase G', state='o')
time.sleep(0.2) # Time for reader to acquire the lock
time.sleep(0.2) # Time fir reader to acquire the lock
# 5. Transition Patroni primary
self.logger.out('Setting Patroni leader to this node', state='i')
tick = 1
@ -515,7 +515,7 @@ class NodeInstance(object):
self.logger.out('Acquiring write lock for synchronization phase B', state='i')
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase B', state='o')
time.sleep(0.2) # Time for reader to acquire the lock
time.sleep(0.2) # Time fir reader to acquire the lock
# 1. Stop DNS aggregator
self.dns_aggregator.stop_aggregator()
# 2. Stop DHCP servers
@ -531,7 +531,7 @@ class NodeInstance(object):
common.run_os_command("systemctl stop pvcapid.service")
# 4. Stop metadata API
self.metadata_api.stop()
time.sleep(0.1) # Time for new writer to acquire the lock
time.sleep(0.1) # Time fir new writer to acquire the lock
# Synchronize nodes C (I am reader)
lock = zkhandler.readlock(self.zk_conn, '/locks/primary_node')
@ -608,7 +608,7 @@ class NodeInstance(object):
try:
lock.acquire(timeout=60) # Don't wait forever and completely block us
self.logger.out('Acquired read lock for synchronization phase G', state='o')
except:
except Exception:
pass
self.logger.out('Releasing read lock for synchronization phase G', state='i')
lock.release()
@ -698,7 +698,7 @@ class NodeInstance(object):
try:
last_node = zkhandler.readdata(self.zk_conn, '/domains/{}/lastnode'.format(dom_uuid))
except:
except Exception:
continue
if last_node != self.name:

4
node-daemon/pvcnoded/VMConsoleWatcherInstance.py
View File

@ -21,16 +21,14 @@
###############################################################################
import os
import uuid
import time
import libvirt
from threading import Thread, Event
from collections import deque
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
class VMConsoleWatcherInstance(object):
# Initialization function
def __init__(self, domuuid, domname, zk_conn, config, logger, this_node):

62
node-daemon/pvcnoded/VMInstance.py
View File

@ -27,7 +27,6 @@ import json
from threading import Thread
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
import pvcnoded.common as common
@ -35,6 +34,7 @@ import pvcnoded.VMConsoleWatcherInstance as VMConsoleWatcherInstance
import daemon_lib.common as daemon_common
def flush_locks(zk_conn, logger, dom_uuid):
logger.out('Flushing RBD locks for VM "{}"'.format(dom_uuid), state='i')
# Get the list of RBD images
@ -65,6 +65,7 @@ def flush_locks(zk_conn, logger, dom_uuid):
return True
# Primary command function
def run_command(zk_conn, logger, this_node, data):
# Get the command and args
@ -93,6 +94,7 @@ def run_command(zk_conn, logger, this_node, data):
# Wait 1 seconds before we free the lock, to ensure the client hits the lock
time.sleep(1)
class VMInstance(object):
# Initialization function
def __init__(self, domuuid, zk_conn, config, logger, this_node):
@ -112,11 +114,11 @@ class VMInstance(object):
self.last_lastnode = zkhandler.readdata(self.zk_conn, '/domains/{}/lastnode'.format(self.domuuid))
try:
self.pinpolicy = zkhandler.readdata(self.zk_conn, '/domains/{}/pinpolicy'.format(self.domuuid))
except:
except Exception:
self.pinpolicy = "none"
try:
self.migration_method = zkhandler.readdata(self.zk_conn, '/domains/{}/migration_method'.format(self.domuuid))
except:
except Exception:
self.migration_method = 'none'
# These will all be set later
@ -166,7 +168,7 @@ class VMInstance(object):
else:
domain_information = daemon_common.getInformationFromXML(self.zk_conn, self.domuuid)
memory = int(domain_information['memory'])
except:
except Exception:
memory = 0
return memory
@ -174,14 +176,14 @@ class VMInstance(object):
def getvcpus(self):
try:
vcpus = int(self.dom.info()[3])
except:
except Exception:
vcpus = 0
return vcpus
# Manage local node domain_list
def addDomainToList(self):
if not self.domuuid in self.this_node.domain_list:
if self.domuuid not in self.this_node.domain_list:
try:
# Add the domain to the domain_list array
self.this_node.domain_list.append(self.domuuid)
@ -211,7 +213,7 @@ class VMInstance(object):
# Start up a new Libvirt connection
libvirt_name = "qemu:///system"
lv_conn = libvirt.open(libvirt_name)
if lv_conn == None:
if lv_conn is None:
self.logger.out('Failed to open local libvirt connection', state='e', prefix='Domain {}'.format(self.domuuid))
self.instart = False
return
@ -220,7 +222,7 @@ class VMInstance(object):
try:
self.dom = self.lookupByUUID(self.domuuid)
curstate = self.dom.state()[0]
except:
except Exception:
curstate = 'notstart'
if curstate == libvirt.VIR_DOMAIN_RUNNING:
@ -255,7 +257,7 @@ class VMInstance(object):
# Start up a new Libvirt connection
libvirt_name = "qemu:///system"
lv_conn = libvirt.open(libvirt_name)
if lv_conn == None:
if lv_conn is None:
self.logger.out('Failed to open local libvirt connection', state='e', prefix='Domain {}'.format(self.domuuid))
self.inrestart = False
return
@ -295,7 +297,7 @@ class VMInstance(object):
self.logger.out('Failed to stop VM', state='e', prefix='Domain {}'.format(self.domuuid))
self.removeDomainFromList()
if self.inrestart == False:
if self.inrestart is False:
zkhandler.writedata(self.zk_conn, {'/domains/{}/state'.format(self.domuuid): 'stop'})
self.logger.out('Successfully stopped VM', state='o', prefix='Domain {}'.format(self.domuuid))
@ -325,7 +327,7 @@ class VMInstance(object):
try:
lvdomstate = self.dom.state()[0]
except:
except Exception:
lvdomstate = None
if lvdomstate != libvirt.VIR_DOMAIN_RUNNING:
@ -422,7 +424,7 @@ class VMInstance(object):
self.logger.out('Acquiring write lock for synchronization phase B', state='i', prefix='Domain {}'.format(self.domuuid))
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase B', state='o', prefix='Domain {}'.format(self.domuuid))
time.sleep(0.5) # Time for reader to acquire the lock
time.sleep(0.5) # Time fir reader to acquire the lock
def migrate_live():
self.logger.out('Setting up live migration', state='i', prefix='Domain {}'.format(self.domuuid))
@ -435,7 +437,7 @@ class VMInstance(object):
dest_lv_conn = libvirt.open(dest_lv)
if not dest_lv_conn:
raise
except:
except Exception:
self.logger.out('Failed to open connection to {}; aborting live migration.'.format(dest_lv), state='e', prefix='Domain {}'.format(self.domuuid))
return False
@ -510,10 +512,10 @@ class VMInstance(object):
self.logger.out('Acquiring write lock for synchronization phase C', state='i', prefix='Domain {}'.format(self.domuuid))
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase C', state='o', prefix='Domain {}'.format(self.domuuid))
time.sleep(0.5) # Time for reader to acquire the lock
time.sleep(0.5) # Time fir reader to acquire the lock
if do_migrate_shutdown:
migrate_shutdown_result = migrate_shutdown()
migrate_shutdown()
self.logger.out('Releasing write lock for synchronization phase C', state='i', prefix='Domain {}'.format(self.domuuid))
lock.release()
@ -548,7 +550,6 @@ class VMInstance(object):
time.sleep(0.1)
self.inreceive = True
live_receive = True
self.logger.out('Receiving VM migration from node "{}"'.format(self.node), state='i', prefix='Domain {}'.format(self.domuuid))
@ -560,11 +561,11 @@ class VMInstance(object):
self.logger.out('Acquiring write lock for synchronization phase A', state='i', prefix='Domain {}'.format(self.domuuid))
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase A', state='o', prefix='Domain {}'.format(self.domuuid))
time.sleep(0.5) # Time for reader to acquire the lock
time.sleep(0.5) # Time fir reader to acquire the lock
self.logger.out('Releasing write lock for synchronization phase A', state='i', prefix='Domain {}'.format(self.domuuid))
lock.release()
self.logger.out('Released write lock for synchronization phase A', state='o', prefix='Domain {}'.format(self.domuuid))
time.sleep(0.1) # Time for new writer to acquire the lock
time.sleep(0.1) # Time fir new writer to acquire the lock
# Synchronize nodes B (I am reader)
lock = zkhandler.readlock(self.zk_conn, '/locks/domain_migrate/{}'.format(self.domuuid))
@ -594,7 +595,7 @@ class VMInstance(object):
self.logger.out('Acquiring write lock for synchronization phase D', state='i', prefix='Domain {}'.format(self.domuuid))
lock.acquire()
self.logger.out('Acquired write lock for synchronization phase D', state='o', prefix='Domain {}'.format(self.domuuid))
time.sleep(0.5) # Time for reader to acquire the lock
time.sleep(0.5) # Time fir reader to acquire the lock
self.state = zkhandler.readdata(self.zk_conn, '/domains/{}/state'.format(self.domuuid))
self.dom = self.lookupByUUID(self.domuuid)
@ -639,11 +640,11 @@ class VMInstance(object):
# Check the current state of the VM
try:
if self.dom != None:
if self.dom is not None:
running, reason = self.dom.state()
else:
raise
except:
except Exception:
running = libvirt.VIR_DOMAIN_NOSTATE
self.logger.out('VM state change for "{}": {} {}'.format(self.domuuid, self.state, self.node), state='i')
@ -663,12 +664,12 @@ class VMInstance(object):
# provision
# Conditional pass one - Are we already performing an action
if self.instart == False \
and self.inrestart == False \
and self.inmigrate == False \
and self.inreceive == False \
and self.inshutdown == False \
and self.instop == False:
if self.instart is False \
and self.inrestart is False \
and self.inmigrate is False \
and self.inreceive is False \
and self.inshutdown is False \
and self.instop is False:
# Conditional pass two - Is this VM configured to run on this node
if self.node == self.this_node.name:
# Conditional pass three - Is this VM currently running on this node
@ -734,7 +735,6 @@ class VMInstance(object):
else:
self.terminate_vm()
# This function is a wrapper for libvirt.lookupByUUID which fixes some problems
# 1. Takes a text UUID and handles converting it to bytes
# 2. Try's it and returns a sensible value if not
@ -753,7 +753,7 @@ class VMInstance(object):
try:
# Open a libvirt connection
lv_conn = libvirt.open(libvirt_name)
if lv_conn == None:
if lv_conn is None:
self.logger.out('Failed to open local libvirt connection', state='e', prefix='Domain {}'.format(self.domuuid))
return None
@ -761,13 +761,13 @@ class VMInstance(object):
dom = lv_conn.lookupByUUID(buuid)
# Fail
except:
except Exception:
dom = None
# After everything
finally:
# Close the libvirt connection
if lv_conn != None:
if lv_conn is not None:
lv_conn.close()
# Return the dom object (or None)

21
node-daemon/pvcnoded/VXNetworkInstance.py
View File

@ -25,10 +25,10 @@ import time
from textwrap import dedent
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
import pvcnoded.common as common
class VXNetworkInstance(object):
# Initialization function
def __init__(self, vni, zk_conn, config, logger, this_node, dns_aggregator):
@ -227,7 +227,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
self.startDHCPServer()
@self.zk_conn.DataWatch('/networks/{}/ip6_gateway'.format(self.vni))
def watch_network_gateway(data, stat, event=''):
def watch_network_gateway6(data, stat, event=''):
if event and event.type == 'DELETED':
# The key has been deleted after existing before; terminate this watcher
# because this class instance is about to be reaped in Daemon.py
@ -249,7 +249,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
self.startDHCPServer()
@self.zk_conn.DataWatch('/networks/{}/dhcp6_flag'.format(self.vni))
def watch_network_dhcp_status(data, stat, event=''):
def watch_network_dhcp6_status(data, stat, event=''):
if event and event.type == 'DELETED':
# The key has been deleted after existing before; terminate this watcher
# because this class instance is about to be reaped in Daemon.py
@ -278,7 +278,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
self.startDHCPServer()
@self.zk_conn.DataWatch('/networks/{}/ip4_gateway'.format(self.vni))
def watch_network_gateway(data, stat, event=''):
def watch_network_gateway4(data, stat, event=''):
if event and event.type == 'DELETED':
# The key has been deleted after existing before; terminate this watcher
# because this class instance is about to be reaped in Daemon.py
@ -300,7 +300,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
self.startDHCPServer()
@self.zk_conn.DataWatch('/networks/{}/dhcp4_flag'.format(self.vni))
def watch_network_dhcp_status(data, stat, event=''):
def watch_network_dhcp4_status(data, stat, event=''):
if event and event.type == 'DELETED':
# The key has been deleted after existing before; terminate this watcher
# because this class instance is about to be reaped in Daemon.py
@ -356,7 +356,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
self.startDHCPServer()
@self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules/in'.format(self.vni))
def watch_network_firewall_rules(new_rules, event=''):
def watch_network_firewall_rules_in(new_rules, event=''):
if event and event.type == 'DELETED':
# The key has been deleted after existing before; terminate this watcher
# because this class instance is about to be reaped in Daemon.py
@ -368,7 +368,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
self.updateFirewallRules()
@self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules/out'.format(self.vni))
def watch_network_firewall_rules(new_rules, event=''):
def watch_network_firewall_rules_out(new_rules, event=''):
if event and event.type == 'DELETED':
# The key has been deleted after existing before; terminate this watcher
# because this class instance is about to be reaped in Daemon.py
@ -409,7 +409,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
try:
os.remove(filename)
self.dhcp_server_daemon.signal('hup')
except:
except Exception:
pass
def updateFirewallRules(self):
@ -453,8 +453,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
output = "{}\n# User rules\n{}\n".format(
firewall_rules,
'\n'.join(full_ordered_rules)
)
'\n'.join(full_ordered_rules))
with open(self.nftables_netconf_filename, 'w') as nfnetfile:
nfnetfile.write(dedent(output))
@ -802,7 +801,7 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out
try:
os.remove(self.nftables_netconf_filename)
except:
except Exception:
pass
# Reload firewall rules

22
node-daemon/pvcnoded/common.py
View File

@ -22,14 +22,13 @@
import subprocess
import signal
import time
from threading import Thread
from shlex import split as shlex_split
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
class OSDaemon(object):
def __init__(self, command_string, environment, logfile):
command = shlex_split(command_string)
@ -57,10 +56,12 @@ class OSDaemon(object):
}
self.proc.send_signal(signal_map[sent_signal])
def run_os_daemon(command_string, environment=None, logfile=None):
daemon = OSDaemon(command_string, environment, logfile)
return daemon
# Run a oneshot command, optionally without blocking
def run_os_command(command_string, background=False, environment=None, timeout=None):
command = shlex_split(command_string)
@ -94,14 +95,15 @@ def run_os_command(command_string, background=False, environment=None, timeout=N
try:
stdout = command_output.stdout.decode('ascii')
except:
except Exception:
stdout = ''
try:
stderr = command_output.stderr.decode('ascii')
except:
except Exception:
stderr = ''
return retcode, stdout, stderr
# Reload the firewall rules of the system
def reload_firewall_rules(logger, rules_file):
logger.out('Reloading firewall configuration', state='o')
@ -109,6 +111,7 @@ def reload_firewall_rules(logger, rules_file):
if retcode != 0:
logger.out('Failed to reload configuration: {}'.format(stderr), state='e')
# Create IP address
def createIPAddress(ipaddr, cidrnetmask, dev):
run_os_command(
@ -125,6 +128,7 @@ def createIPAddress(ipaddr, cidrnetmask, dev):
)
)
# Remove IP address
def removeIPAddress(ipaddr, cidrnetmask, dev):
run_os_command(
@ -135,6 +139,7 @@ def removeIPAddress(ipaddr, cidrnetmask, dev):
)
)
#
# Find a migration target
#
@ -144,14 +149,14 @@ def findTargetNode(zk_conn, config, logger, dom_uuid):
node_limit = zkhandler.readdata(zk_conn, '/domains/{}/node_limit'.format(dom_uuid)).split(',')
if not any(node_limit):
node_limit = ''
except:
except Exception:
node_limit = ''
zkhandler.writedata(zk_conn, {'/domains/{}/node_limit'.format(dom_uuid): ''})
# Determine VM search field
try:
search_field = zkhandler.readdata(zk_conn, '/domains/{}/node_selector'.format(dom_uuid))
except Exception as e:
except Exception:
search_field = None
# If our search field is invalid, use and set the default (for next time)
@ -175,6 +180,7 @@ def findTargetNode(zk_conn, config, logger, dom_uuid):
# Nothing was found
return None
# Get the list of valid target nodes
def getNodes(zk_conn, node_limit, dom_uuid):
valid_node_list = []
@ -198,6 +204,7 @@ def getNodes(zk_conn, node_limit, dom_uuid):
return valid_node_list
# via free memory (relative to allocated memory)
def findTargetNodeMem(zk_conn, config, logger, node_limit, dom_uuid):
most_provfree = 0
@ -224,6 +231,7 @@ def findTargetNodeMem(zk_conn, config, logger, node_limit, dom_uuid):
logger.out('Selected node {}'.format(target_node), state='d', prefix='node-flush')
return target_node
# via load average
def findTargetNodeLoad(zk_conn, config, logger, node_limit, dom_uuid):
least_load = 9999.0
@ -246,6 +254,7 @@ def findTargetNodeLoad(zk_conn, config, logger, node_limit, dom_uuid):
logger.out('Selected node {}'.format(target_node), state='d', prefix='node-flush')
return target_node
# via total vCPUs
def findTargetNodeVCPUs(zk_conn, config, logger, node_limit, dom_uuid):
least_vcpus = 9999
@ -268,6 +277,7 @@ def findTargetNodeVCPUs(zk_conn, config, logger, node_limit, dom_uuid):
logger.out('Selected node {}'.format(target_node), state='d', prefix='node-flush')
return target_node
# via total VMs
def findTargetNodeVMs(zk_conn, config, logger, node_limit, dom_uuid):
least_vms = 9999

21
node-daemon/pvcnoded/dnsmasq-zookeeper-leases.py
View File

@ -20,11 +20,13 @@
#
###############################################################################
import argparse
import os, sys
import os
import sys
import kazoo.client
import re
import yaml
#
# Variables
#
@ -39,30 +41,33 @@ def get_zookeeper_key():
print('ERROR: DNSMASQ_BRIDGE_INTERFACE environment variable not found: {}'.format(e), file=sys.stderr)
exit(1)
# Get the ID of the interface (the digits)
network_vni = re.findall('\d+', interface)[0]
network_vni = re.findall(r'\d+', interface)[0]
# Create the key
zookeeper_key = '/networks/{}/dhcp4_leases'.format(network_vni)
return zookeeper_key
def get_lease_expiry():
try:
expiry = os.environ['DNSMASQ_LEASE_EXPIRES']
except:
except Exception:
expiry = '0'
return expiry
def get_client_id():
try:
client_id = os.environ['DNSMASQ_CLIENT_ID']
except:
except Exception:
client_id = '*'
return client_id
def connect_zookeeper():
# We expect the environ to contain the config file
try:
pvcnoded_config_file = os.environ['PVCD_CONFIG_FILE']
except:
except Exception:
# Default place
pvcnoded_config_file = '/etc/pvc/pvcnoded.yaml'
@ -82,9 +87,11 @@ def connect_zookeeper():
return zk_conn
def read_data(zk_conn, key):
return zk_conn.get(key)[0].decode('ascii')
def get_lease(zk_conn, zk_leases_key, macaddr):
expiry = read_data(zk_conn, '{}/{}/expiry'.format(zk_leases_key, macaddr))
ipaddr = read_data(zk_conn, '{}/{}/ipaddr'.format(zk_leases_key, macaddr))
@ -92,6 +99,7 @@ def get_lease(zk_conn, zk_leases_key, macaddr):
clientid = read_data(zk_conn, '{}/{}/clientid'.format(zk_leases_key, macaddr))
return expiry, ipaddr, hostname, clientid
#
# Command Functions
#
@ -107,6 +115,7 @@ def read_lease_database(zk_conn, zk_leases_key):
# Output list
print('\n'.join(output_list))
def add_lease(zk_conn, zk_leases_key, expiry, macaddr, ipaddr, hostname, clientid):
if not hostname:
hostname = ''
@ -118,9 +127,11 @@ def add_lease(zk_conn, zk_leases_key, expiry, macaddr, ipaddr, hostname, clienti
transaction.create('{}/{}/clientid'.format(zk_leases_key, macaddr), clientid.encode('ascii'))
transaction.commit()
def del_lease(zk_conn, zk_leases_key, macaddr, expiry):
zk_conn.delete('{}/{}'.format(zk_leases_key, macaddr), recursive=True)
#
# Instantiate the parser
#

4
node-daemon/pvcnoded/fencing.py
View File

@ -26,6 +26,7 @@ import pvcnoded.zkhandler as zkhandler
import pvcnoded.common as common
import pvcnoded.VMInstance as VMInstance
#
# Fence thread entry function
#
@ -74,6 +75,7 @@ def fenceNode(node_name, zk_conn, config, logger):
if not fence_status and config['failed_fence'] == 'migrate' and config['suicide_intervals'] != '0':
migrateFromFencedNode(zk_conn, node_name, config, logger)
# Migrate hosts away from a fenced node
def migrateFromFencedNode(zk_conn, node_name, config, logger):
logger.out('Migrating VMs from dead node "{}" to new hosts'.format(node_name), state='i')
@ -111,6 +113,7 @@ def migrateFromFencedNode(zk_conn, node_name, config, logger):
# Set node in flushed state for easy remigrating when it comes back
zkhandler.writedata(zk_conn, {'/nodes/{}/domainstate'.format(node_name): 'flushed'})
#
# Perform an IPMI fence
#
@ -145,6 +148,7 @@ def rebootViaIPMI(ipmi_hostname, ipmi_user, ipmi_password, logger):
print(ipmi_reset_stderr)
return False
#
# Verify that IPMI connectivity to this host exists (used during node init)
#

1
node-daemon/pvcnoded/log.py
View File

@ -22,6 +22,7 @@
import datetime
class Logger(object):
# Define a logger class for a daemon instance
# Keeps record of where to log, and is passed messages which are

12
node-daemon/pvcnoded/zkhandler.py
View File

@ -22,6 +22,7 @@
import uuid
# Child list function
def listchildren(zk_conn, key):
try:
@ -30,6 +31,7 @@ def listchildren(zk_conn, key):
except Exception:
return None
# Key deletion function
def deletekey(zk_conn, key, recursive=True):
try:
@ -38,16 +40,17 @@ def deletekey(zk_conn, key, recursive=True):
except Exception:
return False
# Data read function
def readdata(zk_conn, key):
try:
data_raw = zk_conn.get(key)
data = data_raw[0].decode('utf8')
meta = data_raw[1]
return data
except Exception:
return None
# Data write function
def writedata(zk_conn, kv):
# Commit the transaction
@ -88,6 +91,7 @@ def writedata(zk_conn, kv):
except Exception:
return False
# Key rename function
def renamekey(zk_conn, kv):
# This one is not transactional because, inexplicably, transactions don't
@ -102,8 +106,9 @@ def renamekey(zk_conn, kv):
old_data = zk_conn.get(old_name)[0]
# Find the children of old_name recursively
child_keys = list()
# Find the children of old_name recursively
def get_children(key):
children = zk_conn.get_children(key)
if not children:
@ -133,6 +138,7 @@ def renamekey(zk_conn, kv):
except Exception:
return False
# Write lock function
def writelock(zk_conn, key):
count = 1
@ -149,6 +155,7 @@ def writelock(zk_conn, key):
continue
return lock
# Read lock function
def readlock(zk_conn, key):
count = 1
@ -165,6 +172,7 @@ def readlock(zk_conn, key):
continue
return lock
# Exclusive lock function
def exclusivelock(zk_conn, key):
count = 1