parallelvirtualcluster/pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 115 Wiki Activity

Port remaining tasks to new task handler

Browse Source 
Move the create_vm and run_benchmark tasks to use the new Celery
subsystem, handlers, and wait command. Remove the obsolete, dedicated
API endpoints.

Standardize the CLI client and move the repeated handler code into a
separate common function.
This commit is contained in:
Joshua Boniface
2023-11-16 01:57:56 -05:00
parent aef38639cf
commit 484e6542c2
7 changed files with 67 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
client-cli/pvc/cli/cli.py
View File

@ -3296,15 +3296,26 @@ def cli_storage_benchmark():
@click.command(name="run", short_help="Run a storage benchmark.")
@connection_req
@click.argument("pool")
@click.option(
"--wait/--no-wait",
"wait_flag",
is_flag=True,
default=True,
show_default=True,
help="Wait or don't wait for task to complete, showing progress",
)
@confirm_opt(
"Storage benchmarks take approximately 10 minutes to run and generate significant load on the cluster; they should be run sparingly. Continue"
)
def cli_storage_benchmark_run(pool):
def cli_storage_benchmark_run(pool, wait_flag):
"""
Run a storage benchmark on POOL in the background.
"""
retcode, retmsg = pvc.lib.storage.ceph_benchmark_run(CLI_CONFIG, pool)
retcode, retmsg = pvc.lib.storage.ceph_benchmark_run(CLI_CONFIG, pool, wait_flag)
if retcode and wait_flag:
retmsg = wait_for_celery_task(CLI_CONFIG, retmsg)
finish(retcode, retmsg)
@ -5581,15 +5592,15 @@ def cli_provisioner_profile_list(limit, format_function):
help="Start the VM automatically upon completion of provisioning.",
)
@click.option(
"-w",
"--wait",
"--wait/--no-wait",
"wait_flag",
is_flag=True,
default=False,
help="Wait for provisioning to complete, showing progress",
default=True,
show_default=True,
help="Wait or don't wait for task to complete, showing progress",
)
def cli_provisioner_create(
name, profile, wait_flag, define_flag, start_flag, script_args
name, profile, define_flag, start_flag, script_args, wait_flag
):
"""
Create a new VM NAME with profile PROFILE.
@ -5610,15 +5621,13 @@ def cli_provisioner_create(
if not define_flag:
start_flag = False
retcode, retdata = pvc.lib.provisioner.vm_create(
CLI_CONFIG, name, profile, wait_flag, define_flag, start_flag, script_args
retcode, retmsg = pvc.lib.provisioner.vm_create(
CLI_CONFIG, name, profile, define_flag, start_flag, script_args, wait_flag
)
if retcode and wait_flag:
task_id = retdata
retdata = wait_for_provisioner(CLI_CONFIG, task_id)
finish(retcode, retdata)
retmsg = wait_for_celery_task(CLI_CONFIG, retmsg)
finish(retcode, retmsg)
###############################################################################

57
client-cli/pvc/cli/waiters.py
View File

@ -132,60 +132,3 @@ def wait_for_celery_task(CLI_CONFIG, task_detail):
retdata = task_status.get("state") + ": " + task_status.get("status")
return retdata
def wait_for_provisioner(CLI_CONFIG, task_id):
"""
Wait for a provisioner task to complete
"""
echo(CLI_CONFIG, f"Task ID: {task_id}")
echo(CLI_CONFIG, "")
# Wait for the task to start
echo(CLI_CONFIG, "Waiting for task to start...", newline=False)
while True:
sleep(1)
task_status = pvc.lib.provisioner.task_status(
CLI_CONFIG, task_id, is_watching=True
)
if task_status.get("state") != "PENDING":
break
echo(CLI_CONFIG, ".", newline=False)
echo(CLI_CONFIG, " done.")
echo(CLI_CONFIG, "")
# Start following the task state, updating progress as we go
total_task = task_status.get("total")
with progressbar(length=total_task, show_eta=False) as bar:
last_task = 0
maxlen = 0
while True:
sleep(1)
if task_status.get("state") != "RUNNING":
break
if task_status.get("current") > last_task:
current_task = int(task_status.get("current"))
bar.update(current_task - last_task)
last_task = current_task
# The extensive spaces at the end cause this to overwrite longer previous messages
curlen = len(str(task_status.get("status")))
if curlen > maxlen:
maxlen = curlen
lendiff = maxlen - curlen
overwrite_whitespace = " " * lendiff
echo(
CLI_CONFIG,
" " + task_status.get("status") + overwrite_whitespace,
newline=False,
)
task_status = pvc.lib.provisioner.task_status(
CLI_CONFIG, task_id, is_watching=True
)
if task_status.get("state") == "SUCCESS":
bar.update(total_task - last_task)
echo(CLI_CONFIG, "")
retdata = task_status.get("state") + ": " + task_status.get("status")
return retdata

18
client-cli/pvc/lib/common.py
View File

@ -202,6 +202,24 @@ def call_api(
return response
def get_wait_retdata(response, wait_flag):
if response.status_code == 202:
retvalue = True
retjson = response.json()
if not wait_flag:
retdata = (
f"Task ID: {retjson['task_id']} assigned to node {retjson['run_on']}"
)
else:
# Just return the task JSON without formatting
retdata = response.json()
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
def task_status(config, task_id=None, is_watching=False):
"""
Get information about Celery job {task_id}, or all tasks if None

17
client-cli/pvc/lib/provisioner.py
View File

@ -25,7 +25,7 @@ from requests_toolbelt.multipart.encoder import (
)
import pvc.lib.ansiprint as ansiprint
from pvc.lib.common import UploadProgressBar, call_api
from pvc.lib.common import UploadProgressBar, call_api, get_wait_retdata
from ast import literal_eval
@ -700,7 +700,7 @@ def profile_remove(config, name):
return retvalue, response.json().get("message", "")
def vm_create(config, name, profile, wait_flag, define_flag, start_flag, script_args):
def vm_create(config, name, profile, define_flag, start_flag, script_args, wait_flag):
"""
Create a new VM named {name} with profile {profile}
@ -717,18 +717,7 @@ def vm_create(config, name, profile, wait_flag, define_flag, start_flag, script_
}
response = call_api(config, "post", "/provisioner/create", params=params)
if response.status_code == 202:
retvalue = True
if not wait_flag:
retdata = "Task ID: {}".format(response.json()["task_id"])
else:
# Just return the task_id raw, instead of formatting it
retdata = response.json()["task_id"]
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
def task_status(config, task_id=None, is_watching=False):

93
client-cli/pvc/lib/storage.py
View File

@ -29,7 +29,7 @@ from requests_toolbelt.multipart.encoder import (
)
import pvc.lib.ansiprint as ansiprint
from pvc.lib.common import UploadProgressBar, call_api
from pvc.lib.common import UploadProgressBar, call_api, get_wait_retdata
#
# Supplemental functions
@ -175,21 +175,7 @@ def ceph_osd_db_vg_add(config, node, device, wait_flag):
params = {"node": node, "device": device}
response = call_api(config, "post", "/storage/ceph/osddb", params=params)
if response.status_code == 202:
retvalue = True
retjson = response.json()
if not wait_flag:
retdata = (
f"Task ID: {retjson['task_id']} assigned to node {retjson['run_on']}"
)
else:
# Just return the task JSON without formatting
retdata = response.json()
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
#
@ -265,21 +251,7 @@ def ceph_osd_add(
response = call_api(config, "post", "/storage/ceph/osd", params=params)
if response.status_code == 202:
retvalue = True
retjson = response.json()
if not wait_flag:
retdata = (
f"Task ID: {retjson['task_id']} assigned to node {retjson['run_on']}"
)
else:
# Just return the task JSON without formatting
retdata = response.json()
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
def ceph_osd_replace(
@ -308,21 +280,7 @@ def ceph_osd_replace(
response = call_api(config, "post", f"/storage/ceph/osd/{osdid}", params=params)
if response.status_code == 202:
retvalue = True
retjson = response.json()
if not wait_flag:
retdata = (
f"Task ID: {retjson['task_id']} assigned to node {retjson['run_on']}"
)
else:
# Just return the task JSON without formatting
retdata = response.json()
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
def ceph_osd_refresh(config, osdid, device, wait_flag):
@ -338,21 +296,7 @@ def ceph_osd_refresh(config, osdid, device, wait_flag):
}
response = call_api(config, "put", f"/storage/ceph/osd/{osdid}", params=params)
if response.status_code == 202:
retvalue = True
retjson = response.json()
if not wait_flag:
retdata = (
f"Task ID: {retjson['task_id']} assigned to node {retjson['run_on']}"
)
else:
# Just return the task JSON without formatting
retdata = response.json()
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
def ceph_osd_remove(config, osdid, force_flag, wait_flag):
@ -368,21 +312,7 @@ def ceph_osd_remove(config, osdid, force_flag, wait_flag):
config, "delete", "/storage/ceph/osd/{osdid}".format(osdid=osdid), params=params
)
if response.status_code == 202:
retvalue = True
retjson = response.json()
if not wait_flag:
retdata = (
f"Task ID: {retjson['task_id']} assigned to node {retjson['run_on']}"
)
else:
# Just return the task JSON without formatting
retdata = response.json()
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
def ceph_osd_state(config, osdid, state):
@ -1765,7 +1695,7 @@ def format_list_snapshot(config, snapshot_list):
#
# Benchmark functions
#
def ceph_benchmark_run(config, pool):
def ceph_benchmark_run(config, pool, wait_flag):
"""
Run a storage benchmark against {pool}
@ -1776,14 +1706,7 @@ def ceph_benchmark_run(config, pool):
params = {"pool": pool}
response = call_api(config, "post", "/storage/ceph/benchmark", params=params)
if response.status_code == 202:
retvalue = True
retdata = "Task ID: {}".format(response.json()["task_id"])
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
def ceph_benchmark_list(config, job):

18
client-cli/pvc/lib/vm.py
View File

@ -23,7 +23,7 @@ import time
import re
import pvc.lib.ansiprint as ansiprint
from pvc.lib.common import call_api, format_bytes, format_metric
from pvc.lib.common import call_api, format_bytes, format_metric, get_wait_retdata
#
@ -425,21 +425,7 @@ def vm_locks(config, vm, wait_flag):
"""
response = call_api(config, "post", f"/vm/{vm}/locks")
if response.status_code == 202:
retvalue = True
retjson = response.json()
if not wait_flag:
retdata = (
f"Task ID: {retjson['task_id']} assigned to node {retjson['run_on']}"
)
else:
# Just return the task JSON without formatting
retdata = response.json()
else:
retvalue = False
retdata = response.json().get("message", "")
return retvalue, retdata
return get_wait_retdata(response, wait_flag)
def vm_backup(config, vm, backup_path, incremental_parent=None, retain_snapshot=False):