Joshua Boniface
31f252ad9b
Implements the ability for a client to watch almost-live domain
console logs from the hypervisors. It does this using a deque-based
"tail -f" mechanism (with a configurable buffer per-VM) that watches
the domain console logfile in the (configurable) directory every
half-second. It then stores the current buffer in Zookeeper when
changed, where a client can then request it, either as a static piece
of text in the `less` pager, or via a similar "tail -f" functionality
implemented using fixed line splitting and comparison to provide a
generally-seamless output.
Enabling this feature requires each guest VM to implement a Libvirt
serial log and write its (text) console to it, for example using the
default logging directory:
```
<serial type='pty'>
<log file='/var/log/libvirt/vmname.log' append='off'/>
<serial>
```
The append mode can be either on or off; on grows files unbounded,
off causes the log (and hence the PVC log data) to be truncated on
initial VM startup from offline. The administrator must choose how
they best want to handle this until Libvirt implements their own
clog-type logging format.
101 lines
3.4 KiB
Python
101 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# DomainConsoleWatcherInstance.py - Class implementing a console log watcher for PVC domains
|
|
# Part of the Parallel Virtual Cluster (PVC) system
|
|
#
|
|
# Copyright (C) 2018 Joshua M. Boniface <joshua@boniface.me>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
#
|
|
###############################################################################
|
|
|
|
import os
|
|
import sys
|
|
import uuid
|
|
import time
|
|
import threading
|
|
import libvirt
|
|
|
|
from collections import deque
|
|
|
|
import fcntl
|
|
import signal
|
|
|
|
import pvcd.log as log
|
|
import pvcd.zkhandler as zkhandler
|
|
|
|
class DomainConsoleWatcherInstance(object):
|
|
# Initialization function
|
|
def __init__(self, domuuid, domname, zk_conn, config, logger, this_node):
|
|
self.domuuid = domuuid
|
|
self.domname = domname
|
|
self.zk_conn = zk_conn
|
|
self.config = config
|
|
self.logfile = '{}/{}.log'.format(config['console_log_directory'], self.domname)
|
|
self.console_log_lines = config['console_log_lines']
|
|
self.logger = logger
|
|
self.this_node = this_node
|
|
|
|
# Try to append (create) the logfile and set its permissions
|
|
open(self.logfile, 'a').close()
|
|
os.chmod(self.logfile, 0o600)
|
|
|
|
self.logdeque = deque(open(self.logfile), self.console_log_lines)
|
|
|
|
self.stamp = None
|
|
self.cached_stamp = None
|
|
|
|
# Set up the deque with the current contents of the log
|
|
self.last_loglines = None
|
|
self.loglines = None
|
|
|
|
# Thread options
|
|
self.thread = None
|
|
self.thread_stopper = threading.Event()
|
|
|
|
# Start execution thread
|
|
def start(self):
|
|
self.thread_stopper.clear()
|
|
self.thread = threading.Thread(target=self.run, args=(), kwargs={})
|
|
self.logger.out('Starting VM log parser', state='i', prefix='Domain {}:'.format(self.domuuid))
|
|
self.thread.start()
|
|
|
|
# Stop execution thread
|
|
def stop(self):
|
|
self.logger.out('Stopping VM log parser', state='i', prefix='Domain {}:'.format(self.domuuid))
|
|
self.thread_stopper.set()
|
|
# Do one final flush
|
|
self.update()
|
|
|
|
# Main entrypoint
|
|
def run(self):
|
|
# Main loop
|
|
while not self.thread_stopper.is_set():
|
|
self.update()
|
|
time.sleep(0.5)
|
|
|
|
def update(self):
|
|
self.stamp = os.stat(self.logfile).st_mtime
|
|
if self.stamp != self.cached_stamp:
|
|
self.cached_stamp = self.stamp
|
|
self.fetch_lines()
|
|
# Update Zookeeper with the new loglines if they changed
|
|
if self.loglines != self.last_loglines:
|
|
zkhandler.writedata(self.zk_conn, { '/domains/{}/consolelog'.format(self.domuuid): self.loglines })
|
|
self.last_loglines = self.loglines
|
|
|
|
def fetch_lines(self):
|
|
self.logdeque = deque(open(self.logfile), self.console_log_lines)
|
|
self.loglines = ''.join(self.logdeque)
|