Integrate metadata API into node daemon

debian/control

@@ -8,8 +8,8 @@ X-Python3-Version: >= 3.2
 
 Package: pvc-daemon
 Architecture: all
-Depends: python3-kazoo, python3-psutil, python3-apscheduler, python3-libvirt, python3-psycopg2, python3-dnspython, python3-yaml, ipmitool, libvirt-daemon-system, arping, vlan, bridge-utils, dnsmasq, nftables, pdns-server, pdns-backend-pgsql
+Depends: pvc-client-common, python3-kazoo, python3-psutil, python3-apscheduler, python3-libvirt, python3-psycopg2, python3-dnspython, python3-yaml, ipmitool, libvirt-daemon-system, arping, vlan, bridge-utils, dnsmasq, nftables, pdns-server, pdns-backend-pgsql
-Suggests: pvc-client-cli
+Suggests: pvc-client-api, pvc-client-cli
 Description: Parallel Virtual Cluster virtualization daemon (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .

node-daemon/pvcd.sample.yaml

@@ -25,8 +25,6 @@ pvc:
     enable_storage: True
     # enable_api: Enable or disable the API client, if installed, when node is Primary
     enable_api: True
-    # enable_provisioner: Enable or disable the Provisioner client, if installed, when node is Primary
-    enable_provisioner: True
   # cluster: Cluster-level configuration
   cluster:
     # coordinators: The list of cluster coordinator hostnames
@@ -80,6 +78,20 @@ pvc:
         user: pvcdns
         # pass: PostgreSQL user password, randomly generated
         pass: pvcdns
+    # metadata: Metadata API subsystem
+    metadata:
+      # database: Patroni PostgreSQL database configuration
+      database:
+        # host: PostgreSQL hostname, invariably 'localhost'
+        host: localhost
+        # port: PostgreSQL port, invariably 'localhost'
+        port: 5432
+        # name: PostgreSQL database name, invariably 'pvcprov'
+        name: pvcprov
+        # user: PostgreSQL username, invariable 'pvcprov'
+        user: pvcprov
+        # pass: PostgreSQL user password, randomly generated
+        pass: pvcprov
   # system: Local PVC instance configuration
   system:
     # intervals: Intervals for keepalives and fencing

node-daemon/pvcd/Daemon.py

@@ -52,6 +52,7 @@ import pvcd.NodeInstance as NodeInstance
 import pvcd.VXNetworkInstance as VXNetworkInstance
 import pvcd.DNSAggregatorInstance as DNSAggregatorInstance
 import pvcd.CephInstance as CephInstance
+import pvcd.MetadataAPIInstance as MetadataAPIInstance
 
 ###############################################################################
 # PVCD - node daemon startup program
@@ -194,6 +195,11 @@ def readConfig(pvcd_config_file, myhostname):
                 'pdns_postgresql_dbname': o_config['pvc']['coordinator']['dns']['database']['name'],
                 'pdns_postgresql_user': o_config['pvc']['coordinator']['dns']['database']['user'],
                 'pdns_postgresql_password': o_config['pvc']['coordinator']['dns']['database']['pass'],
+                'metadata_postgresql_host': o_config['pvc']['coordinator']['metadata']['database']['host'],
+                'metadata_postgresql_port': o_config['pvc']['coordinator']['metadata']['database']['port'],
+                'metadata_postgresql_dbname': o_config['pvc']['coordinator']['metadata']['database']['name'],
+                'metadata_postgresql_user': o_config['pvc']['coordinator']['metadata']['database']['user'],
+                'metadata_postgresql_password': o_config['pvc']['coordinator']['metadata']['database']['pass'],
                 'vni_dev': o_config['pvc']['system']['configuration']['networking']['cluster']['device'],
                 'vni_mtu': o_config['pvc']['system']['configuration']['networking']['cluster']['mtu'],
                 'vni_dev_ip': o_config['pvc']['system']['configuration']['networking']['cluster']['address'],
@@ -726,13 +732,16 @@ pool_list = []
 volume_list = dict() # Dict of Lists
 
 if enable_networking:
-    # Create an instance of the DNS Aggregator if we're a coordinator
+    # Create an instance of the DNS Aggregator and Metadata API if we're a coordinator
     if config['daemon_mode'] == 'coordinator':
         dns_aggregator = DNSAggregatorInstance.DNSAggregatorInstance(zk_conn, config, logger)
+        metadata_api = MetadataAPIInstance.MetadataAPIInstance(zk_conn, config, logger)
     else:
         dns_aggregator = None
+        metadata_api = None
 else:
     dns_aggregator = None
+    metadata_api = None
 
 # Node objects
 @zk_conn.ChildrenWatch('/nodes')
@@ -742,7 +751,7 @@ def update_nodes(new_node_list):
     # Add any missing nodes to the list
     for node in new_node_list:
         if not node in node_list:
-            d_node[node] = NodeInstance.NodeInstance(node, myhostname, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator)
+            d_node[node] = NodeInstance.NodeInstance(node, myhostname, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator, metadata_api)
 
     # Remove any deleted nodes from the list
     for node in node_list:

node-daemon/pvcd/MetadataAPIInstance.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+
+# MetadataAPIInstance.py - Class implementing an EC2-compatible cloud-init Metadata server
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2019 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import gevent.pywsgi
+import flask
+import threading
+import sys
+import psycopg2
+from psycopg2.extras import RealDictCursor
+
+# The metadata server requires client libraries
+import client_lib.vm as pvc_vm
+import client_lib.network as pvc_network
+
+class MetadataAPIInstance(object):
+    mdapi = flask.Flask(__name__)
+
+    # Initialization function
+    def __init__(self, zk_conn, config, logger):
+        self.zk_conn = zk_conn
+        self.config = config
+        self.logger = logger
+        self.thread = None
+        self.md_http_server = None
+
+    # Add flask routes inside our instance
+    def add_routes(self):
+        @self.mdapi.route('/', methods=['GET'])
+        def api_root():
+            return flask.jsonify({"message": "PVC Provisioner Metadata API version 1"}), 209
+        
+        @self.mdapi.route('/<version>/meta-data/', methods=['GET'])
+        def api_metadata_root(version):
+            metadata = """instance-id\nname\nprofile"""
+            return metadata, 200
+        
+        @self.mdapi.route('/<version>/meta-data/instance-id', methods=['GET'])
+        def api_metadata_instanceid(version):
+            source_address = flask.request.__dict__['environ']['REMOTE_ADDR']
+            vm_details = self.get_vm_details(source_address)
+            instance_id = vm_details['uuid']
+            return instance_id, 200
+        
+        @self.mdapi.route('/<version>/meta-data/name', methods=['GET'])
+        def api_metadata_hostname(version):
+            source_address = flask.request.__dict__['environ']['REMOTE_ADDR']
+            vm_details = self.get_vm_details(source_address)
+            vm_name = vm_details['name']
+            return vm_name, 200
+        
+        @self.mdapi.route('/<version>/meta-data/profile', methods=['GET'])
+        def api_metadata_profile(version):
+            source_address = flask.request.__dict__['environ']['REMOTE_ADDR']
+            vm_details = self.get_vm_details(source_address)
+            vm_profile = vm_details['profile']
+            return vm_profile, 200
+        
+        @self.mdapi.route('/<version>/user-data', methods=['GET'])
+        def api_userdata(version):
+            source_address = flask.request.__dict__['environ']['REMOTE_ADDR']
+            vm_details = self.get_vm_details(source_address)
+            vm_profile = vm_details['profile']
+            # Get the userdata
+            userdata = self.get_profile_userdata(vm_profile)
+            self.logger.out("Returning userdata for profile {}".format(vm_profile), state='i', prefix='Metadata API')
+            return flask.Response(userdata)
+        
+    def launch_wsgi(self):
+        try:
+            self.add_routes()
+            self.md_http_server = gevent.pywsgi.WSGIServer(
+                ('169.254.169.254', 80),
+                self.mdapi,
+                log=sys.stdout,
+                error_log=sys.stdout
+            )
+            self.md_http_server.serve_forever()
+        except Exception as e:
+            self.logger.out('Error starting Metadata API: {}'.format(e), state='e')
+
+    # WSGI start/stop
+    def start(self):
+        # Launch Metadata API
+        self.logger.out('Starting Metadata API at 169.254.169.254:80', state='i')
+        self.thread = threading.Thread(target=self.launch_wsgi)
+        self.thread.start()
+        self.logger.out('Successfully started Metadata API thread', state='o')
+
+    def stop(self):
+        self.logger.out('Stopping Metadata API at 169.254.169.254:80', state='i')
+        if self.thread and self.md_http_server:
+            try:
+                self.md_http_server.stop()
+                self.md_http_server.close()
+                self.logger.out('Successfully stopped Metadata API', state='o')
+            except Exception as e:
+                self.logger.out('Error stopping Metadata API: {}'.format(e), state='e')
+
+    # Helper functions
+    def open_database(self):
+        conn = psycopg2.connect(
+            host=self.config['metadata_postgresql_host'],
+            port=self.config['metadata_postgresql_port'],
+            dbname=self.config['metadata_postgresql_dbname'],
+            user=self.config['metadata_postgresql_user'],
+            password=self.config['metadata_postgresql_password']
+        )
+        cur = conn.cursor(cursor_factory=RealDictCursor)
+        return conn, cur
+
+    def close_database(self, conn, cur):
+        cur.close()
+        conn.close()
+
+    # Obtain a list of templates
+    def get_profile_userdata(self, vm_profile):
+        query = """SELECT userdata FROM profile 
+        JOIN userdata_template ON profile.userdata_template = userdata_template.id
+        WHERE profile.name = %s;
+        """
+        args = (vm_profile,)
+
+        conn, cur = self.open_database()
+        cur.execute(query, args)
+        data_raw = cur.fetchone()
+        self.close_database(conn, cur)
+        data = data_raw['userdata']
+        return data
+
+    # VM details function
+    def get_vm_details(self, source_address):
+        # Start connection to Zookeeper
+        _discard, networks = pvc_network.get_list(self.zk_conn, None)
+    
+        # Figure out which server this is via the DHCP address
+        host_information = dict()
+        networks_managed = (x for x in networks if x['type'] == 'managed')
+        for network in networks_managed:
+            network_leases = pvc_network.getNetworkDHCPLeases(self.zk_conn, network['vni'])
+            for network_lease in network_leases:
+                information = pvc_network.getDHCPLeaseInformation(self.zk_conn, network['vni'], network_lease)
+                try:
+                    if information['ip4_address'] == source_address:
+                        host_information = information
+                except:
+                    pass
+    
+        # Get our real information on the host; now we can start querying about it
+        client_hostname = host_information['hostname']
+        client_macaddr = host_information['mac_address']
+        client_ipaddr = host_information['ip4_address']
+    
+        # Find the VM with that MAC address - we can't assume that the hostname is actually right
+        _discard, vm_list = pvc_vm.get_list(self.zk_conn, None, None, None)
+        vm_name = None
+        vm_details = dict()
+        for vm in vm_list:
+            try:
+                for network in vm['networks']:
+                    if network['mac'] == client_macaddr:
+                        vm_name = vm['name']
+                        vm_details = vm
+            except:
+                pass
+        
+        return vm_details
+   

node-daemon/pvcd/NodeInstance.py

@@ -34,7 +34,7 @@ import pvcd.common as common
 
 class NodeInstance(object):
     # Initialization function
-    def __init__(self, name, this_node, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator):
+    def __init__(self, name, this_node, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator, metadata_api):
         # Passed-in variables on creation
         self.name = name
         self.this_node = this_node
@@ -53,6 +53,7 @@ class NodeInstance(object):
         self.d_network = d_network
         self.d_domain = d_domain
         self.dns_aggregator = dns_aggregator
+        self.metadata_api = metadata_api
         # Printable lists
         self.active_node_list = []
         self.flushed_node_list = []
@@ -269,8 +270,9 @@ class NodeInstance(object):
         for network in self.d_network:
             self.d_network[network].stopDHCPServer()
             self.d_network[network].removeGateways()
-        self.removeFloatingAddresses()
         self.dns_aggregator.stop_aggregator()
+        self.metadata_api.stop()
+        self.removeFloatingAddresses()
 
     def become_primary(self):
         # Establish a lock
@@ -318,6 +320,7 @@ class NodeInstance(object):
             # Start the DNS aggregator instance
             time.sleep(1)
             self.dns_aggregator.start_aggregator()
+            self.metadata_api.start()
 
             # Start the clients
             if self.config['enable_api']:
@@ -327,6 +330,17 @@ class NodeInstance(object):
                 common.run_os_command("systemctl start pvc-provisioner-worker.service")
 
     def createFloatingAddresses(self):
+        # Metadata link-local IP
+        self.logger.out(
+            'Creating Metadata link-local IP {}/{} on interface {}'.format(
+                '169.254.169.254',
+                '32',
+                'lo'
+            ),
+            state='o'
+        )
+        common.createIPAddress('169.254.169.254', '32', 'lo')
+
         # VNI floating IP
         self.logger.out(
             'Creating floating management IP {}/{} on interface {}'.format(
@@ -337,6 +351,7 @@ class NodeInstance(object):
             state='o'
         )
         common.createIPAddress(self.vni_ipaddr, self.vni_cidrnetmask, 'brcluster')
+
         # Upstream floating IP
         self.logger.out(
             'Creating floating upstream IP {}/{} on interface {}'.format(
@@ -349,6 +364,17 @@ class NodeInstance(object):
         common.createIPAddress(self.upstream_ipaddr, self.upstream_cidrnetmask, self.upstream_dev)
 
     def removeFloatingAddresses(self):
+        # Metadata link-local IP
+        self.logger.out(
+            'Removing Metadata link-local IP {}/{} from interface {}'.format(
+                '169.254.169.254',
+                '32',
+                'lo'
+            ),
+            state='o'
+        )
+        common.removeIPAddress('169.254.169.254', '32', 'lo')
+
         # VNI floating IP
         self.logger.out(
             'Removing floating management IP {}/{} from interface {}'.format(
@@ -359,6 +385,7 @@ class NodeInstance(object):
             state='o'
         )
         common.removeIPAddress(self.vni_ipaddr, self.vni_cidrnetmask, 'brcluster')
+
         # Upstream floating IP
         self.logger.out(
             'Removing floating upstream IP {}/{} from interface {}'.format(