From 5c44b8a75484b2d06e264f43a063ab2acc781c66 Mon Sep 17 00:00:00 2001
From: Joshua Boniface <joshua@boniface.me>
Date: Mon, 1 Oct 2018 22:43:14 -0400
Subject: [PATCH] Fix bug with DataWatches holding up instance del

Fixes a very esoteric bug I was having with duplicate class instances
not being reaped properly due to held-open DataWatch elements.
Workaround for the bug(?) described in
https://github.com/python-zk/kazoo/issues/525
---
 router-daemon/pvcrd/VXNetworkInstance.py | 49 ++++++++++++++++++++----
 1 file changed, 42 insertions(+), 7 deletions(-)

diff --git a/router-daemon/pvcrd/VXNetworkInstance.py b/router-daemon/pvcrd/VXNetworkInstance.py
index 37e4a1d7..a229f5af 100644
--- a/router-daemon/pvcrd/VXNetworkInstance.py
+++ b/router-daemon/pvcrd/VXNetworkInstance.py
@@ -57,27 +57,47 @@ class VXNetworkInstance():
         self.createNetwork()
 
         # Zookeper handlers for changed states
-        @zk_conn.DataWatch('/networks/{}'.format(self.vni))
+        @self.zk_conn.DataWatch('/networks/{}'.format(self.vni))
         def watch_network_description(data, stat, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
             if data and self.description != data.decode('ascii'):
                 self.old_description = self.description
                 self.description = data.decode('ascii')
 
-        @zk_conn.DataWatch('/networks/{}/domain'.format(self.vni))
+        @self.zk_conn.DataWatch('/networks/{}/domain'.format(self.vni))
         def watch_network_domain(data, stat, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
             if data and self.domain != data.decode('ascii'):
                 domain = data.decode('ascii')
                 self.domain = domain
 
-        @zk_conn.DataWatch('/networks/{}/ip_network'.format(self.vni))
+        @self.zk_conn.DataWatch('/networks/{}/ip_network'.format(self.vni))
         def watch_network_ip_network(data, stat, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
             if data and self.ip_network != data.decode('ascii'):
                 ip_network = data.decode('ascii')
                 self.ip_network = ip_network
                 self.ip_cidrnetmask = ip_network.split('/')[-1]
 
-        @zk_conn.DataWatch('/networks/{}/ip_gateway'.format(self.vni))
+        @self.zk_conn.DataWatch('/networks/{}/ip_gateway'.format(self.vni))
         def watch_network_gateway(data, stat, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
             if data and self.ip_gateway != data.decode('ascii'):
                 orig_gateway = self.ip_gateway
                 self.ip_gateway = data.decode('ascii')
@@ -86,8 +106,13 @@ class VXNetworkInstance():
                         self.removeGatewayAddress()
                     self.createGatewayAddress()
 
-        @zk_conn.DataWatch('/networks/{}/dhcp_flag'.format(self.vni))
+        @self.zk_conn.DataWatch('/networks/{}/dhcp_flag'.format(self.vni))
         def watch_network_dhcp_status(data, stat, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
             if data and self.dhcp_flag != data.decode('ascii'):
                 self.dhcp_flag = ( data.decode('ascii') == 'True' )
                 if self.dhcp_flag and self.this_router.network_state == 'primary':
@@ -95,13 +120,23 @@ class VXNetworkInstance():
                 elif self.this_router.network_state == 'primary':
                     self.stopDHCPServer()
 
-        @zk_conn.DataWatch('/networks/{}/dhcp_start'.format(self.vni))
+        @self.zk_conn.DataWatch('/networks/{}/dhcp_start'.format(self.vni))
         def watch_network_dhcp_start(data, stat, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
             if data and self.dhcp_start != data.decode('ascii'):
                 self.dhcp_start = data.decode('ascii')
 
-        @zk_conn.DataWatch('/networks/{}/dhcp_end'.format(self.vni))
+        @self.zk_conn.DataWatch('/networks/{}/dhcp_end'.format(self.vni))
         def watch_network_dhcp_end(data, stat, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
             if data and self.dhcp_end != data.decode('ascii'):
                 self.dhcp_end = data.decode('ascii')