From bbf90f0ffe83f58f7fb197b2b4357a744f20917c Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 30 Jun 2015 13:03:23 -0600 Subject: [PATCH] Handle GC in weakref iter There is an issue that's popping up during automated testing: https://jenkins.saltstack.com/job/salt-linode-cent7/lastCompletedBuild/testReport/integration.modules.grains/TestModulesGrains/test_set_val/ This could be caused by the iteration of the dict keys triggering a GC, thus cleaning up the value we are about to return. This attempts to handle that condition by simply falling back to re-creating a channel if this is the case. cc: @jacksontj --- salt/transport/zeromq.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index fa8c336d5a..97b8aef9b1 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -10,6 +10,7 @@ import os import errno import hashlib import weakref +import gc from random import randint # Import Salt Libs @@ -61,6 +62,7 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): ''' Only create one instance of channel per __key() ''' + # do we have any mapping for this io_loop io_loop = kwargs.get('io_loop') or tornado.ioloop.IOLoop.current() if io_loop not in cls.instance_map: @@ -78,7 +80,18 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): loop_instance_map[key] = new_obj else: log.debug('Re-using AsyncZeroMQReqChannel for {0}'.format(key)) - return loop_instance_map[key] + try: + return loop_instance_map[key] + except KeyError: + # In iterating over the loop_instance_map, we may have triggered + # garbage collection. Therefore, the key is no longer present in + # the map. Re-gen and add to map. + log.debug('Initializing new AsyncZeroMQReqChannel due to GC for {0}'.format(key)) + new_obj = object.__new__(cls) + new_obj.__singleton_init__(opts, **kwargs) + loop_instance_map[key] = new_obj + return loop_instance_map[key] + @classmethod def __key(cls, opts, **kwargs): @@ -118,7 +131,10 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): Since the message_client creates sockets and assigns them to the IOLoop we have to specifically destroy them, since we aren't the only ones with references to the FDs ''' - self.message_client.destroy() + if hasattr(self, 'message_client'): + self.message_client.destroy() + else: + log.debug('No message_client attr for AsyncZeroMQReqChannel found. Not destroying sockets.') @property def master_uri(self):