Merge pull request #39505 from cachedout/issue_38758

Threadsafety option for context dictionaries

conf/minion

@@ -647,6 +647,10 @@
 ###########################################
 # Disable multiprocessing support, by default when a minion receives a
 # publication a new process is spawned and the command is executed therein.
+#
+# WARNING: Disabling multiprocessing may result in substantial slowdowns
+# when processing large pillars. See https://github.com/saltstack/salt/issues/38758
+# for a full explanation.
 #multiprocessing: True
 
 

salt/loader.py

@@ -1037,7 +1037,8 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         self.pack = {} if pack is None else pack
         if opts is None:
             opts = {}
-        self.context_dict = salt.utils.context.ContextDict()
+        threadsafety = not opts.get('multiprocessing')
+        self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafety)
         self.opts = self.__prep_mod_opts(opts)
 
         self.module_dirs = module_dirs

salt/utils/context.py

@@ -66,12 +66,15 @@ class ContextDict(collections.MutableMapping):
     then allow any children to override the values of the parent.
     '''
 
-    def __init__(self, **data):
+    def __init__(self, threadsafe=False, **data):
         # state should be thread local, so this object can be threadsafe
         self._state = threading.local()
         # variable for the overridden data
         self._state.data = None
         self.global_data = {}
+        # Threadsafety indicates whether or not we should protect data stored
+        # in child context dicts from being leaked
+        self._threadsafe = threadsafe
 
     @property
     def active(self):
@@ -89,7 +92,7 @@ class ContextDict(collections.MutableMapping):
         '''
         Clone this context, and return the ChildContextDict
         '''
-        child = ChildContextDict(parent=self, overrides=kwargs)
+        child = ChildContextDict(parent=self, threadsafe=self._threadsafe, overrides=kwargs)
         return child
 
     def __setitem__(self, key, val):
@@ -127,19 +130,24 @@ class ChildContextDict(collections.MutableMapping):
     '''An overrideable child of ContextDict
 
     '''
-    def __init__(self, parent, overrides=None):
+    def __init__(self, parent, overrides=None, threadsafe=False):
         self.parent = parent
         self._data = {} if overrides is None else overrides
         self._old_data = None
 
         # merge self.global_data into self._data
-        for k, v in six.iteritems(self.parent.global_data):
-            if k not in self._data:
-                # A deepcopy is necessary to avoid using the same
-                # objects in globals as we do in thread local storage.
-                # Otherwise, changing one would automatically affect
-                # the other.
-                self._data[k] = copy.deepcopy(v)
+        if threadsafe:
+            for k, v in six.iteritems(self.parent.global_data):
+                if k not in self._data:
+                    # A deepcopy is necessary to avoid using the same
+                    # objects in globals as we do in thread local storage.
+                    # Otherwise, changing one would automatically affect
+                    # the other.
+                    self._data[k] = copy.deepcopy(v)
+        else:
+            for k, v in six.iteritems(self.parent.global_data):
+                if k not in self._data:
+                    self._data[k] = v
 
     def __setitem__(self, key, val):
         self._data[key] = val