Merge pull request #34920 from cachedout/key_cache

Key cache

conf/master

@@ -45,6 +45,13 @@
 # Directory used to store public key data:
 #pki_dir: /etc/salt/pki/master
 
+# Key cache. Increases master speed for large numbers of accepted
+# keys. Available options: 'sched'. (Updates on a fixed schedule.)
+# Note that enabling this feature means that minions will not be
+# available to target for up to the length of the maintanence loop
+# which by default is 60s.
+#key_cache: ''
+
 # Directory to store job and cache data:
 # This directory may contain sensitive data and should be protected accordingly.
 #

doc/topics/tutorials/intro_scale.rst

@@ -271,3 +271,17 @@ If the job cache is necessary there are (currently) 2 options:
   into a returner (not sent through the Master)
 - master_job_cache (New in `2014.7.0`): this will make the Master store the job
   data using a returner (instead of the local job cache on disk).
+
+If a master has many accepted keys, it may take a long time to publish a job
+because the master much first determine the matching minions and deliver
+that information back to the waiting client before the job can be published.
+
+To mitigate this, a key cache may be enabled. This will reduce the load
+on the master to a single file open instead of thousands or tens of thousands.
+
+This cache is updated by the maintanence process, however, which means that
+minions with keys that are accepted may not be targeted by the master
+for up to sixty seconds by default.
+
+To enable the master key cache, set `key_cache: 'sched'` in the master
+configuration file.

salt/config/__init__.py

@@ -159,6 +159,12 @@ VALID_OPTS = {
     # master
     'syndic_finger': str,
 
+    # The caching mechanism to use for the PKI key store. Can substantially decrease master publish
+    # times. Available types:
+    # 'maint': Runs on a schedule as a part of the maintanence process.
+    # '': Disable the key cache [default]
+    'key_cache': str,
+
     # The user under which the daemon should run
     'user': str,
 
@@ -1138,6 +1144,7 @@ DEFAULT_MASTER_OPTS = {
     'keep_jobs': 24,
     'root_dir': salt.syspaths.ROOT_DIR,
     'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'master'),
+    'key_cache': '',
     'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'master'),
     'file_roots': {
         'base': [salt.syspaths.BASE_FILE_ROOTS_DIR,

salt/master.py

@@ -157,6 +157,8 @@ class Maintenance(SignalHandlingMultiprocessingProcess):
         self.loop_interval = int(self.opts['loop_interval'])
         # Track key rotation intervals
         self.rotate = int(time.time())
+        # A serializer for general maint operations
+        self.serial = salt.payload.Serial(self.opts)
 
     # __setstate__ and __getstate__ are only used on Windows.
     # We do this so that __init__ will be invoked on Windows in the child
@@ -237,6 +239,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess):
             self.handle_search(now, last)
             self.handle_git_pillar()
             self.handle_schedule()
+            self.handle_key_cache()
             self.handle_presence(old_present)
             self.handle_key_rotate(now)
             salt.daemons.masterapi.fileserver_update(self.fileserver)
@@ -252,6 +255,27 @@ class Maintenance(SignalHandlingMultiprocessingProcess):
             if now - last >= self.opts['search_index_interval']:
                 self.search.index()
 
+    def handle_key_cache(self):
+        '''
+        Evaluate accepted keys and create a msgpack file
+        which contains a list
+        '''
+        if self.opts['key_cache'] == 'sched':
+            keys = []
+            #TODO DRY from CKMinions
+            if self.opts['transport'] in ('zeromq', 'tcp'):
+                acc = 'minions'
+            else:
+                acc = 'accepted'
+
+            for fn_ in os.listdir(os.path.join(self.opts['pki_dir'], acc)):
+                if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], acc, fn_)):
+                    keys.append(fn_)
+            log.debug('Writing master key cache')
+            # Write a temporary file securely
+            with salt.utils.atomicfile.atomic_open(os.path.join(self.opts['pki_dir'], acc, '.key_cache')) as cache_file:
+                self.serial.dump(keys, cache_file)
+
     def handle_key_rotate(self, now):
         '''
         Rotate the AES key rotation

salt/utils/minions.py

@@ -205,15 +205,7 @@ class CkMinions(object):
         '''
         Return the minions found by looking via globs
         '''
-        pki_dir = os.path.join(self.opts['pki_dir'], self.acc)
+        return fnmatch.filter(self._pki_minions(), expr)
-        try:
-            files = []
-            for fn_ in salt.utils.isorted(os.listdir(pki_dir)):
-                if not fn_.startswith('.') and os.path.isfile(os.path.join(pki_dir, fn_)):
-                    files.append(fn_)
-            return fnmatch.filter(files, expr)
-        except OSError:
-            return []
 
     def _check_list_minions(self, expr, greedy):  # pylint: disable=unused-argument
         '''
@@ -221,25 +213,35 @@ class CkMinions(object):
         '''
         if isinstance(expr, six.string_types):
             expr = [m for m in expr.split(',') if m]
-        ret = []
+        return [x for x in expr if x in self._pki_minions()]
-        for minion in expr:
-            if os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, minion)):
-                ret.append(minion)
-        return ret
 
     def _check_pcre_minions(self, expr, greedy):  # pylint: disable=unused-argument
         '''
         Return the minions found by looking via regular expressions
         '''
-        try:
+        reg = re.compile(expr)
+        return [m for m in self._pki_minions() if reg.match(m)]
+
+    def _pki_minions(self):
+        '''
+        Retreive complete minion list from PKI dir.
+        Respects cache if configured
+        '''
         minions = []
+        pki_cache_fn = os.path.join(self.opts['pki_dir'], self.acc, '.key_cache')
+        try:
+            if self.opts['key_cache'] and os.path.exists(pki_cache_fn):
+                log.debug('Returning cached minion list')
+                with salt.utils.fopen(pki_cache_fn) as fn_:
+                    return self.serial.load(fn_)
+            else:
                 for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))):
                     if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)):
                         minions.append(fn_)
-            reg = re.compile(expr)
+            return minions
-            return [m for m in minions if reg.match(m)]
+        except OSError as exc:
-        except OSError:
+            log.error('Encountered OSError while evaluating  minions in PKI dir: {0}'.format(exc))
-            return []
+            return minions
 
     def _check_cache_minions(self,
                              expr,
@@ -335,11 +337,7 @@ class CkMinions(object):
         cache_enabled = self.opts.get('minion_data_cache', False)
 
         if greedy:
-            mlist = []
+            minions = set(self._pki_minions())
-            for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))):
-                if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)):
-                    mlist.append(fn_)
-            minions = set(mlist)
         elif cache_enabled:
             minions = os.listdir(os.path.join(self.opts['cachedir'], 'minions'))
         else:
@@ -441,11 +439,7 @@ class CkMinions(object):
         if not isinstance(expr, six.string_types) and not isinstance(expr, (list, tuple)):
             log.error('Compound target that is neither string, list nor tuple')
             return []
-        mlist = []
+        minions = set(self._pki_minions())
-        for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))):
-            if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)):
-                mlist.append(fn_)
-        minions = set(mlist)
         log.debug('minions: {0}'.format(minions))
 
         if self.opts.get('minion_data_cache', False):