valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-06 16:45:27 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #34920 from cachedout/key_cache

Browse Source 
Key cache
This commit is contained in:
Mike Place 2016-08-02 12:11:18 -06:00 committed by GitHub
commit 222449cd0e
5 changed files with 78 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
conf/master
View File

@ -45,6 +45,13 @@
# Directory used to store public key data:
#pki_dir: /etc/salt/pki/master
# Key cache. Increases master speed for large numbers of accepted
# keys. Available options: 'sched'. (Updates on a fixed schedule.)
# Note that enabling this feature means that minions will not be
# available to target for up to the length of the maintanence loop
# which by default is 60s.
#key_cache: ''
# Directory to store job and cache data:
# This directory may contain sensitive data and should be protected accordingly.
#

14
doc/topics/tutorials/intro_scale.rst
View File

@ -271,3 +271,17 @@ If the job cache is necessary there are (currently) 2 options:
into a returner (not sent through the Master)
- master_job_cache (New in `2014.7.0`): this will make the Master store the job
data using a returner (instead of the local job cache on disk).
If a master has many accepted keys, it may take a long time to publish a job
because the master much first determine the matching minions and deliver
that information back to the waiting client before the job can be published.
To mitigate this, a key cache may be enabled. This will reduce the load
on the master to a single file open instead of thousands or tens of thousands.
This cache is updated by the maintanence process, however, which means that
minions with keys that are accepted may not be targeted by the master
for up to sixty seconds by default.
To enable the master key cache, set `key_cache: 'sched'` in the master
configuration file.

7
salt/config/__init__.py
View File

@ -159,6 +159,12 @@ VALID_OPTS = {
# master
'syndic_finger': str,
# The caching mechanism to use for the PKI key store. Can substantially decrease master publish
# times. Available types:
# 'maint': Runs on a schedule as a part of the maintanence process.
# '': Disable the key cache [default]
'key_cache': str,
# The user under which the daemon should run
'user': str,
@ -1138,6 +1144,7 @@ DEFAULT_MASTER_OPTS = {
'keep_jobs': 24,
'root_dir': salt.syspaths.ROOT_DIR,
'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'master'),
'key_cache': '',
'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'master'),
'file_roots': {
'base': [salt.syspaths.BASE_FILE_ROOTS_DIR,

24
salt/master.py
View File

@ -157,6 +157,8 @@ class Maintenance(SignalHandlingMultiprocessingProcess):
self.loop_interval = int(self.opts['loop_interval'])
# Track key rotation intervals
self.rotate = int(time.time())
# A serializer for general maint operations
self.serial = salt.payload.Serial(self.opts)
# __setstate__ and __getstate__ are only used on Windows.
# We do this so that __init__ will be invoked on Windows in the child
@ -237,6 +239,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess):
self.handle_search(now, last)
self.handle_git_pillar()
self.handle_schedule()
self.handle_key_cache()
self.handle_presence(old_present)
self.handle_key_rotate(now)
salt.daemons.masterapi.fileserver_update(self.fileserver)
@ -252,6 +255,27 @@ class Maintenance(SignalHandlingMultiprocessingProcess):
if now - last >= self.opts['search_index_interval']:
self.search.index()
def handle_key_cache(self):
'''
Evaluate accepted keys and create a msgpack file
which contains a list
'''
if self.opts['key_cache'] == 'sched':
keys = []
#TODO DRY from CKMinions
if self.opts['transport'] in ('zeromq', 'tcp'):
acc = 'minions'
else:
acc = 'accepted'
for fn_ in os.listdir(os.path.join(self.opts['pki_dir'], acc)):
if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], acc, fn_)):
keys.append(fn_)
log.debug('Writing master key cache')
# Write a temporary file securely
with salt.utils.atomicfile.atomic_open(os.path.join(self.opts['pki_dir'], acc, '.key_cache')) as cache_file:
self.serial.dump(keys, cache_file)
def handle_key_rotate(self, now):
'''
Rotate the AES key rotation

58
salt/utils/minions.py
View File

@ -205,15 +205,7 @@ class CkMinions(object):
'''
Return the minions found by looking via globs
'''
pki_dir = os.path.join(self.opts['pki_dir'], self.acc)
try:
files = []
for fn_ in salt.utils.isorted(os.listdir(pki_dir)):
if not fn_.startswith('.') and os.path.isfile(os.path.join(pki_dir, fn_)):
files.append(fn_)
return fnmatch.filter(files, expr)
except OSError:
return []
return fnmatch.filter(self._pki_minions(), expr)
def _check_list_minions(self, expr, greedy): # pylint: disable=unused-argument
'''
@ -221,25 +213,35 @@ class CkMinions(object):
'''
if isinstance(expr, six.string_types):
expr = [m for m in expr.split(',') if m]
ret = []
for minion in expr:
if os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, minion)):
ret.append(minion)
return ret
return [x for x in expr if x in self._pki_minions()]
def _check_pcre_minions(self, expr, greedy): # pylint: disable=unused-argument
'''
Return the minions found by looking via regular expressions
'''
reg = re.compile(expr)
return [m for m in self._pki_minions() if reg.match(m)]
def _pki_minions(self):
'''
Retreive complete minion list from PKI dir.
Respects cache if configured
'''
minions = []
pki_cache_fn = os.path.join(self.opts['pki_dir'], self.acc, '.key_cache')
try:
minions = []
for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))):
if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)):
minions.append(fn_)
reg = re.compile(expr)
return [m for m in minions if reg.match(m)]
except OSError:
return []
if self.opts['key_cache'] and os.path.exists(pki_cache_fn):
log.debug('Returning cached minion list')
with salt.utils.fopen(pki_cache_fn) as fn_:
return self.serial.load(fn_)
else:
for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))):
if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)):
minions.append(fn_)
return minions
except OSError as exc:
log.error('Encountered OSError while evaluating minions in PKI dir: {0}'.format(exc))
return minions
def _check_cache_minions(self,
expr,
@ -335,11 +337,7 @@ class CkMinions(object):
cache_enabled = self.opts.get('minion_data_cache', False)
if greedy:
mlist = []
for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))):
if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)):
mlist.append(fn_)
minions = set(mlist)
minions = set(self._pki_minions())
elif cache_enabled:
minions = os.listdir(os.path.join(self.opts['cachedir'], 'minions'))
else:
@ -441,11 +439,7 @@ class CkMinions(object):
if not isinstance(expr, six.string_types) and not isinstance(expr, (list, tuple)):
log.error('Compound target that is neither string, list nor tuple')
return []
mlist = []
for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))):
if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)):
mlist.append(fn_)
minions = set(mlist)
minions = set(self._pki_minions())
log.debug('minions: {0}'.format(minions))
if self.opts.get('minion_data_cache', False):