valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-08 01:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into saltcloud-vmware-nested-folders

Browse Source
This commit is contained in:
Brian Adriance 2017-12-01 11:15:06 -05:00 committed by GitHub
commit d8c8a776f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 319 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
conf/master
View File

@ -297,6 +297,11 @@
#batch_safe_limit: 100
#batch_safe_size: 8
# Master stats enables stats events to be fired from the master at close
# to the defined interval
#master_stats: False
#master_stats_event_iter: 60
##### Security settings #####
##########################################

23
doc/ref/configuration/master.rst
View File

@ -868,6 +868,29 @@ what you are doing! Transports are explained in :ref:`Salt Transports
ret_port: 4606
zeromq: []
.. conf_master:: master_stats
``master_stats``
----------------
Default: False
Turning on the master stats enables runtime throughput and statistics events
to be fired from the master event bus. These events will report on what
functions have been run on the master and how long these runs have, on
average, taken over a given period of time.
.. conf_master:: master_stats_event_iter
``master_stats_event_iter``
---------------------------
Default: 60
The time in seconds to fire master_stats events. This will only fire in
conjunction with receiving a request to the master, idle masters will not
fire these events.
.. conf_master:: sock_pool_size
``sock_pool_size``

2
doc/ref/states/writing.rst
View File

@ -111,6 +111,8 @@ This code will call the `managed` function in the :mod:`file
<salt.states.file>` state module and pass the arguments ``name`` and ``source``
to it.
.. _state-return-data:
Return Data
===========

32
doc/topics/orchestrate/orchestrate_runner.rst
View File

@ -5,10 +5,10 @@ Orchestrate Runner
==================
Executing states or highstate on a minion is perfect when you want to ensure that
minion configured and running the way you want. Sometimes however you want to
minion configured and running the way you want. Sometimes however you want to
configure a set of minions all at once.
For example, if you want to set up a load balancer in front of a cluster of web
For example, if you want to set up a load balancer in front of a cluster of web
servers you can ensure the load balancer is set up first, and then the same
matching configuration is applied consistently across the whole cluster.
@ -222,6 +222,34 @@ To execute with pillar data.
"master": "mymaster"}'
Return Codes in Runner/Wheel Jobs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: Oxygen
State (``salt.state``) jobs are able to report failure via the :ref:`state
return dictionary <state-return-data>`. Remote execution (``salt.function``)
jobs are able to report failure by setting a ``retcode`` key in the
``__context__`` dictionary. However, runner (``salt.runner``) and wheel
(``salt.wheel``) jobs would only report a ``False`` result when the
runner/wheel function raised an exception. As of the Oxygen release, it is now
possible to set a retcode in runner and wheel functions just as you can do in
remote execution functions. Here is some example pseudocode:
.. code-block:: python
def myrunner():
...
do stuff
...
if some_error_condition:
__context__['retcode'] = 1
return result
This allows a custom runner/wheel function to report its failure so that
requisites can accurately tell that a job has failed.
More Complex Orchestration
~~~~~~~~~~~~~~~~~~~~~~~~~~

19
doc/topics/releases/oxygen.rst
View File

@ -25,6 +25,25 @@ by any master tops matches that are not matched via a top file.
To make master tops matches execute first, followed by top file matches, set
the new :conf_minion:`master_tops_first` minion config option to ``True``.
Return Codes for Runner/Wheel Functions
---------------------------------------
When using :ref:`orchestration <orchestrate-runner>`, runner and wheel
functions used to report a ``True`` result if the function ran to completion
without raising an exception. It is now possible to set a return code in the
``__context__`` dictionary, allowing runner and wheel functions to report that
they failed. Here's some example pseudocode:
.. code-block:: python
def myrunner():
...
do stuff
...
if some_error_condition:
__context__['retcode'] = 1
return result
LDAP via External Authentication Changes
----------------------------------------
In this release of Salt, if LDAP Bind Credentials are supplied, then

4
salt/cli/daemons.py
View File

@ -161,7 +161,6 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di
v_dirs,
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
sensitive_dirs=[self.config['pki_dir'], self.config['key_dir']],
)
@ -283,7 +282,6 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di
v_dirs,
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
sensitive_dirs=[self.config['pki_dir']],
)
@ -472,7 +470,6 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): #
v_dirs,
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
sensitive_dirs=[self.config['pki_dir']],
)
@ -582,7 +579,6 @@ class Syndic(salt.utils.parsers.SyndicOptionParser, DaemonsMixin): # pylint: di
],
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
sensitive_dirs=[self.config['pki_dir']],
)

6
salt/client/mixins.py
View File

@ -385,7 +385,11 @@ class SyncClientMixin(object):
# Initialize a context for executing the method.
with tornado.stack_context.StackContext(self.functions.context_dict.clone):
data[u'return'] = self.functions[fun](*args, **kwargs)
data[u'success'] = True
try:
data[u'success'] = self.context.get(u'retcode', 0) == 0
except AttributeError:
# Assume a True result if no context attribute
data[u'success'] = True
if isinstance(data[u'return'], dict) and u'data' in data[u'return']:
# some functions can return boolean values
data[u'success'] = salt.utils.state.check_result(data[u'return'][u'data'])

6
salt/config/__init__.py
View File

@ -165,6 +165,10 @@ VALID_OPTS = {
# The master_pubkey_signature must also be set for this.
'master_use_pubkey_signature': bool,
# Enable master stats eveents to be fired, these events will contain information about
# what commands the master is processing and what the rates are of the executions
'master_stats': bool,
'master_stats_event_iter': int,
# The key fingerprint of the higher-level master for the syndic to verify it is talking to the
# intended master
'syndic_finger': str,
@ -1515,6 +1519,8 @@ DEFAULT_MASTER_OPTS = {
'svnfs_saltenv_whitelist': [],
'svnfs_saltenv_blacklist': [],
'max_event_size': 1048576,
'master_stats': False,
'master_stats_event_iter': 60,
'minionfs_env': 'base',
'minionfs_mountpoint': '',
'minionfs_whitelist': [],

11
salt/loader.py
View File

@ -372,15 +372,18 @@ def tops(opts):
return FilterDictWrapper(ret, u'.top')
def wheels(opts, whitelist=None):
def wheels(opts, whitelist=None, context=None):
'''
Returns the wheels modules
'''
if context is None:
context = {}
return LazyLoader(
_module_dirs(opts, u'wheel'),
opts,
tag=u'wheel',
whitelist=whitelist,
pack={u'__context__': context},
)
@ -836,17 +839,19 @@ def call(fun, **kwargs):
return funcs[fun](*args)
def runner(opts, utils=None):
def runner(opts, utils=None, context=None):
'''
Directly call a function inside a loader directory
'''
if utils is None:
utils = {}
if context is None:
context = {}
ret = LazyLoader(
_module_dirs(opts, u'runners', u'runner', ext_type_dirs=u'runner_dirs'),
opts,
tag=u'runners',
pack={u'__utils__': utils},
pack={u'__utils__': utils, u'__context__': context},
)
# TODO: change from __salt__ to something else, we overload __salt__ too much
ret.pack[u'__salt__'] = ret

39
salt/master.py
View File

@ -16,6 +16,7 @@ import errno
import signal
import stat
import logging
import collections
import multiprocessing
import salt.serializers.msgpack
@ -797,6 +798,7 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess):
:return: Master worker
'''
kwargs[u'name'] = name
self.name = name
super(MWorker, self).__init__(**kwargs)
self.opts = opts
self.req_channels = req_channels
@ -804,6 +806,8 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess):
self.mkey = mkey
self.key = key
self.k_mtime = 0
self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0})
self.stat_clock = time.time()
# We need __setstate__ and __getstate__ to also pickle 'SMaster.secrets'.
# Otherwise, 'SMaster.secrets' won't be copied over to the spawned process
@ -879,6 +883,19 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess):
u'clear': self._handle_clear}[key](load)
raise tornado.gen.Return(ret)
def _post_stats(self, start, cmd):
'''
Calculate the master stats and fire events with stat info
'''
end = time.time()
duration = end - start
self.stats[cmd][u'mean'] = (self.stats[cmd][u'mean'] * (self.stats[cmd][u'runs'] - 1) + duration) / self.stats[cmd][u'runs']
if end - self.stat_clock > self.opts[u'master_stats_event_iter']:
# Fire the event with the stats and wipe the tracker
self.aes_funcs.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(self.name, u'stats'))
self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0})
self.stat_clock = end
def _handle_clear(self, load):
'''
Process a cleartext command
@ -888,9 +905,16 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess):
the command specified in the load's 'cmd' key.
'''
log.trace(u'Clear payload received with command %s', load[u'cmd'])
if load[u'cmd'].startswith(u'__'):
cmd = load[u'cmd']
if cmd.startswith(u'__'):
return False
return getattr(self.clear_funcs, load[u'cmd'])(load), {u'fun': u'send_clear'}
if self.opts[u'master_stats']:
start = time.time()
self.stats[cmd][u'runs'] += 1
ret = getattr(self.clear_funcs, cmd)(load), {u'fun': u'send_clear'}
if self.opts[u'master_stats']:
self._post_stats(start, cmd)
return ret
def _handle_aes(self, data):
'''
@ -903,10 +927,17 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess):
if u'cmd' not in data:
log.error(u'Received malformed command %s', data)
return {}
cmd = data[u'cmd']
log.trace(u'AES payload received with command %s', data[u'cmd'])
if data[u'cmd'].startswith(u'__'):
if cmd.startswith(u'__'):
return False
return self.aes_funcs.run_func(data[u'cmd'], data)
if self.opts[u'master_stats']:
start = time.time()
self.stats[cmd][u'runs'] += 1
ret = self.aes_funcs.run_func(data[u'cmd'], data)
if self.opts[u'master_stats']:
self._post_stats(start, cmd)
return ret
def run(self):
'''

39
salt/modules/saltutil.py
View File

@ -585,6 +585,44 @@ def sync_engines(saltenv=None, refresh=False, extmod_whitelist=None, extmod_blac
return ret
def sync_thorium(saltenv=None, refresh=False, extmod_whitelist=None, extmod_blacklist=None):
'''
.. versionadded:: Oxygen
Sync Thorium modules from ``salt://_thorium`` to the minion
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for engines to sync. If no top files are
found, then the ``base`` environment will be synced.
refresh: ``True``
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new Thorium modules are synced.
Set to ``False`` to prevent this refresh.
extmod_whitelist
comma-seperated list of modules to sync
extmod_blacklist
comma-seperated list of modules to blacklist based on type
CLI Examples:
.. code-block:: bash
salt '*' saltutil.sync_thorium
salt '*' saltutil.sync_thorium saltenv=base,dev
'''
ret = _sync('thorium', saltenv, extmod_whitelist, extmod_blacklist)
if refresh:
refresh_modules()
return ret
def sync_output(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None):
'''
Sync outputters from ``salt://_output`` to the minion
@ -864,6 +902,7 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist
ret['log_handlers'] = sync_log_handlers(saltenv, False, extmod_whitelist, extmod_blacklist)
ret['proxymodules'] = sync_proxymodules(saltenv, False, extmod_whitelist, extmod_blacklist)
ret['engines'] = sync_engines(saltenv, False, extmod_whitelist, extmod_blacklist)
ret['thorium'] = sync_thorium(saltenv, False, extmod_whitelist, extmod_blacklist)
if __opts__['file_client'] == 'local':
ret['pillar'] = sync_pillar(saltenv, False, extmod_whitelist, extmod_blacklist)
if refresh:

7
salt/runner.py
View File

@ -43,6 +43,7 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object):
def __init__(self, opts):
self.opts = opts
self.context = {}
@property
def functions(self):
@ -51,11 +52,13 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object):
self.utils = salt.loader.utils(self.opts)
# Must be self.functions for mixin to work correctly :-/
try:
self._functions = salt.loader.runner(self.opts, utils=self.utils)
self._functions = salt.loader.runner(
self.opts, utils=self.utils, context=self.context)
except AttributeError:
# Just in case self.utils is still not present (perhaps due to
# problems with the loader), load the runner funcs without them
self._functions = salt.loader.runner(self.opts)
self._functions = salt.loader.runner(
self.opts, context=self.context)
return self._functions

27
salt/runners/saltutil.py
View File

@ -52,6 +52,7 @@ def sync_all(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
ret['runners'] = sync_runners(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
ret['wheel'] = sync_wheel(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
ret['engines'] = sync_engines(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
ret['thorium'] = sync_thorium(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
ret['queues'] = sync_queues(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
ret['pillar'] = sync_pillar(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
ret['utils'] = sync_utils(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)
@ -303,6 +304,32 @@ def sync_engines(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
extmod_blacklist=extmod_blacklist)[0]
def sync_thorium(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
'''
.. versionadded:: Oxygen
Sync Thorium from ``salt://_thorium`` to the master
saltenv: ``base``
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
extmod_whitelist
comma-seperated list of modules to sync
extmod_blacklist
comma-seperated list of modules to blacklist based on type
CLI Example:
.. code-block:: bash
salt-run saltutil.sync_thorium
'''
return salt.utils.extmods.sync(__opts__, 'thorium', saltenv=saltenv, extmod_whitelist=extmod_whitelist,
extmod_blacklist=extmod_blacklist)[0]
def sync_queues(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
'''
Sync queue modules from ``salt://_queues`` to the master

51
salt/states/saltmod.py
View File

@ -787,28 +787,15 @@ def runner(name, **kwargs):
runner_return = out.get('return')
if isinstance(runner_return, dict) and 'Error' in runner_return:
out['success'] = False
if not out.get('success', True):
cmt = "Runner function '{0}' failed{1}.".format(
name,
' with return {0}'.format(runner_return) if runner_return else '',
)
ret = {
'name': name,
'result': False,
'changes': {},
'comment': cmt,
}
else:
cmt = "Runner function '{0}' executed{1}.".format(
name,
' with return {0}'.format(runner_return) if runner_return else '',
)
ret = {
'name': name,
'result': True,
'changes': {},
'comment': cmt,
}
success = out.get('success', True)
ret = {'name': name,
'changes': {'return': runner_return},
'result': success}
ret['comment'] = "Runner function '{0}' {1}.".format(
name,
'executed' if success else 'failed',
)
ret['__orchestration__'] = True
if 'jid' in out:
@ -1039,15 +1026,21 @@ def wheel(name, **kwargs):
__env__=__env__,
**kwargs)
ret['result'] = True
wheel_return = out.get('return')
if isinstance(wheel_return, dict) and 'Error' in wheel_return:
out['success'] = False
success = out.get('success', True)
ret = {'name': name,
'changes': {'return': wheel_return},
'result': success}
ret['comment'] = "Wheel function '{0}' {1}.".format(
name,
'executed' if success else 'failed',
)
ret['__orchestration__'] = True
if 'jid' in out:
ret['__jid__'] = out['jid']
runner_return = out.get('return')
ret['comment'] = "Wheel function '{0}' executed{1}.".format(
name,
' with return {0}'.format(runner_return) if runner_return else '',
)
return ret

6
salt/utils/args.py
View File

@ -451,10 +451,10 @@ def format_call(fun,
continue
extra[key] = copy.deepcopy(value)
# We'll be showing errors to the users until Salt Oxygen comes out, after
# We'll be showing errors to the users until Salt Fluorine comes out, after
# which, errors will be raised instead.
salt.utils.versions.warn_until(
'Oxygen',
'Fluorine',
'It\'s time to start raising `SaltInvocationError` instead of '
'returning warnings',
# Let's not show the deprecation warning on the console, there's no
@ -491,7 +491,7 @@ def format_call(fun,
'{0}. If you were trying to pass additional data to be used '
'in a template context, please populate \'context\' with '
'\'key: value\' pairs. Your approach will work until Salt '
'Oxygen is out.{1}'.format(
'Fluorine is out.{1}'.format(
msg,
'' if 'full' not in ret else ' Please update your state files.'
)

3
salt/wheel/__init__.py
View File

@ -43,7 +43,8 @@ class WheelClient(salt.client.mixins.SyncClientMixin,
def __init__(self, opts=None):
self.opts = opts
self.functions = salt.loader.wheels(opts)
self.context = {}
self.functions = salt.loader.wheels(opts, context=self.context)
# TODO: remove/deprecate
def call_func(self, fun, **kwargs):

16
tests/integration/files/file/base/_runners/runtests_helpers.py Normal file
View File

@ -0,0 +1,16 @@
# -*- coding: utf-8 -*-
'''
Runner functions for integration tests
'''
# Import python libs
from __future__ import absolute_import
def failure():
__context__['retcode'] = 1
return False
def success():
return True

16
tests/integration/files/file/base/_wheel/runtests_helpers.py Normal file
View File

@ -0,0 +1,16 @@
# -*- coding: utf-8 -*-
'''
Wheel functions for integration tests
'''
# Import python libs
from __future__ import absolute_import
def failure():
__context__['retcode'] = 1
return False
def success():
return True

15
tests/integration/files/file/base/orch/retcode.sls Normal file
View File

@ -0,0 +1,15 @@
test_runner_success:
salt.runner:
- name: runtests_helpers.success
test_runner_failure:
salt.runner:
- name: runtests_helpers.failure
test_wheel_success:
salt.wheel:
- name: runtests_helpers.success
test_wheel_failure:
salt.wheel:
- name: runtests_helpers.failure

12
tests/integration/modules/test_saltutil.py
View File

@ -93,7 +93,8 @@ class SaltUtilSyncModuleTest(ModuleCase):
'states': [],
'sdb': [],
'proxymodules': [],
'output': []}
'output': [],
'thorium': []}
ret = self.run_function('saltutil.sync_all')
self.assertEqual(ret, expected_return)
@ -113,7 +114,8 @@ class SaltUtilSyncModuleTest(ModuleCase):
'states': [],
'sdb': [],
'proxymodules': [],
'output': []}
'output': [],
'thorium': []}
ret = self.run_function('saltutil.sync_all', extmod_whitelist={'modules': ['salttest']})
self.assertEqual(ret, expected_return)
@ -135,7 +137,8 @@ class SaltUtilSyncModuleTest(ModuleCase):
'states': [],
'sdb': [],
'proxymodules': [],
'output': []}
'output': [],
'thorium': []}
ret = self.run_function('saltutil.sync_all', extmod_blacklist={'modules': ['runtests_decorators']})
self.assertEqual(ret, expected_return)
@ -155,7 +158,8 @@ class SaltUtilSyncModuleTest(ModuleCase):
'states': [],
'sdb': [],
'proxymodules': [],
'output': []}
'output': [],
'thorium': []}
ret = self.run_function('saltutil.sync_all', extmod_whitelist={'modules': ['runtests_decorators']},
extmod_blacklist={'modules': ['runtests_decorators']})
self.assertEqual(ret, expected_return)

29
tests/integration/runners/test_state.py
View File

@ -106,6 +106,35 @@ class StateRunnerTest(ShellCase):
for item in out:
self.assertIn(item, ret)
def test_orchestrate_retcode(self):
'''
Test orchestration with nonzero retcode set in __context__
'''
self.run_run('saltutil.sync_runners')
self.run_run('saltutil.sync_wheel')
ret = '\n'.join(self.run_run('state.orchestrate orch.retcode'))
for result in (' ID: test_runner_success\n'
' Function: salt.runner\n'
' Name: runtests_helpers.success\n'
' Result: True',
' ID: test_runner_failure\n'
' Function: salt.runner\n'
' Name: runtests_helpers.failure\n'
' Result: False',
' ID: test_wheel_success\n'
' Function: salt.wheel\n'
' Name: runtests_helpers.success\n'
' Result: True',
' ID: test_wheel_failure\n'
' Function: salt.wheel\n'
' Name: runtests_helpers.failure\n'
' Result: False'):
self.assertIn(result, ret)
def test_orchestrate_target_doesnt_exists(self):
'''
test orchestration when target doesnt exist

8
tests/unit/states/test_saltmod.py
View File

@ -258,8 +258,8 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin):
'''
name = 'state'
ret = {'changes': {}, 'name': 'state', 'result': True,
'comment': 'Runner function \'state\' executed with return True.',
ret = {'changes': {'return': True}, 'name': 'state', 'result': True,
'comment': 'Runner function \'state\' executed.',
'__orchestration__': True}
runner_mock = MagicMock(return_value={'return': True})
@ -274,8 +274,8 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin):
'''
name = 'state'
ret = {'changes': {}, 'name': 'state', 'result': True,
'comment': 'Wheel function \'state\' executed with return True.',
ret = {'changes': {'return': True}, 'name': 'state', 'result': True,
'comment': 'Wheel function \'state\' executed.',
'__orchestration__': True}
wheel_mock = MagicMock(return_value={'return': True})