From 545026352fd86569187aac8ba041d304fb802123 Mon Sep 17 00:00:00 2001 From: Drew Malone Date: Thu, 23 Feb 2017 18:10:11 -0500 Subject: [PATCH 01/38] Address issue 39622 --- salt/modules/boto_vpc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/modules/boto_vpc.py b/salt/modules/boto_vpc.py index f22efa26ef..8c66fe428a 100644 --- a/salt/modules/boto_vpc.py +++ b/salt/modules/boto_vpc.py @@ -1313,6 +1313,12 @@ def _maybe_set_name_tag(name, obj): def _maybe_set_tags(tags, obj): if tags: - obj.add_tags(tags) + # Not all objects in Boto have an 'add_tags()' method. + try: + obj.add_tags(tags) + + except AttributeError: + for tag, value in tags.items(): + obj.add_tag(tag, value) log.debug('The following tags: {0} were added to {1}'.format(', '.join(tags), obj)) From 13da50be33c74a60dc5c79da0829ac4fdd1b2390 Mon Sep 17 00:00:00 2001 From: Drew Malone Date: Thu, 23 Feb 2017 18:40:52 -0500 Subject: [PATCH 02/38] Fix indention lint errors --- salt/modules/boto_vpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/boto_vpc.py b/salt/modules/boto_vpc.py index 8c66fe428a..9adf56dc44 100644 --- a/salt/modules/boto_vpc.py +++ b/salt/modules/boto_vpc.py @@ -1315,10 +1315,10 @@ def _maybe_set_tags(tags, obj): if tags: # Not all objects in Boto have an 'add_tags()' method. try: - obj.add_tags(tags) + obj.add_tags(tags) except AttributeError: - for tag, value in tags.items(): - obj.add_tag(tag, value) + for tag, value in tags.items(): + obj.add_tag(tag, value) log.debug('The following tags: {0} were added to {1}'.format(', '.join(tags), obj)) From 692c456da3184211bfa607ddaf187d762fa6b79c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 23 Feb 2017 15:38:33 -0600 Subject: [PATCH 03/38] Add a function to simply refresh the grains Also use this function to refresh the grains when editing them using grains.{delval,setval,setvals}, avoiding a full sync. This speeds up these functions by skipping the syncing of custom grains modules. For most use cases, walking all your fileserver environments (which could be a lot if using gitfs) is unnecessary when you're simply using these functions to add/remove grains from /etc/salt/grains. --- salt/minion.py | 14 +++++++++++++- salt/modules/grains.py | 11 +++++++---- salt/modules/saltutil.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 9fd080956e..8090d47197 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1685,6 +1685,15 @@ class Minion(MinionBase): tagify([self.opts['id'], 'start'], 'minion'), ) + def grains_refresh_manual(self): + ''' + Perform a manual grains refresh + ''' + self.opts['grains'] = salt.loader.grains( + self.opts, + force_refresh=True, + proxy=salt.utils.is_proxy()) + def module_refresh(self, force_refresh=False, notify=False): ''' Refresh the functions and returners. @@ -1854,9 +1863,12 @@ class Minion(MinionBase): elif package.startswith('manage_beacons'): self.manage_beacons(tag, data) elif package.startswith('grains_refresh'): + if package == 'grains_refresh_manual': + self.grains_refresh_manual() if self.grains_cache != self.opts['grains']: - self.pillar_refresh(force_refresh=True) self.grains_cache = self.opts['grains'] + if package != 'grains_refresh_manual': + self.pillar_refresh(force_refresh=True) elif package.startswith('environ_setenv'): self.environ_setenv(tag, data) elif package.startswith('_minion_mine'): diff --git a/salt/modules/grains.py b/salt/modules/grains.py index be0b65ba5a..0f8c5f27b6 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -278,8 +278,8 @@ def setvals(grains, destructive=False, refresh=True): msg = 'Unable to write to cache file {0}. Check permissions.' log.error(msg.format(fn_)) if not __opts__.get('local', False): - # Sync the grains - __salt__['saltutil.sync_grains'](refresh=refresh) + # Refresh the grains + __salt__['saltutil.refresh_grains'](refresh=refresh) # Return the grains we just set to confirm everything was OK return new_grains @@ -412,7 +412,7 @@ def remove(key, val, delimiter=DEFAULT_TARGET_DELIM): return setval(key, grains) -def delval(key, destructive=False): +def delval(key, destructive=False, refresh=True): ''' .. versionadded:: 0.17.0 @@ -424,6 +424,9 @@ def delval(key, destructive=False): destructive Delete the key, too. Defaults to False. + refresh + Refresh modules and pillar after removing the grain. + CLI Example: .. code-block:: bash @@ -431,7 +434,7 @@ def delval(key, destructive=False): salt '*' grains.delval key ''' - setval(key, None, destructive=destructive) + setval(key, None, destructive=destructive, refresh=refresh) def ls(): # pylint: disable=C0103 diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index bc7bae6901..39d6c28df3 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -310,6 +310,34 @@ def sync_states(saltenv=None, refresh=True): return ret +def refresh_grains(refresh=True): + ''' + .. versionadded:: 2016.3.6,2016.11.4,Nitrogen + + Refresh the minion's grains without syncing custom grains modules from + ``salt://_grains``. + + refresh : True + If ``True``, refresh the available execution modules and recompile + pillar data for the minion. Set to ``False`` to prevent this refresh. + + CLI Examples: + + .. code-block:: bash + + salt '*' saltutil.refresh_grains + ''' + try: + ret = __salt__['event.fire']({}, 'grains_refresh_manual') + except Exception: + log.error('Failed to refresh grains', exc_info=True) + return False + if refresh: + refresh_modules() + refresh_pillar() + return ret + + def sync_grains(saltenv=None, refresh=True): ''' .. versionadded:: 0.10.0 From 7e0ced3b45ca405ee1934abf5f4d3da42cf5f97c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 23 Feb 2017 16:01:19 -0600 Subject: [PATCH 04/38] Properly hand proxy minions The proxy argument to salt.loader.grains expects a reference to a salt.loader.proxy loader instance, not a bool. Thanks @cro! --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 8090d47197..1524d9da06 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1692,7 +1692,7 @@ class Minion(MinionBase): self.opts['grains'] = salt.loader.grains( self.opts, force_refresh=True, - proxy=salt.utils.is_proxy()) + proxy=getattr(self, 'proxy', None)) def module_refresh(self, force_refresh=False, notify=False): ''' From c7dfb494a67c45f8b053b52b1eb65371c1bd4fae Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sat, 25 Feb 2017 21:04:25 -0600 Subject: [PATCH 05/38] Fix mocking for grains refresh --- tests/unit/modules/grains_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/grains_test.py b/tests/unit/modules/grains_test.py index 35945630d9..4906052b7d 100644 --- a/tests/unit/modules/grains_test.py +++ b/tests/unit/modules/grains_test.py @@ -29,7 +29,7 @@ grainsmod.__opts__ = { grainsmod.__salt__ = {} -@patch.dict(grainsmod.__salt__, {'saltutil.sync_grains': MagicMock()}) +@patch.dict(grainsmod.__salt__, {'saltutil.refresh_grains': MagicMock()}) @skipIf(NO_MOCK, NO_MOCK_REASON) class GrainsModuleTestCase(TestCase): From c9bc8af8f2de5e5c90a22b53657a7339ddbc275a Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 28 Feb 2017 09:01:24 -0700 Subject: [PATCH 06/38] [2016.3] Bump latest release version to 2016.11.3 --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 42ebd743e7..a4e591c4eb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -213,7 +213,7 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.2' # latest release +latest_release = '2016.11.3' # latest release previous_release = '2016.3.5' # latest release from previous branch previous_release_dir = '2016.3' # path on web server for previous branch next_release = '' # next release From c2df29edb23ce787dd45a30051d22423794576ce Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 1 Mar 2017 09:27:07 -0700 Subject: [PATCH 07/38] Properly display error in jboss7 state Refs #33187 --- salt/states/jboss7.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/states/jboss7.py b/salt/states/jboss7.py index 7eb6ad91c0..9c5d2950da 100644 --- a/salt/states/jboss7.py +++ b/salt/states/jboss7.py @@ -448,9 +448,9 @@ def __get_artifact(salt_source): log.debug(traceback.format_exc()) comment = 'Unable to manage file: {0}'.format(e) - else: - resolved_source = salt_source['target_file'] - comment = '' + else: + resolved_source = salt_source['target_file'] + comment = '' return resolved_source, comment From 65b239664e07c10f992f324c78a8677ec6b01a48 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 1 Mar 2017 14:15:07 -0700 Subject: [PATCH 08/38] Restore ipv6 connectivity and "master: :" support --- doc/ref/configuration/minion.rst | 43 ++++++++++++++++++++++++++++++++ salt/minion.py | 22 +++++++++++----- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index b3e562a0fc..992f4128cb 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -40,6 +40,33 @@ Default: ``salt`` master: salt +master:port Syntax +~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 2015.8.0 + +The ``master`` config option can also be set to use the master's IP in +conjunction with a port number by default. + +.. code-block:: yaml + + master: localhost:1234 + +For IPv6 formatting with a port, remember to add brackets around the IP address +before adding the port and enclose the line in single quotes to make it a string: + +.. code-block:: yaml + + master: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]:1234' + +.. note:: + + If a port is specified in the ``master`` as well as :conf_minion:`master_port`, + the ``master_port`` setting will be overridden by the ``master`` configuration. + +List of Masters Syntax +~~~~~~~~~~~~~~~~~~~~~~ + The option can can also be set to a list of masters, enabling :ref:`multi-master ` mode. @@ -90,6 +117,22 @@ will try to automatically detect IPv6 connectivity to master. ipv6: True +.. conf_minion:: master_uri_format + +``master_uri_format`` +--------------------- + +.. versionadded:: 2015.8.0 + +Specify the format in which the master address will be evaluated. Valid options +are ``default`` or ``ip_only``. If ``ip_only`` is specified, then the master +address will not be split into IP and PORT, so be sure that only an IP (or domain +name) is set in the :conf_minion:`master` configuration setting. + +.. code-block:: yaml + + master_uri_format: ip_only + .. conf_minion:: master_type ``master_type`` diff --git a/salt/minion.py b/salt/minion.py index 1524d9da06..3d8fe21c88 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -88,6 +88,7 @@ import salt.utils.jid import salt.pillar import salt.utils.args import salt.utils.event +import salt.utils.network import salt.utils.minion import salt.utils.minions import salt.utils.schedule @@ -179,9 +180,11 @@ def resolve_dns(opts, fallback=True, connect=True): if master == '': master = unknown_str if opts.get('__role') == 'syndic': - err = 'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. Set \'syndic_master\' value in minion config.'.format(master) + err = 'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. ' \ + 'Set \'syndic_master\' value in minion config.'.format(master) else: - err = 'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. Set \'master\' value in minion config.'.format(master) + err = 'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. ' \ + 'Set \'master\' value in minion config.'.format(master) log.error(err) raise SaltSystemExit(code=42, msg=err) else: @@ -199,7 +202,10 @@ def resolve_dns(opts, fallback=True, connect=True): def prep_ip_port(opts): ret = {} - if opts['master_uri_format'] == 'ip_only': + # Use given master IP if "ip_only" is set or if master_ip is an ipv6 address without + # a port specified. The is_ipv6 check returns False if brackets are used in the IP + # definition such as master: '[::1]:1234'. + if opts['master_uri_format'] == 'ip_only' or salt.utils.network.is_ipv6(opts['master']): ret['master'] = opts['master'] else: ip_port = opts['master'].rsplit(":", 1) @@ -209,9 +215,13 @@ def prep_ip_port(opts): else: # e.g. master: localhost:1234 # e.g. master: 127.0.0.1:1234 - # e.g. master: ::1:1234 - ret['master'] = ip_port[0] - ret['master_port'] = ip_port[1] + # e.g. master: [::1]:1234 + # Strip off brackets for ipv6 support + ret['master'] = ip_port[0].strip('[]') + + # Cast port back to an int! Otherwise a TypeError is thrown + # on some of the socket calls elsewhere in the minion and utils code. + ret['master_port'] = int(ip_port[1]) return ret From b71c3fe13ced0df7a80b3d1875b9eaa8567f6c48 Mon Sep 17 00:00:00 2001 From: Sergei Zviagintsev Date: Wed, 1 Mar 2017 18:09:36 +0100 Subject: [PATCH 09/38] Revert "cloud.clouds.ec2: cache each named node (#33164)" This reverts commit b3805d825a23 ("cloud.clouds.ec2: cache each named node (#33164)"). Commit 25771fc03022 ("_get_node return instance info directly, not a dict of instances") from 2016.3 feature release changed how the result of _get_node should be interpreted. Before it was like res = _get_node(...) node = res[name] and after the commit it became node = _get_node(...) Commit b3805d825a23 ("cloud.clouds.ec2: cache each named node (#33164)") submitted into 2015.8 as a bugfix for #33162 added loops over the keys of a dictionary returned by _get_node, fixing salt.utils.cloud.cache_node calls in queue_instances() and show_instance(). But after being merged into 2016.3 with merge commit 679200aeb2fd ("Merge branch '2015.8' into '2016.3'"), the commit in question reintroduced the bug on 2016.3 because of changed return value of _get_node. Fixes #39782 --- salt/cloud/clouds/ec2.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index 41e6cf0e24..af890e3264 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -2647,11 +2647,7 @@ def queue_instances(instances): ''' for instance_id in instances: node = _get_node(instance_id=instance_id) - for name in node: - if instance_id == node[name]['instanceId']: - __utils__['cloud.cache_node'](node[name], - __active_provider_name__, - __opts__) + __utils__['cloud.cache_node'](node, __active_provider_name__, __opts__) def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): @@ -3200,10 +3196,7 @@ def show_instance(name=None, instance_id=None, call=None, kwargs=None): ) node = _get_node(name=name, instance_id=instance_id) - for name in node: - __utils__['cloud.cache_node'](node[name], - __active_provider_name__, - __opts__) + __utils__['cloud.cache_node'](node, __active_provider_name__, __opts__) return node From 3ab4f843bfea374c6ed709f52ce58b364f22bf85 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 2 Mar 2017 12:03:50 -0600 Subject: [PATCH 10/38] load runners if role is master --- salt/utils/schedule.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index a2916c565e..2c1efef805 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -636,7 +636,10 @@ class Schedule(object): # This also needed for ZeroMQ transport to reset all functions # context data that could keep paretns connections. ZeroMQ will # hang on polling parents connections from the child process. - self.functions = salt.loader.minion_mods(self.opts) + if self.opts['__role'] == 'master': + self.functions = salt.loader.runner(self.opts) + else: + self.functions = salt.loader.minion_mods(self.opts) self.returners = salt.loader.returners(self.opts, self.functions) ret = {'id': self.opts.get('id', 'master'), 'fun': func, From 2e683e788b71c8a23c81d36f060008c3e4cd3c3c Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 2 Mar 2017 13:27:32 -0700 Subject: [PATCH 11/38] Clean up errors which might be thrown when the monitor socket shuts down --- salt/transport/zeromq.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 20c7ba88cc..7a7f682e11 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -28,6 +28,7 @@ import salt.transport.mixins.auth from salt.exceptions import SaltReqTimeoutError import zmq +import zmq.error import zmq.eventloop.ioloop # support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x if not hasattr(zmq.eventloop.ioloop, 'ZMQIOLoop'): @@ -1006,9 +1007,14 @@ class ZeroMQSocketMonitor(object): def start_poll(self): log.trace("Event monitor start!") - while self._monitor_socket is not None and self._monitor_socket.poll(): - msg = self._monitor_socket.recv_multipart() - self.monitor_callback(msg) + try: + while self._monitor_socket is not None and self._monitor_socket.poll(): + msg = self._monitor_socket.recv_multipart() + self.monitor_callback(msg) + except (AttributeError, zmq.error.ContextTerminated): + # We cannot log here because we'll get an interrupted system call in trying + # to flush the logging buffer as we terminate + pass @property def event_map(self): From e31d46c1b8e97a8ac7d253aa9ed0d6dabd3586da Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 2 Mar 2017 14:45:57 -0700 Subject: [PATCH 12/38] Stop the process manager when it no longer has processes to manage --- salt/utils/process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/utils/process.py b/salt/utils/process.py index 8a0d0f3d14..9e60d0bad6 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -385,6 +385,8 @@ class ProcessManager(object): yield gen.sleep(10) else: time.sleep(10) + if len(self._process_map) == 0: + break # OSError is raised if a signal handler is called (SIGTERM) during os.wait except OSError: break @@ -396,6 +398,7 @@ class ProcessManager(object): if self._restart_processes is True: for pid, mapping in six.iteritems(self._process_map): if not mapping['Process'].is_alive(): + log.trace('Process restart of {0}'.format(pid)) self.restart_process(pid) def kill_children(self, *args, **kwargs): From 7178e77eeef6f2facbe0bfbba1dc693054bc5706 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 3 Mar 2017 10:17:10 -0600 Subject: [PATCH 13/38] Improve the Top File matching docs In 2014.7.0, default matcher in the top file was changed to the compound matcher, but the docs still suggest that it is the glob matcher. This commit updates the docs to explicitly mention that the compound matcher is now the default, and adds a table describing all of the available matchers that can be set in the top file. --- doc/ref/states/index.rst | 44 ++++++++++++++++------ doc/ref/states/top.rst | 68 ++++++++++++++++++++++++++-------- doc/topics/targeting/range.rst | 2 +- 3 files changed, 87 insertions(+), 27 deletions(-) diff --git a/doc/ref/states/index.rst b/doc/ref/states/index.rst index cdad6b7ea3..f8ee0fe909 100644 --- a/doc/ref/states/index.rst +++ b/doc/ref/states/index.rst @@ -147,16 +147,17 @@ watched file updated. The Top File ```````````` -The top file controls the mapping between minions and the states which should be -applied to them. +The top file controls the mapping between minions and the states which should +be applied to them. -The top file specifies which minions should have which SLS files applied and which -environments they should draw those SLS files from. +The top file specifies which minions should have which SLS files applied and +which environments they should draw those SLS files from. The top file works by specifying environments on the top-level. -Each environment contains globs to match minions. Finally, each glob contains a list of -lists of Salt states to apply to matching minions: +Each environment contains :ref:`target expressions ` to match +minions. Finally, each target expression contains a list of Salt states to +apply to matching minions: .. code-block:: yaml @@ -172,12 +173,33 @@ lists of Salt states to apply to matching minions: This above example uses the base environment which is built into the default Salt setup. -The base environment has two globs. First, the '*' glob contains a list of -SLS files to apply to all minions. +The base environment has target expressions. The first one matches all minions, +and the SLS files below it apply to all minions. -The second glob contains a regular expression that will match all minions with -an ID matching saltmaster.* and specifies that for those minions, the salt.master -state should be applied. +The second expression is a regular expression that will match all minions +with an ID matching ``saltmaster.*`` and specifies that for those minions, the +salt.master state should be applied. To + +.. important:: + Since version 2014.7.0, the default matcher (when one is not explicitly + defined as in the second expression in the above example) is the + :ref:`compound ` matcher. Since this matcher parses + individual words in the expression, minion IDs containing spaces will not + match properly using this matcher. Therefore, if your target expression is + designed to match a minion ID containing spaces, it will be necessary to + specify a different match type (such as ``glob``). For example: + + .. code-block:: yaml + + base: + 'test minion': + - match: glob + - foo + - bar + - baz + +A full table of match types available in the top file can be found :ref:`here +`. .. _reloading-modules: diff --git a/doc/ref/states/top.rst b/doc/ref/states/top.rst index ae02648cc8..0d933ddf0d 100644 --- a/doc/ref/states/top.rst +++ b/doc/ref/states/top.rst @@ -158,8 +158,8 @@ Our top file references the environments: - db As seen above, the top file now declares the three environments and for each, -targets are defined to map globs of minion IDs to state files. For example, -all minions which have an ID beginning with the string ``webserver`` will have the +target expressions are defined to map minions to state files. For example, all +minions which have an ID beginning with the string ``webserver`` will have the webserver state from the requested environment assigned to it. In this manner, a proposed change to a state could first be made in a state @@ -219,12 +219,51 @@ also available: Advanced Minion Targeting ========================= -In addition to globs, minions can be specified in top files a few other -ways. Some common ones are :ref:`compound matches ` -and :ref:`node groups `. +In the examples above, notice that all of the target expressions are globs. The +default match type in top files (since version 2014.7.0) is actually the +:ref:`compound matcher `, not the glob matcher as in the +CLI. -Below is a slightly more complex top file example, showing the different types -of matches you can perform: +A single glob, when passed through the compound matcher, acts the same way as +matching by glob, so in most cases the two are indistinguishable. However, +there is an edge case in which a minion ID contains whitespace. While it is not +recommended to include spaces in a minion ID, Salt will not stop you from doing +so. However, since compound expressions are parsed word-by-word, if a minion ID +contains spaces it will fail to match. In this edge case, it will be necessary +to explicitly use the ``glob`` matcher: + +.. code-block:: yaml + + base: + 'minion 1': + - match: glob + - foo + +.. _top-file-match-types: + +The available match types which can be set for a target expression in the top +file are: + +============ ================================================================================================================ +Match Type Description +============ ================================================================================================================ +glob Full minion ID or glob expression to match multiple minions (e.g. ``minion123`` or ``minion*``) +pcre Perl-compatible regular expression (PCRE) matching a minion ID (e.g. ``web[0-3].domain.com``) +grain Match a :ref:`grain `, optionally using globbing (e.g. ``kernel:Linux`` or ``kernel:*BSD``) +grain_pcre Match a :ref:`grain ` using PCRE (e.g. ``kernel:(Free|Open)BSD``) +list Comma-separated list of minions (e.g. ``minion1,minion2,minion3``) +pillar :ref:`Pillar ` match, optionally using globbing (e.g. ``role:webserver`` or ``role:web*``) +pillar_pcre :ref:`Pillar ` match using PCRE (e.g. ``role:web(server|proxy)`` +pillar_exact :ref:`Pillar ` match with no globbing or PCRE (e.g. ``role:webserver``) +ipcidr Subnet or IP address (e.g. ``172.17.0.0/16`` or ``10.2.9.80``) +data Match values kept in the minion's datastore (created using the :mod:`data ` execution module) +range :ref:`Range ` cluster +compound Complex expression combining multiple match types (see :ref:`here `) +nodegroup Pre-defined compound expressions in the master config file (see :ref:`here `) +============ ================================================================================================================ + +Below is a slightly more complex top file example, showing some of the above +match types: .. code-block:: yaml @@ -232,6 +271,13 @@ of matches you can perform: # environment in the ``file_roots`` configuration value. base: + # All minions which begin with the strings 'nag1' or any minion with + # a grain set called 'role' with the value of 'monitoring' will have + # the 'server.sls' state file applied from the 'nagios/' directory. + + 'nag1* or G@role:monitoring': + - nagios.server + # All minions get the following three state files applied '*': @@ -296,14 +342,6 @@ of matches you can perform: - match: pillar - xyz - # All minions which begin with the strings 'nag1' or any minion with - # a grain set called 'role' with the value of 'monitoring' will have - # the 'server.sls' state file applied from the 'nagios/' directory. - - 'nag1* or G@role:monitoring': - - match: compound - - nagios.server - How Top Files Are Compiled ========================== diff --git a/doc/topics/targeting/range.rst b/doc/topics/targeting/range.rst index 67e9fcd339..e5bd3f9dab 100644 --- a/doc/topics/targeting/range.rst +++ b/doc/topics/targeting/range.rst @@ -1,4 +1,4 @@ -.. _targeting_range: +.. _targeting-range: ========== SECO Range From 804b12048c3bead714662d6d6261e887e738587d Mon Sep 17 00:00:00 2001 From: Jan Kadlec Date: Fri, 3 Mar 2017 17:58:53 +0100 Subject: [PATCH 14/38] Add missing apostrophe --- doc/topics/beacons/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/beacons/index.rst b/doc/topics/beacons/index.rst index c85fd53d1d..1c91dd5c6e 100644 --- a/doc/topics/beacons/index.rst +++ b/doc/topics/beacons/index.rst @@ -220,7 +220,7 @@ Add the following to ``/srv/reactor/revert.sls``: .. note:: - The expression ``{{ data['data']['id] }}`` :ref:`is correct + The expression ``{{ data['data']['id'] }}`` :ref:`is correct ` as it matches the event structure :ref:`shown above `. From 6125eff02d00ec05b5ae84b9058d1b7047f1b3f2 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 3 Mar 2017 16:46:24 -0700 Subject: [PATCH 15/38] Add group func to yubikey auth Because we don't support groups with yubikeys (and can't, in fact) we need to dummy up this to always return false. This allows individual user auths to work, whereas they would fail prior to this change. --- salt/auth/yubico.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/auth/yubico.py b/salt/auth/yubico.py index 9a9c44c753..6c0afcb9cf 100644 --- a/salt/auth/yubico.py +++ b/salt/auth/yubico.py @@ -78,15 +78,16 @@ def auth(username, password): client = Yubico(_cred['id'], _cred['key']) try: - if client.verify(password): - return True - else: - return False + return client.verify(password) except yubico_exceptions.StatusCodeError as e: log.info('Unable to verify YubiKey `{0}`'.format(e)) return False +def groups(username, *args, **kwargs): + return False + + if __name__ == '__main__': __opts__ = {'yubico_users': {'damian': {'id': '12345', 'key': 'ABC123'}}} From 2b2ec69d04bad614833c94a8086baf2b26211f7f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 7 Mar 2017 09:31:43 -0600 Subject: [PATCH 16/38] Squelch warning for pygit2 import RedHat updated cffi for RHEL/CentOS 7.3, which causes a pair of warnings on the pygit2 import. This warning is spit out to the CLI on several commands, when they result in the git fileserver backend to be loaded. This commit squelches that warning as it is just noise. Upgrading pygit2/libgit2 would solve this, but that is not likely to happen in RHEL/CentOS 7. --- salt/utils/gitfs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index 657ced09b3..e035001b7c 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -74,7 +74,11 @@ except ImportError: HAS_GITPYTHON = False try: - import pygit2 + # Squelch warning on cent7 due to them upgrading cffi + import warnings + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + import pygit2 HAS_PYGIT2 = True try: GitError = pygit2.errors.GitError From baf4579e63facdbcad4ada09077004e28b7f6e3b Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Wed, 8 Mar 2017 10:19:06 -0700 Subject: [PATCH 17/38] Update cleanup function for azure --- salt/cloud/clouds/msazure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/clouds/msazure.py b/salt/cloud/clouds/msazure.py index a3bef62eb8..185d739c92 100644 --- a/salt/cloud/clouds/msazure.py +++ b/salt/cloud/clouds/msazure.py @@ -1591,7 +1591,7 @@ def cleanup_unattached_disks(kwargs=None, conn=None, call=None): for disk in disks: if disks[disk]['attached_to'] is None: del_kwargs = { - 'name': disks[disk]['name'][0], + 'name': disks[disk]['name'], 'delete_vhd': kwargs.get('delete_vhd', False) } log.info('Deleting disk {name}, deleting VHD: {delete_vhd}'.format(**del_kwargs)) From 462bdecd33459fd85c3a7d860f3fc0901133241a Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 8 Mar 2017 11:21:11 -0700 Subject: [PATCH 18/38] Namespace the line function properly in win_file --- salt/modules/win_file.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index 30c6a62cdb..e73b2c5785 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -64,7 +64,8 @@ from salt.modules.file import (check_hash, # pylint: disable=W0611 search, _get_flags, extract_hash, _error, _sed_esc, _psed, RE_FLAG_TABLE, blockreplace, prepend, seek_read, seek_write, rename, lstat, path_exists_glob, write, pardir, join, HASHES, HASHES_REVMAP, - comment, uncomment, _add_flags, comment_line, apply_template_on_contents) + comment, uncomment, _add_flags, comment_line, _regex_to_static, + _get_line_indent, apply_template_on_contents) from salt.utils import namespaced_function as _namespaced_function @@ -86,12 +87,13 @@ def __virtual__(): global append, _error, directory_exists, touch, contains global contains_regex, contains_glob, get_source_sum global find, psed, get_sum, check_hash, get_hash, delete_backup - global get_diff, _get_flags, extract_hash, comment_line + global get_diff, line, _get_flags, extract_hash, comment_line global access, copy, readdir, rmdir, truncate, replace, search global _binary_replace, _get_bkroot, list_backups, restore_backup global blockreplace, prepend, seek_read, seek_write, rename, lstat global write, pardir, join, _add_flags, apply_template_on_contents global path_exists_glob, comment, uncomment, _mkstemp_copy + global _regex_to_static, _get_line_indent replace = _namespaced_function(replace, globals()) search = _namespaced_function(search, globals()) @@ -128,6 +130,7 @@ def __virtual__(): check_hash = _namespaced_function(check_hash, globals()) get_hash = _namespaced_function(get_hash, globals()) get_diff = _namespaced_function(get_diff, globals()) + line = _namespaced_function(line, globals()) access = _namespaced_function(access, globals()) copy = _namespaced_function(copy, globals()) readdir = _namespaced_function(readdir, globals()) @@ -146,6 +149,8 @@ def __virtual__(): comment = _namespaced_function(comment, globals()) uncomment = _namespaced_function(uncomment, globals()) comment_line = _namespaced_function(comment_line, globals()) + _regex_to_static = _namespaced_function(_regex_to_static, globals()) + _get_line_indent = _namespaced_function(_get_line_indent, globals()) _mkstemp_copy = _namespaced_function(_mkstemp_copy, globals()) _add_flags = _namespaced_function(_add_flags, globals()) apply_template_on_contents = _namespaced_function(apply_template_on_contents, globals()) From 1a78878b4712e0a4b156fdbd0a35b658b6176f77 Mon Sep 17 00:00:00 2001 From: Johnny Cook Date: Wed, 16 Nov 2016 16:03:29 -0600 Subject: [PATCH 19/38] Adding check for ignore_if_missing param when calling _check_file. --- salt/states/file.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 88ab3de385..c5d1a13026 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3358,7 +3358,10 @@ def replace(name, check_res, check_msg = _check_file(name) if not check_res: - return _error(ret, check_msg) + if ignore_if_missing and 'file not found' in check_msg: + pass + else: + return _error(ret, check_msg) changes = __salt__['file.replace'](name, pattern, From ca306c086072b2a0cf83acd8780d37767fb18c44 Mon Sep 17 00:00:00 2001 From: Johnny Cook Date: Tue, 22 Nov 2016 17:07:42 -0600 Subject: [PATCH 20/38] Replace pass with updated comment for return --- salt/states/file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index c5d1a13026..bbe0ac9118 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3359,7 +3359,8 @@ def replace(name, check_res, check_msg = _check_file(name) if not check_res: if ignore_if_missing and 'file not found' in check_msg: - pass + ret['comment'] = 'No changes needed to be made' + return ret else: return _error(ret, check_msg) From 170cbadc54f4ff2dd50ef38afe0f5943a92f933d Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 9 Mar 2017 13:52:13 +0100 Subject: [PATCH 21/38] CIDR matching supports IPv6, update docs After #22080 IPv6 CIDR matching is supported & functional, docs should('ve) reflected that since 2015.8 --- doc/topics/targeting/ipcidr.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/topics/targeting/ipcidr.rst b/doc/topics/targeting/ipcidr.rst index 27c2a28124..40247bc246 100644 --- a/doc/topics/targeting/ipcidr.rst +++ b/doc/topics/targeting/ipcidr.rst @@ -10,7 +10,7 @@ notation). .. code-block:: bash salt -S 192.168.40.20 test.ping - salt -S 10.0.0.0/24 test.ping + salt -S 2001:db8::/64 test.ping Ipcidr matching can also be used in compound matches @@ -27,7 +27,3 @@ It is also possible to use in both pillar and state-matching - internal .. _CIDR: http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing - -.. note:: - - Only IPv4 matching is supported at this time. From a7e419e35f67699af59ff3c510fdaf859eaf490c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 9 Mar 2017 12:18:51 -0600 Subject: [PATCH 22/38] Scrap event-based approach for refreshing grains The call to refresh_modules() in saltutil.refresh_grains was resulting in a race condition with the event I added to refresh the grains. This meant that, even though self.opts['grains'] was being changed by the new event, before that process could finish the module refresh was itself re-loading the grains and re-packing these new grains into the __grains__ dunder, negating the refresh. Since the module refresh loads the grains anyway, this commit changes saltutil.refresh_grains to refresh the modules. It also removes the refresh argument recently added to the grains module functions, as we're no longer using it to conditionally refresh modules. --- salt/modules/grains.py | 16 ++++------------ salt/modules/saltutil.py | 23 ++++++++++++----------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/salt/modules/grains.py b/salt/modules/grains.py index f1d34abf17..dac1035589 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -203,7 +203,7 @@ def item(*args, **kwargs): return ret -def setvals(grains, destructive=False, refresh=True): +def setvals(grains, destructive=False): ''' Set new grains values in the grains config file @@ -211,10 +211,6 @@ def setvals(grains, destructive=False, refresh=True): If an operation results in a key being removed, delete the key, too. Defaults to False. - refresh - Refresh modules and pillar after adding the new grains. - Defaults to True. - CLI Example: .. code-block:: bash @@ -290,12 +286,12 @@ def setvals(grains, destructive=False, refresh=True): log.error(msg.format(fn_)) if not __opts__.get('local', False): # Refresh the grains - __salt__['saltutil.refresh_grains'](refresh=refresh) + __salt__['saltutil.refresh_grains']() # Return the grains we just set to confirm everything was OK return new_grains -def setval(key, val, destructive=False, refresh=True): +def setval(key, val, destructive=False): ''' Set a grains value in the grains config file @@ -309,10 +305,6 @@ def setval(key, val, destructive=False, refresh=True): If an operation results in a key being removed, delete the key, too. Defaults to False. - refresh - Refresh modules and pillar after adding the new grain. - Defaults to True. - CLI Example: .. code-block:: bash @@ -320,7 +312,7 @@ def setval(key, val, destructive=False, refresh=True): salt '*' grains.setval key val salt '*' grains.setval key "{'sub-key': 'val', 'sub-key2': 'val2'}" ''' - return setvals({key: val}, destructive, refresh) + return setvals({key: val}, destructive) def append(key, val, convert=False, delimiter=DEFAULT_TARGET_DELIM): diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index bb77501129..80261d3b6a 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -310,16 +310,19 @@ def sync_states(saltenv=None, refresh=True): return ret -def refresh_grains(refresh=True): +def refresh_grains(refresh_pillar=True): ''' .. versionadded:: 2016.3.6,2016.11.4,Nitrogen Refresh the minion's grains without syncing custom grains modules from ``salt://_grains``. - refresh : True - If ``True``, refresh the available execution modules and recompile - pillar data for the minion. Set to ``False`` to prevent this refresh. + .. note:: + The available execution modules will be reloaded as part of this + proceess, as grains can affect which modules are available. + + refresh_pillar : True + Set to ``False`` to keep pillar data from being refreshed. CLI Examples: @@ -327,13 +330,11 @@ def refresh_grains(refresh=True): salt '*' saltutil.refresh_grains ''' - try: - ret = __salt__['event.fire']({}, 'grains_refresh_manual') - except Exception: - log.error('Failed to refresh grains', exc_info=True) - return False - if refresh: - refresh_modules() + # Modules and pillar need to be refreshed in case grains changes affected + # them, and the module refresh process reloads the grains and assigns the + # newly-reloaded grains to each execution module's __grains__ dunder. + refresh_modules() + if refresh_pillar: refresh_pillar() return ret From d86b03dc90af78dc1c50a3bcdb7bd4108fc7a00c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 9 Mar 2017 12:42:21 -0600 Subject: [PATCH 23/38] Remove manual refresh code from minion.py --- salt/minion.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 80712a98b2..84dc4f2b86 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1765,15 +1765,6 @@ class Minion(MinionBase): tagify([self.opts['id'], 'start'], 'minion'), ) - def grains_refresh_manual(self): - ''' - Perform a manual grains refresh - ''' - self.opts['grains'] = salt.loader.grains( - self.opts, - force_refresh=True, - proxy=getattr(self, 'proxy', None)) - def module_refresh(self, force_refresh=False, notify=False): ''' Refresh the functions and returners. @@ -1951,8 +1942,6 @@ class Minion(MinionBase): elif tag.startswith('manage_beacons'): self.manage_beacons(tag, data) elif tag.startswith('grains_refresh'): - if tag == 'grains_refresh_manual': - self.grains_refresh_manual() if (data.get('force_refresh', False) or self.grains_cache != self.opts['grains']): self.pillar_refresh(force_refresh=True) From d9f48ac6eaabc08c7fa64964763c99e7ffdea4fc Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 9 Mar 2017 13:22:03 -0600 Subject: [PATCH 24/38] Don't shadow refresh_pillar --- salt/modules/saltutil.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 80261d3b6a..cb1473c4ce 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -310,7 +310,7 @@ def sync_states(saltenv=None, refresh=True): return ret -def refresh_grains(refresh_pillar=True): +def refresh_grains(**kwargs): ''' .. versionadded:: 2016.3.6,2016.11.4,Nitrogen @@ -330,13 +330,17 @@ def refresh_grains(refresh_pillar=True): salt '*' saltutil.refresh_grains ''' + kwargs = salt.utils.clean_kwargs(**kwargs) + _refresh_pillar = kwargs.pop('refresh_pillar', True) + if kwargs: + salt.utils.invalid_kwargs(kwargs) # Modules and pillar need to be refreshed in case grains changes affected # them, and the module refresh process reloads the grains and assigns the # newly-reloaded grains to each execution module's __grains__ dunder. refresh_modules() - if refresh_pillar: + if _refresh_pillar: refresh_pillar() - return ret + return True def sync_grains(saltenv=None, refresh=True): From cf0100dabe8de85ae100f0e06f6aeb51b6a20cbe Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 9 Mar 2017 12:18:51 -0600 Subject: [PATCH 25/38] Scrap event-based approach for refreshing grains The call to refresh_modules() in saltutil.refresh_grains was resulting in a race condition with the event I added to refresh the grains. This meant that, even though self.opts['grains'] was being changed by the new event, before that process could finish the module refresh was itself re-loading the grains and re-packing these new grains into the __grains__ dunder, negating the refresh. Since the module refresh loads the grains anyway, this commit changes saltutil.refresh_grains to refresh the modules. It also removes the refresh argument recently added to the grains module functions, as we're no longer using it to conditionally refresh modules. --- salt/minion.py | 14 +------------- salt/modules/grains.py | 23 ++++++----------------- salt/modules/saltutil.py | 29 +++++++++++++++++------------ 3 files changed, 24 insertions(+), 42 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 3d8fe21c88..6a857bad52 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1695,15 +1695,6 @@ class Minion(MinionBase): tagify([self.opts['id'], 'start'], 'minion'), ) - def grains_refresh_manual(self): - ''' - Perform a manual grains refresh - ''' - self.opts['grains'] = salt.loader.grains( - self.opts, - force_refresh=True, - proxy=getattr(self, 'proxy', None)) - def module_refresh(self, force_refresh=False, notify=False): ''' Refresh the functions and returners. @@ -1873,12 +1864,9 @@ class Minion(MinionBase): elif package.startswith('manage_beacons'): self.manage_beacons(tag, data) elif package.startswith('grains_refresh'): - if package == 'grains_refresh_manual': - self.grains_refresh_manual() if self.grains_cache != self.opts['grains']: self.grains_cache = self.opts['grains'] - if package != 'grains_refresh_manual': - self.pillar_refresh(force_refresh=True) + self.pillar_refresh(force_refresh=True) elif package.startswith('environ_setenv'): self.environ_setenv(tag, data) elif package.startswith('_minion_mine'): diff --git a/salt/modules/grains.py b/salt/modules/grains.py index 0f8c5f27b6..640869ccae 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -192,7 +192,7 @@ def item(*args, **kwargs): return ret -def setvals(grains, destructive=False, refresh=True): +def setvals(grains, destructive=False): ''' Set new grains values in the grains config file @@ -200,10 +200,6 @@ def setvals(grains, destructive=False, refresh=True): If an operation results in a key being removed, delete the key, too. Defaults to False. - refresh - Refresh modules and pillar after adding the new grains. - Defaults to True. - CLI Example: .. code-block:: bash @@ -279,12 +275,12 @@ def setvals(grains, destructive=False, refresh=True): log.error(msg.format(fn_)) if not __opts__.get('local', False): # Refresh the grains - __salt__['saltutil.refresh_grains'](refresh=refresh) + __salt__['saltutil.refresh_grains']() # Return the grains we just set to confirm everything was OK return new_grains -def setval(key, val, destructive=False, refresh=True): +def setval(key, val, destructive=False): ''' Set a grains value in the grains config file @@ -298,10 +294,6 @@ def setval(key, val, destructive=False, refresh=True): If an operation results in a key being removed, delete the key, too. Defaults to False. - refresh - Refresh modules and pillar after adding the new grain. - Defaults to True. - CLI Example: .. code-block:: bash @@ -309,7 +301,7 @@ def setval(key, val, destructive=False, refresh=True): salt '*' grains.setval key val salt '*' grains.setval key "{'sub-key': 'val', 'sub-key2': 'val2'}" ''' - return setvals({key: val}, destructive, refresh) + return setvals({key: val}, destructive) def append(key, val, convert=False, delimiter=DEFAULT_TARGET_DELIM): @@ -412,7 +404,7 @@ def remove(key, val, delimiter=DEFAULT_TARGET_DELIM): return setval(key, grains) -def delval(key, destructive=False, refresh=True): +def delval(key, destructive=False): ''' .. versionadded:: 0.17.0 @@ -424,9 +416,6 @@ def delval(key, destructive=False, refresh=True): destructive Delete the key, too. Defaults to False. - refresh - Refresh modules and pillar after removing the grain. - CLI Example: .. code-block:: bash @@ -434,7 +423,7 @@ def delval(key, destructive=False, refresh=True): salt '*' grains.delval key ''' - setval(key, None, destructive=destructive, refresh=refresh) + setval(key, None, destructive=destructive) def ls(): # pylint: disable=C0103 diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 39d6c28df3..05e8457208 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -310,16 +310,19 @@ def sync_states(saltenv=None, refresh=True): return ret -def refresh_grains(refresh=True): +def refresh_grains(**kwargs): ''' .. versionadded:: 2016.3.6,2016.11.4,Nitrogen Refresh the minion's grains without syncing custom grains modules from ``salt://_grains``. - refresh : True - If ``True``, refresh the available execution modules and recompile - pillar data for the minion. Set to ``False`` to prevent this refresh. + .. note:: + The available execution modules will be reloaded as part of this + proceess, as grains can affect which modules are available. + + refresh_pillar : True + Set to ``False`` to keep pillar data from being refreshed. CLI Examples: @@ -327,15 +330,17 @@ def refresh_grains(refresh=True): salt '*' saltutil.refresh_grains ''' - try: - ret = __salt__['event.fire']({}, 'grains_refresh_manual') - except Exception: - log.error('Failed to refresh grains', exc_info=True) - return False - if refresh: - refresh_modules() + kwargs = salt.utils.clean_kwargs(**kwargs) + _refresh_pillar = kwargs.pop('refresh_pillar', True) + if kwargs: + salt.utils.invalid_kwargs(kwargs) + # Modules and pillar need to be refreshed in case grains changes affected + # them, and the module refresh process reloads the grains and assigns the + # newly-reloaded grains to each execution module's __grains__ dunder. + refresh_modules() + if _refresh_pillar: refresh_pillar() - return ret + return True def sync_grains(saltenv=None, refresh=True): From 50e51b5b9d5a3aeb2cf7b6e26a5ed1dcfbddfc99 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 9 Mar 2017 16:06:29 +0000 Subject: [PATCH 26/38] The beacons configuration is now a list. Handle it! --- salt/beacons/ps.py | 29 ++++++++++++++-------------- tests/integration/modules/beacons.py | 11 +++++++---- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/salt/beacons/ps.py b/salt/beacons/ps.py index b61d459fda..479abed8f4 100644 --- a/salt/beacons/ps.py +++ b/salt/beacons/ps.py @@ -60,18 +60,19 @@ def beacon(config): if _name not in procs: procs.append(_name) - for process in config: - ret_dict = {} - if config[process] == 'running': - if process not in procs: - ret_dict[process] = 'Stopped' - ret.append(ret_dict) - elif config[process] == 'stopped': - if process in procs: - ret_dict[process] = 'Running' - ret.append(ret_dict) - else: - if process not in procs: - ret_dict[process] = False - ret.append(ret_dict) + for entry in config: + for process in entry: + ret_dict = {} + if entry[process] == 'running': + if process not in procs: + ret_dict[process] = 'Stopped' + ret.append(ret_dict) + elif entry[process] == 'stopped': + if process in procs: + ret_dict[process] = 'Running' + ret.append(ret_dict) + else: + if process not in procs: + ret_dict[process] = False + ret.append(ret_dict) return ret diff --git a/tests/integration/modules/beacons.py b/tests/integration/modules/beacons.py index d233a32763..3e130bab10 100644 --- a/tests/integration/modules/beacons.py +++ b/tests/integration/modules/beacons.py @@ -47,7 +47,7 @@ class BeaconsAddDeleteTest(integration.ModuleCase): ''' Test adding and deleting a beacon ''' - _add = self.run_function('beacons.add', ['ps', {'apache2': 'stopped'}]) + _add = self.run_function('beacons.add', ['ps', [{'apache2': 'stopped'}]]) self.assertTrue(_add['result']) # save added beacon @@ -71,7 +71,7 @@ class BeaconsTest(integration.ModuleCase): def setUp(self): try: # Add beacon to disable - self.run_function('beacons.add', ['ps', {'apache2': 'stopped'}]) + self.run_function('beacons.add', ['ps', [{'apache2': 'stopped'}]]) self.run_function('beacons.save') except CommandExecutionError: self.skipTest('Unable to add beacon') @@ -102,7 +102,10 @@ class BeaconsTest(integration.ModuleCase): # assert beacon ps is disabled _list = self.run_function('beacons.list', return_yaml=False) - self.assertFalse(_list['ps']['enabled']) + for bdict in _list['ps']: + if 'enabled' in bdict: + self.assertFalse(bdict['enabled']) + break def test_enable(self): ''' @@ -140,7 +143,7 @@ class BeaconsTest(integration.ModuleCase): # list beacons ret = self.run_function('beacons.list', return_yaml=False) if 'enabled' in ret: - self.assertEqual(ret, {'ps': {'apache2': 'stopped'}, 'enabled': True}) + self.assertEqual(ret, {'ps': [{'apache2': 'stopped'}], 'enabled': True}) else: self.assertEqual(ret, {'ps': {'apache': 'stopped'}}) From 6408b123e775335ccafef305d0a7166f28e16436 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 9 Mar 2017 17:50:37 +0000 Subject: [PATCH 27/38] These tests are not destructive Conflicts: - tests/unit/beacons/inotify_beacon_test.py --- tests/unit/beacons/inotify_beacon_test.py | 190 +++++++++------------- 1 file changed, 81 insertions(+), 109 deletions(-) diff --git a/tests/unit/beacons/inotify_beacon_test.py b/tests/unit/beacons/inotify_beacon_test.py index c40fedcd52..ef1dcdf7f1 100644 --- a/tests/unit/beacons/inotify_beacon_test.py +++ b/tests/unit/beacons/inotify_beacon_test.py @@ -33,6 +33,10 @@ class INotifyBeaconTestCase(TestCase): ''' def setUp(self): inotify.__context__ = {} + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) def test_empty_config(self, *args, **kwargs): config = {} @@ -52,122 +56,90 @@ class INotifyBeaconTestCase(TestCase): self.assertEqual(ret[0]['path'], path) self.assertEqual(ret[0]['change'], 'IN_OPEN') - @destructiveTest def test_dir_no_auto_add(self, *args, **kwargs): - tmpdir = None - try: - tmpdir = tempfile.mkdtemp() - config = {tmpdir: {'mask': ['create']}} - ret = inotify.beacon(config) - self.assertEqual(ret, []) - fp = os.path.join(tmpdir, 'tmpfile') - with open(fp, 'w') as f: - pass - ret = inotify.beacon(config) - self.assertEqual(len(ret), 1) - self.assertEqual(ret[0]['path'], fp) - self.assertEqual(ret[0]['change'], 'IN_CREATE') - with open(fp, 'r') as f: - pass - ret = inotify.beacon(config) - self.assertEqual(ret, []) + config = {self.tmpdir: {'mask': ['create']}} + ret = inotify.beacon(config) + self.assertEqual(ret, []) + fp = os.path.join(self.tmpdir, 'tmpfile') + with open(fp, 'w') as f: + pass + ret = inotify.beacon(config) + self.assertEqual(len(ret), 1) + self.assertEqual(ret[0]['path'], fp) + self.assertEqual(ret[0]['change'], 'IN_CREATE') + with open(fp, 'r') as f: + pass + ret = inotify.beacon(config) + self.assertEqual(ret, []) - finally: - if tmpdir: - shutil.rmtree(tmpdir) - - @destructiveTest def test_dir_auto_add(self, *args, **kwargs): - tmpdir = None - try: - tmpdir = tempfile.mkdtemp() - config = {tmpdir: {'mask': ['create', 'open'], 'auto_add': True}} - ret = inotify.beacon(config) - self.assertEqual(ret, []) - fp = os.path.join(tmpdir, 'tmpfile') - with open(fp, 'w') as f: - pass - ret = inotify.beacon(config) - self.assertEqual(len(ret), 2) - self.assertEqual(ret[0]['path'], fp) - self.assertEqual(ret[0]['change'], 'IN_CREATE') - self.assertEqual(ret[1]['path'], fp) - self.assertEqual(ret[1]['change'], 'IN_OPEN') - with open(fp, 'r') as f: - pass - ret = inotify.beacon(config) - self.assertEqual(len(ret), 1) - self.assertEqual(ret[0]['path'], fp) - self.assertEqual(ret[0]['change'], 'IN_OPEN') + config = {self.tmpdir: {'mask': ['create', 'open'], 'auto_add': True}} + ret = inotify.beacon(config) + self.assertEqual(ret, []) + fp = os.path.join(self.tmpdir, 'tmpfile') + with open(fp, 'w') as f: + pass + ret = inotify.beacon(config) + self.assertEqual(len(ret), 2) + self.assertEqual(ret[0]['path'], fp) + self.assertEqual(ret[0]['change'], 'IN_CREATE') + self.assertEqual(ret[1]['path'], fp) + self.assertEqual(ret[1]['change'], 'IN_OPEN') + with open(fp, 'r') as f: + pass + ret = inotify.beacon(config) + self.assertEqual(len(ret), 1) + self.assertEqual(ret[0]['path'], fp) + self.assertEqual(ret[0]['change'], 'IN_OPEN') - finally: - if tmpdir: - shutil.rmtree(tmpdir) - - @destructiveTest def test_dir_recurse(self, *args, **kwargs): - tmpdir = None - try: - tmpdir = tempfile.mkdtemp() - dp1 = os.path.join(tmpdir, 'subdir1') - os.mkdir(dp1) - dp2 = os.path.join(dp1, 'subdir2') - os.mkdir(dp2) - fp = os.path.join(dp2, 'tmpfile') - with open(fp, 'w') as f: - pass - config = {tmpdir: {'mask': ['open'], 'recurse': True}} - ret = inotify.beacon(config) - self.assertEqual(ret, []) - with open(fp) as f: - pass - ret = inotify.beacon(config) - self.assertEqual(len(ret), 3) - self.assertEqual(ret[0]['path'], dp1) - self.assertEqual(ret[0]['change'], 'IN_OPEN|IN_ISDIR') - self.assertEqual(ret[1]['path'], dp2) - self.assertEqual(ret[1]['change'], 'IN_OPEN|IN_ISDIR') - self.assertEqual(ret[2]['path'], fp) - self.assertEqual(ret[2]['change'], 'IN_OPEN') + dp1 = os.path.join(self.tmpdir, 'subdir1') + os.mkdir(dp1) + dp2 = os.path.join(dp1, 'subdir2') + os.mkdir(dp2) + fp = os.path.join(dp2, 'tmpfile') + with open(fp, 'w') as f: + pass + config = {self.tmpdir: {'mask': ['open'], 'recurse': True}} + ret = inotify.beacon(config) + self.assertEqual(ret, []) + with open(fp) as f: + pass + ret = inotify.beacon(config) + self.assertEqual(len(ret), 3) + self.assertEqual(ret[0]['path'], dp1) + self.assertEqual(ret[0]['change'], 'IN_OPEN|IN_ISDIR') + self.assertEqual(ret[1]['path'], dp2) + self.assertEqual(ret[1]['change'], 'IN_OPEN|IN_ISDIR') + self.assertEqual(ret[2]['path'], fp) + self.assertEqual(ret[2]['change'], 'IN_OPEN') - finally: - if tmpdir: - shutil.rmtree(tmpdir) - - @destructiveTest def test_dir_recurse_auto_add(self, *args, **kwargs): - tmpdir = None - try: - tmpdir = tempfile.mkdtemp() - dp1 = os.path.join(tmpdir, 'subdir1') - os.mkdir(dp1) - config = {tmpdir: {'mask': ['create', 'delete'], - 'recurse': True, - 'auto_add': True}} - ret = inotify.beacon(config) - self.assertEqual(ret, []) - dp2 = os.path.join(dp1, 'subdir2') - os.mkdir(dp2) - ret = inotify.beacon(config) - self.assertEqual(len(ret), 1) - self.assertEqual(ret[0]['path'], dp2) - self.assertEqual(ret[0]['change'], 'IN_CREATE|IN_ISDIR') - fp = os.path.join(dp2, 'tmpfile') - with open(fp, 'w') as f: - pass - ret = inotify.beacon(config) - self.assertEqual(len(ret), 1) - self.assertEqual(ret[0]['path'], fp) - self.assertEqual(ret[0]['change'], 'IN_CREATE') - os.remove(fp) - ret = inotify.beacon(config) - self.assertEqual(len(ret), 1) - self.assertEqual(ret[0]['path'], fp) - self.assertEqual(ret[0]['change'], 'IN_DELETE') - - finally: - if tmpdir: - shutil.rmtree(tmpdir) + dp1 = os.path.join(self.tmpdir, 'subdir1') + os.mkdir(dp1) + config = {self.tmpdir: {'mask': ['create', 'delete'], + 'recurse': True, + 'auto_add': True}} + ret = inotify.beacon(config) + self.assertEqual(ret, []) + dp2 = os.path.join(dp1, 'subdir2') + os.mkdir(dp2) + ret = inotify.beacon(config) + self.assertEqual(len(ret), 1) + self.assertEqual(ret[0]['path'], dp2) + self.assertEqual(ret[0]['change'], 'IN_CREATE|IN_ISDIR') + fp = os.path.join(dp2, 'tmpfile') + with open(fp, 'w') as f: + pass + ret = inotify.beacon(config) + self.assertEqual(len(ret), 1) + self.assertEqual(ret[0]['path'], fp) + self.assertEqual(ret[0]['change'], 'IN_CREATE') + os.remove(fp) + ret = inotify.beacon(config) + self.assertEqual(len(ret), 1) + self.assertEqual(ret[0]['path'], fp) + self.assertEqual(ret[0]['change'], 'IN_DELETE') if __name__ == '__main__': From 4a242829ee2e5728636bf8493feafc30402c8e27 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 9 Mar 2017 17:51:43 +0000 Subject: [PATCH 28/38] These tests aren't even using mock! Conflicts: - tests/unit/beacons/inotify_beacon_test.py --- tests/unit/beacons/inotify_beacon_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/beacons/inotify_beacon_test.py b/tests/unit/beacons/inotify_beacon_test.py index ef1dcdf7f1..b4d32d8224 100644 --- a/tests/unit/beacons/inotify_beacon_test.py +++ b/tests/unit/beacons/inotify_beacon_test.py @@ -12,7 +12,6 @@ from salt.beacons import inotify # Salt testing libs from salttesting import skipIf, TestCase from salttesting.helpers import destructiveTest, ensure_in_syspath -from salttesting.mock import NO_MOCK, NO_MOCK_REASON # Third-party libs try: @@ -26,7 +25,6 @@ ensure_in_syspath('../../') @skipIf(not HAS_PYINOTIFY, 'pyinotify is not available') -@skipIf(NO_MOCK, NO_MOCK_REASON) class INotifyBeaconTestCase(TestCase): ''' Test case for salt.beacons.inotify From be06df9b64b94e59f2570a96c5bfd6d44bc0ca8e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 9 Mar 2017 17:59:00 +0000 Subject: [PATCH 29/38] Remove `*args, **kwargs`. Not needed, not useful. --- tests/unit/beacons/inotify_beacon_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/beacons/inotify_beacon_test.py b/tests/unit/beacons/inotify_beacon_test.py index b4d32d8224..bb99363842 100644 --- a/tests/unit/beacons/inotify_beacon_test.py +++ b/tests/unit/beacons/inotify_beacon_test.py @@ -36,12 +36,12 @@ class INotifyBeaconTestCase(TestCase): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - def test_empty_config(self, *args, **kwargs): + def test_empty_config(self): config = {} ret = inotify.beacon(config) self.assertEqual(ret, []) - def test_file_open(self, *args, **kwargs): + def test_file_open(self): path = os.path.realpath(__file__) config = {path: {'mask': ['open']}} ret = inotify.beacon(config) @@ -54,7 +54,7 @@ class INotifyBeaconTestCase(TestCase): self.assertEqual(ret[0]['path'], path) self.assertEqual(ret[0]['change'], 'IN_OPEN') - def test_dir_no_auto_add(self, *args, **kwargs): + def test_dir_no_auto_add(self): config = {self.tmpdir: {'mask': ['create']}} ret = inotify.beacon(config) self.assertEqual(ret, []) @@ -70,7 +70,7 @@ class INotifyBeaconTestCase(TestCase): ret = inotify.beacon(config) self.assertEqual(ret, []) - def test_dir_auto_add(self, *args, **kwargs): + def test_dir_auto_add(self): config = {self.tmpdir: {'mask': ['create', 'open'], 'auto_add': True}} ret = inotify.beacon(config) self.assertEqual(ret, []) @@ -90,7 +90,7 @@ class INotifyBeaconTestCase(TestCase): self.assertEqual(ret[0]['path'], fp) self.assertEqual(ret[0]['change'], 'IN_OPEN') - def test_dir_recurse(self, *args, **kwargs): + def test_dir_recurse(self): dp1 = os.path.join(self.tmpdir, 'subdir1') os.mkdir(dp1) dp2 = os.path.join(dp1, 'subdir2') @@ -112,12 +112,12 @@ class INotifyBeaconTestCase(TestCase): self.assertEqual(ret[2]['path'], fp) self.assertEqual(ret[2]['change'], 'IN_OPEN') - def test_dir_recurse_auto_add(self, *args, **kwargs): + def test_dir_recurse_auto_add(self): dp1 = os.path.join(self.tmpdir, 'subdir1') os.mkdir(dp1) config = {self.tmpdir: {'mask': ['create', 'delete'], - 'recurse': True, - 'auto_add': True}} + 'recurse': True, + 'auto_add': True}} ret = inotify.beacon(config) self.assertEqual(ret, []) dp2 = os.path.join(dp1, 'subdir2') From c7fc09f97d000404724de9bf1670b09c06524cc6 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 9 Mar 2017 19:20:22 +0000 Subject: [PATCH 30/38] Support the new list configuration format. Refs #38121 Conflicts: - salt/beacons/status.py --- salt/beacons/status.py | 47 +++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/salt/beacons/status.py b/salt/beacons/status.py index 7d92e1fd7c..951f46e673 100644 --- a/salt/beacons/status.py +++ b/salt/beacons/status.py @@ -117,34 +117,43 @@ def beacon(config): ctime = datetime.datetime.utcnow().isoformat() if len(config) < 1: - config = { + config = [{ 'loadavg': ['all'], 'cpustats': ['all'], 'meminfo': ['all'], 'vmstats': ['all'], 'time': ['all'], - } + }] + + if not isinstance(config, list): + # To support the old dictionary config format + config = [config] ret = {} - for func in config: - try: - data = __salt__['status.{0}'.format(func)]() - except salt.exceptions.CommandExecutionError as exc: - log.debug('Status beacon attempted to process function {0} \ - but encountered error: {1}'.format(func, exc)) - continue - ret[func] = {} - for item in config[func]: - if item == 'all': - ret[func] = data + for entry in config: + for func in entry: + ret[func] = {} + try: + data = __salt__['status.{0}'.format(func)]() + except salt.exceptions.CommandExecutionError as exc: + log.debug('Status beacon attempted to process function {0} ' + 'but encountered error: {1}'.format(func, exc)) + continue + if not isinstance(entry[func], list): + func_items = [entry[func]] else: - try: + func_items = entry[func] + for item in func_items: + if item == 'all': + ret[func] = data + else: try: - ret[func][item] = data[item] - except TypeError: - ret[func][item] = data[int(item)] - except KeyError as exc: - ret[func] = 'Status beacon is incorrectly configured: {0}'.format(exc) + try: + ret[func][item] = data[item] + except TypeError: + ret[func][item] = data[int(item)] + except KeyError as exc: + ret[func] = 'Status beacon is incorrectly configured: {0}'.format(exc) return [{ 'tag': ctime, From dbaea3de73c7dd153c11f0fc6ff735419b98b05d Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 9 Mar 2017 13:28:37 -0700 Subject: [PATCH 31/38] Remove extra refresh reference that snuck in --- salt/modules/grains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/grains.py b/salt/modules/grains.py index dac1035589..ab230ca909 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -437,7 +437,7 @@ def delval(key, destructive=False, refresh=True): salt '*' grains.delval key ''' - setval(key, None, destructive=destructive, refresh=refresh) + setval(key, None, destructive=destructive) def ls(): # pylint: disable=C0103 From 4d0ddcd110ee2be8c305cc8d5663d77a56c2e140 Mon Sep 17 00:00:00 2001 From: hkrist Date: Thu, 9 Mar 2017 21:32:40 +0100 Subject: [PATCH 32/38] Fixed rawfile_json returner output format. It outputted python object instead of standard json. --- salt/returners/rawfile_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/returners/rawfile_json.py b/salt/returners/rawfile_json.py index c3d64bd717..0c61d85f78 100644 --- a/salt/returners/rawfile_json.py +++ b/salt/returners/rawfile_json.py @@ -58,7 +58,7 @@ def returner(ret): opts = _get_options({}) # Pass in empty ret, since this is a list of events try: with salt.utils.flopen(opts['filename'], 'a') as logfile: - logfile.write(str(ret)+'\n') + logfile.write(json.dumps(ret)+'\n') except: log.error('Could not write to rawdata_json file {0}'.format(opts['filename'])) raise From 4627c4ea6d5aacd9770fbff144f771746a7bdc23 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 9 Mar 2017 15:42:33 +0000 Subject: [PATCH 33/38] Code cleanup and make sure the beacons config file is deleted after testing Conflicts: - tests/integration/modules/beacons.py --- tests/integration/modules/beacons.py | 47 ++++++++++++++++------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/tests/integration/modules/beacons.py b/tests/integration/modules/beacons.py index 3e130bab10..2b3c9ac459 100644 --- a/tests/integration/modules/beacons.py +++ b/tests/integration/modules/beacons.py @@ -11,38 +11,32 @@ import os from salt.modules import beacons from salt.exceptions import CommandExecutionError import integration -import salt.utils # Salttesting libs from salttesting import skipIf from salttesting.helpers import destructiveTest, ensure_in_syspath + ensure_in_syspath('../../') beacons.__opts__ = {} -BEACON_CONF_DIR = os.path.join(integration.TMP, 'minion.d') -if not os.path.exists(BEACON_CONF_DIR): - os.makedirs(BEACON_CONF_DIR) - -IS_ADMIN = False -if salt.utils.is_windows(): - import salt.utils.win_functions - current_user = salt.utils.win_functions.get_current_user() - if current_user == 'SYSTEM': - IS_ADMIN = True - else: - IS_ADMIN = salt.utils.win_functions.is_admin(current_user) -else: - IS_ADMIN = os.geteuid() == 0 - - -@skipIf(not IS_ADMIN, 'You must be root to run these tests') -@destructiveTest class BeaconsAddDeleteTest(integration.ModuleCase): ''' Tests the add and delete functions ''' + def setUp(self): + self.minion_conf_d_dir = os.path.join( + self.minion_opts['config_dir'], + os.path.dirname(self.minion_opts['default_include'])) + if not os.path.isdir(self.minion_conf_d_dir): + os.makedirs(self.minion_conf_d_dir) + self.beacons_config_file_path = os.path.join(self.minion_conf_d_dir, 'beacons.conf') + + def tearDown(self): + if os.path.isfile(self.beacons_config_file_path): + os.unlink(self.beacons_config_file_path) + def test_add_and_delete(self): ''' Test adding and deleting a beacon @@ -62,13 +56,26 @@ class BeaconsAddDeleteTest(integration.ModuleCase): self.run_function('beacons.save') -@skipIf(not IS_ADMIN, 'You must be root to run these tests') @destructiveTest class BeaconsTest(integration.ModuleCase): ''' Tests the beacons execution module ''' + beacons_config_file_path = minion_conf_d_dir = None + + @classmethod + def tearDownClass(cls): + if os.path.isfile(cls.beacons_config_file_path): + os.unlink(cls.beacons_config_file_path) + def setUp(self): + if self.minion_conf_d_dir is None: + self.minion_conf_d_dir = os.path.join( + self.minion_opts['config_dir'], + os.path.dirname(self.minion_opts['default_include'])) + if not os.path.isdir(self.minion_conf_d_dir): + os.makedirs(self.minion_conf_d_dir) + self.__class__.beacons_config_file_path = os.path.join(self.minion_conf_d_dir, 'beacons.conf') try: # Add beacon to disable self.run_function('beacons.add', ['ps', [{'apache2': 'stopped'}]]) From e7b9a45079b4cfda01b88c3245cd144053c70dbc Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Thu, 9 Mar 2017 14:20:35 -0700 Subject: [PATCH 34/38] Correct comment lines output got list_hosts --- salt/modules/hosts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/hosts.py b/salt/modules/hosts.py index b8a3c0dd1b..0d7162c6ad 100644 --- a/salt/modules/hosts.py +++ b/salt/modules/hosts.py @@ -53,7 +53,7 @@ def _list_hosts(): if not line: continue if line.startswith('#'): - ret.setdefault('comment-{0}'.format(count), []).extend(line) + ret.setdefault('comment-{0}'.format(count), []).append(line) count += 1 continue if '#' in line: From 4a52cca926e7cac893f863c903812798fa70add1 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 9 Mar 2017 14:55:33 -0700 Subject: [PATCH 35/38] Pylint fixes --- tests/integration/modules/beacons.py | 1 + tests/unit/beacons/inotify_beacon_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/modules/beacons.py b/tests/integration/modules/beacons.py index 2b3c9ac459..e04aa92dd6 100644 --- a/tests/integration/modules/beacons.py +++ b/tests/integration/modules/beacons.py @@ -21,6 +21,7 @@ ensure_in_syspath('../../') beacons.__opts__ = {} + class BeaconsAddDeleteTest(integration.ModuleCase): ''' Tests the add and delete functions diff --git a/tests/unit/beacons/inotify_beacon_test.py b/tests/unit/beacons/inotify_beacon_test.py index bb99363842..6e5e4daa6c 100644 --- a/tests/unit/beacons/inotify_beacon_test.py +++ b/tests/unit/beacons/inotify_beacon_test.py @@ -11,7 +11,7 @@ from salt.beacons import inotify # Salt testing libs from salttesting import skipIf, TestCase -from salttesting.helpers import destructiveTest, ensure_in_syspath +from salttesting.helpers import ensure_in_syspath # Third-party libs try: From e4aef54c73d99f34d800d13269c92152e1addfb4 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Thu, 9 Mar 2017 15:31:16 -0700 Subject: [PATCH 36/38] Add special token to insert the minion id into the default_include path --- doc/ref/configuration/minion.rst | 11 +++++++++++ salt/config/__init__.py | 4 ++++ salt/utils/schedule.py | 3 --- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index ac3fb0e485..cded829e40 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -2110,6 +2110,17 @@ file. files are prefixed with an underscore. A common example of this is the ``_schedule.conf`` file. +.. note:: + + The configuration system supports adding the special token ``{id}`` to this + option. At startup ``{id}`` will be replaced by the minion's ID, and the + default_include directory will be set here. For example, if the minion's + ID is 'webserver' and ``default_include`` is set to ``minion.d/{id}/*.conf`` + then the default_include directive will be set to ``minion.d/webserver/*.conf``. + This is for situations when there are multiple minions or proxy minions + running on a single machine that need different configurations, specifically for + their schedulers. + ``include`` ----------- diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 468e0481ba..ff5a348a9d 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1930,6 +1930,7 @@ def minion_config(path, overrides = load_config(path, env_var, DEFAULT_MINION_OPTS['conf_file']) default_include = overrides.get('default_include', defaults['default_include']) + include = overrides.get('include', []) overrides.update(include_config(default_include, path, verbose=False, @@ -3162,6 +3163,9 @@ def apply_minion_config(overrides=None, newdirectory = os.path.join(opts[directory], opts['id']) opts[directory] = newdirectory + if 'default_include' in overrides and '{id}' in overrides['default_include']: + opts['default_include'] = overrides['default_include'].replace('{id}', opts['id']) + # pidfile can be in the list of append_minionid_config_dirs, but pidfile # is the actual path with the filename, not a directory. if 'pidfile' in opts.get('append_minionid_config_dirs', []): diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 20ccbc562e..7d26612aee 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -443,9 +443,6 @@ class Schedule(object): config_dir, os.path.dirname(self.opts.get('default_include', salt.config.DEFAULT_MINION_OPTS['default_include']))) - if salt.utils.is_proxy(): - # each proxy will have a separate _schedule.conf file - minion_d_dir = os.path.join(minion_d_dir, self.opts['proxyid']) if not os.path.isdir(minion_d_dir): os.makedirs(minion_d_dir) From d989d749d6a700a17a28f7bc85a4ab842410108d Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 10 Mar 2017 16:05:42 +0200 Subject: [PATCH 37/38] Fix #7997: describe how to upgrade Salt Minion in a proper way --- doc/faq.rst | 143 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 87 insertions(+), 56 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 87600164a1..f7e80d697a 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -168,6 +168,7 @@ information. Module ``X`` isn't available, even though the shell command it uses is installed. Why? -------------------------------------------------------------------------------------- + This is most likely a PATH issue. Did you custom-compile the software which the module requires? RHEL/CentOS/etc. in particular override the root user's path in ``/etc/init.d/functions``, setting it to ``/sbin:/usr/sbin:/bin:/usr/bin``, @@ -246,86 +247,116 @@ specifying the pillar variable is the same one used for :py:func:`pillar.get ` state is only supported in Salt 2015.8.4 and newer. -What is the best way to restart a Salt daemon using Salt? ---------------------------------------------------------- +What is the best way to restart a Salt Minion daemon using Salt after upgrade? +------------------------------------------------------------------------------ -Updating the salt-minion package requires a restart of the salt-minion service. -But restarting the service while in the middle of a state run interrupts the -process of the minion running states and sending results back to the master. -It's a tricky problem to solve, and we're working on it, but in the meantime -one way of handling this (on Linux and UNIX-based operating systems) is to use -**at** (a job scheduler which predates cron) to schedule a restart of the -service. **at** is not installed by default on most distros, and requires a -service to be running (usually called **atd**) in order to schedule jobs. -Here's an example of how to upgrade the salt-minion package at the end of a -Salt run, and schedule a service restart for one minute after the package -update completes. +Updating the ``salt-minion`` package requires a restart of the ``salt-minion`` +service. But restarting the service while in the middle of a state run +interrupts the process of the Minion running states and sending results back to +the Master. A common way to workaround that is to schedule restarting of the +Minion service using :ref:`masterless mode ` after all +other states have been applied. This allows to keep Minion to Master connection +alive for the Minion to report the final results to the Master, while the +service is restarting in the background. -Linux/Unix -********** +Upgrade without automatic restart +********************************* + +Doing the Minion upgrade seems to be a simplest state in your SLS file at +first. But the operating systems such as Debian GNU/Linux, Ununtu and their +derivatives start the service after the package installation by default. +To prevent this, we need to create policy layer which will prevent the Minion +service to restart right after the upgrade: .. code-block:: yaml - salt-minion: + {%- if grains['os_family'] == 'Debian' %} + + Disable starting services: + file.managed: + - name: /usr/sbin/policy-rc.d + - user: root + - group: root + - mode: 0755 + - contents: + - '#!/bin/sh' + - exit 101 + # do not touch if already exists + - replace: False + - prereq: + - pkg: Upgrade Salt Minion + + {%- endif %} + + Upgrade Salt Minion: pkg.installed: - name: salt-minion - - version: 2014.1.7-3.el6 + - version: 2016.11.3{% if grains['os_family'] == 'Debian' %}+ds-1{% endif %} - order: last - service.running: + + Enable Salt Minion: + service.enabled: - name: salt-minion - require: - - pkg: salt-minion - cmd.run: - - name: echo service salt-minion restart | at now + 1 minute + - pkg: Upgrade Salt Minion + + {%- if grains['os_family'] == 'Debian' %} + + Enable starting services: + file.absent: + - name: /usr/sbin/policy-rc.d - onchanges: - - pkg: salt-minion + - pkg: Upgrade Salt Minion -To ensure that **at** is installed and **atd** is running, the following states -can be used (be sure to double-check the package name and service name for the -distro the minion is running, in case they differ from the example below. + {%- endif %} + +Restart using states +******************** + +Now we can apply the workaround to restart the Minion in reliable way. +The following example works on both UNIX-like and Windows operating systems: .. code-block:: yaml - at: - pkg.installed: - - name: at - service.running: - - name: atd - - enable: True - -An alternative to using the :program:`atd` daemon is to fork and disown the -process. - -.. code-block:: yaml - - restart_minion: + Restart Salt Minion: cmd.run: - - name: | + {%- if grains['kernel'] == 'Windows' %} + - name: 'C:\salt\salt-call.bat --local service.restart salt-minion' + {%- else %} + - name: 'salt-call --local service.restart salt-minion' + {%- endif %} + - bg: True + - onchanges: + - pkg: Upgrade Salt Minion + +However, it requires more advanced tricks to upgrade from legacy version of +Salt (before ``2016.3.0``), where executing commands in the background is not +supported: + +.. code-block:: yaml + + Restart Salt Minion: + cmd.run: + {%- if grains['kernel'] == 'Windows' %} + - name: 'start powershell "Restart-Service -Name salt-minion"' + {%- else %} + # fork and disown the process + - name: |- exec 0>&- # close stdin exec 1>&- # close stdout exec 2>&- # close stderr - nohup /bin/sh -c 'sleep 10 && salt-call --local service.restart salt-minion' & - - python_shell: True - - order: last + nohup salt-call --local service.restart salt-minion & + {%- endif %} -Windows -******* +Restart using remote executions +******************************* -For Windows machines, restarting the minion can be accomplished using the -following state: - -.. code-block:: yaml - - schedule-start: - cmd.run: - - name: 'start powershell "Restart-Service -Name salt-minion"' - - order: last - -or running immediately from the command line: +Restart the Minion from the command line: .. code-block:: bash - salt -G kernel:Windows cmd.run 'start powershell "Restart-Service -Name salt-minion"' + salt -G kernel:Windows cmd.run_bg 'C:\salt\salt-call.bat --local service.restart salt-minion' + salt -C 'not G@kernel:Windows' cmd.run_bg 'salt-call --local service.restart salt-minion' Salting the Salt Master ----------------------- From 5fcea05691c5f707a2480a65e3f1818cb85b34c4 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 10 Mar 2017 13:24:39 -0700 Subject: [PATCH 38/38] Mention bot delay disable for 2016.11 --- .mention-bot | 2 -- 1 file changed, 2 deletions(-) diff --git a/.mention-bot b/.mention-bot index c3181cdc9f..1c9a214fd5 100644 --- a/.mention-bot +++ b/.mention-bot @@ -1,7 +1,5 @@ { "skipTitle": "Merge forward", - "delayed": true, - "delayedUntil": "2h", "userBlacklist": [] }