From 1fd9c716d9b566a45869e43ad8310e98b9c32579 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 21 Dec 2016 19:49:32 -0600 Subject: [PATCH 1/6] Add pillarenv_from_saltenv minion config parameter --- doc/ref/configuration/minion.rst | 17 +++++++++++++++++ salt/config/__init__.py | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index a88965e035..4240ce0c8e 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -1599,6 +1599,23 @@ the environment setting, but for pillar instead of states. pillarenv: None +.. conf_minion:: pillarenv_from_saltenv + +``pillarenv_from_saltenv`` +-------------------------- + +Default: ``False`` + +When set to ``True``, the :conf_minion:`pillarenv` value will assume the value +of the effective saltenv when running states. This essentially makes ``salt '*' +state.sls mysls saltenv=dev`` equivalent to ``salt '*' state.sls mysls +saltenv=dev pillarenv=dev``. If :conf_minion:`pillarenv` is set, either in the +minion config file or via the CLI, it will override this option. + +.. code-block:: yaml + + pillarenv_from_saltenv: True + .. conf_minion:: pillar_raise_on_missing ``pillar_raise_on_missing`` diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 55255e7839..1bba90e695 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -224,6 +224,9 @@ VALID_OPTS = { # Force the minion into a single pillar root when it fetches pillar data from the master 'pillarenv': str, + # Make the pillarenv always match the effective saltenv + 'pillarenv_from_saltenv': bool, + # Allows a user to provide an alternate name for top.sls 'state_top': str, @@ -1001,6 +1004,7 @@ DEFAULT_MINION_OPTS = { 'autoload_dynamic_modules': True, 'environment': None, 'pillarenv': None, + 'pillarenv_from_saltenv': False, 'pillar_opts': False, # ``pillar_cache``, ``pillar_cache_ttl`` and ``pillar_cache_backend`` # are not used on the minion but are unavoidably in the code path From f66d2a960399f7070f5ad33b98550cdfbeadf22b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 21 Dec 2016 19:50:08 -0600 Subject: [PATCH 2/6] Add pillarenv_from_saltenv to salt.pillar --- salt/pillar/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 784dc37dd6..f15ddbbe79 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -78,8 +78,12 @@ class AsyncRemotePillar(object): self.grains = grains self.minion_id = minion_id self.channel = salt.transport.client.AsyncReqChannel.factory(opts) - if pillarenv is not None or 'pillarenv' not in self.opts: + if pillarenv is not None: self.opts['pillarenv'] = pillarenv + elif self.opts.get('pillarenv_from_saltenv', False): + self.opts['pillarenv'] = saltenv + elif 'pillarenv' not in self.opts: + self.opts['pillarenv'] = None self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -131,8 +135,12 @@ class RemotePillar(object): self.grains = grains self.minion_id = minion_id self.channel = salt.transport.Channel.factory(opts) - if pillarenv is not None or 'pillarenv' not in self.opts: + if pillarenv is not None: self.opts['pillarenv'] = pillarenv + elif self.opts.get('pillarenv_from_saltenv', False): + self.opts['pillarenv'] = saltenv + elif 'pillarenv' not in self.opts: + self.opts['pillarenv'] = None self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): From 475981f9d0d414664cdecff3bd958689219e4e49 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 21 Dec 2016 19:53:53 -0600 Subject: [PATCH 3/6] Add pillarenv support to state.top Also fix some incorrect saltenv/pillarenv behavior in some of the other funcs, and expand documentation for functions that support pillarenv but contain no documentation that it is supported. --- salt/modules/state.py | 113 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 14 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index e4523d3e92..431ba09387 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -511,7 +511,7 @@ def apply_(mods=None, multiple state runs can safely be run at the same time. Do *not* use this flag for performance optimization. - saltenv : None + saltenv Specify a salt fileserver environment to be used when applying states .. versionchanged:: 0.17.0 @@ -525,8 +525,11 @@ def apply_(mods=None, saltenv was not explicitly set. pillarenv - Specify a Pillar environment to be used when applying states. By - default, all Pillar environments will be merged together and used. + Specify a Pillar environment to be used when applying states. This + can also be set in the minion config file using the + :conf_minion:`pillarenv` option. When neither the + :conf_minion:`pillarenv` minion config option nor this CLI argument is + used, all Pillar environments will be merged together. localconfig Optionally, instead of using the minion config, load minion opts from @@ -711,6 +714,26 @@ def highstate(test=None, .. versionadded:: 2016.3.0 + saltenv + Specify a salt fileserver environment to be used when applying states + + .. versionchanged:: 0.17.0 + Argument name changed from ``env`` to ``saltenv``. + + .. versionchanged:: 2014.7.0 + If no saltenv is specified, the minion config will be checked for a + ``saltenv`` parameter and if found, it will be used. If none is + found, ``base`` will be used. In prior releases, the minion config + was not checked and ``base`` would always be assumed when the + saltenv was not explicitly set. + + pillarenv + Specify a Pillar environment to be used when applying states. This + can also be set in the minion config file using the + :conf_minion:`pillarenv` option. When neither the + :conf_minion:`pillarenv` minion config option nor this CLI argument is + used, all Pillar environments will be merged together. + queue : False Instead of failing immediately when another state run is in progress, queue the new state run to begin running once the other has finished. @@ -774,6 +797,9 @@ def highstate(test=None, if 'saltenv' in kwargs: opts['environment'] = kwargs['saltenv'] + if 'pillarenv' in kwargs: + opts['pillarenv'] = kwargs['pillarenv'] + pillar = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ @@ -784,9 +810,6 @@ def highstate(test=None, 'is specified.' ) - if 'pillarenv' in kwargs: - opts['pillarenv'] = kwargs['pillarenv'] - try: st_ = salt.state.HighState(opts, pillar, @@ -881,7 +904,7 @@ def sls(mods, multiple state runs can safely be run at the same time. Do *not* use this flag for performance optimization. - saltenv : None + saltenv Specify a salt fileserver environment to be used when applying states .. versionchanged:: 0.17.0 @@ -895,9 +918,11 @@ def sls(mods, saltenv was not explicitly set. pillarenv - - Specify a Pillar environment to be used when applying states. By - default, all Pillar environments will be merged together and used. + Specify a Pillar environment to be used when applying states. This + can also be set in the minion config file using the + :conf_minion:`pillarenv` option. When neither the + :conf_minion:`pillarenv` minion config option nor this CLI argument is + used, all Pillar environments will be merged together. localconfig @@ -1076,12 +1101,32 @@ def top(topfn, test=None, queue=False, saltenv=None, + pillarenv=None, **kwargs): ''' Execute a specific top file instead of the default. This is useful to apply configurations from a different environment (for example, dev or prod), without modifying the default top file. + queue : False + Instead of failing immediately when another state run is in progress, + queue the new state run to begin running once the other has finished. + + This option starts a new thread for each queued state run, so use this + option sparingly. + + saltenv + Specify a salt fileserver environment to be used when applying states + + pillarenv + Specify a Pillar environment to be used when applying states. This + can also be set in the minion config file using the + :conf_minion:`pillarenv` option. When neither the + :conf_minion:`pillarenv` minion config option nor this CLI argument is + used, all Pillar environments will be merged together. + + .. versionadded:: Nitrogen + CLI Example: .. code-block:: bash @@ -1102,6 +1147,12 @@ def top(topfn, opts = _get_opts(kwargs.get('localconfig')) opts['test'] = _get_test_value(test, **kwargs) + if saltenv is not None: + opts['environment'] = saltenv + + if pillarenv is not None: + opts['pillarenv'] = pillarenv + pillar = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ @@ -1235,6 +1286,7 @@ def sls_id( id_, mods, saltenv='base', + pillarenv=None, test=None, queue=False, **kwargs): @@ -1251,6 +1303,16 @@ def sls_id( .. versionadded:: 2014.7.0 + saltenv : base + Specify a salt fileserver environment to be used when applying states + + pillarenv + Specify a Pillar environment to be used when applying states. This + can also be set in the minion config file using the + :conf_minion:`pillarenv` option. When neither the + :conf_minion:`pillarenv` minion config option nor this CLI argument is + used, all Pillar environments will be merged together. + CLI Example: .. code-block:: bash @@ -1265,8 +1327,9 @@ def sls_id( orig_test = __opts__.get('test', None) opts = _get_opts(kwargs.get('localconfig')) opts['test'] = _get_test_value(test, **kwargs) - if 'pillarenv' in kwargs: - opts['pillarenv'] = kwargs['pillarenv'] + opts['environment'] = saltenv + if pillarenv is not None: + opts['pillarenv'] = pillarenv try: st_ = salt.state.HighState(opts, proxy=__proxy__) except NameError: @@ -1302,6 +1365,7 @@ def sls_id( def show_low_sls(mods, saltenv='base', + pillarenv=None, test=None, queue=False, **kwargs): @@ -1309,6 +1373,16 @@ def show_low_sls(mods, Display the low data from a specific sls. The default environment is ``base``, use ``saltenv`` to specify a different environment. + saltenv + Specify a salt fileserver environment to be used when applying states + + pillarenv + Specify a Pillar environment to be used when applying states. This + can also be set in the minion config file using the + :conf_minion:`pillarenv` option. When neither the + :conf_minion:`pillarenv` minion config option nor this CLI argument is + used, all Pillar environments will be merged together. + CLI Example: .. code-block:: bash @@ -1330,8 +1404,9 @@ def show_low_sls(mods, orig_test = __opts__.get('test', None) opts = _get_opts(kwargs.get('localconfig')) opts['test'] = _get_test_value(test, **kwargs) - if 'pillarenv' in kwargs: - opts['pillarenv'] = kwargs['pillarenv'] + opts['environment'] = saltenv + if pillarenv is not None: + opts['pillarenv'] = pillarenv st_ = salt.state.HighState(opts) if isinstance(mods, six.string_types): mods = mods.split(',') @@ -1362,6 +1437,16 @@ def show_sls(mods, saltenv='base', test=None, queue=False, **kwargs): Custom Pillar data can be passed with the ``pillar`` kwarg. + saltenv + Specify a salt fileserver environment to be used when applying states + + pillarenv + Specify a Pillar environment to be used when applying states. This + can also be set in the minion config file using the + :conf_minion:`pillarenv` option. When neither the + :conf_minion:`pillarenv` minion config option nor this CLI argument is + used, all Pillar environments will be merged together. + CLI Example: .. code-block:: bash From 4049bf7288951ee22790e56527ed57b729e12dd6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 21 Dec 2016 19:54:48 -0600 Subject: [PATCH 4/6] Add pillarenv_from_saltenv to Nitrogen release notes --- doc/topics/releases/nitrogen.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/topics/releases/nitrogen.rst b/doc/topics/releases/nitrogen.rst index 14a32d7fe9..f07f19251c 100644 --- a/doc/topics/releases/nitrogen.rst +++ b/doc/topics/releases/nitrogen.rst @@ -60,8 +60,22 @@ Execution Module Changes Master Configuration Additions ============================== -- ``syndic_forward_all_events``: Option on multi-syndic or single when connected - to multiple masters to be able to send events to all connected masters. +- :conf_master:`syndic_forward_all_events` - Option on multi-syndic or single + when connected to multiple masters to be able to send events to all connected + masters. + +Minion Configuration Additions +============================== + +- :conf_minion:`pillarenv_from_saltenv` - When set to ``True`` (default is + ``False``), the :conf_minion:`pillarenv` option will take the same value as + the effective saltenv when running states. This would allow a user to run + ``salt '*' state.apply mysls saltenv=dev``, and the SLS for both the state + and pillar data would be sourced from the ``dev`` environment, essentially + the equivalent of running ``salt '*' state.apply mysls saltenv=dev + pillarenv=dev``. Note that if :conf_minion:`pillarenv` is set in the minion + config file, or if ``pillarenv`` is provided on the CLI, it will override + this option. Python API Changes ================== From d744ad7f46894e7977862875a49877587a8bcbe4 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 21 Dec 2016 22:34:28 -0600 Subject: [PATCH 5/6] Make pillar.items support pillarenv and pillarenv_from_saltenv --- salt/modules/pillar.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index f442757486..7133be980c 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import collections # Import third party libs +import copy import os import yaml import salt.ext.six as six @@ -119,11 +120,13 @@ def items(*args, **kwargs): .. versionadded:: 2015.5.0 saltenv + Included only for compatibility with + :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored. + + .. versionadded:: Nitrogen + + pillarenv Pass a specific pillar environment from which to compile pillar data. - If unspecified, the minion's :conf_minion:`environment` option is used, - and if that also is not specified then all configured pillar - environments will be merged into a single pillar dictionary and - returned. .. versionadded:: Nitrogen @@ -137,11 +140,20 @@ def items(*args, **kwargs): if args: return item(*args) + pillarenv = kwargs.get('pillarenv') + if pillarenv is None: + if __opts__.get('pillarenv_from_saltenv', False): + pillarenv = kwargs.get('saltenv') or __opts__['environment'] + else: + pillarenv = __opts__.get('pillarenv') + + opts = copy.copy(__opts__) + opts['pillarenv'] = pillarenv pillar = salt.pillar.get_pillar( - __opts__, + opts, __grains__, - __opts__['id'], - kwargs.get('saltenv') or __opts__['environment'], + opts['id'], + saltenv=pillarenv, pillar=kwargs.get('pillar')) return pillar.compile_pillar() From dde66d3705f66aa6af57a407ace0a2bcb2f56fa2 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 22 Dec 2016 01:02:44 -0600 Subject: [PATCH 6/6] Add pillarenv support to pillar.get --- salt/modules/pillar.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index 7133be980c..75f3cd9f03 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -26,7 +26,8 @@ def get(key, default=KeyError, merge=False, delimiter=DEFAULT_TARGET_DELIM, - saltenv=None): + saltenv=None, + pillarenv=None): ''' .. versionadded:: 0.14 @@ -60,6 +61,12 @@ def get(key, .. versionadded:: 2014.7.0 saltenv + Included only for compatibility with + :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored. + + .. versionadded:: Nitrogen + + pillarenv If specified, this function will query the master to generate fresh pillar data on the fly, specifically from the requested pillar environment. Note that this can produce different pillar data than @@ -85,7 +92,9 @@ def get(key, if default is KeyError: default = '' opt_merge_lists = __opts__.get('pillar_merge_lists', False) - pillar_dict = __pillar__ if saltenv is None else items(saltenv=saltenv) + pillar_dict = __pillar__ \ + if all(x is None for x in (saltenv, pillarenv)) \ + else items(saltenv=saltenv, pillarenv=pillarenv) if merge: ret = salt.utils.traverse_dict_and_list(pillar_dict, key, {}, delimiter)