From abab6fd91c4080d596930445a56566918b9e411d Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 1 Jun 2017 12:55:01 +0000 Subject: [PATCH 1/6] Override minion opts with pillar data --- salt/minion.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index c9cfc6cb1f..428b84bb85 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -3118,6 +3118,9 @@ class ProxyMinion(Minion): if 'proxy' not in self.opts: self.opts['proxy'] = self.opts['pillar']['proxy'] + # update opts, to override data with pillar data + self.opts.update(self.opts['pillar']) + fq_proxyname = self.opts['proxy']['proxytype'] # Need to load the modules so they get all the dunder variables From fd499887f9903ddf530063bdf3c0ec4c2d185110 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 5 Jul 2017 10:14:50 +0000 Subject: [PATCH 2/6] Define new proxy merge pillar in opts... opts --- salt/config/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 0f06f9ccca..c100741347 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -569,6 +569,20 @@ VALID_OPTS = { # False in 2016.3.0 'add_proxymodule_to_opts': bool, + # Merge pillar data into configuration opts. + # As multiple proxies can run on the same server, we may need different + # configuration options for each, while there's one single configuration file. + # The solution is merging the pillar data of each proxy minion into the opts. + 'proxy_merge_pillar_in_opts': bool, + + # Deep merge of pillar data into configuration opts. + # Evaluated only when `proxy_merge_pillar_in_opts` is True. + 'proxy_deep_merge_pillar_in_opts': bool, + + # The strategy used when merging pillar into opts. + # Considered only when `proxy_merge_pillar_in_opts` is True. + 'proxy_merge_pillar_in_opts_strategy': str, + # In some particular cases, always alive proxies are not beneficial. # This option can be used in those less dynamic environments: # the user can request the connection @@ -1637,6 +1651,10 @@ DEFAULT_PROXY_MINION_OPTS = { 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include', 'extension_modules'], 'default_include': 'proxy.d/*.conf', + 'proxy_merge_pillar_in_opts': False, + 'proxy_deep_merge_pillar_in_opts': False, + 'proxy_merge_pillar_in_opts_strategy': 'smart', + # By default, proxies will preserve the connection. # If this option is set to False, # the connection with the remote dumb device From 96b31d5643bb9bffc9f2c913c356c8643664555c Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 5 Jul 2017 10:27:49 +0000 Subject: [PATCH 3/6] Override proxy opts with pillar data when required --- salt/minion.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 428b84bb85..f57e61f2f3 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -100,6 +100,7 @@ import salt.defaults.exitcodes import salt.cli.daemons import salt.log.setup +import salt.utils.dictupdate from salt.config import DEFAULT_MINION_OPTS from salt.defaults import DEFAULT_TARGET_DELIM from salt.executors import FUNCTION_EXECUTORS @@ -3118,8 +3119,11 @@ class ProxyMinion(Minion): if 'proxy' not in self.opts: self.opts['proxy'] = self.opts['pillar']['proxy'] - # update opts, to override data with pillar data - self.opts.update(self.opts['pillar']) + if self.opts.get('proxy_merge_pillar_in_opts'): + self.opts = salt.utils.dictupdate.merge(self.opts, + self.opts['pillar'], + strategy=self.opts.get('proxy_merge_pillar_in_opts_strategy'), + merge_lists=self.opts.get('proxy_deep_merge_pillar_in_opts', False)) fq_proxyname = self.opts['proxy']['proxytype'] From 732b63b0b945430b759431831bde7b716d2633a8 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 5 Jul 2017 10:52:27 +0000 Subject: [PATCH 4/6] Merge mine details whenever possible --- salt/minion.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index f57e61f2f3..a6fb6a2797 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -3120,10 +3120,24 @@ class ProxyMinion(Minion): self.opts['proxy'] = self.opts['pillar']['proxy'] if self.opts.get('proxy_merge_pillar_in_opts'): + # Override proxy opts with pillar data when the user required. self.opts = salt.utils.dictupdate.merge(self.opts, self.opts['pillar'], strategy=self.opts.get('proxy_merge_pillar_in_opts_strategy'), merge_lists=self.opts.get('proxy_deep_merge_pillar_in_opts', False)) + else: + # Even when not required, some details such as mine configuration + # should be merged anyway whenever possible. + if 'mine_interval' in self.opts['pillar']: + self.opts['mine_interval'] = self.opts['pillar']['mine_interval'] + if 'mine_functions' in self.opts['pillar']: + general_proxy_mines = self.opts.get('mine_functions', []) + specific_proxy_mines = self.opts['pillar']['mine_functions'] + try: + self.opts['mine_functions'] = general_proxy_mines + specific_proxy_mines + except TypeError as terr: + log.error('Unable to merge mine functions from the pillar in the opts, for proxy {}'.format( + self.opts['id'])) fq_proxyname = self.opts['proxy']['proxytype'] From cdc0d9674a3d2bd9e40bbe5d0fda39686c5f8f93 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 5 Jul 2017 10:54:45 +0000 Subject: [PATCH 5/6] Allow disabling the mines details merge --- salt/config/__init__.py | 5 +++++ salt/minion.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index c100741347..5216323b5c 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -583,6 +583,9 @@ VALID_OPTS = { # Considered only when `proxy_merge_pillar_in_opts` is True. 'proxy_merge_pillar_in_opts_strategy': str, + # Allow enabling mine details using pillar data. + 'proxy_mines_pillar': bool, + # In some particular cases, always alive proxies are not beneficial. # This option can be used in those less dynamic environments: # the user can request the connection @@ -1655,6 +1658,8 @@ DEFAULT_PROXY_MINION_OPTS = { 'proxy_deep_merge_pillar_in_opts': False, 'proxy_merge_pillar_in_opts_strategy': 'smart', + 'proxy_mines_pillar': True, + # By default, proxies will preserve the connection. # If this option is set to False, # the connection with the remote dumb device diff --git a/salt/minion.py b/salt/minion.py index a6fb6a2797..6b7c82a8d7 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -3125,7 +3125,7 @@ class ProxyMinion(Minion): self.opts['pillar'], strategy=self.opts.get('proxy_merge_pillar_in_opts_strategy'), merge_lists=self.opts.get('proxy_deep_merge_pillar_in_opts', False)) - else: + elif self.opts.get('proxy_mines_pillar'): # Even when not required, some details such as mine configuration # should be merged anyway whenever possible. if 'mine_interval' in self.opts['pillar']: From aad39ba665b7bd85c4ed54d3ee8f896ed5b9b008 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Mon, 4 Sep 2017 14:17:14 +0000 Subject: [PATCH 6/6] Document the new opts --- doc/ref/configuration/proxy.rst | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/doc/ref/configuration/proxy.rst b/doc/ref/configuration/proxy.rst index 974e0af890..e55f3fc01b 100644 --- a/doc/ref/configuration/proxy.rst +++ b/doc/ref/configuration/proxy.rst @@ -118,3 +118,53 @@ has to be closed after every command. .. code-block:: yaml proxy_always_alive: False + +``proxy_merge_pillar_in_opts`` +------------------------------ + +.. versionadded:: 2017.7.3 + +Default: ``False``. + +Wheter the pillar data to be merged into the proxy configuration options. +As multiple proxies can run on the same server, we may need different +configuration options for each, while there's one single configuration file. +The solution is merging the pillar data of each proxy minion into the opts. + +.. code-block:: yaml + + proxy_merge_pillar_in_opts: True + +``proxy_deep_merge_pillar_in_opts`` +----------------------------------- + +.. versionadded:: 2017.7.3 + +Default: ``False``. + +Deep merge of pillar data into configuration opts. +This option is evaluated only when :conf_proxy:`proxy_merge_pillar_in_opts` is +enabled. + +``proxy_merge_pillar_in_opts_strategy`` +--------------------------------------- + +.. versionadded:: 2017.7.3 + +Default: ``smart``. + +The strategy used when merging pillar configuration into opts. +This option is evaluated only when :conf_proxy:`proxy_merge_pillar_in_opts` is +enabled. + +``proxy_mines_pillar`` +---------------------- + +.. versionadded:: 2017.7.3 + +Default: ``True``. + +Allow enabling mine details using pillar data. This evaluates the mine +configuration under the pillar, for the following regular minion options that +are also equally available on the proxy minion: :conf_minion:`mine_interval`, +and :conf_minion:`mine_functions`.