valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-06 16:45:27 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #48809 from cro/matcher_in_loader2

Browse Source 
Matcher in loader, take 4
This commit is contained in:
Nicole Thomas 2018-09-26 09:22:39 -04:00 committed by GitHub
commit 1db2d2e4d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 1022 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
doc/topics/matchers/index.rst Normal file
View File

@ -0,0 +1,81 @@
.. _matchers:
========
Matchers
========
.. versionadded:: Flourine
Matchers are modules that provide Salt's targeting abilities. As of the
Flourine release, matchers can be dynamically loaded. Currently new matchers
cannot be created because the required plumbing for the CLI does not exist yet.
Existing matchers may have their functionality altered or extended.
For details of targeting methods, see the :ref:`Targeting <targeting>` topic.
A matcher module must have a function called ``match()``. This function ends up
becoming a method on the Matcher class. All matcher functions require at least
two arguments, ``self`` (because the function will be turned into a method), and
``tgt``, which is the actual target string. The grains and pillar matchers also
take a ``delimiter`` argument and should default to ``DEFAULT_TARGET_DELIM``.
Like other Salt loadable modules, modules that override built-in functionality
can be placed in ``file_roots`` in a special directory and then copied to the
minion through the normal sync process. :py:func:`saltutil.sync_all <salt.modules.saltutil.sync_all>`
will transfer all loadable modules, and the Flourine release introduces
:py:func:`saltutil.sync_matchers <salt.modules.saltutil.sync_matchers>`. For matchers, the directory is
``/srv/salt/_matchers`` (assuming your ``file_roots`` is set to the default
``/srv/salt``).
As an example, let's modify the ``list`` matcher to have the separator be a
'``/``' instead of the default '``,``'.
.. code-block:: python
from __future__ import absolute_import, print_function, unicode_literals
from salt.ext import six # pylint: disable=3rd-party-module-not-gated
def match(self, tgt):
'''
Determines if this host is on the list
'''
if isinstance(tgt, six.string_types):
# The stock matcher splits on `,`. Change to `/` below.
tgt = tgt.split('/')
return bool(self.opts['id'] in tgt)
Place this code in a file called ``list_matcher.py`` in ``_matchers`` in your
``file_roots``. Sync this down to your minions with
:py:func:`saltutil.sync_matchers <salt.modules.saltutil.sync_matchers>`.
Then attempt to match with the following, replacing ``minionX`` with three of your minions.
.. code-block:: shell
salt -L 'minion1/minion2/minion3' test.ping
Three of your minions should respond.
The current supported matchers and associated filenames are
=============== ====================== ===================
Salt CLI Switch Match Type Filename
=============== ====================== ===================
<none> Glob glob_match.py
-C Compound compound_match.py
-E Perl-Compatible pcre_match.py
Regular Expressions
-L List list_match.py
-G Grain grain_match.py
-P Grain Perl-Compatible grain_pcre_match.py
Regular Expressions
-N Nodegroup nodegroup_match.py
-R Range range_match.py
-I Pillar pillar_match.py
-J Pillar Perl-Compatible pillar_pcre.py
Regular Expressions
-S IP-Classless Internet ipcidr_match.py
Domain Routing
=============== ====================== ===================

16
doc/topics/targeting/index.rst
View File

@ -111,3 +111,19 @@ There are many ways to target individual minions or groups of minions in Salt:
nodegroups
batch
range
Loadable Matchers
=================
.. versionadded:: Flourine
Internally targeting is implemented with chunks of code called Matchers. As of
the Flourine release, matchers can be loaded dynamically. Currently new matchers
cannot be created, but existing matchers can have their functionality altered or
extended. For more information on Matchers see
.. toctree::
:maxdepth: 2
Loadable Matchers <../matchers/index.rst>

2
salt/client/ssh/state.py
View File

@ -74,7 +74,7 @@ class SSHHighState(salt.state.BaseHighState):
self.client = fsclient
salt.state.BaseHighState.__init__(self, opts)
self.state = SSHState(opts, pillar, wrapper)
self.matcher = salt.minion.Matcher(self.opts)
self.matchers = salt.loader.matchers(self.opts)
self.tops = salt.loader.tops(self.opts)
self._pydsl_all_decls = {}

11
salt/loader.py
View File

@ -294,6 +294,17 @@ def raw_mod(opts, name, functions, mod='modules'):
return dict(loader._dict) # return a copy of *just* the funcs for `name`
def matchers(opts):
'''
Return the matcher services plugins
'''
return LazyLoader(
_module_dirs(opts, 'matchers'),
opts,
tag='matchers'
)
def engines(opts, functions, runners, utils, proxy=None):
'''
Return the master services plugins

97
salt/matchers/__init__.py Normal file
View File

@ -0,0 +1,97 @@
# -*- coding: utf-8 -*-
'''
Salt package
'''
# Import Python libs
from __future__ import absolute_import, print_function, unicode_literals
import warnings
# All salt related deprecation warnings should be shown once each!
warnings.filterwarnings(
'once', # Show once
'', # No deprecation message match
DeprecationWarning, # This filter is for DeprecationWarnings
r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.<whatever>'
)
# While we are supporting Python2.6, hide nested with-statements warnings
warnings.filterwarnings(
'ignore',
'With-statements now directly support multiple context managers',
DeprecationWarning
)
# Filter the backports package UserWarning about being re-imported
warnings.filterwarnings(
'ignore',
'^Module backports was already imported from (.*), but (.*) is being added to sys.path$',
UserWarning
)
def __define_global_system_encoding_variable__():
import sys
# This is the most trustworthy source of the system encoding, though, if
# salt is being imported after being daemonized, this information is lost
# and reset to None
encoding = None
if not sys.platform.startswith('win') and sys.stdin is not None:
# On linux we can rely on sys.stdin for the encoding since it
# most commonly matches the filesystem encoding. This however
# does not apply to windows
encoding = sys.stdin.encoding
if not encoding:
# If the system is properly configured this should return a valid
# encoding. MS Windows has problems with this and reports the wrong
# encoding
import locale
try:
encoding = locale.getdefaultlocale()[-1]
except ValueError:
# A bad locale setting was most likely found:
# https://github.com/saltstack/salt/issues/26063
pass
# This is now garbage collectable
del locale
if not encoding:
# This is most likely ascii which is not the best but we were
# unable to find a better encoding. If this fails, we fall all
# the way back to ascii
encoding = sys.getdefaultencoding()
if not encoding:
if sys.platform.startswith('darwin'):
# Mac OS X uses UTF-8
encoding = 'utf-8'
elif sys.platform.startswith('win'):
# Windows uses a configurable encoding; on Windows, Python uses the name “mbcs”
# to refer to whatever the currently configured encoding is.
encoding = 'mbcs'
else:
# On linux default to ascii as a last resort
encoding = 'ascii'
# We can't use six.moves.builtins because these builtins get deleted sooner
# than expected. See:
# https://github.com/saltstack/salt/issues/21036
if sys.version_info[0] < 3:
import __builtin__ as builtins # pylint: disable=incompatible-py3-code
else:
import builtins # pylint: disable=import-error
# Define the detected encoding as a built-in variable for ease of use
setattr(builtins, '__salt_system_encoding__', encoding)
# This is now garbage collectable
del sys
del builtins
del encoding
__define_global_system_encoding_variable__()
# This is now garbage collectable
del __define_global_system_encoding_variable__

30
salt/matchers/cache_match.py Normal file
View File

@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
'''
This is the default cache matcher function. It only exists for the master,
this is why there is only a ``mmatch()`` but not ``match()``.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
import salt.utils.data # pylint: disable=3rd-party-module-not-gated
import salt.utils.minions # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def mmatch(expr,
delimiter,
greedy,
search_type,
regex_match=False,
exact_match=False):
'''
Helper function to search for minions in master caches
If 'greedy' return accepted minions that matched by the condition or absent in the cache.
If not 'greedy' return the only minions have cache data and matched by the condition.
'''
ckminions = salt.utils.minions.CkMinions(__opts__)
return ckminions._check_cache_minions(expr, delimiter, greedy,
search_type, regex_match=regex_match,
exact_match=exact_match)

112
salt/matchers/compound_match.py Normal file
View File

@ -0,0 +1,112 @@
# -*- coding: utf-8 -*-
'''
This is the default compound matcher function.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
from salt.ext import six # pylint: disable=3rd-party-module-not-gated
import salt.loader
import salt.utils.minions # pylint: disable=3rd-party-module-not-gated
HAS_RANGE = False
try:
import seco.range # pylint: disable=unused-import
HAS_RANGE = True
except ImportError:
pass
log = logging.getLogger(__name__)
def match(tgt):
'''
Runs the compound target check
'''
nodegroups = __opts__.get('nodegroups', {})
matchers = salt.loader.matchers(__opts__)
if not isinstance(tgt, six.string_types) and not isinstance(tgt, (list, tuple)):
log.error('Compound target received that is neither string, list nor tuple')
return False
log.debug('compound_match: %s ? %s', __opts__['id'], tgt)
ref = {'G': 'grain',
'P': 'grain_pcre',
'I': 'pillar',
'J': 'pillar_pcre',
'L': 'list',
'N': None, # Nodegroups should already be expanded
'S': 'ipcidr',
'E': 'pcre'}
if HAS_RANGE:
ref['R'] = 'range'
results = []
opers = ['and', 'or', 'not', '(', ')']
if isinstance(tgt, six.string_types):
words = tgt.split()
else:
# we make a shallow copy in order to not affect the passed in arg
words = tgt[:]
while words:
word = words.pop(0)
target_info = salt.utils.minions.parse_target(word)
# Easy check first
if word in opers:
if results:
if results[-1] == '(' and word in ('and', 'or'):
log.error('Invalid beginning operator after "(": %s', word)
return False
if word == 'not':
if not results[-1] in ('and', 'or', '('):
results.append('and')
results.append(word)
else:
# seq start with binary oper, fail
if word not in ['(', 'not']:
log.error('Invalid beginning operator: %s', word)
return False
results.append(word)
elif target_info and target_info['engine']:
if 'N' == target_info['engine']:
# if we encounter a node group, just evaluate it in-place
decomposed = salt.utils.minions.nodegroup_comp(target_info['pattern'], nodegroups)
if decomposed:
words = decomposed + words
continue
engine = ref.get(target_info['engine'])
if not engine:
# If an unknown engine is called at any time, fail out
log.error(
'Unrecognized target engine "%s" for target '
'expression "%s"', target_info['engine'], word
)
return False
engine_args = [target_info['pattern']]
engine_kwargs = {}
if target_info['delimiter']:
engine_kwargs['delimiter'] = target_info['delimiter']
results.append(
six.text_type(matchers['{0}_match.match'.format(engine)](*engine_args, **engine_kwargs))
)
else:
# The match is not explicitly defined, evaluate it as a glob
results.append(six.text_type(matchers['glob_match.match'](word)))
results = ' '.join(results)
log.debug('compound_match %s ? "%s" => "%s"', __opts__['id'], tgt, results)
try:
return eval(results) # pylint: disable=W0123
except Exception:
log.error(
'Invalid compound target: %s for results: %s', tgt, results)
return False
return False

23
salt/matchers/compound_pillar_exact_match.py Normal file
View File

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
'''
This is the default pillar exact matcher for compound matches.
There is no minion-side equivalent for this, so consequently there is no ``match()``
function below, only an ``mmatch()``
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
import salt.utils.minions # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def mmatch(expr, delimiter, greedy):
'''
Return the minions found by looking via pillar
'''
ckminions = salt.utils.minions.CkMinions(__opts__)
return ckminions._check_compound_minions(expr, delimiter, greedy,
pillar_exact=True)

38
salt/matchers/confirm_top.py Normal file
View File

@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
'''
The matcher subsystem needs a function called 'confirm_top', which
takes the data passed to a top file environment and determines if that
data matches this minion.
'''
from __future__ import absolute_import
import logging
import salt.loader
log = logging.getLogger(__file__)
def confirm_top(match, data, nodegroups=None):
'''
Takes the data passed to a top file environment and determines if the
data matches this minion
'''
matcher = 'compound'
if not data:
log.error('Received bad data when setting the match from the top '
'file')
return False
for item in data:
if isinstance(item, dict):
if 'match' in item:
matcher = item['match']
matchers = salt.loader.matchers(__opts__)
funcname = matcher + '_match.match'
if matcher == 'nodegroup':
return matchers[funcname](match, nodegroups)
else:
m = matchers[funcname]
return m(match)
# except TypeError, KeyError:
# log.error('Attempting to match with unknown matcher: %s', matcher)

46
salt/matchers/data_match.py Normal file
View File

@ -0,0 +1,46 @@
# -*- coding: utf-8 -*-
'''
This is the default data matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import fnmatch
import logging
from salt.ext import six # pylint: disable=3rd-party-module-not-gated
import salt.utils.data # pylint: disable=3rd-party-module-not-gated
import salt.utils.minions # pylint: disable=3rd-party-module-not-gated
import salt.utils.network # pylint: disable=3rd-party-module-not-gated
import salt.loader # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def match(tgt, functions=None):
'''
Match based on the local data store on the minion
'''
if functions is None:
utils = salt.loader.utils(__opts__)
functions = salt.loader.minion_mods(__opts__, utils=utils)
comps = tgt.split(':')
if len(comps) < 2:
return False
val = functions['data.getval'](comps[0])
if val is None:
# The value is not defined
return False
if isinstance(val, list):
# We are matching a single component to a single list member
for member in val:
if fnmatch.fnmatch(six.text_type(member).lower(), comps[1].lower()):
return True
return False
if isinstance(val, dict):
if comps[1] in val:
return True
return False
return bool(fnmatch.fnmatch(
val,
comps[1],
))

18
salt/matchers/glob_match.py Normal file
View File

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
'''
This is the default glob matcher function.
'''
from __future__ import absolute_import, print_function, unicode_literals
import fnmatch
from salt.ext import six # pylint: disable=3rd-party-module-not-gated
def match(tgt):
'''
Returns true if the passed glob matches the id
'''
if not isinstance(tgt, six.string_types):
return False
return fnmatch.fnmatch(__opts__['id'], tgt)

26
salt/matchers/grain_match.py Normal file
View File

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
'''
This is the default grains matcher function.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
from salt.defaults import DEFAULT_TARGET_DELIM # pylint: disable=3rd-party-module-not-gated
import salt.utils.data # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def match(tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Reads in the grains glob match
'''
log.debug('grains target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for grains match '
'statement from master')
return False
return salt.utils.data.subdict_match(
__opts__['grains'], tgt, delimiter=delimiter
)

25
salt/matchers/grain_pcre_match.py Normal file
View File

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
'''
This is the default grains PCRE matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
from salt.defaults import DEFAULT_TARGET_DELIM # pylint: disable=3rd-party-module-not-gated
import salt.utils.data # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def match(tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Matches a grain based on regex
'''
log.debug('grains pcre target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for grains pcre match '
'statement from master')
return False
return salt.utils.data.subdict_match(
__opts__['grains'], tgt, delimiter=delimiter, regex_match=True)

45
salt/matchers/ipcidr_match.py Normal file
View File

@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
'''
This is the default ipcidr matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
from salt.ext import six # pylint: disable=3rd-party-module-not-gated
import salt.utils.network # pylint: disable=3rd-party-module-not-gated
if six.PY3:
import ipaddress
else:
import salt.ext.ipaddress as ipaddress
log = logging.getLogger(__name__)
def match(tgt):
'''
Matches based on IP address or CIDR notation
'''
try:
# Target is an address?
tgt = ipaddress.ip_address(tgt)
except: # pylint: disable=bare-except
try:
# Target is a network?
tgt = ipaddress.ip_network(tgt)
except: # pylint: disable=bare-except
log.error('Invalid IP/CIDR target: %s', tgt)
return []
proto = 'ipv{0}'.format(tgt.version)
grains = __opts__['grains']
if proto not in grains:
match = False
elif isinstance(tgt, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
match = six.text_type(tgt) in grains[proto]
else:
match = salt.utils.network.in_subnet(tgt, grains[proto])
return match

39
salt/matchers/list_match.py Normal file
View File

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
'''
This is the default list matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
log = logging.getLogger(__name__)
def match(tgt):
'''
Determines if this host is on the list
'''
try:
if ',' + __opts__['id'] + ',' in tgt \
or tgt.startswith(__opts__['id'] + ',') \
or tgt.endswith(',' + __opts__['id']):
return True
# tgt is a string, which we know because the if statement above did not
# cause one of the exceptions being caught. Therefore, look for an
# exact match. (e.g. salt -L foo test.ping)
return __opts__['id'] == tgt
except (AttributeError, TypeError):
# tgt is not a string, maybe it's a sequence type?
try:
return __opts__['id'] in tgt
except Exception:
# tgt was likely some invalid type
return False
# We should never get here based on the return statements in the logic
# above. If we do, it is because something above changed, and should be
# considered as a bug. Log a warning to help us catch this.
log.warning(
'List matcher unexpectedly did not return, for target %s, '
'this is probably a bug.', tgt
)
return False

22
salt/matchers/nodegroup_match.py Normal file
View File

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
'''
This is the default nodegroup matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import salt.utils.minions # pylint: disable=3rd-party-module-not-gated
import salt.loader
def match(tgt, nodegroups):
'''
This is a compatibility matcher and is NOT called when using
nodegroups for remote execution, but is called when the nodegroups
matcher is used in states
'''
if tgt in nodegroups:
matchers = salt.loader.matchers(__opts__)
return matchers['compound_match.match'](
salt.utils.minions.nodegroup_comp(tgt, nodegroups)
)
return False

14
salt/matchers/pcre_match.py Normal file
View File

@ -0,0 +1,14 @@
# -*- coding: utf-8 -*-
'''
This is the default pcre matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import re
def match(tgt):
'''
Returns true if the passed pcre regex matches
'''
return bool(re.match(tgt, __opts__['id']))

25
salt/matchers/pillar_exact_match.py Normal file
View File

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
'''
This is the default pillar exact matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
import salt.utils.data # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def match(tgt, delimiter=':'):
'''
Reads in the pillar match, no globbing, no PCRE
'''
log.debug('pillar target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for pillar match '
'statement from master')
return False
return salt.utils.data.subdict_match(__opts__['pillar'],
tgt,
delimiter=delimiter,
exact_match=True)

25
salt/matchers/pillar_match.py Normal file
View File

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
'''
This is the default pillar matcher function.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
from salt.defaults import DEFAULT_TARGET_DELIM # pylint: disable=3rd-party-module-not-gated
import salt.utils.data # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def match(tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Reads in the pillar glob match
'''
log.debug('pillar target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for pillar match '
'statement from master')
return False
return salt.utils.data.subdict_match(
__opts__['pillar'], tgt, delimiter=delimiter
)

25
salt/matchers/pillar_pcre_match.py Normal file
View File

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
'''
This is the default pillar PCRE matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
from salt.defaults import DEFAULT_TARGET_DELIM # pylint: disable=3rd-party-module-not-gated
import salt.utils.data # pylint: disable=3rd-party-module-not-gated
log = logging.getLogger(__name__)
def match(tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Reads in the pillar pcre match
'''
log.debug('pillar PCRE target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for pillar PCRE match '
'statement from master')
return False
return salt.utils.data.subdict_match(
__opts__['pillar'], tgt, delimiter=delimiter, regex_match=True
)

30
salt/matchers/range_match.py Normal file
View File

@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
'''
This is the default range matcher.
'''
from __future__ import absolute_import, print_function, unicode_literals
import logging
HAS_RANGE = False
try:
import seco.range
HAS_RANGE = True
except ImportError:
pass
log = logging.getLogger(__name__)
def match(tgt):
'''
Matches based on range cluster
'''
if HAS_RANGE:
range_ = seco.range.Range(__opts__['range_server'])
try:
return __opts__['grains']['fqdn'] in range_.expand(tgt)
except seco.range.RangeException as exc:
log.debug('Range exception in compound match: %s', exc)
return False
return False

331
salt/minion.py
View File

@ -6,14 +6,12 @@ Routines to set up a minion
from __future__ import absolute_import, print_function, with_statement, unicode_literals
import functools
import os
import re
import sys
import copy
import time
import types
import signal
import random
import fnmatch
import logging
import threading
import traceback
@ -27,10 +25,6 @@ from binascii import crc32
# Import Salt Libs
# pylint: disable=import-error,no-name-in-module,redefined-builtin
from salt.ext import six
if six.PY3:
import ipaddress
else:
import salt.ext.ipaddress as ipaddress
from salt.ext.six.moves import range
from salt.utils.zeromq import zmq, ZMQDefaultLoop, install_zmq, ZMQ_VERSION_INFO
import salt.defaults.exitcodes
@ -40,13 +34,6 @@ from salt.utils.ctx import RequestContext
# pylint: enable=no-name-in-module,redefined-builtin
import tornado
HAS_RANGE = False
try:
import seco.range
HAS_RANGE = True
except ImportError:
pass
HAS_PSUTIL = False
try:
import salt.utils.psutil_compat as psutil
@ -862,7 +849,8 @@ class SMinion(MinionBase):
self.utils,
self.serializers)
self.rend = salt.loader.render(self.opts, self.functions)
self.matcher = Matcher(self.opts, self.functions)
# self.matcher = Matcher(self.opts, self.functions)
self.matchers = salt.loader.matchers(self.opts)
self.functions['sys.reload_modules'] = self.gen_modules
self.executors = salt.loader.executors(self.opts)
@ -924,7 +912,7 @@ class MasterMinion(object):
if self.mk_rend:
self.rend = salt.loader.render(self.opts, self.functions)
if self.mk_matcher:
self.matcher = Matcher(self.opts, self.functions)
self.matchers = salt.loader.matchers(self.opts)
self.functions['sys.reload_modules'] = self.gen_modules
@ -2158,6 +2146,13 @@ class Minion(MinionBase):
log.debug('Refreshing beacons.')
self.beacons = salt.beacons.Beacon(self.opts, self.functions)
def matchers_refresh(self):
'''
Refresh the matchers
'''
log.debug('Refreshing matchers.')
self.matchers = salt.loader.matchers(self.opts)
# TODO: only allow one future in flight at a time?
@tornado.gen.coroutine
def pillar_refresh(self, force_refresh=False):
@ -2334,6 +2329,8 @@ class Minion(MinionBase):
)
elif tag.startswith('beacons_refresh'):
self.beacons_refresh()
elif tag.startswith('matchers_refresh'):
self.matchers_refresh()
elif tag.startswith('manage_schedule'):
self.manage_schedule(tag, data)
elif tag.startswith('manage_beacons'):
@ -2531,7 +2528,8 @@ class Minion(MinionBase):
self.functions, self.returners, self.function_errors, self.executors = self._load_modules()
self.serial = salt.payload.Serial(self.opts)
self.mod_opts = self._prep_mod_opts()
self.matcher = Matcher(self.opts, self.functions)
# self.matcher = Matcher(self.opts, self.functions)
self.matchers = salt.loader.matchers(self.opts)
self.beacons = salt.beacons.Beacon(self.opts, self.functions)
uid = salt.utils.user.get_uid(user=self.opts.get('user', None))
self.proc_dir = get_proc_dir(self.opts['cachedir'], uid=uid)
@ -2741,8 +2739,7 @@ class Minion(MinionBase):
# publication if the master does not determine that it should.
if 'tgt_type' in load:
match_func = getattr(self.matcher,
'{0}_match'.format(load['tgt_type']), None)
match_func = self.matchers.get('{0}_match.match'.format(load['tgt_type']), None)
if match_func is None:
return False
if load['tgt_type'] in ('grain', 'grain_pcre', 'pillar'):
@ -2752,7 +2749,7 @@ class Minion(MinionBase):
elif not match_func(load['tgt']):
return False
else:
if not self.matcher.glob_match(load['tgt']):
if not self.matchers['glob_match.match'](load['tgt']):
return False
return True
@ -3250,298 +3247,6 @@ class SyndicManager(MinionBase):
del self.job_rets[master]
class Matcher(object):
'''
Use to return the value for matching calls from the master
'''
def __init__(self, opts, functions=None):
self.opts = opts
self.functions = functions
def confirm_top(self, match, data, nodegroups=None):
'''
Takes the data passed to a top file environment and determines if the
data matches this minion
'''
matcher = 'compound'
if not data:
log.error('Received bad data when setting the match from the top '
'file')
return False
for item in data:
if isinstance(item, dict):
if 'match' in item:
matcher = item['match']
if hasattr(self, matcher + '_match'):
funcname = '{0}_match'.format(matcher)
if matcher == 'nodegroup':
return getattr(self, funcname)(match, nodegroups)
return getattr(self, funcname)(match)
else:
log.error('Attempting to match with unknown matcher: %s', matcher)
return False
def glob_match(self, tgt):
'''
Returns true if the passed glob matches the id
'''
if not isinstance(tgt, six.string_types):
return False
return fnmatch.fnmatch(self.opts['id'], tgt)
def pcre_match(self, tgt):
'''
Returns true if the passed pcre regex matches
'''
return bool(re.match(tgt, self.opts['id']))
def list_match(self, tgt):
'''
Determines if this host is on the list
'''
if isinstance(tgt, six.string_types):
tgt = tgt.split(',')
return bool(self.opts['id'] in tgt)
def grain_match(self, tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Reads in the grains glob match
'''
log.debug('grains target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for grains match '
'statement from master')
return False
return salt.utils.data.subdict_match(
self.opts['grains'], tgt, delimiter=delimiter
)
def grain_pcre_match(self, tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Matches a grain based on regex
'''
log.debug('grains pcre target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for grains pcre match '
'statement from master')
return False
return salt.utils.data.subdict_match(
self.opts['grains'], tgt, delimiter=delimiter, regex_match=True)
def data_match(self, tgt):
'''
Match based on the local data store on the minion
'''
if self.functions is None:
utils = salt.loader.utils(self.opts)
self.functions = salt.loader.minion_mods(self.opts, utils=utils)
comps = tgt.split(':')
if len(comps) < 2:
return False
val = self.functions['data.getval'](comps[0])
if val is None:
# The value is not defined
return False
if isinstance(val, list):
# We are matching a single component to a single list member
for member in val:
if fnmatch.fnmatch(six.text_type(member).lower(), comps[1].lower()):
return True
return False
if isinstance(val, dict):
if comps[1] in val:
return True
return False
return bool(fnmatch.fnmatch(
val,
comps[1],
))
def pillar_match(self, tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Reads in the pillar glob match
'''
log.debug('pillar target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for pillar match '
'statement from master')
return False
return salt.utils.data.subdict_match(
self.opts['pillar'], tgt, delimiter=delimiter
)
def pillar_pcre_match(self, tgt, delimiter=DEFAULT_TARGET_DELIM):
'''
Reads in the pillar pcre match
'''
log.debug('pillar PCRE target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for pillar PCRE match '
'statement from master')
return False
return salt.utils.data.subdict_match(
self.opts['pillar'], tgt, delimiter=delimiter, regex_match=True
)
def pillar_exact_match(self, tgt, delimiter=':'):
'''
Reads in the pillar match, no globbing, no PCRE
'''
log.debug('pillar target: %s', tgt)
if delimiter not in tgt:
log.error('Got insufficient arguments for pillar match '
'statement from master')
return False
return salt.utils.data.subdict_match(self.opts['pillar'],
tgt,
delimiter=delimiter,
exact_match=True)
def ipcidr_match(self, tgt):
'''
Matches based on IP address or CIDR notation
'''
try:
# Target is an address?
tgt = ipaddress.ip_address(tgt)
except Exception:
try:
# Target is a network?
tgt = ipaddress.ip_network(tgt)
except Exception:
log.error('Invalid IP/CIDR target: %s', tgt)
return []
proto = 'ipv{0}'.format(tgt.version)
grains = self.opts['grains']
if proto not in grains:
match = False
elif isinstance(tgt, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
match = six.text_type(tgt) in grains[proto]
else:
match = salt.utils.network.in_subnet(tgt, grains[proto])
return match
def range_match(self, tgt):
'''
Matches based on range cluster
'''
if HAS_RANGE:
range_ = seco.range.Range(self.opts['range_server'])
try:
return self.opts['grains']['fqdn'] in range_.expand(tgt)
except seco.range.RangeException as exc:
log.debug('Range exception in compound match: %s', exc)
return False
return False
def compound_match(self, tgt):
'''
Runs the compound target check
'''
nodegroups = self.opts.get('nodegroups', {})
if not isinstance(tgt, six.string_types) and not isinstance(tgt, (list, tuple)):
log.error('Compound target received that is neither string, list nor tuple')
return False
log.debug('compound_match: %s ? %s', self.opts['id'], tgt)
ref = {'G': 'grain',
'P': 'grain_pcre',
'I': 'pillar',
'J': 'pillar_pcre',
'L': 'list',
'N': None, # Nodegroups should already be expanded
'S': 'ipcidr',
'E': 'pcre'}
if HAS_RANGE:
ref['R'] = 'range'
results = []
opers = ['and', 'or', 'not', '(', ')']
if isinstance(tgt, six.string_types):
words = tgt.split()
else:
# we make a shallow copy in order to not affect the passed in arg
words = tgt[:]
while words:
word = words.pop(0)
target_info = salt.utils.minions.parse_target(word)
# Easy check first
if word in opers:
if results:
if results[-1] == '(' and word in ('and', 'or'):
log.error('Invalid beginning operator after "(": %s', word)
return False
if word == 'not':
if not results[-1] in ('and', 'or', '('):
results.append('and')
results.append(word)
else:
# seq start with binary oper, fail
if word not in ['(', 'not']:
log.error('Invalid beginning operator: %s', word)
return False
results.append(word)
elif target_info and target_info['engine']:
if 'N' == target_info['engine']:
# if we encounter a node group, just evaluate it in-place
decomposed = salt.utils.minions.nodegroup_comp(target_info['pattern'], nodegroups)
if decomposed:
words = decomposed + words
continue
engine = ref.get(target_info['engine'])
if not engine:
# If an unknown engine is called at any time, fail out
log.error(
'Unrecognized target engine "%s" for target '
'expression "%s"', target_info['engine'], word
)
return False
engine_args = [target_info['pattern']]
engine_kwargs = {}
if target_info['delimiter']:
engine_kwargs['delimiter'] = target_info['delimiter']
results.append(
six.text_type(getattr(self, '{0}_match'.format(engine))(*engine_args, **engine_kwargs))
)
else:
# The match is not explicitly defined, evaluate it as a glob
results.append(six.text_type(self.glob_match(word)))
results = ' '.join(results)
log.debug('compound_match %s ? "%s" => "%s"', self.opts['id'], tgt, results)
try:
return eval(results) # pylint: disable=W0123
except Exception:
log.error(
'Invalid compound target: %s for results: %s', tgt, results)
return False
return False
def nodegroup_match(self, tgt, nodegroups):
'''
This is a compatibility matcher and is NOT called when using
nodegroups for remote execution, but is called when the nodegroups
matcher is used in states
'''
if tgt in nodegroups:
return self.compound_match(
salt.utils.minions.nodegroup_comp(tgt, nodegroups)
)
return False
class ProxyMinionManager(MinionManager):
'''
Create the multi-minion interface but for proxy minions
@ -3677,7 +3382,7 @@ class ProxyMinion(Minion):
self.serial = salt.payload.Serial(self.opts)
self.mod_opts = self._prep_mod_opts()
self.matcher = Matcher(self.opts, self.functions)
self.matchers = salt.loader.matchers(self.opts)
self.beacons = salt.beacons.Beacon(self.opts, self.functions)
uid = salt.utils.user.get_uid(user=self.opts.get('user', None))
self.proc_dir = get_proc_dir(self.opts['cachedir'], uid=uid)
@ -3887,7 +3592,7 @@ class SProxyMinion(SMinion):
self.functions = salt.loader.minion_mods(self.opts, utils=self.utils, notify=False, proxy=self.proxy)
self.returners = salt.loader.returners(self.opts, self.functions, proxy=self.proxy)
self.matcher = Matcher(self.opts, self.functions)
self.matchers = salt.loader.matchers(self.opts)
self.functions['sys.reload_modules'] = self.gen_modules
self.executors = salt.loader.executors(self.opts, self.functions, proxy=self.proxy)

61
salt/modules/match.py
View File

@ -8,9 +8,11 @@ from __future__ import absolute_import, print_function, unicode_literals
import inspect
import logging
import sys
import copy
# Import salt libs
import salt.minion
import salt.loader
from salt.defaults import DEFAULT_TARGET_DELIM
from salt.ext import six
@ -36,16 +38,16 @@ def compound(tgt, minion_id=None):
salt '*' match.compound 'L@cheese,foo and *'
'''
opts = {'grains': __grains__, 'pillar': __pillar__}
if minion_id is not None:
opts = copy.copy(__opts__)
if not isinstance(minion_id, six.string_types):
minion_id = six.text_type(minion_id)
opts['id'] = minion_id
else:
minion_id = __grains__['id']
opts['id'] = minion_id
matcher = salt.minion.Matcher(opts, __salt__)
opts = __opts__
matchers = salt.loader.matchers(opts)
try:
return matcher.compound_match(tgt)
return matchers['compound_match.match'](tgt)
except Exception as exc:
log.exception(exc)
return False
@ -71,9 +73,9 @@ def ipcidr(tgt):
- nodeclass: internal
'''
matcher = salt.minion.Matcher({'grains': __grains__}, __salt__)
matchers = salt.loader.matchers(__opts__)
try:
return matcher.ipcidr_match(tgt)
return matchers['ipcidr_match.match'](tgt)
except Exception as exc:
log.exception(exc)
return False
@ -102,9 +104,9 @@ def pillar_pcre(tgt, delimiter=DEFAULT_TARGET_DELIM):
.. versionadded:: 0.16.4
.. deprecated:: 2015.8.0
'''
matcher = salt.minion.Matcher({'pillar': __pillar__}, __salt__)
matchers = salt.loader.matchers(__opts__)
try:
return matcher.pillar_pcre_match(tgt, delimiter=delimiter)
return matchers['pillar_pcre_match.match'](tgt, delimiter=delimiter)
except Exception as exc:
log.exception(exc)
return False
@ -133,9 +135,9 @@ def pillar(tgt, delimiter=DEFAULT_TARGET_DELIM):
.. versionadded:: 0.16.4
.. deprecated:: 2015.8.0
'''
matcher = salt.minion.Matcher({'pillar': __pillar__}, __salt__)
matchers = salt.loader.matchers(__opts__)
try:
return matcher.pillar_match(tgt, delimiter=delimiter)
return matchers['pillar_match.match'](tgt, delimiter=delimiter)
except Exception as exc:
log.exception(exc)
return False
@ -151,9 +153,9 @@ def data(tgt):
salt '*' match.data 'spam:eggs'
'''
matcher = salt.minion.Matcher(__opts__, __salt__)
matchers = salt.loader.matchers(__opts__)
try:
return matcher.data_match(tgt)
return matchers['data_match.match'](tgt)
except Exception as exc:
log.exception(exc)
return False
@ -182,9 +184,9 @@ def grain_pcre(tgt, delimiter=DEFAULT_TARGET_DELIM):
.. versionadded:: 0.16.4
.. deprecated:: 2015.8.0
'''
matcher = salt.minion.Matcher({'grains': __grains__}, __salt__)
matchers = salt.loader.matchers(__opts__)
try:
return matcher.grain_pcre_match(tgt, delimiter=delimiter)
return matchers['grain_pcre_match.match'](tgt, delimiter=delimiter)
except Exception as exc:
log.exception(exc)
return False
@ -213,9 +215,9 @@ def grain(tgt, delimiter=DEFAULT_TARGET_DELIM):
.. versionadded:: 0.16.4
.. deprecated:: 2015.8.0
'''
matcher = salt.minion.Matcher({'grains': __grains__}, __salt__)
matchers = salt.loader.matchers(__opts__)
try:
return matcher.grain_match(tgt, delimiter=delimiter)
return matchers['grain_match.match'](tgt, delimiter=delimiter)
except Exception as exc:
log.exception(exc)
return False
@ -237,13 +239,15 @@ def list_(tgt, minion_id=None):
salt '*' match.list 'server1,server2'
'''
if minion_id is not None:
opts = copy.copy(__opts__)
if not isinstance(minion_id, six.string_types):
minion_id = six.text_type(minion_id)
opts['id'] = minion_id
else:
minion_id = __grains__['id']
matcher = salt.minion.Matcher({'id': minion_id}, __salt__)
opts = __opts__
matchers = salt.loader.matchers(opts)
try:
return matcher.list_match(tgt)
return matchers['list_match.match'](tgt)
except Exception as exc:
log.exception(exc)
return False
@ -265,13 +269,15 @@ def pcre(tgt, minion_id=None):
salt '*' match.pcre '.*'
'''
if minion_id is not None:
opts = copy.copy(__opts__)
if not isinstance(minion_id, six.string_types):
minion_id = six.text_type(minion_id)
opts['id'] = minion_id
else:
minion_id = __grains__['id']
matcher = salt.minion.Matcher({'id': minion_id}, __salt__)
opts = __opts__
matchers = salt.loader.matchers(opts)
try:
return matcher.pcre_match(tgt)
return matchers['pcre_match.match'](tgt)
except Exception as exc:
log.exception(exc)
return False
@ -293,13 +299,16 @@ def glob(tgt, minion_id=None):
salt '*' match.glob '*'
'''
if minion_id is not None:
opts = copy.copy(__opts__)
if not isinstance(minion_id, six.string_types):
minion_id = six.text_type(minion_id)
opts['id'] = minion_id
else:
minion_id = __grains__['id']
matcher = salt.minion.Matcher({'id': minion_id}, __salt__)
opts = __opts__
matchers = salt.loader.matchers(opts)
try:
return matcher.glob_match(tgt)
return matchers['glob_match.match'](tgt)
except Exception as exc:
log.exception(exc)
return False

57
salt/modules/saltutil.py
View File

@ -547,6 +547,44 @@ def sync_proxymodules(saltenv=None, refresh=False, extmod_whitelist=None, extmod
return ret
def sync_matchers(saltenv=None, refresh=False, extmod_whitelist=None, extmod_blacklist=None):
'''
.. versionadded:: Flourine
Sync engine modules from ``salt://_matchers`` to the minion
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for engines to sync. If no top files are
found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new matcher modules are synced.
Set to ``False`` to prevent this refresh.
extmod_whitelist : None
comma-separated list of modules to sync
extmod_blacklist : None
comma-separated list of modules to blacklist based on type
CLI Examples:
.. code-block:: bash
salt '*' saltutil.sync_matchers
salt '*' saltutil.sync_matchers saltenv=base,dev
'''
ret = _sync('matchers', saltenv, extmod_whitelist, extmod_blacklist)
if refresh:
refresh_modules()
return ret
def sync_engines(saltenv=None, refresh=False, extmod_whitelist=None, extmod_blacklist=None):
'''
.. versionadded:: 2016.3.0
@ -943,6 +981,7 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist
ret['engines'] = sync_engines(saltenv, False, extmod_whitelist, extmod_blacklist)
ret['thorium'] = sync_thorium(saltenv, False, extmod_whitelist, extmod_blacklist)
ret['serializers'] = sync_serializers(saltenv, False, extmod_whitelist, extmod_blacklist)
ret['matchers'] = sync_matchers(saltenv, False, extmod_whitelist, extmod_blacklist)
if __opts__['file_client'] == 'local':
ret['pillar'] = sync_pillar(saltenv, False, extmod_whitelist, extmod_blacklist)
if refresh:
@ -969,6 +1008,24 @@ def refresh_beacons():
return ret
def refresh_matchers():
'''
Signal the minion to refresh its matchers.
CLI Example:
.. code-block:: bash
salt '*' saltutil.refresh_matchers
'''
try:
ret = __salt__['event.fire']({}, 'matchers_refresh')
except KeyError:
log.error('Event module not available. Matcher refresh failed.')
ret = False # Effectively a no-op, since we can't really return without an event system
return ret
def refresh_pillar():
'''
Signal the minion to refresh the pillar data.

4
salt/pillar/__init__.py
View File

@ -366,7 +366,7 @@ class Pillar(object):
else:
self.functions = functions
self.matcher = salt.minion.Matcher(self.opts, self.functions)
self.matchers = salt.loader.matchers(self.opts)
self.rend = salt.loader.render(self.opts, self.functions)
ext_pillar_opts = copy.deepcopy(self.opts)
# Fix self.opts['file_roots'] so that ext_pillars know the real
@ -649,7 +649,7 @@ class Pillar(object):
if saltenv != self.opts['pillarenv']:
continue
for match, data in six.iteritems(body):
if self.matcher.confirm_top(
if self.matchers['confirm_top.confirm_top'](
match,
data,
self.opts.get('nodegroups', {}),

4
salt/state.py
View File

@ -3379,7 +3379,7 @@ class BaseHighState(object):
def _filter_matches(_match, _data, _opts):
if isinstance(_data, six.string_types):
_data = [_data]
if self.matcher.confirm_top(
if self.matchers['confirm_top.confirm_top'](
_match,
_data,
_opts
@ -4068,7 +4068,7 @@ class HighState(BaseHighState):
mocked=mocked,
loader=loader,
initial_pillar=initial_pillar)
self.matcher = salt.minion.Matcher(self.opts)
self.matchers = salt.loader.matchers(self.opts)
self.proxy = proxy
# tracks all pydsl state declarations globally across sls files

15
salt/utils/data.py
View File

@ -515,8 +515,8 @@ def subdict_match(data,
Check for a match in a dictionary using a delimiter character to denote
levels of subdicts, and also allowing the delimiter character to be
matched. Thus, 'foo:bar:baz' will match data['foo'] == 'bar:baz' and
data['foo']['bar'] == 'baz'. The former would take priority over the
latter.
data['foo']['bar'] == 'baz'. The latter would take priority over the
former, as more deeply-nested matches are tried first.
'''
def _match(target, pattern, regex_match=False, exact_match=False):
if regex_match:
@ -568,8 +568,15 @@ def subdict_match(data,
return True
return False
for idx in range(1, expr.count(delimiter) + 1):
splits = expr.split(delimiter)
splits = expr.split(delimiter)
num_splits = len(splits)
if num_splits == 1:
# Delimiter not present, this can't possibly be a match
return False
# If we have 4 splits, then we have three delimiters. Thus, the indexes we
# want to use are 3, 2, and 1, in that order.
for idx in range(num_splits - 1, 0, -1):
key = delimiter.join(splits[:idx])
matchstr = delimiter.join(splits[idx:])
log.debug("Attempting to match '%s' in '%s' using delimiter '%s'",

11
salt/utils/network.py
View File

@ -1234,19 +1234,16 @@ def in_subnet(cidr, addr=None):
try:
cidr = ipaddress.ip_network(cidr)
except ValueError:
log.error('Invalid CIDR \'{0}\''.format(cidr))
log.error('Invalid CIDR \'%s\'', cidr)
return False
if addr is None:
addr = ip_addrs()
addr.extend(ip_addrs6())
elif isinstance(addr, six.string_types):
return ipaddress.ip_address(addr) in cidr
elif not isinstance(addr, (list, tuple)):
addr = (addr,)
for ip_addr in addr:
if ipaddress.ip_address(ip_addr) in cidr:
return True
return False
return any(ipaddress.ip_address(item) in cidr for item in addr)
def _ip_addrs(interface=None, include_loopback=False, interface_data=None, proto='inet'):

4
tests/integration/modules/test_saltutil.py
View File

@ -93,6 +93,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
'modules.salttest'],
'renderers': [],
'log_handlers': [],
'matchers': [],
'states': [],
'sdb': [],
'proxymodules': [],
@ -115,6 +116,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
'modules': ['modules.salttest'],
'renderers': [],
'log_handlers': [],
'matchers': [],
'states': [],
'sdb': [],
'proxymodules': [],
@ -140,6 +142,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
'modules.salttest'],
'renderers': [],
'log_handlers': [],
'matchers': [],
'states': [],
'sdb': [],
'proxymodules': [],
@ -162,6 +165,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
'modules': [],
'renderers': [],
'log_handlers': [],
'matchers': [],
'states': [],
'sdb': [],
'proxymodules': [],

2
tests/integration/netapi/rest_tornado/test_app.py
View File

@ -279,7 +279,7 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase):
request_timeout=30,
)
response_obj = salt.utils.json.loads(response.body)
self.assertEqual(response_obj['return'], [{'localhost': True, 'minion': True, 'sub_minion': True}])
self.assertEqual(response_obj['return'], [{'minion': True, 'sub_minion': True}])
# runner tests
def test_simple_local_runner_post(self):

249
tests/unit/test_pillar.py
View File

@ -9,19 +9,43 @@
# Import python libs
from __future__ import absolute_import
import shutil
import tempfile
# Import Salt Testing libs
from tests.support.helpers import with_tempdir
from tests.support.unit import skipIf, TestCase
from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch
from tests.support.paths import TMP
# Import salt libs
import salt.fileclient
import salt.pillar
import salt.utils.stringutils
import salt.exceptions
class MockFileclient(object):
def __init__(self, cache_file=None, get_state=None, list_states=None):
if cache_file is not None:
self.cache_file = lambda *x, **y: cache_file
if get_state is not None:
self.get_state = lambda sls, env: get_state[sls]
if list_states is not None:
self.list_states = lambda *x, **y: list_states
# pylint: disable=unused-argument,no-method-argument,method-hidden
def cache_file(*args, **kwargs):
raise NotImplementedError()
def get_state(*args, **kwargs):
raise NotImplementedError()
def list_states(*args, **kwargs):
raise NotImplementedError()
# pylint: enable=unused-argument,no-method-argument,method-hidden
@skipIf(NO_MOCK, NO_MOCK_REASON)
class PillarTestCase(TestCase):
@ -509,40 +533,57 @@ class PillarTestCase(TestCase):
)
def test_topfile_order(self):
with patch('salt.pillar.salt.fileclient.get_file_client', autospec=True) as get_file_client, \
patch('salt.pillar.salt.minion.Matcher') as Matcher: # autospec=True disabled due to py3 mock bug
opts = {
'optimization_order': [0, 1, 2],
'renderer': 'yaml',
'renderer_blacklist': [],
'renderer_whitelist': [],
'state_top': '',
'pillar_roots': [],
'extension_modules': '',
'saltenv': 'base',
'file_roots': [],
}
grains = {
'os': 'Ubuntu',
'os_family': 'Debian',
'oscodename': 'raring',
'osfullname': 'Ubuntu',
'osrelease': '13.04',
'kernel': 'Linux'
}
# glob match takes precedence
self._setup_test_topfile_mocks(Matcher, get_file_client, 1, 2)
pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base')
self.assertEqual(pillar.compile_pillar()['ssh'], 'bar')
# nodegroup match takes precedence
self._setup_test_topfile_mocks(Matcher, get_file_client, 2, 1)
pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base')
self.assertEqual(pillar.compile_pillar()['ssh'], 'foo')
opts = {
'optimization_order': [0, 1, 2],
'renderer': 'yaml',
'renderer_blacklist': [],
'renderer_whitelist': [],
'state_top': '',
'pillar_roots': [],
'extension_modules': '',
'saltenv': 'base',
'file_roots': [],
}
grains = {
'os': 'Ubuntu',
'os_family': 'Debian',
'oscodename': 'raring',
'osfullname': 'Ubuntu',
'osrelease': '13.04',
'kernel': 'Linux'
}
def _setup_test_topfile_mocks(self, Matcher, get_file_client,
nodegroup_order, glob_order):
def _run_test(nodegroup_order, glob_order, expected):
tempdir = tempfile.mkdtemp(dir=TMP)
try:
sls_files = self._setup_test_topfile_sls(
tempdir,
nodegroup_order,
glob_order)
fc_mock = MockFileclient(
cache_file=sls_files['top']['dest'],
list_states=['top', 'ssh', 'ssh.minion',
'generic', 'generic.minion'],
get_state=sls_files)
with patch.object(salt.fileclient, 'get_file_client',
MagicMock(return_value=fc_mock)):
pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base')
# Make sure that confirm_top.confirm_top returns True
pillar.matchers['confirm_top.confirm_top'] = lambda *x, **y: True
self.assertEqual(pillar.compile_pillar()['ssh'], expected)
finally:
shutil.rmtree(tempdir, ignore_errors=True)
# test case where glob match happens second and therefore takes
# precedence over nodegroup match.
_run_test(nodegroup_order=1, glob_order=2, expected='bar')
# test case where nodegroup match happens second and therefore takes
# precedence over glob match.
_run_test(nodegroup_order=2, glob_order=1, expected='foo')
def _setup_test_topfile_sls(self, tempdir, nodegroup_order, glob_order):
# Write a simple topfile and two pillar state files
self.top_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
top_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
s = '''
base:
group:
@ -557,22 +598,22 @@ base:
- ssh.minion
- generic.minion
'''.format(nodegroup_order=nodegroup_order, glob_order=glob_order)
self.top_file.write(salt.utils.stringutils.to_bytes(s))
self.top_file.flush()
self.ssh_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
self.ssh_file.write(b'''
top_file.write(salt.utils.stringutils.to_bytes(s))
top_file.flush()
ssh_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
ssh_file.write(b'''
ssh:
foo
''')
self.ssh_file.flush()
self.ssh_minion_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
self.ssh_minion_file.write(b'''
ssh_file.flush()
ssh_minion_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
ssh_minion_file.write(b'''
ssh:
bar
''')
self.ssh_minion_file.flush()
self.generic_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
self.generic_file.write(b'''
ssh_minion_file.flush()
generic_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
generic_file.write(b'''
generic:
key1:
- value1
@ -580,67 +621,65 @@ generic:
key2:
sub_key1: []
''')
self.generic_file.flush()
self.generic_minion_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
self.generic_minion_file.write(b'''
generic_file.flush()
generic_minion_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
generic_minion_file.write(b'''
generic:
key1:
- value3
key2:
sub_key2: []
''')
self.generic_minion_file.flush()
generic_minion_file.flush()
# Setup Matcher mock
matcher = Matcher.return_value
matcher.confirm_top.return_value = True
return {
'top': {'path': '', 'dest': top_file.name},
'ssh': {'path': '', 'dest': ssh_file.name},
'ssh.minion': {'path': '', 'dest': ssh_minion_file.name},
'generic': {'path': '', 'dest': generic_file.name},
'generic.minion': {'path': '', 'dest': generic_minion_file.name},
}
# Setup fileclient mock
client = get_file_client.return_value
client.cache_file.return_value = self.top_file.name
def get_state(sls, env):
return {
'ssh': {'path': '', 'dest': self.ssh_file.name},
'ssh.minion': {'path': '', 'dest': self.ssh_minion_file.name},
'generic': {'path': '', 'dest': self.generic_file.name},
'generic.minion': {'path': '', 'dest': self.generic_minion_file.name},
}[sls]
client.get_state.side_effect = get_state
def test_include(self):
with patch('salt.pillar.salt.fileclient.get_file_client', autospec=True) as get_file_client, \
patch('salt.pillar.salt.minion.Matcher') as Matcher: # autospec=True disabled due to py3 mock bug
opts = {
'optimization_order': [0, 1, 2],
'renderer': 'yaml',
'renderer_blacklist': [],
'renderer_whitelist': [],
'state_top': '',
'pillar_roots': [],
'extension_modules': '',
'saltenv': 'base',
'file_roots': [],
}
grains = {
'os': 'Ubuntu',
'os_family': 'Debian',
'oscodename': 'raring',
'osfullname': 'Ubuntu',
'osrelease': '13.04',
'kernel': 'Linux'
}
self._setup_test_include_mocks(Matcher, get_file_client)
@with_tempdir()
def test_include(self, tempdir):
opts = {
'optimization_order': [0, 1, 2],
'renderer': 'yaml',
'renderer_blacklist': [],
'renderer_whitelist': [],
'state_top': '',
'pillar_roots': [],
'extension_modules': '',
'saltenv': 'base',
'file_roots': [],
}
grains = {
'os': 'Ubuntu',
'os_family': 'Debian',
'oscodename': 'raring',
'osfullname': 'Ubuntu',
'osrelease': '13.04',
'kernel': 'Linux'
}
sls_files = self._setup_test_include_sls(tempdir)
fc_mock = MockFileclient(
cache_file=sls_files['top']['dest'],
get_state=sls_files,
list_states=['top', 'test.init', 'test.sub1',
'test.sub2', 'test.sub_wildcard_1'],
)
with patch.object(salt.fileclient, 'get_file_client',
MagicMock(return_value=fc_mock)):
pillar = salt.pillar.Pillar(opts, grains, 'minion', 'base')
# Make sure that confirm_top.confirm_top returns True
pillar.matchers['confirm_top.confirm_top'] = lambda *x, **y: True
compiled_pillar = pillar.compile_pillar()
self.assertEqual(compiled_pillar['foo_wildcard'], 'bar_wildcard')
self.assertEqual(compiled_pillar['foo1'], 'bar1')
self.assertEqual(compiled_pillar['foo2'], 'bar2')
def _setup_test_include_mocks(self, Matcher, get_file_client):
self.top_file = top_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
def _setup_test_include_sls(self, tempdir):
top_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
top_file.write(b'''
base:
'*':
@ -651,50 +690,40 @@ base:
- test
''')
top_file.flush()
self.init_sls = init_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
init_sls = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
init_sls.write(b'''
include:
- test.sub1
- test.sub_wildcard*
''')
init_sls.flush()
self.sub1_sls = sub1_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
sub1_sls = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
sub1_sls.write(b'''
foo1:
bar1
''')
sub1_sls.flush()
self.sub2_sls = sub2_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
sub2_sls = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
sub2_sls.write(b'''
foo2:
bar2
''')
sub2_sls.flush()
self.sub_wildcard_1_sls = sub_wildcard_1_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False)
sub_wildcard_1_sls = tempfile.NamedTemporaryFile(dir=tempdir, delete=False)
sub_wildcard_1_sls.write(b'''
foo_wildcard:
bar_wildcard
''')
sub_wildcard_1_sls.flush()
# Setup Matcher mock
matcher = Matcher.return_value
matcher.confirm_top.return_value = True
# Setup fileclient mock
client = get_file_client.return_value
client.cache_file.return_value = self.top_file.name
client.list_states.return_value = ['top', 'test.init', 'test.sub1', 'test.sub2', 'test.sub_wildcard_1']
def get_state(sls, env):
return {
'test': {'path': '', 'dest': init_sls.name},
'test.sub1': {'path': '', 'dest': sub1_sls.name},
'test.sub2': {'path': '', 'dest': sub2_sls.name},
'test.sub_wildcard_1': {'path': '', 'dest': sub_wildcard_1_sls.name},
}[sls]
client.get_state.side_effect = get_state
return {
'top': {'path': '', 'dest': top_file.name},
'test': {'path': '', 'dest': init_sls.name},
'test.sub1': {'path': '', 'dest': sub1_sls.name},
'test.sub2': {'path': '', 'dest': sub2_sls.name},
'test.sub_wildcard_1': {'path': '', 'dest': sub_wildcard_1_sls.name},
}
@skipIf(NO_MOCK, NO_MOCK_REASON)