valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-06 16:45:27 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #49987 from terminalmage/pillar_roots

Browse Source 
Make Pillar no longer munge file_roots
This commit is contained in:
Daniel Wallace 2018-10-15 16:58:08 -05:00 committed by GitHub
commit 4c4bb5a489
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 54 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/ref/configuration/master.rst
View File

@ -2402,6 +2402,12 @@ Master will not be returned to the Minion.
------------------------------
.. versionadded:: 2014.1.0
.. deprecated:: 2018.3.4
This option is now ignored. Firstly, it only traversed
:conf_master:`file_roots`, which means it did not work for the other
fileserver backends. Secondly, since this option was added we have added
caching to the code that traverses the file_roots (and gitfs, etc.), which
greatly reduces the amount of traversal that is done.
Default: ``False``

103
salt/fileclient.py
View File

@ -61,7 +61,7 @@ def get_file_client(opts, pillar=False):
return {
'remote': RemoteClient,
'local': FSClient,
'pillar': LocalClient,
'pillar': PillarClient,
}.get(client, RemoteClient)(opts)
@ -347,58 +347,17 @@ class Client(object):
Return a list of all available sls modules on the master for a given
environment
'''
limit_traversal = self.opts.get('fileserver_limit_traversal', False)
states = []
if limit_traversal:
if saltenv not in self.opts['file_roots']:
log.warning(
'During an attempt to list states for saltenv \'%s\', '
'the environment could not be found in the configured '
'file roots', saltenv
)
return states
for path in self.opts['file_roots'][saltenv]:
for root, dirs, files in os.walk(path, topdown=True): # future lint: disable=blacklisted-function
root = salt.utils.data.decode(root)
files = salt.utils.data.decode(files)
log.debug(
'Searching for states in dirs %s and files %s',
salt.utils.data.decode(dirs), files
)
if not [filename.endswith('.sls') for filename in files]:
# Use shallow copy so we don't disturb the memory used
# by os.walk. Otherwise this breaks!
del dirs[:]
else:
for found_file in files:
stripped_root = os.path.relpath(root, path)
if salt.utils.platform.is_windows():
stripped_root = stripped_root.replace('\\', '/')
stripped_root = stripped_root.replace('/', '.')
if found_file.endswith(('.sls')):
if found_file.endswith('init.sls'):
if stripped_root.endswith('.'):
stripped_root = stripped_root.rstrip('.')
states.append(stripped_root)
else:
if not stripped_root.endswith('.'):
stripped_root += '.'
if stripped_root.startswith('.'):
stripped_root = stripped_root.lstrip('.')
states.append(stripped_root + found_file[:-4])
else:
for path in self.file_list(saltenv):
if salt.utils.platform.is_windows():
path = path.replace('\\', '/')
if path.endswith('.sls'):
# is an sls module!
if path.endswith('/init.sls'):
states.append(path.replace('/', '.')[:-9])
else:
states.append(path.replace('/', '.')[:-4])
return states
states = set()
for path in self.file_list(saltenv):
if salt.utils.platform.is_windows():
path = path.replace('\\', '/')
if path.endswith('.sls'):
# is an sls module!
if path.endswith('/init.sls'):
states.add(path.replace('/', '.')[:-9])
else:
states.add(path.replace('/', '.')[:-4])
return sorted(states)
def get_state(self, sls, saltenv, cachedir=None):
'''
@ -844,13 +803,10 @@ class Client(object):
)
class LocalClient(Client):
class PillarClient(Client):
'''
Use the local_roots option to parse a local file root
Used by pillar to handle fileclient requests
'''
def __init__(self, opts):
Client.__init__(self, opts)
def _find_file(self, path, saltenv='base'):
'''
Locate the file path
@ -858,12 +814,12 @@ class LocalClient(Client):
fnd = {'path': '',
'rel': ''}
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return fnd
if salt.utils.url.is_escaped(path):
# The path arguments are escaped
path = salt.utils.url.unescape(path)
for root in self.opts['file_roots'][saltenv]:
for root in self.opts['pillar_roots'][saltenv]:
full = os.path.join(root, path)
if os.path.isfile(full):
fnd['path'] = full
@ -896,10 +852,10 @@ class LocalClient(Client):
with optional relative prefix path to limit directory traversal
'''
ret = []
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return ret
prefix = prefix.strip('/')
for path in self.opts['file_roots'][saltenv]:
for path in self.opts['pillar_roots'][saltenv]:
for root, dirs, files in salt.utils.path.os_walk(
os.path.join(path, prefix), followlinks=True
):
@ -912,14 +868,14 @@ class LocalClient(Client):
def file_list_emptydirs(self, saltenv='base', prefix=''):
'''
List the empty dirs in the file_roots
List the empty dirs in the pillar_roots
with optional relative prefix path to limit directory traversal
'''
ret = []
prefix = prefix.strip('/')
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return ret
for path in self.opts['file_roots'][saltenv]:
for path in self.opts['pillar_roots'][saltenv]:
for root, dirs, files in salt.utils.path.os_walk(
os.path.join(path, prefix), followlinks=True
):
@ -931,14 +887,14 @@ class LocalClient(Client):
def dir_list(self, saltenv='base', prefix=''):
'''
List the dirs in the file_roots
List the dirs in the pillar_roots
with optional relative prefix path to limit directory traversal
'''
ret = []
if saltenv not in self.opts['file_roots']:
if saltenv not in self.opts['pillar_roots']:
return ret
prefix = prefix.strip('/')
for path in self.opts['file_roots'][saltenv]:
for path in self.opts['pillar_roots'][saltenv]:
for root, dirs, files in salt.utils.path.os_walk(
os.path.join(path, prefix), followlinks=True
):
@ -965,7 +921,7 @@ class LocalClient(Client):
def hash_file(self, path, saltenv='base'):
'''
Return the hash of a file, to get the hash of a file in the file_roots
Return the hash of a file, to get the hash of a file in the pillar_roots
prepend the path with salt://<file on server> otherwise, prepend the
file with / for a local file.
'''
@ -988,7 +944,7 @@ class LocalClient(Client):
def hash_and_stat_file(self, path, saltenv='base'):
'''
Return the hash of a file, to get the hash of a file in the file_roots
Return the hash of a file, to get the hash of a file in the pillar_roots
prepend the path with salt://<file on server> otherwise, prepend the
file with / for a local file.
@ -1034,7 +990,7 @@ class LocalClient(Client):
Return the available environments
'''
ret = []
for saltenv in self.opts['file_roots']:
for saltenv in self.opts['pillar_roots']:
ret.append(saltenv)
return ret
@ -1428,6 +1384,11 @@ class FSClient(RemoteClient):
self.auth = DumbAuth()
# Provide backward compatibility for anyone directly using LocalClient (but no
# one should be doing this).
LocalClient = FSClient
class DumbAuth(object):
'''
The dumbauth class is used to stub out auth calls fired from the FSClient

28
salt/pillar/__init__.py
View File

@ -345,8 +345,6 @@ class Pillar(object):
if pillarenv is None:
if opts.get('pillarenv_from_saltenv', False):
opts['pillarenv'] = saltenv
# Store the file_roots path so we can restore later. Issue 5449
self.actual_file_roots = opts['file_roots']
# use the local file client
self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv)
self.saltenv = saltenv
@ -369,9 +367,6 @@ class Pillar(object):
self.matcher = salt.minion.Matcher(self.opts, self.functions)
self.rend = salt.loader.render(self.opts, self.functions)
ext_pillar_opts = copy.deepcopy(self.opts)
# Fix self.opts['file_roots'] so that ext_pillars know the real
# location of file_roots. Issue 5951
ext_pillar_opts['file_roots'] = self.actual_file_roots
# Keep the incoming opts ID intact, ie, the master id
if 'id' in opts:
ext_pillar_opts['id'] = opts['id']
@ -438,7 +433,6 @@ class Pillar(object):
The options need to be altered to conform to the file client
'''
opts = copy.deepcopy(opts_in)
opts['file_roots'] = opts['pillar_roots']
opts['file_client'] = 'local'
if not grains:
opts['grains'] = {}
@ -463,15 +457,15 @@ class Pillar(object):
opts['ext_pillar'].append(self.ext)
else:
opts['ext_pillar'] = [self.ext]
if '__env__' in opts['file_roots']:
if '__env__' in opts['pillar_roots']:
env = opts.get('pillarenv') or opts.get('saltenv') or 'base'
if env not in opts['file_roots']:
if env not in opts['pillar_roots']:
log.debug("pillar environment '%s' maps to __env__ pillar_roots directory", env)
opts['file_roots'][env] = opts['file_roots'].pop('__env__')
opts['pillar_roots'][env] = opts['pillar_roots'].pop('__env__')
else:
log.debug("pillar_roots __env__ ignored (environment '%s' found in pillar_roots)",
env)
opts['file_roots'].pop('__env__')
opts['pillar_roots'].pop('__env__')
return opts
def _get_envs(self):
@ -479,8 +473,8 @@ class Pillar(object):
Pull the file server environments out of the master options
'''
envs = set(['base'])
if 'file_roots' in self.opts:
envs.update(list(self.opts['file_roots']))
if 'pillar_roots' in self.opts:
envs.update(list(self.opts['pillar_roots']))
return envs
def get_tops(self):
@ -496,11 +490,11 @@ class Pillar(object):
if self.opts['pillarenv']:
# If the specified pillarenv is not present in the available
# pillar environments, do not cache the pillar top file.
if self.opts['pillarenv'] not in self.opts['file_roots']:
if self.opts['pillarenv'] not in self.opts['pillar_roots']:
log.debug(
'pillarenv \'%s\' not found in the configured pillar '
'environments (%s)',
self.opts['pillarenv'], ', '.join(self.opts['file_roots'])
self.opts['pillarenv'], ', '.join(self.opts['pillar_roots'])
)
else:
top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv'])
@ -1010,8 +1004,6 @@ class Pillar(object):
mopts = dict(self.opts)
if 'grains' in mopts:
mopts.pop('grains')
# Restore the actual file_roots path. Issue 5449
mopts['file_roots'] = self.actual_file_roots
mopts['saltversion'] = __version__
pillar['master'] = mopts
if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False):
@ -1038,10 +1030,6 @@ class Pillar(object):
if decrypt_errors:
pillar.setdefault('_errors', []).extend(decrypt_errors)
# Reset the file_roots for the renderers
for mod_name in sys.modules:
if mod_name.startswith('salt.loaded.int.render.'):
sys.modules[mod_name].__opts__['file_roots'] = self.actual_file_roots
return pillar
def decrypt_pillar(self, pillar):

8
salt/utils/jinja.py
View File

@ -59,16 +59,16 @@ class SaltCacheLoader(BaseLoader):
self.opts = opts
self.saltenv = saltenv
self.encoding = encoding
if self.opts['file_roots'] is self.opts['pillar_roots']:
if saltenv not in self.opts['file_roots']:
self.pillar_rend = pillar_rend
if self.pillar_rend:
if saltenv not in self.opts['pillar_roots']:
self.searchpath = []
else:
self.searchpath = opts['file_roots'][saltenv]
self.searchpath = opts['pillar_roots'][saltenv]
else:
self.searchpath = [os.path.join(opts['cachedir'], 'files', saltenv)]
log.debug('Jinja search path: %s', self.searchpath)
self.cached = []
self.pillar_rend = pillar_rend
self._file_client = None
# Instantiate the fileclient
self.file_client()

4
tests/integration/pillar/test_git_pillar.py
View File

@ -101,13 +101,13 @@ from salt.utils.gitfs import (
# Check for requisite components
try:
HAS_GITPYTHON = GITPYTHON_VERSION >= GITPYTHON_MINVER
except ImportError:
except Exception:
HAS_GITPYTHON = False
try:
HAS_PYGIT2 = PYGIT2_VERSION >= PYGIT2_MINVER \
and LIBGIT2_VERSION >= LIBGIT2_MINVER
except AttributeError:
except Exception:
HAS_PYGIT2 = False
HAS_SSHD = bool(salt.utils.path.which('sshd'))

21
tests/unit/fileserver/test_roots.py
View File

@ -168,24 +168,3 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix
finally:
if self.test_symlink_list_file_roots:
self.opts['file_roots'] = orig_file_roots
class RootsLimitTraversalTest(TestCase, AdaptedConfigurationTestCaseMixin):
def test_limit_traversal(self):
'''
1) Set up a deep directory structure
2) Enable the configuration option 'fileserver_limit_traversal'
3) Ensure that we can find SLS files in a directory so long as there is
an SLS file in a directory above.
4) Ensure that we cannot find an SLS file in a directory that does not
have an SLS file in a directory above.
'''
file_client_opts = self.get_temp_config('master')
file_client_opts['fileserver_limit_traversal'] = True
ret = salt.fileclient.Client(file_client_opts).list_states('base')
self.assertIn('test_deep.test', ret)
self.assertIn('test_deep.a.test', ret)
self.assertNotIn('test_deep.b.2.test', ret)

4
tests/unit/test_pillar.py
View File

@ -314,7 +314,7 @@ class PillarTestCase(TestCase):
'extension_modules': '',
}
pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'base', pillarenv='dev')
self.assertEqual(pillar.opts['file_roots'],
self.assertEqual(pillar.opts['pillar_roots'],
{'base': ['/srv/pillar/base'], 'dev': ['/srv/pillar/__env__']})
def test_ignored_dynamic_pillarenv(self):
@ -329,7 +329,7 @@ class PillarTestCase(TestCase):
'extension_modules': '',
}
pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'base', pillarenv='base')
self.assertEqual(pillar.opts['file_roots'], {'base': ['/srv/pillar/base']})
self.assertEqual(pillar.opts['pillar_roots'], {'base': ['/srv/pillar/base']})
def test_malformed_pillar_sls(self):
with patch('salt.pillar.compile_template') as compile_template: