From d267569a197da9795ca8a411fcc04c29c704c755 Mon Sep 17 00:00:00 2001 From: Marlowe W Date: Thu, 25 May 2017 15:18:27 +0800 Subject: [PATCH 01/12] pillar: target's state list support wildcard in top.sls --- salt/pillar/__init__.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 6b1c2b9154..cee40d5ebc 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -6,6 +6,7 @@ Render the pillar data # Import python libs from __future__ import absolute_import import copy +import fnmatch import os import collections import logging @@ -289,6 +290,7 @@ class Pillar(object): self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv) self.saltenv = saltenv self.client = salt.fileclient.get_file_client(self.opts, True) + self.avail = self.__gather_avail() if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -359,6 +361,15 @@ class Pillar(object): return False return True + def __gather_avail(self): + ''' + Gather the lists of available sls data from the master + ''' + avail = {} + for saltenv in self._get_envs(): + avail[saltenv] = self.client.list_states(saltenv) + return avail + def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): ''' The options need to be altered to conform to the file client @@ -722,8 +733,20 @@ class Pillar(object): if errors is None: errors = [] for saltenv, pstates in six.iteritems(matches): + pstatesfile = [] mods = set() - for sls in pstates: + for sls_match in pstates: + try: + pstatefiles = fnmatch.filter(self.avail[saltenv], sls_match) + except KeyError: + errors.extend( + ['No matching pillar environment for environment ' + '\'{0}\' found'.format(saltenv)] + ) + if not pstatefiles: + pstatefiles = [sls_match] + + for sls in pstatefiles: pstate, mods, err = self.render_pstate(sls, saltenv, mods) if err: From 21af35d7fee3c86fc9b618d365d71af42666aebe Mon Sep 17 00:00:00 2001 From: Marlowe W Date: Fri, 26 May 2017 10:06:06 +0800 Subject: [PATCH 02/12] fix integration test --- salt/pillar/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index cee40d5ebc..f27eaa7c30 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -733,18 +733,21 @@ class Pillar(object): if errors is None: errors = [] for saltenv, pstates in six.iteritems(matches): - pstatesfile = [] + pstatefiles = [] mods = set() for sls_match in pstates: + matched_pstates = [] try: - pstatefiles = fnmatch.filter(self.avail[saltenv], sls_match) + matched_pstates = fnmatch.filter(self.avail[saltenv], sls_match) except KeyError: errors.extend( ['No matching pillar environment for environment ' '\'{0}\' found'.format(saltenv)] ) - if not pstatefiles: - pstatefiles = [sls_match] + if matched_pstates: + pstatefiles.extend(matched_pstates) + else: + pstatefiles.append(sls_match) for sls in pstatefiles: pstate, mods, err = self.render_pstate(sls, saltenv, mods) From d66281bda60e9fed9d01825b3353214d96a4074e Mon Sep 17 00:00:00 2001 From: Marlowe W Date: Fri, 26 May 2017 13:41:21 +0800 Subject: [PATCH 03/12] pillar: fix tests --- tests/integration/pillar/test_git_pillar.py | 128 ++++++++++++++++++++ tests/unit/pillar/test_git.py | 2 + tests/unit/test_pillar.py | 10 +- 3 files changed, 138 insertions(+), 2 deletions(-) diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py index b1024abd20..ad3394c4e5 100644 --- a/tests/integration/pillar/test_git_pillar.py +++ b/tests/integration/pillar/test_git_pillar.py @@ -127,6 +127,8 @@ class GitPythonMixin(object): Test using a single ext_pillar repo ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython cachedir: {cachedir} extension_modules: {extmods} @@ -152,6 +154,8 @@ class GitPythonMixin(object): pillar_merge_lists disabled. ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython cachedir: {cachedir} extension_modules: {extmods} @@ -180,6 +184,8 @@ class GitPythonMixin(object): pillar_merge_lists disabled. ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython cachedir: {cachedir} extension_modules: {extmods} @@ -208,6 +214,8 @@ class GitPythonMixin(object): pillar_merge_lists enabled. ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython cachedir: {cachedir} extension_modules: {extmods} @@ -236,6 +244,8 @@ class GitPythonMixin(object): pillar_merge_lists enabled. ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython cachedir: {cachedir} extension_modules: {extmods} @@ -260,6 +270,8 @@ class GitPythonMixin(object): Test using pillarenv to restrict results to those from a single branch ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython cachedir: {cachedir} extension_modules: {extmods} @@ -285,6 +297,8 @@ class GitPythonMixin(object): SLS file (included_pillar) in the compiled pillar data. ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython cachedir: {cachedir} extension_modules: {extmods} @@ -313,6 +327,8 @@ class GitPythonMixin(object): message in the compiled data. ''' ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: gitpython git_pillar_includes: False cachedir: {cachedir} @@ -433,6 +449,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} git_pillar_privkey: {privkey_nopass} @@ -446,6 +464,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -463,6 +483,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} git_pillar_privkey: {privkey_withpass} @@ -477,6 +499,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -509,6 +533,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} git_pillar_privkey: {privkey_nopass} @@ -524,6 +550,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -545,6 +573,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} git_pillar_privkey: {privkey_withpass} @@ -561,6 +591,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -598,6 +630,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} git_pillar_privkey: {privkey_nopass} @@ -613,6 +647,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -634,6 +670,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} git_pillar_privkey: {privkey_withpass} @@ -650,6 +688,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -687,6 +727,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} git_pillar_privkey: {privkey_nopass} @@ -702,6 +744,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -723,6 +767,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} git_pillar_privkey: {privkey_withpass} @@ -739,6 +785,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -776,6 +824,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} git_pillar_privkey: {privkey_nopass} @@ -791,6 +841,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -812,6 +864,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} git_pillar_privkey: {privkey_withpass} @@ -828,6 +882,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -860,6 +916,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} git_pillar_privkey: {privkey_nopass} @@ -875,6 +933,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -896,6 +956,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} git_pillar_privkey: {privkey_withpass} @@ -912,6 +974,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -947,6 +1011,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} git_pillar_privkey: {privkey_nopass} @@ -962,6 +1028,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -983,6 +1051,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} git_pillar_privkey: {privkey_withpass} @@ -999,6 +1069,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1037,6 +1109,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_includes: False git_pillar_pubkey: {pubkey_nopass} @@ -1053,6 +1127,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_includes: False cachedir: {cachedir} @@ -1075,6 +1151,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_includes: False git_pillar_pubkey: {pubkey_withpass} @@ -1092,6 +1170,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_includes: False cachedir: {cachedir} @@ -1140,6 +1220,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1167,6 +1249,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1196,6 +1280,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1225,6 +1311,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1254,6 +1342,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1278,6 +1368,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1305,6 +1397,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1335,6 +1429,8 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): } ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_includes: False cachedir: {cachedir} @@ -1384,6 +1480,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_user: {user} git_pillar_password: {password} @@ -1398,6 +1496,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1429,6 +1529,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_user: {user} git_pillar_password: {password} @@ -1445,6 +1547,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1481,6 +1585,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_user: {user} git_pillar_password: {password} @@ -1497,6 +1603,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1533,6 +1641,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_user: {user} git_pillar_password: {password} @@ -1549,6 +1659,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1585,6 +1697,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_user: {user} git_pillar_password: {password} @@ -1601,6 +1715,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1632,6 +1748,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_user: {user} git_pillar_password: {password} @@ -1648,6 +1766,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1682,6 +1802,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_user: {user} git_pillar_password: {password} @@ -1698,6 +1820,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 cachedir: {cachedir} extension_modules: {extmods} @@ -1735,6 +1859,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with global credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_includes: False git_pillar_user: {user} @@ -1752,6 +1878,8 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): # Test with per-repo credential options ret = self.get_pillar('''\ + file_ignore_regex: [] + file_ignore_glob: [] git_pillar_provider: pygit2 git_pillar_includes: False cachedir: {cachedir} diff --git a/tests/unit/pillar/test_git.py b/tests/unit/pillar/test_git.py index 7c55175416..1ea9185c84 100644 --- a/tests/unit/pillar/test_git.py +++ b/tests/unit/pillar/test_git.py @@ -56,6 +56,8 @@ class GitPillarTestCase(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModul 'cachedir': cachedir, 'pillar_roots': {}, 'hash_type': 'sha256', + 'file_ignore_regex': [], + 'file_ignore_glob': [], 'file_roots': {}, 'state_top': 'top.sls', 'extension_modules': '', diff --git a/tests/unit/test_pillar.py b/tests/unit/test_pillar.py index 2c385f8085..4cfaa6ef3b 100644 --- a/tests/unit/test_pillar.py +++ b/tests/unit/test_pillar.py @@ -37,8 +37,14 @@ class PillarTestCase(TestCase): 'renderer_blacklist': [], 'renderer_whitelist': [], 'state_top': '', - 'pillar_roots': ['dev', 'base'], - 'file_roots': ['dev', 'base'], + 'pillar_roots': { + 'dev': [], + 'base': [] + }, + 'file_roots': { + 'dev': [], + 'base': [] + }, 'extension_modules': '', 'pillarenv_from_saltenv': True } From c4ae14446ca1541e32402402d0fd63f56678a0d6 Mon Sep 17 00:00:00 2001 From: Andreas Tsaridas Date: Fri, 26 May 2017 18:36:03 +0200 Subject: [PATCH 04/12] try to fix scheduled jobs --- salt/config/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index bd905d3935..6b6e19a95b 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2051,7 +2051,11 @@ def include_config(include, orig_path, verbose, exit_on_config_errors=False): else: # Initialize default config if we wish to skip config errors opts = {} - + schedule = opts.get('schedule', {}) + if schedule and 'schedule' in configuration: + for i in schedule: + if i in configuration['schedule']: + configuration['schedule'].pop(i) include = opts.get('include', []) if include: opts.update(include_config(include, fn_, verbose)) From 5bd77be6965dfcd3b41b0ca27c865a151cd754dc Mon Sep 17 00:00:00 2001 From: Andreas Tsaridas Date: Fri, 26 May 2017 22:34:41 +0200 Subject: [PATCH 05/12] easy fix --- salt/config/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 6b6e19a95b..81f7718324 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2053,9 +2053,7 @@ def include_config(include, orig_path, verbose, exit_on_config_errors=False): opts = {} schedule = opts.get('schedule', {}) if schedule and 'schedule' in configuration: - for i in schedule: - if i in configuration['schedule']: - configuration['schedule'].pop(i) + configuration['schedule'].update(schedule) include = opts.get('include', []) if include: opts.update(include_config(include, fn_, verbose)) From c362dbc2cf8ac57df1cf1d2dff34b7975e5e9536 Mon Sep 17 00:00:00 2001 From: m03 Date: Tue, 30 May 2017 00:03:51 -0700 Subject: [PATCH 07/12] Improve http_json and http_yaml external pillars --- salt/pillar/http_json.py | 63 +++++++++++++++++++++++++++++++-------- salt/pillar/http_yaml.py | 64 ++++++++++++++++++++++++++++++++-------- 2 files changed, 103 insertions(+), 24 deletions(-) diff --git a/salt/pillar/http_json.py b/salt/pillar/http_json.py index 362da3daa7..d931d4a3d0 100644 --- a/salt/pillar/http_json.py +++ b/salt/pillar/http_json.py @@ -6,7 +6,7 @@ A module that adds data to the Pillar structure retrieved by an http request Configuring the HTTP_JSON ext_pillar ==================================== -Set the following Salt config to setup Foreman as external pillar source: +Set the following Salt config to setup http json result as external pillar source: .. code-block:: yaml @@ -17,6 +17,16 @@ Set the following Salt config to setup Foreman as external pillar source: username: username password: password +If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped +in <> brackets) in the url in order to populate pillar data based on the grain value. + +.. code-block:: yaml + + ext_pillar: + - http_json: + url: http://example.com/api/ + with_grains: True + Module Documentation ==================== ''' @@ -24,32 +34,61 @@ Module Documentation # Import python libs from __future__ import absolute_import import logging +import re # Import Salt libs -import salt.ext.six as six +try: + from salt.ext.six.moves.urllib.parse import quote as _quote + _HAS_DEPENDENCIES = True +except ImportError: + _HAS_DEPENDENCIES = False + +# Set up logging +_LOG = logging.getLogger(__name__) + + +def __virtual__(): + return _HAS_DEPENDENCIES def ext_pillar(minion_id, pillar, # pylint: disable=W0613 - url=None): + url, + with_grains=False): ''' Read pillar data from HTTP response. - :param url String to make request - :returns dict with pillar data to add - :returns empty if error - ''' - # Set up logging - log = logging.getLogger(__name__) + :param str url: Url to request. + :param bool with_grains: Whether to substitute strings in the url with their grain values. + :return: A dictionary of the pillar data to add. + :rtype: dict + ''' + grain_pattern = r'<(?P.*?)>' + + if with_grains: + # Get the value of the grain and substitute each grain + # name for the url-encoded version of its grain value. + for match in re.finditer(grain_pattern, url): + grain_name = match.group('grain_name') + grain_value = __salt__['grains.get'](grain_name, None) + + if not grain_value: + _LOG.error("Unable to get minion '%s' grain: %s", minion_id, grain_name) + return {} + + grain_value = _quote(str(grain_value)) + url = re.sub('<{0}>'.format(grain_name), grain_value, url) + + _LOG.debug('Getting url: %s', url) data = __salt__['http.query'](url=url, decode=True, decode_type='json') if 'dict' in data: return data['dict'] - log.error('Error caught on query to' + url + '\nMore Info:\n') + _LOG.error("Error on minion '%s' http query: %s\nMore Info:\n", minion_id, url) - for k, v in six.iteritems(data): - log.error(k + ' : ' + v) + for key in data: + _LOG.error('%s: %s', key, data[key]) return {} diff --git a/salt/pillar/http_yaml.py b/salt/pillar/http_yaml.py index ed961b69b5..f62d298ea2 100644 --- a/salt/pillar/http_yaml.py +++ b/salt/pillar/http_yaml.py @@ -17,6 +17,16 @@ Set the following Salt config to setup an http endpoint as the external pillar s username: username password: password +If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped +in <> brackets) in the url in order to populate pillar data based on the grain value. + +.. code-block:: yaml + + ext_pillar: + - http_yaml: + url: http://example.com/api/ + with_grains: True + Module Documentation ==================== ''' @@ -24,32 +34,62 @@ Module Documentation # Import python libs from __future__ import absolute_import import logging +import re # Import Salt libs -import salt.ext.six as six +try: + from salt.ext.six.moves.urllib.parse import quote as _quote + _HAS_DEPENDENCIES = True +except ImportError: + _HAS_DEPENDENCIES = False + + +# Set up logging +_LOG = logging.getLogger(__name__) + + +def __virtual__(): + return _HAS_DEPENDENCIES def ext_pillar(minion_id, pillar, # pylint: disable=W0613 - url): - """ + url, + with_grains=False): + ''' Read pillar data from HTTP response. - :param url String to make request - :returns dict with pillar data to add - :returns empty if error - """ - # Set up logging - log = logging.getLogger(__name__) + :param str url: Url to request. + :param bool with_grains: Whether to substitute strings in the url with their grain values. + :return: A dictionary of the pillar data to add. + :rtype: dict + ''' + grain_pattern = r'<(?P.*?)>' + + if with_grains: + # Get the value of the grain and substitute each grain + # name for the url-encoded version of its grain value. + for match in re.finditer(grain_pattern, url): + grain_name = match.group('grain_name') + grain_value = __salt__['grains.get'](grain_name, None) + + if not grain_value: + _LOG.error("Unable to get minion '%s' grain: %s", minion_id, grain_name) + return {} + + grain_value = _quote(str(grain_value)) + url = re.sub('<{0}>'.format(grain_name), grain_value, url) + + _LOG.debug('Getting url: %s', url) data = __salt__['http.query'](url=url, decode=True, decode_type='yaml') if 'dict' in data: return data['dict'] - log.error('Error caught on query to' + url + '\nMore Info:\n') + _LOG.error("Error on minion '%s' http query: %s\nMore Info:\n", minion_id, url) - for k, v in six.iteritems(data): - log.error(k + ' : ' + v) + for key in data: + _LOG.error('%s: %s', key, data[key]) return {} From 1ca90885d05f4ca957dbf777053626278a7fb19f Mon Sep 17 00:00:00 2001 From: Neil Rhine Date: Tue, 6 Jun 2017 18:51:18 -0700 Subject: [PATCH 08/12] Adding ability to specify an encryption key for KMS --- salt/cloud/clouds/ec2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index 061f3e2c3e..3ddd46c885 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -2863,6 +2863,8 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): volume_dict['iops'] = volume['iops'] if 'encrypted' in volume: volume_dict['encrypted'] = volume['encrypted'] + if 'kmskeyid' in volume: + volume_dict['kmskeyid'] = volume['kmskeyid'] if 'volume_id' not in volume_dict: created_volume = create_volume(volume_dict, call='function', wait_to_finish=wait_to_finish) @@ -4059,6 +4061,13 @@ def create_volume(kwargs=None, call=None, wait_to_finish=False): # You can't set `encrypted` if you pass a snapshot if 'encrypted' in kwargs and 'snapshot' not in kwargs: params['Encrypted'] = kwargs['encrypted'] + if 'kmskeyid' in kwargs: + params['KmsKeyId'] = kwargs['kmskeyid'] + if 'kmskeyid' in kwargs and 'encrypted' not in kwargs: + log.error( + 'If a KMS Key ID is specified, encryption must be enabled' + ) + return False log.debug(params) From 7cc965aec4b346aad672fb12dae408d8eedbf8bf Mon Sep 17 00:00:00 2001 From: "Markus Kramer (DPDHL IT Services)" Date: Thu, 8 Jun 2017 13:29:50 +0200 Subject: [PATCH 09/12] Use dump instead of dumps for json files --- salt/returners/rawfile_json.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/returners/rawfile_json.py b/salt/returners/rawfile_json.py index f6144fbae1..26db42cb9c 100644 --- a/salt/returners/rawfile_json.py +++ b/salt/returners/rawfile_json.py @@ -72,7 +72,8 @@ def event_return(events): try: with salt.utils.flopen(opts['filename'], 'a') as logfile: for event in events: - logfile.write(str(json.dumps(event))+'\n') + json.dump(event, logfile) + logfile.write('\n') except: log.error('Could not write to rawdata_json file {0}'.format(opts['filename'])) raise From 87f87b781ee5d0ce578c25085f66335c17de26ab Mon Sep 17 00:00:00 2001 From: Vasili Syrakis Date: Fri, 9 Jun 2017 00:11:19 +1000 Subject: [PATCH 10/12] Allowing salted annotation stamp on VMWare VM #35568 --- salt/cloud/clouds/vmware.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/cloud/clouds/vmware.py b/salt/cloud/clouds/vmware.py index 7b646525f1..4b4bb24625 100644 --- a/salt/cloud/clouds/vmware.py +++ b/salt/cloud/clouds/vmware.py @@ -2359,6 +2359,9 @@ def create(vm_): extra_config = config.get_cloud_config_value( 'extra_config', vm_, __opts__, default=None ) + annotation = config.get_cloud_config_value( + 'annotation', vm_, __opts__, default=None + ) power = config.get_cloud_config_value( 'power_on', vm_, __opts__, default=True ) @@ -2569,6 +2572,9 @@ def create(vm_): option = vim.option.OptionValue(key=key, value=value) config_spec.extraConfig.append(option) + if annotation: + config_spec.annotation = annotation + if 'clonefrom' in vm_: clone_spec = handle_snapshot( config_spec, From 4fbcc3b31882fa53409a026fbe16de00d3f6c8ed Mon Sep 17 00:00:00 2001 From: Vasili Syrakis Date: Fri, 9 Jun 2017 00:25:10 +1000 Subject: [PATCH 11/12] Added documentation for the annotation --- doc/topics/cloud/vmware.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/topics/cloud/vmware.rst b/doc/topics/cloud/vmware.rst index a039708598..2b510528a2 100644 --- a/doc/topics/cloud/vmware.rst +++ b/doc/topics/cloud/vmware.rst @@ -194,6 +194,7 @@ Set up an initial profile at ``/etc/salt/cloud.profiles`` or guestinfo.foo: bar guestinfo.domain: foobar.com guestinfo.customVariable: customValue + annotation: Created by Salt-Cloud deploy: True customization: True @@ -451,6 +452,11 @@ Set up an initial profile at ``/etc/salt/cloud.profiles`` or present, it will be reset with the new value provided. Otherwise, a new option is added. Keys with empty values will be removed. +``annotation`` + User-provided description of the virtual machine. This will store a message in the + vSphere interface, under the annotations section in the Summary view of the virtual + machine. + ``deploy`` Specifies if salt should be installed on the newly created VM. Default is ``True`` so salt will be installed using the bootstrap script. If ``template: True`` or From 27a49edd3484483f153403976fe4905ad90bd8df Mon Sep 17 00:00:00 2001 From: Vasili Syrakis Date: Fri, 9 Jun 2017 01:27:20 +1000 Subject: [PATCH 12/12] explicitly cast annotation to string --- salt/cloud/clouds/vmware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/clouds/vmware.py b/salt/cloud/clouds/vmware.py index 4b4bb24625..9736b91686 100644 --- a/salt/cloud/clouds/vmware.py +++ b/salt/cloud/clouds/vmware.py @@ -2573,7 +2573,7 @@ def create(vm_): config_spec.extraConfig.append(option) if annotation: - config_spec.annotation = annotation + config_spec.annotation = str(annotation) if 'clonefrom' in vm_: clone_spec = handle_snapshot(