From f9a116ee9c3666a9dda6a3b34b4ca3c8c99d605b Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Tue, 25 Feb 2014 19:52:47 +0000 Subject: [PATCH 01/39] Support multiline cron comments This fixes #10721. --- salt/modules/cron.py | 3 ++- tests/unit/states/cron_test.py | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/salt/modules/cron.py b/salt/modules/cron.py index 40a1e5d1fd..e46f72eb7a 100644 --- a/salt/modules/cron.py +++ b/salt/modules/cron.py @@ -94,7 +94,8 @@ def _render_tab(lst): if cron['comment'] is not None or cron['identifier'] is not None: comment = '#' if cron['comment']: - comment += ' {0}'.format(cron['comment']) + comment += ' {0}'.format( + cron['comment'].rstrip().replace('\n', '\n# ')) if cron['identifier']: comment += ' {0}:{1}'.format(SALT_CRON_IDENTIFIER, cron['identifier']) diff --git a/tests/unit/states/cron_test.py b/tests/unit/states/cron_test.py index 585adce47e..fe1aa79054 100644 --- a/tests/unit/states/cron_test.py +++ b/tests/unit/states/cron_test.py @@ -76,6 +76,14 @@ def write_crontab(*args, **kw): @skipIf(NO_MOCK, NO_MOCK_REASON) class CronTestCase(TestCase): + def setUp(self): + super(CronTestCase, self).setUp() + set_crontab('') + + def tearDown(self): + super(CronTestCase, self).tearDown() + set_crontab('') + @patch('salt.modules.cron.raw_cron', new=MagicMock(side_effect=get_crontab)) @patch('salt.modules.cron._write_cron_lines', @@ -178,6 +186,42 @@ class CronTestCase(TestCase): '# Lines below here are managed by Salt, do not edit' ) + @patch('salt.modules.cron.raw_cron', + new=MagicMock(side_effect=get_crontab)) + @patch('salt.modules.cron._write_cron_lines', + new=MagicMock(side_effect=write_crontab)) + def test_aissue_1072(self): + ( + '# Lines below here are managed by Salt, do not edit\n' + '# I have a multi-line comment SALT_CRON_IDENTIFIER:1\n' + '* 1 * * * foo' + ) + cron.present( + name='foo', + hour='1', + comment='1I have a multi-line comment\n2about my script here.\n', + identifier='1', + user='root') + cron.present( + name='foo', + hour='1', + comment='3I have a multi-line comment\n3about my script here.\n', + user='root') + cron.present( + name='foo', + hour='1', + comment='I have a multi-line comment\nabout my script here.\n', + identifier='2', + user='root') + self.assertEqual( + get_crontab(), + '# Lines below here are managed by Salt, do not edit\n' + '# 2about my script here. SALT_CRON_IDENTIFIER:1\n' + '* 1 * * * foo\n' + '# I have a multi-line comment\n' + '# about my script here. SALT_CRON_IDENTIFIER:2\n' + '* 1 * * * foo') + if __name__ == '__main__': from integration import run_tests From ba444828d0dcaebd9f24e3bedc5887268186cf0d Mon Sep 17 00:00:00 2001 From: Daniel Bradshaw Date: Tue, 25 Feb 2014 20:55:17 +0000 Subject: [PATCH 02/39] This should really be an else --- salt/state.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/state.py b/salt/state.py index bab1f057df..46df34a6a8 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1380,11 +1380,9 @@ class State(object): if len(cdata['args']) > 0: name = cdata['args'][0] elif 'name' in cdata['kwargs']: - name = cdata['kwargs'].get( - 'name', - low.get('name', - low.get('__id__')) - ) + name = cdata['kwargs']['name'] + else: + name = low.get('name', low.get('__id__')) ret = { 'result': False, 'name': name, From ef48008bbc7e641edb2f4f304f749d5378c3de2e Mon Sep 17 00:00:00 2001 From: Vye Wilson Date: Tue, 25 Feb 2014 14:41:52 -0800 Subject: [PATCH 03/39] added explicit git revision support to winrepo --- salt/modules/win_repo.py | 5 +++++ salt/runners/winrepo.py | 5 +++++ salt/states/git.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_repo.py b/salt/modules/win_repo.py index 328d5ade3e..5fe1777ecd 100644 --- a/salt/modules/win_repo.py +++ b/salt/modules/win_repo.py @@ -105,9 +105,14 @@ def update_git_repos(): #else: #targetname = gitrepo targetname = gitrepo + rev = None + # If a revision is specified, use it. + if len(gitrepo.strip().split(' ')) > 1: + rev, gitrepo = gitrepo.strip().split(' ') gittarget = os.path.join(repo, targetname) #result = mminion.states['git.latest'](gitrepo, result = __salt__['git.latest'](gitrepo, + rev=rev, target=gittarget, force=True) ret[result['name']] = result['result'] diff --git a/salt/runners/winrepo.py b/salt/runners/winrepo.py index 8b7a780678..09650a3a5b 100644 --- a/salt/runners/winrepo.py +++ b/salt/runners/winrepo.py @@ -91,8 +91,13 @@ def update_git_repos(): targetname = gitrepo.split('/')[-1] else: targetname = gitrepo + rev = None + # If a revision is specified, use it. + if len(gitrepo.strip().split(' ')) > 1: + rev, gitrepo = gitrepo.strip().split(' ') gittarget = os.path.join(repo, targetname) result = mminion.states['git.latest'](gitrepo, + rev=rev, target=gittarget, force=True) ret[result['name']] = result['result'] diff --git a/salt/states/git.py b/salt/states/git.py index 8da78ec012..b27be56246 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -213,7 +213,7 @@ def latest(name, identity=identity) elif rev: - cmd = 'git rev-parse {0} ^{{commit}}'.format(rev) + cmd = 'git rev-parse {0}'.format(rev) retcode = __salt__['cmd.retcode'](cmd, cwd=target, runas=user) From 580d7eb3ac0b320789e946bbaa74c9cec6543621 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 25 Feb 2014 17:42:33 -0500 Subject: [PATCH 04/39] deleted duplicate parser option for -v, --verbose --- salt/utils/parsers.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 0e7c79c06c..1db648cd23 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -2176,13 +2176,6 @@ class SaltSSHOptionParser(OptionParser, ConfigDirMixIn, MergeConfigMixIn, 'been changed and the auto refresh timeframe has not been ' 'reached.') ) - self.add_option( - '-v', '--verbose', - default=False, - action='store_true', - help=('Turn on command verbosity, display jid') - ) - self.add_option( '--max-procs', dest='ssh_max_procs', From 2ce1f8d558ca5e433e30777baba969ccd02d0b02 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 15:54:41 -0700 Subject: [PATCH 05/39] Make salt-key work with raet keys --- salt/key.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/key.py b/salt/key.py index 3308bf5054..14c27da299 100644 --- a/salt/key.py +++ b/salt/key.py @@ -24,7 +24,10 @@ class KeyCLI(object): ''' def __init__(self, opts): self.opts = opts - self.key = Key(opts) + if self.opts['transport'] == 'zeromq': + self.key = Key(opts) + else: + self.key = RaetKey(opts) def list_status(self, status): ''' From 0b90fd2a5bc2323f803d9f9d9e87f3accfb4e1a6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 15:59:46 -0700 Subject: [PATCH 06/39] Create raet key dirs --- salt/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/__init__.py b/salt/__init__.py index 69eff73dd9..b443e1ec8d 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -71,6 +71,9 @@ class Master(parsers.MasterOptionParser): os.path.join(self.config['pki_dir'], 'minions'), os.path.join(self.config['pki_dir'], 'minions_pre'), os.path.join(self.config['pki_dir'], 'minions_denied'), + os.path.join(self.config['pki_dir'], 'accepted'), + os.path.join(self.config['pki_dir'], 'pending'), + os.path.join(self.config['pki_dir'], 'rejected'), os.path.join(self.config['pki_dir'], 'minions_rejected'), self.config['cachedir'], From b2c345290f89590eec786c79d958eb663a355454 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 17:02:12 -0700 Subject: [PATCH 07/39] Add auto_accept option to minion config (only works with raet handshake) --- salt/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/config.py b/salt/config.py index 9d17cc9f8a..88a0c2d3b9 100644 --- a/salt/config.py +++ b/salt/config.py @@ -253,6 +253,7 @@ DEFAULT_MINION_OPTS = { 'providers': {}, 'clean_dynamic_modules': True, 'open_mode': False, + 'auto_accept': True, 'multiprocessing': True, 'mine_interval': 60, 'ipc_mode': 'ipc', From e62862f9920875441056fd89d306d02070cfaa90 Mon Sep 17 00:00:00 2001 From: Nahum Shalman Date: Tue, 25 Feb 2014 13:22:34 -0500 Subject: [PATCH 08/39] update documentation about SmartOS esky packaging --- pkg/smartos/esky/BUILD_ENVIRONMENT.md | 26 +++++++++++--------------- pkg/smartos/esky/requirements.txt | 3 +++ 2 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 pkg/smartos/esky/requirements.txt diff --git a/pkg/smartos/esky/BUILD_ENVIRONMENT.md b/pkg/smartos/esky/BUILD_ENVIRONMENT.md index 2eda546a85..aac1967223 100644 --- a/pkg/smartos/esky/BUILD_ENVIRONMENT.md +++ b/pkg/smartos/esky/BUILD_ENVIRONMENT.md @@ -13,7 +13,7 @@ HERE=$(pwd) mv /opt/local /opt/local.backup ; hash -r cd / -curl http://pkgsrc.joyent.com/packages/SmartOS/bootstrap/bootstrap-2013Q3-x86_64.tar.gz | gtar xz +curl http://pkgsrc.joyent.com/packages/SmartOS/bootstrap/bootstrap-2013Q4-x86_64.tar.gz | gtar xz hash -r pkgin -y up @@ -23,13 +23,8 @@ pkgin -y rm salt cd /opt/local/bin curl -kO 'https://us-east.manta.joyent.com/nahamu/public/smartos/bins/patchelf' chmod +x patchelf -cat >swig <<"EOF" -#!/bin/bash -exec /opt/local/bin/swig2.0 -I/opt/local/include "$@" -EOF -pip install esky -yes | pip uninstall bbfreeze +pip install esky bbfreeze cd $HERE curl -kO 'https://pypi.python.org/packages/source/b/bbfreeze-loader/bbfreeze-loader-1.1.0.zip' @@ -41,16 +36,17 @@ $COMPILE -c bbfreeze-loader-1.1.0/_bbfreeze_loader/getpath.c -o $HERE/getpath.o gcc $HERE/console.o $HERE/getpath.o /opt/local/lib/python2.7/config/libpython2.7.a -L/opt/local/lib -L/opt/local/lib/python2.7/config -L/opt/local/lib -lsocket -lnsl -ldl -lrt -lm -static-libgcc -o $HERE/console.exe patchelf --set-rpath '$ORIGIN:$ORIGIN/../lib' $HERE/console.exe -git clone git://github.com/schmir/bbfreeze -b master -( cd $HERE/bbfreeze && easy_install-2.7 . ) find /opt/local -name console.exe -exec mv $HERE/console.exe {} \; -git clone git://github.com/saltstack/salt -b 0.17 -( cd $HERE/salt && python2.7 setup.py bdist && python2.7 setup.py bdist_esky ) - -mv /opt/local /opt/local.build ; hash -r -mv /opt/local.backup /opt/local ; hash -r +git clone git://github.com/saltstack/salt -b 2014.1 +cd $HERE/salt +pip install -r requirements.txt +# packages not in main requirements file that are nice to have +pip install -r pkg/smartos/esky/requirements.txt +bash pkg/smartos/esky/build-tarball.sh +# Upload packages into Manta +pkgin -y in sdc-manta mmkdir -p /$MANTA_USER/public/salt -mput /$MANTA_USER/public/salt -f $(ls salt/dist/*.zip) +for file in dist/salt*; do mput -m /$MANTA_USER/public/salt -f $file; done; ``` diff --git a/pkg/smartos/esky/requirements.txt b/pkg/smartos/esky/requirements.txt new file mode 100644 index 0000000000..5f5263b971 --- /dev/null +++ b/pkg/smartos/esky/requirements.txt @@ -0,0 +1,3 @@ +GitPython==0.3.2.RC1 +halite +cherrypy From e495b46aac94f9c57e7025eb788e0e07f4029e4b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 17:41:53 -0700 Subject: [PATCH 09/39] Add local read and write methods --- salt/key.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/salt/key.py b/salt/key.py index 14c27da299..c0cae7ebe0 100644 --- a/salt/key.py +++ b/salt/key.py @@ -991,3 +991,36 @@ class RaetKey(Key): path = os.path.join(self.opts['pki_dir'], status, key) ret[status][key] = self._get_key_finger(path) return ret + + def read_remote(self, minion_id): + ''' + Read in a remote accepted key + ''' + path = os.path.join(self.opts['pki_dir'], 'accepted', minion_id) + if not os.path.isfile(path): + return {} + with salt.utils.fopen(path, 'rb') as fp_: + return self.serial.loads(fp_.read()) + + def read_local(self): + ''' + Read in the local private keys, return an empy dict if the keys do not + exist + ''' + path = os.path.join(self.opts['pki_dir'], 'local.key') + if not os.path.isfile(path): + return {} + with salt.utils.fopen(path, 'rb') as fp_: + return self.serial.loads(fp_.read()) + + def write_local(self, priv, sign): + ''' + Write the private key and the signing key to a file on disk + ''' + keydata = {'priv': priv, + 'sign': sign} + path = os.path.join(self.opts['pki_dir'], 'local.key') + c_umask = os.umask(191) + with salt.utils.fopen(path, 'w+') as fp_: + fp_.write(self.serial.dumps(keydata)) + os.umask(c_umask) From 20eefa5ec6694e9b9e140a5907710a041e375e7e Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Tue, 25 Feb 2014 17:49:16 -0700 Subject: [PATCH 10/39] Fixing typo in formatted string --- salt/states/iptables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/iptables.py b/salt/states/iptables.py index 5b975c4a45..b1c36c5a14 100644 --- a/salt/states/iptables.py +++ b/salt/states/iptables.py @@ -450,7 +450,7 @@ def set_policy(name, family='ipv4', **kwargs): family): ret['changes'] = {'locale': name} ret['result'] = True - ret['comment'] = 'Set default policy for {0} to {1} family {2]'.format( + ret['comment'] = 'Set default policy for {0} to {1} family {2}'.format( kwargs['chain'], kwargs['policy'], family @@ -487,7 +487,7 @@ def flush(name, family='ipv4', **kwargs): if not __salt__['iptables.flush'](kwargs['table'], kwargs['chain'], family): ret['changes'] = {'locale': name} ret['result'] = True - ret['comment'] = 'Flush iptables rules in {0} table {1} chain {2] family'.format( + ret['comment'] = 'Flush iptables rules in {0} table {1} chain {2} family'.format( kwargs['table'], kwargs['chain'], family From 7823040a251e964190db9f68860837cfcd73467d Mon Sep 17 00:00:00 2001 From: tallmad Date: Wed, 26 Feb 2014 11:26:17 +0800 Subject: [PATCH 11/39] fix bug: manage_file dir_mode does not work --- salt/modules/file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index c8e0be23dc..20a94635fb 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -2635,7 +2635,8 @@ def manage_file(name, else: if not os.path.isdir(os.path.dirname(name)): if makedirs: - makedirs(name, user=user, group=group, mode=mode) + makedirs(name, user=user, group=group, + mode=dir_mode or mode) else: __clean_tmp(sfn) return _error(ret, 'Parent directory not present') From c5f294f1dfca75a83953b354a45eb0183efc7ec0 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 20:54:03 -0700 Subject: [PATCH 12/39] Add pillar pass-through to the state.sls runner --- salt/runners/state.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/runners/state.py b/salt/runners/state.py index 854af4ce7d..fa5e3bda6c 100644 --- a/salt/runners/state.py +++ b/salt/runners/state.py @@ -50,7 +50,7 @@ def over(saltenv='base', os_fn=None): return overstate.over_run -def sls(mods, saltenv='base', test=None, exclude=None): +def sls(mods, saltenv='base', test=None, exclude=None, pillar=None): ''' Execute a state run from the master, used as a powerful orchestration system. @@ -64,7 +64,12 @@ def sls(mods, saltenv='base', test=None, exclude=None): ''' __opts__['file_client'] = 'local' minion = salt.minion.MasterMinion(__opts__) - running = minion.functions['state.sls'](mods, saltenv, test, exclude) + running = minion.functions['state.sls']( + mods, + saltenv, + test, + exclude, + pillar=pillar) ret = {minion.opts['id']: running} salt.output.display_output(ret, 'highstate', opts=__opts__) return ret From 4e7e7a00320ab2c954721ae3f97019976f0c5e4d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Feb 2014 17:13:13 -0800 Subject: [PATCH 13/39] Add salt.utils.expr_match() --- salt/utils/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 971a11d417..2543e9b4bb 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1074,6 +1074,25 @@ def flopen(*args, **kwargs): fcntl.flock(fp_.fileno(), fcntl.LOCK_UN) +def expr_match(expr, line): + ''' + Evaluate a line of text against an expression. First try a full-string + match, next try globbing, and then try to match assuming expr is a regular + expression. Originally designed to match minion IDs for + whitelists/blacklists. + ''' + if line == expr: + return True + if fnmatch.fnmatch(line, expr): + return True + try: + if re.match(r'\A{0}\Z'.format(expr), line): + return True + except re.error: + pass + return False + + def subdict_match(data, expr, delim=':', regex_match=False): ''' Check for a match in a dictionary using a delimiter character to denote From 94c3a0bb4f89bd5bb58a1ace761f26224af8de57 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Feb 2014 17:13:31 -0800 Subject: [PATCH 14/39] Replace expression matching with salt.utils.expr_match() --- salt/master.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/salt/master.py b/salt/master.py index 63c68fee70..1a598550b1 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1729,24 +1729,11 @@ class ClearFuncs(object): with salt.utils.fopen(signing_file, 'r') as fp_: for line in fp_: line = line.strip() - if line.startswith('#'): continue - - if line == keyid: - return True - if fnmatch.fnmatch(keyid, line): - return True - try: - if re.match(r'\A{0}\Z'.format(line), keyid): + else: + if salt.utils.expr_match(keyid, line): return True - except re.error: - log.warn( - '{0} is not a valid regular expression, ignoring line ' - 'in {1}'.format(line, signing_file) - ) - continue - return False def __check_autoreject(self, keyid): From a9d01266783928a8bce8f4bc7ee061c6c9d37563 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 25 Feb 2014 22:27:50 -0600 Subject: [PATCH 15/39] Add salt.utils.check_whitelist_blacklist() --- salt/utils/__init__.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 2543e9b4bb..e955daa0ff 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1093,6 +1093,50 @@ def expr_match(expr, line): return False +def check_whitelist_blacklist(value, whitelist=None, blacklist=None): + ''' + Check a whitelist and/or blacklist to see if the value matches it. + ''' + if not any((whitelist, blacklist)): + return True + in_whitelist = False + in_blacklist = False + if whitelist: + try: + for expr in whitelist: + if expr_match(expr, value): + in_whitelist = True + break + except TypeError: + log.error('Non-iterable whitelist {0}'.format(whitelist)) + whitelist = None + else: + whitelist = None + + if blacklist: + try: + for expr in blacklist: + if expr_match(expr, value): + in_blacklist = True + break + except TypeError: + log.error('Non-iterable blacklist {0}'.format(whitelist)) + blacklist = None + else: + blacklist = None + + if whitelist and not blacklist: + ret = in_whitelist + elif blacklist and not whitelist: + ret = not in_blacklist + elif whitelist and blacklist: + ret = in_whitelist and not in_blacklist + else: + ret = True + + return ret + + def subdict_match(data, expr, delim=':', regex_match=False): ''' Check for a match in a dictionary using a delimiter character to denote From 3d9c5e28cf074b3603dbbe8f46e9c83f8976ab5a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 25 Feb 2014 22:28:24 -0600 Subject: [PATCH 16/39] Add env whitelist/blacklist to gitfs --- salt/fileserver/gitfs.py | 149 +++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 59 deletions(-) diff --git a/salt/fileserver/gitfs.py b/salt/fileserver/gitfs.py index 8dd0293373..7e6bc2b652 100644 --- a/salt/fileserver/gitfs.py +++ b/salt/fileserver/gitfs.py @@ -323,15 +323,20 @@ _dulwich_env_refs = lambda refs: [x for x in refs def _get_tree_gitpython(repo, short): ''' - Return a git.Tree object if the branch/tag/SHA is found, otherwise False + Return a git.Tree object if the branch/tag/SHA is found, otherwise None ''' - for ref in repo.refs: - if isinstance(ref, (git.RemoteReference, git.TagReference)): - parted = ref.name.partition('/') - refname = parted[2] if parted[2] else parted[0] - if short == refname: - return ref.commit.tree + if short in envs(): + for ref in repo.refs: + if isinstance(ref, (git.RemoteReference, git.TagReference)): + parted = ref.name.partition('/') + rspec = parted[2] if parted[2] else parted[0] + rspec = rspec.replace('/', '_') + if rspec == short: + return ref.commit.tree + # Branch or tag not matched, check if 'short' is a commit + if not _env_is_exposed(short): + return None try: commit = repo.rev_parse(short) except gitdb.exc.BadObject: @@ -343,16 +348,21 @@ def _get_tree_gitpython(repo, short): def _get_tree_pygit2(repo, short): ''' - Return a pygit2.Tree object if the branch/tag/SHA is found, otherwise False + Return a pygit2.Tree object if the branch/tag/SHA is found, otherwise None ''' - for ref in repo.listall_references(): - _, rtype, rspec = ref.split('/', 2) - if rtype in ('remotes', 'tags'): - parted = rspec.partition('/') - refname = parted[2] if parted[2] else parted[0] - if short == refname: - return repo.lookup_reference(ref).get_object().tree + if short in envs(): + for ref in repo.listall_references(): + _, rtype, rspec = ref.split('/', 2) + if rtype in ('remotes', 'tags'): + parted = rspec.partition('/') + rspec = parted[2] if parted[2] else parted[0] + rspec = rspec.replace('/', '_') + if rspec == short and _env_is_exposed(rspec): + return repo.lookup_reference(ref).get_object().tree + # Branch or tag not matched, check if 'short' is a commit + if not _env_is_exposed(short): + return None try: commit = repo.revparse_single(short) except (KeyError, TypeError): @@ -366,35 +376,39 @@ def _get_tree_pygit2(repo, short): def _get_tree_dulwich(repo, short): ''' Return a dulwich.objects.Tree object if the branch/tag/SHA is found, - otherwise False + otherwise None ''' - refs = repo.get_refs() - # Sorting ensures we check heads (branches) before tags - for ref in sorted(_dulwich_env_refs(refs)): - # ref will be something like 'refs/heads/master' - rtype, rspec = ref[5:].split('/') - if rspec == short: - if rtype == 'heads': - commit = repo.get_object(refs[ref]) - elif rtype == 'tags': - tag = repo.get_object(refs[ref]) - if isinstance(tag, dulwich.objects.Tag): - # Tag.get_object() returns a 2-tuple, the 2nd element of - # which is the commit SHA to which the tag refers - commit = repo.get_object(tag.object[1]) - elif isinstance(tag, dulwich.objects.Commit): - commit = tag - else: - log.error( - 'Unhandled object type {0!r} in _get_tree_dulwich. ' - 'This is a bug, please report it.' - .format(tag.type_name) - ) - return repo.get_object(commit.tree) + if short in envs(): + refs = repo.get_refs() + # Sorting ensures we check heads (branches) before tags + for ref in sorted(_dulwich_env_refs(refs)): + # ref will be something like 'refs/heads/master' + rtype, rspec = ref[5:].split('/', 1) + rspec = rspec.replace('/', '_') + if rspec == short and _env_is_exposed(rspec): + if rtype == 'heads': + commit = repo.get_object(refs[ref]) + elif rtype == 'tags': + tag = repo.get_object(refs[ref]) + if isinstance(tag, dulwich.objects.Tag): + # Tag.get_object() returns a 2-tuple, the 2nd element + # of which is the commit SHA to which the tag refers + commit = repo.get_object(tag.object[1]) + elif isinstance(tag, dulwich.objects.Commit): + commit = tag + else: + log.error( + 'Unhandled object type {0!r} in ' + '_get_tree_dulwich. This is a bug, please report ' + 'it.'.format(tag.type_name) + ) + return repo.get_object(commit.tree) # Branch or tag not matched, check if 'short' is a commit. This is more # difficult with Dulwich because of its inability to deal with shortened # SHA-1 hashes. + if not _env_is_exposed(short): + return None try: int(short, 16) except ValueError: @@ -407,18 +421,18 @@ def _get_tree_dulwich(repo, short): if isinstance(sha_obj, dulwich.objects.Commit): sha_commit = sha_obj else: - matches = [ + matches = set([ x for x in ( repo.get_object(x) for x in repo.object_store if x.startswith(short) ) if isinstance(x, dulwich.objects.Commit) - ] + ]) if len(matches) > 1: log.warning('Ambiguous commit ID {0!r}'.format(short)) return None try: - sha_commit = matches[0] + sha_commit = matches.pop() except IndexError: pass except TypeError as exc: @@ -802,6 +816,18 @@ def update(): pass +def _env_is_exposed(env): + ''' + Check if an environment is exposed by comparing it against a whitelist and + blacklist. + ''' + return salt.utils.check_whitelist_blacklist( + env, + whitelist=__opts__['gitfs_env_whitelist'], + blacklist=__opts__['gitfs_env_blacklist'] + ) + + def envs(ignore_cache=False): ''' Return a list of refs that can be used as environments @@ -841,14 +867,15 @@ def _envs_gitpython(repo, base_branch): remote = repo.remotes[0] for ref in repo.refs: parted = ref.name.partition('/') - short = parted[2] if parted[2] else parted[0] + rspec = parted[2] if parted[2] else parted[0] + rspec = rspec.replace('/', '_') if isinstance(ref, git.Head): - if short == base_branch: - short = 'base' - if ref not in remote.stale_refs: - ret.add(short) - elif isinstance(ref, git.Tag): - ret.add(short) + if rspec == base_branch: + rspec = 'base' + if ref not in remote.stale_refs and _env_is_exposed(rspec): + ret.add(rspec) + elif isinstance(ref, git.Tag) and _env_is_exposed(rspec): + ret.add(rspec) return ret @@ -863,15 +890,17 @@ def _envs_pygit2(repo, base_branch): for ref in repo.listall_references(): ref = re.sub('^refs/', '', ref) rtype, rspec = ref.split('/', 1) - if rtype == 'tags': - ret.add(rspec) - elif rtype == 'remotes': + if rtype == 'remotes': if rspec not in stale_refs: parted = rspec.partition('/') - short = parted[2] if parted[2] else parted[0] - if short == base_branch: - short = 'base' - ret.add(short) + rspec = parted[2] if parted[2] else parted[0] + rspec = rspec.replace('/', '_') + if rspec == base_branch: + rspec = 'base' + if _env_is_exposed(rspec): + ret.add(rspec) + elif rtype == 'tags' and _env_is_exposed(rspec): + ret.add(rspec) return ret @@ -884,11 +913,13 @@ def _envs_dulwich(repo, base_branch): for ref in _dulwich_env_refs(repo.get_refs()): # ref will be something like 'refs/heads/master' rtype, rspec = ref[5:].split('/', 1) - if rtype == 'tags': - ret.add(rspec) - elif rtype == 'heads': + rspec = rspec.replace('/', '_') + if rtype == 'heads': if rspec == base_branch: rspec = 'base' + if _env_is_exposed(rspec): + ret.add(rspec) + elif rtype == 'tags' and _env_is_exposed(rspec): ret.add(rspec) return ret From b8c66247cdc78f677b46de431a194acd09c6bb55 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 25 Feb 2014 22:28:55 -0600 Subject: [PATCH 17/39] Add default values for new config parameters --- salt/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/config.py b/salt/config.py index 0add19b6bc..b49112972f 100644 --- a/salt/config.py +++ b/salt/config.py @@ -137,6 +137,8 @@ VALID_OPTS = { 'gitfs_mountpoint': str, 'gitfs_root': str, 'gitfs_base': str, + 'gitfs_env_whitelist': list, + 'gitfs_env_blacklist': list, 'hgfs_remotes': list, 'hgfs_mountpoint': str, 'hgfs_root': str, @@ -337,6 +339,8 @@ DEFAULT_MASTER_OPTS = { 'gitfs_mountpoint': '', 'gitfs_root': '', 'gitfs_base': 'master', + 'gitfs_env_whitelist': [], + 'gitfs_env_blacklist': [], 'hgfs_remotes': [], 'hgfs_mountpoint': '', 'hgfs_root': '', From 9d52f04a43c8e5027dec69ca471707d40644fe5d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 25 Feb 2014 22:29:44 -0600 Subject: [PATCH 18/39] Clear remote fs backend env cache on master start Clear remote fileserver backend env cache so it gets recreated during the initial loop_interval. Allows envs to be rebuilt if changes are made to an env whitelist/blacklist. --- salt/master.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/master.py b/salt/master.py index 1a598550b1..65d1e9db34 100644 --- a/salt/master.py +++ b/salt/master.py @@ -165,6 +165,25 @@ class Master(SMaster): br, loc = opts_dict['git'].strip().split() pillargitfs.append(git_pillar.GitPillar(br, loc, self.opts)) + # Clear remote fileserver backend env cache so it gets recreated during + # the first loop_interval + for backend in ('git', 'hg', 'svn'): + if backend in self.opts['fileserver_backend']: + env_cache = os.path.join( + self.opts['cachedir'], + '{0}fs'.format(backend), + 'envs.p' + ) + if os.path.isfile(env_cache): + log.debug('Clearing {0}fs env cache'.format(backend)) + try: + os.remove(env_cache) + except (IOError, OSError) as exc: + log.critical( + 'Unable to clear env cache file {0}: {1}' + .format(env_cache, exc) + ) + old_present = set() while True: now = int(time.time()) From f725f00614fe84c8639990b9345371cf159b0ae7 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 25 Feb 2014 22:49:07 -0600 Subject: [PATCH 19/39] Add new config parameters to the docs --- doc/ref/configuration/master.rst | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 69375b512b..466523d38d 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -933,6 +933,60 @@ Defines which branch/tag should be used as the ``base`` environment. gitfs_base: salt +.. conf_master:: gitfs_env_whitelist + +``gitfs_env_whitelist`` +*********************** + +.. versionadded:: Helium + +Default: ``[]`` + +Used to restrict which environments are made available. Can speed up state runs +if your gitfs remotes contain many branches/tags. Full names, globs, and +regular expressions are accepted. + +If used, only branches/tags/SHAs which match one of the specified expressions +will be exposed as fileserver environments. + +If used in conjunction with :conf_master:`gitfs_env_blacklist`, then the subset +of hosts which match the whitelist but do *not* match the blacklist will be +exposed as fileserver environments. + +.. code-block:: yaml + + gitfs_env_whitelist: + - base + - v1.* + - 'mybranch\d+' + +.. conf_master:: gitfs_env_blacklist + +``gitfs_env_blacklist`` +*********************** + +.. versionadded:: Helium + +Default: ``[]`` + +Used to restrict which environments are made available. Can speed up state runs +if your gitfs remotes contain many branches/tags. Full names, globs, and +regular expressions are accepted. + +If used, branches/tags/SHAs which match one of the specified expressions will +*not* be exposed as fileserver environments. + +If used in conjunction with :conf_master:`gitfs_env_whitelist`, then the subset +of hosts which match the whitelist but do *not* match the blacklist will be +exposed as fileserver environments. + +.. code-block:: yaml + + gitfs_env_blacklist: + - base + - v1.* + - 'mybranch\d+' + hg: Mercurial Remote File Server Backend ---------------------------------------- From 695cf1aace135ab345665fbc7ca54a40a0bf53c0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 26 Feb 2014 05:11:01 +0000 Subject: [PATCH 20/39] `{a, b}` `set()` notation is not available in Python 2.6 Allow running the testcase filename alone. --- tests/unit/modules/mac_user_test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/modules/mac_user_test.py b/tests/unit/modules/mac_user_test.py index 695f4099d3..955bd7f8ba 100644 --- a/tests/unit/modules/mac_user_test.py +++ b/tests/unit/modules/mac_user_test.py @@ -5,8 +5,11 @@ # Import Salt Testing Libs from salttesting import TestCase, skipIf +from salttesting.helpers import ensure_in_syspath from salttesting.mock import MagicMock, patch +ensure_in_syspath('../../') + # Import Salt Libs from salt.modules import mac_user from salt.exceptions import SaltInvocationError, CommandExecutionError @@ -272,7 +275,7 @@ class MacUserTestCase(TestCase): @patch('salt.modules.mac_user.info', MagicMock(return_value=mock_info_ret)) @patch('salt.modules.mac_user.list_groups', - MagicMock(return_value={'wheel', 'root'})) + MagicMock(return_value=('wheel', 'root'))) def test_chgroups_same_desired(self): ''' Tests if the user's list of groups is the same as the arguments @@ -324,3 +327,8 @@ class MacUserTestCase(TestCase): ''' ret = ['_amavisd', '_appleevents', '_appowner'] self.assertEqual(mac_user.list_users(), ret) + + +if __name__ == '__main__': + from integration import run_tests + run_tests(MacUserTestCase, needs_daemon=False) From 5cd1d75ffb33d83736e2e09d7b2aee362efd430d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 25 Feb 2014 23:53:12 -0600 Subject: [PATCH 21/39] remove unused import --- salt/master.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index 65d1e9db34..2a019b142e 100644 --- a/salt/master.py +++ b/salt/master.py @@ -9,7 +9,6 @@ import os import re import time import errno -import fnmatch import signal import shutil import stat From 08620e75410dddb8c98c5199e9ce0092b4de4bd1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 25 Feb 2014 23:57:11 -0600 Subject: [PATCH 22/39] Lint fixes --- salt/daemons/test/test_master.py | 11 ++++------- salt/daemons/test/test_minion.py | 11 ++++------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/salt/daemons/test/test_master.py b/salt/daemons/test/test_master.py index b2aef05c08..122d6ae398 100644 --- a/salt/daemons/test/test_master.py +++ b/salt/daemons/test/test_master.py @@ -1,18 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - -""" +''' Runs minion floscript - -""" +''' import os import salt.daemons.flo -import ioflo.app.run FLO_DIR_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - , 'flo') + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'flo' +) def test(): diff --git a/salt/daemons/test/test_minion.py b/salt/daemons/test/test_minion.py index 1be9b962f6..0824451d24 100644 --- a/salt/daemons/test/test_minion.py +++ b/salt/daemons/test/test_minion.py @@ -1,18 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - -""" +''' Runs minion floscript - -""" +''' import os import salt.daemons.flo -import ioflo.app.run FLO_DIR_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - , 'flo') + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'flo' +) def test(): From 13cfa54065c4c01ddbee32aa5521d1881cec55af Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 25 Feb 2014 22:52:02 +0000 Subject: [PATCH 23/39] Properly handle PyZMQ >= 0.14 when freezing --- setup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/setup.py b/setup.py index d6b303fba7..755c68b919 100755 --- a/setup.py +++ b/setup.py @@ -22,6 +22,12 @@ from distutils.command.clean import clean from distutils.command.sdist import sdist # pylint: enable=E0611 +try: + import zmq + HAS_ZMQ = True +except ImportError: + HAS_ZMQ = False + # Change to salt source's directory prior to running any command try: SETUP_DIRNAME = os.path.dirname(__file__) @@ -517,6 +523,13 @@ FREEZER_INCLUDES = [ 'email.mime.*', ] +if HAS_ZMQ and zmq.pyzmq_version_info() >= (0, 14): + # We're freezing, and when freezing ZMQ needs to be installed, so this + # works fine + if 'zmq.core.*' in FREEZER_INCLUDES: + # For PyZMQ >= 0.14, freezing does not need 'zmq.core.*' + FREEZER_INCLUDES.remove('zmq.core.*') + if IS_WINDOWS_PLATFORM: FREEZER_INCLUDES.extend([ 'win32api', From b717319e82b268737f57703886d5b28098a06699 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 26 Feb 2014 00:51:05 -0600 Subject: [PATCH 24/39] Add "if __name__ == '__main__'" section to mac_group_test.py --- tests/unit/modules/mac_group_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/modules/mac_group_test.py b/tests/unit/modules/mac_group_test.py index 175b63915d..3d0ed77cba 100644 --- a/tests/unit/modules/mac_group_test.py +++ b/tests/unit/modules/mac_group_test.py @@ -77,3 +77,6 @@ class MacGroupTestCase(TestCase): self.assertEqual(mac_group._format_info(data), ret) +if __name__ == '__main__': + from integration import run_tests + run_tests(MacGroupTestCase, needs_daemon=False) From aeb2ef5384b4d8adfc3c03bb2ba0af2f6d648268 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 26 Feb 2014 09:29:37 +0000 Subject: [PATCH 25/39] Make PyLint ignore the test files --- salt/daemons/test/test_master.py | 1 + salt/daemons/test/test_minion.py | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/daemons/test/test_master.py b/salt/daemons/test/test_master.py index b2aef05c08..3b8e099159 100644 --- a/salt/daemons/test/test_master.py +++ b/salt/daemons/test/test_master.py @@ -6,6 +6,7 @@ Runs minion floscript """ +# pylint: skip-file import os import salt.daemons.flo import ioflo.app.run diff --git a/salt/daemons/test/test_minion.py b/salt/daemons/test/test_minion.py index 1be9b962f6..c29af32683 100644 --- a/salt/daemons/test/test_minion.py +++ b/salt/daemons/test/test_minion.py @@ -6,6 +6,7 @@ Runs minion floscript """ +# pylint: skip-file import os import salt.daemons.flo import ioflo.app.run From 95b376a9f2d78c387457eb6b8700857828b17ed3 Mon Sep 17 00:00:00 2001 From: Tomas Havlas Date: Wed, 26 Feb 2014 12:44:44 +0100 Subject: [PATCH 26/39] iptables append --reject-with after --jump --- salt/modules/iptables.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/modules/iptables.py b/salt/modules/iptables.py index 2a6f6f99a1..f1f36368ca 100644 --- a/salt/modules/iptables.py +++ b/salt/modules/iptables.py @@ -192,6 +192,10 @@ def build_rule(table=None, chain=None, command=None, position='', full=None, fam after_jump.append('--to-destination {0} '.format(kwargs['to-destination'])) del kwargs['to-destination'] + if 'reject-with' in kwargs: + after_jump.append('--reject-with {0} '.format(kwargs['reject-with'])) + del kwargs['reject-with'] + for item in kwargs: if len(item) == 1: rule += '-{0} {1} '.format(item, kwargs[item]) From 07ef0a9129efe98d78c1200da08e0c8a878e3de2 Mon Sep 17 00:00:00 2001 From: Jens Rantil Date: Wed, 26 Feb 2014 12:46:17 +0100 Subject: [PATCH 27/39] module/timezone: add newline in `/etc/timezone` This follows the convention used by Debian. I've also stripped/trimmed the timezone string in case someone's already manually added a newline. --- salt/modules/timezone.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/timezone.py b/salt/modules/timezone.py index d2ca3d485d..c0033639c6 100644 --- a/salt/modules/timezone.py +++ b/salt/modules/timezone.py @@ -128,7 +128,8 @@ def set_zone(timezone): '/etc/sysconfig/clock', '^ZONE=.*', 'ZONE="{0}"'.format(timezone)) elif 'Debian' in __grains__['os_family']: with salt.utils.fopen('/etc/timezone', 'w') as ofh: - ofh.write(timezone) + ofh.write(timezone.strip()) + ofh.write('\n') elif 'Gentoo' in __grains__['os_family']: with salt.utils.fopen('/etc/timezone', 'w') as ofh: ofh.write(timezone) From d9954ec73f7b94d683674dfc7dee07133ca4d434 Mon Sep 17 00:00:00 2001 From: Tomas Havlas Date: Wed, 26 Feb 2014 13:59:00 +0100 Subject: [PATCH 28/39] fixed arguments for build_rule in insert --- salt/states/iptables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/iptables.py b/salt/states/iptables.py index b1c36c5a14..10a80ab15a 100644 --- a/salt/states/iptables.py +++ b/salt/states/iptables.py @@ -296,7 +296,7 @@ def insert(name, family='ipv4', **kwargs): for ignore in _STATE_INTERNAL_KEYWORDS: if ignore in kwargs: del kwargs[ignore] - rule = __salt__['iptables.build_rule'](family, **kwargs) + rule = __salt__['iptables.build_rule'](family=family, **kwargs) command = __salt__['iptables.build_rule'](full=True, family=family, command='I', **kwargs) if __salt__['iptables.check'](kwargs['table'], kwargs['chain'], From 77a96ea8689f70d49005693b7c2da0833d77340c Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Wed, 26 Feb 2014 15:11:09 +0100 Subject: [PATCH 29/39] Expanduser on key_filename in openstack provider --- salt/cloud/clouds/openstack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index 3aa3aedc37..1bbc40213f 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -326,6 +326,7 @@ def create(vm_): key_filename = config.get_cloud_config_value( 'ssh_key_file', vm_, __opts__, search_global=False, default=None ) + key_filename = os.path.expanduser(key_filename) if key_filename is not None and not os.path.isfile(key_filename): raise SaltCloudConfigError( 'The defined ssh_key_file {0!r} does not exist'.format( From 595d3b89514114d22714cdf1313767957fbfe661 Mon Sep 17 00:00:00 2001 From: Tomas Havlas Date: Wed, 26 Feb 2014 15:50:07 +0100 Subject: [PATCH 30/39] iptables --tcp-flags parse whitespace separated values --- salt/modules/iptables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/iptables.py b/salt/modules/iptables.py index f1f36368ca..96539c0885 100644 --- a/salt/modules/iptables.py +++ b/salt/modules/iptables.py @@ -885,7 +885,7 @@ def _parser(): add_arg('--string', dest='string', action='append') add_arg('--hex-string', dest='hex-string', action='append') ## tcp - add_arg('--tcp-flags', dest='tcp-flags', action='append') + add_arg('--tcp-flags', dest='tcp-flags', action='append', nargs='*') add_arg('--syn', dest='syn', action='append') add_arg('--tcp-option', dest='tcp-option', action='append') ## tcpmss From 08bb9a880dd28ad87a5618ba747d605e982fae7f Mon Sep 17 00:00:00 2001 From: Tomas Havlas Date: Wed, 26 Feb 2014 16:04:00 +0100 Subject: [PATCH 31/39] iptables --tcp-flags nargs fix for 2.6 version --- salt/modules/iptables.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/iptables.py b/salt/modules/iptables.py index 96539c0885..9678132546 100644 --- a/salt/modules/iptables.py +++ b/salt/modules/iptables.py @@ -885,7 +885,10 @@ def _parser(): add_arg('--string', dest='string', action='append') add_arg('--hex-string', dest='hex-string', action='append') ## tcp - add_arg('--tcp-flags', dest='tcp-flags', action='append', nargs='*') + if sys.version.startswith('2.6'): + add_arg('--tcp-flags', dest='tcp-flags', action='append') + else: + add_arg('--tcp-flags', dest='tcp-flags', action='append', nargs='*') add_arg('--syn', dest='syn', action='append') add_arg('--tcp-option', dest='tcp-option', action='append') ## tcpmss From 09866fee22d097e0ccf3d331e698fb9bd258b238 Mon Sep 17 00:00:00 2001 From: "Miguel A. Guillen" Date: Wed, 26 Feb 2014 16:20:50 +0100 Subject: [PATCH 32/39] Initial version of nagios module --- salt/modules/nagios.py | 282 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 salt/modules/nagios.py diff --git a/salt/modules/nagios.py b/salt/modules/nagios.py new file mode 100644 index 0000000000..e09af61446 --- /dev/null +++ b/salt/modules/nagios.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- +''' +Run nagios plugins/checks from salt and get the return as data. +''' + +# Import python libs +import os +import stat + +# Import salt libs +import salt.utils +from salt._compat import string_types +import logging + +log = logging.getLogger(__name__) + +PLUGINDIR = '/usr/lib/nagios/plugins/' + + +def __virtual__(): + """ + Only load if nagios-plugins are installed + """ + + if os.path.isdir('/usr/lib/nagios/'): + return 'nagios' + return False + + +def _execute_cmd(plugin, args='', run_type='cmd.retcode', key_name=None): + """ + Execute nagios plugin if it's in the directory with salt command specified in run_type + """ + data = {} + + all_plugins = list_plugins() + if plugin in all_plugins: + data = __salt__[run_type]('{0}{1} {2}'.format(PLUGINDIR,plugin,args)) + + return data + + +def _execute_pillar(pillar_name, cmd_command): + """ + Run one or more nagios plugins from pillar data and get the result of run_all + The pillar have to be in this format: + ------ + webserver: + Ping_google: + - check_icmp:8.8.8.8 + - check_icmp:google.com + Load: + - check_load:-w 0.8 -c 1 + APT: + - check_apt + ------- + webserver is the role to check, the next keys are the group and the items the check with the arguments if needed + You have to group different checks in a group + CLI Example: + + .. code-block:: bash + + salt '*' nagios.run webserver + + """ + + groups = __salt__['pillar.get'](pillar_name) + + data = {} + for group in groups: + data[group]={} + commands = groups[group] + for command in commands: + #Check if is a dict to get the arguments + #in command if not set the arguments to empty string + if isinstance(command,dict): + plugin = command.keys()[0] + args = command[plugin] + else: + plugin = command + args = '' + command_key=_format_dict_key(args, plugin) + data[group][command_key]=cmd_command(plugin, args, group) + return data + + +def _format_dict_key(args, plugin): + + key_name = plugin + args_key = args.replace(' ','') + if args != '': + args_key = '_'+args_key + key_name = plugin+args_key + + return key_name + + +def run(plugin, args='', key_name=None): + """ + Run nagios plugin and return all the data execution with cmd.run + + """ + + data = _execute_cmd(plugin, args, 'cmd.run', key_name) + + return data + + +def retcode(plugin, args='', key_name=None): + """ + Run one nagios plugin and return retcode of the execution + + CLI Example: + + .. code-block:: bash + + salt '*' nagios.run check_apt + salt '*' nagios.run check_icmp '8.8.8.8' + + """ + data = {} + + # Remove all the spaces, the key must not have any space + if key_name is None: + key_name= _format_dict_key(args, plugin) + + data[key_name] = {} + + status = _execute_cmd(plugin, args,'cmd.retcode', key_name) + data[key_name]['status'] = status + + return data + + +def run_all(plugin, args='', key_name=None): + """ + Run nagios plugin and return all the data execution with cmd.run_all + + """ + + data = _execute_cmd(plugin, args, 'cmd.run_all', key_name) + + return data + + +def retcode_pillar(pillar_name): + """ + Run one or more nagios plugins from pillar data and get the result of cdm.retcode + The pillar have to be in this format: + ------ + webserver: + Ping_google: + - check_icmp:8.8.8.8 + - check_icmp:google.com + Load: + - check_load:-w 0.8 -c 1 + APT: + - check_apt + ------- + webserver is the role to check, the next keys are the group and the items the check with the arguments if needed + You must to group different checks(one o more) and always it will return the highest value of all the checks + CLI Example: + + .. code-block:: bash + + salt '*' nagios.retcode webserver + + """ + + groups = __salt__['pillar.get'](pillar_name) + + check = {} + data = {} + + for group in groups: + commands = groups[group] + for command in commands: + #Check if is a dict to get the arguments + #in command if not set the arguments to empty string + if isinstance(command,dict): + plugin = command.keys()[0] + args = command[plugin] + else: + plugin = command + args = '' + + check.update(retcode(plugin, args, group)) + + current_value = 0 + new_value = int(check[group]['status']) + if group in data: + current_value = int(data[group]['status']) + + log.debug('Value to check in {0}: {1}'.format(group,new_value)) + + if (new_value > current_value) or (group not in data): + + log.debug('Updating {0}'.format(group)) + + if group not in data: + data[group] = {} + data[group]['status'] = new_value + + return data + + +def run_pillar(pillar_name): + """ + Run one or more nagios plugins from pillar data and get the result of cdm.run + The pillar have to be in this format: + ------ + webserver: + Ping_google: + - check_icmp:8.8.8.8 + - check_icmp:google.com + Load: + - check_load:-w 0.8 -c 1 + APT: + - check_apt + ------- + webserver is the role to check, the next keys are the group and the items the check with the arguments if needed + You have to group different checks in a group + CLI Example: + + .. code-block:: bash + + salt '*' nagios.run webserver + + """ + data = _execute_pillar(pillar_name, run) + + return data + + +def run_all_pillar(pillar_name): + """ + Run one or more nagios plugins from pillar data and get the result of cdm.run_all + The pillar have to be in this format: + ------ + webserver: + Ping_google: + - check_icmp:8.8.8.8 + - check_icmp:google.com + Load: + - check_load:-w 0.8 -c 1 + APT: + - check_apt + ------- + webserver is the role to check, the next keys are the group and the items the check with the arguments if needed + You have to group different checks in a group + CLI Example: + + .. code-block:: bash + + salt '*' nagios.run webserver + + """ + data = _execute_pillar(pillar_name, run_all) + return data + + +def list_plugins(): + """ + List all the nagios plugins + + CLI Example: + + .. code-block:: bash + + salt '*' nagios.list_plugins + + """ + + plugin_list = os.listdir(PLUGINDIR) + ret = [] + for plugin in plugin_list: + # Check if execute bit + stat_f = os.path.join(PLUGINDIR, plugin) + execute_bit = stat.S_IXUSR & os.stat(stat_f)[stat.ST_MODE] + if execute_bit: + ret.append(plugin) + return ret From 565ecdbf7e8954a054aedd91be93138ebf99777f Mon Sep 17 00:00:00 2001 From: "Miguel A. Guillen" Date: Wed, 26 Feb 2014 17:23:38 +0100 Subject: [PATCH 33/39] Fix format issues --- salt/modules/nagios.py | 54 ++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/salt/modules/nagios.py b/salt/modules/nagios.py index e09af61446..53352f11a8 100644 --- a/salt/modules/nagios.py +++ b/salt/modules/nagios.py @@ -1,15 +1,14 @@ # -*- coding: utf-8 -*- -''' +""" Run nagios plugins/checks from salt and get the return as data. -''' +""" # Import python libs import os import stat # Import salt libs -import salt.utils -from salt._compat import string_types + import logging log = logging.getLogger(__name__) @@ -35,14 +34,14 @@ def _execute_cmd(plugin, args='', run_type='cmd.retcode', key_name=None): all_plugins = list_plugins() if plugin in all_plugins: - data = __salt__[run_type]('{0}{1} {2}'.format(PLUGINDIR,plugin,args)) + data = __salt__[run_type]('{0}{1} {2}'.format(PLUGINDIR, plugin, args)) return data -def _execute_pillar(pillar_name, cmd_command): +def _execute_pillar(pillar_name, run_type): """ - Run one or more nagios plugins from pillar data and get the result of run_all + Run one or more nagios plugins from pillar data and get the result of run_type The pillar have to be in this format: ------ webserver: @@ -54,43 +53,34 @@ def _execute_pillar(pillar_name, cmd_command): APT: - check_apt ------- - webserver is the role to check, the next keys are the group and the items the check with the arguments if needed - You have to group different checks in a group - CLI Example: - - .. code-block:: bash - - salt '*' nagios.run webserver - """ groups = __salt__['pillar.get'](pillar_name) data = {} for group in groups: - data[group]={} + data[group] = {} commands = groups[group] for command in commands: #Check if is a dict to get the arguments #in command if not set the arguments to empty string - if isinstance(command,dict): + if isinstance(command, dict): plugin = command.keys()[0] args = command[plugin] else: plugin = command args = '' - command_key=_format_dict_key(args, plugin) - data[group][command_key]=cmd_command(plugin, args, group) + command_key = _format_dict_key(args, plugin) + data[group][command_key] = run_type(plugin, args, group) return data def _format_dict_key(args, plugin): - key_name = plugin - args_key = args.replace(' ','') + args_key = args.replace(' ', '') if args != '': - args_key = '_'+args_key - key_name = plugin+args_key + args_key = '_' + args_key + key_name = plugin + args_key return key_name @@ -122,11 +112,11 @@ def retcode(plugin, args='', key_name=None): # Remove all the spaces, the key must not have any space if key_name is None: - key_name= _format_dict_key(args, plugin) + key_name = _format_dict_key(args, plugin) data[key_name] = {} - status = _execute_cmd(plugin, args,'cmd.retcode', key_name) + status = _execute_cmd(plugin, args, 'cmd.retcode', key_name) data[key_name]['status'] = status return data @@ -177,26 +167,22 @@ def retcode_pillar(pillar_name): for command in commands: #Check if is a dict to get the arguments #in command if not set the arguments to empty string - if isinstance(command,dict): + if isinstance(command, dict): plugin = command.keys()[0] args = command[plugin] else: plugin = command args = '' - + check.update(retcode(plugin, args, group)) - + current_value = 0 new_value = int(check[group]['status']) if group in data: current_value = int(data[group]['status']) - - log.debug('Value to check in {0}: {1}'.format(group,new_value)) - + if (new_value > current_value) or (group not in data): - - log.debug('Updating {0}'.format(group)) - + if group not in data: data[group] = {} data[group]['status'] = new_value From 087c1c1e66d06e6c91efedde312418d52bab4b3b Mon Sep 17 00:00:00 2001 From: "Miguel A. Guillen" Date: Wed, 26 Feb 2014 17:23:38 +0100 Subject: [PATCH 34/39] Fix format issues --- salt/modules/nagios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/nagios.py b/salt/modules/nagios.py index 53352f11a8..92daa37ed4 100644 --- a/salt/modules/nagios.py +++ b/salt/modules/nagios.py @@ -165,7 +165,7 @@ def retcode_pillar(pillar_name): for group in groups: commands = groups[group] for command in commands: - #Check if is a dict to get the arguments + #Check if is a dict to get the arguments #in command if not set the arguments to empty string if isinstance(command, dict): plugin = command.keys()[0] From 48ff3b1f3ce6a2307c922b4d8fda68aae6beacb2 Mon Sep 17 00:00:00 2001 From: "Miguel A. Guillen" Date: Wed, 26 Feb 2014 18:09:54 +0100 Subject: [PATCH 35/39] Added carbon returner information --- salt/modules/nagios.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/nagios.py b/salt/modules/nagios.py index 92daa37ed4..68202edec9 100644 --- a/salt/modules/nagios.py +++ b/salt/modules/nagios.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """ -Run nagios plugins/checks from salt and get the return as data. +Run nagios plugins/checks from salt and get the return as datai, +retcode commands are compatible with carbon returner. """ # Import python libs From ec375affa5d429497dbc82cb9447c62cec06befa Mon Sep 17 00:00:00 2001 From: "Miguel A. Guillen" Date: Wed, 26 Feb 2014 18:16:44 +0100 Subject: [PATCH 36/39] Added carbon returner information --- salt/modules/nagios.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/modules/nagios.py b/salt/modules/nagios.py index 68202edec9..92daa37ed4 100644 --- a/salt/modules/nagios.py +++ b/salt/modules/nagios.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ -Run nagios plugins/checks from salt and get the return as datai, -retcode commands are compatible with carbon returner. +Run nagios plugins/checks from salt and get the return as data. """ # Import python libs From fb98e76fb0280da35cda3f5be3d600e6a90ae6ec Mon Sep 17 00:00:00 2001 From: Evan Borgstrom Date: Wed, 26 Feb 2014 12:19:00 -0500 Subject: [PATCH 37/39] Adding pyobjects documentation to the tutorial --- doc/.tx/config | 6 ++++++ doc/topics/tutorials/starting_states.rst | 26 +++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/doc/.tx/config b/doc/.tx/config index d56f35c123..cc5ecbaf9a 100644 --- a/doc/.tx/config +++ b/doc/.tx/config @@ -1466,6 +1466,12 @@ source_file = _build/locale/ref/renderers/all/salt.renderers.pydsl.pot source_lang = en source_name = ref/renderers/all/salt.renderers.pydsl.rst +[salt.ref--renderers--all--salt_renderers_pyobjects] +file_filter = locale//LC_MESSAGES/ref/renderers/all/salt.renderers.pyobjects.po +source_file = _build/locale/ref/renderers/all/salt.renderers.pyobjects.pot +source_lang = en +source_name = ref/renderers/all/salt.renderers.pyobjects.rst + [salt.ref--renderers--all--salt_renderers_stateconf] file_filter = locale//LC_MESSAGES/ref/renderers/all/salt.renderers.stateconf.po source_file = _build/locale/ref/renderers/all/salt.renderers.stateconf.pot diff --git a/doc/topics/tutorials/starting_states.rst b/doc/topics/tutorials/starting_states.rst index bb058dcceb..7bda036e46 100644 --- a/doc/topics/tutorials/starting_states.rst +++ b/doc/topics/tutorials/starting_states.rst @@ -327,16 +327,19 @@ full programming constructs are available when creating SLS files. Other renderers available are ``yaml_mako`` and ``yaml_wempy`` which each use the `Mako`_ or `Wempy`_ templating system respectively rather than the jinja -templating system, and more notably, the pure Python or ``py`` and ``pydsl`` -renderers. +templating system, and more notably, the pure Python or ``py``, ``pydsl`` & +``pyobjects`` renderers. The ``py`` renderer allows for SLS files to be written in pure Python, allowing for the utmost level of flexibility and power when preparing SLS data; while the :doc:`pydsl` renderer -provides a flexible, domain-specific language for authoring SLS data in Python. +provides a flexible, domain-specific language for authoring SLS data in Python; +and the :doc:`pyobjects` renderer +gives you a `"Pythonic"`_ interface to building state data. .. _`Jinja2`: http://jinja.pocoo.org/ .. _`Mako`: http://www.makotemplates.org/ .. _`Wempy`: https://fossil.secution.com/u/gcw/wempy/doc/tip/README.wiki +.. _`"Pythonic"`: http://legacy.python.org/dev/peps/pep-0008/ .. note:: The templating engines described above aren't just available in SLS files. @@ -467,8 +470,8 @@ and set them up to be mounted, and the ``salt`` object is used multiple times to call shell commands to gather data. -Introducing the Python and the PyDSL Renderers ----------------------------------------------- +Introducing the Python, PyDSL and the Pyobjects Renderers +--------------------------------------------------------- Sometimes the chosen default renderer might not have enough logical power to accomplish the needed task. When this happens, the Python renderer can be @@ -499,8 +502,6 @@ must be a Salt friendly data structure, or better known as a Salt Alternatively, using the :doc:`pydsl` renderer, the above example can be written more succinctly as: -``python/django.sls:`` - .. code-block:: python #!pydsl @@ -508,6 +509,17 @@ renderer, the above example can be written more succinctly as: include('python', delayed=True) state('django').pkg.installed() +The :doc:`pyobjects` renderer +provides an `"Pythonic"`_ object based approach for building the state data. +The above example could be written as: + +.. code-block:: python + + #!pyobjects + + include('python') + Pkg.installed("django") + This Python examples would look like this if they were written in YAML: From 927b6275af82eff8db2ad25bbcdec1c9e7409396 Mon Sep 17 00:00:00 2001 From: Evan Borgstrom Date: Wed, 26 Feb 2014 12:22:32 -0500 Subject: [PATCH 38/39] Ensure the code-blocks work properly --- salt/renderers/pyobjects.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/renderers/pyobjects.py b/salt/renderers/pyobjects.py index 8b5b305849..33c7a6a817 100644 --- a/salt/renderers/pyobjects.py +++ b/salt/renderers/pyobjects.py @@ -9,6 +9,7 @@ example that ensures the ``/tmp`` directory is in the correct state. .. code-block:: python :linenos: + #!pyobjects File.managed("/tmp", user='root', group='root', mode='1777') @@ -46,6 +47,7 @@ core of what makes pyobjects the best way to write states. .. code-block:: python :linenos: + #!pyobjects with Pkg.installed("nginx"): @@ -66,6 +68,7 @@ The above could have also been written use direct requisite statements as. .. code-block:: python :linenos: + #!pyobjects Pkg.installed("nginx") @@ -80,6 +83,7 @@ generated outside of the current file. .. code-block:: python :linenos: + #!pyobjects # some-other-package is defined in some other state file @@ -91,6 +95,7 @@ watch_in, use & use_in) when using the requisite as a context manager. .. code-block:: python :linenos: + #!pyobjects with Service("my-service", "watch_in"): @@ -111,6 +116,7 @@ a state. .. code-block:: python :linenos: + #!pyobjects include('http', 'ssh') @@ -130,6 +136,7 @@ The following lines are functionally equivalent: .. code-block:: python :linenos: + #!pyobjects ret = salt.cmd.run(bar) @@ -148,6 +155,7 @@ The following pairs of lines are functionally equivalent: .. code-block:: python :linenos: + #!pyobjects value = pillar('foo:bar:baz', 'qux') From 2cec616f312a0bc452fe2b590fcefcf7097d88cc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 26 Feb 2014 12:49:19 -0700 Subject: [PATCH 39/39] Whitespace and double quote cleanup #10781 --- salt/modules/nagios.py | 57 ++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/salt/modules/nagios.py b/salt/modules/nagios.py index 92daa37ed4..b6c5efa88d 100644 --- a/salt/modules/nagios.py +++ b/salt/modules/nagios.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -""" +''' Run nagios plugins/checks from salt and get the return as data. -""" +''' # Import python libs import os @@ -17,19 +17,18 @@ PLUGINDIR = '/usr/lib/nagios/plugins/' def __virtual__(): - """ + ''' Only load if nagios-plugins are installed - """ - + ''' if os.path.isdir('/usr/lib/nagios/'): return 'nagios' return False def _execute_cmd(plugin, args='', run_type='cmd.retcode', key_name=None): - """ + ''' Execute nagios plugin if it's in the directory with salt command specified in run_type - """ + ''' data = {} all_plugins = list_plugins() @@ -40,7 +39,7 @@ def _execute_cmd(plugin, args='', run_type='cmd.retcode', key_name=None): def _execute_pillar(pillar_name, run_type): - """ + ''' Run one or more nagios plugins from pillar data and get the result of run_type The pillar have to be in this format: ------ @@ -53,8 +52,7 @@ def _execute_pillar(pillar_name, run_type): APT: - check_apt ------- - """ - + ''' groups = __salt__['pillar.get'](pillar_name) data = {} @@ -86,18 +84,17 @@ def _format_dict_key(args, plugin): def run(plugin, args='', key_name=None): - """ + ''' Run nagios plugin and return all the data execution with cmd.run - """ - + ''' data = _execute_cmd(plugin, args, 'cmd.run', key_name) return data def retcode(plugin, args='', key_name=None): - """ + ''' Run one nagios plugin and return retcode of the execution CLI Example: @@ -106,8 +103,7 @@ def retcode(plugin, args='', key_name=None): salt '*' nagios.run check_apt salt '*' nagios.run check_icmp '8.8.8.8' - - """ + ''' data = {} # Remove all the spaces, the key must not have any space @@ -123,18 +119,15 @@ def retcode(plugin, args='', key_name=None): def run_all(plugin, args='', key_name=None): - """ + ''' Run nagios plugin and return all the data execution with cmd.run_all - - """ - + ''' data = _execute_cmd(plugin, args, 'cmd.run_all', key_name) - return data def retcode_pillar(pillar_name): - """ + ''' Run one or more nagios plugins from pillar data and get the result of cdm.retcode The pillar have to be in this format: ------ @@ -154,9 +147,7 @@ def retcode_pillar(pillar_name): .. code-block:: bash salt '*' nagios.retcode webserver - - """ - + ''' groups = __salt__['pillar.get'](pillar_name) check = {} @@ -191,7 +182,7 @@ def retcode_pillar(pillar_name): def run_pillar(pillar_name): - """ + ''' Run one or more nagios plugins from pillar data and get the result of cdm.run The pillar have to be in this format: ------ @@ -211,15 +202,14 @@ def run_pillar(pillar_name): .. code-block:: bash salt '*' nagios.run webserver - - """ + ''' data = _execute_pillar(pillar_name, run) return data def run_all_pillar(pillar_name): - """ + ''' Run one or more nagios plugins from pillar data and get the result of cdm.run_all The pillar have to be in this format: ------ @@ -239,14 +229,13 @@ def run_all_pillar(pillar_name): .. code-block:: bash salt '*' nagios.run webserver - - """ + ''' data = _execute_pillar(pillar_name, run_all) return data def list_plugins(): - """ + ''' List all the nagios plugins CLI Example: @@ -254,9 +243,7 @@ def list_plugins(): .. code-block:: bash salt '*' nagios.list_plugins - - """ - + ''' plugin_list = os.listdir(PLUGINDIR) ret = [] for plugin in plugin_list: