From b5c6ead7a65fb03e1fbff773e24aa8611aa7c682 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 18 Oct 2013 12:56:54 -0600 Subject: [PATCH 01/58] First pass at an automated check in the minion tune-in loop, using a threaded approach to periodically refresh the master if the grains on a minion have changed. --- conf/minion | 9 +++++++++ salt/config.py | 2 ++ salt/minion.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/conf/minion b/conf/minion index de7e1b5f1b..d4e7876381 100644 --- a/conf/minion +++ b/conf/minion @@ -182,6 +182,15 @@ # often lower this value #loop_interval: 60 +# The grains_refresh_every setting allows for a minion to periodically check +# its grains to see if they have changed and, if so, to inform the master +# of the new grains. This operation is moderately expensive, therefore +# care should be taken not to set this value too low. A value of 600 seconds +# is a reasonable place to start. +# +# If the value is set to zero, this check is disabled. +grains_refresh_every = 0 + # When healing, a dns_check is run. This is to make sure that the originally # resolved dns has not changed. If this is something that does not happen in # your environment, set this value to False. diff --git a/salt/config.py b/salt/config.py index 34770eedb6..675aace554 100644 --- a/salt/config.py +++ b/salt/config.py @@ -165,6 +165,7 @@ VALID_OPTS = { 'win_repo_mastercachefile': str, 'win_gitrepos': list, 'modules_max_memory': int, + 'grains_refresh_every': int, } # default configurations @@ -252,6 +253,7 @@ DEFAULT_MINION_OPTS = { 'tcp_keepalive_cnt': -1, 'tcp_keepalive_intvl': -1, 'modules_max_memory': -1, + 'grains_refresh_every': 0, } DEFAULT_MASTER_OPTS = { diff --git a/salt/minion.py b/salt/minion.py index d3eef00234..07bac2af94 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -911,6 +911,24 @@ class Minion(object): data['arg'] = [] self._handle_decoded_payload(data) + def _refresh_grains_watcher(self, refresh_interval_in_seconds): + ''' + Create a loop that will fire a pillar refresh to inform a master about a change in the grains of this minion + :param refresh_interval_in_seconds: + :return: None + ''' + + def _do_refresh(): + if grain_cache != self.opts['grains']: + log.debug('Grain refresh is launching Pillar refresh!') + self.pillar_refresh() + + threading.Timer(refresh_interval_in_seconds, _do_refresh).start() + + grain_cache = self.opts['grains'] + threading.Timer(refresh_interval_in_seconds, _do_refresh).start() + log.debug('Starting grain refresh routine') + @property def master_pub(self): ''' @@ -982,6 +1000,7 @@ class Minion(object): def tune_in(self): ''' Lock onto the publisher. This is the main event loop for the minion + :rtype : None ''' try: log.info( @@ -1134,6 +1153,21 @@ class Minion(object): time.sleep(.5) loop_interval = int(self.opts['loop_interval']) + + # Calculate the refresh interval for grain refresh + if loop_interval and self.opts['grains_refresh_every']: + grains_refresh_interval = self.opts['grains_refresh_every'] + if loop_interval > grains_refresh_interval: # We can't refresh grains more frequently + # than we check for events + grains_refresh_interval = int(int(self.opts['grains_refresh_every']) / (loop_interval * 1000)) + + try: + if grains_refresh_interval: + self._refresh_grains_watcher(grains_refresh_interval) + except Exception: + log.error( + 'Exception occurred in attempt to initialize grain refresh routine during minion tune-in' + ) while True: try: self.schedule.eval() From 2d37f86a99aac621609132dc99ebef658085e9f1 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 18 Oct 2013 14:03:21 -0600 Subject: [PATCH 02/58] Make an effort to return a warning message if we get arguments passed to us that even Python's optparse can't deal with. Refs #6538. --- salt/utils/parsers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index b0fc789046..8ea13701e2 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -1080,10 +1080,14 @@ class SaltCMDOptionParser(OptionParser, ConfigDirMixIn, MergeConfigMixIn, ) def _mixin_after_parsed(self): - # Catch invalid invocations of salt such as: salt run if len(self.args) <= 1 and not self.options.doc: - self.print_help() - self.exit(1) + try: + self.print_help() + except: # We get an argument that Python's optparser just can't deal with. + # Perhaps stdout was redirected, or a file glob was passed in. Regardless, we're in an unknown + # state here. + sys.stdout.write("Invalid options passed. Please try -h for help.") # Try to warn if we can. + sys.exit(1) if self.options.doc: # Include the target From 2209f421eeace76ad1ed991c42f6b328b11a7806 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 18 Oct 2013 15:53:27 -0600 Subject: [PATCH 03/58] Re-write to use Scheduler instead of threads. --- conf/minion | 7 ++++-- salt/config.py | 2 +- salt/minion.py | 60 +++++++++++++++++++++++++++++++------------------- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/conf/minion b/conf/minion index d4e7876381..32cb8a8900 100644 --- a/conf/minion +++ b/conf/minion @@ -185,8 +185,11 @@ # The grains_refresh_every setting allows for a minion to periodically check # its grains to see if they have changed and, if so, to inform the master # of the new grains. This operation is moderately expensive, therefore -# care should be taken not to set this value too low. A value of 600 seconds -# is a reasonable place to start. +# care should be taken not to set this value too low. +# +# Note: This value is expressed in __minutes__! +# +# A value of 10 minutes is a reasonable default. # # If the value is set to zero, this check is disabled. grains_refresh_every = 0 diff --git a/salt/config.py b/salt/config.py index 675aace554..12dfa47181 100644 --- a/salt/config.py +++ b/salt/config.py @@ -253,7 +253,7 @@ DEFAULT_MINION_OPTS = { 'tcp_keepalive_cnt': -1, 'tcp_keepalive_intvl': -1, 'modules_max_memory': -1, - 'grains_refresh_every': 0, + 'grains_refresh_every': 1, } DEFAULT_MASTER_OPTS = { diff --git a/salt/minion.py b/salt/minion.py index 07bac2af94..8557902421 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -276,6 +276,7 @@ class MasterMinion(object): self.mk_rend = rend self.mk_matcher = matcher self.gen_modules() + self.grains_cache = self.opts['grains'] def gen_modules(self): ''' @@ -911,23 +912,24 @@ class Minion(object): data['arg'] = [] self._handle_decoded_payload(data) - def _refresh_grains_watcher(self, refresh_interval_in_seconds): + def _refresh_grains_watcher(self, refresh_interval_in_minutes): ''' Create a loop that will fire a pillar refresh to inform a master about a change in the grains of this minion - :param refresh_interval_in_seconds: + :param refresh_interval_in_minutes: :return: None ''' + if '__update_grains' not in self.opts.get('schedule', {}): + if not 'schedule' in self.opts: + self.opts['schedule'] = {} + self.opts['schedule'].update({ + '__update_grains': + { + 'function': 'event.fire', + 'args': [{}, "grains_refresh"], + 'minutes': refresh_interval_in_minutes + } + }) - def _do_refresh(): - if grain_cache != self.opts['grains']: - log.debug('Grain refresh is launching Pillar refresh!') - self.pillar_refresh() - - threading.Timer(refresh_interval_in_seconds, _do_refresh).start() - - grain_cache = self.opts['grains'] - threading.Timer(refresh_interval_in_seconds, _do_refresh).start() - log.debug('Starting grain refresh routine') @property def master_pub(self): @@ -1154,20 +1156,28 @@ class Minion(object): loop_interval = int(self.opts['loop_interval']) - # Calculate the refresh interval for grain refresh - if loop_interval and self.opts['grains_refresh_every']: - grains_refresh_interval = self.opts['grains_refresh_every'] - if loop_interval > grains_refresh_interval: # We can't refresh grains more frequently - # than we check for events - grains_refresh_interval = int(int(self.opts['grains_refresh_every']) / (loop_interval * 1000)) - try: - if grains_refresh_interval: - self._refresh_grains_watcher(grains_refresh_interval) - except Exception: + if self.opts['grains_refresh_every']: # If exists and is not zero. In minutes, not seconds! + if self.opts['grains_refresh_every'] > 1: + log.debug( + "Enabling the grains refresher. Will run every {0} minutes.".format\ + (self.opts['grains_refresh_every']) + ) + else: # Clean up minute vs. minutes in log message + log.debug( + "Enabling the grains refresher. Will run every {0} minute.".format\ + (self.opts['grains_refresh_every']) + + ) + self._refresh_grains_watcher( + abs(self.opts['grains_refresh_every']) + ) + except Exception as exc: log.error( - 'Exception occurred in attempt to initialize grain refresh routine during minion tune-in' + 'Exception occurred in attempt to initialize grain refresh routine during minion tune-in: {0}'.format( + exc) ) + while True: try: self.schedule.eval() @@ -1198,6 +1208,10 @@ class Minion(object): self.module_refresh() elif package.startswith('pillar_refresh'): self.pillar_refresh() + elif package.startswith('grains_refresh'): + if self.grains_cache != self.opts['grains']: + self.pillar_refresh() + self.grains_cache = self.opts['grains'] self.epub_sock.send(package) except Exception: pass From 9ba86c7b90b87b909423f664b1119b374cbeea82 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 20 Oct 2013 13:37:26 +0200 Subject: [PATCH 04/58] Fix(fileclient): friendly renegotiate authentication In case of authentication error, try to renegotiate the crypticle handshake to mitigate master restart This fixes #7987 --- salt/fileclient.py | 113 ++++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 63 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index f19daa0611..2a80e51f3b 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -357,10 +357,12 @@ class Client(object): if url_data.username is not None \ and url_data.scheme in ('http', 'https'): _, netloc = url_data.netloc.split('@', 1) - fixed_url = urlunparse((url_data.scheme, netloc, url_data.path, - url_data.params, url_data.query, url_data.fragment)) + fixed_url = urlunparse( + (url_data.scheme, netloc, url_data.path, + url_data.params, url_data.query, url_data.fragment)) passwd_mgr = url_passwd_mgr() - passwd_mgr.add_password(None, fixed_url, url_data.username, url_data.password) + passwd_mgr.add_password( + None, fixed_url, url_data.username, url_data.password) auth_handler = url_auth_handler(passwd_mgr) opener = url_build_opener(auth_handler) url_install_opener(opener) @@ -482,7 +484,9 @@ class LocalClient(Client): return ret prefix = prefix.strip('/') for path in self.opts['file_roots'][env]: - for root, dirs, files in os.walk(os.path.join(path, prefix), followlinks=True): + for root, dirs, files in os.walk( + os.path.join(path, prefix), followlinks=True + ): for fname in files: ret.append( os.path.relpath( @@ -502,7 +506,9 @@ class LocalClient(Client): if env not in self.opts['file_roots']: return ret for path in self.opts['file_roots'][env]: - for root, dirs, files in os.walk(os.path.join(path, prefix), followlinks=True): + for root, dirs, files in os.walk( + os.path.join(path, prefix), followlinks=True + ): if len(dirs) == 0 and len(files) == 0: ret.append(os.path.relpath(root, path)) return ret @@ -517,7 +523,9 @@ class LocalClient(Client): return ret prefix = prefix.strip('/') for path in self.opts['file_roots'][env]: - for root, dirs, files in os.walk(os.path.join(path, prefix), followlinks=True): + for root, dirs, files in os.walk( + os.path.join(path, prefix), followlinks=True + ): ret.append(os.path.relpath(root, path)) return ret @@ -605,6 +613,26 @@ class RemoteClient(Client): self.auth = salt.crypt.SAuth(opts) self.sreq = salt.payload.SREQ(self.opts['master_uri']) + def _crypted_transfer(self, load, tries=3, timeout=60, payload='aes'): + ''' + In case of authentication errors, try to renogiate authentication + and retry the method. + Indeed, we can fail too early in case of a master restart during a + minion state executon call + ''' + def _do_transfer(): + return self.auth.crypticle.loads( + self.sreq.send(payload, + self.auth.crypticle.dumps(load), + tries, + timeout) + ) + try: + return _do_transfer() + except salt.crypt.AuthenticationError: + self.auth = salt.crypt.SAuth(self.opts) + return _do_transfer() + def get_file(self, path, dest='', makedirs=False, env='base', gzip=None): ''' Get a single file from the salt-master @@ -612,7 +640,8 @@ class RemoteClient(Client): dest is omitted, then the downloaded file will be placed in the minion cache ''' - #-- Hash compare local copy with master and skip download if no diference found. + #-- Hash compare local copy with master and skip download + # if no diference found. dest2check = dest if not dest2check: rel_path = self._check_proto(path) @@ -623,7 +652,9 @@ class RemoteClient(Client): hash_local = self.hash_file(dest2check, env) hash_server = self.hash_file(path, env) if hash_local == hash_server: - log.info('Fetching file ** skipped **, latest already in cache \'{0}\''.format(path)) + log.info( + 'Fetching file ** skipped **, ' + 'latest already in cache \'{0}\''.format(path)) return dest2check log.debug('Fetching file ** attempting ** \'{0}\''.format(path)) @@ -651,12 +682,7 @@ class RemoteClient(Client): else: load['loc'] = fn_.tell() try: - data = self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + data = self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -707,12 +733,7 @@ class RemoteClient(Client): 'prefix': prefix, 'cmd': '_file_list'} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + return self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -724,12 +745,7 @@ class RemoteClient(Client): 'prefix': prefix, 'cmd': '_file_list_emptydirs'} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -741,12 +757,7 @@ class RemoteClient(Client): 'prefix': prefix, 'cmd': '_dir_list'} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + return self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -758,12 +769,7 @@ class RemoteClient(Client): 'prefix': prefix, 'cmd': '_symlink_list'} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + return self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -782,19 +788,15 @@ class RemoteClient(Client): return {} else: ret = {} - ret['hsum'] = salt.utils.get_hash(path, form='md5', chunk_size=4096) + ret['hsum'] = salt.utils.get_hash( + path, form='md5', chunk_size=4096) ret['hash_type'] = 'md5' return ret load = {'path': path, 'env': env, 'cmd': '_file_hash'} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + return self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -805,12 +807,7 @@ class RemoteClient(Client): load = {'env': env, 'cmd': '_file_list'} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + return self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -820,12 +817,7 @@ class RemoteClient(Client): ''' load = {'cmd': '_master_opts'} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + return self._crypted_transfer(load) except SaltReqTimeoutError: return '' @@ -839,11 +831,6 @@ class RemoteClient(Client): 'opts': self.opts, 'tok': self.auth.gen_token('salt')} try: - return self.auth.crypticle.loads( - self.sreq.send('aes', - self.auth.crypticle.dumps(load), - 3, - 60) - ) + return self._crypted_transfer(load) except SaltReqTimeoutError: return '' From e6da57906a7dfe8b3d2a5e9451093292422e359c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 20 Oct 2013 06:54:18 -0600 Subject: [PATCH 05/58] whitespace fix per pylint --- salt/states/ssh_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/ssh_auth.py b/salt/states/ssh_auth.py index 53339b013a..1bf92744db 100644 --- a/salt/states/ssh_auth.py +++ b/salt/states/ssh_auth.py @@ -8,7 +8,7 @@ controlled via the ssh_auth state. Defaults can be set by the enc, options, and comment keys. These defaults can be overridden by including them in the name. -Since the YAML specification limits the length of simple keys to 1024 +Since the YAML specification limits the length of simple keys to 1024 characters, and since SSH keys are often longer than that, you may have to use a YAML 'explicit key', as demonstrated in the second example below. From c7fcc79747e357d82591f2851b29224780f3efed Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 14:35:10 -0500 Subject: [PATCH 06/58] Add tests for pkg.installed with version specification --- tests/integration/states/pkg.py | 63 ++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 95fdb9444b..69094f6521 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -25,8 +25,7 @@ _PKG_TARGETS = { } -@requires_salt_modules('pkg.latest_version') -@requires_salt_modules('pkg.version') +@requires_salt_modules('pkg.version', 'pkg.latest_version') class PkgTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ''' @@ -55,9 +54,37 @@ class PkgTest(integration.ModuleCase, # needs to not be installed before we run the states below self.assertFalse(version) - ret = self.run_state('pkg.installed', name=pkg_targets[0]) + ret = self.run_state('pkg.installed', name=target) self.assertSaltTrueReturn(ret) - ret = self.run_state('pkg.removed', name=pkg_targets[0]) + ret = self.run_state('pkg.removed', name=target) + self.assertSaltTrueReturn(ret) + + @destructiveTest + @skipIf(salt.utils.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_installed_with_version(self, grains=None): + ''' + This is a destructive test as it installs and then removes a package + ''' + os_family = grains.get('os_family', '') + pkg_targets = _PKG_TARGETS.get(os_family, []) + + # Make sure that we have targets that match the os_family. If this + # fails then the _PKG_TARGETS dict above needs to have an entry added, + # with two packages that are not installed before these tests are run + self.assertTrue(pkg_targets) + + target = pkg_targets[0] + version = self.run_function('pkg.latest_version', [target]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertTrue(version) + + ret = self.run_state('pkg.installed', name=target) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) @destructiveTest @@ -87,6 +114,34 @@ class PkgTest(integration.ModuleCase, ret = self.run_state('pkg.removed', name=None, pkgs=pkg_targets) self.assertSaltTrueReturn(ret) + @destructiveTest + @skipIf(salt.utils.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_installed_multipkg_with_version(self, grains=None): + ''' + This is a destructive test as it installs and then removes two packages + ''' + os_family = grains.get('os_family', '') + pkg_targets = _PKG_TARGETS.get(os_family, []) + + # Make sure that we have targets that match the os_family. If this + # fails then the _PKG_TARGETS dict above needs to have an entry added, + # with two packages that are not installed before these tests are run + self.assertTrue(pkg_targets) + + version = self.run_function('pkg.latest_version', [pkg_targets[0]]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so these + # packages need to not be installed before we run the states below + self.assertTrue(version) + + pkgs = [{pkg_targets[0]: version}, pkg_targets[1]] + + ret = self.run_state('pkg.installed', name=None, pkgs=pkgs) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pkg.removed', name=None, pkgs=pkg_targets) + self.assertSaltTrueReturn(ret) if __name__ == '__main__': from integration import run_tests From fe1c4487e3bc43d17757f42ab1ac689081e54fb4 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 15:16:36 -0500 Subject: [PATCH 07/58] add tests for installing 32-bit packages --- tests/integration/states/pkg.py | 64 ++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 69094f6521..102112c6d6 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -24,6 +24,11 @@ _PKG_TARGETS = { 'Suse': ['aalib', 'finch'] } +_PKG_TARGETS_32 = { + 'Debian': '', + 'RedHat': 'zlib.i686' +} + @requires_salt_modules('pkg.version', 'pkg.latest_version') class PkgTest(integration.ModuleCase, @@ -82,7 +87,7 @@ class PkgTest(integration.ModuleCase, # needs to not be installed before we run the states below self.assertTrue(version) - ret = self.run_state('pkg.installed', name=target) + ret = self.run_state('pkg.installed', name=target, version=version) self.assertSaltTrueReturn(ret) ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) @@ -143,6 +148,63 @@ class PkgTest(integration.ModuleCase, ret = self.run_state('pkg.removed', name=None, pkgs=pkg_targets) self.assertSaltTrueReturn(ret) + @destructiveTest + @skipIf(salt.utils.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_installed_32bit(self, grains=None): + ''' + This is a destructive test as it installs and then removes a package + ''' + os_family = grains.get('os_family', '') + target = _PKG_TARGETS_32.get(os_family, '') + + # _PKG_TARGETS_32 is only populated for the OS families for which Salt + # has to munge package names for 32-bit-on-x86_64 (Currently only + # Debian and Redhat). Don't actually perform this test on other + # platforms. + if target: + version = self.run_function('pkg.version', [target]) + + # If this assert fails, we need to find a new target. This test + # needs to be able to test successful installation of packages, so + # the target needs to not be installed before we run the states + # below + self.assertFalse(version) + + ret = self.run_state('pkg.installed', name=target) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pkg.removed', name=target) + self.assertSaltTrueReturn(ret) + + @destructiveTest + @skipIf(salt.utils.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_installed_32bit_with_version(self, grains=None): + ''' + This is a destructive test as it installs and then removes a package + ''' + os_family = grains.get('os_family', '') + target = _PKG_TARGETS_32.get(os_family, '') + + # _PKG_TARGETS_32 is only populated for the OS families for which Salt + # has to munge package names for 32-bit-on-x86_64 (Currently only + # Debian and Redhat). Don't actually perform this test on other + # platforms. + if target: + version = self.run_function('pkg.latest_version', [target]) + + # If this assert fails, we need to find a new target. This test + # needs to be able to test successful installation of the package, so + # the target needs to not be installed before we run the states + # below + self.assertTrue(version) + + ret = self.run_state('pkg.installed', name=target, version=version) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pkg.removed', name=target) + self.assertSaltTrueReturn(ret) + + if __name__ == '__main__': from integration import run_tests run_tests(PkgTest) From 5dc08d552a613a06ebc23420384f0c700838bd1a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 15:48:57 -0500 Subject: [PATCH 08/58] adjust pkg targets for pkg tests --- tests/integration/states/pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 102112c6d6..49c72a5be9 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -17,9 +17,9 @@ import integration import salt.utils _PKG_TARGETS = { - 'Arch': ['bzr', 'finch'], + 'Arch': ['python2-django', 'finch'], 'Debian': ['python-plist', 'finch'], - 'RedHat': ['bzr', 'finch'], + 'RedHat': ['python-cli', 'finch'], 'FreeBSD': ['aalib', 'pth'], 'Suse': ['aalib', 'finch'] } From d5120f1041226ca0829076baea7f26c6d8343623 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 15:53:24 -0500 Subject: [PATCH 09/58] Fix generic exception: Use "except Exception" instead of simply "except". Fixes pylint violation (see below link for details). http://jenkins.saltstack.com/job/salt-lint/644/violations/file/salt/utils/parsers.py/ --- salt/utils/parsers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 8ea13701e2..1cd1e4c6b8 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -1083,10 +1083,12 @@ class SaltCMDOptionParser(OptionParser, ConfigDirMixIn, MergeConfigMixIn, if len(self.args) <= 1 and not self.options.doc: try: self.print_help() - except: # We get an argument that Python's optparser just can't deal with. - # Perhaps stdout was redirected, or a file glob was passed in. Regardless, we're in an unknown - # state here. - sys.stdout.write("Invalid options passed. Please try -h for help.") # Try to warn if we can. + except Exception: + # We get an argument that Python's optparser just can't deal + # with. Perhaps stdout was redirected, or a file glob was + # passed in. Regardless, we're in an unknown state here. + sys.stdout.write('Invalid options passed. Please try -h for ' + 'help.') # Try to warn if we can. sys.exit(1) if self.options.doc: From 62a9385ddf7c382da82747e74509eea903b6818a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 15:58:59 -0500 Subject: [PATCH 10/58] Don't perform version spec pkg tests on FreeBSD pkg_add does not support this, so this test will fail when pkg.latest_version is run and an assertion is done to determine the targeted version. --- tests/integration/states/pkg.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 49c72a5be9..d39cdcc31e 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -74,6 +74,11 @@ class PkgTest(integration.ModuleCase, os_family = grains.get('os_family', '') pkg_targets = _PKG_TARGETS.get(os_family, []) + # Don't perform this test on FreeBSD since version specification is not + # supported. + if os_family == 'FreeBSD': + return + # Make sure that we have targets that match the os_family. If this # fails then the _PKG_TARGETS dict above needs to have an entry added, # with two packages that are not installed before these tests are run @@ -129,6 +134,11 @@ class PkgTest(integration.ModuleCase, os_family = grains.get('os_family', '') pkg_targets = _PKG_TARGETS.get(os_family, []) + # Don't perform this test on FreeBSD since version specification is not + # supported. + if os_family == 'FreeBSD': + return + # Make sure that we have targets that match the os_family. If this # fails then the _PKG_TARGETS dict above needs to have an entry added, # with two packages that are not installed before these tests are run From c5f5b400e91e60450b8a4159927b8896f44b3dc2 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 17:04:27 -0500 Subject: [PATCH 11/58] adjust redhat pkg targets for pkg tests --- tests/integration/states/pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index d39cdcc31e..00e7605237 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -19,7 +19,7 @@ import salt.utils _PKG_TARGETS = { 'Arch': ['python2-django', 'finch'], 'Debian': ['python-plist', 'finch'], - 'RedHat': ['python-cli', 'finch'], + 'RedHat': ['python-meh', 'finch'], 'FreeBSD': ['aalib', 'pth'], 'Suse': ['aalib', 'finch'] } From 63ee0f1e83682c407b1dbdea71037a2ada0e79e3 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 18:18:35 -0500 Subject: [PATCH 12/58] Add function to wait for pkgdb lock to clear Arch pkg tests that call pkg.latest_version seem to fail on running this function due to the package database still being locked from previous tests. This adds a function that waits (in 5 second increments) for up to 60 seconds for the pkgdb to be unlocked. http://jenkins.saltstack.com/job/salt-rs-arch/474/artifact/salt-runtests.log/*view*/ --- tests/integration/states/pkg.py | 37 +++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 00e7605237..449d5e3076 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -12,6 +12,10 @@ from salttesting.helpers import ( ) ensure_in_syspath('../../') +# Import python libs +import os +import time + # Import salt libs import integration import salt.utils @@ -30,6 +34,18 @@ _PKG_TARGETS_32 = { } +def _wait_for_pkgdb_lock(): + ''' + Package tests tend to fail on Arch Linux due to pkgdb lockfile being + present. This will wait up to 60 seconds before bailing. + ''' + for idx in xrange(12): + if not os.path.isfile('/var/lib/pacman/db.lck'): + return + time.sleep(5) + raise Exception('Package database locked after 60 seconds, bailing out') + + @requires_salt_modules('pkg.version', 'pkg.latest_version') class PkgTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): @@ -39,7 +55,7 @@ class PkgTest(integration.ModuleCase, @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains - def test_pkg_installed(self, grains=None): + def test_pkg_001_installed(self, grains=None): ''' This is a destructive test as it installs and then removes a package ''' @@ -67,7 +83,7 @@ class PkgTest(integration.ModuleCase, @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains - def test_pkg_installed_with_version(self, grains=None): + def test_pkg_002_installed_with_version(self, grains=None): ''' This is a destructive test as it installs and then removes a package ''' @@ -84,6 +100,9 @@ class PkgTest(integration.ModuleCase, # with two packages that are not installed before these tests are run self.assertTrue(pkg_targets) + if os_family == 'Arch': + _wait_for_pkgdb_unlock() + target = pkg_targets[0] version = self.run_function('pkg.latest_version', [target]) @@ -100,7 +119,7 @@ class PkgTest(integration.ModuleCase, @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains - def test_pkg_installed_multipkg(self, grains=None): + def test_pkg_003_installed_multipkg(self, grains=None): ''' This is a destructive test as it installs and then removes two packages ''' @@ -127,7 +146,7 @@ class PkgTest(integration.ModuleCase, @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains - def test_pkg_installed_multipkg_with_version(self, grains=None): + def test_pkg_004_installed_multipkg_with_version(self, grains=None): ''' This is a destructive test as it installs and then removes two packages ''' @@ -144,6 +163,9 @@ class PkgTest(integration.ModuleCase, # with two packages that are not installed before these tests are run self.assertTrue(pkg_targets) + if os_family == 'Arch': + _wait_for_pkgdb_unlock() + version = self.run_function('pkg.latest_version', [pkg_targets[0]]) # If this assert fails, we need to find new targets, this test needs to @@ -161,7 +183,7 @@ class PkgTest(integration.ModuleCase, @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains - def test_pkg_installed_32bit(self, grains=None): + def test_pkg_005_installed_32bit(self, grains=None): ''' This is a destructive test as it installs and then removes a package ''' @@ -189,7 +211,7 @@ class PkgTest(integration.ModuleCase, @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains - def test_pkg_installed_32bit_with_version(self, grains=None): + def test_pkg_006_installed_32bit_with_version(self, grains=None): ''' This is a destructive test as it installs and then removes a package ''' @@ -201,6 +223,9 @@ class PkgTest(integration.ModuleCase, # Debian and Redhat). Don't actually perform this test on other # platforms. if target: + if os_family == 'Arch': + _wait_for_pkgdb_unlock() + version = self.run_function('pkg.latest_version', [target]) # If this assert fails, we need to find a new target. This test From 90d6d045f1595729a783bceea8672a6ed8590308 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 18:48:01 -0500 Subject: [PATCH 13/58] Move _wait_for_pkgdb_unlock inside of PkgTest class --- tests/integration/states/pkg.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 449d5e3076..01e3a4a61b 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -34,24 +34,23 @@ _PKG_TARGETS_32 = { } -def _wait_for_pkgdb_lock(): - ''' - Package tests tend to fail on Arch Linux due to pkgdb lockfile being - present. This will wait up to 60 seconds before bailing. - ''' - for idx in xrange(12): - if not os.path.isfile('/var/lib/pacman/db.lck'): - return - time.sleep(5) - raise Exception('Package database locked after 60 seconds, bailing out') - - @requires_salt_modules('pkg.version', 'pkg.latest_version') class PkgTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ''' pkg.installed state tests ''' + def _wait_for_pkgdb_lock(): + ''' + Package tests tend to fail on Arch Linux due to pkgdb lockfile being + present. This will wait up to 60 seconds before bailing. + ''' + for idx in xrange(12): + if not os.path.isfile('/var/lib/pacman/db.lck'): + return + time.sleep(5) + raise Exception('Package database locked after 60 seconds, bailing out') + @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains @@ -101,7 +100,7 @@ class PkgTest(integration.ModuleCase, self.assertTrue(pkg_targets) if os_family == 'Arch': - _wait_for_pkgdb_unlock() + self._wait_for_pkgdb_unlock() target = pkg_targets[0] version = self.run_function('pkg.latest_version', [target]) @@ -164,7 +163,7 @@ class PkgTest(integration.ModuleCase, self.assertTrue(pkg_targets) if os_family == 'Arch': - _wait_for_pkgdb_unlock() + self._wait_for_pkgdb_unlock() version = self.run_function('pkg.latest_version', [pkg_targets[0]]) @@ -224,7 +223,7 @@ class PkgTest(integration.ModuleCase, # platforms. if target: if os_family == 'Arch': - _wait_for_pkgdb_unlock() + self._wait_for_pkgdb_unlock() version = self.run_function('pkg.latest_version', [target]) From 939784002191155ee2058a14a19197d0f20d2c21 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 20:16:28 -0500 Subject: [PATCH 14/58] More tweaks to package tests This makes package tests work better on CentOS 5, and also adds a 32-bit target for Ubuntu. --- tests/integration/states/pkg.py | 40 ++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 01e3a4a61b..4276509760 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -23,14 +23,14 @@ import salt.utils _PKG_TARGETS = { 'Arch': ['python2-django', 'finch'], 'Debian': ['python-plist', 'finch'], - 'RedHat': ['python-meh', 'finch'], + 'RedHat': ['xz-devel', 'zsh-html'], 'FreeBSD': ['aalib', 'pth'], 'Suse': ['aalib', 'finch'] } _PKG_TARGETS_32 = { - 'Debian': '', - 'RedHat': 'zlib.i686' + 'Ubuntu': 'memtest86+:i386', + 'CentOS': 'xz-devel.i686' } @@ -186,14 +186,18 @@ class PkgTest(integration.ModuleCase, ''' This is a destructive test as it installs and then removes a package ''' - os_family = grains.get('os_family', '') - target = _PKG_TARGETS_32.get(os_family, '') + os_name = grains.get('os', '') + target = _PKG_TARGETS_32.get(os_name, '') - # _PKG_TARGETS_32 is only populated for the OS families for which Salt - # has to munge package names for 32-bit-on-x86_64 (Currently only - # Debian and Redhat). Don't actually perform this test on other - # platforms. + # _PKG_TARGETS_32 is only populated for platforms for which Salt has to + # munge package names for 32-bit-on-x86_64 (Currently only Ubuntu and + # RHEL-based). Don't actually perform this test on other platforms. if target: + # CentOS 5 has .i386 arch designation for 32-bit pkgs + elif os_name == 'CentOS' \ + and grains['osrelease'].startswith('5.'): + target = target.replace('.i686', '.i386') + version = self.run_function('pkg.version', [target]) # If this assert fails, we need to find a new target. This test @@ -214,17 +218,21 @@ class PkgTest(integration.ModuleCase, ''' This is a destructive test as it installs and then removes a package ''' - os_family = grains.get('os_family', '') - target = _PKG_TARGETS_32.get(os_family, '') + os_name = grains.get('os', '') + target = _PKG_TARGETS_32.get(os_name, '') - # _PKG_TARGETS_32 is only populated for the OS families for which Salt - # has to munge package names for 32-bit-on-x86_64 (Currently only - # Debian and Redhat). Don't actually perform this test on other - # platforms. + # _PKG_TARGETS_32 is only populated for platforms for which Salt has to + # munge package names for 32-bit-on-x86_64 (Currently only Ubuntu and + # RHEL-based). Don't actually perform this test on other platforms. if target: - if os_family == 'Arch': + if grains.get('os_family', '') == 'Arch': self._wait_for_pkgdb_unlock() + # CentOS 5 has .i386 arch designation for 32-bit pkgs + elif os_name == 'CentOS' \ + and grains['osrelease'].startswith('5.'): + target = target.replace('.i686', '.i386') + version = self.run_function('pkg.latest_version', [target]) # If this assert fails, we need to find a new target. This test From 1c0d279c5954b60080221d9b75e593742d240ddb Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 20:50:28 -0500 Subject: [PATCH 15/58] Fix import error --- tests/integration/states/pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 4276509760..ca84b3b8cf 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -194,7 +194,7 @@ class PkgTest(integration.ModuleCase, # RHEL-based). Don't actually perform this test on other platforms. if target: # CentOS 5 has .i386 arch designation for 32-bit pkgs - elif os_name == 'CentOS' \ + if os_name == 'CentOS' \ and grains['osrelease'].startswith('5.'): target = target.replace('.i686', '.i386') @@ -229,7 +229,7 @@ class PkgTest(integration.ModuleCase, self._wait_for_pkgdb_unlock() # CentOS 5 has .i386 arch designation for 32-bit pkgs - elif os_name == 'CentOS' \ + if os_name == 'CentOS' \ and grains['osrelease'].startswith('5.'): target = target.replace('.i686', '.i386') From 74ce0cb604191385084996f7a85a8672fe1b2c5f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 21:09:58 -0500 Subject: [PATCH 16/58] Add CLI example for ports.list_all --- salt/modules/freebsdports.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/modules/freebsdports.py b/salt/modules/freebsdports.py index efd7e1fbe0..b8070fe6f8 100644 --- a/salt/modules/freebsdports.py +++ b/salt/modules/freebsdports.py @@ -404,6 +404,12 @@ def list_all(): ''' Lists all ports available. + CLI Example: + + .. code-block:: bash + + salt '*' ports.list_all + .. warning:: Takes a while to run, and returns a **LOT** of output From 81434578fc7b4da246360b71371298e2348a116a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 21:29:36 -0500 Subject: [PATCH 17/58] Fix function name --- tests/integration/states/pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index ca84b3b8cf..5586ec7ff0 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -40,7 +40,7 @@ class PkgTest(integration.ModuleCase, ''' pkg.installed state tests ''' - def _wait_for_pkgdb_lock(): + def _wait_for_pkgdb_unlock(): ''' Package tests tend to fail on Arch Linux due to pkgdb lockfile being present. This will wait up to 60 seconds before bailing. From 7984a8c2cf45c3409415fe7ff4af5eabf8e45eec Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 21:46:06 -0500 Subject: [PATCH 18/58] Apparently the test framework doesn't like private member functions --- tests/integration/states/pkg.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 5586ec7ff0..1926fc4841 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -40,17 +40,6 @@ class PkgTest(integration.ModuleCase, ''' pkg.installed state tests ''' - def _wait_for_pkgdb_unlock(): - ''' - Package tests tend to fail on Arch Linux due to pkgdb lockfile being - present. This will wait up to 60 seconds before bailing. - ''' - for idx in xrange(12): - if not os.path.isfile('/var/lib/pacman/db.lck'): - return - time.sleep(5) - raise Exception('Package database locked after 60 seconds, bailing out') - @destructiveTest @skipIf(salt.utils.is_windows(), 'minion is windows') @requires_system_grains @@ -100,7 +89,13 @@ class PkgTest(integration.ModuleCase, self.assertTrue(pkg_targets) if os_family == 'Arch': - self._wait_for_pkgdb_unlock() + for idx in xrange(13): + if idx == 12: + raise Exception('Package database locked after 60 seconds, ' + 'bailing out') + if not os.path.isfile('/var/lib/pacman/db.lck'): + break + time.sleep(5) target = pkg_targets[0] version = self.run_function('pkg.latest_version', [target]) @@ -163,7 +158,13 @@ class PkgTest(integration.ModuleCase, self.assertTrue(pkg_targets) if os_family == 'Arch': - self._wait_for_pkgdb_unlock() + for idx in xrange(13): + if idx == 12: + raise Exception('Package database locked after 60 seconds, ' + 'bailing out') + if not os.path.isfile('/var/lib/pacman/db.lck'): + break + time.sleep(5) version = self.run_function('pkg.latest_version', [pkg_targets[0]]) From 95efdad8bac553d81d6fcc11c35507c6310ba2c5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 20 Oct 2013 21:47:05 -0500 Subject: [PATCH 19/58] Remove ubuntu 32-bit target --- tests/integration/states/pkg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 1926fc4841..cc4310bc62 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -29,7 +29,6 @@ _PKG_TARGETS = { } _PKG_TARGETS_32 = { - 'Ubuntu': 'memtest86+:i386', 'CentOS': 'xz-devel.i686' } From 71184923ee6708775705f7bf4406062d940404a0 Mon Sep 17 00:00:00 2001 From: Puluto Date: Mon, 21 Oct 2013 15:05:02 +0800 Subject: [PATCH 20/58] Update state.py https://github.com/saltstack/salt/issues/7991 --- salt/client/ssh/state.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/client/ssh/state.py b/salt/client/ssh/state.py index 8aeace87ac..b8c46c92d2 100644 --- a/salt/client/ssh/state.py +++ b/salt/client/ssh/state.py @@ -142,11 +142,16 @@ def prep_trans_tar(opts, chunks, file_refs): break cwd = os.getcwd() os.chdir(gendir) - with tarfile.open(trans_tar, 'w:gz') as tfp: + try: + tfp = tarfile.open(trans_tar, 'w:gz') for root, dirs, files in os.walk(gendir): for name in files: full = os.path.join(root, name) tfp.add(full[len(gendir):].lstrip(os.sep)) + except Exception, ex: + print ex + finally: + tfp.close() os.chdir(cwd) shutil.rmtree(gendir) return trans_tar From 498d9013b719eef85782f34425fce1afaaf9cd14 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 21 Oct 2013 10:24:29 -0500 Subject: [PATCH 21/58] Add versionadded directive to dockerio module Also changed the top line of the docstring, as this affects its appearance in the docs, and this top line should be a simple explanation of the module. Having "overview of this module" in this line is redundant. --- salt/modules/dockerio.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/modules/dockerio.py b/salt/modules/dockerio.py index 648e9bdc0a..e73ea9675a 100644 --- a/salt/modules/dockerio.py +++ b/salt/modules/dockerio.py @@ -1,8 +1,9 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- ''' -Management of dockers: overview of this module -============================================== +Management of dockers +===================== + +.. versionadded:: Hydrogen .. note:: From 67ca15ecc2b0d7affe1ad795c140387398116283 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 21 Oct 2013 10:43:55 -0500 Subject: [PATCH 22/58] Fix sphinx build warning Section headers must have their underline the same length as the section header itself. --- doc/topics/releases/0.17.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/releases/0.17.1.rst b/doc/topics/releases/0.17.1.rst index 32ee231136..48c4e5b906 100644 --- a/doc/topics/releases/0.17.1.rst +++ b/doc/topics/releases/0.17.1.rst @@ -33,7 +33,7 @@ Be advised that these security issues all apply to a small subset of Salt users and mostly apply to Salt SSH. Insufficient Argument Validation -------------------------------- +-------------------------------- This issue allowed for a user with limited privileges to embed executions inside of routines to execute routines that should be restricted. This applies From c6a5fca0d3cf0ccce5cda54654a68e757128aeab Mon Sep 17 00:00:00 2001 From: Elias Probst Date: Mon, 21 Oct 2013 20:56:13 +0200 Subject: [PATCH 23/58] =?UTF-8?q?Corrected=20typo=20(extended=20auth=20?= =?UTF-8?q?=E2=86=92=20external=20auth).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- salt/utils/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index cfecf7130e..b8a8683f3c 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -1003,10 +1003,10 @@ class SaltCMDOptionParser(OptionParser, ConfigDirMixIn, MergeConfigMixIn, 'minions to have running') ) self.add_option( - '-a', '--auth', '--eauth', '--extended-auth', + '-a', '--auth', '--eauth', '--external-auth', default='', dest='eauth', - help=('Specify an extended authentication system to use.') + help=('Specify an external authentication system to use.') ) self.add_option( '-T', '--make-token', From c4453b330700a6cd46ce3fbd5690938ddc39be70 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 21 Oct 2013 13:16:54 -0600 Subject: [PATCH 24/58] Add a lexical parser for MySQL grants to the MySQL module. Change the behavior of the grant-checker to use this new method by default and fallback to the strict method if it fails. --- salt/modules/mysql.py | 78 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index c708bf88d4..b30489ed53 100644 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -34,6 +34,9 @@ import sys # Import salt libs import salt.utils +#import shlex which should be distributed with Python +import shlex + # Import third party libs try: import MySQLdb @@ -148,6 +151,59 @@ def _connect(**kwargs): return dbc +def _grant_to_tokens(grant): + ''' + + This should correspond fairly closely to the YAML rendering of a mysql_grants state which comes out + as follows: + + OrderedDict([('whatever_identifier', OrderedDict([('mysql_grants.present', + [OrderedDict([('database', 'testdb.*')]), OrderedDict([('user', 'testuser')]), + OrderedDict([('grant', 'ALTER, SELECT, LOCK TABLES')]), OrderedDict([('host', 'localhost')])])]))]) + + :param grant: An un-parsed MySQL GRANT statement str, like + "GRANT SELECT, ALTER, LOCK TABLES ON `testdb`.* TO 'testuser'@'localhost'" + :return: + A Python dict with the following keys/values: + - user: MySQL User + - host: MySQL host + - grant: [grant1, grant2] (ala SELECT, USAGE, etc) + - database: MySQL DB + ''' + exploded_grant = shlex.split(grant) + grant_tokens = [] + position_tracker = 1 # Skip the initial 'GRANT' word token + phrase = 'grants' + + for token in exploded_grant[position_tracker:]: + if phrase == 'grants': + cleaned_token = token.rstrip(',') + grant_tokens.append(cleaned_token) + position_tracker += 1 + + if phrase == 'db': + database = token.strip('`') + phrase = 'tables' + + if phrase == 'user': + user, host = token.split('@') + + if token == 'ON': + phrase = 'db' + + if token == 'TO': + phrase = 'user' + + return { + 'user': user, + 'host': host, + 'grant': grant_tokens, + 'database': database, + + } + + + def query(database, query, **connection_args): ''' Run an arbitrary SQL query and return the results or @@ -1080,14 +1136,30 @@ def grant_exists(grant, ) grants = user_grants(user, host, **connection_args) - if grants is not False and target in grants: - log.debug('Grant exists.') - return True + + for grant in grants: + try: + target_tokens = None + if not target_tokens: # Avoid the overhead of re-calc in loop + target_tokens = _grant_to_tokens(target) + grant_tokens = _grant_to_tokens(grant) + if grant_tokens['user'] == target_tokens['user'] and \ + grant_tokens['database'] == target_tokens['database'] and \ + grant_tokens['host'] == target_tokens['host']: + if set(grant_tokens['grant']) == set(target_tokens['grant']): + return True + + except Exception as exc: # Fallback to strict parsing + log.debug("OH NO CAUGHT EXCEPTION: {0}".format(exc)) + if grants is not False and target in grants: + log.debug('Grant exists.') + return True log.debug('Grant does not exist, or is perhaps not ordered properly?') return False + def grant_add(grant, database, user, From 5678f46744c23688dee800d819f072cba150f2e5 Mon Sep 17 00:00:00 2001 From: Sebastien Pahl Date: Mon, 21 Oct 2013 12:25:11 -0700 Subject: [PATCH 25/58] Generate jid in _wait in state module if not specified (can happen when called in the mine with queue=True) --- salt/modules/state.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index 161f78566b..7eed15a518 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -11,6 +11,7 @@ import shutil import time import logging import tarfile +import datetime import tempfile # Import salt libs @@ -66,6 +67,8 @@ def _wait(jid): ''' Wait for all previously started state jobs to finish running ''' + if jid is None: + jid = '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now()) states = _prior_running_states(jid) while states: time.sleep(1) @@ -126,7 +129,7 @@ def low(data, queue=False, **kwargs): salt '*' state.low '{"state": "pkg", "fun": "installed", "name": "vi"}' ''' if queue: - _wait(kwargs['__pub_jid']) + _wait(kwargs.get('__pub_jid')) else: conflict = running() if conflict: @@ -157,7 +160,7 @@ def high(data, queue=False, **kwargs): salt '*' state.high '{"vim": {"pkg": ["installed"]}}' ''' if queue: - _wait(kwargs['__pub_jid']) + _wait(kwargs.get('__pub_jid')) else: conflict = running() if conflict: @@ -180,7 +183,7 @@ def template(tem, queue=False, **kwargs): salt '*' state.template '' ''' if queue: - _wait(kwargs['__pub_jid']) + _wait(kwargs.get('__pub_jid')) else: conflict = running() if conflict: @@ -203,7 +206,7 @@ def template_str(tem, queue=False, **kwargs): salt '*' state.template_str '