From 3602af1e1bd7a2badae4365f2ae9ba57891a566a Mon Sep 17 00:00:00 2001 From: Andreas Ulm Date: Wed, 28 Jun 2017 00:32:20 +0200 Subject: [PATCH 001/508] Fix rabbitmqctl output handler for 3.6.10 Fix the output sanatizer for rabbitmqctl output generated by rabbitmq-server 3.6.10 and later. #41955 Signed-off-by: Andreas Ulm --- salt/modules/rabbitmq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index 801237789d..4cfdeb4d3d 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -72,6 +72,7 @@ def _safe_output(line): ''' return not any([ line.startswith('Listing') and line.endswith('...'), + line.startswith('Listing') and '\t' not in line, '...done' in line, line.startswith('WARNING:') ]) From 76fd941d915c518f3565c19f05037c767af8b04f Mon Sep 17 00:00:00 2001 From: Andreas Ulm Date: Wed, 28 Jun 2017 01:13:30 +0200 Subject: [PATCH 002/508] added tests for rabbitmq 3.6.10 output handler Signed-off-by: Andreas Ulm --- tests/unit/modules/rabbitmq_test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/modules/rabbitmq_test.py b/tests/unit/modules/rabbitmq_test.py index b0a665fc11..661a4cd9c8 100644 --- a/tests/unit/modules/rabbitmq_test.py +++ b/tests/unit/modules/rabbitmq_test.py @@ -52,6 +52,17 @@ class RabbitmqTestCase(TestCase): with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}): self.assertDictEqual(rabbitmq.list_users(), {'guest': ['administrator', 'user']}) + # 'list_users_rabbitmq4' function tests: 1 + + def test_list_users_rabbitmq4(self): + ''' + Test if it return a list of users based off of rabbitmqctl user_list. + Output changed in rabbitmq-server version 3.6.10 + ''' + mock_run = MagicMock(return_value='Listing users\nguest\t[administrator user]\n') + with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}): + self.assertDictEqual(rabbitmq.list_users(), {'guest': ['administrator', 'user']}) + # 'list_users_with_warning_rabbitmq2' function tests: 1 def test_list_users_with_warning_rabbitmq2(self): @@ -220,6 +231,19 @@ class RabbitmqTestCase(TestCase): self.assertDictEqual(rabbitmq.list_user_permissions('myuser'), {'saltstack': ['saltstack']}) + # 'list_user_permissions2' function tests: 1 + + def test_list_user_permissions2(self): + ''' + Test if it list permissions for a user + via rabbitmqctl list_user_permissions. + Output changed in rabbitmq-server version 3.6.10 + ''' + mock_run = MagicMock(return_value='Listing stuff\nsaltstack\tsaltstack\n...done') + with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}): + self.assertDictEqual(rabbitmq.list_user_permissions('myuser'), + {'saltstack': ['saltstack']}) + # 'set_user_tags' function tests: 1 def test_set_user_tags(self): From 1cc2aa503a8e451e3c7f2be382593bd9298c2372 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Wed, 28 Jun 2017 15:59:17 +0100 Subject: [PATCH 003/508] Fix dockerng.network_* ignoring of tests=True Fixes #41976 --- salt/states/dockerng.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 3826cce670..78e6483b9f 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2123,6 +2123,11 @@ def network_present(name, driver=None, containers=None): ret['result'] = result else: + if __opts__['test']: + ret['result'] = None + ret['comment'] = ('The network \'{0}\' will be created'.format(name)) + return ret + try: ret['changes']['created'] = __salt__['dockerng.create_network']( name, driver=driver) @@ -2169,6 +2174,11 @@ def network_absent(name, driver=None): ret['comment'] = 'Network \'{0}\' already absent'.format(name) return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = ('The network \'{0}\' will be removed'.format(name)) + return ret + for container in networks[0]['Containers']: try: ret['changes']['disconnected'] = __salt__['dockerng.disconnect_container_from_network'](container, name) From 515c612808485f9a089ebb185d96441416e7922c Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Wed, 28 Jun 2017 17:28:22 +0100 Subject: [PATCH 004/508] Fix dockerng.network_* name matching Updating the states so that they loop through the list of networks returned from `dockerng.networks` to check that one of the networks returned fully matches the name of the network that needs to be ensured to be present or absent. Another option would be to add an argument to the `dockerng.networks` execution module to enable strict matching. This would probably be preferable to repeating the logic in the states, but I didn't want to meddle too much as the execution module is currently just wrapping the docker-py library. Fixes #41982 --- salt/states/dockerng.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 17559800f7..124e45f3e1 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2242,8 +2242,17 @@ def network_present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['dockerng.inspect_container'](c)['Id'] for c in containers] networks = __salt__['dockerng.networks'](names=[name]) + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None if networks: - network = networks[0] # we expect network's name to be unique + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is not None: if all(c in network['Containers'] for c in containers): ret['result'] = True ret['comment'] = 'Network \'{0}\' already exists.'.format(name) @@ -2302,7 +2311,17 @@ def network_absent(name, driver=None): 'comment': ''} networks = __salt__['dockerng.networks'](names=[name]) - if not networks: + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None + if networks: + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is None: ret['result'] = True ret['comment'] = 'Network \'{0}\' already absent'.format(name) return ret From a5f7288ad9ab60d13cab8a4ff33655992f5724df Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 28 Jun 2017 15:35:35 -0600 Subject: [PATCH 005/508] Skip test that uses pwd, not available on Windows --- tests/unit/modules/test_cmdmod.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/modules/test_cmdmod.py b/tests/unit/modules/test_cmdmod.py index ac57e059d7..2d5ae79472 100644 --- a/tests/unit/modules/test_cmdmod.py +++ b/tests/unit/modules/test_cmdmod.py @@ -263,6 +263,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.utils.fopen', mock_open(read_data=MOCK_SHELL_FILE)): self.assertFalse(cmdmod._is_valid_shell('foo')) + @skipIf(salt.utils.is_windows(), 'Do not run on Windows') def test_os_environment_remains_intact(self): ''' Make sure the OS environment is not tainted after running a command From 8c00c63b55006887861207b6e4716db0f69f8ed4 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Thu, 29 Jun 2017 13:36:29 +0100 Subject: [PATCH 006/508] Fix dockerng.network_* name matching Updating the states so that they loop through the list of networks returned from `dockerng.networks` to check that one of the networks returned fully matches the name of the network that needs to be ensured to be present or absent. Another option would be to add an argument to the `dockerng.networks` execution module to enable strict matching. This would probably be preferable to repeating the logic in the states, but I didn't want to meddle too much as the execution module is currently just wrapping the docker-py library. Fixes #41982 --- salt/states/docker_network.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/salt/states/docker_network.py b/salt/states/docker_network.py index 9d78d66b21..a10162ba9d 100644 --- a/salt/states/docker_network.py +++ b/salt/states/docker_network.py @@ -90,8 +90,17 @@ def present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['docker.inspect_container'](c)['Id'] for c in containers] networks = __salt__['docker.networks'](names=[name]) + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None if networks: - network = networks[0] # we expect network's name to be unique + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is not None: if all(c in network['Containers'] for c in containers): ret['result'] = True ret['comment'] = 'Network \'{0}\' already exists.'.format(name) @@ -150,7 +159,17 @@ def absent(name, driver=None): 'comment': ''} networks = __salt__['docker.networks'](names=[name]) - if not networks: + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None + if networks: + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is None: ret['result'] = True ret['comment'] = 'Network \'{0}\' already absent'.format(name) return ret From 494765e93950b2c8d41f63f8461a36943481ef44 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 26 Jun 2017 12:26:50 -0700 Subject: [PATCH 007/508] Updating the git module to allow an identity file to be used when passing the user parameter --- salt/modules/git.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index d164cd6cef..7cc2ddcc6c 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -170,13 +170,24 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, identity = [identity] # try each of the identities, independently + tmp_identity_file = None for id_file in identity: if 'salt://' in id_file: + tmp_identity_file = salt.utils.files.mkstemp() _id_file = id_file - id_file = __salt__['cp.cache_file'](id_file, saltenv) + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) if not id_file: log.error('identity {0} does not exist.'.format(_id_file)) + __salt__['file.remove'](tmp_identity_file) continue + else: + __salt__['file.set_mode'](id_file, '0600') + if user: + os.chown(id_file, + __salt__['file.user_to_uid'](user), + -1) else: if not __salt__['file.file_exists'](id_file): missing_keys.append(id_file) @@ -247,6 +258,11 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, if not salt.utils.is_windows() and 'GIT_SSH' in env: os.remove(env['GIT_SSH']) + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) + # If the command was successful, no need to try additional IDs if result['retcode'] == 0: return result From 8faa9f6d92bf7957c94d3e87d5604e804330291d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 26 Jun 2017 14:49:54 -0700 Subject: [PATCH 008/508] Updating PR with requested changes. --- salt/modules/git.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 7cc2ddcc6c..c50e61afa9 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,11 +169,15 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] + # Override umask + if not salt.utils.is_windows(): + _old_umask = os.umask(0o077) + # try each of the identities, independently tmp_identity_file = None for id_file in identity: if 'salt://' in id_file: - tmp_identity_file = salt.utils.files.mkstemp() + tmp_identity_file = salt.utils.mkstemp() _id_file = id_file id_file = __salt__['cp.get_file'](id_file, tmp_identity_file, @@ -183,7 +187,6 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, __salt__['file.remove'](tmp_identity_file) continue else: - __salt__['file.set_mode'](id_file, '0600') if user: os.chown(id_file, __salt__['file.user_to_uid'](user), @@ -255,13 +258,17 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, redirect_stderr=redirect_stderr, **kwargs) finally: - if not salt.utils.is_windows() and 'GIT_SSH' in env: - os.remove(env['GIT_SSH']) + if not salt.utils.is_windows(): + # Reset the umask + os.umask(_old_umask) - # Cleanup the temporary identify file - if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) + if 'GIT_SSH' in env: + os.remove(env['GIT_SSH']) + + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From 4949bf3ff322aa27028aa0af2b16cb0748ae0117 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 29 Jun 2017 10:08:58 -0700 Subject: [PATCH 009/508] Updating to swap on the new salt.utils.files.set_umask context_manager --- salt/modules/git.py | 189 +++++++++++++++++++++----------------------- 1 file changed, 91 insertions(+), 98 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index c50e61afa9..161de3d1d5 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,114 +169,107 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] - # Override umask - if not salt.utils.is_windows(): - _old_umask = os.umask(0o077) - - # try each of the identities, independently - tmp_identity_file = None - for id_file in identity: - if 'salt://' in id_file: - tmp_identity_file = salt.utils.mkstemp() - _id_file = id_file - id_file = __salt__['cp.get_file'](id_file, - tmp_identity_file, - saltenv) - if not id_file: - log.error('identity {0} does not exist.'.format(_id_file)) - __salt__['file.remove'](tmp_identity_file) - continue + with salt.utils.files.set_umask(0o177): + # try each of the identities, independently + tmp_identity_file = None + for id_file in identity: + if 'salt://' in id_file: + tmp_identity_file = salt.utils.mkstemp() + _id_file = id_file + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) + if not id_file: + log.error('identity {0} does not exist.'.format(_id_file)) + __salt__['file.remove'](tmp_identity_file) + continue + else: + if user: + os.chown(id_file, + __salt__['file.user_to_uid'](user), + -1) else: - if user: - os.chown(id_file, - __salt__['file.user_to_uid'](user), - -1) - else: - if not __salt__['file.file_exists'](id_file): - missing_keys.append(id_file) - log.error('identity {0} does not exist.'.format(id_file)) - continue + if not __salt__['file.file_exists'](id_file): + missing_keys.append(id_file) + log.error('identity {0} does not exist.'.format(id_file)) + continue - env = { - 'GIT_IDENTITY': id_file - } + env = { + 'GIT_IDENTITY': id_file + } - # copy wrapper to area accessible by ``runas`` user - # currently no suppport in windows for wrapping git ssh - ssh_id_wrapper = os.path.join( - salt.utils.templates.TEMPLATE_DIRNAME, - 'git/ssh-id-wrapper' - ) - if salt.utils.is_windows(): - for suffix in ('', ' (x86)'): - ssh_exe = ( - 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' - .format(suffix) - ) - if os.path.isfile(ssh_exe): - env['GIT_SSH_EXE'] = ssh_exe - break - else: - raise CommandExecutionError( - 'Failed to find ssh.exe, unable to use identity file' - ) - # Use the windows batch file instead of the bourne shell script - ssh_id_wrapper += '.bat' - env['GIT_SSH'] = ssh_id_wrapper - else: - tmp_file = salt.utils.mkstemp() - salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) - os.chmod(tmp_file, 0o500) - os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) - env['GIT_SSH'] = tmp_file - - if 'salt-call' not in _salt_cli \ - and __salt__['ssh.key_is_encrypted'](id_file): - errors.append( - 'Identity file {0} is passphrase-protected and cannot be ' - 'used in a non-interactive command. Using salt-call from ' - 'the minion will allow a passphrase-protected key to be ' - 'used.'.format(id_file) + # copy wrapper to area accessible by ``runas`` user + # currently no suppport in windows for wrapping git ssh + ssh_id_wrapper = os.path.join( + salt.utils.templates.TEMPLATE_DIRNAME, + 'git/ssh-id-wrapper' ) - continue + if salt.utils.is_windows(): + for suffix in ('', ' (x86)'): + ssh_exe = ( + 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' + .format(suffix) + ) + if os.path.isfile(ssh_exe): + env['GIT_SSH_EXE'] = ssh_exe + break + else: + raise CommandExecutionError( + 'Failed to find ssh.exe, unable to use identity file' + ) + # Use the windows batch file instead of the bourne shell script + ssh_id_wrapper += '.bat' + env['GIT_SSH'] = ssh_id_wrapper + else: + tmp_file = salt.utils.mkstemp() + salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) + os.chmod(tmp_file, 0o500) + os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) + env['GIT_SSH'] = tmp_file - log.info( - 'Attempting git authentication using identity file {0}' - .format(id_file) - ) + if 'salt-call' not in _salt_cli \ + and __salt__['ssh.key_is_encrypted'](id_file): + errors.append( + 'Identity file {0} is passphrase-protected and cannot be ' + 'used in a non-interactive command. Using salt-call from ' + 'the minion will allow a passphrase-protected key to be ' + 'used.'.format(id_file) + ) + continue - try: - result = __salt__['cmd.run_all']( - command, - cwd=cwd, - runas=user, - password=password, - env=env, - python_shell=False, - log_callback=salt.utils.url.redact_http_basic_auth, - ignore_retcode=ignore_retcode, - redirect_stderr=redirect_stderr, - **kwargs) - finally: - if not salt.utils.is_windows(): - # Reset the umask - os.umask(_old_umask) + log.info( + 'Attempting git authentication using identity file {0}' + .format(id_file) + ) - if 'GIT_SSH' in env: + try: + result = __salt__['cmd.run_all']( + command, + cwd=cwd, + runas=user, + password=password, + env=env, + python_shell=False, + log_callback=salt.utils.url.redact_http_basic_auth, + ignore_retcode=ignore_retcode, + redirect_stderr=redirect_stderr, + **kwargs) + finally: + if not salt.utils.is_windows() and 'GIT_SSH' in env: os.remove(env['GIT_SSH']) - # Cleanup the temporary identify file - if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) - # If the command was successful, no need to try additional IDs - if result['retcode'] == 0: - return result - else: - err = result['stdout' if redirect_stderr else 'stderr'] - if err: - errors.append(salt.utils.url.redact_http_basic_auth(err)) + # If the command was successful, no need to try additional IDs + if result['retcode'] == 0: + return result + else: + err = result['stdout' if redirect_stderr else 'stderr'] + if err: + errors.append(salt.utils.url.redact_http_basic_auth(err)) # We've tried all IDs and still haven't passed, so error out if failhard: From 68549f3496d5784a762b3aef4b0923ae059ec272 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 29 Jun 2017 10:15:50 -0700 Subject: [PATCH 010/508] Fixing umask to we can set files as executable. --- salt/modules/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 161de3d1d5..53bea307d1 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,7 +169,7 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] - with salt.utils.files.set_umask(0o177): + with salt.utils.files.set_umask(0o077): # try each of the identities, independently tmp_identity_file = None for id_file in identity: From 26beb18aa505bcc196fbd303c895c95d8f8a4680 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 29 Jun 2017 11:35:28 -0600 Subject: [PATCH 011/508] Set concurrent to True when running states with sudo Fixes #25842 --- salt/executors/sudo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/executors/sudo.py b/salt/executors/sudo.py index d3321847a5..825fee979a 100644 --- a/salt/executors/sudo.py +++ b/salt/executors/sudo.py @@ -73,7 +73,7 @@ class SudoExecutor(ModuleExecutorBase): '-c', salt.syspaths.CONFIG_DIR, '--', data.get('fun')] - if data['fun'] == 'state.sls': + if data['fun'] in ('state.sls', 'state.highstate', 'state.apply'): kwargs['concurrent'] = True for arg in args: self.cmd.append(_cmd_quote(str(arg))) From 1b6026177c09e2bf852c4c8cda1dc3da1653c7f2 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 29 Jun 2017 13:04:33 -0500 Subject: [PATCH 012/508] Restrict set_umask to mkstemp call only --- salt/modules/git.py | 190 ++++++++++++++++++++++---------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 53bea307d1..7b0f37f3f4 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,107 +169,107 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] - with salt.utils.files.set_umask(0o077): - # try each of the identities, independently - tmp_identity_file = None - for id_file in identity: - if 'salt://' in id_file: + # try each of the identities, independently + tmp_identity_file = None + for id_file in identity: + if 'salt://' in id_file: + with salt.utils.files.set_umask(0o077): tmp_identity_file = salt.utils.mkstemp() - _id_file = id_file - id_file = __salt__['cp.get_file'](id_file, - tmp_identity_file, - saltenv) - if not id_file: - log.error('identity {0} does not exist.'.format(_id_file)) - __salt__['file.remove'](tmp_identity_file) - continue - else: - if user: - os.chown(id_file, - __salt__['file.user_to_uid'](user), - -1) + _id_file = id_file + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) + if not id_file: + log.error('identity {0} does not exist.'.format(_id_file)) + __salt__['file.remove'](tmp_identity_file) + continue else: - if not __salt__['file.file_exists'](id_file): - missing_keys.append(id_file) - log.error('identity {0} does not exist.'.format(id_file)) - continue - - env = { - 'GIT_IDENTITY': id_file - } - - # copy wrapper to area accessible by ``runas`` user - # currently no suppport in windows for wrapping git ssh - ssh_id_wrapper = os.path.join( - salt.utils.templates.TEMPLATE_DIRNAME, - 'git/ssh-id-wrapper' - ) - if salt.utils.is_windows(): - for suffix in ('', ' (x86)'): - ssh_exe = ( - 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' - .format(suffix) - ) - if os.path.isfile(ssh_exe): - env['GIT_SSH_EXE'] = ssh_exe - break - else: - raise CommandExecutionError( - 'Failed to find ssh.exe, unable to use identity file' - ) - # Use the windows batch file instead of the bourne shell script - ssh_id_wrapper += '.bat' - env['GIT_SSH'] = ssh_id_wrapper - else: - tmp_file = salt.utils.mkstemp() - salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) - os.chmod(tmp_file, 0o500) - os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) - env['GIT_SSH'] = tmp_file - - if 'salt-call' not in _salt_cli \ - and __salt__['ssh.key_is_encrypted'](id_file): - errors.append( - 'Identity file {0} is passphrase-protected and cannot be ' - 'used in a non-interactive command. Using salt-call from ' - 'the minion will allow a passphrase-protected key to be ' - 'used.'.format(id_file) - ) + if user: + os.chown(id_file, + __salt__['file.user_to_uid'](user), + -1) + else: + if not __salt__['file.file_exists'](id_file): + missing_keys.append(id_file) + log.error('identity {0} does not exist.'.format(id_file)) continue - log.info( - 'Attempting git authentication using identity file {0}' - .format(id_file) - ) + env = { + 'GIT_IDENTITY': id_file + } - try: - result = __salt__['cmd.run_all']( - command, - cwd=cwd, - runas=user, - password=password, - env=env, - python_shell=False, - log_callback=salt.utils.url.redact_http_basic_auth, - ignore_retcode=ignore_retcode, - redirect_stderr=redirect_stderr, - **kwargs) - finally: - if not salt.utils.is_windows() and 'GIT_SSH' in env: - os.remove(env['GIT_SSH']) - - # Cleanup the temporary identify file - if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) - - # If the command was successful, no need to try additional IDs - if result['retcode'] == 0: - return result + # copy wrapper to area accessible by ``runas`` user + # currently no suppport in windows for wrapping git ssh + ssh_id_wrapper = os.path.join( + salt.utils.templates.TEMPLATE_DIRNAME, + 'git/ssh-id-wrapper' + ) + if salt.utils.is_windows(): + for suffix in ('', ' (x86)'): + ssh_exe = ( + 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' + .format(suffix) + ) + if os.path.isfile(ssh_exe): + env['GIT_SSH_EXE'] = ssh_exe + break else: - err = result['stdout' if redirect_stderr else 'stderr'] - if err: - errors.append(salt.utils.url.redact_http_basic_auth(err)) + raise CommandExecutionError( + 'Failed to find ssh.exe, unable to use identity file' + ) + # Use the windows batch file instead of the bourne shell script + ssh_id_wrapper += '.bat' + env['GIT_SSH'] = ssh_id_wrapper + else: + tmp_file = salt.utils.mkstemp() + salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) + os.chmod(tmp_file, 0o500) + os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) + env['GIT_SSH'] = tmp_file + + if 'salt-call' not in _salt_cli \ + and __salt__['ssh.key_is_encrypted'](id_file): + errors.append( + 'Identity file {0} is passphrase-protected and cannot be ' + 'used in a non-interactive command. Using salt-call from ' + 'the minion will allow a passphrase-protected key to be ' + 'used.'.format(id_file) + ) + continue + + log.info( + 'Attempting git authentication using identity file {0}' + .format(id_file) + ) + + try: + result = __salt__['cmd.run_all']( + command, + cwd=cwd, + runas=user, + password=password, + env=env, + python_shell=False, + log_callback=salt.utils.url.redact_http_basic_auth, + ignore_retcode=ignore_retcode, + redirect_stderr=redirect_stderr, + **kwargs) + finally: + if not salt.utils.is_windows() and 'GIT_SSH' in env: + os.remove(env['GIT_SSH']) + + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) + + # If the command was successful, no need to try additional IDs + if result['retcode'] == 0: + return result + else: + err = result['stdout' if redirect_stderr else 'stderr'] + if err: + errors.append(salt.utils.url.redact_http_basic_auth(err)) # We've tried all IDs and still haven't passed, so error out if failhard: From 44841e5626defb0e330372d344554dab2b2c432d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 29 Jun 2017 11:22:34 -0700 Subject: [PATCH 013/508] Moving the call to cp.get_file inside the with block to ensure the umask is preserved when we grab the file. --- salt/modules/git.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 7b0f37f3f4..4f1101b755 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -175,10 +175,10 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, if 'salt://' in id_file: with salt.utils.files.set_umask(0o077): tmp_identity_file = salt.utils.mkstemp() - _id_file = id_file - id_file = __salt__['cp.get_file'](id_file, - tmp_identity_file, - saltenv) + _id_file = id_file + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) if not id_file: log.error('identity {0} does not exist.'.format(_id_file)) __salt__['file.remove'](tmp_identity_file) @@ -261,7 +261,7 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # Cleanup the temporary identify file if tmp_identity_file and os.path.exists(tmp_identity_file): log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) + #__salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From 26f848e1115fdcdd471ad863ffc604678dd6c304 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 29 Jun 2017 14:49:39 -0500 Subject: [PATCH 014/508] Mock socket.getaddrinfo in unit.utils.network_test.NetworkTestCase.test_host_to_ips This will keep future IP address changes from impacting this test. It also tests a host which returns an IPv6 address, and a host which cannot be resolved. --- tests/unit/utils/network_test.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils/network_test.py b/tests/unit/utils/network_test.py index a02675c92f..7a3d278f66 100644 --- a/tests/unit/utils/network_test.py +++ b/tests/unit/utils/network_test.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Import Python libs from __future__ import absolute_import +import socket # Import Salt Testing libs from salttesting import skipIf @@ -111,8 +112,31 @@ class NetworkTestCase(TestCase): changed. In these cases, we just need to update the IP address in the assertion. ''' - ret = network.host_to_ips('www.saltstack.com') - self.assertEqual(ret, ['104.197.168.128']) + def _side_effect(host, *args): + try: + return { + 'github.com': [ + (2, 1, 6, '', ('192.30.255.112', 0)), + (2, 1, 6, '', ('192.30.255.113', 0)), + ], + 'ipv6host.foo': [ + (10, 1, 6, '', ('2001:a71::1', 0, 0, 0)), + ], + }[host] + except KeyError: + raise socket.gaierror(-2, 'Name or service not known') + + getaddrinfo_mock = MagicMock(side_effect=_side_effect) + with patch.object(socket, 'getaddrinfo', getaddrinfo_mock): + # Test host that can be resolved + ret = network.host_to_ips('github.com') + self.assertEqual(ret, ['192.30.255.112', '192.30.255.113']) + # Test ipv6 + ret = network.host_to_ips('ipv6host.foo') + self.assertEqual(ret, ['2001:a71::1']) + # Test host that can't be resolved + ret = network.host_to_ips('someothersite.com') + self.assertEqual(ret, None) def test_generate_minion_id(self): self.assertTrue(network.generate_minion_id()) From 0ef6cf634c43d2d8265fe76bda3cc434532a519d Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 30 Jun 2017 10:57:47 +0100 Subject: [PATCH 015/508] Add trace logging of dockerng.networks result Added during debugging of failing test to determine what dockerng.networks is expected to return, keeping as it seems useful. --- salt/states/dockerng.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 124e45f3e1..2facade3ab 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2242,6 +2242,9 @@ def network_present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['dockerng.inspect_container'](c)['Id'] for c in containers] networks = __salt__['dockerng.networks'](names=[name]) + log.trace( + 'dockerng.network_present: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. @@ -2311,6 +2314,9 @@ def network_absent(name, driver=None): 'comment': ''} networks = __salt__['dockerng.networks'](names=[name]) + log.trace( + 'dockerng.network_present: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. From 3369f0072ff9afb22b083a66b42cbcafd6051895 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 30 Jun 2017 11:01:10 +0100 Subject: [PATCH 016/508] Fix broken unit test test_network_absent This started failing following commit 515c612, which relied on the 'Name' key being present in the return value of dockerng.networks - as the mock didn't have this set the test started failing. --- tests/unit/states/dockerng_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/dockerng_test.py b/tests/unit/states/dockerng_test.py index 884776bfb8..c8d5dddef0 100644 --- a/tests/unit/states/dockerng_test.py +++ b/tests/unit/states/dockerng_test.py @@ -724,9 +724,13 @@ class DockerngTestCase(TestCase): ''' dockerng_remove_network = Mock(return_value='removed') dockerng_disconnect_container_from_network = Mock(return_value='disconnected') + dockerng_networks = Mock(return_value=[{ + 'Name': 'network_foo', + 'Containers': {'container': {}} + }]) __salt__ = {'dockerng.remove_network': dockerng_remove_network, 'dockerng.disconnect_container_from_network': dockerng_disconnect_container_from_network, - 'dockerng.networks': Mock(return_value=[{'Containers': {'container': {}}}]), + 'dockerng.networks': dockerng_networks, } with patch.dict(dockerng_state.__dict__, {'__salt__': __salt__}): From 9eea796da8bdab8e25b141c7d3aa7041d422e387 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 30 Jun 2017 11:03:45 +0100 Subject: [PATCH 017/508] Add regression tests for #41982 These test the scenarios where another network with a similar name already exists, verifying that network_absent doesn't attempt to remove a network which isn't specified, and network_present still attempts to create the specified network despite a similarly named network already being present. --- tests/unit/states/dockerng_test.py | 40 +++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/dockerng_test.py b/tests/unit/states/dockerng_test.py index c8d5dddef0..d8e409cefa 100644 --- a/tests/unit/states/dockerng_test.py +++ b/tests/unit/states/dockerng_test.py @@ -698,10 +698,18 @@ class DockerngTestCase(TestCase): dockerng_create_network = Mock(return_value='created') dockerng_connect_container_to_network = Mock(return_value='connected') dockerng_inspect_container = Mock(return_value={'Id': 'abcd'}) + # Get dockerng.networks to return a network with a name which is a superset of the name of + # the network which is to be created, despite this network existing we should still expect + # that the new network will be created. + # Regression test for #41982. + dockerng_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) __salt__ = {'dockerng.create_network': dockerng_create_network, 'dockerng.inspect_container': dockerng_inspect_container, 'dockerng.connect_container_to_network': dockerng_connect_container_to_network, - 'dockerng.networks': Mock(return_value=[]), + 'dockerng.networks': dockerng_networks, } with patch.dict(dockerng_state.__dict__, {'__salt__': __salt__}): @@ -746,6 +754,36 @@ class DockerngTestCase(TestCase): 'removed': 'removed'}, 'result': True}) + def test_network_absent_with_matching_network(self): + ''' + Test dockerng.network_absent when the specified network does not exist, + but another network with a name which is a superset of the specified + name does exist. In this case we expect there to be no attempt to remove + any network. + Regression test for #41982. + ''' + dockerng_remove_network = Mock(return_value='removed') + dockerng_disconnect_container_from_network = Mock(return_value='disconnected') + dockerng_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) + __salt__ = {'dockerng.remove_network': dockerng_remove_network, + 'dockerng.disconnect_container_from_network': dockerng_disconnect_container_from_network, + 'dockerng.networks': dockerng_networks, + } + with patch.dict(dockerng_state.__dict__, + {'__salt__': __salt__}): + ret = dockerng_state.network_absent( + 'network_foo', + ) + dockerng_disconnect_container_from_network.assert_not_called() + dockerng_remove_network.assert_not_called() + self.assertEqual(ret, {'name': 'network_foo', + 'comment': 'Network \'network_foo\' already absent', + 'changes': {}, + 'result': True}) + def test_volume_present(self): ''' Test dockerng.volume_present From 1de5e008a06f2c73b73f5c7fde7200920dba7348 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 30 Jun 2017 11:32:10 -0400 Subject: [PATCH 018/508] Add initial 2016.11.7 Release Notes --- doc/topics/releases/2016.11.7.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/topics/releases/2016.11.7.rst diff --git a/doc/topics/releases/2016.11.7.rst b/doc/topics/releases/2016.11.7.rst new file mode 100644 index 0000000000..6ef9587133 --- /dev/null +++ b/doc/topics/releases/2016.11.7.rst @@ -0,0 +1,5 @@ +============================ +Salt 2016.11.7 Release Notes +============================ + +Version 2016.11.7 is a bugfix release for :ref:`2016.11.0 `. From e20cea635079ee1b66d28996b3502dca40ba1f88 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Fri, 30 Jun 2017 11:21:49 -0600 Subject: [PATCH 019/508] Upgrade support for gnupg v2.1 and higher --- salt/modules/debbuild.py | 115 ++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/salt/modules/debbuild.py b/salt/modules/debbuild.py index 2cb9deed2a..2124cbf259 100644 --- a/salt/modules/debbuild.py +++ b/salt/modules/debbuild.py @@ -60,17 +60,22 @@ def __virtual__(): return (False, 'The debbuild module could not be loaded: unsupported OS family') -def _check_repo_sign_utils_support(): - util_name = 'debsign' - if salt.utils.which(util_name): +def _check_repo_sign_utils_support(name): + ''' + Check for specified command name in search path + ''' + if salt.utils.which(name): return True else: raise CommandExecutionError( - 'utility \'{0}\' needs to be installed'.format(util_name) + 'utility \'{0}\' needs to be installed or made available in search path'.format(name) ) def _check_repo_gpg_phrase_utils_support(): + ''' + Check for /usr/lib/gnupg2/gpg-preset-passphrase is installed + ''' util_name = '/usr/lib/gnupg2/gpg-preset-passphrase' if __salt__['file.file_exists'](util_name): return True @@ -170,8 +175,8 @@ def _get_repo_dists_env(env): 'ORIGIN': ('O', 'Origin', 'SaltStack'), 'LABEL': ('O', 'Label', 'salt_debian'), 'SUITE': ('O', 'Suite', 'stable'), - 'VERSION': ('O', 'Version', '8.1'), - 'CODENAME': ('M', 'Codename', 'jessie'), + 'VERSION': ('O', 'Version', '9.0'), + 'CODENAME': ('M', 'Codename', 'stretch'), 'ARCHS': ('M', 'Architectures', 'i386 amd64 source'), 'COMPONENTS': ('M', 'Components', 'main'), 'DESCRIPTION': ('O', 'Description', 'SaltStack debian package repo'), @@ -205,7 +210,7 @@ def _get_repo_dists_env(env): else: env_dists += '{0}: {1}\n'.format(key, value) - ## ensure mandatories are included + # ensure mandatories are included env_keys = list(env.keys()) for key in env_keys: if key in dflts_keys and dflts_dict[key][0] == 'M' and key not in env_man_seen: @@ -312,7 +317,7 @@ def make_src_pkg(dest_dir, spec, sources, env=None, template=None, saltenv='base for src in sources: _get_src(tree_base, src, saltenv) - #.dsc then assumes sources already build + # .dsc then assumes sources already build if spec_pathfile.endswith('.dsc'): for efile in os.listdir(tree_base): full = os.path.join(tree_base, efile) @@ -578,7 +583,8 @@ def make_repo(repodir, with salt.utils.fopen(repoconfopts, 'w') as fow: fow.write('{0}'.format(repocfg_opts)) - local_fingerprint = None + local_keygrip_to_use = None + local_key_fingerprint = None local_keyid = None phrase = '' @@ -587,17 +593,14 @@ def make_repo(repodir, gpg_tty_info_file = '{0}/gpg-tty-info-salt'.format(gnupghome) gpg_tty_info_dict = {} - # test if using older than gnupg 2.1, env file exists + # if using older than gnupg 2.1, then env file exists older_gnupg = __salt__['file.file_exists'](gpg_info_file) - # interval of 0.125 is really too fast on some systems - interval = 0.5 - if keyid is not None: with salt.utils.fopen(repoconfdist, 'a') as fow: fow.write('SignWith: {0}\n'.format(keyid)) - ## import_keys + # import_keys pkg_pub_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_pub_keyname', None)) pkg_priv_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_priv_keyname', None)) @@ -621,21 +624,37 @@ def make_repo(repodir, local_keys = __salt__['gpg.list_keys'](user=runas, gnupghome=gnupghome) for gpg_key in local_keys: if keyid == gpg_key['keyid'][8:]: - local_fingerprint = gpg_key['fingerprint'] + local_keygrip_to_use = gpg_key['fingerprint'] + local_key_fingerprint = gpg_key['fingerprint'] local_keyid = gpg_key['keyid'] break + if not older_gnupg: + _check_repo_sign_utils_support('gpg2') + cmd = '{0} --with-keygrip --list-secret-keys'.format(salt.utils.which('gpg2')) + local_keys2_keygrip = __salt__['cmd.run'](cmd, runas=runas) + local_keys2 = iter(local_keys2_keygrip.splitlines()) + try: + for line in local_keys2: + if line.startswith('sec'): + line_fingerprint = next(local_keys2).lstrip().rstrip() + if local_key_fingerprint == line_fingerprint: + lkeygrip = next(local_keys2).split('=') + local_keygrip_to_use = lkeygrip[1].lstrip().rstrip() + break + except StopIteration: + raise SaltInvocationError( + 'unable to find keygrip associated with fingerprint \'{0}\' for keyid \'{1}\'' + .format(local_key_fingerprint, local_keyid) + ) + if local_keyid is None: raise SaltInvocationError( 'The key ID \'{0}\' was not found in GnuPG keyring at \'{1}\'' .format(keyid, gnupghome) ) - _check_repo_sign_utils_support() - - if use_passphrase: - _check_repo_gpg_phrase_utils_support() - phrase = __salt__['pillar.get']('gpg_passphrase') + _check_repo_sign_utils_support('debsign') if older_gnupg: with salt.utils.fopen(gpg_info_file, 'r') as fow: @@ -656,10 +675,30 @@ def make_repo(repodir, __salt__['environ.setenv'](gpg_tty_info_dict) break - ## sign_it_here - for file in os.listdir(repodir): - if file.endswith('.dsc'): - abs_file = os.path.join(repodir, file) + if use_passphrase: + _check_repo_gpg_phrase_utils_support() + phrase = __salt__['pillar.get']('gpg_passphrase') + cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --preset --passphrase "{0}" {1}'.format(phrase, local_keygrip_to_use) + __salt__['cmd.run'](cmd, runas=runas) + + for debfile in os.listdir(repodir): + abs_file = os.path.join(repodir, debfile) + if debfile.endswith('.changes'): + os.remove(abs_file) + + if debfile.endswith('.dsc'): + # sign_it_here + if older_gnupg: + if local_keyid is not None: + cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file) + __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) + + cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(codename, abs_file) + __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) + else: + # interval of 0.125 is really too fast on some systems + interval = 0.5 + if local_keyid is not None: number_retries = timeout / interval times_looped = 0 error_msg = 'Failed to debsign file {0}'.format(abs_file) @@ -702,27 +741,6 @@ def make_repo(repodir, finally: proc.close(terminate=True, kill=True) - if use_passphrase: - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --forget {0}'.format(local_fingerprint) - __salt__['cmd.run'](cmd, runas=runas) - - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --preset --passphrase "{0}" {1}'.format(phrase, local_fingerprint) - __salt__['cmd.run'](cmd, runas=runas) - - for debfile in os.listdir(repodir): - abs_file = os.path.join(repodir, debfile) - if debfile.endswith('.changes'): - os.remove(abs_file) - - if debfile.endswith('.dsc'): - if older_gnupg: - if local_keyid is not None: - cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file) - __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) - - cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(codename, abs_file) - __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) - else: number_retries = timeout / interval times_looped = 0 error_msg = 'Failed to reprepro includedsc file {0}'.format(abs_file) @@ -747,8 +765,7 @@ def make_repo(repodir, if times_looped > number_retries: raise SaltInvocationError( - 'Attemping to reprepro includedsc for file {0} failed, timed out after {1} loops' - .format(abs_file, int(times_looped * interval)) + 'Attemping to reprepro includedsc for file {0} failed, timed out after {1} loops'.format(abs_file, times_looped) ) time.sleep(interval) @@ -770,8 +787,4 @@ def make_repo(repodir, cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedeb {0} {1}'.format(codename, abs_file) res = __salt__['cmd.run_all'](cmd, cwd=repodir, use_vt=True) - if use_passphrase and local_keyid is not None: - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --forget {0}'.format(local_fingerprint) - res = __salt__['cmd.run_all'](cmd, runas=runas) - return res From fd4458b6c7c1878946d4572003311bb29535da0c Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 29 Jun 2017 10:19:08 -0600 Subject: [PATCH 020/508] import salt.minion for EventReturn for Windows Because of the way our multiprocessing works on Windows, this module is not available. In Linux, the namespace is copied over, and the module is available, so we only need the import on Windows. --- salt/utils/event.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/utils/event.py b/salt/utils/event.py index 0b64ee1833..3e0ad6a95f 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -59,6 +59,7 @@ import fnmatch import hashlib import logging import datetime +import sys from collections import MutableMapping from multiprocessing.util import Finalize from salt.ext.six.moves import range @@ -1159,6 +1160,16 @@ class EventReturn(salt.utils.process.SignalHandlingMultiprocessingProcess): A dedicated process which listens to the master event bus and queues and forwards events to the specified returner. ''' + def __new__(cls, *args, **kwargs): + if sys.platform.startswith('win'): + # This is required for Windows. On Linux, when a process is + # forked, the module namespace is copied and the current process + # gets all of sys.modules from where the fork happens. This is not + # the case for Windows. + import salt.minion + instance = super(EventReturn, cls).__new__(cls, *args, **kwargs) + return instance + def __init__(self, opts, log_queue=None): ''' Initialize the EventReturn system From b1960cea442c5d00ec55c6f4187ab1010e54f801 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Sun, 2 Jul 2017 17:16:22 +0300 Subject: [PATCH 021/508] Fix scheduled job run on Master if `when` parameter is a list --- salt/utils/schedule.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index a69e5ca418..08fcf36edb 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1090,23 +1090,35 @@ class Schedule(object): log.error('Invalid date string {0}. ' 'Ignoring job {1}.'.format(i, job)) continue - when = int(time.mktime(when__.timetuple())) - if when >= now: - _when.append(when) + _when.append(int(time.mktime(when__.timetuple()))) + + # Sort the list of "whens" from earlier to later schedules _when.sort() + + for i in _when: + if i < now and len(_when) > 1: + # Remove all missed schedules except the latest one. + # We need it to detect if it was triggered previously. + _when.remove(i) + if _when: - # Grab the first element - # which is the next run time + # Grab the first element, which is the next run time or + # last scheduled time in the past. when = _when[0] # If we're switching to the next run in a list - # ensure the job can run - if '_when' in data and data['_when'] != when: + # ensure the job can run. + if '_when' in data and \ + (data['_when'] != when or len(_when) == 1): data['_when_run'] = True - data['_when'] = when - seconds = when - now + # Calculate time to previously scheduled job + # (which has not been run yet), or it should be + # the last time definition in the list. + seconds = data['_when'] - now + else: + seconds = when - now - # scheduled time is in the past and the run was not triggered before + # Scheduled time is in the past and the run was not triggered before if seconds < 0 and not data.get('_when_run', False): continue @@ -1121,6 +1133,9 @@ class Schedule(object): if when > data['_when']: data['_when'] = when data['_when_run'] = True + # At last scheduled time, we disable all subsequent job runs + elif len(_when) == 1: + del data['_when'] else: continue From cf55c3361c1c0db95e49fcc0085548c786b6524b Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Mon, 3 Jul 2017 19:27:00 +1000 Subject: [PATCH 022/508] pkg.install and pkg.remove on the command line take number version numbers, store them within a float. However version is a string, to support versions numbers like 1.3.4 The following for example would not work as version would be a float, and a float does not match a string a dict with key of strings. c:\salt\salt-call -l debug pkg.install name=softwarename version=3.1 --- salt/modules/win_pkg.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 8451a6722d..8f8b962444 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1020,10 +1020,15 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ret[pkg_name] = 'Unable to locate package {0}'.format(pkg_name) continue - # Get the version number passed or the latest available + # Get the version number passed or the latest available (must be a string) version_num = '' if options: - version_num = options.get('version', False) + version_num = options.get('version', '') + # The user user salt cmdline with version=5.3 might be interpreted + # as a float it must be converted to a string in order for + # string matching to work. + if not isinstance(version, six.string_types): + version_num = str(version_num) if not version_num: version_num = _get_latest_pkg_version(pkginfo) @@ -1355,6 +1360,11 @@ def remove(name=None, pkgs=None, version=None, **kwargs): continue if version_num is not None: + # The user user salt cmdline with version=5.3 might be interpreted + # as a float it must be converted to a string in order for + # string matching to work. + if not isinstance(version, six.string_types): + version_num = str(version_num) if version_num not in pkginfo and 'latest' in pkginfo: version_num = 'latest' elif 'latest' in pkginfo: From 8d549685a7a4e71819b3701654bfb1522c5384f1 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Mon, 3 Jul 2017 16:08:16 +0100 Subject: [PATCH 023/508] Make result=true if Docker volume already exists In other states the result is always returned as `True` if no change is required, whether `test=True` is set or not. Fixes #42076 --- salt/states/dockerng.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 17559800f7..2f10abd444 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2442,7 +2442,7 @@ def volume_present(name, driver=None, driver_opts=None, force=False): ret['result'] = result return ret - ret['result'] = None if __opts__['test'] else True + ret['result'] = True ret['comment'] = 'Volume \'{0}\' already exists.'.format(name) return ret From 2e1dc95500bc56262208402b0103b8fa01b0ab0d Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Mon, 3 Jul 2017 16:19:04 +0100 Subject: [PATCH 024/508] Make result=true if Docker volume already exists In other states the result is always returned as `True` if no change is required, whether `test=True` is set or not. Fixes #42076 --- salt/states/docker_volume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/docker_volume.py b/salt/states/docker_volume.py index 84a4f2154d..209b3839ac 100644 --- a/salt/states/docker_volume.py +++ b/salt/states/docker_volume.py @@ -187,7 +187,7 @@ def present(name, driver=None, driver_opts=None, force=False): ret['result'] = result return ret - ret['result'] = None if __opts__['test'] else True + ret['result'] = True ret['comment'] = 'Volume \'{0}\' already exists.'.format(name) return ret From 4fb2bb1856f527b27d5b1eedbc3969fc9a40decf Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Tue, 4 Jul 2017 03:08:08 +1000 Subject: [PATCH 025/508] Fix typo --- salt/modules/win_pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 8f8b962444..1c0be8a941 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1027,7 +1027,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version, six.string_types): + if not isinstance(version_num, six.string_types): version_num = str(version_num) if not version_num: @@ -1363,7 +1363,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version, six.string_types): + if not isinstance(version_num, six.string_types): version_num = str(version_num) if version_num not in pkginfo and 'latest' in pkginfo: version_num = 'latest' From 71675494251ef1d40a992b4068cb5fc45ec59035 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Tue, 4 Jul 2017 03:30:44 +1000 Subject: [PATCH 026/508] Handle version=None when converted to a string it becomes 'None' parm should default to empty string rather than None, it would fix better with existing code. --- salt/modules/win_pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 1c0be8a941..e13baed361 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1027,7 +1027,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version_num, six.string_types): + if not isinstance(version_num, six.string_types) and version_num is not None: version_num = str(version_num) if not version_num: @@ -1363,7 +1363,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version_num, six.string_types): + if not isinstance(version_num, six.string_types) and version_num is not None: version_num = str(version_num) if version_num not in pkginfo and 'latest' in pkginfo: version_num = 'latest' From 09d37dd8922a5b3df4c14c119ef3aa5c22796529 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Tue, 4 Jul 2017 03:48:04 +1000 Subject: [PATCH 027/508] Fix comment typo --- salt/modules/win_pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index e13baed361..b0d4052496 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1024,7 +1024,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): version_num = '' if options: version_num = options.get('version', '') - # The user user salt cmdline with version=5.3 might be interpreted + # Using the salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. if not isinstance(version_num, six.string_types) and version_num is not None: @@ -1360,7 +1360,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): continue if version_num is not None: - # The user user salt cmdline with version=5.3 might be interpreted + # Using the salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. if not isinstance(version_num, six.string_types) and version_num is not None: From 47d61f4edf800602d913933c5d6a9eb5fd0fe577 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 3 Jul 2017 13:16:57 -0500 Subject: [PATCH 028/508] Prevent command from showing in exception when output_loglevel=quiet When a command is being executed using salt.utils.timed_subprocess.TimedProc, and that command does not exist in the PATH, it will result in an OSError/IOError being raised. We catch this and then raise a CommandExecutionError with some information, including the command being run. However, when output_loglevel=quiet, we don't want any details of the command to be visible. This commit redacts the command details in these corner cases. --- salt/modules/cmdmod.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 82898edfeb..65b6767e33 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -526,10 +526,25 @@ def _run(cmd, try: proc = salt.utils.timed_subprocess.TimedProc(cmd, **kwargs) except (OSError, IOError) as exc: - raise CommandExecutionError( + msg = ( 'Unable to run command \'{0}\' with the context \'{1}\', ' - 'reason: {2}'.format(cmd, kwargs, exc) + 'reason: '.format( + cmd if _check_loglevel(output_loglevel) is not None + else 'REDACTED', + kwargs + ) ) + try: + if exc.filename is None: + msg += 'command not found' + else: + msg += '{0}: {1}'.format(exc, exc.filename) + except AttributeError: + # Both IOError and OSError have the filename attribute, so this + # is a precaution in case the exception classes in the previous + # try/except are changed. + msg += 'unknown' + raise CommandExecutionError(msg) try: proc.run() From bd27870a71300113a327d98e1abd2c2b1c424847 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 3 Jul 2017 13:30:01 -0500 Subject: [PATCH 029/508] Add debug logging to dockerng.login This can be used with debug logging enabled to troubleshoot which registries this function is attempting to authenticate. --- salt/modules/dockerng.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/modules/dockerng.py b/salt/modules/dockerng.py index 22aa4146ac..0cf393c4db 100644 --- a/salt/modules/dockerng.py +++ b/salt/modules/dockerng.py @@ -1899,6 +1899,10 @@ def login(*registries): cmd = ['docker', 'login', '-u', username, '-p', password] if registry.lower() != 'hub': cmd.append(registry) + log.debug( + 'Attempting to login to docker registry \'%s\' as user \'%s\'', + registry, username + ) login_cmd = __salt__['cmd.run_all']( cmd, python_shell=False, From c2822e05adde419c7ffedcab56b046af13a2833e Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 3 Jul 2017 14:47:36 -0600 Subject: [PATCH 030/508] Remove references in docs to pip install salt-cloud Instead, provide the option to use "-L" with the Bootstrap script. Fixes #41885 --- doc/topics/cloud/index.rst | 4 +++- doc/topics/cloud/qs.rst | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/topics/cloud/index.rst b/doc/topics/cloud/index.rst index e3f2959028..3dbb0c3aaa 100644 --- a/doc/topics/cloud/index.rst +++ b/doc/topics/cloud/index.rst @@ -64,7 +64,9 @@ automatically installed salt-cloud for you. Use your distribution's package manager to install the ``salt-cloud`` package from the same repo that you used to install Salt. These repos will automatically be setup by Salt Bootstrap. -If there is no salt-cloud package, install with ``pip install salt-cloud``. +Alternatively, the ``-L`` option can be passed to the `Salt Bootstrap`_ script when +installing Salt. The ``-L`` option will install ``salt-cloud`` and the required +``libcloud`` package. .. _`Salt Bootstrap`: https://github.com/saltstack/salt-bootstrap diff --git a/doc/topics/cloud/qs.rst b/doc/topics/cloud/qs.rst index 8cac85fe4f..3e6b40d37b 100644 --- a/doc/topics/cloud/qs.rst +++ b/doc/topics/cloud/qs.rst @@ -12,7 +12,9 @@ automatically installed salt-cloud for you. Use your distribution's package manager to install the ``salt-cloud`` package from the same repo that you used to install Salt. These repos will automatically be setup by Salt Bootstrap. -If there is no salt-cloud package, install with ``pip install salt-cloud``. +Alternatively, the ``-L`` option can be passed to the `Salt Bootstrap`_ script when +installing Salt. The ``-L`` option will install ``salt-cloud`` and the required +``libcloud`` package. .. _`Salt Bootstrap`: https://github.com/saltstack/salt-bootstrap From d4e7b91608cef279da68dc9e29065876d161716a Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 3 Jul 2017 23:06:54 -0400 Subject: [PATCH 031/508] Update releasecanddiate doc with new 2017.7.0rc1 Release --- doc/topics/releases/releasecandidate.rst | 34 +++++------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/doc/topics/releases/releasecandidate.rst b/doc/topics/releases/releasecandidate.rst index 0b042b86da..e2a628920a 100644 --- a/doc/topics/releases/releasecandidate.rst +++ b/doc/topics/releases/releasecandidate.rst @@ -8,7 +8,7 @@ Installing/Testing a Salt Release Candidate It's time for a new feature release of Salt! Follow the instructions below to install the latest release candidate of Salt, and try :ref:`all the shiny new -features `! Be sure to report any bugs you find on `Github +features `! Be sure to report any bugs you find on `Github `_. Installing Using Packages @@ -32,32 +32,10 @@ Builds for a few platforms are available as part of the RC at https://repo.salts Available builds: -- Amazon Linux -- Debian 8 -- macOS -- RHEL 7 -- SmartOS (see below) -- Ubuntu 16.04 - Windows .. FreeBSD -SmartOS -------- -Release candidate builds for SmartOS are available at http://pkg.blackdot.be/extras/salt-2016.11rc/. - -On a base64 2015Q4-x86_64 based native zone the package can be installed by the following: - -.. code-block:: bash - - pfexec pkg_add -U https://pkg.blackdot.be/extras/salt-2016.11rc/salt-2016.11.0rc2_2015Q4_x86_64.tgz - -When using the 2016Q2-tools release on the global zone by the following: - -.. code-block:: bash - - pfexec pkg_add -U https://pkg.blackdot.be/extras/salt-2016.11rc/salt-2016.11.0rc2_2016Q2_TOOLS.tgz - Installing Using Bootstrap ========================== @@ -67,14 +45,14 @@ You can install a release candidate of Salt using `Salt Bootstrap .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P git v2016.11.0rc2 + sudo sh install_salt.sh -P git v2017.7.0rc1 If you want to also install a master using Salt Bootstrap, use the ``-M`` flag: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M git v2016.11.0rc2 + sudo sh install_salt.sh -P -M git v2017.7.0rc1 If you want to install only a master and not a minion using Salt Bootstrap, use the ``-M`` and ``-N`` flags: @@ -82,13 +60,13 @@ the ``-M`` and ``-N`` flags: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M -N git v2016.11.0rc2 + sudo sh install_salt.sh -P -M -N git v2017.7.0rc1 Installing Using PyPI ===================== Installing from the `source archive -`_ on +`_ on `PyPI `_ is fairly straightforward. .. note:: @@ -126,4 +104,4 @@ Then install salt using the following command: .. code-block:: bash - sudo pip install salt==2016.11.0rc2 + sudo pip install salt==2017.7.0rc1 From 905be493d4586de841892ee44151083c169b4df9 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 4 Jul 2017 11:09:45 +0300 Subject: [PATCH 032/508] [2017.7] Fix scheduled jobs if `when` parameter is a list --- salt/utils/schedule.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 231f2331c0..888ba3369b 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1129,21 +1129,28 @@ class Schedule(object): log.error('Invalid date string {0}. ' 'Ignoring job {1}.'.format(i, job)) continue - when = int(time.mktime(when__.timetuple())) - if when >= now: - _when.append(when) + _when.append(int(time.mktime(when__.timetuple()))) if data['_splay']: _when.append(data['_splay']) + # Sort the list of "whens" from earlier to later schedules _when.sort() + + for i in _when: + if i < now and len(_when) > 1: + # Remove all missed schedules except the latest one. + # We need it to detect if it was triggered previously. + _when.remove(i) + if _when: - # Grab the first element - # which is the next run time + # Grab the first element, which is the next run time or + # last scheduled time in the past. when = _when[0] if '_run' not in data: - data['_run'] = True + # Prevent run of jobs from the past + data['_run'] = bool(when >= now) if not data['_next_fire_time']: data['_next_fire_time'] = when From 1bb42bb6097596cd1d5920b4cd5ea0a3d26ecb0f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Jul 2017 00:19:24 -0500 Subject: [PATCH 033/508] Fix regression when CLI pillar override is used with salt-call --- salt/state.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 06e63658f1..2a7e4a4507 100644 --- a/salt/state.py +++ b/salt/state.py @@ -660,8 +660,17 @@ class State(object): .format(', '.join(VALID_PILLAR_ENC)) ) self._pillar_enc = pillar_enc - self.opts['pillar'] = initial_pillar if initial_pillar is not None \ - else self._gather_pillar() + if initial_pillar is not None: + self.opts['pillar'] = initial_pillar + if self._pillar_override: + self.opts['pillar'] = salt.utils.dictupdate.merge( + self.opts['pillar'], + self._pillar_override, + self.opts.get('pillar_source_merging_strategy', 'smart'), + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + else: + self.opts['pillar'] = self._gather_pillar() self.state_con = context or {} self.load_modules() self.active = set() From 9a268949e3c5c430247143d7c7bb1da25f92c45d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Jul 2017 00:41:20 -0500 Subject: [PATCH 034/508] Add integration test for 42116 --- .../file/base/issue-42116-cli-pillar-override.sls | 2 ++ tests/integration/shell/call.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/integration/files/file/base/issue-42116-cli-pillar-override.sls diff --git a/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls b/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls new file mode 100644 index 0000000000..d3b2b239ec --- /dev/null +++ b/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls @@ -0,0 +1,2 @@ +ping -c 2 {{ pillar['myhost'] }}: + cmd.run diff --git a/tests/integration/shell/call.py b/tests/integration/shell/call.py index fd73dd92c9..7ed5875ba1 100644 --- a/tests/integration/shell/call.py +++ b/tests/integration/shell/call.py @@ -430,6 +430,21 @@ class CallTest(integration.ShellCase, testprogram.TestProgramCase, integration.S # Restore umask os.umask(current_umask) + @skipIf(sys.platform.startswith('win'), 'This test does not apply on Win') + def test_42116_cli_pillar_override(self): + ret = self.run_call( + 'state.apply issue-42116-cli-pillar-override ' + 'pillar=\'{"myhost": "localhost"}\'' + ) + for line in ret: + line = line.lstrip() + if line == 'Comment: Command "ping -c 2 localhost" run': + # Successful test + break + else: + log.debug('salt-call output:\n\n%s', '\n'.join(ret)) + self.fail('CLI pillar override not found in pillar data') + def tearDown(self): ''' Teardown method to remove installed packages From d14291267f5141715aefe90460959e9ca92abc86 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Jul 2017 02:00:54 -0500 Subject: [PATCH 035/508] Fix pillar.get when saltenv is passed --- salt/modules/pillar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index 5ad8afe27c..e8d6e0a0e1 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -110,7 +110,7 @@ def get(key, default = '' opt_merge_lists = __opts__.get('pillar_merge_lists', False) if \ merge_nested_lists is None else merge_nested_lists - pillar_dict = __pillar__ if saltenv is None else items(saltenv=saltenv) + pillar_dict = __pillar__ if saltenv is None else items(pillarenv=saltenv) if merge: if isinstance(default, dict): From 1435bf177ec4bdbcbba2614d7edde5074868a84e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 3 Jul 2017 14:17:13 -0600 Subject: [PATCH 036/508] require large timediff for ipv6 warning --- salt/grains/core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index ac6f98c043..c7bc2fe49f 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1688,12 +1688,14 @@ def ip_fqdn(): ret[key] = [] else: try: + start_time = datetime.datetime.utcnow() info = socket.getaddrinfo(_fqdn, None, socket_type) ret[key] = list(set(item[4][0] for item in info)) except socket.error: - if __opts__['__role'] == 'master': - log.warning('Unable to find IPv{0} record for "{1}" causing a 10 second timeout when rendering grains. ' - 'Set the dns or /etc/hosts for IPv{0} to clear this.'.format(ipv_num, _fqdn)) + timediff = datetime.datetime.utcnow() - start_time + if timediff.seconds > 5 and __opts__['__role'] == 'master': + log.warning('Unable to find IPv{0} record for "{1}" causing a {2} second timeout when rendering grains. ' + 'Set the dns or /etc/hosts for IPv{0} to clear this.'.format(ipv_num, _fqdn, timediff)) ret[key] = [] return ret From 9c4e132540ff8036a1514bcd4096458abf9b0883 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 3 Jul 2017 16:37:31 -0600 Subject: [PATCH 037/508] Import datetime --- salt/grains/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index c7bc2fe49f..9ed045f45d 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -20,6 +20,7 @@ import re import platform import logging import locale +import datetime import salt.exceptions __proxyenabled__ = ['*'] From bd802432339e648c8854e349f600b8c61fbc43ec Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 3 Jul 2017 14:15:35 -0600 Subject: [PATCH 038/508] Change repo_ng to repo-ng --- doc/topics/windows/windows-package-manager.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index d4ce6b5a07..063c8b44eb 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -140,7 +140,7 @@ packages: - 2015.8.0 and later minions: https://github.com/saltstack/salt-winrepo-ng - Earlier releases: https://github.com/saltstack/salt-winrepo -By default, these repositories are mirrored to ``/srv/salt/win/repo_ng`` +By default, these repositories are mirrored to ``/srv/salt/win/repo-ng`` and ``/srv/salt/win/repo``. This location can be changed in the master config file by setting the From e1694af39c12de643d939f3460ba65ed38068741 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Wed, 5 Jul 2017 17:06:06 -0400 Subject: [PATCH 039/508] Update builds available for rc1 --- doc/topics/releases/releasecandidate.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/topics/releases/releasecandidate.rst b/doc/topics/releases/releasecandidate.rst index e2a628920a..60918d6ac3 100644 --- a/doc/topics/releases/releasecandidate.rst +++ b/doc/topics/releases/releasecandidate.rst @@ -32,6 +32,8 @@ Builds for a few platforms are available as part of the RC at https://repo.salts Available builds: +- Ubuntu16 +- Redhat7 - Windows .. FreeBSD From 4ee24202fc78b0c2a50fc9657520f8a087ba4aa0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 5 Jul 2017 15:49:53 -0600 Subject: [PATCH 040/508] Fix unit tests for test_pip --- tests/unit/modules/test_pip.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/unit/modules/test_pip.py b/tests/unit/modules/test_pip.py index ec4b379928..8a5229b0e6 100644 --- a/tests/unit/modules/test_pip.py +++ b/tests/unit/modules/test_pip.py @@ -10,6 +10,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs +import salt.utils import salt.modules.pip as pip from salt.exceptions import CommandExecutionError @@ -289,17 +290,23 @@ class PipTestCase(TestCase, LoaderModuleMockMixin): mock_path.isdir.return_value = True pkg = 'mock' - venv_path = '/test_env' def join(*args): - return '/'.join(args) + return os.sep.join(args) + mock_path.join = join mock = MagicMock(return_value={'retcode': 0, 'stdout': ''}) with patch.dict(pip.__salt__, {'cmd.run_all': mock}): + if salt.utils.is_windows(): + venv_path = 'c:\\test_env' + bin_path = os.path.join(venv_path, 'Scripts', 'pip.exe').encode('string-escape') + else: + venv_path = '/test_env' + bin_path = os.path.join(venv_path, 'bin', 'pip') pip.install(pkg, bin_env=venv_path) mock.assert_called_once_with( - [os.path.join(venv_path, 'bin', 'pip'), 'install', pkg], - env={'VIRTUAL_ENV': '/test_env'}, + [bin_path, 'install', pkg], + env={'VIRTUAL_ENV': venv_path}, saltenv='base', runas=None, use_vt=False, From 00d9a52802e6d01c6f0826c14c7470d95958e4c4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 5 Jul 2017 17:44:06 -0600 Subject: [PATCH 041/508] Fix problem with handling REG_QWORD in list values --- salt/modules/reg.py | 4 +++- tests/unit/modules/test_reg_win.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/modules/reg.py b/salt/modules/reg.py index 63249dc5fe..644859f141 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -150,7 +150,9 @@ class Registry(object): # pylint: disable=R0903 _winreg.REG_DWORD: 'REG_DWORD', _winreg.REG_EXPAND_SZ: 'REG_EXPAND_SZ', _winreg.REG_MULTI_SZ: 'REG_MULTI_SZ', - _winreg.REG_SZ: 'REG_SZ' + _winreg.REG_SZ: 'REG_SZ', + # REG_QWORD isn't in the winreg library + 11: 'REG_QWORD' } self.opttype_reverse = { _winreg.REG_OPTION_NON_VOLATILE: 'REG_OPTION_NON_VOLATILE', diff --git a/tests/unit/modules/test_reg_win.py b/tests/unit/modules/test_reg_win.py index 960c00e5d0..e83d89b127 100644 --- a/tests/unit/modules/test_reg_win.py +++ b/tests/unit/modules/test_reg_win.py @@ -27,7 +27,7 @@ except ImportError: PY2 = sys.version_info[0] == 2 # The following used to make sure we are not # testing already existing data -# Note strftime retunrns a str, so we need to make it unicode +# Note strftime returns a str, so we need to make it unicode TIMEINT = int(time.time()) if PY2: From fb2cb78a31ce657448e812cc2c7c84695a8ef2d6 Mon Sep 17 00:00:00 2001 From: Steve Katz Date: Wed, 5 Jul 2017 19:56:11 -0400 Subject: [PATCH 042/508] Fix docs for puppet.plugin_sync so code-block renders properly and sync is spelled consistently --- salt/modules/puppet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/puppet.py b/salt/modules/puppet.py index 11067686be..aef4587381 100644 --- a/salt/modules/puppet.py +++ b/salt/modules/puppet.py @@ -337,9 +337,10 @@ def summary(): def plugin_sync(): ''' - Runs a plugin synch between the puppet master and agent + Runs a plugin sync between the puppet master and agent CLI Example: + .. code-block:: bash salt '*' puppet.plugin_sync From b27b1e340a09e8cb3b8b69c1bd5129a92b145632 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 6 Jul 2017 16:43:07 +0300 Subject: [PATCH 043/508] Fix #42115: parse libcloud "rc" version correctly --- salt/cloud/libcloudfuncs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/libcloudfuncs.py b/salt/cloud/libcloudfuncs.py index f627c4cf96..3191251c99 100644 --- a/salt/cloud/libcloudfuncs.py +++ b/salt/cloud/libcloudfuncs.py @@ -25,7 +25,7 @@ try: ) HAS_LIBCLOUD = True LIBCLOUD_VERSION_INFO = tuple([ - int(part) for part in libcloud.__version__.replace('-', '.').split('.')[:3] + int(part) for part in libcloud.__version__.replace('-', '.').replace('rc', '.').split('.')[:3] ]) except ImportError: From 7c0fb248ec025f0043c901badac8370aadfb6f4d Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Thu, 6 Jul 2017 09:46:04 -0400 Subject: [PATCH 044/508] Fix kerberos create_keytab doc --- salt/modules/kerberos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/kerberos.py b/salt/modules/kerberos.py index a5a77a5891..284033daaf 100644 --- a/salt/modules/kerberos.py +++ b/salt/modules/kerberos.py @@ -262,7 +262,7 @@ def create_keytab(name, keytab, enctypes=None): .. code-block:: bash - salt 'kdc.example.com' host/host1.example.com host1.example.com.keytab + salt 'kdc.example.com' kerberos.create_keytab host/host1.example.com host1.example.com.keytab ''' ret = {} From 01addb6053b5defa5ae556439f72e5d3406ecbb1 Mon Sep 17 00:00:00 2001 From: "Michael A. Smith" Date: Wed, 2 Nov 2016 15:41:52 -0400 Subject: [PATCH 045/508] Avoid Early Convert ret['comment'] to String Fixes this exception: An exception occurred in this state: Traceback (most recent call last): File "/var/tmp/.syops_b1274f_salt/py2/salt/state.py", line 1733, in call **cdata['kwargs']) File "/var/tmp/.syops_b1274f_salt/py2/salt/loader.py", line 1652, in wrapper return f(*args, **kwargs) File "/var/tmp/.syops_b1274f_salt/py2/salt/states/gpg.py", line 119, in present ret['comment'].append('Adding {0} to GPG keychain'.format(name)) AttributeError: 'str' object has no attribute 'append' --- salt/states/gpg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/gpg.py b/salt/states/gpg.py index 71b12c3aeb..a5aa4aa4cd 100644 --- a/salt/states/gpg.py +++ b/salt/states/gpg.py @@ -132,7 +132,7 @@ def present(name, else: ret['comment'].append('Invalid trust level {0}'.format(trust)) - ret['comment'] = '\n'.join(ret['comment']) + ret['comment'] = '\n'.join(ret['comment']) return ret @@ -188,5 +188,5 @@ def absent(name, ret['comment'].append('Deleting {0} from GPG keychain'.format(name)) else: ret['comment'].append('{0} not found in GPG keychain'.format(name)) - ret['comment'] = '\n'.join(ret['comment']) + ret['comment'] = '\n'.join(ret['comment']) return ret From 53f7b987e857faf5ae61ae2ae0293fb4999a9380 Mon Sep 17 00:00:00 2001 From: Dan Lloyd Date: Mon, 13 Feb 2017 12:39:09 -0500 Subject: [PATCH 046/508] Pass sig to service.status in after_toggle As on line 350, pass `sig` to `service.status` to ensure that the correct behavior is used. If we don't pass `sig`, upstart/sysv jobs that do not implement `service status` will fail. --- salt/states/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/service.py b/salt/states/service.py index acfc647bc2..0602360864 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -379,7 +379,7 @@ def running(name, enable=None, sig=None, init_delay=None, **kwargs): time.sleep(init_delay) # only force a change state if we have explicitly detected them - after_toggle_status = __salt__['service.status'](name) + after_toggle_status = __salt__['service.status'](name, sig) if 'service.enabled' in __salt__: after_toggle_enable_status = __salt__['service.enabled'](name) else: From 686926daf7f29464f4618e0205ad3fde2c1a4c11 Mon Sep 17 00:00:00 2001 From: Arthur Lutz Date: Tue, 4 Jul 2017 14:00:10 +0200 Subject: [PATCH 047/508] Update aws.rst - add Debian default username --- doc/topics/cloud/aws.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/topics/cloud/aws.rst b/doc/topics/cloud/aws.rst index c2131715f1..b5eb10178e 100644 --- a/doc/topics/cloud/aws.rst +++ b/doc/topics/cloud/aws.rst @@ -78,6 +78,7 @@ parameters are discussed in more detail below. # RHEL -> ec2-user # CentOS -> ec2-user # Ubuntu -> ubuntu + # Debian -> admin # ssh_username: ec2-user From 1bbff572ab003bf13e5e06712cb58c9725d50297 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 6 Jul 2017 11:46:23 -0600 Subject: [PATCH 048/508] Fix some documentation issues found in jinja filters doc topic These doc issues were all found in issue #42151. This commit takes care of all of the issues presented in #42151 except for the last task. --- doc/topics/jinja/index.rst | 38 +++++++++++++++++++++++++++++++------- salt/utils/__init__.py | 18 ++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 5548b29646..75641e4203 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -335,7 +335,7 @@ Returns: .. versionadded:: 2017.7.0 -Wraps a text around quoutes. +This text will be wrapped in quotes. .. jinja_ref:: regex_search @@ -750,19 +750,43 @@ Returns: Check a whitelist and/or blacklist to see if the value matches it. -Example: +This filter can be used with either a whitelist or a blacklist individually, +or a whitelist and a blacklist can be passed simultaneously. + +If whitelist is used alone, value membership is checked against the +whitelist only. If the value is found, the function returns ``True``. +Otherwise, it returns ``False``. + +If blacklist is used alone, value membership is checked against the +blacklist only. If the value is found, the function returns ``False``. +Otherwise, it returns ``True``. + +If both a whitelist and a blacklist are provided, value membership in the +blacklist will be examined first. If the value is not found in the blacklist, +then the whitelist is checked. If the value isn't found in the whitelist, +the function returns ``False``. + +Whitelist Example: .. code-block:: jinja - {{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} - {{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} + {{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} Returns: .. code-block:: python - True + True +Blacklist Example: + +.. code-block:: jinja + + {{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} + +.. code-block:: python + + False .. jinja_ref:: date_format @@ -1186,7 +1210,7 @@ Example: .. code-block:: jinja - {{ ['192.168.0.1', 'foo', 'bar', 'fe80::'] | ipv4 }} + {{ ['192.168.0.1', 'foo', 'bar', 'fe80::'] | ipv6 }} Returns: @@ -1224,7 +1248,7 @@ Returns: .. versionadded:: 2017.7.0 -Return the size of the network. +Return the size of the network. This utility works for both IPv4 and IPv6. Example: diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index b4e3c58abb..ec018f1ad7 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1444,6 +1444,24 @@ def expr_match(line, expr): def check_whitelist_blacklist(value, whitelist=None, blacklist=None): ''' Check a whitelist and/or blacklist to see if the value matches it. + + value + The item to check the whitelist and/or blacklist against. + + whitelist + The list of items that are white-listed. If ``value`` is found + in the whitelist, then the function returns ``True``. Otherwise, + it returns ``False``. + + blacklist + The list of items that are black-listed. If ``value`` is found + in the blacklist, then the function returns ``False``. Otherwise, + it returns ``True``. + + If both a whitelist and a blacklist are provided, value membership + in the blacklist will be examined first. If the value is not found + in the blacklist, then the whitelist is checked. If the value isn't + found in the whitelist, the function returns ``False``. ''' if blacklist is not None: if not hasattr(blacklist, '__iter__'): From 832a3d86ddb1f6d56596ffa18649a0779d5d7779 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 6 Jul 2017 13:11:02 -0600 Subject: [PATCH 049/508] Skip tests that use os.symlink on Windows --- tests/unit/modules/test_timezone.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/modules/test_timezone.py b/tests/unit/modules/test_timezone.py index 7cc6a18d19..0474ca9627 100644 --- a/tests/unit/modules/test_timezone.py +++ b/tests/unit/modules/test_timezone.py @@ -161,6 +161,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_aix', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -175,6 +176,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^ZONE=.*', 'ZONE="UTC"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -189,6 +191,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^TIMEZONE=.*', 'TIMEZONE="UTC"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -207,6 +210,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = _fopen.return_value.__enter__.return_value.write.mock_calls[0] assert args == ('UTC',) + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -225,6 +229,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = _fopen.return_value.__enter__.return_value.write.mock_calls[0] assert args == ('UTC',) + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=True)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -239,6 +244,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._timedatectl', MagicMock(return_value={'stdout': 'rtc in local tz:yes'})): assert timezone.get_hwclock() == 'localtime' + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -254,6 +260,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['tail', '-n', '1', '/etc/adjtime'],) assert kwarg == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -284,6 +291,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['tail', '-n', '1', '/etc/adjtime'],) assert kwarg == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -299,6 +307,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.utils.fopen', mock_open()): assert timezone.get_hwclock() == 'localtime' + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -315,6 +324,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(timezone.__grains__, {'os_family': ['AIX']}): assert timezone.get_hwclock() == hwclock + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -329,6 +339,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert timezone.set_hwclock('forty two') assert timezone.set_hwclock('UTC') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -348,6 +359,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['rtc', '-z', 'GMT'],) assert kwargs == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -364,6 +376,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['timezonectl', 'set-local-rtc', 'false'],) assert kwargs == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -379,6 +392,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^ZONE=.*', 'ZONE="TEST_TIMEZONE"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -394,6 +408,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^TIMEZONE=.*', 'TIMEZONE="TEST_TIMEZONE"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -413,6 +428,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[1] assert args == ('/etc/default/rcS', '^UTC=.*', 'UTC=no') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) From a34970b45b0dfb32788761f66241470a3042b474 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 6 Jul 2017 12:12:50 -0700 Subject: [PATCH 050/508] Back porting the fix for 2017.7 that ensures the order of the names parameter. --- salt/state.py | 12 ++++++++---- .../files/file/base/issue-7649-handle-iorder.sls | 12 ++++++------ tests/integration/states/handle_iorder.py | 7 +++---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/salt/state.py b/salt/state.py index 2a7e4a4507..82c180486b 100644 --- a/salt/state.py +++ b/salt/state.py @@ -547,7 +547,7 @@ class Compiler(object): continue for state, run in six.iteritems(body): funcs = set() - names = set() + names = [] if state.startswith('__'): continue chunk = {'state': state, @@ -564,7 +564,9 @@ class Compiler(object): if isinstance(arg, dict): for key, val in six.iteritems(arg): if key == 'names': - names.update(val) + for _name in val: + if _name not in names: + names.append(_name) continue else: chunk.update(arg) @@ -1287,7 +1289,7 @@ class State(object): continue for state, run in six.iteritems(body): funcs = set() - names = set() + names = [] if state.startswith('__'): continue chunk = {'state': state, @@ -1306,7 +1308,9 @@ class State(object): if isinstance(arg, dict): for key, val in six.iteritems(arg): if key == 'names': - names.update(val) + for _name in val: + if _name not in names: + names.append(_name) elif key == 'state': # Don't pass down a state override continue diff --git a/tests/integration/files/file/base/issue-7649-handle-iorder.sls b/tests/integration/files/file/base/issue-7649-handle-iorder.sls index 4cfcdd11a0..49e6dfe900 100644 --- a/tests/integration/files/file/base/issue-7649-handle-iorder.sls +++ b/tests/integration/files/file/base/issue-7649-handle-iorder.sls @@ -1,9 +1,9 @@ handle-iorder: cmd: - - cwd: /tmp/ruby-1.9.2-p320 - - names: - - ./configure - - make - - make install - - run + - cwd: /tmp/ruby-1.9.2-p320 + - names: + - ./configure + - make + - make install + - run diff --git a/tests/integration/states/handle_iorder.py b/tests/integration/states/handle_iorder.py index 7ef1864cdd..1865f85a12 100644 --- a/tests/integration/states/handle_iorder.py +++ b/tests/integration/states/handle_iorder.py @@ -24,11 +24,10 @@ class HandleOrderTest(integration.ModuleCase): ''' ret = self.run_function('state.show_low_sls', mods='issue-7649-handle-iorder') - sorted_chunks = sorted(ret, key=lambda c: c.get('order')) + sorted_chunks = [chunk['name'] for chunk in sorted(ret, key=lambda c: c.get('order'))] - self.assertEqual(sorted_chunks[0]['name'], './configure') - self.assertEqual(sorted_chunks[1]['name'], 'make') - self.assertEqual(sorted_chunks[2]['name'], 'make install') + expected = ['./configure', 'make', 'make install'] + self.assertEqual(expected, sorted_chunks) if __name__ == '__main__': From 8260a71c07c3bdc22d6af6fcad737ed199cd9706 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 6 Jul 2017 13:19:40 -0600 Subject: [PATCH 051/508] Disable tests that require pwd in Windows --- tests/unit/modules/test_useradd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/modules/test_useradd.py b/tests/unit/modules/test_useradd.py index 2bcbee38d1..7fd457425f 100644 --- a/tests/unit/modules/test_useradd.py +++ b/tests/unit/modules/test_useradd.py @@ -72,6 +72,7 @@ class UserAddTestCase(TestCase, LoaderModuleMockMixin): # 'getent' function tests: 2 + @skipIf(HAS_PWD is False, 'The pwd module is not available') def test_getent(self): ''' Test if user.getent already have a value @@ -359,6 +360,7 @@ class UserAddTestCase(TestCase, LoaderModuleMockMixin): # 'list_users' function tests: 1 + @skipIf(HAS_PWD is False, 'The pwd module is not available') def test_list_users(self): ''' Test if it returns a list of all users From 11862743c2d7fc610f34541e5447cc94b1c1dea5 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 6 Jul 2017 15:03:54 -0600 Subject: [PATCH 052/508] Use long_range function for IPv6Network hosts() function Previous implementation used xrange, which requires that the range fit in a C long data type. Given that an IPv6 subnet is massive, the ``salt.ext.ipadress.IPv6Network.hosts()`` function would stacktrace with an OverflowError due to the use of xrange. However, ``/64`` is a common netmask. This PR uses the ``long_range`` function for the ``IPv6Network.hosts()`` function instead of ``xrange``. While the funciton will no longer stacktrace on large IPv6 subnets, the user should be aware that these large subnets will take a very long time to list. Fixes #42166 --- salt/ext/ipaddress.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/ext/ipaddress.py b/salt/ext/ipaddress.py index d2dfc5ee50..75011f7606 100644 --- a/salt/ext/ipaddress.py +++ b/salt/ext/ipaddress.py @@ -28,9 +28,6 @@ bytes = bytearray # Python 2 does not support exception chaining. # s/ from None$// -# Python 2 ranges need to fit in a C long -# 'fix' hosts() for IPv6Network - # When checking for instances of int, also allow Python 2's long. _builtin_isinstance = isinstance @@ -2259,7 +2256,7 @@ class IPv6Network(_BaseV6, _BaseNetwork): """ network = int(self.network_address) broadcast = int(self.broadcast_address) - for x in range(1, broadcast - network + 1): + for x in long_range(1, broadcast - network + 1): yield self._address_class(network + x) @property From b8bcc0d5990557e993d0fc785a9528278f038604 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 6 Jul 2017 17:36:18 -0600 Subject: [PATCH 053/508] Add note to various network_hosts docs about long_run for IPv6 networks --- doc/topics/jinja/index.rst | 7 ++++++- salt/utils/network.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 5548b29646..7081ce2154 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -1202,7 +1202,12 @@ Returns: .. versionadded:: 2017.7.0 -Return the list of hosts within a networks. +Return the list of hosts within a networks. This utility works for both IPv4 and IPv6. + +.. note:: + + When running this command with a large IPv6 network, the command will + take a long time to gather all of the hosts. Example: diff --git a/salt/utils/network.py b/salt/utils/network.py index aee5be16ab..33c7a14c9f 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -453,6 +453,11 @@ def _network_hosts(ip_addr_entry): def network_hosts(value, options=None, version=None): ''' Return the list of hosts within a network. + + .. note:: + + When running this command with a large IPv6 network, the command will + take a long time to gather all of the hosts. ''' ipaddr_filter_out = _filter_ipaddr(value, options=options, version=version) if not ipaddr_filter_out: From 407b8f4bb385ee398c30d06586c7524f67db625f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FUSIL-DELAHAYE?= Date: Fri, 7 Jul 2017 16:36:42 +0200 Subject: [PATCH 054/508] Fix #42198 If where_args is not set, not using it in the delete request. --- salt/states/sqlite3.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/states/sqlite3.py b/salt/states/sqlite3.py index 7c612cc9d0..abf1953e7d 100644 --- a/salt/states/sqlite3.py +++ b/salt/states/sqlite3.py @@ -147,9 +147,13 @@ def row_absent(name, db, table, where_sql, where_args=None): changes['changes']['old'] = rows[0] else: - cursor = conn.execute("DELETE FROM `" + - table + "` WHERE " + where_sql, - where_args) + if where_args is None: + cursor = conn.execute("DELETE FROM `" + + table + "` WHERE " + where_sql) + else: + cursor = conn.execute("DELETE FROM `" + + table + "` WHERE " + where_sql, + where_args) conn.commit() if cursor.rowcount == 1: changes['result'] = True From 2be4865f48e714b2f1e6743f4d6aaaedd5524bbb Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 7 Jul 2017 09:43:52 -0600 Subject: [PATCH 055/508] [PY3] Fix test that is flaky in Python 3 We can't rely on lists having the same order in Python3 the same way we rely on them in Python2. If we sort them first, and then compare them, this test will be more reliable. --- tests/unit/templates/test_jinja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 471d626540..6817c0c731 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -486,7 +486,7 @@ class TestCustomExtensions(TestCase): env = Environment(extensions=[SerializerExtension]) if six.PY3: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset).strip("'{}").split("', '") - self.assertEqual(rendered, list(unique)) + self.assertEqual(sorted(rendered), sorted(list(unique))) else: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset) self.assertEqual(rendered, u"{0}".format(unique)) From 771ade5d7325ff7bc285968ef6630dd6afe2930a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 7 Jul 2017 11:32:13 -0500 Subject: [PATCH 056/508] Only pass a saltenv in orchestration if one was explicitly passed (2017.7) Also, only pass pillarenv if explicitly passed. _get_opts() in the state module will apply the correct saltenv (if applicable) when no explicit saltenv/pillarenv is passed. --- salt/states/saltmod.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index d9c41f8cb3..ec430d7a60 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -245,14 +245,12 @@ def state(name, if pillar: cmd_kw['kwarg']['pillar'] = pillar - # If pillarenv is directly defined, use it - if pillarenv: + if pillarenv is not None: cmd_kw['kwarg']['pillarenv'] = pillarenv - # Use pillarenv if it's passed from __opts__ (via state.orchestrate for example) - elif __opts__.get('pillarenv'): - cmd_kw['kwarg']['pillarenv'] = __opts__['pillarenv'] - cmd_kw['kwarg']['saltenv'] = saltenv if saltenv is not None else __env__ + if saltenv is not None: + cmd_kw['kwarg']['saltenv'] = saltenv + cmd_kw['kwarg']['queue'] = queue if isinstance(concurrent, bool): From c07e22041aa13d256c5a011e3f5d47792857dd0e Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 7 Jul 2017 12:37:22 -0600 Subject: [PATCH 057/508] Add missing config to example --- salt/states/win_iis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/states/win_iis.py b/salt/states/win_iis.py index 071899886f..0d2661d6c4 100644 --- a/salt/states/win_iis.py +++ b/salt/states/win_iis.py @@ -495,6 +495,7 @@ def container_setting(name, container, settings=None): processModel.maxProcesses: 1 processModel.userName: TestUser processModel.password: TestPassword + processModel.identityType: SpecificUser Example of usage for the ``Sites`` container: From 22a18fa2ed473ae18b15fedd383facce46eb3d27 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 7 Jul 2017 11:32:13 -0500 Subject: [PATCH 058/508] Only pass a saltenv in orchestration if one was explicitly passed (2016.11) Also, only pass pillarenv if explicitly passed. _get_opts() in the state module will apply the correct saltenv (if applicable) when no explicit saltenv/pillarenv is passed. --- salt/states/saltmod.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index 92f46af465..25b0dd1040 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -73,6 +73,7 @@ def state( saltenv=None, test=False, pillar=None, + pillarenv=None, expect_minions=False, fail_minions=None, allow_fail=0, @@ -230,7 +231,12 @@ def state( if pillar: cmd_kw['kwarg']['pillar'] = pillar - cmd_kw['kwarg']['saltenv'] = saltenv if saltenv is not None else __env__ + if pillarenv is not None: + cmd_kw['kwarg']['pillarenv'] = pillarenv + + if saltenv is not None: + cmd_kw['kwarg']['saltenv'] = saltenv + cmd_kw['kwarg']['queue'] = queue if isinstance(concurrent, bool): From 798d29276ed29e3dd209d08952fcaf7633cad87e Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 7 Jul 2017 15:14:06 -0600 Subject: [PATCH 059/508] Add note about "to_bytes" jinja filter issues when using yaml_jinja renderer --- doc/topics/jinja/index.rst | 14 ++++++++++++++ doc/topics/troubleshooting/yaml_idiosyncrasies.rst | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 75641e4203..bcab6c2322 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -849,6 +849,13 @@ Example: {{ 'wall of text' | to_bytes }} +.. note:: + + This option may have adverse effects when using the default renderer, ``yaml_jinja``. + This is due to the fact that YAML requires proper handling in regard to special + characters. Please see the section on :ref:`YAML ASCII support ` + in the :ref:`YAML Idiosyncracies ` documentation for more + information. .. jinja_ref:: json_decode_list @@ -1308,6 +1315,13 @@ Example: {{ '00:11:22:33:44:55' | mac_str_to_bytes }} +.. note:: + + This option may have adverse effects when using the default renderer, ``yaml_jinja``. + This is due to the fact that YAML requires proper handling in regard to special + characters. Please see the section on :ref:`YAML ASCII support ` + in the :ref:`YAML Idiosyncracies ` documentation for more + information. .. jinja_ref:: dns_check diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index e1c6383459..df5eff3d01 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -248,8 +248,10 @@ Alternatively, they can be defined the "old way", or with multiple - require: - user: fred -YAML support only plain ASCII -============================= +.. _yaml_plain_ascii: + +YAML supports only plain ASCII +============================== According to YAML specification, only ASCII characters can be used. From 08e7d8351a2d6a9fde4fad6757954d5e2f7da35d Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Mon, 10 Jul 2017 12:46:32 -0400 Subject: [PATCH 060/508] Abolish references to `dig` in examples. --- salt/modules/dnsutil.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/dnsutil.py b/salt/modules/dnsutil.py index 36b1469a4b..1cd2b6a032 100644 --- a/salt/modules/dnsutil.py +++ b/salt/modules/dnsutil.py @@ -232,7 +232,7 @@ def check_ip(ip_addr): .. code-block:: bash - salt ns1 dig.check_ip 127.0.0.1 + salt ns1 dnsutil.check_ip 127.0.0.1 ''' if _has_dig(): return __salt__['dig.check_ip'](ip_addr) @@ -302,7 +302,7 @@ def NS(domain, resolve=True, nameserver=None): .. code-block:: bash - salt ns1 dig.NS google.com + salt ns1 dnsutil.NS google.com ''' if _has_dig(): @@ -323,7 +323,7 @@ def SPF(domain, record='SPF', nameserver=None): .. code-block:: bash - salt ns1 dig.SPF google.com + salt ns1 dnsutil.SPF google.com ''' if _has_dig(): return __salt__['dig.SPF'](domain, record, nameserver) @@ -346,7 +346,7 @@ def MX(domain, resolve=False, nameserver=None): .. code-block:: bash - salt ns1 dig.MX google.com + salt ns1 dnsutil.MX google.com ''' if _has_dig(): return __salt__['dig.MX'](domain, resolve, nameserver) From 4cb51bd03af6a6a3a4dc0ce07eb109cd07597d28 Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Mon, 10 Jul 2017 13:55:06 -0400 Subject: [PATCH 061/508] Make note of dig partial requirement. --- salt/modules/dnsutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/dnsutil.py b/salt/modules/dnsutil.py index 1cd2b6a032..948cd720ce 100644 --- a/salt/modules/dnsutil.py +++ b/salt/modules/dnsutil.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- ''' -Compendium of generic DNS utilities +Compendium of generic DNS utilities. + +.. note: + Some functions in the `dnsutil` execution module depend on `dig`. ''' from __future__ import absolute_import From 4bf4b14161be83fad83e1c1ca4f8974f2cfae4ae Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Mon, 10 Jul 2017 20:29:35 +0000 Subject: [PATCH 062/508] New option for napalm proxy/minion: provider This option will turn very handy for environemnts when the user requires a library with a different name than napalm-base (eventually on top of the public lib). The sole requirement right now is to preserve the get_network_driver function. However, this can be enhnaced even from this perspective later, if really needed. --- salt/utils/napalm.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index c69a92432e..36914be042 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import traceback import logging +import importlib from functools import wraps log = logging.getLogger(__file__) @@ -264,6 +265,7 @@ def get_device_opts(opts, salt_obj=None): network_device['TIMEOUT'] = device_dict.get('timeout', 60) network_device['OPTIONAL_ARGS'] = device_dict.get('optional_args', {}) network_device['ALWAYS_ALIVE'] = device_dict.get('always_alive', True) + network_device['PROVIDER'] = device_dict.get('provider') network_device['UP'] = False # get driver object form NAPALM if 'config_lock' not in network_device['OPTIONAL_ARGS']: @@ -281,7 +283,24 @@ def get_device(opts, salt_obj=None): ''' log.debug('Setting up NAPALM connection') network_device = get_device_opts(opts, salt_obj=salt_obj) - _driver_ = napalm_base.get_network_driver(network_device.get('DRIVER_NAME')) + provider_lib = napalm_base + if network_device.get('PROVIDER'): + # In case the user requires a different provider library, + # other than napalm-base. + # For example, if napalm-base does not satisfy the requirements + # and needs to be enahanced with more specific features, + # we may need to define a custom library on top of napalm-base + # with the constraint that it still needs to provide the + # `get_network_driver` function. However, even this can be + # extended later, if really needed. + # Configuration example: + # provider: napalm_base_example + try: + provider_lib = importlib.import_module(network_device.get('PROVIDER')) + except ImportError as ierr: + log.error('Unable to import {0}'.format(network_device.get('PROVIDER')), exc_info=True) + log.error('Falling back to napalm-base') + _driver_ = provider_lib.get_network_driver(network_device.get('DRIVER_NAME')) try: network_device['DRIVER'] = _driver_( network_device.get('HOSTNAME', ''), From 1ac69bd7370e05ff278cc995c05133300a7da8de Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Tue, 11 Jul 2017 14:44:51 +0000 Subject: [PATCH 063/508] Document the provider option and rearrange the doc --- salt/proxy/napalm.py | 78 +++++++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index b0cd8d6546..9ae4fd5316 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -23,37 +23,77 @@ Please check Installation_ for complete details. Pillar ------ -The napalm proxy configuration requires four mandatory parameters in order to connect to the network device: +The napalm proxy configuration requires the following parameters in order to connect to the network device: -* driver: specifies the network device operating system. For a complete list of the supported operating systems \ -please refer to the `NAPALM Read the Docs page`_. -* host: hostname -* username: username to be used when connecting to the device -* passwd: the password needed to establish the connection -* optional_args: dictionary with the optional arguments. Check the complete list of supported `optional arguments`_ +driver + Specifies the network device operating system. + For a complete list of the supported operating systems please refer to the + `NAPALM Read the Docs page`_. -.. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems -.. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments +host + The IP Address or FQDN to use when connecting to the device. Alternatively, + the following field names can be used instead: ``hostname``, ``fqdn``, ``ip``. -.. versionadded:: 2017.7.0 +username + The username to be used when connecting to the device. -* always_alive: in certain less dynamic environments, maintaining the remote connection permanently +passwd + The password needed to establish the connection. + + .. note:: + + This field may not be mandatory when working with SSH-based drivers, and + the username has a SSH key properly configured on the device targeted to + be managed. + +optional_args + Dictionary with the optional arguments. + Check the complete list of supported `optional arguments`_. + +always_alive: ``True`` + In certain less dynamic environments, maintaining the remote connection permanently open with the network device is not always beneficial. In that case, the user can select to initialize the connection only when needed, by specifying this field to ``false``. Default: ``true`` (maintains the connection with the remote network device). -Example: + .. versionadded:: 2017.7.0 + +provider: ``napalm_base`` + The module that provides the ``get_network_device`` function. + This option is useful when the user has more specific needs and requires + to extend the NAPALM capabilities using a private library implementation. + The only constraint is that the alternative library needs to have the + ``get_network_device`` function available. + + .. versionadded:: 2017.7.1 + +.. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems +.. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments + +Proxy pillar file example: .. code-block:: yaml proxy: - proxytype: napalm - driver: junos - host: core05.nrt02 - username: my_username - passwd: my_password - optional_args: - port: 12201 + proxytype: napalm + driver: junos + host: core05.nrt02 + username: my_username + passwd: my_password + optional_args: + port: 12201 + +Example using a user-specific library, extending NAPALM's capabilities, e.g. ``custom_napalm_base``: + +.. code-block:: yaml + + proxy: + proxytype: napalm + driver: ios + fqdn: cr1.th2.par.as1234.net + username: salt + password: '' + provider: custom_napalm_base .. seealso:: From 97261bfe69bf018e19082a7616d221d12c908fb2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 10:04:37 -0600 Subject: [PATCH 064/508] Fix win_inet_pton check for malformatted ip addresses --- salt/ext/win_inet_pton.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index ffb47d440a..8c4d7dde3e 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import socket import ctypes import os +import ipaddress class sockaddr(ctypes.Structure): @@ -31,6 +32,24 @@ else: def inet_pton(address_family, ip_string): + # Verify IP Address + # This will catch IP Addresses such as 10.1.2 + if address_family == socket.AF_INET: + try: + ipaddress.ip_address(ip_string.decode()) + except ValueError: + raise socket.error('illegal IP address string passed to inet_pton') + return socket.inet_aton(ip_string) + + # Verify IP Address + # The `WSAStringToAddressA` function handles notations used by Berkeley + # software which includes 3 part IP Addresses such as `10.1.2`. That's why + # the above check is needed to enforce more strict IP Address validation as + # used by the `inet_pton` function in Unix. + # See the following: + # https://stackoverflow.com/a/29286098 + # Docs for the `inet_addr` function on MSDN + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms738563.aspx addr = sockaddr() addr.sa_family = address_family addr_size = ctypes.c_int(ctypes.sizeof(addr)) From e6a9563d472912ebad8eaa658ee2fda1e1401936 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Tue, 11 Jul 2017 10:19:22 -0600 Subject: [PATCH 065/508] simple doc updates --- salt/fileserver/svnfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/fileserver/svnfs.py b/salt/fileserver/svnfs.py index fc17b979b7..5dd1ccc10d 100644 --- a/salt/fileserver/svnfs.py +++ b/salt/fileserver/svnfs.py @@ -2,7 +2,7 @@ ''' Subversion Fileserver Backend -After enabling this backend, branches, and tags in a remote subversion +After enabling this backend, branches and tags in a remote subversion repository are exposed to salt as different environments. To enable this backend, add ``svn`` to the :conf_master:`fileserver_backend` option in the Master config file. @@ -694,7 +694,7 @@ def file_hash(load, fnd): def _file_lists(load, form): ''' - Return a dict containing the file lists for files, dirs, emtydirs and symlinks + Return a dict containing the file lists for files, dirs, emptydirs and symlinks ''' if 'env' in load: salt.utils.warn_until( From 53e25760beb3a504e9451a0826166c41f631a553 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 11 Jul 2017 10:17:51 -0600 Subject: [PATCH 066/508] Only use unassociated ips when unable to allocate When checking the unassociated ips first, in Parallel mode, each server will use the same one unallocated ip. --- salt/cloud/clouds/openstack.py | 55 ++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index 5eaff82c7e..e73af9e0e5 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -135,6 +135,14 @@ Alternatively, one could use the private IP to connect by specifying: ssh_interface: private_ips +.. note:: + + When using floating ips from networks, if the OpenStack driver is unable to + allocate a new ip address for the server, it will check that for + unassociated ip addresses in the floating ip pool. If SaltCloud is running + in parallel mode, it is possible that more than one server will attempt to + use the same ip address. + ''' # Import python libs @@ -866,40 +874,43 @@ def _assign_floating_ips(vm_, conn, kwargs): pool = OpenStack_1_1_FloatingIpPool( net['floating'], conn.connection ) - for idx in pool.list_floating_ips(): - if idx.node_id is None: - floating.append(idx) + try: + floating.append(pool.create_floating_ip()) + except Exception as e: + log.debug('Cannot allocate IP from floating pool \'%s\'. Checking for unassociated ips.', + net['floating']) + for idx in pool.list_floating_ips(): + if idx.node_id is None: + floating.append(idx) + break if not floating: - try: - floating.append(pool.create_floating_ip()) - except Exception as e: - raise SaltCloudSystemExit( - 'Floating pool \'{0}\' does not have any more ' - 'please create some more or use a different ' - 'pool.'.format(net['floating']) - ) + raise SaltCloudSystemExit( + 'There are no more floating IP addresses ' + 'available, please create some more' + ) # otherwise, attempt to obtain list without specifying pool # this is the same as 'nova floating-ip-list' elif ssh_interface(vm_) != 'private_ips': try: # This try/except is here because it appears some - # *cough* Rackspace *cough* # OpenStack providers return a 404 Not Found for the # floating ip pool URL if there are no pools setup pool = OpenStack_1_1_FloatingIpPool( '', conn.connection ) - for idx in pool.list_floating_ips(): - if idx.node_id is None: - floating.append(idx) + try: + floating.append(pool.create_floating_ip()) + except Exception as e: + log.debug('Cannot allocate IP from the default floating pool. Checking for unassociated ips.') + for idx in pool.list_floating_ips(): + if idx.node_id is None: + floating.append(idx) + break if not floating: - try: - floating.append(pool.create_floating_ip()) - except Exception as e: - raise SaltCloudSystemExit( - 'There are no more floating IP addresses ' - 'available, please create some more' - ) + raise SaltCloudSystemExit( + 'There are no more floating IP addresses ' + 'available, please create some more' + ) except Exception as e: if str(e).startswith('404'): pass From acc0345bc87be0e4f619bcc9b78601ef0ac601f0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 12:08:13 -0600 Subject: [PATCH 067/508] Fix unit tests --- salt/modules/win_system.py | 98 +++++++++++++----- tests/unit/modules/test_win_system.py | 143 +++++++++++++------------- 2 files changed, 145 insertions(+), 96 deletions(-) diff --git a/salt/modules/win_system.py b/salt/modules/win_system.py index a92f0923e5..993ab39830 100644 --- a/salt/modules/win_system.py +++ b/salt/modules/win_system.py @@ -37,6 +37,7 @@ except ImportError: import salt.utils import salt.utils.locales import salt.ext.six as six +from salt.exceptions import CommandExecutionError # Set up logging log = logging.getLogger(__name__) @@ -622,7 +623,11 @@ def join_domain(domain, .. versionadded:: 2015.8.2/2015.5.7 Returns: - dict: Dictionary if successful, otherwise False + dict: Dictionary if successful + + Raises: + CommandExecutionError: Raises an error if _join_domain returns anything + other than 0 CLI Example: @@ -655,6 +660,56 @@ def join_domain(domain, account_ou = account_ou.split('\\') account_ou = ''.join(account_ou) + err = _join_domain(domain=domain, username=username, password=password, + account_ou=account_ou, account_exists=account_exists) + + if not err: + ret = {'Domain': domain, + 'Restart': False} + if restart: + ret['Restart'] = reboot() + return ret + + raise CommandExecutionError(win32api.FormatMessage(err).rstrip()) + + +def _join_domain(domain, + username=None, + password=None, + account_ou=None, + account_exists=False): + ''' + Helper function to join the domain. + + Args: + domain (str): The domain to which the computer should be joined, e.g. + ``example.com`` + + username (str): Username of an account which is authorized to join + computers to the specified domain. Need to be either fully qualified + like ``user@domain.tld`` or simply ``user`` + + password (str): Password of the specified user + + account_ou (str): The DN of the OU below which the account for this + computer should be created when joining the domain, e.g. + ``ou=computers,ou=departm_432,dc=my-company,dc=com`` + + account_exists (bool): If set to ``True`` the computer will only join + the domain if the account already exists. If set to ``False`` the + computer account will be created if it does not exist, otherwise it + will use the existing account. Default is False. + + Returns: + int: + + :param domain: + :param username: + :param password: + :param account_ou: + :param account_exists: + :return: + ''' NETSETUP_JOIN_DOMAIN = 0x1 # pylint: disable=invalid-name NETSETUP_ACCOUNT_CREATE = 0x2 # pylint: disable=invalid-name NETSETUP_DOMAIN_JOIN_IF_JOINED = 0x20 # pylint: disable=invalid-name @@ -670,23 +725,13 @@ def join_domain(domain, pythoncom.CoInitialize() conn = wmi.WMI() comp = conn.Win32_ComputerSystem()[0] - err = comp.JoinDomainOrWorkgroup(Name=domain, - Password=password, - UserName=username, - AccountOU=account_ou, - FJoinOptions=join_options) - # you have to do this because JoinDomainOrWorkgroup returns a strangely - # formatted value that looks like (0,) - if not err[0]: - ret = {'Domain': domain, - 'Restart': False} - if restart: - ret['Restart'] = reboot() - return ret - - log.error(win32api.FormatMessage(err[0]).rstrip()) - return False + # Return the results of the command as an error + # JoinDomainOrWorkgroup returns a strangely formatted value that looks like + # (0,) so return the first item + return comp.JoinDomainOrWorkgroup( + Name=domain, Password=password, UserName=username, AccountOU=account_ou, + FJoinOptions=join_options)[0] def unjoin_domain(username=None, @@ -919,7 +964,11 @@ def set_system_date_time(years=None, seconds (int): Seconds digit: 0 - 59 Returns: - bool: True if successful, otherwise False. + bool: True if successful + + Raises: + CommandExecutionError: Raises an error if ``SetLocalTime`` function + fails CLI Example: @@ -972,12 +1021,15 @@ def set_system_date_time(years=None, system_time.wSecond = int(seconds) system_time_ptr = ctypes.pointer(system_time) succeeded = ctypes.windll.kernel32.SetLocalTime(system_time_ptr) - return succeeded is not 0 - except OSError: + if succeeded is not 0: + return True + else: + log.error('Failed to set local time') + raise CommandExecutionError( + win32api.FormatMessage(succeeded).rstrip()) + except OSError as err: log.error('Failed to set local time') - return False - - return True + raise CommandExecutionError(err) def get_system_date(): diff --git a/tests/unit/modules/test_win_system.py b/tests/unit/modules/test_win_system.py index 3506a4ddf6..8b8a05356f 100644 --- a/tests/unit/modules/test_win_system.py +++ b/tests/unit/modules/test_win_system.py @@ -67,30 +67,31 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to reboot the system ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(), True) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_timeout_in_minutes(self): ''' Test to reboot the system with a timeout ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(5, in_seconds=False), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(5, in_seconds=False), True) + shutdown.assert_called_with(timeout=5, in_seconds=False, reboot=True, + only_on_pending_reboot=False) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_timeout_in_seconds(self): ''' Test to reboot the system with a timeout ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(5, in_seconds=True), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '5'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(5, in_seconds=True), True) + shutdown.assert_called_with(timeout=5, in_seconds=True, reboot=True, + only_on_pending_reboot=False) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_wait(self): @@ -98,50 +99,49 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): Test to reboot the system with a timeout and wait for it to finish ''' - mock = MagicMock(return_value='salt') - sleep_mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - with patch('time.sleep', sleep_mock): - self.assertEqual(win_system.reboot(wait_for_reboot=True), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) - sleep_mock.assert_called_once_with(330) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)), \ + patch('salt.modules.win_system.time.sleep', + MagicMock()) as time: + self.assertEqual(win_system.reboot(wait_for_reboot=True), True) + time.assert_called_with(330) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_shutdown(self): ''' Test to shutdown a running system ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.shutdown(), 'salt') + with patch('salt.modules.win_system.win32api.InitiateSystemShutdown', + MagicMock()): + self.assertEqual(win_system.shutdown(), True) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_shutdown_hard(self): ''' Test to shutdown a running system with no timeout or warning ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.shutdown_hard(), 'salt') + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.shutdown_hard(), True) + shutdown.assert_called_with(timeout=0) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_set_computer_name(self): ''' Test to set the Windows computer name ''' - mock = MagicMock(side_effect=[{'Computer Name': {'Current': ""}, - 'ReturnValue = 0;': True}, - {'Computer Name': {'Current': 'salt'}}]) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - mock = MagicMock(return_value='salt') - with patch.object(win_system, 'get_computer_name', mock): - mock = MagicMock(return_value=True) - with patch.object(win_system, - 'get_pending_computer_name', mock): - self.assertDictEqual(win_system.set_computer_name("salt"), + with patch('salt.modules.win_system.windll.kernel32.SetComputerNameExW', + MagicMock(return_value=True)): + with patch.object(win_system, 'get_computer_name', + MagicMock(return_value='salt')): + with patch.object(win_system, 'get_pending_computer_name', + MagicMock(return_value='salt_new')): + self.assertDictEqual(win_system.set_computer_name("salt_new"), {'Computer Name': {'Current': 'salt', - 'Pending': True}}) - + 'Pending': 'salt_new'}}) + # Test set_computer_name failure + with patch('salt.modules.win_system.windll.kernel32.SetComputerNameExW', + MagicMock(return_value=False)): self.assertFalse(win_system.set_computer_name("salt")) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') @@ -149,25 +149,25 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to get a pending computer name. ''' - mock = MagicMock(return_value='salt') - with patch.object(win_system, 'get_computer_name', mock): - mock = MagicMock(side_effect=['salt0', - 'ComputerName REG_SZ (salt)']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): + with patch.object(win_system, 'get_computer_name', + MagicMock(return_value='salt')): + reg_mock = MagicMock(return_value={'vdata': 'salt'}) + with patch.dict(win_system.__salt__, {'reg.read_value': reg_mock}): self.assertFalse(win_system.get_pending_computer_name()) + reg_mock = MagicMock(return_value={'vdata': 'salt_pending'}) + with patch.dict(win_system.__salt__, {'reg.read_value': reg_mock}): self.assertEqual(win_system.get_pending_computer_name(), - '(salt)') + 'salt_pending') @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_get_computer_name(self): ''' Test to get the Windows computer name ''' - mock = MagicMock(side_effect=['Server Name Salt', 'Salt']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.get_computer_name(), 'Salt') - + with patch('salt.modules.win_system.win32api.GetComputerNameEx', + MagicMock(side_effect=['computer name', ''])): + self.assertEqual(win_system.get_computer_name(), 'computer name') self.assertFalse(win_system.get_computer_name()) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') @@ -189,10 +189,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to get the Windows computer description ''' - mock = MagicMock(side_effect=['Server Comment Salt', 'Salt']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.get_computer_desc(), 'Salt') - + with patch('salt.modules.win_system.get_system_info', + MagicMock(side_effect=[{'description': 'salt description'}, + {'description': None}])): + self.assertEqual(win_system.get_computer_desc(), 'salt description') self.assertFalse(win_system.get_computer_desc()) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs w32net and other windows libraries') @@ -200,17 +200,20 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to join a computer to an Active Directory domain ''' - mock = MagicMock(side_effect=[{'ReturnValue = 0;': True}, - {'Salt': True}]) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertDictEqual(win_system.join_domain("saltstack", - "salt", - "salt@123"), - {'Domain': 'saltstack'}) + with patch('salt.modules.win_system._join_domain', + MagicMock(return_value=0)): + with patch('salt.modules.win_system.get_domain_workgroup', + MagicMock(return_value={'Workgroup': 'Workgroup'})): + self.assertDictEqual( + win_system.join_domain( + "saltstack", "salt", "salt@123"), + {'Domain': 'saltstack', 'Restart': False}) - self.assertFalse(win_system.join_domain("saltstack", - "salt", - "salt@123")) + with patch('salt.modules.win_system.get_domain_workgroup', + MagicMock(return_value={'Domain': 'saltstack'})): + self.assertEqual( + win_system.join_domain("saltstack", "salt", "salt@123"), + 'Already joined to saltstack') def test_get_system_time(self): ''' @@ -230,13 +233,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to set system time ''' - mock = MagicMock(side_effect=[False, True]) - with patch.object(win_system, '_validate_time', mock): + with patch('salt.modules.win_system.set_system_date_time', + MagicMock(side_effect=[False, True])): self.assertFalse(win_system.set_system_time("11:31:15 AM")) - - mock = MagicMock(return_value=True) - with patch.dict(win_system.__salt__, {'cmd.retcode': mock}): - self.assertFalse(win_system.set_system_time("11:31:15 AM")) + self.assertTrue(win_system.set_system_time("11:31:15 AM")) def test_get_system_date(self): ''' @@ -250,13 +250,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to set system date ''' - mock = MagicMock(side_effect=[False, True]) - with patch.object(win_system, '_validate_date', mock): + with patch('salt.modules.win_system.set_system_date_time', + MagicMock(side_effect=[False, True])): self.assertFalse(win_system.set_system_date("03-28-13")) - - mock = MagicMock(return_value=True) - with patch.dict(win_system.__salt__, {'cmd.retcode': mock}): - self.assertFalse(win_system.set_system_date("03-28-13")) + self.assertTrue(win_system.set_system_date("03-28-13")) def test_start_time_service(self): ''' From 45be32666a78d82dfc174accf102d608ae0e172f Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 13:08:00 -0600 Subject: [PATCH 068/508] Add error-handling function to shutil.rmtree --- tests/unit/pillar/test_git.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/pillar/test_git.py b/tests/unit/pillar/test_git.py index 7c55175416..f46011d6da 100644 --- a/tests/unit/pillar/test_git.py +++ b/tests/unit/pillar/test_git.py @@ -16,6 +16,7 @@ import tempfile import shutil import subprocess import yaml +import stat # Import Salt Testing libs from tests.integration import AdaptedConfigurationTestCaseMixin @@ -72,9 +73,13 @@ class GitPillarTestCase(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModul git_pillar._update('master', 'file://{0}'.format(self.repo_path)) def tearDown(self): - shutil.rmtree(self.tmpdir) + shutil.rmtree(self.tmpdir, onerror=self._rmtree_error) super(GitPillarTestCase, self).tearDown() + def _rmtree_error(self, func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + def _create_repo(self): 'create source Git repo in temp directory' repo = os.path.join(self.tmpdir, 'repo_pillar') From 55b278c478cb2e1925e6ef417f3b2104112dae8f Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 13:23:11 -0600 Subject: [PATCH 069/508] Mock the reg.read_value function --- tests/unit/states/test_environ.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/states/test_environ.py b/tests/unit/states/test_environ.py index 28c35c525f..45a8e5fb1a 100644 --- a/tests/unit/states/test_environ.py +++ b/tests/unit/states/test_environ.py @@ -96,11 +96,14 @@ class TestEnvironState(TestCase, LoaderModuleMockMixin): ret = envstate.setenv('notimportant', {'foo': 'bar'}) self.assertEqual(ret['changes'], {'foo': 'bar'}) - ret = envstate.setenv('notimportant', {'test': False, 'foo': 'baz'}, false_unsets=True) + with patch.dict(envstate.__salt__, {'reg.read_value': MagicMock()}): + ret = envstate.setenv( + 'notimportant', {'test': False, 'foo': 'baz'}, false_unsets=True) self.assertEqual(ret['changes'], {'test': None, 'foo': 'baz'}) self.assertEqual(envstate.os.environ, {'INITIAL': 'initial', 'foo': 'baz'}) - ret = envstate.setenv('notimportant', {'test': False, 'foo': 'bax'}) + with patch.dict(envstate.__salt__, {'reg.read_value': MagicMock()}): + ret = envstate.setenv('notimportant', {'test': False, 'foo': 'bax'}) self.assertEqual(ret['changes'], {'test': '', 'foo': 'bax'}) self.assertEqual(envstate.os.environ, {'INITIAL': 'initial', 'foo': 'bax', 'test': ''}) From 8c76bbb53d3ce4b3b8eb384f75309a9e793c7ece Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 15:57:32 -0600 Subject: [PATCH 070/508] Some minor doc fixes for dnsutil module so they'll render correctly --- salt/modules/dnsutil.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/modules/dnsutil.py b/salt/modules/dnsutil.py index 948cd720ce..390fbef9b9 100644 --- a/salt/modules/dnsutil.py +++ b/salt/modules/dnsutil.py @@ -2,8 +2,9 @@ ''' Compendium of generic DNS utilities. -.. note: - Some functions in the `dnsutil` execution module depend on `dig`. +.. note:: + + Some functions in the ``dnsutil`` execution module depend on ``dig``. ''' from __future__ import absolute_import @@ -245,7 +246,7 @@ def check_ip(ip_addr): def A(host, nameserver=None): ''' - Return the A record(s) for `host`. + Return the A record(s) for ``host``. Always returns a list. @@ -270,7 +271,7 @@ def A(host, nameserver=None): def AAAA(host, nameserver=None): ''' - Return the AAAA record(s) for `host`. + Return the AAAA record(s) for ``host``. Always returns a list. From c31ded341cf8066a4c70cd2b898e1a7c7ce5d672 Mon Sep 17 00:00:00 2001 From: Thomas Dutrion Date: Sun, 9 Jul 2017 11:34:14 +0200 Subject: [PATCH 071/508] Remove duplicate instruction in Openstack Rackspace config example --- doc/topics/cloud/config.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/topics/cloud/config.rst b/doc/topics/cloud/config.rst index a1739bc701..d7d3ca821e 100644 --- a/doc/topics/cloud/config.rst +++ b/doc/topics/cloud/config.rst @@ -371,7 +371,6 @@ both. compute_name: cloudServersOpenStack protocol: ipv4 compute_region: DFW - protocol: ipv4 user: myuser tenant: 5555555 password: mypass From 30d62f43da5f3b5f824bb763035b5180bcde2201 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 16:24:29 -0600 Subject: [PATCH 072/508] Update minion restart section in FAQ doc for windows Fixes #41116 --- doc/faq.rst | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 8237417816..9911679bb0 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -319,7 +319,27 @@ Restart using states ******************** Now we can apply the workaround to restart the Minion in reliable way. -The following example works on both UNIX-like and Windows operating systems: +The following example works on UNIX-like operating systems: + +.. code-block:: jinja + + {%- if grains['os'] != 'Windows' % + Restart Salt Minion: + cmd.run: + - name: 'salt-call --local service.restart salt-minion' + - bg: True + - onchanges: + - pkg: Upgrade Salt Minion + {%- endif %} + +Note that restarting the salt-minion service on Windows operating systems is +not always necessary when performing an upgrade. The installer stops the +``salt-minion`` service, removes it, deletes the contents of the ``\salt\bin`` +directory, installs the new code, re-creates the ``salt-minion`` service, and +starts it (by default). The restart step **would** be necessary during the +upgrade process, however, if the minion config was edited after the upgrade or +installation. If a minion restart is necessary, the state above can be edited +as follows: .. code-block:: jinja @@ -335,8 +355,8 @@ The following example works on both UNIX-like and Windows operating systems: - pkg: Upgrade Salt Minion However, it requires more advanced tricks to upgrade from legacy version of -Salt (before ``2016.3.0``), where executing commands in the background is not -supported: +Salt (before ``2016.3.0``) on UNIX-like operating systems, where executing +commands in the background is not supported: .. code-block:: jinja From 0c484f89795a5479a80fe683eb5202aa6a170323 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 16:55:43 -0600 Subject: [PATCH 073/508] Fix unit tests on Windows --- salt/modules/win_file.py | 3 - tests/unit/states/test_file.py | 104 +++++++++++++++++++++++++-------- 2 files changed, 81 insertions(+), 26 deletions(-) diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index 12d9319031..461fb5310d 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -1687,9 +1687,6 @@ def check_perms(path, perms = changes[user]['perms'] try: - log.debug('*' * 68) - log.debug(perms) - log.debug('*' * 68) salt.utils.win_dacl.set_permissions( path, user, perms, 'deny', applies_to) ret['changes']['deny_perms'][user] = changes[user] diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index dc58ca2500..2ccce34c29 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -100,7 +100,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): def test_contents_pillar_doesnt_add_more_newlines(self): # make sure the newline - pillar_value = 'i am the pillar value\n' + pillar_value = 'i am the pillar value{0}'.format(os.linesep) self.run_contents_pillar(pillar_value, expected=pillar_value) @@ -167,7 +167,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_empty, - 'file.group_to_gid': mock_empty}): + 'file.group_to_gid': mock_empty, + 'user.info': mock_empty, + 'user.current': mock_user}): comt = ('User {0} does not exist. Group {1} does not exist.'.format(user, group)) ret.update({'comment': comt, 'name': name}) self.assertDictEqual(filestate.symlink(name, target, user=user, @@ -176,7 +178,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, - 'file.is_link': mock_f}): + 'file.is_link': mock_f, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': True}): with patch.object(os.path, 'exists', mock_f): comt = ('Symlink {0} to {1}' @@ -191,7 +195,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, - 'file.is_link': mock_f}): + 'file.is_link': mock_f, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_f): with patch.object(os.path, 'exists', mock_f): @@ -207,7 +213,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_t, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(salt.states.file, '_check_symlink_ownership', mock_t): @@ -224,7 +232,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_f, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(os.path, 'exists', mock_f): @@ -243,7 +253,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_f, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(os.path, 'exists', mock_f): @@ -538,7 +550,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'G12', 'G12', 'G12', 'G12', 'G12']) mock_if = MagicMock(side_effect=[True, False, False, False, False, False, False, False]) - mock_ret = MagicMock(return_value=(ret, None)) + mock_ret = MagicMock(return_value=ret) mock_dict = MagicMock(return_value={}) mock_cp = MagicMock(side_effect=[Exception, True]) mock_ex = MagicMock(side_effect=[Exception, {'changes': {name: name}}, @@ -573,8 +585,14 @@ class TestFileState(TestCase, LoaderModuleMockMixin): self.assertDictEqual(filestate.managed(name, create=False), ret) - comt = ('User salt is not available Group saltstack' - ' is not available') + # Group argument is ignored on Windows systems. Group is set to + # user + if salt.utils.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt, 'result': False}) self.assertDictEqual(filestate.managed(name, user=user, group=group), ret) @@ -689,7 +707,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' Test to ensure that a named directory is present and has the right perms ''' - name = '/etc/grub.conf' + if salt.utils.is_windows(): + name = 'C:\\Windows\\temp\\grub_test' + else: + name = '/etc/grub.conf' user = 'salt' group = 'saltstack' @@ -711,17 +732,28 @@ class TestFileState(TestCase, LoaderModuleMockMixin): mock_t = MagicMock(return_value=True) mock_f = MagicMock(return_value=False) - mock_perms = MagicMock(return_value=(ret, '')) + if salt.utils.is_windows(): + mock_perms = MagicMock(return_value=ret) + else: + mock_perms = MagicMock(return_value=(ret, '')) mock_uid = MagicMock(side_effect=['', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12']) mock_gid = MagicMock(side_effect=['', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12']) + mock_check = MagicMock(return_value=( + None, + 'The directory "{0}" will be changed'.format(name), + {'directory': 'new'})) + mock_error = CommandExecutionError with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.stats': mock_f, 'file.check_perms': mock_perms, - 'file.mkdir': mock_t}): + 'file.mkdir': mock_t}), \ + patch('salt.utils.win_dacl.get_sid', mock_error), \ + patch('os.path.isdir', mock_t), \ + patch('salt.states.file._check_directory_win', mock_check): if salt.utils.is_windows(): comt = ('User salt is not available Group salt' ' is not available') @@ -729,8 +761,8 @@ class TestFileState(TestCase, LoaderModuleMockMixin): comt = ('User salt is not available Group saltstack' ' is not available') ret.update({'comment': comt, 'name': name}) - self.assertDictEqual(filestate.directory(name, user=user, - group=group), ret) + self.assertDictEqual( + filestate.directory(name, user=user, group=group), ret) with patch.object(os.path, 'isabs', mock_f): comt = ('Specified file {0} is not an absolute path' @@ -771,9 +803,19 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.object(os.path, 'isfile', mock_f): with patch.dict(filestate.__opts__, {'test': True}): - comt = ('The following files will be changed:\n{0}:' - ' directory - new\n'.format(name)) - ret.update({'comment': comt, 'result': None, 'pchanges': {'/etc/grub.conf': {'directory': 'new'}}}) + if salt.utils.is_windows(): + comt = 'The directory "{0}" will be changed' \ + ''.format(name) + p_chg = {'directory': 'new'} + ret.update({ + 'comment': comt, + 'result': None, + 'pchanges': p_chg + }) + else: + comt = ('The following files will be changed:\n{0}:' + ' directory - new\n'.format(name)) + ret.update({'comment': comt, 'result': None, 'pchanges': {'/etc/grub.conf': {'directory': 'new'}}}) self.assertDictEqual(filestate.directory(name, user=user, group=group), @@ -845,8 +887,14 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.source_list': mock_lst, 'cp.list_master_dirs': mock_emt, 'cp.list_master': mock_l}): - comt = ('User salt is not available Group saltstack' - ' is not available') + + # Group argument is ignored on Windows systems. Group is set to user + if salt.utils.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt}) self.assertDictEqual(filestate.recurse(name, source, user=user, group=group), ret) @@ -935,7 +983,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' with patch('salt.states.file._load_accumulators', MagicMock(return_value=([], []))): - name = '/etc/hosts' + if salt.utils.is_windows(): + name = 'C:\\Windows\\System32\\drivers\\etc\\hosts' + else: + name = '/etc/hosts' ret = {'name': name, 'result': False, @@ -1312,8 +1363,15 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.get_group': mock_grp, 'file.get_mode': mock_grp, 'file.check_perms': mock_t}): - comt = ('User salt is not available Group ' - 'saltstack is not available') + + # Group argument is ignored on Windows systems. Group is set + # to user + if salt.utils.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt, 'result': False}) self.assertDictEqual(filestate.copy(name, source, user=user, group=group), ret) From a4231c9827acb5a94cf29c9ada21346558cadd3b Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 16:59:42 -0600 Subject: [PATCH 074/508] Fix ret mock for linux --- tests/unit/states/test_file.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index 2ccce34c29..0d190d48e6 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -550,7 +550,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'G12', 'G12', 'G12', 'G12', 'G12']) mock_if = MagicMock(side_effect=[True, False, False, False, False, False, False, False]) - mock_ret = MagicMock(return_value=ret) + if salt.utils.is_windows(): + mock_ret = MagicMock(return_value=ret) + else: + mock_ret = MagicMock(return_value=(ret, None)) mock_dict = MagicMock(return_value={}) mock_cp = MagicMock(side_effect=[Exception, True]) mock_ex = MagicMock(side_effect=[Exception, {'changes': {name: name}}, From 78cdee51d5559b1764ef857b40bcd7c45a26d799 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 17:00:50 -0600 Subject: [PATCH 075/508] Gate boto_elb tests if proper version of moto isn't installed For Python 2 tests, we can use an older version. But when running these tests of Python 3, we need a newer version of moto that supports Python 3. This gates the tests if the expected version of moto is missing. --- tests/unit/modules/test_boto_elb.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/modules/test_boto_elb.py b/tests/unit/modules/test_boto_elb.py index 5d1ee5a9d1..4582155c6f 100644 --- a/tests/unit/modules/test_boto_elb.py +++ b/tests/unit/modules/test_boto_elb.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import logging from copy import deepcopy +import pkg_resources # import Python Third Party Libs # pylint: disable=import-error @@ -45,8 +46,10 @@ except ImportError: # Import Salt Libs import salt.config +import salt.ext.six as six import salt.loader import salt.modules.boto_elb as boto_elb +import salt.utils.versions # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -63,11 +66,32 @@ conn_parameters = {'region': region, 'key': access_key, 'keyid': secret_key, boto_conn_parameters = {'aws_access_key_id': access_key, 'aws_secret_access_key': secret_key} instance_parameters = {'instance_type': 't1.micro'} +required_moto = '0.3.7' +required_moto_py3 = '1.0.1' + + +def _has_required_moto(): + ''' + Returns True or False depending on if ``moto`` is installed and at the correct version, + depending on what version of Python is running these tests. + ''' + if not HAS_MOTO: + return False + else: + moto_version = salt.utils.versions.LooseVersion(pkg_resources.get_distribution('moto').version) + if moto_version < required_moto: + return False + elif six.PY3 and moto_version < required_moto_py3: + return False + + return True @skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(HAS_BOTO is False, 'The boto module must be installed.') @skipIf(HAS_MOTO is False, 'The moto module must be installed.') +@skipIf(_has_required_moto() is False, 'The moto module must be >= to {0} for ' + 'PY2 or {1} for PY3.'.format(required_moto, required_moto_py3)) class BotoElbTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for salt.modules.boto_elb module From 669aaee10d746176e977e4ea16a4c51159cc1768 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 17:09:21 -0600 Subject: [PATCH 076/508] Mock file exists properly --- tests/unit/states/test_file.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index 0d190d48e6..e106423c81 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -710,10 +710,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' Test to ensure that a named directory is present and has the right perms ''' - if salt.utils.is_windows(): - name = 'C:\\Windows\\temp\\grub_test' - else: - name = '/etc/grub.conf' + name = '/etc/grub.conf' user = 'salt' group = 'saltstack' @@ -810,15 +807,15 @@ class TestFileState(TestCase, LoaderModuleMockMixin): comt = 'The directory "{0}" will be changed' \ ''.format(name) p_chg = {'directory': 'new'} - ret.update({ - 'comment': comt, - 'result': None, - 'pchanges': p_chg - }) else: comt = ('The following files will be changed:\n{0}:' ' directory - new\n'.format(name)) - ret.update({'comment': comt, 'result': None, 'pchanges': {'/etc/grub.conf': {'directory': 'new'}}}) + p_chg = {'/etc/grub.conf': {'directory': 'new'}} + ret.update({ + 'comment': comt, + 'result': None, + 'pchanges': p_chg + }) self.assertDictEqual(filestate.directory(name, user=user, group=group), @@ -986,10 +983,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' with patch('salt.states.file._load_accumulators', MagicMock(return_value=([], []))): - if salt.utils.is_windows(): - name = 'C:\\Windows\\System32\\drivers\\etc\\hosts' - else: - name = '/etc/hosts' + name = '/etc/hosts' ret = {'name': name, 'result': False, @@ -1008,7 +1002,8 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ret.update({'comment': comt, 'name': name}) self.assertDictEqual(filestate.blockreplace(name), ret) - with patch.object(os.path, 'isabs', mock_t): + with patch.object(os.path, 'isabs', mock_t), \ + patch.object(os.path, 'exists', mock_t): with patch.dict(filestate.__salt__, {'file.blockreplace': mock_t}): with patch.dict(filestate.__opts__, {'test': True}): comt = ('Changes would be made') From 38d9b3d553a424d8163d86db9b0a3e7e32c8e19a Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 17:28:45 -0600 Subject: [PATCH 077/508] Add some clarity to "multiple quotes" section of yaml docs Fixes #41721 --- .../troubleshooting/yaml_idiosyncrasies.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index e1c6383459..bd48691669 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -28,6 +28,7 @@ hit `Enter`. Also, you can convert tabs to 2 spaces by these commands in Vim: Indentation =========== + The suggested syntax for YAML files is to use 2 spaces for indentation, but YAML will follow whatever indentation system that the individual file uses. Indentation of two spaces works very well for SLS files given the @@ -112,8 +113,19 @@ PyYAML will load these values as boolean ``True`` or ``False``. Un-capitalized versions will also be loaded as booleans (``true``, ``false``, ``yes``, ``no``, ``on``, and ``off``). This can be especially problematic when constructing Pillar data. Make sure that your Pillars which need to use the string versions -of these values are enclosed in quotes. Pillars will be parsed twice by salt, -so you'll need to wrap your values in multiple quotes, for example '"false"'. +of these values are enclosed in quotes. Pillars will be parsed twice by salt, +so you'll need to wrap your values in multiple quotes, including double quotation +marks (``" "``) and single quotation marks (``' '``). note that spaces are included +in the quotation type examples for clarity. + +Multiple quoting examples looks like this: + +.. code-block:: yaml + + - '"false"' + - "'True'" + - "'YES'" + - '"No"' The '%' Sign ============ From d1f2a93368f8e06986ce04efe2a6238bb1a8ce51 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Wed, 12 Jul 2017 11:03:34 +0300 Subject: [PATCH 078/508] DOCS: unify hash sum with hash type format --- salt/modules/file.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 9a0f68c7a6..6044296361 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -710,18 +710,21 @@ def check_hash(path, file_hash): hash The hash to check against the file specified in the ``path`` argument. - For versions 2016.11.4 and newer, the hash can be specified without an + + .. versionchanged:: 2016.11.4 + + For this and newer versions the hash can be specified without an accompanying hash type (e.g. ``e138491e9d5b97023cea823fe17bac22``), but for earlier releases it is necessary to also specify the hash type - in the format ``:`` (e.g. - ``md5:e138491e9d5b97023cea823fe17bac22``). + in the format ``=`` (e.g. + ``md5=e138491e9d5b97023cea823fe17bac22``). CLI Example: .. code-block:: bash salt '*' file.check_hash /etc/fstab e138491e9d5b97023cea823fe17bac22 - salt '*' file.check_hash /etc/fstab md5:e138491e9d5b97023cea823fe17bac22 + salt '*' file.check_hash /etc/fstab md5=e138491e9d5b97023cea823fe17bac22 ''' path = os.path.expanduser(path) From f1bc58f6d5a2aec896cf116745452943bec07797 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 6 Jul 2017 11:53:54 +0300 Subject: [PATCH 079/508] Utils: add example of module import --- doc/topics/utils/index.rst | 51 +++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/doc/topics/utils/index.rst b/doc/topics/utils/index.rst index 48728174f5..44380f3541 100644 --- a/doc/topics/utils/index.rst +++ b/doc/topics/utils/index.rst @@ -54,7 +54,7 @@ types like so: salt '*' mymodule.observe_the_awesomeness ''' - print __utils__['foo.bar']() + return __utils__['foo.bar']() Utility modules, like any other kind of Salt extension, support using a :ref:`__virtual__ function ` to conditionally load them, @@ -81,11 +81,56 @@ the ``foo`` utility module with a ``__virtual__`` function. def bar(): return 'baz' +Also you could even write your utility modules in object oriented fashion: + +.. code-block:: python + + # -*- coding: utf-8 -*- + ''' + My utils module + --------------- + + This module contains common functions for use in my other custom types. + ''' + + class Foo(object): + + def __init__(self): + pass + + def bar(self): + return 'baz' + +And import them into other custom modules: + +.. code-block:: python + + # -*- coding: utf-8 -*- + ''' + My awesome execution module + --------------------------- + ''' + + import mymodule + + def observe_the_awesomeness(): + ''' + Prints information from my utility module + + CLI Example: + + .. code-block:: bash + + salt '*' mymodule.observe_the_awesomeness + ''' + foo = mymodule.Foo() + return foo.bar() + These are, of course, contrived examples, but they should serve to show some of the possibilities opened up by writing utility modules. Keep in mind though -that States still have access to all of the execution modules, so it is not +that states still have access to all of the execution modules, so it is not necessary to write a utility module to make a function available to both a -state and an execution module. One good use case for utililty modules is one +state and an execution module. One good use case for utility modules is one where it is necessary to invoke the same function from a custom :ref:`outputter `/returner, as well as an execution module. From 6bb8b8f98cc7e37d08f0f81b4a52cbae1f5409ba Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 7 Jul 2017 18:54:04 +0300 Subject: [PATCH 080/508] Add missing doc for ``utils_dirs`` Minion config option --- doc/ref/configuration/minion.rst | 14 ++++++++++++++ salt/loader.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 6e5dea3ffa..29fbe01dd7 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -1272,6 +1272,20 @@ A list of extra directories to search for Salt renderers render_dirs: - /var/lib/salt/renderers +.. conf_minion:: utils_dirs + +``utils_dirs`` +--------------- + +Default: ``[]`` + +A list of extra directories to search for Salt utilities + +.. code-block:: yaml + + utils_dirs: + - /var/lib/salt/utils + .. conf_minion:: cython_enable ``cython_enable`` diff --git a/salt/loader.py b/salt/loader.py index 60bf78cc56..566597dca4 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -182,7 +182,7 @@ def minion_mods( generated modules in __context__ :param dict utils: Utility functions which should be made available to - Salt modules in __utils__. See `utils_dir` in + Salt modules in __utils__. See `utils_dirs` in salt.config for additional information about configuration. From 5f8f98a01fa2522ba4ef8fd329ec56618b4e2632 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Wed, 12 Jul 2017 18:01:55 +0300 Subject: [PATCH 081/508] Fix #38839: remove `state` from Reactor runner kwags --- salt/utils/reactor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/utils/reactor.py b/salt/utils/reactor.py index 4289f097b4..6d4010d450 100644 --- a/salt/utils/reactor.py +++ b/salt/utils/reactor.py @@ -278,6 +278,9 @@ class ReactWrap(object): # Update called function's low data with event user to # segregate events fired by reactor and avoid reaction loops kwargs['__user__'] = self.event_user + # Replace ``state`` kwarg which comes from high data compiler. + # It breaks some runner functions and seems unnecessary. + kwargs['__state__'] = kwargs.pop('state') l_fun(*f_call.get('args', ()), **kwargs) except Exception: From bd638880e3ceda8c01be33dbf59fa585ebbc2e4a Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 12 Jul 2017 09:36:07 -0600 Subject: [PATCH 082/508] Add mono-spacing to salt-minion reference for consistency --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index 9911679bb0..f2a8bace5a 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -332,7 +332,7 @@ The following example works on UNIX-like operating systems: - pkg: Upgrade Salt Minion {%- endif %} -Note that restarting the salt-minion service on Windows operating systems is +Note that restarting the ``salt-minion`` service on Windows operating systems is not always necessary when performing an upgrade. The installer stops the ``salt-minion`` service, removes it, deletes the contents of the ``\salt\bin`` directory, installs the new code, re-creates the ``salt-minion`` service, and From ed89cd0b93ffeee891e91862af47730a575a3c41 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 12 Jul 2017 10:13:26 -0600 Subject: [PATCH 083/508] Use os.sep for path seps --- tests/unit/states/test_winrepo.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/states/test_winrepo.py b/tests/unit/states/test_winrepo.py index a702bbd2de..d5e4321620 100644 --- a/tests/unit/states/test_winrepo.py +++ b/tests/unit/states/test_winrepo.py @@ -67,9 +67,8 @@ class WinrepoTestCase(TestCase, LoaderModuleMockMixin): 'changes': {}, 'result': False, 'comment': ''} - ret.update({'comment': - '{file_roots}/win/repo is ' - 'missing'.format(file_roots=BASE_FILE_ROOTS_DIR)}) + ret.update({'comment': '{0} is missing'.format( + os.sep.join([BASE_FILE_ROOTS_DIR, 'win', 'repo']))}) self.assertDictEqual(winrepo.genrepo('salt'), ret) mock = MagicMock(return_value={'winrepo_dir': 'salt', From f03222384319916b2917be3751ec20a3a1c6f577 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 12 Jul 2017 14:19:35 -0600 Subject: [PATCH 084/508] Handle libcloud objects that throw RepresenterErrors with --out=yaml Fixes #42152 --- salt/cloud/libcloudfuncs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/cloud/libcloudfuncs.py b/salt/cloud/libcloudfuncs.py index 3191251c99..f6465b8f66 100644 --- a/salt/cloud/libcloudfuncs.py +++ b/salt/cloud/libcloudfuncs.py @@ -150,7 +150,7 @@ def avail_locations(conn=None, call=None): ret[img_name] = {} for attr in dir(img): - if attr.startswith('_'): + if attr.startswith('_') or attr == 'driver': continue attr_value = getattr(img, attr) @@ -187,7 +187,7 @@ def avail_images(conn=None, call=None): ret[img_name] = {} for attr in dir(img): - if attr.startswith('_'): + if attr.startswith('_') or attr in ('driver', 'get_uuid'): continue attr_value = getattr(img, attr) if isinstance(attr_value, string_types) and not six.PY3: @@ -222,7 +222,7 @@ def avail_sizes(conn=None, call=None): ret[size_name] = {} for attr in dir(size): - if attr.startswith('_'): + if attr.startswith('_') or attr in ('driver', 'get_uuid'): continue try: From f2250d474acee9ba90e68963109fb7c7847da2ca Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 12 Jul 2017 17:08:23 -0600 Subject: [PATCH 085/508] Add a note about using different styles of quotes. --- doc/topics/troubleshooting/yaml_idiosyncrasies.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index bd48691669..31168afa13 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -115,7 +115,7 @@ versions will also be loaded as booleans (``true``, ``false``, ``yes``, ``no``, Pillar data. Make sure that your Pillars which need to use the string versions of these values are enclosed in quotes. Pillars will be parsed twice by salt, so you'll need to wrap your values in multiple quotes, including double quotation -marks (``" "``) and single quotation marks (``' '``). note that spaces are included +marks (``" "``) and single quotation marks (``' '``). Note that spaces are included in the quotation type examples for clarity. Multiple quoting examples looks like this: @@ -127,6 +127,11 @@ Multiple quoting examples looks like this: - "'YES'" - '"No"' +.. note:: + + When using multiple quotes in this manner, they must be different. Using ``"" ""`` + or ``'' ''`` won't work in this case (spaces are included in examples for clarity). + The '%' Sign ============ From 74689e34626a9c841b9656c51bb55165ec56193c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:12:24 +0200 Subject: [PATCH 086/508] Add ability to use tagged functions in the same set --- salt/states/module.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/states/module.py b/salt/states/module.py index 31a4355464..621c048d16 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -262,6 +262,7 @@ def run(**kwargs): missing = [] tests = [] for func in functions: + func = func.split(':')[0] if func not in __salt__: missing.append(func) elif __opts__['test']: @@ -284,8 +285,9 @@ def run(**kwargs): failures = [] success = [] for func in functions: + _func = func.split(':')[0] try: - func_ret = _call_function(func, returner=kwargs.get('returner'), + func_ret = _call_function(_func, returner=kwargs.get('returner'), func_args=kwargs.get(func)) if not _get_result(func_ret, ret['changes'].get('ret', {})): if isinstance(func_ret, dict): From ea8351362c5aa09f4c622f025e4ed79515a9d544 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:13:19 +0200 Subject: [PATCH 087/508] Bugfix: args gets ignored alongside named parameters --- salt/states/module.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/salt/states/module.py b/salt/states/module.py index 621c048d16..202999e7d8 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -315,22 +315,35 @@ def _call_function(name, returner=None, **kwargs): ''' argspec = salt.utils.args.get_function_argspec(__salt__[name]) func_kw = dict(zip(argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code - argspec.defaults or [])) - func_args = [] - for funcset in kwargs.get('func_args') or {}: - if isinstance(funcset, dict): - func_kw.update(funcset) + argspec.defaults or [])) + arg_type, na_type, kw_type = [], {}, False + for funcset in reversed(kwargs.get('func_args') or []): + if not isinstance(funcset, dict): + kw_type = True + if kw_type: + if isinstance(funcset, dict): + arg_type += funcset.values() + na_type.update(funcset) + else: + arg_type.append(funcset) else: - func_args.append(funcset) + func_kw.update(funcset) + arg_type.reverse() + _exp_prm = len(argspec.args or []) - len(argspec.defaults or []) + _passed_prm = len(arg_type) missing = [] - for arg in argspec.args: - if arg not in func_kw: - missing.append(arg) + if na_type and _exp_prm > _passed_prm: + for arg in argspec.args: + if arg not in func_kw: + missing.append(arg) if missing: raise SaltInvocationError('Missing arguments: {0}'.format(', '.join(missing))) + elif _exp_prm > _passed_prm: + raise SaltInvocationError('Function expects {0} parameters, got only {1}'.format( + _exp_prm, _passed_prm)) - mret = __salt__[name](*func_args, **func_kw) + mret = __salt__[name](*arg_type, **func_kw) if returner is not None: returners = salt.loader.returners(__opts__, __salt__) if returner in returners: From 94c97a8f254e44f1ce40e5919146c5762e419474 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:22:28 +0200 Subject: [PATCH 088/508] Update and correct the error message --- tests/unit/states/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py index 54f3768d37..208829273a 100644 --- a/tests/unit/states/test_module.py +++ b/tests/unit/states/test_module.py @@ -122,7 +122,7 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): ret = module.run(**{CMD: None}) - assert ret['comment'] == "'{0}' failed: Missing arguments: name".format(CMD) + assert ret['comment'] == "'{0}' failed: Function expects 1 parameters, got only 0".format(CMD) def test_run_correct_arg(self): ''' From 8c71257a4b77c5e3ac0a5dbe4f9c1e6c8c60245e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:22:46 +0200 Subject: [PATCH 089/508] Call unnamed parameters properly --- tests/unit/states/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py index 208829273a..4588e20ca8 100644 --- a/tests/unit/states/test_module.py +++ b/tests/unit/states/test_module.py @@ -131,7 +131,7 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): ''' with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): - ret = module.run(**{CMD: [{'name': 'Fred'}]}) + ret = module.run(**{CMD: ['Fred']}) assert ret['comment'] == '{0}: Success'.format(CMD) assert ret['result'] From 1391a05d5e9b4d2609d5fc5f818bf718615ecaf0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 18:04:05 +0200 Subject: [PATCH 090/508] Bugfix: syntax error in the example --- doc/topics/releases/2017.7.0.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 52e026e490..7e1d0d3a2d 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -124,13 +124,12 @@ State Module Changes # After run_something: module.run: - mymodule.something: + - mymodule.something: - name: some name - first_arg: one - second_arg: two - do_stuff: True - Since a lot of users are already using :py:func:`module.run ` states, this new behavior must currently be explicitly turned on, to allow users to take their time updating their SLS From 1d7233224b565725b4f87933ff1ce2ae0d25e878 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 18:04:42 +0200 Subject: [PATCH 091/508] Describe function batching --- doc/topics/releases/2017.7.0.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 7e1d0d3a2d..2cce0dbf1a 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -137,6 +137,19 @@ State Module Changes the next feature release of Salt (Oxygen) and the old usage will no longer be supported at that time. + Another feature of the new :py:func:`module.run ` is that + it allows to call many functions in a single batch, such as: + + .. code-block:: yaml + + run_something: + module.run: + - mymodule.function_without_parameters: + - mymodule.another_function: + - myparam + - my_other_param + + In a rare case you have a function that needs to be called several times but To enable the new behavior for :py:func:`module.run `, add the following to the minion config file: From 1e8a56eda5aec6e65b54b034efd6b6c28cd29428 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 18:04:49 +0200 Subject: [PATCH 092/508] Describe function tagging --- doc/topics/releases/2017.7.0.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 2cce0dbf1a..4acd3997b8 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -150,6 +150,23 @@ State Module Changes - my_other_param In a rare case you have a function that needs to be called several times but + with the different parameters, an additional feature of "tagging" is to the + rescue. In order to tag a function, use a colon delimeter. For example + + .. code-block:: yaml + + run_something: + module.run: + - mymodule.same_function:1: + - mymodule.same_function:2: + - myparam + - my_other_param + - mymodule.same_function:3: + - foo: bar + + The example above will run `mymodule.same_function` three times with the + different parameters. + To enable the new behavior for :py:func:`module.run `, add the following to the minion config file: @@ -157,6 +174,7 @@ State Module Changes use_superseded: - module.run + - The default for the ``fingerprint_hash_type`` option used in the ``present`` function in the :mod:`ssh ` state changed from ``md5`` to ``sha256``. From e38d432f906f99447b0beac523952df4199a6072 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 13 Jul 2017 09:19:58 +0200 Subject: [PATCH 093/508] Fix docs --- doc/topics/releases/2017.7.0.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 4acd3997b8..cbda765fd4 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -138,7 +138,7 @@ State Module Changes supported at that time. Another feature of the new :py:func:`module.run ` is that - it allows to call many functions in a single batch, such as: + it allows calling many functions in a single batch, such as: .. code-block:: yaml @@ -149,9 +149,9 @@ State Module Changes - myparam - my_other_param - In a rare case you have a function that needs to be called several times but + In a rare case that you have a function that needs to be called several times but with the different parameters, an additional feature of "tagging" is to the - rescue. In order to tag a function, use a colon delimeter. For example + rescue. In order to tag a function, use a colon delimeter. For example: .. code-block:: yaml From 357dc22f0597b0473b094b4a23b9c5416546250e Mon Sep 17 00:00:00 2001 From: Nicolas Geniteau Date: Thu, 1 Jun 2017 11:39:52 +0200 Subject: [PATCH 094/508] Fix user creation with empty password Emptying the password was only done if the user already existed, not when creating. Signed-off-by: Nicolas Geniteau --- salt/states/user.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/salt/states/user.py b/salt/states/user.py index 57623b2e94..ef02f3db4d 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -126,12 +126,14 @@ def _changes(name, if shell and lusr['shell'] != shell: change['shell'] = shell if 'shadow.info' in __salt__ and 'shadow.default_hash' in __salt__: - if password: + if password and not empty_password: default_hash = __salt__['shadow.default_hash']() if lshad['passwd'] == default_hash \ or lshad['passwd'] != default_hash and enforce_password: if lshad['passwd'] != password: change['passwd'] = password + if empty_password and lshad['passwd'] != '': + change['empty_password'] = True if date and date is not 0 and lshad['lstchg'] != date: change['date'] = date if mindays and mindays is not 0 and lshad['min'] != mindays: @@ -444,9 +446,6 @@ def present(name, if gid_from_name: gid = __salt__['file.group_to_gid'](name) - if empty_password: - __salt__['shadow.del_password'](name) - changes = _changes(name, uid, gid, @@ -497,6 +496,12 @@ def present(name, if key == 'passwd' and not empty_password: __salt__['shadow.set_password'](name, password) continue + if key == 'passwd' and empty_password: + log.warning("No password will be set when empty_password=True") + continue + if key == 'empty_password' and val: + __salt__['shadow.del_password'](name) + continue if key == 'date': __salt__['shadow.set_date'](name, date) continue @@ -662,6 +667,14 @@ def present(name, ' {1}'.format(name, 'XXX-REDACTED-XXX') ret['result'] = False ret['changes']['password'] = 'XXX-REDACTED-XXX' + if empty_password and not password: + __salt__['shadow.del_password'](name) + spost = __salt__['shadow.info'](name) + if spost['passwd'] != '': + ret['comment'] = 'User {0} created but failed to ' \ + 'empty password'.format(name) + ret['result'] = False + ret['changes']['password'] = '' if date: __salt__['shadow.set_date'](name, date) spost = __salt__['shadow.info'](name) From 663874908aee77901378ee744e8079fde608d14c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 12 Jul 2017 10:01:35 -0500 Subject: [PATCH 095/508] pkg.installed: pack name/version into pkgs argument This allows a version of 'latest' to work when just a name and version is passed. --- salt/states/pkg.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 93c7286d9b..260aca1387 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -1248,6 +1248,15 @@ def installed( 'result': True, 'comment': 'No packages to install provided'} + # If just a name (and optionally a version) is passed, just pack them into + # the pkgs argument. + if name and not any((pkgs, sources)): + if version: + pkgs = [{name: version}] + version = None + else: + pkgs = [name] + kwargs['saltenv'] = __env__ refresh = salt.utils.pkg.check_refresh(__opts__, refresh) if not isinstance(pkg_verify, list): @@ -1414,7 +1423,7 @@ def installed( if salt.utils.is_freebsd(): force = True # Downgrades need to be forced. try: - pkg_ret = __salt__['pkg.install'](name, + pkg_ret = __salt__['pkg.install'](name=None, refresh=refresh, version=version, force=force, From 026ccf401a2fc1505392017899f4cc18ce26e07f Mon Sep 17 00:00:00 2001 From: Johannes Scholz Date: Fri, 14 Jul 2017 15:55:23 +0200 Subject: [PATCH 096/508] Force file removal on Windows. Fixes #42295 --- salt/states/file.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index b950acb173..026cd1a913 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -1040,7 +1040,10 @@ def absent(name): ret['comment'] = 'File {0} is set for removal'.format(name) return ret try: - __salt__['file.remove'](name) + if salt.utils.is_windows(): + __salt__['file.remove'](name, force=True) + else: + __salt__['file.remove'](name) ret['comment'] = 'Removed file {0}'.format(name) ret['changes']['removed'] = name return ret From 603f5b7de6fcc97aabc4ec528283a7ea2543033e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 14 Jul 2017 10:21:33 -0500 Subject: [PATCH 097/508] Change "TBD" in versionadded to "2017.7.0" --- salt/modules/boto_elbv2.py | 2 +- salt/states/boto_elbv2.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/boto_elbv2.py b/salt/modules/boto_elbv2.py index 954be55b8d..f9410e45c6 100644 --- a/salt/modules/boto_elbv2.py +++ b/salt/modules/boto_elbv2.py @@ -2,7 +2,7 @@ ''' Connection module for Amazon ALB -.. versionadded:: TBD +.. versionadded:: 2017.7.0 :configuration: This module accepts explicit elb credentials but can also utilize IAM roles assigned to the instance through Instance Profiles. Dynamic diff --git a/salt/states/boto_elbv2.py b/salt/states/boto_elbv2.py index 41566f5916..5146c9d695 100644 --- a/salt/states/boto_elbv2.py +++ b/salt/states/boto_elbv2.py @@ -2,7 +2,7 @@ ''' Manage AWS Application Load Balancer -.. versionadded:: TBD +.. versionadded:: 2017.7.0 Add and remove targets from an ALB target group. @@ -54,6 +54,8 @@ def __virtual__(): def targets_registered(name, targets, region=None, key=None, keyid=None, profile=None): ''' + .. versionadded:: 2017.7.0 + Add targets to an Application Load Balancer target group. This state will not remove targets. name @@ -63,8 +65,6 @@ def targets_registered(name, targets, region=None, key=None, keyid=None, A list of target IDs or a string of a single target that this target group should distribute traffic to. - .. versionadded:: TBD - .. code-block:: yaml add-targets: From c40604694047231691f525b8009a8cc67bf26466 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 14 Jul 2017 11:57:58 -0600 Subject: [PATCH 098/508] Add clarification to salt ssh docs about key auto-generation. Fixes #42267 --- doc/topics/ssh/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/topics/ssh/index.rst b/doc/topics/ssh/index.rst index e55734762f..fa08a96b8c 100644 --- a/doc/topics/ssh/index.rst +++ b/doc/topics/ssh/index.rst @@ -64,7 +64,8 @@ Deploy ssh key for salt-ssh =========================== By default, salt-ssh will generate key pairs for ssh, the default path will be -/etc/salt/pki/master/ssh/salt-ssh.rsa +``/etc/salt/pki/master/ssh/salt-ssh.rsa``. The key generation happens when you run +``salt-ssh`` for the first time. You can use ssh-copy-id, (the OpenSSH key deployment tool) to deploy keys to your servers. From b40f980632ccbac43147bd6071b7658de52f41ef Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 14 Jul 2017 17:26:02 -0600 Subject: [PATCH 099/508] Add more documentation for config options that are missing from master/minion docs --- conf/master | 18 ++ conf/minion | 18 ++ doc/ref/configuration/master.rst | 460 +++++++++++++++++++++++++++---- doc/ref/configuration/minion.rst | 283 ++++++++++++++++++- salt/config/__init__.py | 2 +- 5 files changed, 717 insertions(+), 64 deletions(-) diff --git a/conf/master b/conf/master index 7c09e858b1..1fc76cb89a 100644 --- a/conf/master +++ b/conf/master @@ -302,6 +302,9 @@ # public keys from the minions. Note that this is insecure. #auto_accept: False +# The size of key that should be generated when creating new keys. +#keysize: 2048 + # Time in minutes that an incoming public key with a matching name found in # pki_dir/minion_autosign/keyid is automatically accepted. Expired autosign keys # are removed when the master checks the minion_autosign directory. @@ -916,6 +919,21 @@ #pillar_cache_backend: disk +###### Reactor Settings ##### +########################################### +# Define a salt reactor. See https://docs.saltstack.com/en/latest/topics/reactor/ +#reactor: [] + +#Set the TTL for the cache of the reactor configuration. +#reactor_refresh_interval: 60 + +#Configure the number of workers for the runner/wheel in the reactor. +#reactor_worker_threads: 10 + +#Define the queue size for workers in the reactor. +#reactor_worker_hwm: 10000 + + ##### Syndic settings ##### ########################################## # The Salt syndic is used to pass commands through a master from a higher diff --git a/conf/minion b/conf/minion index 44e5f4e52f..ed2cfde7dc 100644 --- a/conf/minion +++ b/conf/minion @@ -615,6 +615,9 @@ # you do so at your own risk! #open_mode: False +# The size of key that should be generated when creating new keys. +#keysize: 2048 + # Enable permissive access to the salt keys. This allows you to run the # master or minion as root, but have a non-root group be given access to # your pki_dir. To make the access explicit, root must belong to the group @@ -656,6 +659,21 @@ # ssl_version: PROTOCOL_TLSv1_2 +###### Reactor Settings ##### +########################################### +# Define a salt reactor. See https://docs.saltstack.com/en/latest/topics/reactor/ +#reactor: [] + +#Set the TTL for the cache of the reactor configuration. +#reactor_refresh_interval: 60 + +#Configure the number of workers for the runner/wheel in the reactor. +#reactor_worker_threads: 10 + +#Define the queue size for workers in the reactor. +#reactor_worker_hwm: 10000 + + ###### Thread settings ##### ########################################### # Disable multiprocessing support, by default when a minion receives a diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 0ee2f2a4c0..fd310773bf 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -91,64 +91,6 @@ The user to run the Salt processes user: root -.. conf_master:: max_open_files - -``max_open_files`` ------------------- - -Default: ``100000`` - -Each minion connecting to the master uses AT LEAST one file descriptor, the -master subscription connection. If enough minions connect you might start -seeing on the console(and then salt-master crashes): - -.. code-block:: bash - - Too many open files (tcp_listener.cpp:335) - Aborted (core dumped) - -.. code-block:: yaml - - max_open_files: 100000 - -By default this value will be the one of `ulimit -Hn`, i.e., the hard limit for -max open files. - -To set a different value than the default one, uncomment, and configure this -setting. Remember that this value CANNOT be higher than the hard limit. Raising -the hard limit depends on the OS and/or distribution, a good way to find the -limit is to search the internet for something like this: - -.. code-block:: text - - raise max open files hard limit debian - -.. conf_master:: worker_threads - -``worker_threads`` ------------------- - -Default: ``5`` - -The number of threads to start for receiving commands and replies from minions. -If minions are stalling on replies because you have many minions, raise the -worker_threads value. - -Worker threads should not be put below 3 when using the peer system, but can -drop down to 1 worker otherwise. - -.. note:: - When the master daemon starts, it is expected behaviour to see - multiple salt-master processes, even if 'worker_threads' is set to '1'. At - a minimum, a controlling process will start along with a Publisher, an - EventPublisher, and a number of MWorker processes will be started. The - number of MWorker processes is tuneable by the 'worker_threads' - configuration value while the others are not. - -.. code-block:: yaml - - worker_threads: 5 - .. conf_master:: ret_port ``ret_port`` @@ -869,6 +811,74 @@ to socket concurrently. sock_pool_size: 15 +.. conf_master:: ipc_mode + +``ipc_mode`` +------------ + +Default: ``ipc`` + +The ipc strategy. (i.e., sockets versus tcp, etc.) Windows platforms lack +POSIX IPC and must rely on TCP based inter-process communications. ``ipc_mode`` +is set to ``tcp`` by default on Windows. + +.. code-block:: yaml + + ipc_mode: ipc + +.. conf_master:: + +``tcp_master_pub_port`` +----------------------- + +Default: ``4512`` + +The TCP port on which events for the master should be published if ``ipc_mode`` is TCP. + +.. code-block:: yaml + + tcp_master_pub_port: 4512 + +.. conf_master:: tcp_master_pull_port + +``tcp_master_pull_port`` +------------------------ + +Default: ``4513`` + +The TCP port on which events for the master should be pulled if ``ipc_mode`` is TCP. + +.. code-block:: yaml + + tcp_master_pull_port: 4513 + +.. conf_master:: tcp_master_publish_pull + +``tcp_master_publish_pull`` +--------------------------- + +Default: ``4514`` + +The TCP port on which events for the master should be pulled fom and then republished onto +the event bus on the master. + +.. code-block:: yaml + + tcp_master_publish_pull: 4514 + +.. conf_master:: tcp_master_workers + +``tcp_master_workers`` +---------------------- + +Default: ``4515`` + +The TCP port for ``mworkers`` to connect to on the master. + +.. code-block:: yaml + + tcp_master_workers: 4515 + Salt-SSH Configuration ====================== @@ -1103,6 +1113,19 @@ public keys from minions. auto_accept: False +.. conf_master:: keysize + +``keysize`` +----------- + +Default: ``2048`` + +The size of key that should be generated when creating new keys. + +.. code-block:: yaml + + keysize: 2048 + .. conf_master:: autosign_timeout ``autosign_timeout`` @@ -1147,6 +1170,24 @@ minion IDs for which keys will automatically be rejected. Will override both membership in the :conf_master:`autosign_file` and the :conf_master:`auto_accept` setting. +.. conf_master:: permissive_pki_access + +``permissive_pki_access`` +------------------------- + +Default: ``False`` + +Enable permissive access to the salt keys. This allows you to run the +master or minion as root, but have a non-root group be given access to +your pki_dir. To make the access explicit, root must belong to the group +you've given access to. This is potentially quite insecure. If an autosign_file +is specified, enabling permissive_pki_access will allow group access to that +specific file. + +.. code-block:: yaml + + permissive_pki_access: False + .. conf_master:: publisher_acl ``publisher_acl`` @@ -1191,6 +1232,20 @@ This is completely disabled by default. - cmd.* - test.echo +.. conf_master:: sudo_acl + +``sudo_acl`` +------------ + +Default: ``False`` + +Enforce ``publisher_acl`` and ``publisher_acl_blacklist`` when users have sudo +access to the salt command. + +.. code-block:: yaml + + sudo_acl: False + .. conf_master:: external_auth ``external_auth`` @@ -1347,6 +1402,18 @@ Do not disable this unless it is absolutely clear what this does. rotate_aes_key: True +.. conf_master:: publish_session + +``publish_session`` +------------------- + +Default: ``86400`` + +The number of seconds between AES key rotations on the master. + +.. code-block:: yaml + + publish_session: Default: 86400 .. conf_master:: ssl @@ -1378,6 +1445,24 @@ constant names without ssl module prefix: ``CERT_REQUIRED`` or ``PROTOCOL_SSLv23 ``allow_minion_key_revoke`` --------------------------- +Default: ``False`` + +By default, the master deletes its cache of minion data when the key for that +minion is removed. To preserve the cache after key deletion, set +``preserve_minion_cache`` to True. + +WARNING: This may have security implications if compromised minions auth with +a previous deleted minion ID. + +.. code-block:: yaml + + preserve_minion_cache: False + +.. conf_master:: allow_minion_key_revoke + +``allow_minion_key_revoke`` +--------------------------- + Default: ``True`` Controls whether a minion can request its own key revocation. When True @@ -1389,6 +1474,128 @@ the master will drop the request and the minion's key will remain accepted. rotate_aes_key: True + +Master Large Scale Tuning Settings +================================== + +.. conf_master:: max_open_files + +``max_open_files`` +------------------ + +Default: ``100000`` + +Each minion connecting to the master uses AT LEAST one file descriptor, the +master subscription connection. If enough minions connect you might start +seeing on the console(and then salt-master crashes): + +.. code-block:: bash + + Too many open files (tcp_listener.cpp:335) + Aborted (core dumped) + +.. code-block:: yaml + + max_open_files: 100000 + +By default this value will be the one of `ulimit -Hn`, i.e., the hard limit for +max open files. + +To set a different value than the default one, uncomment, and configure this +setting. Remember that this value CANNOT be higher than the hard limit. Raising +the hard limit depends on the OS and/or distribution, a good way to find the +limit is to search the internet for something like this: + +.. code-block:: text + + raise max open files hard limit debian + +.. conf_master:: worker_threads + +``worker_threads`` +------------------ + +Default: ``5`` + +The number of threads to start for receiving commands and replies from minions. +If minions are stalling on replies because you have many minions, raise the +worker_threads value. + +Worker threads should not be put below 3 when using the peer system, but can +drop down to 1 worker otherwise. + +.. note:: + When the master daemon starts, it is expected behaviour to see + multiple salt-master processes, even if 'worker_threads' is set to '1'. At + a minimum, a controlling process will start along with a Publisher, an + EventPublisher, and a number of MWorker processes will be started. The + number of MWorker processes is tuneable by the 'worker_threads' + configuration value while the others are not. + +.. code-block:: yaml + + worker_threads: 5 + +.. conf_master:: pub_hwm + +``pub_hwm`` +----------- + +Default: ``1000`` + +The zeromq high water mark on the publisher interface. + +.. code-block:: yaml + + pub_hwm: 1000 + +.. conf_master:: zmq_backlog + +``zmq_backlog`` +--------------- + +Default: ``1000`` + +The listen queue size of the ZeroMQ backlog. + +.. code-block:: yaml + + zmq_backlog: 1000 + +.. conf_master:: salt_event_pub_hwm +.. conf_master:: event_publisher_pub_hwm + +``salt_event_pub_hwm`` and ``event_publisher_pub_hwm`` +------------------------------------------------------ + +These two ZeroMQ High Water Mark settings, ``salt_event_pub_hwm`` and +``event_publisher_pub_hwm`` are significant for masters with thousands of +minions. When these are insufficiently high it will manifest in random +responses missing in the CLI and even missing from the job cache. Masters +that have fast CPUs and many cores with appropriate ``worker_threads`` +will not need these set as high. + +The ZeroMQ high-water-mark for the ``SaltEvent`` pub socket default is: + +.. code-block:: yaml + + salt_event_pub_hwm: 20000 + +The ZeroMQ high-water-mark for the ``EventPublisher`` pub socket default is: + +.. code-block:: yaml + + event_publisher_pub_hwm: 10000 + +As an example, on single master deployment with 8,000 minions, 2.4GHz CPUs, +24 cores, and 32GiB memory has these settings: + +.. code-block:: yaml + + salt_event_pub_hwm: 128000 + event_publisher_pub_hwm: 64000 + + Master Module Management ======================== @@ -2942,6 +3149,26 @@ configuration. pillar_opts: False +.. conf_master:: pillar_safe_render_error + +``pillar_safe_render_error`` +---------------------------- + +Default: ``True`` + +The pillar_safe_render_error option prevents the master from passing pillar +render errors to the minion. This is set on by default because the error could +contain templating data which would give that minion information it shouldn't +have, like a password! When set ``True`` the error message will only show: + +.. code-block:: shell + + Rendering SLS 'my.sls' failed. Please see master log for details. + +.. code-block:: yaml + + pillar_safe_render_error: True + .. _master-configuration-ext-pillar: .. conf_master:: ext_pillar @@ -3525,6 +3752,63 @@ can be utilized: pillar_cache_backend: disk + +Master Reactor Settings +======================= + +.. conf_master:: reactor + +``reactor`` +----------- + +Default: ``[]`` + +Defines a salt reactor. See the :ref:`Reactor ` documentation for more +information. + +.. code-block:: yaml + + reactor: [] + +.. conf_master:: reactor_refresh_interval + +``reactor_refresh_interval`` +---------------------------- + +Default: ``60`` + +The TTL for the cache of the reactor configuration. + +.. code-block:: yaml + + reactor_refresh_interval: 60 + +.. conf_master:: reactor_worker_threads + +``reactor_worker_threads`` +-------------------------- + +Default: ``10`` + +The number of workers for the runner/wheel in the reactor. + +.. code-block:: yaml + reactor_worker_threads: 10 + +.. conf_master:: reactor_worker_hwm + +``reactor_worker_hwm`` +---------------------- + +Default: ``10000`` + +The queue size for workers in the reactor. + +.. code-block:: yaml + + reactor_worker_hwm: 10000 + + Syndic Server Settings ====================== @@ -3970,6 +4254,64 @@ option then the master will log a warning message. - master.d/* - /etc/roles/webserver + +Keepalive Settings +================== + +.. conf_master:: tcp_keepalive + +``tcp_keepalive`` +----------------- + +Default: ``True`` + +The tcp keepalive interval to set on TCP ports. This setting can be used to tune Salt +connectivity issues in messy network environments with misbehaving firewalls. + +.. code-block:: yaml + + tcp_keepalive: True + +.. conf_master:: tcp_keepalive_cnt + +``tcp_keepalive_cnt`` +--------------------- + +Default: ``-1`` + +Sets the ZeroMQ TCP keepalive count. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_cnt: -1 + +.. conf_master:: tcp_keepalive_idle + +``tcp_keepalive_idle`` +---------------------- + +Default: ``300`` + +Sets ZeroMQ TCP keepalive idle. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_idle: 300 + +.. conf_master:: tcp_keepalive_intvl + +``tcp_keepalive_intvl`` +----------------------- + +Default: ``-1`` + +Sets ZeroMQ TCP keepalive interval. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_intvl': -1 + + .. _winrepo-master-config-opts: Windows Software Repo Settings @@ -4108,7 +4450,7 @@ URL of the repository: .. code-block:: yaml - winrepo_remotes: + winrepo_remotes_ng: - ' https://github.com/saltstack/salt-winrepo-ng.git' Replace ```` with the SHA1 hash of a commit ID. Specifying a commit diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 29fbe01dd7..087f41ef7e 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -750,6 +750,20 @@ seconds each iteration. acceptance_wait_time_max: 0 +.. conf_minion:: rejected_retry + +``rejected_retry`` +------------------ + +Default: ``False`` + +If the master rejects the minion's public key, retry instead of exiting. +Rejected keys will be handled the same as waiting on acceptance. + +.. code-block:: yaml + + rejected_retry: False + .. conf_minion:: random_reauth_delay ``random_reauth_delay`` @@ -1180,7 +1194,7 @@ If certain returners should be disabled, this is the place .. conf_minion:: enable_whitelist_modules ``whitelist_modules`` ----------------------------- +--------------------- Default: ``[]`` (Module whitelisting is disabled. Adding anything to the config option will cause only the listed modules to be enabled. Modules not in the list will @@ -1275,7 +1289,7 @@ A list of extra directories to search for Salt renderers .. conf_minion:: utils_dirs ``utils_dirs`` ---------------- +-------------- Default: ``[]`` @@ -1334,6 +1348,20 @@ below. providers: service: systemd +.. conf_minion:: modules_max_memory + +``modules_max_memory`` +---------------------- + +Default: ``-1`` + +Specify a max size (in bytes) for modules on import. This feature is currently +only supported on *nix operating systems and requires psutil. + +.. code-block:: yaml + + modules_max_memory: -1 + Top File Settings ================= @@ -1465,6 +1493,52 @@ environment lacks one. default_top: dev +.. conf_minion:: startup_states + +``startup_states`` +------------------ + +Default: ``''`` + +States to run when the minion daemon starts. To enable, set ``startup_states`` to: + +- ``highstate``: Execute state.highstate +- ``sls``: Read in the sls_list option and execute the named sls files +- ``top``: Read top_file option and execute based on that file on the Master + +.. code-block:: yaml + + startup_states: '' + +.. conf_minion:: sls_list + +``sls_list`` +------------ + +Default: ``[]`` + +List of states to run when the minion starts up if ``startup_states`` is set to ``sls``. + +.. code-block:: yaml + + sls_list: + - edit.vim + - hyper + +.. conf_minion:: top_file + +``top_file`` +------------ + +Default: ``''`` + +Top file to execute if ``startup_states`` is set to ``top``. + +.. code-block:: yaml + + top_file: '' + + State Management Settings ========================= @@ -1481,7 +1555,7 @@ The default renderer used for local state executions renderer: yaml_jinja -.. conf_master:: test +.. conf_minion:: test ``test`` -------- @@ -1902,6 +1976,35 @@ before the initial key exchange. The master fingerprint can be found by running master_finger: 'ba:30:65:2a:d6:9e:20:4f:d8:b2:f3:a7:d4:65:11:13' +.. conf_minion:: keysize + +``keysize`` +----------- + +Default: ``2048`` + +The size of key that should be generated when creating new keys. + +.. code-block:: yaml + + keysize: 2048 + +.. conf_minion:: permissive_pki_access + +``permissive_pki_access`` +------------------------- + +Default: ``False`` + +Enable permissive access to the salt keys. This allows you to run the +master or minion as root, but have a non-root group be given access to +your pki_dir. To make the access explicit, root must belong to the group +you've given access to. This is potentially quite insecure. + +.. code-block:: yaml + + permissive_pki_access: False + .. conf_minion:: verify_master_pubkey_sign ``verify_master_pubkey_sign`` @@ -2009,7 +2112,7 @@ blocked. If `cmd_whitelist_glob` is NOT SET, then all shell commands are permitt - 'cat /etc/fstab' -.. conf_master:: ssl +.. conf_minion:: ssl ``ssl`` ------- @@ -2035,6 +2138,62 @@ constant names without ssl module prefix: ``CERT_REQUIRED`` or ``PROTOCOL_SSLv23 ssl_version: PROTOCOL_TLSv1_2 +Reactor Settings +================ + +.. conf_minion:: reactor + +``reactor`` +----------- + +Default: ``[]`` + +Defines a salt reactor. See the :ref:`Reactor ` documentation for more +information. + +.. code-block:: yaml + + reactor: [] + +.. conf_minion:: reactor_refresh_interval + +``reactor_refresh_interval`` +---------------------------- + +Default: ``60`` + +The TTL for the cache of the reactor configuration. + +.. code-block:: yaml + + reactor_refresh_interval: 60 + +.. conf_minion:: reactor_worker_threads + +``reactor_worker_threads`` +-------------------------- + +Default: ``10`` + +The number of workers for the runner/wheel in the reactor. + +.. code-block:: yaml + reactor_worker_threads: 10 + +.. conf_minion:: reactor_worker_hwm + +``reactor_worker_hwm`` +---------------------- + +Default: ``10000`` + +The queue size for workers in the reactor. + +.. code-block:: yaml + + reactor_worker_hwm: 10000 + + Thread Settings =============== @@ -2305,6 +2464,62 @@ option then the minion will log a warning message. - /etc/roles/webserver +Keepalive Settings +================== + +.. conf_minion:: tcp_keepalive + +``tcp_keepalive`` +----------------- + +Default: ``True`` + +The tcp keepalive interval to set on TCP ports. This setting can be used to tune Salt +connectivity issues in messy network environments with misbehaving firewalls. + +.. code-block:: yaml + + tcp_keepalive: True + +.. conf_minion:: tcp_keepalive_cnt + +``tcp_keepalive_cnt`` +--------------------- + +Default: ``-1`` + +Sets the ZeroMQ TCP keepalive count. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_cnt: -1 + +.. conf_minion:: tcp_keepalive_idle + +``tcp_keepalive_idle`` +---------------------- + +Default: ``300`` + +Sets ZeroMQ TCP keepalive idle. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_idle: 300 + +.. conf_minion:: tcp_keepalive_intvl + +``tcp_keepalive_intvl`` +----------------------- + +Default: ``-1`` + +Sets ZeroMQ TCP keepalive interval. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_intvl': -1 + Frozen Build Update Settings ============================ @@ -2406,6 +2621,36 @@ out. winrepo_dir: 'D:\winrepo' +.. conf_minion:: winrepo_dir_ng + +``winrepo_dir_ng`` +------------------ + +.. versionadded:: 2015.8.0 + A new :ref:`ng ` repo was added. + +Default: ``/srv/salt/win/repo-ng`` + +Location on the minion where the :conf_minion:`winrepo_remotes_ng` are checked +out for 2015.8.0 and later minions. + +.. code-block:: yaml + + winrepo_dir_ng: /srv/salt/win/repo-ng + +.. conf_minion:: winrepo_source_dir + +``winrepo_source_dir`` +---------------------- + +Default: ``salt://win/repo-ng/`` + +The source location for the winrepo sls files. + +.. code-block:: yaml + + winrepo_source_dir: salt://win/repo-ng/ + .. conf_minion:: winrepo_cachefile .. conf_minion:: win_repo_cachefile @@ -2458,3 +2703,33 @@ URL of the the repository: Replace ```` with the SHA1 hash of a commit ID. Specifying a commit ID is useful in that it allows one to revert back to a previous version in the event that an error is introduced in the latest revision of the repo. + +.. conf_minion:: winrepo_remotes_ng + +``winrepo_remotes_ng`` +---------------------- + +.. versionadded:: 2015.8.0 + A new :ref:`ng ` repo was added. + +Default: ``['https://github.com/saltstack/salt-winrepo-ng.git']`` + +List of git repositories to checkout and include in the winrepo for +2015.8.0 and later minions. + +.. code-block:: yaml + + winrepo_remotes_ng: + - https://github.com/saltstack/salt-winrepo-ng.git + +To specify a specific revision of the repository, prepend a commit ID to the +URL of the repository: + +.. code-block:: yaml + + winrepo_remotes_ng: + - ' https://github.com/saltstack/salt-winrepo-ng.git' + +Replace ```` with the SHA1 hash of a commit ID. Specifying a commit +ID is useful in that it allows one to revert back to a previous version in the +event that an error is introduced in the latest revision of the repo. diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 7c40f82c81..5417790f06 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -331,7 +331,7 @@ VALID_OPTS = { # The TCP port on which minion events should be pulled if ipc_mode is TCP 'tcp_pull_port': int, - # The TCP port on which events for the master should be pulled if ipc_mode is TCP + # The TCP port on which events for the master should be published if ipc_mode is TCP 'tcp_master_pub_port': int, # The TCP port on which events for the master should be pulled if ipc_mode is TCP From c83e6fc696560c674e66eaa2b7ea506e00676b4e Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sat, 15 Jul 2017 23:24:07 +0100 Subject: [PATCH 100/508] Default skip_verify to False This has been partially tackled in https://github.com/saltstack/salt/pull/41528, but exactly the main entry point was missed. --- salt/states/netconfig.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/states/netconfig.py b/salt/states/netconfig.py index 9318e227b4..ed20c1ea6b 100644 --- a/salt/states/netconfig.py +++ b/salt/states/netconfig.py @@ -109,7 +109,7 @@ def managed(name, template_mode='755', saltenv=None, template_engine='jinja', - skip_verify=True, + skip_verify=False, defaults=None, test=False, commit=True, @@ -194,9 +194,11 @@ def managed(name, - :mod:`py` - :mod:`wempy` - skip_verify: True + skip_verify: False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. + + .. versionchanged:: 2017.7.1 test: False Dry run? If set to ``True``, will apply the config, discard and return the changes. Default: ``False`` From c830573a2cb449182b315b33df23a6db28353b02 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sun, 16 Jul 2017 23:09:19 +0100 Subject: [PATCH 101/508] Trailing whitespaces --- salt/states/netconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/netconfig.py b/salt/states/netconfig.py index ed20c1ea6b..1b2ed44d56 100644 --- a/salt/states/netconfig.py +++ b/salt/states/netconfig.py @@ -197,7 +197,7 @@ def managed(name, skip_verify: False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. - + .. versionchanged:: 2017.7.1 test: False From bbba84ce2d3646609a7f3ff84934be1417f970c0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 17 Jul 2017 15:26:54 +0200 Subject: [PATCH 102/508] Bugfix: Jobs scheduled to run at a future time stay pending for Salt minions (bsc#1036125) --- salt/utils/schedule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index a69e5ca418..88118e5b58 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -333,6 +333,7 @@ import logging import errno import random import yaml +import copy # Import Salt libs import salt.config @@ -844,7 +845,7 @@ class Schedule(object): if argspec.keywords: # this function accepts **kwargs, pack in the publish data for key, val in six.iteritems(ret): - kwargs['__pub_{0}'.format(key)] = val + kwargs['__pub_{0}'.format(key)] = copy.deepcopy(val) ret['return'] = self.functions[func](*args, **kwargs) From 774d204d6565924f065fb4b768a7f6c72c20ebe4 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 17 Jul 2017 15:26:54 +0200 Subject: [PATCH 103/508] Bugfix: Jobs scheduled to run at a future time stay pending for Salt minions (bsc#1036125) --- salt/utils/schedule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 231f2331c0..2bb75ffcf1 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -333,6 +333,7 @@ import logging import errno import random import yaml +import copy # Import Salt libs import salt.config @@ -841,7 +842,7 @@ class Schedule(object): if argspec.keywords: # this function accepts **kwargs, pack in the publish data for key, val in six.iteritems(ret): - kwargs['__pub_{0}'.format(key)] = val + kwargs['__pub_{0}'.format(key)] = copy.deepcopy(val) ret['return'] = self.functions[func](*args, **kwargs) From ef1f663fc911bf2ab3a607b92f62652eeea284cc Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 17 Jul 2017 13:44:25 -0600 Subject: [PATCH 104/508] Detect server OS with a desktop release name --- salt/grains/core.py | 28 ++++++++++------------------ salt/version.py | 28 ++++++++++++++++------------ 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 9b4235da10..71ad3d0721 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -984,28 +984,20 @@ def _windows_platform_data(): os_release = platform.release() info = salt.utils.win_osinfo.get_os_version_info() + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function # started reporting the Desktop version instead of the Server version on # Server versions of Windows, so we need to look those up - # Check for Python >=2.7.12 or >=3.5.2 - ver = pythonversion()['pythonversion'] - if ((six.PY2 and - salt.utils.compare_versions(ver, '>=', [2, 7, 12, 'final', 0])) - or - (six.PY3 and - salt.utils.compare_versions(ver, '>=', [3, 5, 2, 'final', 0]))): - # (Product Type 1 is Desktop, Everything else is Server) - if info['ProductType'] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - os_release = server.get(os_release, - 'Grain not found. Update lookup table ' - 'in the `_windows_platform_data` ' - 'function in `grains\\core.py`') + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # (Product Type 1 is Desktop, Everything else is Server) + if info['ProductType'] > 1 and os_release in server: + os_release = server[os_release] service_pack = None if info['ServicePackMajor'] > 0: diff --git a/salt/version.py b/salt/version.py index 6fc4ccbf61..6e5d4e74af 100644 --- a/salt/version.py +++ b/salt/version.py @@ -654,18 +654,22 @@ def system_information(): release = platform.release() if platform.win32_ver()[0]: import win32api # pylint: disable=3rd-party-module-not-gated - if ((sys.version_info.major == 2 and sys.version_info >= (2, 7, 12)) or - (sys.version_info.major == 3 and sys.version_info >= (3, 5, 2))): - if win32api.GetVersionEx(1)[8] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - release = server.get(platform.release(), - 'UNKServer') - _, ver, sp, extra = platform.win32_ver() - version = ' '.join([release, ver, sp, extra]) + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} + # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function + # started reporting the Desktop version instead of the Server version on + # Server versions of Windows, so we need to look those up + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # If this is a Server Platform then `GetVersionEx` will return a number + # greater than 1. + if win32api.GetVersionEx(1)[8] > 1 and release in server: + release = server[release] + _, ver, sp, extra = platform.win32_ver() + version = ' '.join([release, ver, sp, extra]) system = [ ('system', platform.system()), From dc85b5edbe89034778a5d625cced4c5cd9fba6de Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 17 Jul 2017 16:50:54 -0400 Subject: [PATCH 105/508] [2016.3] Update version numbers in doc config for 2017.7.0 release --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 29168d62f6..210bca239e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -213,7 +213,7 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.6' # latest release +latest_release = '2017.7.0' # latest release previous_release = '2016.3.6' # latest release from previous branch previous_release_dir = '2016.3' # path on web server for previous branch next_release = '' # next release From b90b7a7506e6baa9bad7eb21eae72dfe7979e546 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 17 Jul 2017 16:54:46 -0400 Subject: [PATCH 106/508] [2016.11] Update version numbers in doc config for 2017.7.0 release --- doc/conf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index fa2296499c..e891986dc6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -240,9 +240,9 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.6' # latest release -previous_release = '2016.3.6' # latest release from previous branch -previous_release_dir = '2016.3' # path on web server for previous branch +latest_release = '2017.7.0' # latest release +previous_release = '2016.11.6' # latest release from previous branch +previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch @@ -253,8 +253,8 @@ if on_saltstack: copyright = time.strftime("%Y") # < --- START do not merge these settings to other branches START ---> # -build_type = 'latest' # latest, previous, develop, next -release = latest_release # version, latest_release, previous_release +build_type = 'previous' # latest, previous, develop, next +release = previous_release # version, latest_release, previous_release # < --- END do not merge these settings to other branches END ---> # # Set google custom search engine From dc5bb301f7c93814af2122ac86cdde4b0859e8d1 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 17 Jul 2017 16:56:56 -0400 Subject: [PATCH 107/508] [2017.7] Update version numbers in doc config for 2017.7.0 release --- doc/conf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 50357ddcf6..1ecfdfcd99 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -245,9 +245,9 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.6' # latest release -previous_release = '2016.3.6' # latest release from previous branch -previous_release_dir = '2016.3' # path on web server for previous branch +latest_release = '2017.7.0' # latest release +previous_release = '2016.11.6' # latest release from previous branch +previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch @@ -258,8 +258,8 @@ if on_saltstack: copyright = time.strftime("%Y") # < --- START do not merge these settings to other branches START ---> # -build_type = 'develop' # latest, previous, develop, next -release = version # version, latest_release, previous_release +build_type = 'latest' # latest, previous, develop, next +release = latest_release # version, latest_release, previous_release # < --- END do not merge these settings to other branches END ---> # # Set google custom search engine From 8c048403d7327c47e67a9e74961db414ce221e4d Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 17 Jul 2017 14:58:16 -0600 Subject: [PATCH 108/508] Detect Server OS with a desktop release name --- salt/grains/core.py | 27 +++++++++------------------ salt/version.py | 28 ++++++++++++++++------------ 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 9ed045f45d..3f40256299 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -920,28 +920,19 @@ def _windows_platform_data(): os_release = platform.release() info = salt.utils.win_osinfo.get_os_version_info() + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function # started reporting the Desktop version instead of the Server version on # Server versions of Windows, so we need to look those up - # Check for Python >=2.7.12 or >=3.5.2 - ver = pythonversion()['pythonversion'] - if ((six.PY2 and - salt.utils.compare_versions(ver, '>=', [2, 7, 12, 'final', 0])) - or - (six.PY3 and - salt.utils.compare_versions(ver, '>=', [3, 5, 2, 'final', 0]))): - # (Product Type 1 is Desktop, Everything else is Server) - if info['ProductType'] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - os_release = server.get(os_release, - 'Grain not found. Update lookup table ' - 'in the `_windows_platform_data` ' - 'function in `grains\\core.py`') + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + if info['ProductType'] > 1 and os_release in server: + os_release = server[os_release] service_pack = None if info['ServicePackMajor'] > 0: diff --git a/salt/version.py b/salt/version.py index 8549ed9bdf..0c7695568a 100644 --- a/salt/version.py +++ b/salt/version.py @@ -647,18 +647,22 @@ def system_information(): release = platform.release() if platform.win32_ver()[0]: import win32api - if ((sys.version_info.major == 2 and sys.version_info >= (2, 7, 12)) or - (sys.version_info.major == 3 and sys.version_info >= (3, 5, 2))): - if win32api.GetVersionEx(1)[8] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - release = server.get(platform.release(), - 'UNKServer') - _, ver, sp, extra = platform.win32_ver() - version = ' '.join([release, ver, sp, extra]) + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} + # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function + # started reporting the Desktop version instead of the Server version on + # Server versions of Windows, so we need to look those up + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # If this is a Server Platform then `GetVersionEx` will return a number + # greater than 1. + if win32api.GetVersionEx(1)[8] > 1 and release in server: + release = server[release] + _, ver, sp, extra = platform.win32_ver() + version = ' '.join([release, ver, sp, extra]) system = [ ('system', platform.system()), From 526b6ee14d30f34c820f6ece2690bf90756d0a55 Mon Sep 17 00:00:00 2001 From: Corvin Mcpherson Date: Mon, 17 Jul 2017 19:10:15 -0400 Subject: [PATCH 109/508] Multiple documentation fixes The master config 'engines' option takes a list of engines but will accept a single engine instead of the list but adds a warning that it requires a list. The documentation for some of the engine modules pass a single engine instead of a list of engines. This commit fixes the documentation of these engine modules to give correct usage of the 'engines' option. Fixes #42333 Additionally, A few of the documentation pages were missing code-blocks where they were needed to ensure proper formatting. This commit adds the code blocks where they were missing in the engine modules documentation. --- salt/engines/docker_events.py | 4 ++-- salt/engines/hipchat.py | 32 ++++++++++++++++---------------- salt/engines/http_logstash.py | 14 +++++++------- salt/engines/logentries.py | 3 +++ salt/engines/logstash.py | 3 +++ salt/engines/reactor.py | 8 ++++---- salt/engines/redis_sentinel.py | 3 +++ salt/engines/slack.py | 30 +++++++++++++++--------------- 8 files changed, 53 insertions(+), 44 deletions(-) diff --git a/salt/engines/docker_events.py b/salt/engines/docker_events.py index cb6c6dcd2c..9028ad8d82 100644 --- a/salt/engines/docker_events.py +++ b/salt/engines/docker_events.py @@ -50,8 +50,8 @@ def start(docker_url='unix://var/run/docker.sock', .. code-block:: yaml engines: - docker_events: - docker_url: unix://var/run/docker.sock + - docker_events: + docker_url: unix://var/run/docker.sock The config above sets up engines to listen for events from the Docker daemon and publish diff --git a/salt/engines/hipchat.py b/salt/engines/hipchat.py index aede7c9fdc..4493d8b322 100644 --- a/salt/engines/hipchat.py +++ b/salt/engines/hipchat.py @@ -14,22 +14,22 @@ keys make the engine interactive. .. code-block:: yaml engines: - hipchat: - token: 'XXXXXX' - room: 'salt' - control: True - valid_users: - - SomeUser - valid_commands: - - test.ping - - cmd.run - - list_jobs - - list_commands - aliases: - list_jobs: - cmd: jobs.list_jobs - list_commands: - cmd: pillar.get salt:engines:hipchat:valid_commands target=saltmaster + - hipchat: + token: 'XXXXXX' + room: 'salt' + control: True + valid_users: + - SomeUser + valid_commands: + - test.ping + - cmd.run + - list_jobs + - list_commands + aliases: + list_jobs: + cmd: jobs.list_jobs + list_commands: + cmd: pillar.get salt:engines:hipchat:valid_commands target=saltmaster ''' from __future__ import absolute_import diff --git a/salt/engines/http_logstash.py b/salt/engines/http_logstash.py index 3e4b89ad78..78edc0d1b9 100644 --- a/salt/engines/http_logstash.py +++ b/salt/engines/http_logstash.py @@ -12,13 +12,13 @@ them onto a logstash endpoint via HTTP requests. engines: - http_logstash: - url: http://blabla.com/salt-stuff - tags: - - salt/job/*/new - - salt/job/*/ret/* - funs: - - probes.results - - bgp.config + url: http://blabla.com/salt-stuff + tags: + - salt/job/*/new + - salt/job/*/ret/* + funs: + - probes.results + - bgp.config ''' from __future__ import absolute_import diff --git a/salt/engines/logentries.py b/salt/engines/logentries.py index 0fe422edfe..9dd0fc3735 100644 --- a/salt/engines/logentries.py +++ b/salt/engines/logentries.py @@ -24,6 +24,9 @@ master config. :configuration: Example configuration + + .. code-block:: yaml + engines: - logentries: endpoint: data.logentries.com diff --git a/salt/engines/logstash.py b/salt/engines/logstash.py index 7de753b4c8..8965e4dd3d 100644 --- a/salt/engines/logstash.py +++ b/salt/engines/logstash.py @@ -8,6 +8,9 @@ them onto a logstash endpoint. :configuration: Example configuration + + .. code-block:: yaml + engines: - logstash: host: log.my_network.com diff --git a/salt/engines/reactor.py b/salt/engines/reactor.py index b95a8213e3..43829f0a27 100644 --- a/salt/engines/reactor.py +++ b/salt/engines/reactor.py @@ -7,10 +7,10 @@ Example Config in Master or Minion config .. code-block:: yaml engines: - reactor: - refresh_interval: 60 - worker_threads: 10 - worker_hwm: 10000 + - reactor: + refresh_interval: 60 + worker_threads: 10 + worker_hwm: 10000 reactor: - 'salt/cloud/*/destroyed': diff --git a/salt/engines/redis_sentinel.py b/salt/engines/redis_sentinel.py index 8f4e807313..596f610d38 100644 --- a/salt/engines/redis_sentinel.py +++ b/salt/engines/redis_sentinel.py @@ -8,6 +8,9 @@ events based on the channels they are subscribed to. :configuration: Example configuration + + .. code-block:: yaml + engines: - redis_sentinel: hosts: diff --git a/salt/engines/slack.py b/salt/engines/slack.py index d5df1bd574..6b37244daf 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -12,21 +12,21 @@ prefaced with a ``!``. .. code-block:: yaml engines: - slack: - token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' - control: True - valid_users: - - garethgreenaway - valid_commands: - - test.ping - - cmd.run - - list_jobs - - list_commands - aliases: - list_jobs: - cmd: jobs.list_jobs - list_commands: - cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster + - slack: + token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' + control: True + valid_users: + - garethgreenaway + valid_commands: + - test.ping + - cmd.run + - list_jobs + - list_commands + aliases: + list_jobs: + cmd: jobs.list_jobs + list_commands: + cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster :depends: slackclient ''' From 14cf6ce3220394b84b0f0ea4d7d581d70beb471e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 17 Jul 2017 23:36:21 -0500 Subject: [PATCH 110/508] is_windows is a function, not a propery/attribute While not fatal, this could potentially cause problems running this test on Windows. --- tests/integration/modules/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/git.py b/tests/integration/modules/git.py index d404d7101c..548bcd1995 100644 --- a/tests/integration/modules/git.py +++ b/tests/integration/modules/git.py @@ -41,7 +41,7 @@ def _git_version(): git_version = subprocess.Popen( ['git', '--version'], shell=False, - close_fds=False if salt.utils.is_windows else True, + close_fds=False if salt.utils.is_windows() else True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] except OSError: From 915d94219e1915617db5372bc28d610650437353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= Date: Tue, 18 Jul 2017 12:20:34 +0100 Subject: [PATCH 111/508] Allow to check whether a function is available on the AliasesLoader wrapper --- salt/utils/templates.py | 7 +++++++ .../files/file/base/jinja_salt_contains_function.sls | 10 ++++++++++ tests/integration/states/renderers.py | 8 ++++++++ 3 files changed, 25 insertions(+) create mode 100644 tests/integration/files/file/base/jinja_salt_contains_function.sls diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 6370729779..37aa4c22be 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -82,6 +82,13 @@ class AliasedLoader(object): else: return getattr(self.wrapped, name) + def __contains__(self, name): + if name in ALIASES: + salt.utils.warn_until('Nitrogen', ALIAS_WARN) + return ALIASES[name] in self.wrapped + else: + return name in self.wrapped + class AliasedModule(object): ''' diff --git a/tests/integration/files/file/base/jinja_salt_contains_function.sls b/tests/integration/files/file/base/jinja_salt_contains_function.sls new file mode 100644 index 0000000000..24978d1799 --- /dev/null +++ b/tests/integration/files/file/base/jinja_salt_contains_function.sls @@ -0,0 +1,10 @@ +{% set salt_foo_bar_exist = 'foo.bar' in salt %} +{% set salt_test_ping_exist = 'test.ping' in salt %} + +test-ping-exist: + test.succeed_without_changes: + - name: salt_test_ping_exist_{{ salt_test_ping_exist }} + +foo-bar-not-exist: + test.succeed_without_changes: + - name: salt_foo_bar_exist_{{ salt_foo_bar_exist }} diff --git a/tests/integration/states/renderers.py b/tests/integration/states/renderers.py index 61ea0ffb72..1a305ada15 100644 --- a/tests/integration/states/renderers.py +++ b/tests/integration/states/renderers.py @@ -26,6 +26,14 @@ class TestJinjaRenderer(integration.ModuleCase): for state_ret in ret.values(): self.assertTrue(state_ret['result']) + def test_salt_contains_function(self): + ''' + Test if we are able to check if a function exists inside the "salt" + wrapper (AliasLoader) which is available on Jinja templates. + ''' + ret = self.run_function('state.sls', ['jinja_salt_contains_function']) + for state_ret in ret.values(): + self.assertTrue(state_ret['result']) if __name__ == '__main__': from integration import run_tests From 96517d1355eb70efbd2e7d2ac1ef87b267aacbc7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 10:06:55 -0600 Subject: [PATCH 112/508] Add note about patched windows packages --- doc/topics/releases/2017.7.0.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 52e026e490..2d06b20140 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -945,3 +945,13 @@ The ``glusterfs`` state had the following function removed: The ``openvswitch_port`` state had the following change: - The ``type`` option was removed from the ``present`` function. Please use ``tunnel_type`` instead. + +Build Notes +=========== + +Windows Installer Packages +-------------------------- + +Windows Installer packages have been patched with the following PR: 42347_ + +.. _42347: https://github.com/saltstack/salt/pull/42347 From f7c0bb4f46644b19ba18cd7a34e12739cb10a308 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 11:23:16 -0600 Subject: [PATCH 113/508] Remove build and dist directories before install --- pkg/windows/build.bat | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index 9dde8b2e72..cc9a4b34d7 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -110,6 +110,12 @@ if not %errorLevel%==0 ( ) @echo. +:: Remove build and dist directories +@echo %0 :: Remove build and dist directories... +@echo --------------------------------------------------------------------- +rd /s /q "%SrcDir%\build" +rd /s /q "%SrcDir%\dist" + :: Install Current Version of salt @echo %0 :: Install Current Version of salt... @echo --------------------------------------------------------------------- From 0946002713c75caa95a8f5d29c0c72232944dc19 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 11:28:49 -0600 Subject: [PATCH 114/508] Add blank line after delete --- pkg/windows/build.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index cc9a4b34d7..0117718539 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -115,6 +115,7 @@ if not %errorLevel%==0 ( @echo --------------------------------------------------------------------- rd /s /q "%SrcDir%\build" rd /s /q "%SrcDir%\dist" +@echo. :: Install Current Version of salt @echo %0 :: Install Current Version of salt... From a7c910c31e6d88763185789006a49753175e9883 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 11:36:38 -0600 Subject: [PATCH 115/508] Remove build and dist directories before install --- pkg/windows/build.bat | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index 07cf831612..d27d036474 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -42,6 +42,13 @@ if not %errorLevel%==0 ( ) @echo. +:: Remove build and dist directories +@echo %0 :: Remove build and dist directories... +@echo --------------------------------------------------------------------- +rd /s /q "%SrcDir%\build" +rd /s /q "%SrcDir%\dist" +@echo. + :: Install Current Version of salt @echo %0 :: Install Current Version of salt... @echo --------------------------------------------------------------------- From ce1c1b6d282d2c40bb71d124c2594b5a27a7075b Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 18 Jul 2017 18:51:27 -0400 Subject: [PATCH 116/508] Add initial 2017.7.1 Release Notes File --- doc/topics/releases/2017.7.1.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/topics/releases/2017.7.1.rst diff --git a/doc/topics/releases/2017.7.1.rst b/doc/topics/releases/2017.7.1.rst new file mode 100644 index 0000000000..e879d8639a --- /dev/null +++ b/doc/topics/releases/2017.7.1.rst @@ -0,0 +1,5 @@ +============================ +Salt 2017.7.1 Release Notes +============================ + +Version 2017.7.1 is a bugfix release for :ref:`2017.7.0 `. From e721c7eee2b8bdc38a4a5084a5b53fba7faf3870 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 19 Jul 2017 13:21:29 +0300 Subject: [PATCH 117/508] Don't use `key in weakvaluedict` because it could lie. Example: ```python import weakref class O(object): pass d = weakref.WeakValueDictionary() key = 'key' value = O() d[key] = value d[key] # del value key in d # True d.get(key) # None ``` --- salt/crypt.py | 22 ++++++++++++---------- salt/transport/ipc.py | 11 ++++++----- salt/transport/tcp.py | 11 ++++++----- salt/transport/zeromq.py | 21 ++++++--------------- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/salt/crypt.py b/salt/crypt.py index a5411f6ab9..d330594a2a 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -372,17 +372,18 @@ class AsyncAuth(object): loop_instance_map = AsyncAuth.instance_map[io_loop] key = cls.__key(opts) - if key not in loop_instance_map: + auth = loop_instance_map.get(key) + if auth is None: log.debug('Initializing new AsyncAuth for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_auth = object.__new__(cls) - new_auth.__singleton_init__(opts, io_loop=io_loop) - loop_instance_map[key] = new_auth + auth = object.__new__(cls) + auth.__singleton_init__(opts, io_loop=io_loop) + loop_instance_map[key] = auth else: log.debug('Re-using AsyncAuth for {0}'.format(key)) - return loop_instance_map[key] + return auth @classmethod def __key(cls, opts, io_loop=None): @@ -1008,14 +1009,15 @@ class SAuth(AsyncAuth): Only create one instance of SAuth per __key() ''' key = cls.__key(opts) - if key not in SAuth.instances: + auth = SAuth.instances.get(key) + if auth is None: log.debug('Initializing new SAuth for {0}'.format(key)) - new_auth = object.__new__(cls) - new_auth.__singleton_init__(opts) - SAuth.instances[key] = new_auth + auth = object.__new__(cls) + auth.__singleton_init__(opts) + SAuth.instances[key] = auth else: log.debug('Re-using SAuth for {0}'.format(key)) - return SAuth.instances[key] + return auth @classmethod def __key(cls, opts, io_loop=None): diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index c92904196f..2fa7f03e64 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -248,15 +248,16 @@ class IPCClient(object): # FIXME key = str(socket_path) - if key not in loop_instance_map: + client = loop_instance_map.get(key) + if client is None: log.debug('Initializing new IPCClient for path: {0}'.format(key)) - new_client = object.__new__(cls) + client = object.__new__(cls) # FIXME - new_client.__singleton_init__(io_loop=io_loop, socket_path=socket_path) - loop_instance_map[key] = new_client + client.__singleton_init__(io_loop=io_loop, socket_path=socket_path) + loop_instance_map[key] = client else: log.debug('Re-using IPCClient for {0}'.format(key)) - return loop_instance_map[key] + return client def __singleton_init__(self, socket_path, io_loop=None): ''' diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 8aadfef45e..a9001f03a5 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -221,17 +221,18 @@ class AsyncTCPReqChannel(salt.transport.client.ReqChannel): loop_instance_map = cls.instance_map[io_loop] key = cls.__key(opts, **kwargs) - if key not in loop_instance_map: + obj = loop_instance_map.get(key) + if obj is None: log.debug('Initializing new AsyncTCPReqChannel for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj + obj = object.__new__(cls) + obj.__singleton_init__(opts, **kwargs) + loop_instance_map[key] = obj else: log.debug('Re-using AsyncTCPReqChannel for {0}'.format(key)) - return loop_instance_map[key] + return obj @classmethod def __key(cls, opts, **kwargs): diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 5200c07236..94aeb3e21a 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -80,28 +80,19 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): loop_instance_map = cls.instance_map[io_loop] key = cls.__key(opts, **kwargs) - if key not in loop_instance_map: + obj = loop_instance_map.get(key) + if obj is None: log.debug('Initializing new AsyncZeroMQReqChannel for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj + obj = object.__new__(cls) + obj.__singleton_init__(opts, **kwargs) + loop_instance_map[key] = obj log.trace('Inserted key into loop_instance_map id {0} for key {1} and process {2}'.format(id(loop_instance_map), key, os.getpid())) else: log.debug('Re-using AsyncZeroMQReqChannel for {0}'.format(key)) - try: - return loop_instance_map[key] - except KeyError: - # In iterating over the loop_instance_map, we may have triggered - # garbage collection. Therefore, the key is no longer present in - # the map. Re-gen and add to map. - log.debug('Initializing new AsyncZeroMQReqChannel due to GC for {0}'.format(key)) - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj - return loop_instance_map[key] + return obj def __deepcopy__(self, memo): cls = self.__class__ From 664f4b577be632332d428e5d64742ce7ddb4b402 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Wed, 19 Jul 2017 13:24:15 +0200 Subject: [PATCH 118/508] pillar.items pillar_env & pillar_override are never used Fixes a few bugs in pillar_env determination and pillar_enc decryption --- salt/modules/pillar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index d0beb92a06..7293f39244 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -257,8 +257,8 @@ def items(*args, **kwargs): __opts__, __grains__, __opts__['id'], - pillar_override=kwargs.get('pillar'), - pillarenv=kwargs.get('pillarenv') or __opts__['pillarenv']) + pillar_override=pillar_override, + pillarenv=pillarenv) return pillar.compile_pillar() From aa4eed93c82ac4cd8e8206ed5440ab1d257b9d2f Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 19 Jul 2017 11:22:07 -0600 Subject: [PATCH 119/508] Update Python and other reqs --- pkg/osx/build.sh | 2 +- pkg/osx/{build_env.sh => build_env_2.sh} | 24 +++++----- pkg/osx/req.txt | 47 +++++++++---------- pkg/osx/req_ext.txt | 4 +- pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 | 1 - pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 | 1 + .../shasums/libsodium-1.0.12.tar.gz.sha512 | 1 - .../shasums/libsodium-1.0.13.tar.gz.sha512 | 1 + pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 | 1 - pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 | 1 + .../shasums/pkg-config-0.29.2.tar.gz.sha512 | 1 + pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 | 1 - pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 | 1 - pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 | 1 + 14 files changed, 42 insertions(+), 45 deletions(-) rename pkg/osx/{build_env.sh => build_env_2.sh} (95%) delete mode 100644 pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 create mode 100644 pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 delete mode 100644 pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 create mode 100644 pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 delete mode 100644 pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 create mode 100644 pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 create mode 100644 pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 delete mode 100644 pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 delete mode 100644 pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 create mode 100644 pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index e304248328..ab1f51a16b 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -66,7 +66,7 @@ fi # Create the Build Environment ############################################################################ echo -n -e "\033]0;Build: Build Environment\007" -sudo $PKGRESOURCES/build_env.sh +sudo $PKGRESOURCES/build_env_2.sh ############################################################################ # Install Salt diff --git a/pkg/osx/build_env.sh b/pkg/osx/build_env_2.sh similarity index 95% rename from pkg/osx/build_env.sh rename to pkg/osx/build_env_2.sh index f886bd59ad..3e101fd751 100755 --- a/pkg/osx/build_env.sh +++ b/pkg/osx/build_env_2.sh @@ -121,8 +121,8 @@ BUILDDIR=$SCRIPTDIR/build ############################################################################ echo -n -e "\033]0;Build_Env: pkg-config\007" -PKGURL="http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.tar.gz" -PKGDIR="pkg-config-0.29" +PKGURL="http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" +PKGDIR="pkg-config-0.29.2" download $PKGURL @@ -140,8 +140,8 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: libsodium\007" -PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.12.tar.gz" -PKGDIR="libsodium-1.0.12" +PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.13.tar.gz" +PKGDIR="libsodium-1.0.13" download $PKGURL @@ -159,8 +159,8 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: zeromq\007" -PKGURL="http://download.zeromq.org/zeromq-4.1.3.tar.gz" -PKGDIR="zeromq-4.1.3" +PKGURL="http://download.zeromq.org/zeromq-4.1.4.tar.gz" +PKGDIR="zeromq-4.1.4" download $PKGURL @@ -178,13 +178,13 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: OpenSSL\007" -PKGURL="http://openssl.org/source/openssl-1.0.2f.tar.gz" -PKGDIR="openssl-1.0.2f" +PKGURL="http://openssl.org/source/openssl-1.0.2l.tar.gz" +PKGDIR="openssl-1.0.2l" download $PKGURL echo "################################################################################" -echo "Building OpenSSL 1.0.2f" +echo "Building OpenSSL" echo "################################################################################" cd $PKGDIR ./Configure darwin64-x86_64-cc --prefix=/opt/salt --openssldir=/opt/salt/openssl @@ -197,13 +197,13 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: Python\007" -PKGURL="https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tar.xz" -PKGDIR="Python-2.7.12" +PKGURL="https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz" +PKGDIR="Python-2.7.13" download $PKGURL echo "################################################################################" -echo "Building Python 2.7.12" +echo "Building Python" echo "################################################################################" echo "Note there are some test failures" cd $PKGDIR diff --git a/pkg/osx/req.txt b/pkg/osx/req.txt index 3bee76f5b1..2224830a5a 100644 --- a/pkg/osx/req.txt +++ b/pkg/osx/req.txt @@ -1,34 +1,31 @@ -apache-libcloud==0.20.1 +apache-libcloud==2.1.0 backports.ssl_match_hostname==3.5.0.1 -backports_abc==0.4 +backports_abc==0.5 certifi -cffi==1.5.0 -CherryPy==4.0.0 -click==6.2 -enum34==1.1.2 +cffi==1.10.0 +CherryPy==11.0.0 +click==6.7 +enum34==1.1.6 gitdb==0.6.4 -GitPython==1.0.1 -idna==2.0 -ioflo==1.5.0 -ipaddress==1.0.16 -Jinja2==2.9.4 -libnacl==1.4.4 +GitPython==2.1.5 +idna==2.5 +ipaddress==1.0.18 +Jinja2==2.9.6 linode-python==1.1.1 -Mako==1.0.3 -MarkupSafe==0.23 -msgpack-python==0.4.7 -pyasn1==0.1.9 -pycparser==2.14 +Mako==1.0.7 +MarkupSafe==1.0 +msgpack-python==0.4.8 +pyasn1==0.2.3 +pycparser==2.18 pycrypto==2.6.1 -python-dateutil==2.4.2 -python-gnupg==0.3.8 -PyYAML==3.11 -pyzmq==15.2.0 -raet==0.6.5 -requests==2.9.1 +python-dateutil==2.6.1 +python-gnupg==0.4.1 +PyYAML==3.12 +pyzmq==16.0.2 +requests==2.18.1 singledispatch==3.4.0.3 six==1.10.0 smmap==0.9.0 timelib==0.2.4 -tornado==4.3 -vultr==0.1.2 +tornado==4.5.1 +vultr==1.0rc1 diff --git a/pkg/osx/req_ext.txt b/pkg/osx/req_ext.txt index b31fff2b1d..fac429a942 100644 --- a/pkg/osx/req_ext.txt +++ b/pkg/osx/req_ext.txt @@ -1,2 +1,2 @@ -cryptography==1.2.2 -pyOpenSSL==0.15.1 +cryptography==2.0 +pyOpenSSL==17.1.0 diff --git a/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 b/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 deleted file mode 100644 index 513cf3de5a..0000000000 --- a/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 +++ /dev/null @@ -1 +0,0 @@ -6ddbbce47cc49597433d98ca05c2f62f07ed1070807b645602a8e9e9b996adc6fa66fa20a33cd7d23d4e7e925e25071d7301d288149fbe4e8c5f06d5438dda1f ./Python-2.7.12.tar.xz diff --git a/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 b/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 new file mode 100644 index 0000000000..a57f8db9b9 --- /dev/null +++ b/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 @@ -0,0 +1 @@ +f37c9a28ce129d01e63c84d7db627a06402854578f62d17927334ea21ede318e04bbf66e890e3f47c85333e6b19f6e5581fb3f3e27efd24be27017d1b6529c4b ./Python-2.7.13.tar.xz diff --git a/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 b/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 deleted file mode 100644 index 948a5da6b6..0000000000 --- a/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -1e63960da42bcc90945463ae1f5b1355849881dce5bba6d293391f8d6f0932063a5bfd433a071cb184af90ebeab469acc34710587116922144d61f3d7661901b ./libsodium-1.0.12.tar.gz diff --git a/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 b/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 new file mode 100644 index 0000000000..1b5270adbf --- /dev/null +++ b/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 @@ -0,0 +1 @@ +c619b12fdf0b2e59174b6e383a62d5499ebcd720fdbb2c1a41a98a46c285df075202423454b294fefee185432441e943805397d7656f7cd7837de425da623929 ./libsodium-1.0.13.tar.gz diff --git a/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 b/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 deleted file mode 100644 index b107e52b8a..0000000000 --- a/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -50abf6dc94cafd06e7fd20770808bdc675c88daa369e4f752bd584ab17f72a57357c1ca1eca3c83e6745b5a3c9c73c99dce70adaa904d73f6df4c75bc7138351 ./openssl-1.0.2f.tar.gz diff --git a/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 b/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 new file mode 100644 index 0000000000..9dac0d205f --- /dev/null +++ b/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 @@ -0,0 +1 @@ +047d964508ad6025c79caabd8965efd2416dc026a56183d0ef4de7a0a6769ce8e0b4608a3f8393d326f6d03b26a2b067e6e0c750f35b20be190e595e8290c0e3 ./openssl-1.0.2l.tar.gz diff --git a/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 b/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 new file mode 100644 index 0000000000..beb4354b5a --- /dev/null +++ b/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 @@ -0,0 +1 @@ +4861ec6428fead416f5cbbbb0bbad10b9152967e481d4b0ff2eb396a9f297f552984c9bb72f6864a37dcd8fca1d9ccceda3ef18d8f121938dbe4fdf2b870fe75 ./pkg-config-0.29.2.tar.gz diff --git a/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 b/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 deleted file mode 100644 index 8e19abaabb..0000000000 --- a/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -c2857cd67801c0db5d204912453ff6bdc7da3ea61f8b1c6b38983d48dffb958725e7723f909abbc057c7b34a85c27290eec6943808312a75909306076064aa63 ./pkg-config-0.29.tar.gz diff --git a/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 b/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 deleted file mode 100644 index ea85a1581d..0000000000 --- a/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -2c993d18ea44e1cba890e024176af65b85b842ca4f8a22d319be4ace8388ab8828dd706b065f02754025bf271b1d7aa878c3f6655878248f7826452cb2a6134c ./zeromq-4.1.3.tar.gz diff --git a/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 b/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 new file mode 100644 index 0000000000..4f8932f829 --- /dev/null +++ b/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 @@ -0,0 +1 @@ +8a8cf4f52ad78dddfff104bfba0f80bbc12566920906a0fafb9fc340aa92f5577c2923cb2e5346c69835cd2ea1609647a8893c2883cd22c1f0340a720511460c ./zeromq-4.1.4.tar.gz From d9d94fe02fd5610873e21c34b468a910ec2784c8 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 19 Jul 2017 13:29:42 -0600 Subject: [PATCH 120/508] Update old "ref" references to "rev" in git.detached state Fixes #42381 The "rev" kwarg was added to replace "ref" in #38898, however, when switching the state over to "rev", some stacktraces occur due to some remaining "ref" references. --- salt/states/git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 7a710fee10..a269c9b2d9 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -2213,7 +2213,7 @@ def detached(name, # Determine if supplied ref is a hash remote_rev_type = 'ref' - if len(ref) <= 40 \ + if len(rev) <= 40 \ and all(x in string.hexdigits for x in rev): rev = rev.lower() remote_rev_type = 'hash' @@ -2419,7 +2419,7 @@ def detached(name, https_pass=https_pass, ignore_retcode=False) - if 'refs/remotes/'+remote+'/'+ref in all_remote_refs: + if 'refs/remotes/'+remote+'/'+rev in all_remote_refs: checkout_commit_id = all_remote_refs['refs/remotes/' + remote + '/' + rev] elif 'refs/tags/' + rev in all_remote_refs: checkout_commit_id = all_remote_refs['refs/tags/' + rev] From 246a2b3e748e66e081c61d644f62bd26437f03d1 Mon Sep 17 00:00:00 2001 From: Corvin Mcpherson Date: Wed, 19 Jul 2017 19:29:20 -0400 Subject: [PATCH 121/508] Fix documentation misformat in salt.states.file.replace --- salt/states/file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index af3d58d988..731e891b77 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3797,12 +3797,13 @@ def replace(name, A regular expression, to be matched using Python's :py:func:`~re.search`. - ..note:: + .. note:: + If you need to match a literal string that contains regex special characters, you may want to use salt's custom Jinja filter, ``escape_regex``. - ..code-block:: jinja + .. code-block:: jinja {{ 'http://example.com?foo=bar%20baz' | escape_regex }} From 7b8d6cbbd2ff1c2829e83858409072bc32542242 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 19 Jul 2017 17:30:47 -0600 Subject: [PATCH 122/508] Add support for Py3 --- pkg/osx/build.sh | 29 ++++++++--- pkg/osx/{build_env_2.sh => build_env.sh} | 56 ++++++++++++++------ pkg/osx/build_pkg.sh | 59 ++++++++++++++++------ pkg/osx/distribution.xml.dist | 8 +-- pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 | 1 + 5 files changed, 111 insertions(+), 42 deletions(-) rename pkg/osx/{build_env_2.sh => build_env.sh} (86%) create mode 100644 pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index ab1f51a16b..aaf8c78e0d 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -19,14 +19,16 @@ # $1 : : the version of salt to build # (a git tag, not a branch) # (defaults to git-repo state) -# $2 : : the staging area for the package +# $2 : : The version of Python to use in the +# build. Default is 2 +# $3 : : the staging area for the package # defaults to /tmp/salt_pkg # # Example: -# The following will build Salt v2015.8.3 and stage all files -# in /tmp/custom_pkg: +# The following will build Salt v2015.8.3 with Python 2 and +# stage all files in /tmp/custom_pkg: # -# ./build.sh v2015.8.3 /tmp/custom_pkg +# ./build.sh v2015.8.3 2 /tmp/custom_pkg # ############################################################################ echo -n -e "\033]0;Build: Variables\007" @@ -41,9 +43,15 @@ else fi if [ "$2" == "" ]; then + PYVER=2 +else + PYVER=$2 +fi + +if [ "$3" == "" ]; then PKGDIR=/tmp/salt_pkg else - PKGDIR=$2 + PKGDIR=$3 fi ############################################################################ @@ -51,6 +59,11 @@ fi ############################################################################ SRCDIR=`git rev-parse --show-toplevel` PKGRESOURCES=$SRCDIR/pkg/osx +if [ "$PYVER" == "2" ]; then + PYTHON=/opt/salt/bin/python +else + PYTHON=/opt/salt/bin/python3 +fi ############################################################################ # Make sure this is the Salt Repository @@ -66,16 +79,16 @@ fi # Create the Build Environment ############################################################################ echo -n -e "\033]0;Build: Build Environment\007" -sudo $PKGRESOURCES/build_env_2.sh +sudo $PKGRESOURCES/build_env.sh $PYVER ############################################################################ # Install Salt ############################################################################ echo -n -e "\033]0;Build: Install Salt\007" -sudo /opt/salt/bin/python $SRCDIR/setup.py install +sudo $PYTHON $SRCDIR/setup.py install ############################################################################ # Build Package ############################################################################ echo -n -e "\033]0;Build: Package Salt\007" -sudo $PKGRESOURCES/build_pkg.sh $VERSION $PKGDIR +sudo $PKGRESOURCES/build_pkg.sh $VERSION $PYVER $PKGDIR diff --git a/pkg/osx/build_env_2.sh b/pkg/osx/build_env.sh similarity index 86% rename from pkg/osx/build_env_2.sh rename to pkg/osx/build_env.sh index 3e101fd751..3e8ad8dfa5 100755 --- a/pkg/osx/build_env_2.sh +++ b/pkg/osx/build_env.sh @@ -6,18 +6,21 @@ # Authors: CR Oldham, Shane Lee # Date: December 2015 # -# Description: This script sets up a build environment for salt on macOS. +# Description: This script sets up a build environment for Salt on macOS. # # Requirements: # - XCode Command Line Tools (xcode-select --install) # # Usage: -# This script is not passed any parameters +# This script can be passed 1 parameter +# $1 : : the version of Python to use for the +# build environment. Default is 2 # # Example: -# The following will set up a build environment for salt on macOS +# The following will set up a Python 3 build environment for Salt +# on macOS # -# ./dev_env.sh +# ./dev_env.sh 3 # ############################################################################ @@ -31,6 +34,15 @@ quit_on_error() { exit -1 } +############################################################################ +# Check passed parameters, set defaults +############################################################################ +if [ "$1" == "" ]; then + PYVER=2 +else + PYVER=$1 +fi + ############################################################################ # Parameters Required for the script to function properly ############################################################################ @@ -45,6 +57,15 @@ SHADIR=$SCRIPTDIR/shasums PKG_CONFIG_PATH=/opt/salt/lib/pkgconfig CFLAGS="-I/opt/salt/include" LDFLAGS="-L/opt/salt/lib" +if [ "$PYVER" == "2" ]; then + PYDIR=/opt/salt/lib/python2.7 + PYTHON=/opt/salt/bin/python + PIP=/opt/salt/bin/pip +else + PYDIR=/opt/salt/lib/python3.5 + PYTHON=/opt/salt/bin/python3 + PIP=/opt/salt/bin/pip3 +fi ############################################################################ # Determine Which XCode is being used (XCode or XCode Command Line Tools) @@ -197,8 +218,13 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: Python\007" -PKGURL="https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz" -PKGDIR="Python-2.7.13" +if [ "$PYVER" == "2" ]; then + PKGURL="https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz" + PKGDIR="Python-2.7.13" +else + PKGURL="https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tar.xz" + PKGDIR="Python-3.5.3" +fi download $PKGURL @@ -227,23 +253,23 @@ cd $BUILDDIR echo "################################################################################" echo "Installing Salt Dependencies with pip (normal)" echo "################################################################################" -sudo -H /opt/salt/bin/pip install \ - -r $SRCDIR/pkg/osx/req.txt \ - --no-cache-dir +sudo -H $PIP install \ + -r $SRCDIR/pkg/osx/req.txt \ + --no-cache-dir echo "################################################################################" echo "Installing Salt Dependencies with pip (build_ext)" echo "################################################################################" -sudo -H /opt/salt/bin/pip install \ - -r $SRCDIR/pkg/osx/req_ext.txt \ - --global-option=build_ext \ - --global-option="-I/opt/salt/include" \ - --no-cache-dir +sudo -H $PIP install \ + -r $SRCDIR/pkg/osx/req_ext.txt \ + --global-option=build_ext \ + --global-option="-I/opt/salt/include" \ + --no-cache-dir echo "--------------------------------------------------------------------------------" echo "Create Symlink to certifi for openssl" echo "--------------------------------------------------------------------------------" -sudo ln -s /opt/salt/lib/python2.7/site-packages/certifi/cacert.pem /opt/salt/openssl/cert.pem +sudo ln -s $PYDIR/site-packages/certifi/cacert.pem /opt/salt/openssl/cert.pem echo -n -e "\033]0;Build_Env: Finished\007" diff --git a/pkg/osx/build_pkg.sh b/pkg/osx/build_pkg.sh index 86ad9aa450..80f5c4f734 100755 --- a/pkg/osx/build_pkg.sh +++ b/pkg/osx/build_pkg.sh @@ -15,13 +15,16 @@ # This script can be passed 2 parameters # $1 : : the version name to give the package (overrides # version of the git repo) (Defaults to the git repo version) -# $2 : : the staging area for the package defaults to +# $2 : : the version of python that was built (defaults +# to 2) +# $3 : : the staging area for the package defaults to # /tmp/salt_pkg # # Example: -# The following will build Salt and stage all files in /tmp/salt_pkg: +# The following will build Salt version 2017.7.0 with Python 3 and +# stage all files in /tmp/salt_pkg: # -# ./build.sh +# ./build.sh 2017.7.0 3 # ############################################################################ @@ -45,11 +48,18 @@ else VERSION=$1 fi -# Get/Set temp directory +# Get/Set Python Version if [ "$2" == "" ]; then + PYVER=2 +else + PYVER=$2 +fi + +# Get/Set temp directory +if [ "$3" == "" ]; then PKGDIR=/tmp/salt_pkg else - PKGDIR=$2 + PKGDIR=$3 fi CPUARCH=`uname -m` @@ -114,7 +124,11 @@ sudo rm -rdf $PKGDIR/opt/salt/lib/engines sudo rm -rdf $PKGDIR/opt/salt/share/aclocal sudo rm -rdf $PKGDIR/opt/salt/share/doc sudo rm -rdf $PKGDIR/opt/salt/share/man/man1/pkg-config.1 -sudo rm -rdf $PKGDIR/opt/salt/lib/python2.7/test +if [ "$PYVER" == "2" ]; then + sudo rm -rdf $PKGDIR/opt/salt/lib/python2.7/test +else + sudo rm -rdf $PKGDIR/opt/salt/lib/python3.5/test +fi echo -n -e "\033]0;Build_Pkg: Remove compiled python files\007" sudo find $PKGDIR/opt/salt -name '*.pyc' -type f -delete @@ -133,15 +147,30 @@ cp $SRCDIR/conf/master $PKGDIR/etc/salt/master.dist ############################################################################ echo -n -e "\033]0;Build_Pkg: Add Version to .xml\007" +if [ "$PYVER" == "2" ]; then + TITLE="Salt $VERSION" + DESC="Salt $VERSION with Python 2" +else + TITLE="Salt $VERSION (Python 3)" + DESC="Salt $VERSION with Python 3" +fi + cd $PKGRESOURCES cp distribution.xml.dist distribution.xml -SEDSTR="s/@VERSION@/$VERSION/" -echo $SEDSTR -sed -i '' $SEDSTR distribution.xml +SEDSTR="s/@TITLE@/$TITLE/g" +sed -E -i '' "$SEDSTR" distribution.xml -SEDSTR="s/@CPUARCH@/$CPUARCH/" -echo $SEDSTR -sed -i '' $SEDSTR distribution.xml +SEDSTR="s/@DESC@/$DESC/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@VERSION@/$VERSION/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@PYVER@/$PYVER/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@CPUARCH@/$CPUARCH/g" +sed -i '' "$SEDSTR" distribution.xml ############################################################################ # Build the Package @@ -152,10 +181,10 @@ pkgbuild --root=$PKGDIR \ --scripts=pkg-scripts \ --identifier=com.saltstack.salt \ --version=$VERSION \ - --ownership=recommended salt-src-$VERSION-$CPUARCH.pkg + --ownership=recommended salt-src-$VERSION-py$PYVER-$CPUARCH.pkg productbuild --resources=pkg-resources \ --distribution=distribution.xml \ - --package-path=salt-src-$VERSION-$CPUARCH.pkg \ - --version=$VERSION salt-$VERSION-$CPUARCH.pkg + --package-path=salt-src-$VERSION-py$PYVER-$CPUARCH.pkg \ + --version=$VERSION salt-$VERSION-py$PYVER-$CPUARCH.pkg diff --git a/pkg/osx/distribution.xml.dist b/pkg/osx/distribution.xml.dist index 083fef44f6..d31063f5f4 100644 --- a/pkg/osx/distribution.xml.dist +++ b/pkg/osx/distribution.xml.dist @@ -1,6 +1,6 @@ - Salt @VERSION@ + @TITLE@ com.saltstack.salt @@ -25,7 +25,7 @@ salt-src-@VERSION@-@CPUARCH@.pkg + auth="root">salt-src-@VERSION@-py@PYVER@-@CPUARCH@.pkg @@ -34,8 +34,8 @@ diff --git a/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 b/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 new file mode 100644 index 0000000000..d4ab61d5f7 --- /dev/null +++ b/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 @@ -0,0 +1 @@ +bbcc20e315c63dbc8901d7e7bfa29d4dbdad9335720757d8d679730319fd1d9fcfdb55cf62d620c9b052134170f162c28d653a8af60923185b8932524d827864 ./Python-3.5.3.tar.xz From 4ae3911f0143c2e31e4559a0ae263e069d83afe0 Mon Sep 17 00:00:00 2001 From: Richard Clark Date: Wed, 19 Jul 2017 20:30:49 -0700 Subject: [PATCH 123/508] Fix file.managed check_cmd file not found - Issue #42404 --- salt/states/file.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 731e891b77..de86f8c68b 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -2433,12 +2433,16 @@ def managed(name, if sfn and os.path.isfile(sfn): os.remove(sfn) return ret + + if sfn and os.path.isfile(sfn): + os.remove(sfn) + # Since we generated a new tempfile and we are not returning here # lets change the original sfn to the new tempfile or else we will # get file not found - if sfn and os.path.isfile(sfn): - os.remove(sfn) - sfn = tmp_filename + + sfn = tmp_filename + else: ret = {'changes': {}, 'comment': '', From 1c0574d05e68a2f519c6b5f0857fc66894c48711 Mon Sep 17 00:00:00 2001 From: Emmanuel Bouton Date: Thu, 20 Jul 2017 17:01:10 +0200 Subject: [PATCH 124/508] Fix error message when tornado or pycurl is not installed --- salt/utils/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index 0a3c32e3a9..e34280cbfb 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -456,8 +456,8 @@ def query(url, # We want to use curl_http if we have a proxy defined if proxy_host and proxy_port: if HAS_CURL_HTTPCLIENT is False: - ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl, but the ' - 'pycurl library does not seem to be installed') + ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl and tornado, ' + 'but the libraries does not seem to be installed') log.error(ret['error']) return ret From 9d66e273c413527468c16881a5a2ba2496fac855 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 20 Jul 2017 10:37:08 -0600 Subject: [PATCH 125/508] Fix hard coded pip path --- pkg/osx/build_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osx/build_env.sh b/pkg/osx/build_env.sh index 3e8ad8dfa5..5d7f1bac5a 100755 --- a/pkg/osx/build_env.sh +++ b/pkg/osx/build_env.sh @@ -241,7 +241,7 @@ sudo -H $MAKE install ############################################################################ # upgrade pip ############################################################################ -sudo -H /opt/salt/bin/pip install --upgrade pip +sudo -H $PIP install --upgrade pip ############################################################################ # Download and install salt python dependencies From ac0e04af72caf61ca8c172b1f0dfda8663711bac Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 20 Jul 2017 10:59:24 -0600 Subject: [PATCH 126/508] Remove build and dist, sign pkgs --- pkg/osx/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index aaf8c78e0d..c411840a91 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -64,6 +64,7 @@ if [ "$PYVER" == "2" ]; then else PYTHON=/opt/salt/bin/python3 fi +CPUARCH=`uname -m` ############################################################################ # Make sure this is the Salt Repository @@ -85,6 +86,8 @@ sudo $PKGRESOURCES/build_env.sh $PYVER # Install Salt ############################################################################ echo -n -e "\033]0;Build: Install Salt\007" +sudo rm -rm $SRCDIR/build +sudo rm -rm $SRCDIR/dist sudo $PYTHON $SRCDIR/setup.py install ############################################################################ @@ -92,3 +95,8 @@ sudo $PYTHON $SRCDIR/setup.py install ############################################################################ echo -n -e "\033]0;Build: Package Salt\007" sudo $PKGRESOURCES/build_pkg.sh $VERSION $PYVER $PKGDIR + +############################################################################ +# Sign Package +############################################################################ +sudo $PKGRESOURCES/build_sig.sh salt-$VERSION-py$PYVER-$CPUARCH.pkg salt-$VERSION-py$PYVER-$CPUARCH-signed.pkg From 9f4eb80d9072a3fe0e7f3d23528f57b928561f79 Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 20 Jul 2017 14:26:30 -0600 Subject: [PATCH 127/508] Add a test.arg variant that cleans the pub kwargs by default --- salt/modules/test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/modules/test.py b/salt/modules/test.py index 1dc0c28a33..6094b05981 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -311,6 +311,18 @@ def arg_repr(*args, **kwargs): return {"args": repr(args), "kwargs": repr(kwargs)} +def arg_clean(*args, **kwargs): + ''' + Like test.arg but cleans kwargs of the __pub* items + CLI Example: + + .. code-block:: bash + + salt '*' test.arg_clean 1 "two" 3.1 txt="hello" wow='{a: 1, b: "hello"}' + ''' + return dict(args=args, kwargs=salt.utils.clean_kwargs(**kwargs)) + + def fib(num): ''' Return the num-th Fibonacci number, and the time it took to compute in From 622ff5be40df34dadf1f83c4f1dfb77fabdcf25a Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 20 Jul 2017 10:48:04 -0600 Subject: [PATCH 128/508] Add LocalClient.cmd test for arg/kwarg parsing --- tests/integration/client/test_kwarg.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/integration/client/test_kwarg.py b/tests/integration/client/test_kwarg.py index bb5b369de0..a9ec8d27ff 100644 --- a/tests/integration/client/test_kwarg.py +++ b/tests/integration/client/test_kwarg.py @@ -6,7 +6,6 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.case import ModuleCase - class StdTest(ModuleCase): ''' Test standard client calls @@ -94,3 +93,25 @@ class StdTest(ModuleCase): ret = self.client.cmd('minion', 'test.ping', full_return=True) for mid, data in ret.items(): self.assertIn('retcode', data) + + def test_cmd_arg_kwarg_parsing(self): + ret = self.client.cmd('minion', 'test.arg_clean', + arg=[ + 'foo', + 'bar=off', + 'baz={qux: 123}' + ], + kwarg={ + 'quux': 'Quux', + }) + + self.assertEqual(ret['minion'], { + 'args': ['foo'], + 'kwargs': { + 'bar': False, + 'baz': { + 'qux': 123, + }, + 'quux': 'Quux', + }, + }) From 31273c7ec1ee14cdde4f8780280cb0c8019d6b30 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 20 Jul 2017 15:03:14 -0500 Subject: [PATCH 129/508] Modify our custom YAML loader to treat unicode literals as unicode strings --- salt/utils/yamlloader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/utils/yamlloader.py b/salt/utils/yamlloader.py index a51d1f1f25..3d07c005c7 100644 --- a/salt/utils/yamlloader.py +++ b/salt/utils/yamlloader.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import warnings # Import third party libs +import re import yaml from yaml.nodes import MappingNode, SequenceNode from yaml.constructor import ConstructorError @@ -101,6 +102,11 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): # an empty string. Change it to '0'. if node.value == '': node.value = '0' + elif node.tag == 'tag:yaml.org,2002:str': + # If any string comes in as a quoted unicode literal, eval it into + # the proper unicode string type. + if re.match(r'^u([\'"]).+\1$', node.value, flags=re.IGNORECASE): + node.value = eval(node.value, {}, {}) # pylint: disable=W0123 return super(SaltYamlSafeLoader, self).construct_scalar(node) def flatten_mapping(self, node): From 0fd39498c016e1cb3373872bc8fc9385a8ecea99 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Jul 2017 13:48:01 -0700 Subject: [PATCH 130/508] Updating the versions function inside the manage runner to account for when a minion is offline and we are unable to determine it's version. --- salt/runners/manage.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index 249c875c57..ebf6471d9a 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -652,6 +652,7 @@ def versions(): return ret labels = { + -2: 'Minion offline', -1: 'Minion requires update', 0: 'Up to date', 1: 'Minion newer than master', @@ -663,12 +664,19 @@ def versions(): master_version = salt.version.__saltstack_version__ for minion in minions: - minion_version = salt.version.SaltStackVersion.parse(minions[minion]) - ver_diff = cmp(minion_version, master_version) + if not minions[minion]: + minion_version = False + ver_diff = -2 + else: + minion_version = salt.version.SaltStackVersion.parse(minions[minion]) + ver_diff = cmp(minion_version, master_version) if ver_diff not in version_status: version_status[ver_diff] = {} - version_status[ver_diff][minion] = minion_version.string + if minion_version: + version_status[ver_diff][minion] = minion_version.string + else: + version_status[ver_diff][minion] = minion_version # Add version of Master to output version_status[2] = master_version.string From 540977b4b1bc2468c9eac8f216d0b796e3a4f4df Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Thu, 29 Jun 2017 10:10:47 -0500 Subject: [PATCH 131/508] Fix: Reactor emits critical error Under normal usage, the reactor will emit this critical error: ``` [CRITICAL] kwargs must be passed inside the low data within the 'kwarg' key. See usage of salt.utils.args.parse_input() and salt.minion.load_args_and_kwargs() elsewhere in the codebase. ``` It seems like only `salt.utils.reactor.Reactor` uses `salt.state.Compiler`. Due to this, it appears safe to customize `Compiler.compile_high_data` for usage only by the reactor. Since reactor arguments are always named, we ensure that each 'chunk' has an `arg` field that is an empty list and a `kwarg` field that contains all the named arguments for use with the given function call. This conforms to the format expected by `salt.client.mixins.SyncClientMixin._low`. Signed-off-by: Sergey Kizunov --- salt/state.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 44fb73b97d..d3d7cd3f81 100644 --- a/salt/state.py +++ b/salt/state.py @@ -577,6 +577,8 @@ class Compiler(object): if '__env__' in body: chunk['__env__'] = body['__env__'] chunk['__id__'] = name + chunk['arg'] = [] + chunk['kwarg'] = {} for arg in run: if isinstance(arg, six.string_types): funcs.add(arg) @@ -589,7 +591,7 @@ class Compiler(object): names.append(_name) continue else: - chunk.update(arg) + chunk['kwarg'].update(arg) if names: name_order = 1 for entry in names: From 635810b3e3bc03a9a733ba63203238a983548c9a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Jul 2017 16:50:42 -0700 Subject: [PATCH 132/508] Updating the slack engine in 2016.11 to pass the args and kwrags correctly to LocalClient --- salt/engines/slack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 6b37244daf..1ad9e05ce6 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -202,7 +202,7 @@ def start(token, # Default to trying to run as a client module. else: local = salt.client.LocalClient() - ret = local.cmd('{0}'.format(target), cmd, args, kwargs) + ret = local.cmd('{0}'.format(target), cmd, arg=args, kwarg=kwargs) if ret: return_text = json.dumps(ret, sort_keys=True, indent=1) From f411cfc2a9ce5d65e372bc7c28afec8f09fa15dc Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Jul 2017 16:57:10 -0700 Subject: [PATCH 133/508] Updating the slack engine in 2017.7 to pass the args and kwrags correctly to LocalClient --- salt/engines/slack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 1709364832..efddfda0ea 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -255,7 +255,7 @@ def start(token, # Default to trying to run as a client module. else: local = salt.client.LocalClient() - ret = local.cmd('{0}'.format(target), cmd, args, kwargs, tgt_type='{0}'.format(tgt_type)) + ret = local.cmd('{0}'.format(target), cmd, arg=args, kwarg=kwargs, tgt_type='{0}'.format(tgt_type)) if ret: return_text = json.dumps(ret, sort_keys=True, indent=1) From 3f4a918f733b3875ef31af0c685ad4b779f26998 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 21 Jul 2017 09:45:30 -0400 Subject: [PATCH 134/508] update windows urls to new py2/py3 naming scheme --- doc/conf.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 1ecfdfcd99..6e828afc77 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -320,11 +320,21 @@ rst_prolog = """\ .. _`salt-packagers`: https://groups.google.com/forum/#!forum/salt-packagers .. |windownload| raw:: html -

x86: Salt-Minion-{release}-x86-Setup.exe - | md5

+

Python2 x86: Salt-Minion-{release}-x86-Setup.exe + | md5

+ +

Python2 AMD64: Salt-Minion-{release}-AMD64-Setup.exe + | md5

+

Python3 x86: Salt-Minion-{release}-x86-Setup.exe + | md5

+ +

Python3 AMD64: Salt-Minion-{release}-AMD64-Setup.exe + | md5

-

AMD64: Salt-Minion-{release}-AMD64-Setup.exe - | md5

.. |osxdownload| raw:: html From 98b661406ef90f0a75eef418a0df54af2aab34d5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Jul 2017 09:21:38 -0500 Subject: [PATCH 135/508] Document future renaming of new rand_str jinja filter The name and the documentation for this filter are entirely inaccurate for what it actually does. It will be renamed in the Oxygen release. --- doc/topics/jinja/index.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 5548b29646..9c0dcdfb38 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -876,16 +876,22 @@ Returns: ------------ .. versionadded:: 2017.7.0 +.. versionadded:: Oxygen + Renamed from ``rand_str`` to ``random_hash`` to more accurately describe + what the filter does. -Generate a random string and applies a hash. Default hashing: md5. +Generates a random number between 1 and the number passed to the filter, and +then hashes it. The default hash type is the one specified by the minion's +:conf_minion:`hash_type` config option, but an alternate hash type can be +passed to the filter as an argument. Example: .. code-block:: jinja - {% set passwd_length = 17 %} - {{ passwd_length | rand_str }} - {{ passwd_length | rand_str('sha512') }} + {% set num_range = 99999999 %} + {{ num_range | rand_str }} + {{ num_range | rand_str('sha512') }} Returns: From ea457aa0a564977668791e13e235788e013d55b8 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 21 Jul 2017 09:08:26 -0600 Subject: [PATCH 136/508] Remove ALIASES block from template util These alias warnings were removed in #38218 for 2017.7. --- salt/utils/templates.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 8a2514e796..b4bf049dc1 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -73,11 +73,7 @@ class AliasedLoader(object): return getattr(self.wrapped, name) def __contains__(self, name): - if name in ALIASES: - salt.utils.warn_until('Nitrogen', ALIAS_WARN) - return ALIASES[name] in self.wrapped - else: - return name in self.wrapped + return name in self.wrapped class AliasedModule(object): From c0df0137f5f9c743bbbe512b3904aec0b6264200 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 21 Jul 2017 10:18:20 -0600 Subject: [PATCH 137/508] Bump warning version from Oxygen to Fluorine in roster cache The change was introduced in 2017.7 - we need to provide at least 2 feature releases before removing this warning. Therefore, this PR bumps the version we will warn until from Oxygen to Fluorine. --- salt/roster/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/roster/cache.py b/salt/roster/cache.py index bb03b5f0f3..cf4244ff3a 100644 --- a/salt/roster/cache.py +++ b/salt/roster/cache.py @@ -129,7 +129,7 @@ def targets(tgt, tgt_type='glob', **kwargs): # pylint: disable=W0613 'host': ('ipv6-private', 'ipv6-global', 'ipv4-private', 'ipv4-public') }) if isinstance(roster_order, (tuple, list)): - salt.utils.warn_until('Oxygen', + salt.utils.warn_until('Fluorine', 'Using legacy syntax for roster_order') roster_order = { 'host': roster_order @@ -137,7 +137,7 @@ def targets(tgt, tgt_type='glob', **kwargs): # pylint: disable=W0613 for config_key, order in roster_order.items(): for idx, key in enumerate(order): if key in ('public', 'private', 'local'): - salt.utils.warn_until('Oxygen', + salt.utils.warn_until('Fluorine', 'roster_order {0} will include IPv6 soon. ' 'Set order to ipv4-{0} if needed.'.format(key)) order[idx] = 'ipv4-' + key From 1920dc60793dba34904c686b815ead0b8fbbd94c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Jul 2017 10:00:35 -0700 Subject: [PATCH 138/508] Uncomment the line that removes the temporary identity file. --- salt/modules/git.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index b6592a4607..304276187b 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -273,10 +273,10 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, if not salt.utils.is_windows() and 'GIT_SSH' in env: os.remove(env['GIT_SSH']) - # Cleanup the temporary identify file + # Cleanup the temporary identity file if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - #__salt__['file.remove'](tmp_identity_file) + log.debug('Removing identity file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From ff24102d51206a1db6f5ad6b89360761b4077aec Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Jul 2017 10:01:55 -0700 Subject: [PATCH 139/508] Uncomment the line that removes the temporary identity file. --- salt/modules/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 4f1101b755..4b403fe029 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -261,7 +261,7 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # Cleanup the temporary identify file if tmp_identity_file and os.path.exists(tmp_identity_file): log.debug('Removing identify file {0}'.format(tmp_identity_file)) - #__salt__['file.remove'](tmp_identity_file) + __salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From 66093738c8c6d1b5a84d7c8959b2bf74c132d065 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Jul 2017 10:59:22 -0500 Subject: [PATCH 140/508] Add back support for string kwargs --- salt/minion.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 03e0c0cfa3..bc469b63fd 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -306,14 +306,16 @@ def load_args_and_kwargs(func, args, data=None, ignore_invalid=False): else: string_kwarg = salt.utils.args.parse_input([arg], condition=False)[1] # pylint: disable=W0632 if string_kwarg: - log.critical( - 'String kwarg(s) %s passed to ' - 'salt.minion.load_args_and_kwargs(). This is no longer ' - 'supported, so the kwarg(s) will be ignored. Arguments ' - 'passed to salt.minion.load_args_and_kwargs() should be ' - 'passed to salt.utils.args.parse_input() first to load ' - 'and condition them properly.', string_kwarg - ) + if argspec.keywords or next(six.iterkeys(string_kwarg)) in argspec.args: + # Function supports **kwargs or is a positional argument to + # the function. + _kwargs.update(string_kwarg) + else: + # **kwargs not in argspec and parsed argument name not in + # list of positional arguments. This keyword argument is + # invalid. + for key, val in six.iteritems(string_kwarg): + invalid_kwargs.append('{0}={1}'.format(key, val)) else: _args.append(arg) From 99ec634c6be648bdd68633f560b6cda6bd75684a Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 21 Jul 2017 15:19:31 -0600 Subject: [PATCH 141/508] validate ssh_interface for ec2 and default to public_ips --- salt/cloud/clouds/ec2.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index d6f9edcc6e..5223b80457 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -961,10 +961,18 @@ def ssh_interface(vm_): Return the ssh_interface type to connect to. Either 'public_ips' (default) or 'private_ips'. ''' - return config.get_cloud_config_value( + ret = config.get_cloud_config_value( 'ssh_interface', vm_, __opts__, default='public_ips', search_global=False ) + if not ret in ('public_ips', 'private_ips'): + log.warning(( + 'Invalid ssh_interface: {0}. ' + 'Allowed options are ("public_ips", "private_ips"). ' + 'Defaulting to "public_ips".' + ).format(ret)) + ret = 'public_ips' + return ret def get_ssh_gateway_config(vm_): From 0cc0c0967a8ffc5908d275c9800e98ec1ee4213a Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 21 Jul 2017 16:14:32 -0600 Subject: [PATCH 142/508] Lint fixes --- salt/minion.py | 2 +- tests/integration/client/test_kwarg.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index bc469b63fd..3c5046ee93 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -315,7 +315,7 @@ def load_args_and_kwargs(func, args, data=None, ignore_invalid=False): # list of positional arguments. This keyword argument is # invalid. for key, val in six.iteritems(string_kwarg): - invalid_kwargs.append('{0}={1}'.format(key, val)) + invalid_kwargs.append('{0}={1}'.format(key, val)) else: _args.append(arg) diff --git a/tests/integration/client/test_kwarg.py b/tests/integration/client/test_kwarg.py index a9ec8d27ff..ac4d69a0e7 100644 --- a/tests/integration/client/test_kwarg.py +++ b/tests/integration/client/test_kwarg.py @@ -6,6 +6,7 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.case import ModuleCase + class StdTest(ModuleCase): ''' Test standard client calls From 102509029e5fd32f4a86c8a44c22314082c744ef Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 21 Jul 2017 16:36:07 -0600 Subject: [PATCH 143/508] Remove chown mock, fix path seps --- tests/unit/test_crypt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_crypt.py b/tests/unit/test_crypt.py index a26b7583f8..e5713fb641 100644 --- a/tests/unit/test_crypt.py +++ b/tests/unit/test_crypt.py @@ -88,13 +88,13 @@ SIG = ( class CryptTestCase(TestCase): def test_gen_keys(self): - with patch.multiple(os, umask=MagicMock(), chmod=MagicMock(), chown=MagicMock, + with patch.multiple(os, umask=MagicMock(), chmod=MagicMock(), access=MagicMock(return_value=True)): with patch('salt.utils.fopen', mock_open()): - open_priv_wb = call('/keydir/keyname.pem', 'wb+') - open_pub_wb = call('/keydir/keyname.pub', 'wb+') + open_priv_wb = call('/keydir{0}keyname.pem'.format(os.sep), 'wb+') + open_pub_wb = call('/keydir{0}keyname.pub'.format(os.sep), 'wb+') with patch('os.path.isfile', return_value=True): - self.assertEqual(crypt.gen_keys('/keydir', 'keyname', 2048), '/keydir/keyname.pem') + self.assertEqual(crypt.gen_keys('/keydir', 'keyname', 2048), '/keydir{0}keyname.pem'.format(os.sep)) self.assertNotIn(open_priv_wb, salt.utils.fopen.mock_calls) self.assertNotIn(open_pub_wb, salt.utils.fopen.mock_calls) with patch('os.path.isfile', return_value=False): From 0b548c72e1f201467f0caa656721ae6b58dfa809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Fusil-Delahaye?= Date: Sat, 22 Jul 2017 13:01:49 +0200 Subject: [PATCH 144/508] Fix a potential Exception with an explicit error message if sign_remote_certificate is not authorized on master, execution fail with explicite error. --- salt/modules/x509.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/modules/x509.py b/salt/modules/x509.py index 986f137a82..5fc7a97a63 100644 --- a/salt/modules/x509.py +++ b/salt/modules/x509.py @@ -1373,10 +1373,19 @@ def create_certificate( ['listen_in', 'preqrequired', '__prerequired__']: kwargs.pop(ignore, None) - cert_txt = __salt__['publish.publish']( + certs = __salt__['publish.publish']( tgt=ca_server, fun='x509.sign_remote_certificate', - arg=str(kwargs))[ca_server] + arg=str(kwargs)) + + if not any(certs): + raise salt.exceptions.SaltInvocationError( + 'ca_server did not respond' + ' salt master must permit peers to' + ' call the sign_remote_certificate function.') + + cert_txt = certs[ca_server] + if path: return write_pem( text=cert_txt, From 92f18907011b80c72577524a465c910c8943f6f9 Mon Sep 17 00:00:00 2001 From: Aurelien Fusil-Delahaye Date: Mon, 24 Jul 2017 14:12:23 +0200 Subject: [PATCH 145/508] Fix azurerm query to show IPs azurearm.py edited to show IPs instead of interfaces id at queries --- salt/cloud/clouds/azurearm.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py index 8d346c9a0e..6184a89c49 100644 --- a/salt/cloud/clouds/azurearm.py +++ b/salt/cloud/clouds/azurearm.py @@ -407,13 +407,14 @@ def list_nodes_full(conn=None, call=None): # pylint: disable=unused-argument for group in list_resource_groups(): nodes = compconn.virtual_machines.list(group) for node in nodes: + private_ips, public_ips = __get_ips_from_node(group, node) ret[node.name] = object_to_dict(node) ret[node.name]['id'] = node.id ret[node.name]['name'] = node.name ret[node.name]['size'] = node.hardware_profile.vm_size ret[node.name]['state'] = node.provisioning_state - ret[node.name]['private_ips'] = node.network_profile.network_interfaces - ret[node.name]['public_ips'] = node.network_profile.network_interfaces + ret[node.name]['private_ips'] = private_ips + ret[node.name]['public_ips'] = public_ips ret[node.name]['storage_profile']['data_disks'] = [] ret[node.name]['resource_group'] = group for disk in node.storage_profile.data_disks: @@ -433,6 +434,30 @@ def list_nodes_full(conn=None, call=None): # pylint: disable=unused-argument return ret +def __get_ips_from_node(resource_group, node): + ''' + List private and public IPs from a VM interface + ''' + global netconn # pylint: disable=global-statement,invalid-name + if not netconn: + netconn = get_conn(NetworkManagementClient) + + private_ips = [] + public_ips = [] + for node_iface in node.network_profile.network_interfaces: + node_iface_name = node_iface.id.split('/')[-1] + network_interface = netconn.network_interfaces.get(resource_group, node_iface_name) + for ip_configuration in network_interface.ip_configurations: + if ip_configuration.private_ip_address: + private_ips.append(ip_configuration.private_ip_address) + if ip_configuration.public_ip_address and ip_configuration.public_ip_address.id: + public_iface_name = ip_configuration.public_ip_address.id.split('/')[-1] + public_iface = netconn.public_ip_addresses.get(resource_group, public_iface_name) + public_ips.append(public_iface.ip_address) + + return private_ips, public_ips + + def list_resource_groups(conn=None, call=None): # pylint: disable=unused-argument ''' List resource groups associated with the account From 6352f447ce6669c6951d198e2b39b57c16b698f9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 24 Jul 2017 08:50:44 -0500 Subject: [PATCH 146/508] Add PER_REMOTE_ONLY to init_remotes call in git_pillar runner When the mountpoint feature was added, it added a per-remote-only parameter called `mountpoint`. While this is reflected in salt.pillar.git_pillar, it was not in salt.runners.git_pillar. This corrects that oversight, fixing a traceback when the `git_pillar.update` runner is executed and one or more remotes have a `mountpoint` parameter configured. --- salt/runners/git_pillar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/runners/git_pillar.py b/salt/runners/git_pillar.py index 8dfc4412eb..0e8e97beb3 100644 --- a/salt/runners/git_pillar.py +++ b/salt/runners/git_pillar.py @@ -86,7 +86,8 @@ def update(branch=None, repo=None): else: pillar = salt.utils.gitfs.GitPillar(__opts__) pillar.init_remotes(pillar_conf, - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES) + salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + salt.pillar.git_pillar.PER_REMOTE_ONLY) for remote in pillar.remotes: # Skip this remote if it doesn't match the search criteria if branch is not None: From 72924b06b807a4e2442e227015548324a1b83be2 Mon Sep 17 00:00:00 2001 From: clem-compilatio Date: Mon, 24 Jul 2017 16:59:19 +0200 Subject: [PATCH 147/508] Fix _assign_floating_ips in openstack.py Fixes #42417 In salt/cloud/clouds/openstack.py : - _assign_floating_ips() should not raise SaltCloudSystemExit() just yet. - Instead, it should let _query_node_data() a chance to get the IP from node['public_ips']. --- salt/cloud/clouds/openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index e73af9e0e5..775173a348 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -907,9 +907,9 @@ def _assign_floating_ips(vm_, conn, kwargs): floating.append(idx) break if not floating: - raise SaltCloudSystemExit( + log.warning( 'There are no more floating IP addresses ' - 'available, please create some more' + 'available, please create some more if necessary' ) except Exception as e: if str(e).startswith('404'): From c32c1b2803369baaddf3678533f04ee2a47e7235 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 11:31:25 -0600 Subject: [PATCH 148/508] fix pylint --- salt/cloud/clouds/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index 5223b80457..f47d2d93c3 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -965,7 +965,7 @@ def ssh_interface(vm_): 'ssh_interface', vm_, __opts__, default='public_ips', search_global=False ) - if not ret in ('public_ips', 'private_ips'): + if ret not in ('public_ips', 'private_ips'): log.warning(( 'Invalid ssh_interface: {0}. ' 'Allowed options are ("public_ips", "private_ips"). ' From e3a6717efa7bd77893cec6c0677971a85298cce3 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 24 Jul 2017 12:17:41 -0600 Subject: [PATCH 149/508] Add info about top file to pillar walk-through example to include edit.vim The pillar example in the "Pillar Makes Simple States Grow Easily" section does not meniton that the new file `/srv/pillar/edit/vim.sls` created for the example must be included in the pillar top file in order for the references to work. This PR adds that documentation so that the example works correctly. Fixes #42405 --- doc/topics/tutorials/pillar.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/topics/tutorials/pillar.rst b/doc/topics/tutorials/pillar.rst index e0c97f26bd..3ec2c1ddea 100644 --- a/doc/topics/tutorials/pillar.rst +++ b/doc/topics/tutorials/pillar.rst @@ -75,7 +75,7 @@ The default location for the pillar is in /srv/pillar. .. note:: - The pillar location can be configured via the `pillar_roots` option inside + The pillar location can be configured via the ``pillar_roots`` option inside the master configuration file. It must not be in a subdirectory of the state tree or file_roots. If the pillar is under file_roots, any pillar targeting can be bypassed by minions. @@ -242,7 +242,7 @@ set in the minion's pillar, then the default of ``httpd`` will be used. .. note:: Under the hood, pillar is just a Python dict, so Python dict methods such - as `get` and `items` can be used. + as ``get`` and ``items`` can be used. Pillar Makes Simple States Grow Easily ====================================== @@ -303,6 +303,18 @@ Where the vimrc source location can now be changed via pillar: Ensuring that the right vimrc is sent out to the correct minions. +The pillar top file must include a reference to the new sls pillar file: + +``/srv/pillar/top.sls``: + +.. code-block:: yaml + + base: + '*': + - pkg + - edit.vim + + Setting Pillar Data on the Command Line ======================================= From fa466519c4bdb6baea07e6b9f34b33a1ae4953f1 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 24 Jul 2017 13:22:50 -0600 Subject: [PATCH 150/508] Add a mention of the True/False returns with __virtual__() And their relationship to `__virtualname__`. Fixes #42375 --- doc/ref/modules/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 9f81170fe4..1056ba40f8 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -405,6 +405,10 @@ similar to the following: return __virtualname__ return False +Note that the ``__virtual__()`` function will return either a ``True`` or ``False`` +value. If it returns a ``True`` value, this ``__virtualname__`` module-level attribute +can be set as seen in the above example. This is the name that the module should be +referred to as. Documentation ============= From af3bcc927bd7c728a29a7331d1879734cc56df1c Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 24 Jul 2017 14:34:09 -0600 Subject: [PATCH 151/508] Document changes to Windows Update in 10/2016 --- salt/modules/win_wua.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index a6a5712c38..baf8c8a9ee 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -945,12 +945,26 @@ def set_wu_settings(level=None, Change Windows Update settings. If no parameters are passed, the current value will be returned. + Supported: + - Windows Vista / Server 2008 + - Windows 7 / Server 2008R2 + - Windows 8 / Server 2012 + - Windows 8.1 / Server 2012R2 + + .. note: + Microsoft began using the Unified Update Platform (UUP) starting with + Windows 10 / Server 2016. The Windows Update settings have changed and + the ability to 'Save' Windows Update settings has been removed. Windows + Update settings are read-only. See msdn documentation: + https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx + :param int level: Number from 1 to 4 indicating the update level: 1. Never check for updates 2. Check for updates but let me choose whether to download and install them 3. Download updates but let me choose whether to install them 4. Install updates automatically + :param bool recommended: Boolean value that indicates whether to include optional or recommended updates when a search for updates and installation of updates is @@ -993,8 +1007,28 @@ def set_wu_settings(level=None, salt '*' win_wua.set_wu_settings level=4 recommended=True featured=False """ - ret = {} - ret['Success'] = True + # The AutomaticUpdateSettings.Save() method used in this function does not + # work on Windows 10 / Server 2016. It is called in throughout this function + # like this: + # + # obj_au = win32com.client.Dispatch('Microsoft.Update.AutoUpdate') + # obj_au_settings = obj_au.Settings + # obj_au_settings.Save() + # + # The `Save()` method reports success but doesn't actually change anything. + # Windows Update settings are read-only in Windows 10 / Server 2016. There's + # a little blurb on MSDN that mentions this, but gives no alternative for + # changing these settings in Windows 10 / Server 2016. + # + # https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx + # + # Apparently the Windows Update framework in Windows Vista - Windows 8.1 has + # been changed quite a bit in Windows 10 / Server 2016. It is now called the + # Unified Update Platform (UUP). I haven't found an API or a Powershell + # commandlet for working with the the UUP. Perhaps there will be something + # forthcoming. The `win_lgpo` module might be an option for changing the + # Windows Update settings using local group policy. + ret = {'Success': True} # Initialize the PyCom system pythoncom.CoInitialize() From a02c91adda41c1fca42f62f2359c3415ba1bde7a Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 24 Jul 2017 15:02:15 -0600 Subject: [PATCH 152/508] Namespace `cmp_to_key` in the pkg state for Windows --- salt/states/pkg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 120565f1f4..f5f85b73a0 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -116,6 +116,7 @@ if salt.utils.is_windows(): from salt.modules.win_pkg import _repo_process_pkg_sls from salt.modules.win_pkg import _get_latest_pkg_version from salt.modules.win_pkg import _reverse_cmp_pkg_versions + from functools import cmp_to_key _get_package_info = _namespaced_function(_get_package_info, globals()) get_repo_data = _namespaced_function(get_repo_data, globals()) _get_repo_details = \ @@ -130,6 +131,7 @@ if salt.utils.is_windows(): _namespaced_function(_get_latest_pkg_version, globals()) _reverse_cmp_pkg_versions = \ _namespaced_function(_reverse_cmp_pkg_versions, globals()) + cmp_to_key = _namespaced_function(cmp_to_key, globals()) # The following imports are used by the namespaced win_pkg funcs # and need to be included in their globals. # pylint: disable=import-error,unused-import From e90ca7a114b11b9a01c383b797dfce58b1ccad9c Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 15:00:15 -0600 Subject: [PATCH 153/508] use salt encoding for joyent on 2017.7 --- salt/cloud/clouds/joyent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/cloud/clouds/joyent.py b/salt/cloud/clouds/joyent.py index 3a2f119f9f..5602e24e8c 100644 --- a/salt/cloud/clouds/joyent.py +++ b/salt/cloud/clouds/joyent.py @@ -1071,10 +1071,10 @@ def query(action=None, timenow = datetime.datetime.utcnow() timestamp = timenow.strftime('%a, %d %b %Y %H:%M:%S %Z').strip() with salt.utils.fopen(ssh_keyfile, 'r') as kh_: - rsa_key = RSA.importKey(kh_) + rsa_key = RSA.importKey(kh_.read()) rsa_ = PKCS1_v1_5.new(rsa_key) hash_ = SHA256.new() - hash_.update(timestamp) + hash_.update(timestamp.encode(__salt_system_encoding__)) signed = base64.b64encode(rsa_.sign(hash_)) keyid = '/{0}/keys/{1}'.format(user.split('/')[0], ssh_keyname) @@ -1085,7 +1085,7 @@ def query(action=None, 'Date': timestamp, 'Authorization': 'Signature keyId="{0}",algorithm="rsa-sha256" {1}'.format( keyid, - signed + signed.decode(__salt_system_encoding__) ), } From b7ebb4d81ae4855d1c87fa85c0a3a5d7c6873f00 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 15:01:57 -0600 Subject: [PATCH 154/508] these drivers do not actually have an issue. Joyent does not use apache-libcloud, and is the only one with this issue. --- doc/topics/releases/2017.7.0.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 2d06b20140..2935ef2a11 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -28,8 +28,6 @@ The following salt-cloud drivers have known issues running with Python 3. These - Joyent -- Any driver that relies on the `apache-libcloud` library such as cloudstack, dimenstiondata, gce, nova, and openstack - - When running under Python 3, users who require Unicode support should ensure that a locale is set on their machines. Users using the `C` locale are advised to switch to a UTF-aware locale to ensure proper functionality with Salt with Python 3. From b242d2d6b5f59b3eb6b022def19dcbba122b7070 Mon Sep 17 00:00:00 2001 From: jmarinaro Date: Mon, 24 Jul 2017 18:15:16 -0600 Subject: [PATCH 155/508] Fixes AttributeError thrown by chocolatey state Fixes #42521 --- salt/states/chocolatey.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/states/chocolatey.py b/salt/states/chocolatey.py index 79c69048ef..d83f9bddd3 100644 --- a/salt/states/chocolatey.py +++ b/salt/states/chocolatey.py @@ -92,7 +92,7 @@ def installed(name, version=None, source=None, force=False, pre_versions=False, # Determine action # Package not installed - if name not in [package.split('|')[0].lower() for package in pre_install.splitlines()]: + if name.lower() not in [package.lower() for package in pre_install.keys()]: if version: ret['changes'] = {name: 'Version {0} will be installed' ''.format(version)} @@ -193,9 +193,13 @@ def uninstalled(name, version=None, uninstall_args=None, override_args=False): pre_uninstall = __salt__['chocolatey.list'](local_only=True) # Determine if package is installed - if name in [package.split('|')[0].lower() for package in pre_uninstall.splitlines()]: - ret['changes'] = {name: '{0} version {1} will be removed' - ''.format(name, pre_uninstall[name][0])} + if name.lower() in [package.lower() for package in pre_uninstall.keys()]: + try: + ret['changes'] = {name: '{0} version {1} will be removed' + ''.format(name, pre_uninstall[name][0])} + except KeyError: + ret['changes'] = {name: '{0} will be removed' + ''.format(name)} else: ret['comment'] = 'The package {0} is not installed'.format(name) return ret From 2fd172e07bca3e6379ddac5095130231ff7c7745 Mon Sep 17 00:00:00 2001 From: Marin Hannache Date: Tue, 25 Jul 2017 16:23:34 +0200 Subject: [PATCH 156/508] Avoid confusing warning when using file.line file.line is internaly using file.managed without any source argument to ensure file creation with the appropriate permissions. file.managed has its 'replace' parameter defaulting to True, resulting in the following warning : Neither 'source' nor 'contents' nor 'contents_pillar' nor 'contents_grains' was defined, yet 'replace' was set to 'True'. As there is no source to replace the file with, 'replace' has been set to 'False' to avoid reading the file unnecessarily. Setting explicitly replace to False solve this issue. Signed-off-by: Marin Hannache --- salt/states/file.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 64cffa15a2..86d1e42505 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3732,7 +3732,13 @@ def line(name, content=None, match=None, mode=None, location=None, if not name: return _error(ret, 'Must provide name to file.line') - managed(name, create=create, user=user, group=group, mode=file_mode) + managed( + name, + create=create, + user=user, + group=group, + mode=file_mode, + replace=False) check_res, check_msg = _check_file(name) if not check_res: From 118d5134e2e746e56ded813c12eef49573d09fd5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 25 Jul 2017 11:37:06 -0600 Subject: [PATCH 157/508] Remove namespaced function `cmp_to_key` --- salt/states/pkg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index f5f85b73a0..1562ea2fa8 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -131,7 +131,6 @@ if salt.utils.is_windows(): _namespaced_function(_get_latest_pkg_version, globals()) _reverse_cmp_pkg_versions = \ _namespaced_function(_reverse_cmp_pkg_versions, globals()) - cmp_to_key = _namespaced_function(cmp_to_key, globals()) # The following imports are used by the namespaced win_pkg funcs # and need to be included in their globals. # pylint: disable=import-error,unused-import From a040443fa1fee3b6acb09a0ecd6696329073e1ec Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 25 Jul 2017 11:37:59 -0600 Subject: [PATCH 158/508] Move functools import inside pylint escapes --- salt/states/pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 1562ea2fa8..b586ed5dbe 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -105,6 +105,7 @@ if salt.utils.is_windows(): import datetime import errno import time + from functools import cmp_to_key # pylint: disable=import-error # pylint: enable=unused-import from salt.modules.win_pkg import _get_package_info @@ -116,7 +117,6 @@ if salt.utils.is_windows(): from salt.modules.win_pkg import _repo_process_pkg_sls from salt.modules.win_pkg import _get_latest_pkg_version from salt.modules.win_pkg import _reverse_cmp_pkg_versions - from functools import cmp_to_key _get_package_info = _namespaced_function(_get_package_info, globals()) get_repo_data = _namespaced_function(get_repo_data, globals()) _get_repo_details = \ From 685c2cced680ffb5745b3eaa8754ff9eb887fe50 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 25 Jul 2017 14:44:18 -0600 Subject: [PATCH 159/508] Add information about returning a tuple with an error message --- doc/ref/modules/index.rst | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 1056ba40f8..3bd62b3c41 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -405,10 +405,29 @@ similar to the following: return __virtualname__ return False -Note that the ``__virtual__()`` function will return either a ``True`` or ``False`` -value. If it returns a ``True`` value, this ``__virtualname__`` module-level attribute -can be set as seen in the above example. This is the name that the module should be -referred to as. +The ``__virtual__()`` function can return a ``True`` or ``False`` boolean, a tuple, +or a string. If it returns a ``True`` value, this ``__virtualname__`` module-level +attribute can be set as seen in the above example. This is the string that the module +should be referred to as. + +When ``__virtual__()`` returns a tuple, the first item should be a boolean and the +second should be a string. This is typically done when the module should not load. The +first value of the tuple is ``False`` and the second is the error message to display +for why the module did not load. + +For example: + +.. code-block:: python + + def __virtual__(): + ''' + Only load if git exists on the system + ''' + if salt.utils.which('git') is None: + return (False, + 'The git execution module cannot be loaded: git unavailable.') + else: + return True Documentation ============= From c4fabaa192940d84beb53f4f169c4113bcdb1883 Mon Sep 17 00:00:00 2001 From: Richard Phillis Date: Wed, 26 Jul 2017 09:15:59 +1000 Subject: [PATCH 160/508] Remove '-s' (--script) argument to parted within align_check function The -s argument results in no stdout output from parted in this context, so the user must infer success or failure from the exit status. --- salt/modules/parted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/parted.py b/salt/modules/parted.py index 04a7d37d47..f568c3d001 100644 --- a/salt/modules/parted.py +++ b/salt/modules/parted.py @@ -216,7 +216,7 @@ def align_check(device, part_type, partition): 'Invalid partition passed to partition.align_check' ) - cmd = 'parted -m -s {0} align-check {1} {2}'.format( + cmd = 'parted -m {0} align-check {1} {2}'.format( device, part_type, partition ) out = __salt__['cmd.run'](cmd).splitlines() From fb69e710931f3f4d2bdd0ebcec1aeff30d8fd653 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 25 Jul 2017 21:10:58 -0400 Subject: [PATCH 161/508] add changelog to 2017.7.1 release notes --- doc/topics/releases/2017.7.1.rst | 178 +++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/doc/topics/releases/2017.7.1.rst b/doc/topics/releases/2017.7.1.rst index e879d8639a..dcbe2c8f39 100644 --- a/doc/topics/releases/2017.7.1.rst +++ b/doc/topics/releases/2017.7.1.rst @@ -3,3 +3,181 @@ Salt 2017.7.1 Release Notes ============================ Version 2017.7.1 is a bugfix release for :ref:`2017.7.0 `. + +Changes for v2017.7.0..v2017.7.1 +------------------------------ + +Extended changelog courtesy of Todd Stansell (https://github.com/tjstansell/salt-changelogs): + +*Generated at: 2017-07-26T01:09:40Z* + +Statistics: + +- Total Merges: **11** +- Total Issue references: **9** +- Total PR references: **22** + +Changes: + + +- **PR** `#42548`_: (*gtmanfred*) pass in empty kwarg for reactor + @ *2017-07-26T00:41:20Z* + + - **ISSUE** `#460`_: (*whiteinge*) Add a topic and a ref for modules/states/returners/renderers/runners + | refs: `#42548`_ + * 711b742c54 Merge pull request `#42548`_ from gtmanfred/2017.7.1 + * 0257c1dc32 pass in empty kwarg for reactor + + * b948e980d2 update chunk, not kwarg in chunk + +- **PR** `#42522`_: (*gtmanfred*) pacman wildcard is only for repository installs + @ *2017-07-24T20:51:05Z* + + - **ISSUE** `#42519`_: (*xuhcc*) Error when installing package from file under Arch Linux + | refs: `#42522`_ + * 50c1635dcc Merge pull request `#42522`_ from gtmanfred/2017.7.1 + * 7787fb9e1b pacman wildcard is only for repository installs + +- **PR** `#42508`_: (*rallytime*) Back-port `#42474`_ to 2017.7.1 + @ *2017-07-24T20:49:51Z* + + - **PR** `#42474`_: (*whiteinge*) Cmd arg kwarg parsing test + | refs: `#42508`_ + - **PR** `#39646`_: (*terminalmage*) Handle deprecation of passing string args to load_args_and_kwargs + | refs: `#42474`_ + * 05c07ac049 Merge pull request `#42508`_ from rallytime/`bp-42474`_ + * 76fb074433 Add a test.arg variant that cleans the pub kwargs by default + + * 624f63648e Lint fixes + + * d246a5fc61 Add back support for string kwargs + + * 854e098aa0 Add LocalClient.cmd test for arg/kwarg parsing + +- **PR** `#42472`_: (*rallytime*) Back-port `#42435`_ to 2017.7.1 + @ *2017-07-24T15:11:13Z* + + - **ISSUE** `#42427`_: (*grichmond-salt*) Issue Passing Variables created from load_json as Inline Pillar Between States + | refs: `#42435`_ + - **PR** `#42435`_: (*terminalmage*) Modify our custom YAML loader to treat unicode literals as unicode strings + | refs: `#42472`_ + * 95fe2558e4 Merge pull request `#42472`_ from rallytime/`bp-42435`_ + * 5c47af5b98 Modify our custom YAML loader to treat unicode literals as unicode strings + +- **PR** `#42473`_: (*rallytime*) Back-port `#42436`_ to 2017.7.1 + @ *2017-07-24T15:10:29Z* + + - **ISSUE** `#42374`_: (*tyhunt99*) [2017.7.0] salt-run mange.versions throws exception if minion is offline or unresponsive + | refs: `#42436`_ + - **PR** `#42436`_: (*garethgreenaway*) Fixes to versions function in manage runner + | refs: `#42473`_ + * 5b99d45f54 Merge pull request `#42473`_ from rallytime/`bp-42436`_ + * 82ed919803 Updating the versions function inside the manage runner to account for when a minion is offline and we are unable to determine it's version. + +- **PR** `#42471`_: (*rallytime*) Back-port `#42399`_ to 2017.7.1 + @ *2017-07-24T15:09:50Z* + + - **ISSUE** `#42381`_: (*zebooka*) Git.detached broken in 2017.7.0 + | refs: `#42399`_ + - **ISSUE** `#38878`_: (*tomlaredo*) [Naming consistency] git.latest "rev" option VS git.detached "ref" option + | refs: `#38898`_ + - **PR** `#42399`_: (*rallytime*) Update old "ref" references to "rev" in git.detached state + | refs: `#42471`_ + - **PR** `#38898`_: (*terminalmage*) git.detached: rename ref to rev for consistency + | refs: `#42399`_ + * 3d1a2d3f9f Merge pull request `#42471`_ from rallytime/`bp-42399`_ + * b9a4669e5a Update old "ref" references to "rev" in git.detached state + +- **PR** `#42470`_: (*rallytime*) Back-port `#42031`_ to 2017.7.1 + @ *2017-07-24T15:09:30Z* + + - **ISSUE** `#42400`_: (*Enquier*) Conflict in execution of passing pillar data to orch/reactor event executions 2017.7.0 + | refs: `#42031`_ + - **PR** `#42031`_: (*skizunov*) Fix: Reactor emits critical error + | refs: `#42470`_ + * 09766bccbc Merge pull request `#42470`_ from rallytime/`bp-42031`_ + * 0a0c6287a4 Fix: Reactor emits critical error + +- **PR** `#42469`_: (*rallytime*) Back-port `#42027`_ to 2017.7.1 + @ *2017-07-21T22:41:02Z* + + - **ISSUE** `#41949`_: (*jrporcaro*) Event returner doesn't work with Windows Master + | refs: `#42027`_ + - **PR** `#42027`_: (*gtmanfred*) import salt.minion for EventReturn for Windows + | refs: `#42469`_ + * d7b172a15b Merge pull request `#42469`_ from rallytime/`bp-42027`_ + * ed612b4ee7 import salt.minion for EventReturn for Windows + +- **PR** `#42466`_: (*rallytime*) Back-port `#42452`_ to 2017.7.1 + @ *2017-07-21T19:41:24Z* + + - **PR** `#42452`_: (*Ch3LL*) update windows urls to new py2/py3 naming scheme + | refs: `#42466`_ + * 8777b1a825 Merge pull request `#42466`_ from rallytime/`bp-42452`_ + * c10196f68c update windows urls to new py2/py3 naming scheme + +- **PR** `#42439`_: (*rallytime*) Back-port `#42409`_ to 2017.7.1 + @ *2017-07-21T17:38:10Z* + + - **PR** `#42409`_: (*twangboy*) Add Scripts to build Py3 on Mac + | refs: `#42439`_ + * fceaaf41d0 Merge pull request `#42439`_ from rallytime/`bp-42409`_ + * 8176964b41 Remove build and dist, sign pkgs + + * 2c14d92a07 Fix hard coded pip path + + * 82fdd7c2e1 Add support for Py3 + + * 2478447246 Update Python and other reqs + +- **PR** `#42441`_: (*rallytime*) Back-port `#42433`_ to 2017.7.1 + @ *2017-07-21T17:37:01Z* + + - **ISSUE** `#42403`_: (*astronouth7303*) [2017.7] Pillar empty when state is applied from orchestrate + | refs: `#42433`_ + - **PR** `#42433`_: (*terminalmage*) Only force saltenv/pillarenv to be a string when not None + | refs: `#42441`_ + * 660400560b Merge pull request `#42441`_ from rallytime/`bp-42433`_ + * 17f347123a Only force saltenv/pillarenv to be a string when not None + + +.. _`#38878`: https://github.com/saltstack/salt/issues/38878 +.. _`#38898`: https://github.com/saltstack/salt/pull/38898 +.. _`#39646`: https://github.com/saltstack/salt/pull/39646 +.. _`#41949`: https://github.com/saltstack/salt/issues/41949 +.. _`#42027`: https://github.com/saltstack/salt/pull/42027 +.. _`#42031`: https://github.com/saltstack/salt/pull/42031 +.. _`#42374`: https://github.com/saltstack/salt/issues/42374 +.. _`#42381`: https://github.com/saltstack/salt/issues/42381 +.. _`#42399`: https://github.com/saltstack/salt/pull/42399 +.. _`#42400`: https://github.com/saltstack/salt/issues/42400 +.. _`#42403`: https://github.com/saltstack/salt/issues/42403 +.. _`#42409`: https://github.com/saltstack/salt/pull/42409 +.. _`#42427`: https://github.com/saltstack/salt/issues/42427 +.. _`#42433`: https://github.com/saltstack/salt/pull/42433 +.. _`#42435`: https://github.com/saltstack/salt/pull/42435 +.. _`#42436`: https://github.com/saltstack/salt/pull/42436 +.. _`#42439`: https://github.com/saltstack/salt/pull/42439 +.. _`#42441`: https://github.com/saltstack/salt/pull/42441 +.. _`#42452`: https://github.com/saltstack/salt/pull/42452 +.. _`#42466`: https://github.com/saltstack/salt/pull/42466 +.. _`#42469`: https://github.com/saltstack/salt/pull/42469 +.. _`#42470`: https://github.com/saltstack/salt/pull/42470 +.. _`#42471`: https://github.com/saltstack/salt/pull/42471 +.. _`#42472`: https://github.com/saltstack/salt/pull/42472 +.. _`#42473`: https://github.com/saltstack/salt/pull/42473 +.. _`#42474`: https://github.com/saltstack/salt/pull/42474 +.. _`#42508`: https://github.com/saltstack/salt/pull/42508 +.. _`#42519`: https://github.com/saltstack/salt/issues/42519 +.. _`#42522`: https://github.com/saltstack/salt/pull/42522 +.. _`#42548`: https://github.com/saltstack/salt/pull/42548 +.. _`#460`: https://github.com/saltstack/salt/issues/460 +.. _`bp-42027`: https://github.com/saltstack/salt/pull/42027 +.. _`bp-42031`: https://github.com/saltstack/salt/pull/42031 +.. _`bp-42399`: https://github.com/saltstack/salt/pull/42399 +.. _`bp-42409`: https://github.com/saltstack/salt/pull/42409 +.. _`bp-42433`: https://github.com/saltstack/salt/pull/42433 +.. _`bp-42435`: https://github.com/saltstack/salt/pull/42435 +.. _`bp-42436`: https://github.com/saltstack/salt/pull/42436 +.. _`bp-42452`: https://github.com/saltstack/salt/pull/42452 +.. _`bp-42474`: https://github.com/saltstack/salt/pull/42474 From ee3bc6eb10dcf217395ad07ca3e49c4c25eb665f Mon Sep 17 00:00:00 2001 From: "J. Beard" Date: Sun, 23 Jul 2017 21:02:13 -0700 Subject: [PATCH 162/508] Fixing output so --force-color and --no-color override master and minion config color value --- salt/output/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/output/__init__.py b/salt/output/__init__.py index d9ce631b4e..8a1732c512 100644 --- a/salt/output/__init__.py +++ b/salt/output/__init__.py @@ -168,6 +168,13 @@ def get_printout(out, opts=None, **kwargs): opts['color'] = False else: opts['color'] = True + else: + if opts.get('force_color', False): + opts['color'] = True + elif opts.get('no_color', False) or salt.utils.is_windows(): + opts['color'] = False + else: + pass outputters = salt.loader.outputters(opts) if out not in outputters: From afa7a13ce334874a51cd382dd985f7dbd9d0775b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 21 Jul 2017 08:31:08 -0600 Subject: [PATCH 163/508] use logic from file.directory for makedirs --- salt/states/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/archive.py b/salt/states/archive.py index e36f178f2e..f053d3c207 100644 --- a/salt/states/archive.py +++ b/salt/states/archive.py @@ -1221,7 +1221,7 @@ def extracted(name, return ret if not os.path.isdir(name): - __salt__['file.makedirs'](name, user=user) + __states__['file.directory'](name, user=user, makedirs=True) created_destdir = True log.debug('Extracting {0} to {1}'.format(cached_source, name)) From 0f0b7e3e0ac3ee0aa9c7a28d2c16bc0c3b9e681f Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Tue, 25 Jul 2017 20:56:28 -0500 Subject: [PATCH 164/508] Fix disable_ config option Previously, the logic always looked for `disable_s` in the config options. The issue with this is that the following tag names already end with 's': `beacons`, `engines`, `grains`, `log_handlers`, `serializers`, `states`, and `utils`. So previously, if you wanted to disable a beacon, the config option to set is `disable_beaconss` (with 'ss' at the end). Fix this so that we only append an 's' if the tag name does not already end with an 's'. Signed-off-by: Sergey Kizunov --- salt/loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/loader.py b/salt/loader.py index 000c0fef60..1c03e475be 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -1094,7 +1094,8 @@ class LazyLoader(salt.utils.lazy.LazyDict): virtual_funcs = [] self.virtual_funcs = virtual_funcs - self.disabled = set(self.opts.get('disable_{0}s'.format(self.tag), [])) + self.disabled = set(self.opts.get('disable_{0}{1}'.format( + self.tag, '' if self.tag[-1] == 's' else 's'), [])) self.refresh_file_mapping() From 4e2fb03a95e7cf8249c7a4a49d415859406551f4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 12:21:14 -0600 Subject: [PATCH 165/508] Add pythonpath to batch files and service --- pkg/windows/buildenv/salt-call.bat | 3 +++ pkg/windows/buildenv/salt-cp.bat | 3 +++ pkg/windows/buildenv/salt-key.bat | 3 +++ pkg/windows/buildenv/salt-master.bat | 3 +++ pkg/windows/buildenv/salt-minion-debug.bat | 3 +++ pkg/windows/buildenv/salt-minion.bat | 3 +++ pkg/windows/buildenv/salt-run.bat | 3 +++ pkg/windows/buildenv/salt.bat | 3 +++ pkg/windows/installer/Salt-Minion-Setup.nsi | 2 +- 9 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/windows/buildenv/salt-call.bat b/pkg/windows/buildenv/salt-call.bat index 095f51e4c1..3949253030 100644 --- a/pkg/windows/buildenv/salt-call.bat +++ b/pkg/windows/buildenv/salt-call.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-call +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-cp.bat b/pkg/windows/buildenv/salt-cp.bat index 29274320b1..4c5eae9be3 100644 --- a/pkg/windows/buildenv/salt-cp.bat +++ b/pkg/windows/buildenv/salt-cp.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-cp +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-key.bat b/pkg/windows/buildenv/salt-key.bat index 471e3626b2..08b05f04c5 100644 --- a/pkg/windows/buildenv/salt-key.bat +++ b/pkg/windows/buildenv/salt-key.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-key +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-master.bat b/pkg/windows/buildenv/salt-master.bat index 9a124ffd46..0f66885d91 100644 --- a/pkg/windows/buildenv/salt-master.bat +++ b/pkg/windows/buildenv/salt-master.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-master +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat index ad0ebafee0..bb58e89132 100644 --- a/pkg/windows/buildenv/salt-minion-debug.bat +++ b/pkg/windows/buildenv/salt-minion-debug.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Stop the Salt Minion service net stop salt-minion diff --git a/pkg/windows/buildenv/salt-minion.bat b/pkg/windows/buildenv/salt-minion.bat index 0a1aafa0c0..41c434aa83 100644 --- a/pkg/windows/buildenv/salt-minion.bat +++ b/pkg/windows/buildenv/salt-minion.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-run.bat b/pkg/windows/buildenv/salt-run.bat index 5d62775d5e..c931d87ec5 100644 --- a/pkg/windows/buildenv/salt-run.bat +++ b/pkg/windows/buildenv/salt-run.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-run +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt.bat b/pkg/windows/buildenv/salt.bat index 050387da90..6a7302372a 100644 --- a/pkg/windows/buildenv/salt.bat +++ b/pkg/windows/buildenv/salt.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 070621a52a..8d0c6fd1b6 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -335,7 +335,7 @@ Section -Post ; Register the Salt-Minion Service nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" - nsExec::Exec "nssm.exe set salt-minion AppEnvironmentExtra PYTHONHOME=" + nsExec::Exec "nssm.exe set salt-minion AppEnvironmentExtra PYTHONHOME= PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" From 9af1eb274112d03370c393f35045cec74c7f738f Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 13:42:12 -0600 Subject: [PATCH 166/508] Ignore any PYTHON* environment vars already on the system --- pkg/windows/buildenv/salt-call.bat | 6 +----- pkg/windows/buildenv/salt-cp.bat | 6 +----- pkg/windows/buildenv/salt-key.bat | 6 +----- pkg/windows/buildenv/salt-master.bat | 6 +----- pkg/windows/buildenv/salt-minion-debug.bat | 6 +----- pkg/windows/buildenv/salt-minion.bat | 6 +----- pkg/windows/buildenv/salt-run.bat | 6 +----- pkg/windows/buildenv/salt.bat | 6 +----- pkg/windows/installer/Salt-Minion-Setup.nsi | 3 +-- 9 files changed, 9 insertions(+), 42 deletions(-) diff --git a/pkg/windows/buildenv/salt-call.bat b/pkg/windows/buildenv/salt-call.bat index 3949253030..6fc4086a9d 100644 --- a/pkg/windows/buildenv/salt-call.bat +++ b/pkg/windows/buildenv/salt-call.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-call -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-cp.bat b/pkg/windows/buildenv/salt-cp.bat index 4c5eae9be3..6c63dce338 100644 --- a/pkg/windows/buildenv/salt-cp.bat +++ b/pkg/windows/buildenv/salt-cp.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-cp -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-key.bat b/pkg/windows/buildenv/salt-key.bat index 08b05f04c5..c78b41580b 100644 --- a/pkg/windows/buildenv/salt-key.bat +++ b/pkg/windows/buildenv/salt-key.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-key -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-master.bat b/pkg/windows/buildenv/salt-master.bat index 0f66885d91..c1b9575a56 100644 --- a/pkg/windows/buildenv/salt-master.bat +++ b/pkg/windows/buildenv/salt-master.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-master -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat index bb58e89132..e78887ea63 100644 --- a/pkg/windows/buildenv/salt-minion-debug.bat +++ b/pkg/windows/buildenv/salt-minion-debug.bat @@ -8,12 +8,8 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Stop the Salt Minion service net stop salt-minion :: Launch Script -"%Python%" "%Script%" -l debug - +"%Python%" -E "%Script%" -l debug diff --git a/pkg/windows/buildenv/salt-minion.bat b/pkg/windows/buildenv/salt-minion.bat index 41c434aa83..8ca7ba4d8d 100644 --- a/pkg/windows/buildenv/salt-minion.bat +++ b/pkg/windows/buildenv/salt-minion.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-run.bat b/pkg/windows/buildenv/salt-run.bat index c931d87ec5..ca4741c6a9 100644 --- a/pkg/windows/buildenv/salt-run.bat +++ b/pkg/windows/buildenv/salt-run.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-run -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt.bat b/pkg/windows/buildenv/salt.bat index 6a7302372a..5eab5b5618 100644 --- a/pkg/windows/buildenv/salt.bat +++ b/pkg/windows/buildenv/salt.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 8d0c6fd1b6..24adb6eb08 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -334,8 +334,7 @@ Section -Post WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "Path" "$INSTDIR\bin\" ; Register the Salt-Minion Service - nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" - nsExec::Exec "nssm.exe set salt-minion AppEnvironmentExtra PYTHONHOME= PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages" + nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" From 0293429e24eea2f98ebb54c82a161789f25f5384 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 20 Jul 2017 13:55:06 -0500 Subject: [PATCH 167/508] Only force saltenv/pillarenv to be a string when not None This fixes a regression introduced to make numeric saltenv/pillarenv work properly. --- salt/modules/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index fc64d8d97d..5d6bf5f009 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -263,14 +263,14 @@ def _get_opts(**kwargs): if 'saltenv' in kwargs: saltenv = kwargs['saltenv'] - if not isinstance(saltenv, six.string_types): + if saltenv is not None and not isinstance(saltenv, six.string_types): opts['environment'] = str(kwargs['saltenv']) else: opts['environment'] = kwargs['saltenv'] if 'pillarenv' in kwargs: pillarenv = kwargs['pillarenv'] - if not isinstance(pillarenv, six.string_types): + if pillarenv is not None and not isinstance(pillarenv, six.string_types): opts['pillarenv'] = str(kwargs['pillarenv']) else: opts['pillarenv'] = kwargs['pillarenv'] From d55a44dd1a12933d25c2741f5b2b9f5fc6f6d485 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 14:21:41 -0600 Subject: [PATCH 168/508] Avoid loading user site packages --- pkg/windows/buildenv/salt-call.bat | 2 +- pkg/windows/buildenv/salt-cp.bat | 2 +- pkg/windows/buildenv/salt-key.bat | 2 +- pkg/windows/buildenv/salt-master.bat | 2 +- pkg/windows/buildenv/salt-minion-debug.bat | 2 +- pkg/windows/buildenv/salt-minion.bat | 2 +- pkg/windows/buildenv/salt-run.bat | 2 +- pkg/windows/buildenv/salt.bat | 2 +- pkg/windows/installer/Salt-Minion-Setup.nsi | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/windows/buildenv/salt-call.bat b/pkg/windows/buildenv/salt-call.bat index 6fc4086a9d..55f7cfac3b 100644 --- a/pkg/windows/buildenv/salt-call.bat +++ b/pkg/windows/buildenv/salt-call.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-call :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-cp.bat b/pkg/windows/buildenv/salt-cp.bat index 6c63dce338..61fd1ce444 100644 --- a/pkg/windows/buildenv/salt-cp.bat +++ b/pkg/windows/buildenv/salt-cp.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-cp :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-key.bat b/pkg/windows/buildenv/salt-key.bat index c78b41580b..4928b696d5 100644 --- a/pkg/windows/buildenv/salt-key.bat +++ b/pkg/windows/buildenv/salt-key.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-key :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-master.bat b/pkg/windows/buildenv/salt-master.bat index c1b9575a56..134875c072 100644 --- a/pkg/windows/buildenv/salt-master.bat +++ b/pkg/windows/buildenv/salt-master.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-master :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat index e78887ea63..2e6b5c5bcf 100644 --- a/pkg/windows/buildenv/salt-minion-debug.bat +++ b/pkg/windows/buildenv/salt-minion-debug.bat @@ -12,4 +12,4 @@ Set Script=%SaltDir%\bin\Scripts\salt-minion net stop salt-minion :: Launch Script -"%Python%" -E "%Script%" -l debug +"%Python%" -E -s "%Script%" -l debug diff --git a/pkg/windows/buildenv/salt-minion.bat b/pkg/windows/buildenv/salt-minion.bat index 8ca7ba4d8d..0eb041219c 100644 --- a/pkg/windows/buildenv/salt-minion.bat +++ b/pkg/windows/buildenv/salt-minion.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-run.bat b/pkg/windows/buildenv/salt-run.bat index ca4741c6a9..a8766fc9b0 100644 --- a/pkg/windows/buildenv/salt-run.bat +++ b/pkg/windows/buildenv/salt-run.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-run :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt.bat b/pkg/windows/buildenv/salt.bat index 5eab5b5618..6732ca2968 100644 --- a/pkg/windows/buildenv/salt.bat +++ b/pkg/windows/buildenv/salt.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 24adb6eb08..ab890529d5 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -334,7 +334,7 @@ Section -Post WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "Path" "$INSTDIR\bin\" ; Register the Salt-Minion Service - nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" + nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E -s $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" From 2868061ee4d15e17327ce66a5b2f14b61f6bc379 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 25 Jul 2017 14:05:58 -0600 Subject: [PATCH 169/508] update chunk, not kwarg in chunk --- salt/state.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/state.py b/salt/state.py index d3d7cd3f81..a3b9563011 100644 --- a/salt/state.py +++ b/salt/state.py @@ -571,14 +571,14 @@ class Compiler(object): if state.startswith('__'): continue chunk = {'state': state, - 'name': name} + 'name': name, + 'arg': [], + 'kwarg': {}} if '__sls__' in body: chunk['__sls__'] = body['__sls__'] if '__env__' in body: chunk['__env__'] = body['__env__'] chunk['__id__'] = name - chunk['arg'] = [] - chunk['kwarg'] = {} for arg in run: if isinstance(arg, six.string_types): funcs.add(arg) @@ -591,7 +591,7 @@ class Compiler(object): names.append(_name) continue else: - chunk['kwarg'].update(arg) + chunk.update(arg) if names: name_order = 1 for entry in names: From 63bb0fb2c461908879663b998c8a06d31443e23d Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 25 Jul 2017 15:34:44 -0600 Subject: [PATCH 170/508] pass in empty kwarg for reactor --- salt/state.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/state.py b/salt/state.py index a3b9563011..44fb73b97d 100644 --- a/salt/state.py +++ b/salt/state.py @@ -571,9 +571,7 @@ class Compiler(object): if state.startswith('__'): continue chunk = {'state': state, - 'name': name, - 'arg': [], - 'kwarg': {}} + 'name': name} if '__sls__' in body: chunk['__sls__'] = body['__sls__'] if '__env__' in body: From 69d5973651ca2230fb409ce8e8c5732b83c62f45 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 16:36:25 -0600 Subject: [PATCH 171/508] Compile scripts with -E -s params for python --- pkg/osx/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index c411840a91..7850d48cd8 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -86,9 +86,9 @@ sudo $PKGRESOURCES/build_env.sh $PYVER # Install Salt ############################################################################ echo -n -e "\033]0;Build: Install Salt\007" -sudo rm -rm $SRCDIR/build -sudo rm -rm $SRCDIR/dist -sudo $PYTHON $SRCDIR/setup.py install +sudo rm -rf $SRCDIR/build +sudo rm -rf $SRCDIR/dist +sudo $PYTHON $SRCDIR/setup.py build -e "$PYTHON -E -s" install ############################################################################ # Build Package From a96f7c09e0ee8589ed36c01c9e293ae2d885f131 Mon Sep 17 00:00:00 2001 From: Guillaume DUBROEUCQ Date: Thu, 27 Jul 2017 14:34:56 +0200 Subject: [PATCH 172/508] yumpkg.py: add option to the command "check-update" --- salt/modules/yumpkg.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 448d347bb6..98b618c734 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -896,7 +896,13 @@ def refresh_db(**kwargs): branch_arg = _get_branch_option(**kwargs) clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache'] - update_cmd = [_yum(), '--quiet', 'check-update'] + + if __grains__.get('os_family') == 'RedHat' and __grains__.get('osmajorrelease') == '7': + # This feature is disabled because it is not used by Salt. Furthermore, this feature lasts a lot with using large repo like EPEL + update_cmd = [_yum(), '--quiet', '--setopt=autocheck_running_kernel=false', 'check-update'] + else: + update_cmd = [_yum(), '--quiet', 'check-update'] + for args in (repo_arg, exclude_arg, branch_arg): if args: clean_cmd.extend(args) From d2ef4483e4370749a6757f2a40bd9f24fedc9ab1 Mon Sep 17 00:00:00 2001 From: Guillaume DUBROEUCQ Date: Thu, 27 Jul 2017 15:03:34 +0200 Subject: [PATCH 173/508] yumpkg.py: clean --- salt/modules/yumpkg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 98b618c734..1947d038ce 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -896,12 +896,12 @@ def refresh_db(**kwargs): branch_arg = _get_branch_option(**kwargs) clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache'] + update_cmd = [_yum(), '--quiet', 'check-update'] if __grains__.get('os_family') == 'RedHat' and __grains__.get('osmajorrelease') == '7': - # This feature is disabled because it is not used by Salt. Furthermore, this feature lasts a lot with using large repo like EPEL - update_cmd = [_yum(), '--quiet', '--setopt=autocheck_running_kernel=false', 'check-update'] - else: - update_cmd = [_yum(), '--quiet', 'check-update'] + # This feature is disable because it is not used by Salt and lasts a lot with using large repo like EPEL + update_cmd.append('--setopt=autocheck_running_kernel=false') + for args in (repo_arg, exclude_arg, branch_arg): if args: From 9c0b5cc1d6fbac5f01a6df0058562551bfbd50fb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 27 Jul 2017 10:59:07 -0600 Subject: [PATCH 174/508] Remove extra newline --- salt/modules/yumpkg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 1947d038ce..9ccd66d105 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -902,7 +902,6 @@ def refresh_db(**kwargs): # This feature is disable because it is not used by Salt and lasts a lot with using large repo like EPEL update_cmd.append('--setopt=autocheck_running_kernel=false') - for args in (repo_arg, exclude_arg, branch_arg): if args: clean_cmd.extend(args) From 0373791f2a9d885fb94b9eec62099566d4367be1 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 27 Jul 2017 11:45:29 -0600 Subject: [PATCH 175/508] Correct capatlization --- salt/modules/win_wua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index baf8c8a9ee..782459365a 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -955,7 +955,7 @@ def set_wu_settings(level=None, Microsoft began using the Unified Update Platform (UUP) starting with Windows 10 / Server 2016. The Windows Update settings have changed and the ability to 'Save' Windows Update settings has been removed. Windows - Update settings are read-only. See msdn documentation: + Update settings are read-only. See MSDN documentation: https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx :param int level: From 559d432930b77a0f5389332b6d8b0857852427da Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 27 Jul 2017 12:00:38 -0600 Subject: [PATCH 176/508] fix tests --- tests/unit/states/test_archive.py | 51 ++++++++++++++++--------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/unit/states/test_archive.py b/tests/unit/states/test_archive.py index 2823711fe7..30d256773d 100644 --- a/tests/unit/states/test_archive.py +++ b/tests/unit/states/test_archive.py @@ -102,16 +102,17 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': mock_run, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): - ret = archive.extracted(tmp_dir, - source, - options=test_opts, - enforce_toplevel=False) - ret_opts.append(source) - mock_run.assert_called_with(ret_opts, - cwd=tmp_dir + os.sep, - python_shell=False) + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): + ret = archive.extracted(tmp_dir, + source, + options=test_opts, + enforce_toplevel=False) + ret_opts.append(source) + mock_run.assert_called_with(ret_opts, + cwd=tmp_dir + os.sep, + python_shell=False) def test_tar_gnutar(self): ''' @@ -142,13 +143,14 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': run_all, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stdout') + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + ret = archive.extracted('/tmp/out', + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stdout') def test_tar_bsdtar(self): ''' @@ -179,10 +181,11 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': run_all, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stderr') + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + ret = archive.extracted('/tmp/out', + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stderr') From 928a4808dd85d0fcf5e5cf60d91db4a20802bacd Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 27 Jul 2017 11:25:23 -0700 Subject: [PATCH 177/508] Updating the superseded and deprecated decorators to work when specified as pillar values. --- salt/utils/decorators/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 8c3daeacd3..86e605933e 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -542,8 +542,14 @@ class _WithDeprecated(_DeprecationDecorator): f_name=function.__name__)) opts = self._globals.get('__opts__', '{}') - use_deprecated = full_name in opts.get(self.CFG_USE_DEPRECATED, list()) - use_superseded = full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) + pillar = self._globals.get('__pillar__', '{}') + + use_deprecated = full_name in opts.get(self.CFG_USE_DEPRECATED, list()) or \ + full_name in pillar.get(self.CFG_USE_DEPRECATED, list()) + + use_superseded = full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) or \ + full_name in pillar.get(self.CFG_USE_SUPERSEDED, list()) + if use_deprecated and use_superseded: raise SaltConfigurationError("Function '{0}' is mentioned both in deprecated " "and superseded sections. Please remove any of that.".format(full_name)) @@ -567,6 +573,8 @@ class _WithDeprecated(_DeprecationDecorator): return func_path in self._globals.get('__opts__').get( self.CFG_USE_DEPRECATED, list()) or (self._policy == self.OPT_IN and not (func_path in self._globals.get('__opts__', {}).get( + self.CFG_USE_SUPERSEDED, list())) + and not (func_path in self._globals.get('__pillar__', {}).get( self.CFG_USE_SUPERSEDED, list()))), func_path def __call__(self, function): From 74bae139399da116a6fb0e2dbe3457cd82caa9f7 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 27 Jul 2017 13:19:48 -0700 Subject: [PATCH 178/508] Small update to something I missed in the first commit. Updating tests to also test for pillar values. --- salt/utils/decorators/__init__.py | 1 + tests/unit/utils/test_decorators.py | 80 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 86e605933e..9d459169bb 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -571,6 +571,7 @@ class _WithDeprecated(_DeprecationDecorator): f_name=self._orig_f_name) return func_path in self._globals.get('__opts__').get( + self.CFG_USE_DEPRECATED, list()) or func_path in self._globals.get('__pillar__').get( self.CFG_USE_DEPRECATED, list()) or (self._policy == self.OPT_IN and not (func_path in self._globals.get('__opts__', {}).get( self.CFG_USE_SUPERSEDED, list())) diff --git a/tests/unit/utils/test_decorators.py b/tests/unit/utils/test_decorators.py index a729ec024c..35ea6da502 100644 --- a/tests/unit/utils/test_decorators.py +++ b/tests/unit/utils/test_decorators.py @@ -59,6 +59,7 @@ class DecoratorsTest(TestCase): self.globs = { '__virtualname__': 'test', '__opts__': {}, + '__pillar__': {}, 'old_function': self.old_function, 'new_function': self.new_function, '_new_function': self._new_function, @@ -149,6 +150,23 @@ class DecoratorsTest(TestCase): ['The function "test.new_function" is using its deprecated ' 'version and will expire in version "Beryllium".']) + def test_with_deprecated_notfound_in_pillar(self): + ''' + Test with_deprecated should raise an exception, if a same name + function with the "_" prefix not implemented. + + :return: + ''' + del self.globs['_new_function'] + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + with self.assertRaises(CommandExecutionError): + depr(self.new_function)() + self.assertEqual(self.messages, + ['The function "test.new_function" is using its deprecated ' + 'version and will expire in version "Beryllium".']) + def test_with_deprecated_found(self): ''' Test with_deprecated should not raise an exception, if a same name @@ -166,6 +184,23 @@ class DecoratorsTest(TestCase): 'and will expire in version "Beryllium".'] self.assertEqual(self.messages, log_msg) + def test_with_deprecated_found_in_pillar(self): + ''' + Test with_deprecated should not raise an exception, if a same name + function with the "_" prefix is implemented, but should use + an old version instead, if "use_deprecated" is requested. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['_new_function'] = self.old_function + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + self.assertEqual(depr(self.new_function)(), self.old_function()) + log_msg = ['The function "test.new_function" is using its deprecated version ' + 'and will expire in version "Beryllium".'] + self.assertEqual(self.messages, log_msg) + def test_with_deprecated_found_eol(self): ''' Test with_deprecated should raise an exception, if a same name @@ -185,6 +220,25 @@ class DecoratorsTest(TestCase): 'is configured as its deprecated version. The lifetime of the function ' '"new_function" expired. Please use its successor "new_function" instead.']) + def test_with_deprecated_found_eol_in_pillar(self): + ''' + Test with_deprecated should raise an exception, if a same name + function with the "_" prefix is implemented, "use_deprecated" is requested + and EOL is reached. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['_new_function'] = self.old_function + depr = decorators.with_deprecated(self.globs, "Helium") + depr._curr_version = self._mk_version("Beryllium")[1] + with self.assertRaises(CommandExecutionError): + depr(self.new_function)() + self.assertEqual(self.messages, + ['Although function "new_function" is called, an alias "new_function" ' + 'is configured as its deprecated version. The lifetime of the function ' + '"new_function" expired. Please use its successor "new_function" instead.']) + def test_with_deprecated_no_conf(self): ''' Test with_deprecated should not raise an exception, if a same name @@ -260,6 +314,19 @@ class DecoratorsTest(TestCase): assert depr(self.new_function)() == self.new_function() assert not self.messages + def test_with_deprecated_opt_in_use_superseded_in_pillar(self): + ''' + Test with_deprecated using opt-in policy, + where newer function is used as per configuration. + + :return: + ''' + self.globs['__pillar__']['use_superseded'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium", policy=decorators._DeprecationDecorator.OPT_IN) + depr._curr_version = self._mk_version("Helium")[1] + assert depr(self.new_function)() == self.new_function() + assert not self.messages + def test_with_deprecated_opt_in_use_superseded_and_deprecated(self): ''' Test with_deprecated misconfiguration. @@ -272,3 +339,16 @@ class DecoratorsTest(TestCase): depr._curr_version = self._mk_version("Helium")[1] with self.assertRaises(SaltConfigurationError): assert depr(self.new_function)() == self.new_function() + + def test_with_deprecated_opt_in_use_superseded_and_deprecated_in_pillar(self): + ''' + Test with_deprecated misconfiguration. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['__pillar__']['use_superseded'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + with self.assertRaises(SaltConfigurationError): + assert depr(self.new_function)() == self.new_function() From cc4e45656d7c99f7d1271a864b936c76e5349e9b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 11:54:42 -0600 Subject: [PATCH 179/508] Allow not interpreting backslashes in the repl Without this, if append or prepend is used with file.replace, the backslashes will not be interpreted when appending the file, but the next time the state is run, the backslashes could be interpreted and removed from the line. --- salt/modules/file.py | 20 +++++++++++++++++--- salt/states/file.py | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 9a0f68c7a6..33e7708abe 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1884,6 +1884,7 @@ def replace(path, show_changes=True, ignore_if_missing=False, preserve_inode=True, + backslash_literal=False, ): ''' .. versionadded:: 0.17.0 @@ -1984,6 +1985,14 @@ def replace(path, filename. Hard links will then share an inode with the backup, instead (if using ``backup`` to create a backup copy). + backslash_literal : False + .. versionadded:: 2016.11.7 + + Interpret backslashes as literal backslashes for the repl and not + escape characters. This will help when using append/prepend so that + the backslashes are not interpreted for the repl on the second run of + the state. + If an equal sign (``=``) appears in an argument to a Salt command it is interpreted as a keyword argument in the format ``key=val``. That processing can be bypassed in order to pass an equal sign through to the @@ -2080,7 +2089,10 @@ def replace(path, if re.search(cpattern, r_data): return True # `with` block handles file closure else: - result, nrepl = re.subn(cpattern, repl, r_data, count) + result, nrepl = re.subn(cpattern, + repl.replace('\\', '\\\\') if backslash_literal else repl, + r_data, + count) # found anything? (even if no change) if nrepl > 0: @@ -2138,8 +2150,10 @@ def replace(path, r_data = mmap.mmap(r_file.fileno(), 0, access=mmap.ACCESS_READ) - result, nrepl = re.subn(cpattern, repl, - r_data, count) + result, nrepl = re.subn(cpattern, + repl.replace('\\', '\\\\') if backslash_literal else repl, + r_data, + count) try: w_file.write(salt.utils.to_str(result)) except (OSError, IOError) as exc: diff --git a/salt/states/file.py b/salt/states/file.py index b950acb173..d404f24eb8 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3261,7 +3261,8 @@ def replace(name, not_found_content=None, backup='.bak', show_changes=True, - ignore_if_missing=False): + ignore_if_missing=False, + backslash_literal=False): r''' Maintain an edit in a file. @@ -3351,6 +3352,14 @@ def replace(name, state will display an error raised by the execution module. If set to ``True``, the state will simply report no changes. + backslash_literal : False + .. versionadded:: 2016.11.7 + + Interpret backslashes as literal backslashes for the repl and not + escape characters. This will help when using append/prepend so that + the backslashes are not interpreted for the repl on the second run of + the state. + For complex regex patterns, it can be useful to avoid the need for complex quoting and escape sequences by making use of YAML's multiline string syntax. @@ -3399,7 +3408,8 @@ def replace(name, backup=backup, dry_run=__opts__['test'], show_changes=show_changes, - ignore_if_missing=ignore_if_missing) + ignore_if_missing=ignore_if_missing, + backslash_literal=backslash_literal) if changes: ret['pchanges']['diff'] = changes From b9c91eba60fbd7e8fef4cc086d54b171c33cb8e3 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 27 Jul 2017 15:46:58 -0600 Subject: [PATCH 180/508] Add runas_passwd as a global for states --- doc/ref/states/requisites.rst | 23 ++++++++++++++++++++++- salt/modules/cmdmod.py | 3 +++ salt/state.py | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index 6b00b3e856..4fbc0ad972 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -519,7 +519,8 @@ runas .. versionadded:: 2017.7.0 -The ``runas`` global option is used to set the user which will be used to run the command in the ``cmd.run`` module. +The ``runas`` global option is used to set the user which will be used to run +the command in the ``cmd.run`` module. .. code-block:: yaml @@ -532,6 +533,26 @@ The ``runas`` global option is used to set the user which will be used to run th In the above state, the pip command run by ``cmd.run`` will be run by the daniel user. +runas_passwd +~~~~~~~~~~~~ + +.. versionadded:: 2017.7.2 + +The ``runas_passwd`` global option is used to set the password used by the runas +global option. This is required by ``cmd.run`` on Windows when ``runas`` is +specified. It will be set when ``password`` is defined in the state. + +.. code-block:: yaml + + run_script: + cmd.run: + - name: Powershell -NonInteractive -ExecutionPolicy Bypass -File C:\\Temp\\script.ps1 + - runas: frank + - password: supersecret + +In the above state, the Powershell script run by ``cmd.run`` will be run by the +frank user with the password ``supersecret``. + .. _requisites-require-in: .. _requisites-watch-in: .. _requisites-onchanges-in: diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index b576174a6d..ea75865280 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -294,6 +294,9 @@ def _run(cmd, if runas is None and '__context__' in globals(): runas = __context__.get('runas') + if password is None and '__context__' in globals(): + password = __context__.get('runas_passwd') + # Set the default working directory to the home directory of the user # salt-minion is running as. Defaults to home directory of user under which # the minion is running. diff --git a/salt/state.py b/salt/state.py index 44fb73b97d..af50b2c073 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1751,6 +1751,7 @@ class State(object): ret = {'result': False, 'name': low['name'], 'changes': {}} self.state_con['runas'] = low.get('runas', None) + self.state_con['runas_passwd'] = low.get('runas_passwd', None) if not low.get('__prereq__'): log.info( From 41f0f75a0687af29c5954f90dbe59d5eec14362d Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 27 Jul 2017 18:01:26 -0600 Subject: [PATCH 181/508] Add new var to list, change to runas_password --- salt/modules/cmdmod.py | 2 +- salt/state.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index ea75865280..9f3364c594 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -295,7 +295,7 @@ def _run(cmd, runas = __context__.get('runas') if password is None and '__context__' in globals(): - password = __context__.get('runas_passwd') + password = __context__.get('runas_password') # Set the default working directory to the home directory of the user # salt-minion is running as. Defaults to home directory of user under which diff --git a/salt/state.py b/salt/state.py index af50b2c073..2ce6a3adae 100644 --- a/salt/state.py +++ b/salt/state.py @@ -97,6 +97,7 @@ STATE_RUNTIME_KEYWORDS = frozenset([ 'reload_grains', 'reload_pillar', 'runas', + 'runas_password', 'fire_event', 'saltenv', 'use', @@ -1751,7 +1752,7 @@ class State(object): ret = {'result': False, 'name': low['name'], 'changes': {}} self.state_con['runas'] = low.get('runas', None) - self.state_con['runas_passwd'] = low.get('runas_passwd', None) + self.state_con['runas_password'] = low.get('runas_password', None) if not low.get('__prereq__'): log.info( From d42f781c645bac43e09c6b309b66d70cbfe784b0 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 28 Jul 2017 09:38:55 +0100 Subject: [PATCH 182/508] Add trace logging of docker.networks result Added during debugging of failing test to determine what docker.networks is expected to return, keeping as it seems useful. --- salt/states/docker_network.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/states/docker_network.py b/salt/states/docker_network.py index a10162ba9d..28eaa05f67 100644 --- a/salt/states/docker_network.py +++ b/salt/states/docker_network.py @@ -90,6 +90,9 @@ def present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['docker.inspect_container'](c)['Id'] for c in containers] networks = __salt__['docker.networks'](names=[name]) + log.trace( + 'docker_network.present: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. @@ -159,6 +162,9 @@ def absent(name, driver=None): 'comment': ''} networks = __salt__['docker.networks'](names=[name]) + log.trace( + 'docker_network.absent: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. From d31f2913bdd5c76f64686fe81c576677939b65e1 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 28 Jul 2017 09:57:45 +0100 Subject: [PATCH 183/508] Fix broken unit test test_network_absent This started failing following commit 515c612, which relied on the 'Name' key being present in the return value of docker.networks - as the mock didn't have this set the test started failing. --- tests/unit/states/test_docker_network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/test_docker_network.py b/tests/unit/states/test_docker_network.py index 7b0037a7d4..aca1e9061e 100644 --- a/tests/unit/states/test_docker_network.py +++ b/tests/unit/states/test_docker_network.py @@ -69,10 +69,14 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): ''' docker_remove_network = Mock(return_value='removed') docker_disconnect_container_from_network = Mock(return_value='disconnected') + docker_networks = Mock(return_value=[{ + 'Name': 'network_foo', + 'Containers': {'container': {}} + }]) __salt__ = { 'docker.remove_network': docker_remove_network, 'docker.disconnect_container_from_network': docker_disconnect_container_from_network, - 'docker.networks': Mock(return_value=[{'Containers': {'container': {}}}]), + 'docker.networks': docker_networks, } with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): From c7d364ec5620a907e105d953b5542a6c057e6e06 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 28 Jul 2017 10:41:44 +0100 Subject: [PATCH 184/508] Add regression tests for #41982 These test the scenarios where another network with a similar name already exists, verifying that absent doesn't attempt to remove a network which isn't specified, and present still attempts to create the specified network despite a similarly named network already being present. --- tests/unit/states/test_docker_network.py | 40 +++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/test_docker_network.py b/tests/unit/states/test_docker_network.py index aca1e9061e..367441b937 100644 --- a/tests/unit/states/test_docker_network.py +++ b/tests/unit/states/test_docker_network.py @@ -43,10 +43,18 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): docker_create_network = Mock(return_value='created') docker_connect_container_to_network = Mock(return_value='connected') docker_inspect_container = Mock(return_value={'Id': 'abcd'}) + # Get docker.networks to return a network with a name which is a superset of the name of + # the network which is to be created, despite this network existing we should still expect + # that the new network will be created. + # Regression test for #41982. + docker_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) __salt__ = {'docker.create_network': docker_create_network, 'docker.inspect_container': docker_inspect_container, 'docker.connect_container_to_network': docker_connect_container_to_network, - 'docker.networks': Mock(return_value=[]), + 'docker.networks': docker_networks, } with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): @@ -89,3 +97,33 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): 'changes': {'disconnected': 'disconnected', 'removed': 'removed'}, 'result': True}) + + def test_absent_with_matching_network(self): + ''' + Test docker_network.absent when the specified network does not exist, + but another network with a name which is a superset of the specified + name does exist. In this case we expect there to be no attempt to remove + any network. + Regression test for #41982. + ''' + docker_remove_network = Mock(return_value='removed') + docker_disconnect_container_from_network = Mock(return_value='disconnected') + docker_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) + __salt__ = { + 'docker.remove_network': docker_remove_network, + 'docker.disconnect_container_from_network': docker_disconnect_container_from_network, + 'docker.networks': docker_networks, + } + with patch.dict(docker_state.__dict__, + {'__salt__': __salt__}): + ret = docker_state.absent('network_foo') + docker_disconnect_container_from_network.assert_not_called() + docker_remove_network.assert_not_called() + self.assertEqual(ret, {'name': 'network_foo', + 'comment': 'Network \'network_foo\' already absent', + 'changes': {}, + 'result': True}) + From ab21bd9b5b4b260b0533c0f5249a4969e6143962 Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Fri, 28 Jul 2017 06:00:16 -0600 Subject: [PATCH 185/508] Sync cloud modules when saltutil.sync_all is run Fixes #12587 --- salt/runners/saltutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index a6099e18b3..339a8b44ef 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -38,6 +38,7 @@ def sync_all(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' log.debug('Syncing all') ret = {} + ret['clouds'] = sync_clouds(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['modules'] = sync_modules(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['states'] = sync_states(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['grains'] = sync_grains(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) From f83960c02a510ea79ef153e29a41d72ade8e3591 Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Fri, 28 Jul 2017 09:13:59 -0600 Subject: [PATCH 186/508] Lint: Remove extra line at end of file. --- tests/unit/states/test_docker_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/states/test_docker_network.py b/tests/unit/states/test_docker_network.py index 367441b937..552170bdfb 100644 --- a/tests/unit/states/test_docker_network.py +++ b/tests/unit/states/test_docker_network.py @@ -126,4 +126,3 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): 'comment': 'Network \'network_foo\' already absent', 'changes': {}, 'result': True}) - From 2e132daa7370ee4c67a55768bb731016546fefcf Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 28 Jul 2017 08:33:21 -0700 Subject: [PATCH 187/508] Slight update to formatting --- salt/utils/decorators/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 9d459169bb..855b93a5dc 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -544,11 +544,11 @@ class _WithDeprecated(_DeprecationDecorator): opts = self._globals.get('__opts__', '{}') pillar = self._globals.get('__pillar__', '{}') - use_deprecated = full_name in opts.get(self.CFG_USE_DEPRECATED, list()) or \ - full_name in pillar.get(self.CFG_USE_DEPRECATED, list()) + use_deprecated = (full_name in opts.get(self.CFG_USE_DEPRECATED, list()) or + full_name in pillar.get(self.CFG_USE_DEPRECATED, list())) - use_superseded = full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) or \ - full_name in pillar.get(self.CFG_USE_SUPERSEDED, list()) + use_superseded = (full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) or + full_name in pillar.get(self.CFG_USE_SUPERSEDED, list())) if use_deprecated and use_superseded: raise SaltConfigurationError("Function '{0}' is mentioned both in deprecated " From ee4ea6b860574ec43d8ea53f8a721363e6cbe0a4 Mon Sep 17 00:00:00 2001 From: m03 Date: Sun, 11 Jun 2017 17:22:35 -0700 Subject: [PATCH 188/508] Fix #34245 ini.options_present reporting changes --- salt/states/ini_manage.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/states/ini_manage.py b/salt/states/ini_manage.py index 10cff758ca..8afef101d8 100644 --- a/salt/states/ini_manage.py +++ b/salt/states/ini_manage.py @@ -86,8 +86,9 @@ def options_present(name, sections=None, separator='=', strict=False): changes[section_name].update({key_to_remove: ''}) changes[section_name].update({key_to_remove: {'before': orig_value, 'after': None}}) - changes[section_name].update( - __salt__['ini.set_option'](name, {section_name: section_body}, separator)[section_name]) + options_updated = __salt__['ini.set_option'](name, {section_name: section_body}, separator) + if options_updated: + changes[section_name].update(options_updated[section_name]) else: changes = __salt__['ini.set_option'](name, sections, separator) except IOError as err: @@ -99,8 +100,12 @@ def options_present(name, sections=None, separator='=', strict=False): ret['comment'] = 'Errors encountered. {0}'.format(changes['error']) ret['changes'] = {} else: - ret['comment'] = 'Changes take effect' - ret['changes'] = changes + if all(value == {} for value in changes.values()): + ret['changes'] = {} + ret['comment'] = 'No changes take effect' + else: + ret['changes'] = changes + ret['comment'] = 'Changes take effect' return ret From 22c6a7c7ff5a82df7db094aba300a103e56a4515 Mon Sep 17 00:00:00 2001 From: m03 Date: Sun, 11 Jun 2017 18:01:55 -0700 Subject: [PATCH 189/508] Improve output precision --- salt/states/ini_manage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/states/ini_manage.py b/salt/states/ini_manage.py index 8afef101d8..e22202e0f1 100644 --- a/salt/states/ini_manage.py +++ b/salt/states/ini_manage.py @@ -89,6 +89,8 @@ def options_present(name, sections=None, separator='=', strict=False): options_updated = __salt__['ini.set_option'](name, {section_name: section_body}, separator) if options_updated: changes[section_name].update(options_updated[section_name]) + if not changes[section_name]: + del changes[section_name] else: changes = __salt__['ini.set_option'](name, sections, separator) except IOError as err: @@ -100,12 +102,12 @@ def options_present(name, sections=None, separator='=', strict=False): ret['comment'] = 'Errors encountered. {0}'.format(changes['error']) ret['changes'] = {} else: - if all(value == {} for value in changes.values()): - ret['changes'] = {} - ret['comment'] = 'No changes take effect' - else: + if changes: ret['changes'] = changes ret['comment'] = 'Changes take effect' + else: + ret['changes'] = {} + ret['comment'] = 'No changes take effect' return ret From 61aba35718f1157c1addd80676066c529251c51c Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 10:12:44 -0600 Subject: [PATCH 190/508] Deprecate password, make runas_password a named arg --- salt/states/cmd.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 8f55c85832..e032a6caba 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -654,6 +654,7 @@ def run(name, creates=None, cwd=None, runas=None, + runas_password=None, shell=None, env=None, stateful=False, @@ -687,6 +688,9 @@ def run(name, runas The user name to run the command as + runas_password + The password of the user specified in runas. Required for Windows + shell The shell to use for execution, defaults to the shell grain @@ -829,9 +833,19 @@ def run(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') + if 'password' in kwargs: + runas_password = kwargs.pop('password') + salt.utils.warn_until( + 'Neon', + 'The legacy password argument is deprecated. Use runas_password' + 'instead. This warning will be removed in Salt Neon') + if runas is not None: + __context__['runas'] = runas + if runas_password is not None: + __context__['runas_password'] = runas_password + cmd_kwargs = copy.deepcopy(kwargs) cmd_kwargs.update({'cwd': cwd, - 'runas': runas, 'use_vt': use_vt, 'shell': shell or __grains__['shell'], 'env': env, @@ -891,6 +905,7 @@ def script(name, creates=None, cwd=None, runas=None, + runas_password=None, shell=None, env=None, stateful=False, @@ -933,6 +948,9 @@ def script(name, runas The name of the user to run the command as + runas_password + The password of the user specified in runas. Required for Windows + shell The shell to use for execution. The default is set in grains['shell'] @@ -1065,9 +1083,19 @@ def script(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') + if 'password' in kwargs: + runas_password = kwargs.pop('password') + salt.utils.warn_until( + 'Neon', + 'The legacy password argument is deprecated. Use runas_password' + 'instead. This warning will be removed in Salt Neon') + if runas is not None: + __context__['runas'] = runas + if runas_password is not None: + __context__['runas_password'] = runas_password + cmd_kwargs = copy.deepcopy(kwargs) - cmd_kwargs.update({'runas': runas, - 'shell': shell or __grains__['shell'], + cmd_kwargs.update({'shell': shell or __grains__['shell'], 'env': env, 'onlyif': onlyif, 'unless': unless, @@ -1082,7 +1110,6 @@ def script(name, run_check_cmd_kwargs = { 'cwd': cwd, - 'runas': runas, 'shell': shell or __grains__['shell'] } From 4ea264e3dbc04f5781f72258c88d892344024b81 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 10:16:38 -0600 Subject: [PATCH 191/508] Change to runas_password in docs --- doc/ref/states/requisites.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index 4fbc0ad972..c77b7e44ca 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -538,9 +538,9 @@ runas_passwd .. versionadded:: 2017.7.2 -The ``runas_passwd`` global option is used to set the password used by the runas -global option. This is required by ``cmd.run`` on Windows when ``runas`` is -specified. It will be set when ``password`` is defined in the state. +The ``runas_password`` global option is used to set the password used by the +runas global option. This is required by ``cmd.run`` on Windows when ``runas`` +is specified. It will be set when ``runas_password`` is defined in the state. .. code-block:: yaml @@ -548,7 +548,7 @@ specified. It will be set when ``password`` is defined in the state. cmd.run: - name: Powershell -NonInteractive -ExecutionPolicy Bypass -File C:\\Temp\\script.ps1 - runas: frank - - password: supersecret + - runas_password: supersecret In the above state, the Powershell script run by ``cmd.run`` will be run by the frank user with the password ``supersecret``. From cd5eb9390386f7898d6a368f09269046fcc6ea0b Mon Sep 17 00:00:00 2001 From: Andreas Thienemann Date: Thu, 27 Jul 2017 16:40:35 +0200 Subject: [PATCH 192/508] Fix ssh-salt calls with scan roster for uncached clients When using salt-ssh with a scanning roster uncached minions lead to a traceback indicating an error creating the Single() object. Fix this error by ensuring that the target dictionary always contains a host key and the appropriate value. Fixes #42588 --- salt/client/ssh/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 22c770ea16..a25e8838b5 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -467,6 +467,8 @@ class SSH(object): for default in self.defaults: if default not in self.targets[host]: self.targets[host][default] = self.defaults[default] + if 'host' not in self.targets[host]: + self.targets[host]['host'] = host args = ( que, self.opts, From 181a1beecc30d451c46b96b060f9ec1a66db3adc Mon Sep 17 00:00:00 2001 From: Slawomir Bojarski Date: Wed, 26 Jul 2017 16:16:59 -0400 Subject: [PATCH 193/508] Fixed error reporting in "boto_cfn.present" function. Fixes #41433 --- salt/states/boto_cfn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index fcc7375464..87723f7b7f 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -141,10 +141,14 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi stack_policy_body = _get_template(stack_policy_body, name) stack_policy_during_update_body = _get_template(stack_policy_during_update_body, name) + for i in [template_body, stack_policy_body, stack_policy_during_update_body]: + if isinstance(i, dict): + return i + _valid = _validate(template_body, template_url, region, key, keyid, profile) log.debug('Validate is : {0}.'.format(_valid)) if _valid is not True: - code, message = _get_error(_valid) + code, message = _valid ret['result'] = False ret['comment'] = 'Template could not be validated.\n{0} \n{1}'.format(code, message) return ret From 5c945f10c244011a57dd9d97c41ced41677ea7d0 Mon Sep 17 00:00:00 2001 From: Slawomir Bojarski Date: Fri, 28 Jul 2017 12:57:10 -0400 Subject: [PATCH 194/508] Fix debug message in "boto_cfn._validate" function. --- salt/states/boto_cfn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index 87723f7b7f..c996b9006b 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -254,7 +254,7 @@ def _get_template(template, name): def _validate(template_body=None, template_url=None, region=None, key=None, keyid=None, profile=None): # Validates template. returns true if template syntax is correct. validate = __salt__['boto_cfn.validate_template'](template_body, template_url, region, key, keyid, profile) - log.debug('Validate is result is {0}.'.format(str(validate))) + log.debug('Validate result is {0}.'.format(str(validate))) if isinstance(validate, str): code, message = _get_error(validate) log.debug('Validate error is {0} and message is {1}.'.format(code, message)) From 6c71ab6f803320931b38ac07d642bc0a4adea02e Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:03:21 -0600 Subject: [PATCH 195/508] Remove runas and runas_password after state run --- salt/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/state.py b/salt/state.py index 2ce6a3adae..78cc8d128c 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1866,6 +1866,9 @@ class State(object): sys.modules[self.states[cdata['full']].__module__].__opts__[ 'test'] = test + self.state_con.pop('runas') + self.state_con.pop('runas_password') + # If format_call got any warnings, let's show them to the user if 'warnings' in cdata: ret.setdefault('warnings', []).extend(cdata['warnings']) From 18d6ce4d55ca60c6323aca172531bb520ff4642a Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:15:37 -0600 Subject: [PATCH 196/508] Add global vars to cmd.call --- salt/states/cmd.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index e032a6caba..2289dded97 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -839,6 +839,7 @@ def run(name, 'Neon', 'The legacy password argument is deprecated. Use runas_password' 'instead. This warning will be removed in Salt Neon') + if runas is not None: __context__['runas'] = runas if runas_password is not None: @@ -1089,6 +1090,7 @@ def script(name, 'Neon', 'The legacy password argument is deprecated. Use runas_password' 'instead. This warning will be removed in Salt Neon') + if runas is not None: __context__['runas'] = runas if runas_password is not None: @@ -1212,8 +1214,29 @@ def call(name, 'result': False, 'comment': ''} + if 'user' in kwargs or 'group' in kwargs: + salt.utils.warn_until( + 'Oxygen', + 'The legacy user/group arguments are deprecated. ' + 'Replace them with runas. ' + 'These arguments will be removed in Salt Oxygen.' + ) + if 'user' in kwargs and kwargs['user'] is not None and runas is None: + runas = kwargs.pop('user') + + if 'password' in kwargs: + runas_password = kwargs.pop('password') + salt.utils.warn_until( + 'Neon', + 'The legacy password argument is deprecated. Use runas_password' + 'instead. This warning will be removed in Salt Neon') + + if runas is not None: + __context__['runas'] = runas + if runas_password is not None: + __context__['runas_password'] = runas_password + cmd_kwargs = {'cwd': kwargs.get('cwd'), - 'runas': kwargs.get('user'), 'shell': kwargs.get('shell') or __grains__['shell'], 'env': kwargs.get('env'), 'use_vt': use_vt, From 464ec347139917c0b9c8233021012d7e55245f6f Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:20:48 -0600 Subject: [PATCH 197/508] Fix another instance of runas_passwd --- doc/ref/states/requisites.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index c77b7e44ca..3e3d29213c 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -533,8 +533,8 @@ the command in the ``cmd.run`` module. In the above state, the pip command run by ``cmd.run`` will be run by the daniel user. -runas_passwd -~~~~~~~~~~~~ +runas_password +~~~~~~~~~~~~~~ .. versionadded:: 2017.7.2 From 0c9e40012b473f51570516496b7b70e270c3d86f Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:39:02 -0600 Subject: [PATCH 198/508] Remove deprecation, add logic to state.py --- salt/state.py | 6 ++++- salt/states/cmd.py | 60 ++++------------------------------------------ 2 files changed, 10 insertions(+), 56 deletions(-) diff --git a/salt/state.py b/salt/state.py index 78cc8d128c..8d1ce6dae0 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1752,7 +1752,11 @@ class State(object): ret = {'result': False, 'name': low['name'], 'changes': {}} self.state_con['runas'] = low.get('runas', None) - self.state_con['runas_password'] = low.get('runas_password', None) + + if low['state'] == 'cmd' and 'password' in low: + self.state_con['runas_password'] = low['password'] + else: + self.state_con['runas_password'] = low.get('runas_password', None) if not low.get('__prereq__'): log.info( diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 2289dded97..8f55c85832 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -654,7 +654,6 @@ def run(name, creates=None, cwd=None, runas=None, - runas_password=None, shell=None, env=None, stateful=False, @@ -688,9 +687,6 @@ def run(name, runas The user name to run the command as - runas_password - The password of the user specified in runas. Required for Windows - shell The shell to use for execution, defaults to the shell grain @@ -833,20 +829,9 @@ def run(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') - if 'password' in kwargs: - runas_password = kwargs.pop('password') - salt.utils.warn_until( - 'Neon', - 'The legacy password argument is deprecated. Use runas_password' - 'instead. This warning will be removed in Salt Neon') - - if runas is not None: - __context__['runas'] = runas - if runas_password is not None: - __context__['runas_password'] = runas_password - cmd_kwargs = copy.deepcopy(kwargs) cmd_kwargs.update({'cwd': cwd, + 'runas': runas, 'use_vt': use_vt, 'shell': shell or __grains__['shell'], 'env': env, @@ -906,7 +891,6 @@ def script(name, creates=None, cwd=None, runas=None, - runas_password=None, shell=None, env=None, stateful=False, @@ -949,9 +933,6 @@ def script(name, runas The name of the user to run the command as - runas_password - The password of the user specified in runas. Required for Windows - shell The shell to use for execution. The default is set in grains['shell'] @@ -1084,20 +1065,9 @@ def script(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') - if 'password' in kwargs: - runas_password = kwargs.pop('password') - salt.utils.warn_until( - 'Neon', - 'The legacy password argument is deprecated. Use runas_password' - 'instead. This warning will be removed in Salt Neon') - - if runas is not None: - __context__['runas'] = runas - if runas_password is not None: - __context__['runas_password'] = runas_password - cmd_kwargs = copy.deepcopy(kwargs) - cmd_kwargs.update({'shell': shell or __grains__['shell'], + cmd_kwargs.update({'runas': runas, + 'shell': shell or __grains__['shell'], 'env': env, 'onlyif': onlyif, 'unless': unless, @@ -1112,6 +1082,7 @@ def script(name, run_check_cmd_kwargs = { 'cwd': cwd, + 'runas': runas, 'shell': shell or __grains__['shell'] } @@ -1214,29 +1185,8 @@ def call(name, 'result': False, 'comment': ''} - if 'user' in kwargs or 'group' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'The legacy user/group arguments are deprecated. ' - 'Replace them with runas. ' - 'These arguments will be removed in Salt Oxygen.' - ) - if 'user' in kwargs and kwargs['user'] is not None and runas is None: - runas = kwargs.pop('user') - - if 'password' in kwargs: - runas_password = kwargs.pop('password') - salt.utils.warn_until( - 'Neon', - 'The legacy password argument is deprecated. Use runas_password' - 'instead. This warning will be removed in Salt Neon') - - if runas is not None: - __context__['runas'] = runas - if runas_password is not None: - __context__['runas_password'] = runas_password - cmd_kwargs = {'cwd': kwargs.get('cwd'), + 'runas': kwargs.get('user'), 'shell': kwargs.get('shell') or __grains__['shell'], 'env': kwargs.get('env'), 'use_vt': use_vt, From fcf45889ddf655256fd79d66ff0837c5b49955b8 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 28 Jul 2017 15:35:30 -0500 Subject: [PATCH 199/508] Fix unicode constructor in custom YAML loader 2e7f743 added this in the wrong place, it should not have been added within the if block since we will _always_ want this constructor available. --- salt/utils/yamlloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/utils/yamlloader.py b/salt/utils/yamlloader.py index a51d1f1f25..7beb94b767 100644 --- a/salt/utils/yamlloader.py +++ b/salt/utils/yamlloader.py @@ -45,9 +45,9 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): self.add_constructor( u'tag:yaml.org,2002:omap', type(self).construct_yaml_map) - self.add_constructor( - u'tag:yaml.org,2002:python/unicode', - type(self).construct_unicode) + self.add_constructor( + u'tag:yaml.org,2002:python/unicode', + type(self).construct_unicode) self.dictclass = dictclass def construct_yaml_map(self, node): From b17495c9c86c15cbd30dd0df5709baa5f7b83b5e Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 14:46:01 -0600 Subject: [PATCH 200/508] Fix problem with list when install=True --- salt/modules/win_wua.py | 193 ++++++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 77 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 796da27ee4..74843868a6 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -63,27 +63,31 @@ def available(software=True, Args: - software (bool): Include software updates in the results (default is - True) + software (bool): + Include software updates in the results (default is True) - drivers (bool): Include driver updates in the results (default is False) + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip updates that are already installed. Default - is False. + skip_installed (bool): + Skip updates that are already installed. Default is False. - skip_hidden (bool): Skip updates that have been hidden. Default is True. + skip_hidden (bool): + Skip updates that have been hidden. Default is True. - skip_mandatory (bool): Skip mandatory updates. Default is False. + skip_mandatory (bool): + Skip mandatory updates. Default is False. - skip_reboot (bool): Skip updates that require a reboot. Default is - False. + skip_reboot (bool): + Skip updates that require a reboot. Default is False. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -101,8 +105,9 @@ def available(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -152,19 +157,19 @@ def available(software=True, salt '*' win_wua.available # List all updates with categories of Critical Updates and Drivers - salt '*' win_wua.available categories=['Critical Updates','Drivers'] + salt '*' win_wua.available categories=["Critical Updates","Drivers"] # List all Critical Security Updates - salt '*' win_wua.available categories=['Security Updates'] severities=['Critical'] + salt '*' win_wua.available categories=["Security Updates"] severities=["Critical"] # List all updates with a severity of Critical - salt '*' win_wua.available severities=['Critical'] + salt '*' win_wua.available severities=["Critical"] # A summary of all available updates salt '*' win_wua.available summary=True # A summary of all Feature Packs and Windows 8.1 Updates - salt '*' win_wua.available categories=['Feature Packs','Windows 8.1'] summary=True + salt '*' win_wua.available categories=["Feature Packs","Windows 8.1"] summary=True ''' # Create a Windows Update Agent instance @@ -261,18 +266,21 @@ def get(name, download=False, install=False): Returns details for all updates that match the search criteria Args: - name (str): The name of the update you're searching for. This can be the - GUID, a KB number, or any part of the name of the update. GUIDs and - KBs are preferred. Run ``list`` to get the GUID for the update - you're looking for. + name (str): + The name of the update you're searching for. This can be the GUID, a + KB number, or any part of the name of the update. GUIDs and KBs are + preferred. Run ``list`` to get the GUID for the update you're + looking for. - download (bool): Download the update returned by this function. Run this - function first to see if the update exists, then set ``download=True`` - to download the update. + download (bool): + Download the update returned by this function. Run this function + first to see if the update exists, then set ``download=True`` to + download the update. - install (bool): Install the update returned by this function. Run this - function first to see if the update exists, then set ``install=True`` to - install the update. + install (bool): + Install the update returned by this function. Run this function + first to see if the update exists, then set ``install=True`` to + install the update. Returns: dict: Returns a dict containing a list of updates that match the name if @@ -486,30 +494,34 @@ def list(software=True, install is True the same list will be downloaded and/or installed. Args: - software (bool): Include software updates in the results (default is - True) + software (bool): + Include software updates in the results (default is True) - drivers (bool): Include driver updates in the results (default is False) + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip installed updates in the results (default is - False) + skip_installed (bool): + Skip installed updates in the results (default is False) - download (bool): (Overrides reporting functionality) Download the list - of updates returned by this function. Run this function first with - ``download=False`` to see what will be downloaded, then set - ``download=True`` to download the updates. + download (bool): + (Overrides reporting functionality) Download the list of updates + returned by this function. Run this function first with + ``download=False`` to see what will be downloaded, then set + ``download=True`` to download the updates. - install (bool): (Overrides reporting functionality) Install the list of - updates returned by this function. Run this function first with - ``install=False`` to see what will be installed, then set - ``install=True`` to install the updates. + install (bool): + (Overrides reporting functionality) Install the list of updates + returned by this function. Run this function first with + ``install=False`` to see what will be installed, then set + ``install=True`` to install the updates. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -527,8 +539,9 @@ def list(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -575,22 +588,22 @@ def list(software=True, .. code-block:: bash # Normal Usage (list all software updates) - salt '*' win_wua.list_updates + salt '*' win_wua.list # List all updates with categories of Critical Updates and Drivers - salt '*' win_wua.list_updates categories=['Critical Updates','Drivers'] + salt '*' win_wua.list categories=['Critical Updates','Drivers'] # List all Critical Security Updates - salt '*' win_wua.list_updates categories=['Security Updates'] severities=['Critical'] + salt '*' win_wua.list categories=['Security Updates'] severities=['Critical'] # List all updates with a severity of Critical - salt '*' win_wua.list_updates severities=['Critical'] + salt '*' win_wua.list severities=['Critical'] # A summary of all available updates - salt '*' win_wua.list_updates summary=True + salt '*' win_wua.list summary=True # A summary of all Feature Packs and Windows 8.1 Updates - salt '*' win_wua.list_updates categories=['Feature Packs','Windows 8.1'] summary=True + salt '*' win_wua.list categories=['Feature Packs','Windows 8.1'] summary=True ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -604,11 +617,11 @@ def list(software=True, # Download if download or install: - ret['Download'] = wua.download(updates.updates) + ret['Download'] = wua.download(updates) # Install if install: - ret['Install'] = wua.install(updates.updates) + ret['Install'] = wua.install(updates) if not ret: return updates.summary() if summary else updates.list() @@ -690,9 +703,14 @@ def download(names): Args: - names (str, list): A single update or a list of updates to download. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to download. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. + + .. note:: + An error will be raised if there are more results than there are items + in the names parameter Returns: @@ -703,7 +721,7 @@ def download(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.download guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] + salt '*' win_wua.download names=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -714,6 +732,13 @@ def download(names): if updates.count() == 0: raise CommandExecutionError('No updates found') + # Make sure it's a list so count comparison is correct + if isinstance(names, six.string_types): + names = [names] + + if isinstance(names, six.integer_types): + names = [str(names)] + if updates.count() > len(names): raise CommandExecutionError('Multiple updates found, names need to be ' 'more specific') @@ -795,9 +820,14 @@ def install(names): Args: - names (str, list): A single update or a list of updates to install. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to install. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. + + .. note:: + An error will be raised if there are more results than there are items + in the names parameter Returns: @@ -808,7 +838,7 @@ def install(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.install_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB12323211'] + salt '*' win_wua.install KB12323211 ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -819,6 +849,13 @@ def install(names): if updates.count() == 0: raise CommandExecutionError('No updates found') + # Make sure it's a list so count comparison is correct + if isinstance(names, six.string_types): + names = [names] + + if isinstance(names, six.integer_types): + names = [str(names)] + if updates.count() > len(names): raise CommandExecutionError('Multiple updates found, names need to be ' 'more specific') @@ -834,9 +871,10 @@ def uninstall(names): Args: - names (str, list): A single update or a list of updates to uninstall. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to uninstall. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. Returns: @@ -1110,30 +1148,31 @@ def get_wu_settings(): Boolean value that indicates whether to display notifications for featured updates. Group Policy Required (Read-only): - Boolean value that indicates whether Group Policy requires the Automatic - Updates service. + Boolean value that indicates whether Group Policy requires the + Automatic Updates service. Microsoft Update: Boolean value that indicates whether to turn on Microsoft Update for other Microsoft Products Needs Reboot: - Boolean value that indicates whether the machine is in a reboot pending - state. + Boolean value that indicates whether the machine is in a reboot + pending state. Non Admins Elevated: - Boolean value that indicates whether non-administrators can perform some - update-related actions without administrator approval. + Boolean value that indicates whether non-administrators can perform + some update-related actions without administrator approval. Notification Level: Number 1 to 4 indicating the update level: 1. Never check for updates - 2. Check for updates but let me choose whether to download and install them + 2. Check for updates but let me choose whether to download and + install them 3. Download updates but let me choose whether to install them 4. Install updates automatically Read Only (Read-only): Boolean value that indicates whether the Automatic Update settings are read-only. Recommended Updates: - Boolean value that indicates whether to include optional or recommended - updates when a search for updates and installation of updates is - performed. + Boolean value that indicates whether to include optional or + recommended updates when a search for updates and installation of + updates is performed. Scheduled Day: Days of the week on which Automatic Updates installs or uninstalls updates. From 19f34bda55f18aeb72ce2accb0340887d7689356 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 15:06:52 -0600 Subject: [PATCH 201/508] Fix docs, formatting --- salt/modules/win_wua.py | 156 +++++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 66 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 74843868a6..9ef58881cc 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -188,23 +188,29 @@ def list_update(name, download=False, install=False): ''' .. deprecated:: 2017.7.0 Use :func:`get` instead + Returns details for all updates that match the search criteria Args: - name (str): The name of the update you're searching for. This can be the - GUID, a KB number, or any part of the name of the update. GUIDs and - KBs are preferred. Run ``list_updates`` to get the GUID for the update - you're looking for. - download (bool): Download the update returned by this function. Run this - function first to see if the update exists, then set ``download=True`` - to download the update. + name (str): + The name of the update you're searching for. This can be the GUID, a + KB number, or any part of the name of the update. GUIDs and KBs are + preferred. Run ``list_updates`` to get the GUID for the update + you're looking for. - install (bool): Install the update returned by this function. Run this - function first to see if the update exists, then set ``install=True`` to - install the update. + download (bool): + Download the update returned by this function. Run this function + first to see if the update exists, then set ``download=True`` to + download the update. + + install (bool): + Install the update returned by this function. Run this function + first to see if the update exists, then set ``install=True`` to + install the update. Returns: + dict: Returns a dict containing a list of updates that match the name if download and install are both set to False. Should usually be a single update, but can return multiple if a partial name is given. @@ -263,9 +269,10 @@ def get(name, download=False, install=False): ''' .. versionadded:: 2017.7.0 - Returns details for all updates that match the search criteria + Returns details for the named update Args: + name (str): The name of the update you're searching for. This can be the GUID, a KB number, or any part of the name of the update. GUIDs and KBs are @@ -283,6 +290,7 @@ def get(name, download=False, install=False): install the update. Returns: + dict: Returns a dict containing a list of updates that match the name if download and install are both set to False. Should usually be a single update, but can return multiple if a partial name is given. @@ -365,30 +373,35 @@ def list_updates(software=True, install is True the same list will be downloaded and/or installed. Args: - software (bool): Include software updates in the results (default is - True) - drivers (bool): Include driver updates in the results (default is False) + software (bool): + Include software updates in the results (default is True) + + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip installed updates in the results (default is - False) + skip_installed (bool): + Skip installed updates in the results (default is False) - download (bool): (Overrides reporting functionality) Download the list - of updates returned by this function. Run this function first with - ``download=False`` to see what will be downloaded, then set - ``download=True`` to download the updates. + download (bool): + (Overrides reporting functionality) Download the list of updates + returned by this function. Run this function first with + ``download=False`` to see what will be downloaded, then set + ``download=True`` to download the updates. - install (bool): (Overrides reporting functionality) Install the list of - updates returned by this function. Run this function first with - ``install=False`` to see what will be installed, then set - ``install=True`` to install the updates. + install (bool): + (Overrides reporting functionality) Install the list of updates + returned by this function. Run this function first with + ``install=False`` to see what will be installed, then set + ``install=True`` to install the updates. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -406,8 +419,9 @@ def list_updates(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -494,6 +508,7 @@ def list(software=True, install is True the same list will be downloaded and/or installed. Args: + software (bool): Include software updates in the results (default is True) @@ -638,13 +653,16 @@ def download_update(name): Args: - name (str): The name of the update to download. This can be a GUID, a KB - number, or any part of the name. To ensure a single item is matched the - GUID is preferred. + name (str): + The name of the update to download. This can be a GUID, a KB number, + or any part of the name. To ensure a single item is matched the GUID + is preferred. - .. note:: If more than one result is returned an error will be raised. + .. note:: + If more than one result is returned an error will be raised. Returns: + dict: A dictionary containing the results of the download CLI Examples: @@ -654,7 +672,6 @@ def download_update(name): salt '*' win_wua.download_update 12345678-abcd-1234-abcd-1234567890ab salt '*' win_wua.download_update KB12312321 - ''' salt.utils.warn_until( 'Fluorine', @@ -673,8 +690,9 @@ def download_updates(names): Args: - names (list): A list of updates to download. This can be any combination - of GUIDs, KB numbers, or names. GUIDs or KBs are preferred. + names (list): + A list of updates to download. This can be any combination of GUIDs, + KB numbers, or names. GUIDs or KBs are preferred. Returns: @@ -685,7 +703,7 @@ def download_updates(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.download guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] + salt '*' win_wua.download_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] ''' salt.utils.warn_until( 'Fluorine', @@ -759,10 +777,12 @@ def install_update(name): number, or any part of the name. To ensure a single item is matched the GUID is preferred. - .. note:: If no results or more than one result is returned an error - will be raised. + .. note:: + If no results or more than one result is returned an error will be + raised. Returns: + dict: A dictionary containing the results of the install CLI Examples: @@ -926,34 +946,37 @@ def set_wu_settings(level=None, Update settings are read-only. See MSDN documentation: https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx - :param int level: - Number from 1 to 4 indicating the update level: + Args: + + level (int): + Number from 1 to 4 indicating the update level: + 1. Never check for updates 2. Check for updates but let me choose whether to download and install them 3. Download updates but let me choose whether to install them 4. Install updates automatically - :param bool recommended: - Boolean value that indicates whether to include optional or recommended - updates when a search for updates and installation of updates is - performed. + recommended (bool): + Boolean value that indicates whether to include optional or + recommended updates when a search for updates and installation of + updates is performed. - :param bool featured: - Boolean value that indicates whether to display notifications for - featured updates. + featured (bool): + Boolean value that indicates whether to display notifications for + featured updates. - :param bool elevated: - Boolean value that indicates whether non-administrators can perform some - update-related actions without administrator approval. + elevated (bool): + Boolean value that indicates whether non-administrators can perform + some update-related actions without administrator approval. - :param bool msupdate: - Boolean value that indicates whether to turn on Microsoft Update for - other Microsoft products + msupdate (bool): + Boolean value that indicates whether to turn on Microsoft Update for + other Microsoft products + + day (str): + Days of the week on which Automatic Updates installs or uninstalls + updates. Accepted values: - :param str day: - Days of the week on which Automatic Updates installs or uninstalls - updates. - Accepted values: - Everyday - Monday - Tuesday @@ -962,18 +985,20 @@ def set_wu_settings(level=None, - Friday - Saturday - :param str time: - Time at which Automatic Updates installs or uninstalls updates. Must be - in the ##:## 24hr format, eg. 3:00 PM would be 15:00 + time (str): + Time at which Automatic Updates installs or uninstalls updates. Must + be in the ##:## 24hr format, eg. 3:00 PM would be 15:00. Must be in + 1 hour increments. - :return: Returns a dictionary containing the results. + Returns: + + dict: Returns a dictionary containing the results. CLI Examples: .. code-block:: bash salt '*' win_wua.set_wu_settings level=4 recommended=True featured=False - ''' # The AutomaticUpdateSettings.Save() method used in this function does not # work on Windows 10 / Server 2016. It is called in throughout this function @@ -1255,13 +1280,12 @@ def get_needs_reboot(): Returns: - bool: True if the system requires a reboot, False if not + bool: True if the system requires a reboot, otherwise False CLI Examples: .. code-block:: bash salt '*' win_wua.get_needs_reboot - ''' return salt.utils.win_update.needs_reboot() From 1340c15ce7c0b518e8aed6bb9d89e924e192930d Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 15:38:14 -0600 Subject: [PATCH 202/508] Add general usage instructions --- salt/modules/win_wua.py | 57 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 9ef58881cc..237fb74924 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -2,6 +2,49 @@ ''' Module for managing Windows Updates using the Windows Update Agent. +List updates on the system using the following functions: + +- :ref:`available` +- :ref:`list` + +This is an easy way to find additional information about updates available to +to the system, such as the GUID, KB number, or description. + +Once you have the GUID or a KB number for the update you can get information +about the update, download, install, or uninstall it using these functions: + +- :ref:`get` +- :ref:`download` +- :ref:`install` +- :ref:`uninstall` + +The get function expects a name in the form of a GUID, KB, or Title and should +return information about a single update. The other functions accept either a +single item or a list of items for downloading/installing/uninstalling a +specific list of items. + +The :ref:`list` and :ref:`get` functions are utility functions. In addition to +returning information about updates they can also download and install updates +by setting ``download=True`` or ``install=True``. So, with :ref:`list` for +example, you could run the function with the filters you want to see what is +available. Then just add ``install=True`` to install everything on that list. + +If you want to download, install, or uninstall specific updates, use +:ref:`download`, :ref:`install`, or :ref:`uninstall`. To update your system +with the latest updates use :ref:`list` and set ``install=True`` + +You can also adjust the Windows Update settings using the :ref:`set_wu_settings` +function. This function is only supported on the following operating systems: + +- Windows Vista / Server 2008 +- Windows 7 / Server 2008R2 +- Windows 8 / Server 2012 +- Windows 8.1 / Server 2012R2 + +As of Windows 10 and Windows Server 2016, the ability to modify the Windows +Update settings has been restricted. The settings can be modified in the Local +Group Policy using the ``lgpo`` module. + .. versionadded:: 2015.8.0 :depends: @@ -54,12 +97,12 @@ def available(software=True, skip_mandatory=False, skip_reboot=False, categories=None, - severities=None, - ): + severities=None,): ''' .. versionadded:: 2017.7.0 - List updates that match the passed criteria. + List updates that match the passed criteria. This allows for more filter + options than :func:`list`. Good for finding a specific GUID or KB. Args: @@ -176,9 +219,11 @@ def available(software=True, wua = salt.utils.win_update.WindowsUpdateAgent() # Look for available - updates = wua.available(skip_hidden, skip_installed, skip_mandatory, - skip_reboot, software, drivers, categories, - severities) + updates = wua.available( + skip_hidden=skip_hidden, skip_installed=skip_installed, + skip_mandatory=skip_mandatory, skip_reboot=skip_reboot, + software=software, drivers=drivers, categories=categories, + severities=severities) # Return results as Summary or Details return updates.summary() if summary else updates.list() From 1e13383b95eca138b690433c4ad12b08e123b3a3 Mon Sep 17 00:00:00 2001 From: "xiaofei.sun" Date: Sat, 29 Jul 2017 11:40:02 +0800 Subject: [PATCH 203/508] tornado api --- salt/netapi/rest_tornado/saltnado.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 6622b14635..d70fb6e753 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -521,8 +521,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin try: # Use cgi.parse_header to correctly separate parameters from value - header = cgi.parse_header(self.request.headers['Content-Type']) - value, parameters = header + value, parameters = cgi.parse_header(self.request.headers['Content-Type']) return ct_in_map[value](data) except KeyError: self.send_error(406) @@ -536,7 +535,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin if not self.request.body: return data = self.deserialize(self.request.body) - self.raw_data = copy(data) + self.request_payload = copy(data) if 'arg' in data and not isinstance(data['arg'], list): data['arg'] = [data['arg']] @@ -694,15 +693,13 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 }} ''' try: - request_payload = self.deserialize(self.request.body) - - if not isinstance(request_payload, dict): + if not isinstance(self.request_payload, dict): self.send_error(400) return - creds = {'username': request_payload['username'], - 'password': request_payload['password'], - 'eauth': request_payload['eauth'], + creds = {'username': self.request_payload['username'], + 'password': self.request_payload['password'], + 'eauth': self.request_payload['eauth'], } # if any of the args are missing, its a bad request except KeyError: @@ -1633,7 +1630,7 @@ class WebhookSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 listen=False) ret = self.event.fire_event({ - 'post': self.raw_data, + 'post': self.request_payload, 'get': dict(self.request.query_arguments), # In Tornado >= v4.0.3, the headers come # back as an HTTPHeaders instance, which From 49023deb9438a67c46c92776b09896ce1feb24ad Mon Sep 17 00:00:00 2001 From: Morgan Willcock Date: Mon, 31 Jul 2017 21:17:35 +0100 Subject: [PATCH 204/508] Disable ZFS grain on NetBSD --- salt/grains/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index 71ad3d0721..824ec0fccd 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2351,6 +2351,10 @@ def _zpool_data(grains): if salt.utils.is_windows() or 'proxyminion' in __opts__: return {} + # quickly return if NetBSD (ZFS still under development) + if salt.utils.is_netbsd(): + return {} + # quickly return if no zpool and zfs command if not salt.utils.which('zpool'): return {} From 6bd91c8b03ef14bf1de1e66bcfd23b557a011701 Mon Sep 17 00:00:00 2001 From: Seth House Date: Mon, 31 Jul 2017 14:15:52 -0600 Subject: [PATCH 205/508] Reenable cpstats for rest_cherrypy --- salt/netapi/rest_cherrypy/app.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 48f1330018..5c55433bb3 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -468,11 +468,19 @@ import tarfile from multiprocessing import Process, Pipe # Import third-party libs -# pylint: disable=import-error +# pylint: disable=import-error, 3rd-party-module-not-gated import cherrypy +try: + from cherrypy.lib import cpstats +except ImportError: + cpstats = None + logger.warn('Import of cherrypy.cpstats failed. ' + 'Possible upstream bug: ' + 'https://github.com/cherrypy/cherrypy/issues/1444') + import yaml import salt.ext.six as six -# pylint: enable=import-error +# pylint: enable=import-error, 3rd-party-module-not-gated # Import Salt libs @@ -2446,13 +2454,6 @@ class Stats(object): :status 406: |406| ''' if hasattr(logging, 'statistics'): - # Late import - try: - from cherrypy.lib import cpstats - except ImportError: - logger.error('Import of cherrypy.cpstats failed. Possible ' - 'upstream bug here: https://github.com/cherrypy/cherrypy/issues/1444') - return {} return cpstats.extrapolate_statistics(logging.statistics) return {} @@ -2559,13 +2560,14 @@ class API(object): 'tools.trailing_slash.on': True, 'tools.gzip.on': True, - 'tools.cpstats.on': self.apiopts.get('collect_stats', False), - 'tools.html_override.on': True, 'tools.cors_tool.on': True, }, } + if cpstats and self.apiopts.get('collect_stats', False): + conf['/']['tools.cpstats.on'] = True + if 'favicon' in self.apiopts: conf['/favicon.ico'] = { 'tools.staticfile.on': True, From d73c4b55b7ca288c25b8e21ddfb879ede646cc9b Mon Sep 17 00:00:00 2001 From: Neile Havens Date: Mon, 31 Jul 2017 17:10:06 -0500 Subject: [PATCH 206/508] back-port #42612 to 2017.7 --- salt/states/selinux.py | 12 +++++++++--- tests/unit/states/test_selinux.py | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/states/selinux.py b/salt/states/selinux.py index 868dbc16e3..8187ea8338 100644 --- a/salt/states/selinux.py +++ b/salt/states/selinux.py @@ -171,8 +171,14 @@ def boolean(name, value, persist=False): name, rvalue) return ret - if __salt__['selinux.setsebool'](name, rvalue, persist): + ret['result'] = __salt__['selinux.setsebool'](name, rvalue, persist) + if ret['result']: ret['comment'] = 'Boolean {0} has been set to {1}'.format(name, rvalue) + ret['changes'].update({'State': {'old': bools[name]['State'], + 'new': rvalue}}) + if persist and not default: + ret['changes'].update({'Default': {'old': bools[name]['Default'], + 'new': rvalue}}) return ret ret['comment'] = 'Failed to set the boolean {0} to {1}'.format(name, rvalue) return ret @@ -262,7 +268,7 @@ def module_install(name): name Path to file with module to install - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' ret = {'name': name, 'result': True, @@ -283,7 +289,7 @@ def module_remove(name): name The name of the module to remove - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' ret = {'name': name, 'result': True, diff --git a/tests/unit/states/test_selinux.py b/tests/unit/states/test_selinux.py index 36474b95c4..f6ce3bfb8f 100644 --- a/tests/unit/states/test_selinux.py +++ b/tests/unit/states/test_selinux.py @@ -118,9 +118,11 @@ class SelinuxTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(selinux.__opts__, {'test': False}): comt = ('Boolean samba_create_home_dirs has been set to on') ret.update({'comment': comt, 'result': True}) + ret.update({'changes': {'State': {'old': 'off', 'new': 'on'}}}) self.assertDictEqual(selinux.boolean(name, value), ret) comt = ('Failed to set the boolean ' 'samba_create_home_dirs to on') - ret.update({'comment': comt, 'result': True}) + ret.update({'comment': comt, 'result': False}) + ret.update({'changes': {}}) self.assertDictEqual(selinux.boolean(name, value), ret) From 389c037285190e34ea57374a2464f33cdf0ca894 Mon Sep 17 00:00:00 2001 From: Steven Joseph Date: Mon, 17 Jul 2017 14:07:36 +1000 Subject: [PATCH 207/508] Check remote tags before deciding to do a fetch #42329 --- salt/states/git.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/states/git.py b/salt/states/git.py index 274662e83b..a1bf29cecc 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1308,6 +1308,19 @@ def latest(name, 'if it does not already exist).', comments ) + if set(all_local_tags) != set([ + x.split('/')[-1] for x in __salt__['git.ls_remote']( + cwd=target, + remote=remote, + opts="--tags", + user=user, + password=password, + identity=identity, + saltenv=__env__, + ignore_retcode=True, + ).keys() + ]): + has_remote_rev = False if not has_remote_rev: try: From 24413084e2255c6319e1213fdbdacc21a4ccc8cb Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 1 Aug 2017 12:40:31 -0700 Subject: [PATCH 208/508] Updating the call to shlex_split to pass the posix=False argument so that quotes are preserved. --- salt/modules/augeas_cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/augeas_cfg.py b/salt/modules/augeas_cfg.py index 6b1f1e7b1b..401a11eb1b 100644 --- a/salt/modules/augeas_cfg.py +++ b/salt/modules/augeas_cfg.py @@ -199,7 +199,7 @@ def execute(context=None, lens=None, commands=(), load_path=None): method = METHOD_MAP[cmd] nargs = arg_map[method] - parts = salt.utils.shlex_split(arg) + parts = salt.utils.shlex_split(arg, posix=False) if len(parts) not in nargs: err = '{0} takes {1} args: {2}'.format(method, nargs, parts) From deb6316d6782807a692756347a2ad3889b9d71a8 Mon Sep 17 00:00:00 2001 From: Seth House Date: Tue, 1 Aug 2017 14:17:13 -0600 Subject: [PATCH 209/508] Fix lint errors --- salt/netapi/rest_cherrypy/app.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 5c55433bb3..6eb389242e 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -467,8 +467,10 @@ import signal import tarfile from multiprocessing import Process, Pipe +logger = logging.getLogger(__name__) + # Import third-party libs -# pylint: disable=import-error, 3rd-party-module-not-gated +# pylint: disable=import-error import cherrypy try: from cherrypy.lib import cpstats @@ -480,8 +482,7 @@ except ImportError: import yaml import salt.ext.six as six -# pylint: enable=import-error, 3rd-party-module-not-gated - +# pylint: enable=import-error # Import Salt libs import salt @@ -491,8 +492,6 @@ import salt.utils.event # Import salt-api libs import salt.netapi -logger = logging.getLogger(__name__) - # Imports related to websocket try: from .tools import websockets From a2565ba8e570f3e969ed5660e3a798e6fd852610 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 2 Aug 2017 09:39:18 +0000 Subject: [PATCH 210/508] Add new napalm option: multiprocessing --- salt/utils/napalm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index 36914be042..0ca387cc68 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -243,6 +243,11 @@ def get_device_opts(opts, salt_obj=None): network_device = {} # by default, look in the proxy config details device_dict = opts.get('proxy', {}) or opts.get('napalm', {}) + if opts.get('proxy') or opts.get('napalm'): + opts['multiprocessing'] = device_dict.get('multiprocessing', False) + # Most NAPALM drivers are SSH-based, so multiprocessing should default to False. + # But the user can be allows to have a different value for the multiprocessing, which will + # override the opts. if salt_obj and not device_dict: # get the connection details from the opts device_dict = salt_obj['config.merge']('napalm') From 1598571f52583213ccc6d5d0ef8b403483cdc72f Mon Sep 17 00:00:00 2001 From: Frankie Hui Date: Wed, 2 Aug 2017 17:47:31 +0800 Subject: [PATCH 211/508] Add eos.rst in the installation guide --- doc/topics/installation/eos.rst | 155 ++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 doc/topics/installation/eos.rst diff --git a/doc/topics/installation/eos.rst b/doc/topics/installation/eos.rst new file mode 100644 index 0000000000..f7537c334b --- /dev/null +++ b/doc/topics/installation/eos.rst @@ -0,0 +1,155 @@ + +========================================= +Arista EOS Salt minion installation guide +========================================= + +The Salt minion for Arista EOS is distributed as a SWIX extension and can be installed directly on the switch. The EOS network operating system is based on old Fedora distributions and the installation of the ``salt-minion`` requires backports. This SWIX extension contains the necessary backports, together with the Salt basecode. + +.. note:: + + This SWIX extension has been tested on Arista DCS-7280SE-68-R, running EOS 4.17.5M and vEOS 4.18.3F. + +Important Notes +=============== + +This package is in beta, make sure to test it carefully before running it in production. + +If confirmed working correctly, please report and add a note on this page with the platform model and EOS version. + +If you want to uninstall this package, please refer to the uninstalling_ section. + +Installation from the Official SaltStack Repository +=================================================== + +Download the swix package and save it to flash. + +.. code-block:: bash + + veos#copy https://salt-eos.netops.life/salt-eos-latest.swix flash: + veos#copy https://salt-eos.netops.life/startup.sh flash: + +Install the Extension +===================== + +Copy the Salt package to extension + +.. code-block:: bash + + veos#copy flash:salt-eos-latest.swix extension: + +Install the SWIX + +.. code-block:: bash + + veos#extension salt-eos-latest.swix force + +Verify the installation + +.. code-block:: bash + + veos#show extensions | include salt-eos + salt-eos-2017-07-19.swix 1.0.11/1.fc25 A, F 27 + +Change the Salt master IP address or FQDN, by edit the variable (SALT_MASTER) + +.. code-block:: bash + + veos#bash vi /mnt/flash/startup.sh + +Make sure you enable the eAPI with unix-socket + +.. code-block:: bash + + veos(config)#management api http-commands + protocol unix-socket + no shutdown + +Post-installation tasks +======================= + +Generate Keys and host record and start Salt minion + +.. code-block:: bash + + veos#bash + #sudo /mnt/flash/startup.sh + +``salt-minion`` should be running + +Copy the installed extensions to boot-extensions + +.. code-block:: bash + + veos#copy installed-extensions boot-extensions + +Apply event-handler to let EOS start salt-minion during boot-up + +.. code-block:: bash + + veos(config)#event-handler boot-up-script + trigger on-boot + action bash sudo /mnt/flash/startup.sh + +For more specific installation details of the ``salt-minion``, please refer to :ref:`Configuring Salt`. + +.. _uninstalling: + +Uninstalling +============ + +If you decide to uninstall this package, the following steps are recommended for safety: + +1. Remove the extension from boot-extensions + +.. code-block:: bash + + veos#bash rm /mnt/flash/boot-extensions + +2. Remove the extension from extensions folder + +.. code-block:: bash + + veos#bash rm /mnt/flash/.extensions/salt-eos-latest.swix + +2. Remove boot-up script + +.. code-block:: bash + + veos(config)#no event-handler boot-up-script + +Additional Information +====================== + +This SWIX extension contains the following RPM packages: + +.. code-block:: + + libsodium-1.0.11-1.fc25.i686.rpm + libstdc++-6.2.1-2.fc25.i686.rpm + openpgm-5.2.122-6.fc24.i686.rpm + python-Jinja2-2.8-0.i686.rpm + python-PyYAML-3.12-0.i686.rpm + python-babel-0.9.6-5.fc18.noarch.rpm + python-backports-1.0-3.fc18.i686.rpm + python-backports-ssl_match_hostname-3.4.0.2-1.fc18.noarch.rpm + python-backports_abc-0.5-0.i686.rpm + python-certifi-2016.9.26-0.i686.rpm + python-chardet-2.0.1-5.fc18.noarch.rpm + python-crypto-1.4.1-1.noarch.rpm + python-crypto-2.6.1-1.fc18.i686.rpm + python-futures-3.1.1-1.noarch.rpm + python-jtextfsm-0.3.1-0.noarch.rpm + python-kitchen-1.1.1-2.fc18.noarch.rpm + python-markupsafe-0.18-1.fc18.i686.rpm + python-msgpack-python-0.4.8-0.i686.rpm + python-napalm-base-0.24.3-1.noarch.rpm + python-napalm-eos-0.6.0-1.noarch.rpm + python-netaddr-0.7.18-0.noarch.rpm + python-pyeapi-0.7.0-0.noarch.rpm + python-salt-2017.7.0_1414_g2fb986f-1.noarch.rpm + python-singledispatch-3.4.0.3-0.i686.rpm + python-six-1.10.0-0.i686.rpm + python-tornado-4.4.2-0.i686.rpm + python-urllib3-1.5-7.fc18.noarch.rpm + python2-zmq-15.3.0-2.fc25.i686.rpm + zeromq-4.1.4-5.fc25.i686.rpm From 37bca1b90203f8faf952ed4af17f4047f8dc27dc Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 2 Aug 2017 10:23:53 +0000 Subject: [PATCH 212/508] Add multiprocessing option for NAPALM proxy Overrides the :conf_minion:`multiprocessing` option, per proxy minion. The ``multiprocessing`` option must be turned off for SSH-based proxies. However, some NAPALM drivers (e.g. Arista, NX-OS) are not SSH-based. As multiple proxy minions may share the same configuration file, this option permits the configuration of the ``multiprocessing`` option more specifically, for some proxy minions. Additionally, this helps to default the option to False, in case the user does not configure it explicitly in the proxy config file. --- salt/proxy/napalm.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index 9ae4fd5316..3a9b6a0465 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -67,6 +67,17 @@ provider: ``napalm_base`` .. versionadded:: 2017.7.1 +multiprocessing: ``False`` + Overrides the :conf_minion:`multiprocessing` option, per proxy minion. + The ``multiprocessing`` option must be turned off for SSH-based proxies. + However, some NAPALM drivers (e.g. Arista, NX-OS) are not SSH-based. + As multiple proxy minions may share the same configuration file, + this option permits the configuration of the ``multiprocessing`` option + more specifically, for some proxy minions. + + .. versionadded:: 2017.7.1 + + .. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems .. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments From 9c4566db0c7c69631e30a436645d5bef7766988c Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 2 Aug 2017 14:51:58 +0000 Subject: [PATCH 213/508] multiprocessing option tagged for 2017.7.2 --- salt/proxy/napalm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index 3a9b6a0465..1acee33148 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -75,7 +75,7 @@ multiprocessing: ``False`` this option permits the configuration of the ``multiprocessing`` option more specifically, for some proxy minions. - .. versionadded:: 2017.7.1 + .. versionadded:: 2017.7.2 .. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems From 157fb28851dc4db152975a2efbe11c11d9aa72e3 Mon Sep 17 00:00:00 2001 From: Andy Hibbert Date: Wed, 2 Aug 2017 15:59:08 +0100 Subject: [PATCH 214/508] boto_efs_fix_tags: Fix #42688 invalid type for parameter tags Change-Id: Ibab5417074b1ac98ee025c421d04674070745edf --- salt/modules/boto_efs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/boto_efs.py b/salt/modules/boto_efs.py index 0bb20c31d2..1ef85d9233 100644 --- a/salt/modules/boto_efs.py +++ b/salt/modules/boto_efs.py @@ -158,7 +158,7 @@ def create_file_system(name, import os import base64 creation_token = base64.b64encode(os.urandom(46), ['-', '_']) - tags = {"Key": "Name", "Value": name} + tags = [{"Key": "Name", "Value": name}] client = _get_conn(key=key, keyid=keyid, profile=profile, region=region) From 287b57b5c5b7dca4365543951221454f30eb41e9 Mon Sep 17 00:00:00 2001 From: NLR Date: Wed, 2 Aug 2017 17:50:02 +0200 Subject: [PATCH 215/508] Fix RabbitMQ tags not properly set. --- salt/modules/rabbitmq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index 4cfdeb4d3d..44cf514c9f 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -472,11 +472,11 @@ def set_user_tags(name, tags, runas=None): if runas is None: runas = salt.utils.get_user() - if tags and isinstance(tags, (list, tuple)): - tags = ' '.join(tags) + if not isinstance(tags, (list, tuple)): + tags = [tags] res = __salt__['cmd.run']( - ['rabbitmqctl', 'set_user_tags', name, tags], + ['rabbitmqctl', 'set_user_tags', name] + tags, runas=runas, python_shell=False) msg = "Tag(s) set" From 78fccdc7e2480d59b4625471925f5e792a4e36a8 Mon Sep 17 00:00:00 2001 From: NLR Date: Wed, 2 Aug 2017 17:57:41 +0200 Subject: [PATCH 216/508] Cast to list in case tags is a tuple. --- salt/modules/rabbitmq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index 44cf514c9f..3ae6f71212 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -476,7 +476,7 @@ def set_user_tags(name, tags, runas=None): tags = [tags] res = __salt__['cmd.run']( - ['rabbitmqctl', 'set_user_tags', name] + tags, + ['rabbitmqctl', 'set_user_tags', name] + list(tags), runas=runas, python_shell=False) msg = "Tag(s) set" From 1a0457af51769b4e3f6132dd8bba36eec4546587 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 2 Aug 2017 10:30:29 -0600 Subject: [PATCH 217/508] allow adding extra remotes to a repository --- salt/states/git.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 274662e83b..caf75399de 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1154,13 +1154,22 @@ def latest(name, password=password, https_user=https_user, https_pass=https_pass) - comments.append( - 'Remote \'{0}\' changed from {1} to {2}'.format( - remote, - salt.utils.url.redact_http_basic_auth(fetch_url), - redacted_fetch_url + if fetch_url is None: + comments.append( + 'Remote \'{0}\' set to {1}'.format( + remote, + redacted_fetch_url + ) + ) + ret['changes']['new'] = name + ' => ' + remote + else: + comments.append( + 'Remote \'{0}\' changed from {1} to {2}'.format( + remote, + salt.utils.url.redact_http_basic_auth(fetch_url), + redacted_fetch_url + ) ) - ) if remote_rev is not None: if __opts__['test']: From 03b675a6185fe824808499695444234d5b729b2a Mon Sep 17 00:00:00 2001 From: Seth House Date: Wed, 2 Aug 2017 12:51:59 -0600 Subject: [PATCH 218/508] Add import to work around likely multiprocessing scoping bug Fixes #42649. --- salt/client/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 1073223ae1..2e4aead071 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -544,6 +544,7 @@ class LocalClient(object): {'stewart': {...}} ''' if 'expr_form' in kwargs: + import salt salt.utils.warn_until( 'Fluorine', 'The target type should be passed using the \'tgt_type\' ' From a260e913b5e4ee9fcfb7bd1420ddc53ad7064db2 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Wed, 2 Aug 2017 13:44:32 -0600 Subject: [PATCH 219/508] Do not change the arguments of the function when memoizing --- salt/utils/decorators/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 855b93a5dc..f7066b48bf 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -251,9 +251,8 @@ def memoize(func): str_args.append(str(arg)) else: str_args.append(arg) - args = str_args - args_ = ','.join(list(args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)]) + args_ = ','.join(list(str_args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)]) if args_ not in cache: cache[args_] = func(*args, **kwargs) return cache[args_] From 8604312a7b453ce056a4a0af193c25a2c341dbff Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 2 Aug 2017 17:12:02 -0600 Subject: [PATCH 220/508] Remove master conf in minion install --- pkg/windows/build_pkg.bat | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/windows/build_pkg.bat b/pkg/windows/build_pkg.bat index b6b52f3bda..0d30f047ac 100644 --- a/pkg/windows/build_pkg.bat +++ b/pkg/windows/build_pkg.bat @@ -108,9 +108,9 @@ xcopy /E /Q "%PyDir%" "%BinDir%\" @echo Copying configs to buildenv\conf... @echo ---------------------------------------------------------------------- @echo xcopy /E /Q "%SrcDir%\conf\master" "%CnfDir%\" -xcopy /Q "%SrcDir%\conf\master" "%CnfDir%\" +xcopy /Q /Y "%SrcDir%\conf\master" "%CnfDir%\" @echo xcopy /E /Q "%SrcDir%\conf\minion" "%CnfDir%\" -xcopy /Q "%SrcDir%\conf\minion" "%CnfDir%\" +xcopy /Q /Y "%SrcDir%\conf\minion" "%CnfDir%\" @echo. @echo Copying VCRedist to Prerequisites @@ -582,6 +582,10 @@ If Exist "%BinDir%\Scripts\salt-run*"^ If Exist "%BldDir%\salt-run.bat"^ del /Q "%BldDir%\salt-run.bat" 1>nul +:: Remove the master config file +if Exist "%CnfDir%\master"^ + del /Q "%CnfDir%\master" 1>nul + :: Make the Salt Minion Installer makensis.exe /DSaltVersion=%Version% /DPythonVersion=%Python% "%InsDir%\Salt-Minion-Setup.nsi" @echo. From 834d6c605e221e349ec3454419ee3cd5c9a96b7b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 2 Aug 2017 23:11:52 -0600 Subject: [PATCH 221/508] Set fact gathering style to 'old' for test_junos Without this, we stacktrace because it does not appear that setting 'gather_facts' to False prevents the library from assuming the presence of facts. I believe this to be an upstream bug with jnpr. Because they have listed this as being a deprecated option in the future this may re-break in the future. --- tests/unit/modules/test_junos.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/modules/test_junos.py b/tests/unit/modules/test_junos.py index b40793e71e..34176d9d71 100644 --- a/tests/unit/modules/test_junos.py +++ b/tests/unit/modules/test_junos.py @@ -52,6 +52,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): host='1.1.1.1', user='test', password='test123', + fact_style='old', gather_facts=False) self.dev.open() self.dev.timeout = 30 From f58256802ab1a2c1f00b4c47be3c55bb79fd6a6e Mon Sep 17 00:00:00 2001 From: Andy Hibbert Date: Thu, 3 Aug 2017 12:01:47 +0100 Subject: [PATCH 222/508] allow_no_ip_sg: Allow user to not supply ipaddress or securitygroups when running boto_efs.create_mount_target Change-Id: I558de2e80e71399a442913f33fc69b5d84490361 --- salt/modules/boto_efs.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/salt/modules/boto_efs.py b/salt/modules/boto_efs.py index 0bb20c31d2..8e20a04cec 100644 --- a/salt/modules/boto_efs.py +++ b/salt/modules/boto_efs.py @@ -223,10 +223,23 @@ def create_mount_target(filesystemid, client = _get_conn(key=key, keyid=keyid, profile=profile, region=region) - return client.create_mount_point(FileSystemId=filesystemid, - SubnetId=subnetid, - IpAddress=ipaddress, - SecurityGroups=securitygroups) + if ipaddress is None and securitygroups is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid) + + if ipaddress is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + SecurityGroups=securitygroups) + if securitygroups is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + IpAddress=ipaddress) + + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + IpAddress=ipaddress, + SecurityGroups=securitygroups) def create_tags(filesystemid, From 4bbfc751ae9e8c786229dacb3a7ebda1b303ed16 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 1 Aug 2017 15:26:17 -0600 Subject: [PATCH 223/508] render kubernetes docs --- doc/ref/modules/all/index.rst | 1 + doc/ref/modules/all/salt.modules.kubernetes.rst | 6 ++++++ doc/ref/states/all/index.rst | 1 + doc/ref/states/all/salt.states.kubernetes.rst | 6 ++++++ 4 files changed, 14 insertions(+) create mode 100644 doc/ref/modules/all/salt.modules.kubernetes.rst create mode 100644 doc/ref/states/all/salt.states.kubernetes.rst diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index e0b0198791..f325f8f375 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -195,6 +195,7 @@ execution modules keyboard keystone kmod + kubernetes launchctl layman ldap3 diff --git a/doc/ref/modules/all/salt.modules.kubernetes.rst b/doc/ref/modules/all/salt.modules.kubernetes.rst new file mode 100644 index 0000000000..a0f715d617 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.kubernetes.rst @@ -0,0 +1,6 @@ +======================= +salt.modules.kubernetes +======================= + +.. automodule:: salt.modules.kubernetes + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 03de35190e..61e38a2b52 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -135,6 +135,7 @@ state modules keyboard keystone kmod + kubernetes layman ldap libcloud_dns diff --git a/doc/ref/states/all/salt.states.kubernetes.rst b/doc/ref/states/all/salt.states.kubernetes.rst new file mode 100644 index 0000000000..a0f715d617 --- /dev/null +++ b/doc/ref/states/all/salt.states.kubernetes.rst @@ -0,0 +1,6 @@ +======================= +salt.modules.kubernetes +======================= + +.. automodule:: salt.modules.kubernetes + :members: From bca17902f5d968ed1710820efdfcabc2e422495b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 2 Aug 2017 08:55:52 -0600 Subject: [PATCH 224/508] add version added info --- doc/topics/releases/2017.7.0.rst | 2 ++ salt/modules/kubernetes.py | 1 + salt/states/kubernetes.py | 2 ++ 3 files changed, 5 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index f170176d01..a0e257b347 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -706,6 +706,7 @@ Execution modules - :mod:`salt.modules.grafana4 ` - :mod:`salt.modules.heat ` - :mod:`salt.modules.icinga2 ` +- :mod:`salt.modules.kubernetes ` - :mod:`salt.modules.logmod ` - :mod:`salt.modules.mattermost ` - :mod:`salt.modules.namecheap_dns ` @@ -784,6 +785,7 @@ States - :mod:`salt.states.icinga2 ` - :mod:`salt.states.influxdb_continuous_query ` - :mod:`salt.states.influxdb_retention_policy ` +- :mod:`salt.states.kubernetes ` - :mod:`salt.states.logadm ` - :mod:`salt.states.logrotate ` - :mod:`salt.states.msteams ` diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index b68b83057b..850534bf56 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -16,6 +16,7 @@ or `api_password` parameters when calling a function: .. code-block:: bash salt '*' kubernetes.nodes api_url=http://k8s-api-server:port api_user=myuser api_password=pass +.. versionadded: 2017.7.0 ''' # Import Python Futures diff --git a/salt/states/kubernetes.py b/salt/states/kubernetes.py index 64cd451532..bc62da1ab7 100644 --- a/salt/states/kubernetes.py +++ b/salt/states/kubernetes.py @@ -73,6 +73,8 @@ The kubernetes module is used to manage different kubernetes resources. key1: value1 key2: value2 key3: value3 + +.. versionadded: 2017.7.0 ''' from __future__ import absolute_import From 1958d1863424b95f8673d2d3eec8d54dffc37247 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:25:27 -0600 Subject: [PATCH 225/508] python2- prefix for fedora 26 packages --- salt/modules/yumpkg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 8f68747c75..4b10f34c0b 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,7 +181,10 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - vl_plugin = 'python-dnf-plugins-extras-versionlock' + if int(__grains__.get('osmajorrelease')) < 26: + vl_plugin = 'python-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python2-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From f179b97b52987d124fe522a43ff570f235bdf10b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:56:30 -0600 Subject: [PATCH 226/508] add py3 for versionlock --- salt/modules/yumpkg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 4b10f34c0b..064a6aaa45 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,7 +181,9 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if int(__grains__.get('osmajorrelease')) < 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) < 26: vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'python2-dnf-plugins-extras-versionlock' From 178cc1bd8183b8285c4b6806a4fe093451cbc89a Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 14:15:52 -0600 Subject: [PATCH 227/508] make sure names are correct --- salt/modules/yumpkg.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 064a6aaa45..1495033153 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,12 +181,16 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if six.PY3: - vl_plugin = 'python3-dnf-plugins-extras-versionlock' - elif int(__grains__.get('osmajorrelease')) < 26: - vl_plugin = 'python-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) >= 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugin-versionlock' + else: + vl_plugin = 'python2-dnf-plugin-versionlock' else: - vl_plugin = 'python2-dnf-plugins-extras-versionlock' + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From 8784899942aa6ed48ccd7c99b226df2535a985e5 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 1 Aug 2017 15:23:28 -0600 Subject: [PATCH 228/508] fix syntax --- salt/modules/yumpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 1495033153..c105bbb95b 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,7 +181,7 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - elif int(__grains__.get('osmajorrelease')) >= 26: + if int(__grains__.get('osmajorrelease')) >= 26: if six.PY3: vl_plugin = 'python3-dnf-plugin-versionlock' else: From 683561a711005ec875b9f4175a0aa0d42aa94899 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 2 Aug 2017 14:30:18 -0600 Subject: [PATCH 229/508] use subtraction instead of or If we use or, we will get minions that are in the ret list, but not connected to the master in cases where the syndic is used. --- salt/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 1073223ae1..c27dbf7328 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -738,7 +738,7 @@ class LocalClient(object): ret[mid] = (data if full_return else data.get('ret', {})) - for failed in list(set(pub_data['minions']) ^ set(ret)): + for failed in list(set(pub_data['minions']) - set(ret)): ret[failed] = False return ret finally: From 4b1df2f2235d1f2170dd00c15668f6c71f834dd7 Mon Sep 17 00:00:00 2001 From: Steven Joseph Date: Fri, 4 Aug 2017 17:26:18 +1000 Subject: [PATCH 230/508] Exclude annotated tags from checks --- salt/states/git.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index a1bf29cecc..7e0110b40c 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1308,7 +1308,7 @@ def latest(name, 'if it does not already exist).', comments ) - if set(all_local_tags) != set([ + remote_tags = set([ x.split('/')[-1] for x in __salt__['git.ls_remote']( cwd=target, remote=remote, @@ -1318,9 +1318,14 @@ def latest(name, identity=identity, saltenv=__env__, ignore_retcode=True, - ).keys() - ]): + ).keys() if '^{}' not in x + ]) + if set(all_local_tags) != remote_tags: has_remote_rev = False + ret['changes']['tags'] = { + 'old': all_local_tags, + 'new': list(remote_tags) + } if not has_remote_rev: try: From 58b997c67f5e60dfc7c1079aa5393e88588c7f2c Mon Sep 17 00:00:00 2001 From: Kristjan Koppel Date: Fri, 4 Aug 2017 15:35:41 +0300 Subject: [PATCH 231/508] Added a helper function that removes container names from container HostConfig:Links values to enable compare_container() to make the correct decision about differences in links. Fixes #42741 --- salt/modules/dockermod.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 0f87a9b17f..1127f9a6ca 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -558,6 +558,19 @@ def _prep_pull(): ''' __context__['docker._pull_status'] = [x[:12] for x in images(all=True)] +def _scrub_links(links, name): + ''' + Remove container name from HostConfig:Links values to enable comparing + container configurations correctly. + ''' + if isinstance(links, list): + ret = [] + for l in links: + ret.append(l.replace('/{0}/'.format(name), '/', 1)) + else: + ret = links + + return ret def _size_fmt(num): ''' @@ -884,6 +897,9 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict][item] val2 = result2[conf_dict].get(item) + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) if val1 != val2: ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} # Check for optionally-present items that were in the second container @@ -895,6 +911,9 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict].get(item) val2 = result2[conf_dict][item] + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) if val1 != val2: ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} return ret From f3dcfca4e0f4937b066767321bdcb30e80aa515d Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Fri, 4 Aug 2017 09:09:15 -0600 Subject: [PATCH 232/508] Fix infinite loops on failed Windows deployments --- salt/utils/cloud.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index e8a98ee9c3..25817b26d7 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -806,21 +806,21 @@ def wait_for_winexesvc(host, port, username, password, timeout=900): log.debug('winexe connected...') return True log.debug('Return code was {0}'.format(ret_code)) - time.sleep(1) except socket.error as exc: log.debug('Caught exception in wait_for_winexesvc: {0}'.format(exc)) - time.sleep(1) - if time.time() - start > timeout: - log.error('winexe connection timed out: {0}'.format(timeout)) - return False - log.debug( - 'Retrying winexe connection to host {0} on port {1} ' - '(try {2})'.format( - host, - port, - try_count - ) + + if time.time() - start > timeout: + log.error('winexe connection timed out: {0}'.format(timeout)) + return False + log.debug( + 'Retrying winexe connection to host {0} on port {1} ' + '(try {2})'.format( + host, + port, + try_count ) + ) + time.sleep(1) def wait_for_winrm(host, port, username, password, timeout=900): @@ -846,19 +846,19 @@ def wait_for_winrm(host, port, username, password, timeout=900): log.debug('WinRM session connected...') return s log.debug('Return code was {0}'.format(r.status_code)) - time.sleep(1) except WinRMTransportError as exc: log.debug('Caught exception in wait_for_winrm: {0}'.format(exc)) - if time.time() - start > timeout: - log.error('WinRM connection timed out: {0}'.format(timeout)) - return None - log.debug( - 'Retrying WinRM connection to host {0} on port {1} ' - '(try {2})'.format( - host, port, trycount - ) + + if time.time() - start > timeout: + log.error('WinRM connection timed out: {0}'.format(timeout)) + return None + log.debug( + 'Retrying WinRM connection to host {0} on port {1} ' + '(try {2})'.format( + host, port, trycount ) - time.sleep(1) + ) + time.sleep(1) def validate_windows_cred(host, From de6d3cc0cff9dedd4373e0fec5e8f634ed2f3957 Mon Sep 17 00:00:00 2001 From: garethgreenaway Date: Fri, 4 Aug 2017 08:59:33 -0700 Subject: [PATCH 233/508] Update dockermod.py --- salt/modules/dockermod.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 1127f9a6ca..dda1d6e94a 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -558,6 +558,7 @@ def _prep_pull(): ''' __context__['docker._pull_status'] = [x[:12] for x in images(all=True)] + def _scrub_links(links, name): ''' Remove container name from HostConfig:Links values to enable comparing @@ -572,6 +573,7 @@ def _scrub_links(links, name): return ret + def _size_fmt(num): ''' Format bytes as human-readable file sizes From de60b77c82bfab349c4d60a561c7326f89555ff9 Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 4 Aug 2017 12:36:06 -0600 Subject: [PATCH 234/508] Workaround Orchestrate problem that highstate outputter mutates data --- salt/client/mixins.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 634133c352..78d0bb4bb6 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -408,8 +408,6 @@ class SyncClientMixin(object): ) data['success'] = False - namespaced_event.fire_event(data, 'ret') - if self.store_job: try: salt.utils.job.store_job( @@ -427,6 +425,9 @@ class SyncClientMixin(object): log.error('Could not store job cache info. ' 'Job details for this run may be unavailable.') + # Outputters _can_ mutate data so write to the job cache first! + namespaced_event.fire_event(data, 'ret') + # if we fired an event, make sure to delete the event object. # This will ensure that we call destroy, which will do the 0MQ linger log.info('Runner completed: {0}'.format(data['jid'])) From 4a9f6ba44f1ab7a6e4bc2382b0dcfb3055ed0c53 Mon Sep 17 00:00:00 2001 From: Seth House Date: Wed, 2 Aug 2017 14:19:20 -0600 Subject: [PATCH 235/508] Add token_expire_user_override link to auth runner docstring --- salt/runners/auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/runners/auth.py b/salt/runners/auth.py index 8aec8c3389..30c359558d 100644 --- a/salt/runners/auth.py +++ b/salt/runners/auth.py @@ -20,6 +20,10 @@ def mk_token(**load): ''' Create an eauth token using provided credentials + Non-root users may specify an expiration date (if allowed via the + :conf_master:`token_expire_user_override` setting) by passing an additional + ``token_expire`` param. + CLI Example: .. code-block:: shell From 710bdf611586992829906e704995df9087d0e04d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 4 Aug 2017 10:19:27 -0500 Subject: [PATCH 236/508] docker.compare_container: treat null oom_kill_disable as False Resolves #42705. --- salt/modules/dockermod.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index dda1d6e94a..09056cb927 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -899,11 +899,15 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict][item] val2 = result2[conf_dict].get(item) - if item == 'Links': - val1 = _scrub_links(val1, first) - val2 = _scrub_links(val2, second) - if val1 != val2: - ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + if item in ('OomKillDisable',): + if bool(val1) != bool(val2): + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + else: + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) + if val1 != val2: + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} # Check for optionally-present items that were in the second container # and not the first. for item in result2[conf_dict]: @@ -913,11 +917,15 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict].get(item) val2 = result2[conf_dict][item] - if item == 'Links': - val1 = _scrub_links(val1, first) - val2 = _scrub_links(val2, second) - if val1 != val2: - ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + if item in ('OomKillDisable',): + if bool(val1) != bool(val2): + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + else: + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) + if val1 != val2: + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} return ret From 665de2d1f9e7af0b24bc635a39d98a2b2efba4b5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 7 Aug 2017 09:23:10 -0500 Subject: [PATCH 237/508] Fix domainname parameter input translation Resolves #42538 --- salt/modules/dockermod.py | 6 +++--- salt/states/docker_container.py | 16 +++------------- salt/utils/docker/translate.py | 2 +- tests/unit/utils/test_docker.py | 2 +- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 09056cb927..4d4ca55bac 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -1836,7 +1836,7 @@ def create(image, generate one for you (it will be included in the return data). skip_translate - This function translates Salt CLI input into the format which + This function translates Salt CLI or SLS input into the format which docker-py_ expects. However, in the event that Salt's translation logic fails (due to potential changes in the Docker Remote API, or to bugs in the translation code), this argument can be used to exert granular @@ -2104,9 +2104,9 @@ def create(image, - ``dns_search="[foo1.domain.tld, foo2.domain.tld]"`` domainname - Set custom DNS search domains + The domain name to use for the container - Example: ``domainname=domain.tld,domain2.tld`` + Example: ``domainname=domain.tld`` entrypoint Entrypoint for the container. Either a string (e.g. ``"mycmd --arg1 diff --git a/salt/states/docker_container.py b/salt/states/docker_container.py index 1bfc8cf489..1f5996c3d5 100644 --- a/salt/states/docker_container.py +++ b/salt/states/docker_container.py @@ -145,7 +145,7 @@ def running(name, .. _docker-container-running-skip-translate: skip_translate - This function translates Salt CLI input into the format which + This function translates Salt CLI or SLS input into the format which docker-py_ expects. However, in the event that Salt's translation logic fails (due to potential changes in the Docker Remote API, or to bugs in the translation code), this argument can be used to exert granular @@ -677,24 +677,14 @@ def running(name, - foo2.domain.tld domainname - Set custom DNS search domains. Can be expressed as a comma-separated - list or a YAML list. The below two examples are equivalent: + The domain name to use for the container .. code-block:: yaml foo: docker_container.running: - image: bar/baz:latest - - dommainname: domain.tld,domain2.tld - - .. code-block:: yaml - - foo: - docker_container.running: - - image: bar/baz:latest - - dommainname: - - domain.tld - - domain2.tld + - dommainname: domain.tld entrypoint Entrypoint for the container diff --git a/salt/utils/docker/translate.py b/salt/utils/docker/translate.py index b6d8717316..372596a759 100644 --- a/salt/utils/docker/translate.py +++ b/salt/utils/docker/translate.py @@ -387,7 +387,7 @@ def dns(val, **kwargs): def domainname(val, **kwargs): # pylint: disable=unused-argument - return _translate_stringlist(val) + return _translate_str(val) def entrypoint(val, **kwargs): # pylint: disable=unused-argument diff --git a/tests/unit/utils/test_docker.py b/tests/unit/utils/test_docker.py index 5b0587f18c..5ee6edc236 100644 --- a/tests/unit/utils/test_docker.py +++ b/tests/unit/utils/test_docker.py @@ -820,7 +820,7 @@ class TranslateInputTestCase(TestCase): expected ) - @assert_stringlist + @assert_string def test_domainname(self): ''' Should be a list of strings or converted to one From 7ef691e8da519f0e44783b7f73863048e369ecd2 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 7 Aug 2017 11:43:11 -0600 Subject: [PATCH 238/508] make sure to use the correct out_file When installing more than one package, the out_file is not correct when doing the install. --- salt/spm/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/spm/__init__.py b/salt/spm/__init__.py index b2f367e5ac..2bd2646b61 100644 --- a/salt/spm/__init__.py +++ b/salt/spm/__init__.py @@ -359,6 +359,7 @@ class SPMClient(object): # First we download everything, then we install for package in dl_list: + out_file = dl_list[package]['dest_file'] # Kick off the install self._install_indv_pkg(package, out_file) return From 560510428514c1fab10c35a70a90741480d7f817 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 7 Aug 2017 15:05:46 -0400 Subject: [PATCH 239/508] Add a cmp compatibility function utility The ``cmp`` function has been removed in Python 3. This PR adds the functionality as a salt utility, so the function can be used across all Python versions. Fixes #42697 --- salt/runners/manage.py | 3 ++- salt/utils/compat.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index ebf6471d9a..fbf8ea3f23 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -22,6 +22,7 @@ from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: dis # Import salt libs import salt.key import salt.utils +import salt.utils.compat import salt.utils.minions import salt.client import salt.client.ssh @@ -669,7 +670,7 @@ def versions(): ver_diff = -2 else: minion_version = salt.version.SaltStackVersion.parse(minions[minion]) - ver_diff = cmp(minion_version, master_version) + ver_diff = salt.utils.compat.cmp(minion_version, master_version) if ver_diff not in version_status: version_status[ver_diff] = {} diff --git a/salt/utils/compat.py b/salt/utils/compat.py index b97820db92..09c5cc2828 100644 --- a/salt/utils/compat.py +++ b/salt/utils/compat.py @@ -46,3 +46,15 @@ def deepcopy_bound(name): finally: copy._deepcopy_dispatch = pre_dispatch return ret + + +def cmp(x, y): + ''' + Compatibility helper function to replace the ``cmp`` function from Python 2. The + ``cmp`` function is no longer available in Python 3. + + cmp(x, y) -> integer + + Return negative if xy. + ''' + return (x > y) - (x < y) From d707f9486306ef65245b7eb2970c477bc014c863 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 7 Aug 2017 15:38:39 -0400 Subject: [PATCH 240/508] Update all other calls to "cmp" function Some should just use the new utility function, while others can just be compared more directly. --- salt/states/bigip.py | 4 ++-- salt/states/boto_cfn.py | 5 ++++- salt/states/cisconso.py | 8 +++++++- salt/states/ini_manage.py | 2 +- salt/states/zabbix_usergroup.py | 2 +- salt/utils/cloud.py | 3 ++- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/salt/states/bigip.py b/salt/states/bigip.py index 8a575b0d1d..44462534dc 100644 --- a/salt/states/bigip.py +++ b/salt/states/bigip.py @@ -85,7 +85,7 @@ def _check_for_changes(entity_type, ret, existing, modified): if 'generation' in existing['content'].keys(): del existing['content']['generation'] - if cmp(modified['content'], existing['content']) == 0: + if modified['content'] == existing['content']: ret['comment'] = '{entity_type} is currently enforced to the desired state. No changes made.'.format(entity_type=entity_type) else: ret['comment'] = '{entity_type} was enforced to the desired state. Note: Only parameters specified ' \ @@ -94,7 +94,7 @@ def _check_for_changes(entity_type, ret, existing, modified): ret['changes']['new'] = modified['content'] else: - if cmp(modified, existing) == 0: + if modified == existing: ret['comment'] = '{entity_type} is currently enforced to the desired state. No changes made.'.format(entity_type=entity_type) else: ret['comment'] = '{entity_type} was enforced to the desired state. Note: Only parameters specified ' \ diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index c996b9006b..97f2cf6260 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -43,6 +43,9 @@ from __future__ import absolute_import import logging import json +# Import Salt libs +import salt.utils.compat + # Import 3rd party libs try: from salt._compat import ElementTree as ET @@ -158,7 +161,7 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi template = template['GetTemplateResponse']['GetTemplateResult']['TemplateBody'].encode('ascii', 'ignore') template = json.loads(template) _template_body = json.loads(template_body) - compare = cmp(template, _template_body) + compare = salt.utils.compat.cmp(template, _template_body) if compare != 0: log.debug('Templates are not the same. Compare value is {0}'.format(compare)) # At this point we should be able to run update safely since we already validated the template diff --git a/salt/states/cisconso.py b/salt/states/cisconso.py index eb26cfaf33..7622a3102d 100644 --- a/salt/states/cisconso.py +++ b/salt/states/cisconso.py @@ -8,6 +8,12 @@ For documentation on setting up the cisconso proxy minion look in the documentat for :mod:`salt.proxy.cisconso `. ''' +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +import salt.utils.compat + def __virtual__(): return 'cisconso.set_data_value' in __salt__ @@ -53,7 +59,7 @@ def value_present(name, datastore, path, config): existing = __salt__['cisconso.get_data'](datastore, path) - if cmp(existing, config): + if salt.utils.compat.cmp(existing, config): ret['result'] = True ret['comment'] = 'Config is already set' diff --git a/salt/states/ini_manage.py b/salt/states/ini_manage.py index e22202e0f1..7ea26e05b7 100644 --- a/salt/states/ini_manage.py +++ b/salt/states/ini_manage.py @@ -206,7 +206,7 @@ def sections_present(name, sections=None, separator='='): ret['result'] = False ret['comment'] = "{0}".format(err) return ret - if cmp(dict(sections[section]), cur_section) == 0: + if dict(sections[section]) == cur_section: ret['comment'] += 'Section unchanged {0}.\n'.format(section) continue elif cur_section: diff --git a/salt/states/zabbix_usergroup.py b/salt/states/zabbix_usergroup.py index a5d68b1546..292a2aafc9 100644 --- a/salt/states/zabbix_usergroup.py +++ b/salt/states/zabbix_usergroup.py @@ -84,7 +84,7 @@ def present(name, **kwargs): for right in kwargs['rights']: for key in right: right[key] = str(right[key]) - if cmp(sorted(kwargs['rights']), sorted(usergroup['rights'])) != 0: + if sorted(kwargs['rights']) != sorted(usergroup['rights']): update_rights = True else: update_rights = True diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index cd6d59e9b5..b67f42cedb 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -58,6 +58,7 @@ import salt.config import salt.loader import salt.template import salt.utils +import salt.utils.compat import salt.utils.event from salt.utils import vt from salt.utils.nb_popen import NonBlockingPopen @@ -3041,7 +3042,7 @@ def diff_node_cache(prov_dir, node, new_data, opts): # Perform a simple diff between the old and the new data, and if it differs, # return both dicts. # TODO: Return an actual diff - diff = cmp(new_data, cache_data) + diff = salt.utils.compat.cmp(new_data, cache_data) if diff != 0: fire_event( 'event', From 998834fbacf319c2369750f02559d7d741dd4d1d Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 7 Aug 2017 14:38:10 -0400 Subject: [PATCH 241/508] Sort lists before compairing them in python 3 unit test --- tests/unit/templates/test_jinja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 6817c0c731..7a96608825 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -497,7 +497,7 @@ class TestCustomExtensions(TestCase): env = Environment(extensions=[SerializerExtension]) if six.PY3: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset).strip("'{}").split("', '") - self.assertEqual(rendered, list(unique)) + self.assertEqual(sorted(rendered), sorted(list(unique))) else: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset) self.assertEqual(rendered, u"{0}".format(unique)) From d397c90e92dfc61a36b216cb2049e28fabdf85f7 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 7 Aug 2017 14:37:10 -0600 Subject: [PATCH 242/508] only read file if it is not a string --- salt/utils/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index 87da823099..eaec8df33a 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -779,7 +779,8 @@ def _render(template, render, renderer, template_dict, opts): blacklist = opts.get('renderer_blacklist') whitelist = opts.get('renderer_whitelist') ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict) - ret = ret.read() + if salt.utils.stringio.is_readable(ret): + ret = ret.read() if str(ret).startswith('#!') and not str(ret).startswith('#!/'): ret = str(ret).split('\n', 1)[1] return ret From 90a2fb66a25fe9e39d52f63d9ba5c6df4436513d Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 7 Aug 2017 16:45:42 -0400 Subject: [PATCH 243/508] Fix typo for template_dict in http docs --- doc/topics/tutorials/http.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/topics/tutorials/http.rst b/doc/topics/tutorials/http.rst index 1eaee62071..e6b20c62a2 100644 --- a/doc/topics/tutorials/http.rst +++ b/doc/topics/tutorials/http.rst @@ -110,7 +110,7 @@ To pass through a file that contains jinja + yaml templating (the default): method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) To pass through a file that contains mako templating: @@ -123,7 +123,7 @@ To pass through a file that contains mako templating: data_file='/srv/salt/somefile.mako', data_render=True, data_renderer='mako', - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) Because this function uses Salt's own rendering system, any Salt renderer can @@ -140,7 +140,7 @@ However, this can be changed to ``master`` if necessary. method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'}, + template_dict={'key1': 'value1', 'key2': 'value2'}, opts=__opts__ ) @@ -149,7 +149,7 @@ However, this can be changed to ``master`` if necessary. method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'}, + template_dict={'key1': 'value1', 'key2': 'value2'}, node='master' ) @@ -170,11 +170,11 @@ a Python dict. header_file='/srv/salt/headers.jinja', header_render=True, header_renderer='jinja', - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) Because much of the data that would be templated between headers and data may be -the same, the ``template_data`` is the same for both. Correcting possible +the same, the ``template_dict`` is the same for both. Correcting possible variable name collisions is up to the user. Authentication From c7ea631558d570e1afdc8611bf428f3cb92c5f5c Mon Sep 17 00:00:00 2001 From: Seth House Date: Mon, 7 Aug 2017 14:11:49 -0600 Subject: [PATCH 244/508] Add more docs on the token_expire param --- salt/runners/auth.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/salt/runners/auth.py b/salt/runners/auth.py index 30c359558d..7e8e64855a 100644 --- a/salt/runners/auth.py +++ b/salt/runners/auth.py @@ -17,20 +17,28 @@ import salt.netapi def mk_token(**load): - ''' + r''' Create an eauth token using provided credentials - Non-root users may specify an expiration date (if allowed via the - :conf_master:`token_expire_user_override` setting) by passing an additional - ``token_expire`` param. + Non-root users may specify an expiration date -- if allowed via the + :conf_master:`token_expire_user_override` setting -- by passing an + additional ``token_expire`` param. This overrides the + :conf_master:`token_expire` setting of the same name in the Master config + and is how long a token should live in seconds. CLI Example: .. code-block:: shell salt-run auth.mk_token username=saltdev password=saltdev eauth=auto - salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \\ + + # Create a token valid for three years. + salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \ token_expire=94670856 + + # Calculate the number of seconds using expr. + salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \ + token_expire=$(expr \( 365 \* 24 \* 60 \* 60 \) \* 3) ''' # This will hang if the master daemon is not running. netapi = salt.netapi.NetapiClient(__opts__) From 928b523797edf69c75fecd4c95a5466917211fde Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Fri, 4 Aug 2017 17:01:02 -0600 Subject: [PATCH 245/508] Remove waits and retries from Saltify deployment The waits and retries are there to give a new VM a chance to be created and booted. Saltify works against existing VMs, so there is no need to wait for services to become available. --- salt/utils/cloud.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index 25817b26d7..069533558d 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -311,6 +311,11 @@ def bootstrap(vm_, opts): } } + if vm_.get('driver', 'none:none').split(':')[1] == 'saltify': + saltify_driver = True + else: + saltify_driver = False + key_filename = salt.config.get_cloud_config_value( 'key_filename', vm_, opts, search_global=False, default=salt.config.get_cloud_config_value( @@ -475,6 +480,9 @@ def bootstrap(vm_, opts): 'make_minion', vm_, opts, default=True ) + if saltify_driver: + deploy_kwargs['wait_for_passwd_maxtries'] = 0 # No need to wait/retry with Saltify + win_installer = salt.config.get_cloud_config_value( 'win_installer', vm_, opts ) @@ -499,6 +507,8 @@ def bootstrap(vm_, opts): deploy_kwargs['winrm_port'] = salt.config.get_cloud_config_value( 'winrm_port', vm_, opts, default=5986 ) + if saltify_driver: + deploy_kwargs['port_timeout'] = 1 # No need to wait/retry with Saltify # Store what was used to the deploy the VM event_kwargs = copy.deepcopy(deploy_kwargs) From 0acffc6df54347c3128847da67fe2854c1b040fc Mon Sep 17 00:00:00 2001 From: lomeroe Date: Thu, 3 Aug 2017 12:07:49 -0500 Subject: [PATCH 246/508] fix #42600 in develop attempt to write data to regpol file even if data_to_write is empty (i.e. no policies configured) --- salt/modules/win_lgpo.py | 143 +++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 6bf5d437d8..347ca742f9 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -3982,78 +3982,77 @@ def _write_regpol_data(data_to_write, gpt_extension_guid: admx registry extension guid for the class ''' try: - if data_to_write: - reg_pol_header = u'\u5250\u6765\x01\x00' - if not os.path.exists(policy_file_path): - ret = __salt__['file.makedirs'](policy_file_path) - with salt.utils.fopen(policy_file_path, 'wb') as pol_file: - if not data_to_write.startswith(reg_pol_header): - pol_file.write(reg_pol_header.encode('utf-16-le')) - pol_file.write(data_to_write.encode('utf-16-le')) - try: - gpt_ini_data = '' - if os.path.exists(gpt_ini_path): - with salt.utils.fopen(gpt_ini_path, 'rb') as gpt_file: - gpt_ini_data = gpt_file.read() - if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): - gpt_ini_data = '[General]\r\n' + gpt_ini_data - if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): - # ensure the line contains the ADM guid - gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] - if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), - gpt_ext_str): - gpt_ext_str = gpt_ext_str.split('=') - gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] - gpt_ext_str = '='.join(gpt_ext_str) - gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - gpt_extension, gpt_extension_guid, - gpt_ini_data[general_location.end():]) - # https://technet.microsoft.com/en-us/library/cc978247.aspx - if _regexSearchRegPolData(r'Version=', gpt_ini_data): - version_loc = re.search(r'^Version=.*\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - version_str = gpt_ini_data[version_loc.start():version_loc.end()] - version_str = version_str.split('=') - version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (version_nums[0], version_nums[1] + 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (version_nums[0] + 1, version_nums[1]) - version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[0:version_loc.start()], - 'Version', version_num, - gpt_ini_data[version_loc.end():]) - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (0, 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (1, 0) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - 'Version', - int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), - gpt_ini_data[general_location.end():]) - if gpt_ini_data: - with salt.utils.fopen(gpt_ini_path, 'wb') as gpt_file: - gpt_file.write(gpt_ini_data) - except Exception as e: - msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( - gpt_ini_path, e) - raise CommandExecutionError(msg) + reg_pol_header = u'\u5250\u6765\x01\x00' + if not os.path.exists(policy_file_path): + ret = __salt__['file.makedirs'](policy_file_path) + with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: + if not data_to_write.startswith(reg_pol_header): + pol_file.write(reg_pol_header.encode('utf-16-le')) + pol_file.write(data_to_write.encode('utf-16-le')) + try: + gpt_ini_data = '' + if os.path.exists(gpt_ini_path): + with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + gpt_ini_data = gpt_file.read() + if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): + gpt_ini_data = '[General]\r\n' + gpt_ini_data + if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): + # ensure the line contains the ADM guid + gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] + if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), + gpt_ext_str): + gpt_ext_str = gpt_ext_str.split('=') + gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] + gpt_ext_str = '='.join(gpt_ext_str) + gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + gpt_extension, gpt_extension_guid, + gpt_ini_data[general_location.end():]) + # https://technet.microsoft.com/en-us/library/cc978247.aspx + if _regexSearchRegPolData(r'Version=', gpt_ini_data): + version_loc = re.search(r'^Version=.*\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + version_str = gpt_ini_data[version_loc.start():version_loc.end()] + version_str = version_str.split('=') + version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (version_nums[0], version_nums[1] + 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (version_nums[0] + 1, version_nums[1]) + version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[0:version_loc.start()], + 'Version', version_num, + gpt_ini_data[version_loc.end():]) + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (0, 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (1, 0) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + 'Version', + int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), + gpt_ini_data[general_location.end():]) + if gpt_ini_data: + with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: + gpt_file.write(gpt_ini_data) + except Exception as e: + msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( + gpt_ini_path, e) + raise CommandExecutionError(msg) except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format(policy_file_path, e) raise CommandExecutionError(msg) From 695f8c1ae41cbcc6675930a379d277fb81edbeac Mon Sep 17 00:00:00 2001 From: lomeroe Date: Thu, 3 Aug 2017 12:07:49 -0500 Subject: [PATCH 247/508] fix #42600 in develop attempt to write data to regpol file even if data_to_write is empty (i.e. no policies configured) --- salt/modules/win_lgpo.py | 143 +++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index d941521dda..21f855e3c2 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -3989,78 +3989,77 @@ def _write_regpol_data(data_to_write, gpt_extension_guid: admx registry extension guid for the class ''' try: - if data_to_write: - reg_pol_header = u'\u5250\u6765\x01\x00' - if not os.path.exists(policy_file_path): - ret = __salt__['file.makedirs'](policy_file_path) - with open(policy_file_path, 'wb') as pol_file: - if not data_to_write.startswith(reg_pol_header): - pol_file.write(reg_pol_header.encode('utf-16-le')) - pol_file.write(data_to_write.encode('utf-16-le')) - try: - gpt_ini_data = '' - if os.path.exists(gpt_ini_path): - with open(gpt_ini_path, 'rb') as gpt_file: - gpt_ini_data = gpt_file.read() - if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): - gpt_ini_data = '[General]\r\n' + gpt_ini_data - if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): - # ensure the line contains the ADM guid - gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] - if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), - gpt_ext_str): - gpt_ext_str = gpt_ext_str.split('=') - gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] - gpt_ext_str = '='.join(gpt_ext_str) - gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - gpt_extension, gpt_extension_guid, - gpt_ini_data[general_location.end():]) - # https://technet.microsoft.com/en-us/library/cc978247.aspx - if _regexSearchRegPolData(r'Version=', gpt_ini_data): - version_loc = re.search(r'^Version=.*\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - version_str = gpt_ini_data[version_loc.start():version_loc.end()] - version_str = version_str.split('=') - version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (version_nums[0], version_nums[1] + 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (version_nums[0] + 1, version_nums[1]) - version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[0:version_loc.start()], - 'Version', version_num, - gpt_ini_data[version_loc.end():]) - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (0, 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (1, 0) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - 'Version', - int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), - gpt_ini_data[general_location.end():]) - if gpt_ini_data: - with open(gpt_ini_path, 'wb') as gpt_file: - gpt_file.write(gpt_ini_data) - except Exception as e: - msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( - gpt_ini_path, e) - raise CommandExecutionError(msg) + reg_pol_header = u'\u5250\u6765\x01\x00' + if not os.path.exists(policy_file_path): + ret = __salt__['file.makedirs'](policy_file_path) + with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: + if not data_to_write.startswith(reg_pol_header): + pol_file.write(reg_pol_header.encode('utf-16-le')) + pol_file.write(data_to_write.encode('utf-16-le')) + try: + gpt_ini_data = '' + if os.path.exists(gpt_ini_path): + with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + gpt_ini_data = gpt_file.read() + if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): + gpt_ini_data = '[General]\r\n' + gpt_ini_data + if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): + # ensure the line contains the ADM guid + gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] + if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), + gpt_ext_str): + gpt_ext_str = gpt_ext_str.split('=') + gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] + gpt_ext_str = '='.join(gpt_ext_str) + gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + gpt_extension, gpt_extension_guid, + gpt_ini_data[general_location.end():]) + # https://technet.microsoft.com/en-us/library/cc978247.aspx + if _regexSearchRegPolData(r'Version=', gpt_ini_data): + version_loc = re.search(r'^Version=.*\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + version_str = gpt_ini_data[version_loc.start():version_loc.end()] + version_str = version_str.split('=') + version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (version_nums[0], version_nums[1] + 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (version_nums[0] + 1, version_nums[1]) + version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[0:version_loc.start()], + 'Version', version_num, + gpt_ini_data[version_loc.end():]) + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (0, 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (1, 0) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + 'Version', + int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), + gpt_ini_data[general_location.end():]) + if gpt_ini_data: + with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: + gpt_file.write(gpt_ini_data) + except Exception as e: + msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( + gpt_ini_path, e) + raise CommandExecutionError(msg) except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format(policy_file_path, e) raise CommandExecutionError(msg) From 1cc86592edf48ef85bff122e303d02d26581e6e2 Mon Sep 17 00:00:00 2001 From: Sebastian Sobczynski Date: Tue, 8 Aug 2017 16:19:29 +0100 Subject: [PATCH 248/508] Update return data before calling returners --- salt/utils/schedule.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 920dce8be0..2a818628aa 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -849,6 +849,12 @@ class Schedule(object): ret['return'] = self.functions[func](*args, **kwargs) + # runners do not provide retcode + if 'retcode' in self.functions.pack['__context__']: + ret['retcode'] = self.functions.pack['__context__']['retcode'] + + ret['success'] = True + data_returner = data.get('returner', None) if data_returner or self.schedule_returner: if 'return_config' in data: @@ -865,7 +871,6 @@ class Schedule(object): for returner in OrderedDict.fromkeys(rets): ret_str = '{0}.returner'.format(returner) if ret_str in self.returners: - ret['success'] = True self.returners[ret_str](ret) else: log.info( @@ -874,11 +879,6 @@ class Schedule(object): ) ) - # runners do not provide retcode - if 'retcode' in self.functions.pack['__context__']: - ret['retcode'] = self.functions.pack['__context__']['retcode'] - - ret['success'] = True except Exception: log.exception("Unhandled exception running {0}".format(ret['fun'])) # Although catch-all exception handlers are bad, the exception here From fa5822009f00760802c44d5589b1a1dbac9f3494 Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Tue, 8 Aug 2017 11:18:20 -0500 Subject: [PATCH 249/508] Fix exception when master_type=disable The following exception occasionally occurs when `master_type=disable`: ``` File "/usr/lib/python2.7/site-packages/salt/minion.py", line 1989, in handle_event self._fire_master(data['data'], data['tag'], data['events'], data['pretag']) File "/usr/lib/python2.7/site-packages/salt/minion.py", line 1261, in _fire_master 'tok': self.tok} AttributeError: 'Minion' object has no attribute 'tok' ``` This occurs because it tries to fire a master event when the minion is not connected to the master, in this case due to an action from a beacon. Signed-off-by: Sergey Kizunov --- salt/minion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 3c5046ee93..2c7f4cb212 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1985,8 +1985,9 @@ class Minion(MinionBase): elif tag.startswith('_minion_mine'): self._mine_send(tag, data) elif tag.startswith('fire_master'): - log.debug('Forwarding master event tag={tag}'.format(tag=data['tag'])) - self._fire_master(data['data'], data['tag'], data['events'], data['pretag']) + if self.connected: + log.debug('Forwarding master event tag={tag}'.format(tag=data['tag'])) + self._fire_master(data['data'], data['tag'], data['events'], data['pretag']) elif tag.startswith(master_event(type='disconnected')) or tag.startswith(master_event(type='failback')): # if the master disconnect event is for a different master, raise an exception if tag.startswith(master_event(type='disconnected')) and data['master'] != self.opts['master']: From dbd29e4aaaef6d3ad631959f93392752dff49588 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 7 Aug 2017 14:37:10 -0600 Subject: [PATCH 250/508] only read file if it is not a string --- salt/utils/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index e34280cbfb..51d43dcfdd 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -763,7 +763,8 @@ def _render(template, render, renderer, template_dict, opts): blacklist = opts.get('renderer_blacklist') whitelist = opts.get('renderer_whitelist') ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict) - ret = ret.read() + if salt.utils.stringio.is_readable(ret): + ret = ret.read() if str(ret).startswith('#!') and not str(ret).startswith('#!/'): ret = str(ret).split('\n', 1)[1] return ret From 5a91c1f2d1311d8d16d82f6c101a8c5829b8c088 Mon Sep 17 00:00:00 2001 From: remijouannet Date: Wed, 26 Jul 2017 01:27:38 +0200 Subject: [PATCH 251/508] update consul module following this documentation https://www.consul.io/api/acl.html --- salt/modules/consul.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/modules/consul.py b/salt/modules/consul.py index 4ed96efea9..2629971a58 100644 --- a/salt/modules/consul.py +++ b/salt/modules/consul.py @@ -1972,6 +1972,8 @@ def acl_create(consul_url=None, **kwargs): Create a new ACL token. :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to create + leave it blank to let consul server generate one :param name: Meaningful indicator of the ACL's purpose. :param type: Type is either client or management. A management token is comparable to a root user and has the @@ -2002,6 +2004,9 @@ def acl_create(consul_url=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') + if 'id' in kwargs: + data['ID'] = kwargs['id'] + if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2120,7 +2125,7 @@ def acl_delete(consul_url=None, **kwargs): ret['res'] = False return ret - function = 'acl/delete/{0}'.format(kwargs['id']) + function = 'acl/destroy/{0}'.format(kwargs['id']) res = _query(consul_url=consul_url, data=data, method='PUT', From 8c8640d6b8fa39cb5c0bda1b922788d0c449af15 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 8 Aug 2017 14:35:53 -0400 Subject: [PATCH 252/508] Update doc references in glusterfs.volume_present The "created" option has been deprecated in favor of volume_present and the docs need to match. Fixes #42683 --- salt/states/glusterfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/glusterfs.py b/salt/states/glusterfs.py index 5ee7120c5a..9116243916 100644 --- a/salt/states/glusterfs.py +++ b/salt/states/glusterfs.py @@ -121,13 +121,13 @@ def volume_present(name, bricks, stripe=False, replica=False, device_vg=False, .. code-block:: yaml myvolume: - glusterfs.created: + glusterfs.volume_present: - bricks: - host1:/srv/gluster/drive1 - host2:/srv/gluster/drive2 Replicated Volume: - glusterfs.created: + glusterfs.volume_present: - name: volume2 - bricks: - host1:/srv/gluster/drive2 From 152eb88d9f2ff719a99e5d9f3a0bad94b55dae32 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 8 Aug 2017 14:42:34 -0400 Subject: [PATCH 253/508] Update modules --> states in kubernetes doc module The kubernetes state docs are not rendering/building due to a typo in the doc module. Fixes #42639 --- doc/ref/states/all/salt.states.kubernetes.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ref/states/all/salt.states.kubernetes.rst b/doc/ref/states/all/salt.states.kubernetes.rst index a0f715d617..55bd416492 100644 --- a/doc/ref/states/all/salt.states.kubernetes.rst +++ b/doc/ref/states/all/salt.states.kubernetes.rst @@ -1,6 +1,6 @@ -======================= -salt.modules.kubernetes -======================= +====================== +salt.states.kubernetes +====================== -.. automodule:: salt.modules.kubernetes +.. automodule:: salt.states.kubernetes :members: From 78d826dd147d721e5a041c8fe8f5999758b0b2b6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 13:55:35 -0500 Subject: [PATCH 254/508] Fix regression in yum/dnf version specification Resolves #42774. --- salt/states/pkg.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index b586ed5dbe..97a9d254a0 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -186,15 +186,12 @@ def _fulfills_version_spec(versions, oper, desired_version, if isinstance(versions, dict) and 'version' in versions: versions = versions['version'] for ver in versions: - if oper == '==': - if fnmatch.fnmatch(ver, desired_version): - return True - - elif salt.utils.compare_versions(ver1=ver, - oper=oper, - ver2=desired_version, - cmp_func=cmp_func, - ignore_epoch=ignore_epoch): + if (oper == '==' and fnmatch.fnmatch(ver, desired_version)) \ + or salt.utils.compare_versions(ver1=ver, + oper=oper, + ver2=desired_version, + cmp_func=cmp_func, + ignore_epoch=ignore_epoch): return True return False From c69f17dd183a9301058c5f4ac6e35387b420506b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 14:15:29 -0500 Subject: [PATCH 255/508] Add integration test for #42774 --- tests/integration/states/test_pkg.py | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/integration/states/test_pkg.py b/tests/integration/states/test_pkg.py index c2f112e026..814e3578c8 100644 --- a/tests/integration/states/test_pkg.py +++ b/tests/integration/states/test_pkg.py @@ -673,6 +673,44 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) + @requires_system_grains + def test_pkg_014_installed_missing_release(self, grains=None): # pylint: disable=unused-argument + ''' + Tests that a version number missing the release portion still resolves + as correctly installed. For example, version 2.0.2 instead of 2.0.2-1.el7 + ''' + os_family = grains.get('os_family', '') + + if os_family.lower() != 'redhat': + self.skipTest('Test only runs on RedHat OS family') + + pkg_targets = _PKG_TARGETS.get(os_family, []) + + # Make sure that we have targets that match the os_family. If this + # fails then the _PKG_TARGETS dict above needs to have an entry added, + # with two packages that are not installed before these tests are run + self.assertTrue(pkg_targets) + + target = pkg_targets[0] + version = self.run_function('pkg.version', [target]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + + ret = self.run_state( + 'pkg.installed', + name=target, + version=salt.utils.str_version_to_evr(version)[1], + refresh=False, + ) + self.assertSaltTrueReturn(ret) + + # Clean up + ret = self.run_state('pkg.removed', name=target) + self.assertSaltTrueReturn(ret) + @requires_salt_modules('pkg.group_install') @requires_system_grains def test_group_installed_handle_missing_package_group(self, grains=None): # pylint: disable=unused-argument From dc20e4651b4911be3fc6be715f629826c2e80360 Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Tue, 8 Aug 2017 14:39:34 -0600 Subject: [PATCH 256/508] Ignore error values when listing Windows SNMP community strings --- salt/modules/win_snmp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/modules/win_snmp.py b/salt/modules/win_snmp.py index 15653a7510..6fd3bd515f 100644 --- a/salt/modules/win_snmp.py +++ b/salt/modules/win_snmp.py @@ -303,6 +303,11 @@ def get_community_names(): # Windows SNMP service GUI. if isinstance(current_values, list): for current_value in current_values: + + # Ignore error values + if not isinstance(current_value, dict): + continue + permissions = str() for permission_name in _PERMISSION_TYPES: if current_value['vdata'] == _PERMISSION_TYPES[permission_name]: From 00f93142e4ed39fc0b09a0cdef4339ac227e12aa Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 9 Aug 2017 12:24:06 -0500 Subject: [PATCH 257/508] Fix misspelling of "versions" --- salt/modules/boto_iam.py | 2 +- salt/modules/status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/boto_iam.py b/salt/modules/boto_iam.py index 2bc34168b0..e955504ade 100644 --- a/salt/modules/boto_iam.py +++ b/salt/modules/boto_iam.py @@ -1892,7 +1892,7 @@ def list_policy_versions(policy_name, return ret.get('list_policy_versions_response', {}).get('list_policy_versions_result', {}).get('versions') except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to list {0} policy vesions.' + msg = 'Failed to list {0} policy versions.' log.error(msg.format(policy_name)) return [] diff --git a/salt/modules/status.py b/salt/modules/status.py index 26871084f5..90c10b68b5 100644 --- a/salt/modules/status.py +++ b/salt/modules/status.py @@ -214,7 +214,7 @@ def uptime(): raise CommandExecutionError("File {ut_path} was not found.".format(ut_path=ut_path)) seconds = int(float(salt.utils.fopen(ut_path).read().split()[0])) elif salt.utils.is_sunos(): - # note: some flavors/vesions report the host uptime inside a zone + # note: some flavors/versions report the host uptime inside a zone # https://support.oracle.com/epmos/faces/BugDisplay?id=15611584 res = __salt__['cmd.run_all']('kstat -p unix:0:system_misc:boot_time') if res['retcode'] > 0: From 81fefa6e67cf598c9ca5b6b13cbdddbb2f2245d5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 11:59:32 -0600 Subject: [PATCH 258/508] Add ability to pass version in pkgs list --- salt/modules/win_pkg.py | 122 +++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 38 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index b0d4052496..9872f9891d 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -861,56 +861,93 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): r''' Install the passed package(s) on the system using winrepo - :param name: - The name of a single package, or a comma-separated list of packages to - install. (no spaces after the commas) - :type name: str, list, or None + Args: - :param bool refresh: Boolean value representing whether or not to refresh - the winrepo db + name (str): + The name of a single package, or a comma-separated list of packages + to install. (no spaces after the commas) - :param pkgs: A list of packages to install from a software repository. - All packages listed under ``pkgs`` will be installed via a single - command. + refresh (bool): + Boolean value representing whether or not to refresh the winrepo db - :type pkgs: list or None + pkgs (list): + A list of packages to install from a software repository. All + packages listed under ``pkgs`` will be installed via a single + command. - *Keyword Arguments (kwargs)* + You can specify a version by passing the item as a dict: - :param str version: - The specific version to install. If omitted, the latest version will be - installed. If passed with multiple install, the version will apply to - all packages. Recommended for single installation only. + CLI Example: - :param str cache_file: - A single file to copy down for use with the installer. Copied to the - same location as the installer. Use this over ``cache_dir`` if there - are many files in the directory and you only need a specific file and - don't want to cache additional files that may reside in the installer - directory. Only applies to files on ``salt://`` + .. code-block:: bash - :param bool cache_dir: - True will copy the contents of the installer directory. This is useful - for installations that are not a single file. Only applies to - directories on ``salt://`` + # will install the latest version of foo and bar + salt '*' pkg.install pkgs='["foo", "bar"]' - :param str saltenv: Salt environment. Default 'base' + # will install the latest version of foo and version 1.2.3 of bar + salt '*' pkg.install pkgs='["foo", {"bar": "1.2.3"}]' - :param bool report_reboot_exit_codes: - If the installer exits with a recognized exit code indicating that - a reboot is required, the module function + Kwargs: + + version (str): + The specific version to install. If omitted, the latest version will + be installed. Recommend for use when installing a single package. + + If passed with a list of packages in the ``pkgs`` parameter, the + version will be ignored. + + CLI Example: + + .. code-block:: bash + + # Version is ignored + salt '*' pkg.install pkgs="['foo', 'bar']" version=1.2.3 + + If passed with a comma seperated list in the ``name`` parameter, the + version will apply to all packages in the list. + + CLI Example: + + .. code-block:: bash + + # Version 1.2.3 will apply to packages foo and bar + salt '*' pkg.install foo,bar version=1.2.3 + + cache_file (str): + A single file to copy down for use with the installer. Copied to the + same location as the installer. Use this over ``cache_dir`` if there + are many files in the directory and you only need a specific file + and don't want to cache additional files that may reside in the + installer directory. Only applies to files on ``salt://`` + + cache_dir (bool): + True will copy the contents of the installer directory. This is + useful for installations that are not a single file. Only applies to + directories on ``salt://`` + + extra_install_flags (str): + Additional install flags that will be appended to the + ``install_flags`` defined in the software definition file. Only + applies when single package is passed. + + saltenv (str): + Salt environment. Default 'base' + + report_reboot_exit_codes (bool): + If the installer exits with a recognized exit code indicating that + a reboot is required, the module function *win_system.set_reboot_required_witnessed* - will be called, preserving the knowledge of this event - for the remainder of the current boot session. For the time being, - 3010 is the only recognized exit code. The value of this param - defaults to True. + will be called, preserving the knowledge of this event + for the remainder of the current boot session. For the time being, + 3010 is the only recognized exit code. The value of this param + defaults to True. - .. versionadded:: 2016.11.0 + .. versionadded:: 2016.11.0 - :return: Return a dict containing the new package names and versions:: - :rtype: dict + Returns: + dict: Return a dict containing the new package names and versions: If the package is installed by ``pkg.install``: @@ -989,13 +1026,22 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # "sources" argument pkg_params = __salt__['pkg_resource.parse_targets'](name, pkgs, **kwargs)[0] + if len(pkg_params) > 1: + if kwargs.get('extra_install_flags') is not None: + log.warning('\'extra_install_flags\' argument will be ignored for ' + 'multiple package targets') + + # Windows expects an Options dictionary containing 'version' + for pkg in pkg_params: + pkg_params[pkg] = {'version': pkg_params[pkg]} + if pkg_params is None or len(pkg_params) == 0: log.error('No package definition found') return {} if not pkgs and len(pkg_params) == 1: - # Only use the 'version' param if 'name' was not specified as a - # comma-separated list + # Only use the 'version' param if a single item was passed to the 'name' + # parameter pkg_params = { name: { 'version': kwargs.get('version'), From 83b9b230cd83a044891c95df8fbc1337eb60b8f2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 12:05:42 -0600 Subject: [PATCH 259/508] Add winrepo to docs about supporting versions in pkgs --- salt/states/pkg.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 260aca1387..fe2a67a057 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -1073,8 +1073,11 @@ def installed( ``NOTE:`` For :mod:`apt `, :mod:`ebuild `, - :mod:`pacman `, :mod:`yumpkg `, - and :mod:`zypper `, version numbers can be specified + :mod:`pacman `, + :mod:`winrepo `, + :mod:`yumpkg `, and + :mod:`zypper `, + version numbers can be specified in the ``pkgs`` argument. For example: .. code-block:: yaml From 7de687aa5747a0cac28b58f057fe2e467cd15bc8 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 17:49:20 -0600 Subject: [PATCH 260/508] Document requirements for win_pki --- salt/modules/win_pki.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pki.py b/salt/modules/win_pki.py index 5b046b364a..25228cc567 100644 --- a/salt/modules/win_pki.py +++ b/salt/modules/win_pki.py @@ -1,9 +1,18 @@ # -*- coding: utf-8 -*- ''' -Microsoft certificate management via the Pki PowerShell module. +Microsoft certificate management via the PKIClient PowerShell module. +https://technet.microsoft.com/en-us/itpro/powershell/windows/pkiclient/pkiclient + +The PKI Client PowerShell module is only available on Windows 8+ and Windows +Server 2012+. +https://technet.microsoft.com/en-us/library/hh848636(v=wps.620).aspx :platform: Windows +:depends: + - PowerShell 4 + - PKIClient Module (Windows 8+ / Windows Server 2012+) + .. versionadded:: 2016.11.0 ''' # Import python libs @@ -29,11 +38,17 @@ __virtualname__ = 'win_pki' def __virtual__(): ''' - Only works on Windows systems with the PKI PowerShell module installed. + Requires Windows + Requires Windows 8+ / Windows Server 2012+ + Requires PowerShell + Requires PKIClient PowerShell module installed. ''' if not salt.utils.is_windows(): return False, 'Only available on Windows Systems' + if salt.utils.version_cmp(__grains__['osversion'], '6.2.9200') == -1: + return False, 'Only available on Windows 8+ / Windows Server 2012 +' + if not __salt__['cmd.shell_info']('powershell')['installed']: return False, 'Powershell not available' From f0a1d06b46e87fa44d8784c07348df42cc8bee00 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 17:53:22 -0600 Subject: [PATCH 261/508] Standardize PKI Client --- salt/modules/win_pki.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_pki.py b/salt/modules/win_pki.py index 25228cc567..ca17b6d626 100644 --- a/salt/modules/win_pki.py +++ b/salt/modules/win_pki.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Microsoft certificate management via the PKIClient PowerShell module. +Microsoft certificate management via the PKI Client PowerShell module. https://technet.microsoft.com/en-us/itpro/powershell/windows/pkiclient/pkiclient The PKI Client PowerShell module is only available on Windows 8+ and Windows @@ -11,7 +11,7 @@ https://technet.microsoft.com/en-us/library/hh848636(v=wps.620).aspx :depends: - PowerShell 4 - - PKIClient Module (Windows 8+ / Windows Server 2012+) + - PKI Client Module (Windows 8+ / Windows Server 2012+) .. versionadded:: 2016.11.0 ''' @@ -41,7 +41,7 @@ def __virtual__(): Requires Windows Requires Windows 8+ / Windows Server 2012+ Requires PowerShell - Requires PKIClient PowerShell module installed. + Requires PKI Client PowerShell module installed. ''' if not salt.utils.is_windows(): return False, 'Only available on Windows Systems' From 497241fbcb1226b00f021f7c9082db161e25a701 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Thu, 10 Aug 2017 10:39:43 +0300 Subject: [PATCH 262/508] Fix bug #42818 in win_iis module Exception in function "create_cert_binding". function fails with the following exception: 2017-08-09 00:23:32,096 [salt.state ][ERROR ][2948] An exception occurred in this state: Traceback (most recent call last): File "c:\salt\bin\lib\site-packages\salt\state.py", line 1837, in call **cdata['kwargs']) File "c:\salt\bin\lib\site-packages\salt\loader.py", line 1794, in wrapper return f(*args, **kwargs) File "c:\salt\var\cache\salt\minion\extmods\states\win_iisV2.py", line 326, in create_cert_binding ipaddress, port, sslflags) File "c:\salt\var\cache\salt\minion\extmods\modules\win_iisV2.py", line 861, in create_cert_binding if binding_info not in new_cert_bindings(site): TypeError: 'dict' object is not callable **This is the problematic code: new_cert_bindings = list_cert_bindings(site) if binding_info not in new_cert_bindings(site): Just need to remove (site) from second line as follows and it's fixed: new_cert_bindings = list_cert_bindings(site) if binding_info not in new_cert_bindings:** --- salt/modules/win_iis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index d0fe647b03..9309873ac4 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -856,7 +856,7 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, new_cert_bindings = list_cert_bindings(site) - if binding_info not in new_cert_bindings(site): + if binding_info not in new_cert_bindings: log.error('Binding not present: {0}'.format(binding_info)) return False From d8f7d7a7c0a4f2a95860bddfb8c048da00d16de5 Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Thu, 10 Aug 2017 16:35:45 +0200 Subject: [PATCH 263/508] API changes for Kubernetes version 2.0.0 Switching to the new Kubernetes client lib API introduced with version 2.0.0. --- salt/modules/kubernetes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 850534bf56..10e10acef1 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -2,7 +2,7 @@ ''' Module for handling kubernetes calls. -:optdepends: - kubernetes Python client +:optdepends: - kubernetes Python client >= version 2.0.0 :configuration: The k8s API settings are provided either in a pillar, in the minion's config file, or in master's config file:: @@ -762,7 +762,7 @@ def create_deployment( ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.V1beta1Deployment, + obj_class=kubernetes.client.AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1013,7 +1013,7 @@ def replace_deployment(name, ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.V1beta1Deployment, + obj_class=kubernetes.client.AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1276,9 +1276,9 @@ def __dict_to_object_meta(name, namespace, metadata): def __dict_to_deployment_spec(spec): ''' - Converts a dictionary into kubernetes V1beta1DeploymentSpec instance. + Converts a dictionary into kubernetes AppsV1beta1DeploymentSpec instance. ''' - spec_obj = kubernetes.client.V1beta1DeploymentSpec() + spec_obj = kubernetes.client.AppsV1beta1DeploymentSpec() for key, value in iteritems(spec): if hasattr(spec_obj, key): setattr(spec_obj, key, value) From ff66b7aaf09c2684f11c2f3fcb400aa001bfca72 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 10 Aug 2017 17:55:26 +0300 Subject: [PATCH 264/508] Execute fire_master asynchronously in the main minion thread. In another case it will block minion execution if master is not responding. This is actual for MultiMaster configuration because blocks minion to respond to the active master requests if another one is down. --- salt/minion.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 3c5046ee93..b1d08ab6ff 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1251,7 +1251,7 @@ class Minion(MinionBase): ret = yield channel.send(load, timeout=timeout) raise tornado.gen.Return(ret) - def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True): + def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True, timeout_handler=None): ''' Fire an event on the master, or drop message if unable to send. ''' @@ -1270,9 +1270,10 @@ class Minion(MinionBase): else: return - def timeout_handler(*_): - log.info('fire_master failed: master could not be contacted. Request timed out.') - return True + if timeout_handler is None: + def timeout_handler(*_): + log.info('fire_master failed: master could not be contacted. Request timed out.') + return True if sync: try: @@ -2205,13 +2206,15 @@ class Minion(MinionBase): if ping_interval > 0 and self.connected: def ping_master(): try: - if not self._fire_master('ping', 'minion_ping'): + def ping_timeout_handler(*_): if not self.opts.get('auth_safemode', True): log.error('** Master Ping failed. Attempting to restart minion**') delay = self.opts.get('random_reauth_delay', 5) log.info('delaying random_reauth_delay {0}s'.format(delay)) # regular sys.exit raises an exception -- which isn't sufficient in a thread os._exit(salt.defaults.exitcodes.SALT_KEEPALIVE) + + self._fire_master('ping', 'minion_ping', sync=False, timeout_handler=ping_timeout_handler) except Exception: log.warning('Attempt to ping master failed.', exc_on_loglevel=logging.DEBUG) self.periodic_callbacks['ping'] = tornado.ioloop.PeriodicCallback(ping_master, ping_interval * 1000, io_loop=self.io_loop) @@ -2226,7 +2229,7 @@ class Minion(MinionBase): except Exception: log.critical('The beacon errored: ', exc_info=True) if beacons and self.connected: - self._fire_master(events=beacons) + self._fire_master(events=beacons, sync=False) self.periodic_callbacks['beacons'] = tornado.ioloop.PeriodicCallback(handle_beacons, loop_interval * 1000, io_loop=self.io_loop) From 7f5412c19e5727bee98a16728f1a82178e2661cb Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 10 Aug 2017 18:54:46 +0300 Subject: [PATCH 265/508] Make lint happier. --- salt/minion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index b1d08ab6ff..208ea128d3 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1270,11 +1270,6 @@ class Minion(MinionBase): else: return - if timeout_handler is None: - def timeout_handler(*_): - log.info('fire_master failed: master could not be contacted. Request timed out.') - return True - if sync: try: self._send_req_sync(load, timeout) @@ -1285,6 +1280,12 @@ class Minion(MinionBase): log.info('fire_master failed: {0}'.format(traceback.format_exc())) return False else: + if timeout_handler is None: + def handle_timeout(*_): + log.info('fire_master failed: master could not be contacted. Request timed out.') + return True + timeout_handler = handle_timeout + with tornado.stack_context.ExceptionStackContext(timeout_handler): self._send_req_async(load, timeout, callback=lambda f: None) # pylint: disable=unexpected-keyword-arg return True From 21934f61bbba1243d0b4120939d948f0338deb78 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:25:27 -0600 Subject: [PATCH 266/508] python2- prefix for fedora 26 packages --- salt/modules/yumpkg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 9ccd66d105..890a78fb04 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,7 +180,10 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - vl_plugin = 'python-dnf-plugins-extras-versionlock' + if int(__grains__.get('osmajorrelease')) < 26: + vl_plugin = 'python-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python2-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From f83b553d6e84ec87676354e280d62cb4d2b90859 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:56:30 -0600 Subject: [PATCH 267/508] add py3 for versionlock --- salt/modules/yumpkg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 890a78fb04..f4b80b10c2 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,7 +180,9 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if int(__grains__.get('osmajorrelease')) < 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) < 26: vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'python2-dnf-plugins-extras-versionlock' From 6ecdbcec1d14ecdbf494359243cc8948df45cd99 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 14:15:52 -0600 Subject: [PATCH 268/508] make sure names are correct --- salt/modules/yumpkg.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index f4b80b10c2..371cea8863 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,12 +180,16 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if six.PY3: - vl_plugin = 'python3-dnf-plugins-extras-versionlock' - elif int(__grains__.get('osmajorrelease')) < 26: - vl_plugin = 'python-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) >= 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugin-versionlock' + else: + vl_plugin = 'python2-dnf-plugin-versionlock' else: - vl_plugin = 'python2-dnf-plugins-extras-versionlock' + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From a3da86eea8b4b3ef00edbbd8f5a8e1ab7cd88e28 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 1 Aug 2017 15:23:28 -0600 Subject: [PATCH 269/508] fix syntax --- salt/modules/yumpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 371cea8863..b0a183d70a 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,7 +180,7 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - elif int(__grains__.get('osmajorrelease')) >= 26: + if int(__grains__.get('osmajorrelease')) >= 26: if six.PY3: vl_plugin = 'python3-dnf-plugin-versionlock' else: From b458b89fb84313e787a8ce7578ab67115b569c81 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 10 Aug 2017 13:49:45 -0600 Subject: [PATCH 270/508] skip cache_clean test if npm version is >= 5.0.0 --- tests/integration/states/npm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/states/npm.py b/tests/integration/states/npm.py index 7035ee3047..44f43a9a07 100644 --- a/tests/integration/states/npm.py +++ b/tests/integration/states/npm.py @@ -6,6 +6,7 @@ ''' # Import Python libs from __future__ import absolute_import +from distutils.version import LooseVersion # Import Salt Testing libs from salttesting import skipIf @@ -15,6 +16,9 @@ ensure_in_syspath('../../') # Import salt libs import integration import salt.utils +import salt.modules.cmdmod as cmd + +MAX_NPM_VERSION = '5.0.0' @skipIf(salt.utils.which('npm') is None, 'npm not installed') @@ -50,6 +54,7 @@ class NpmStateTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ret = self.run_state('npm.installed', name=None, pkgs=['pm2', 'grunt']) self.assertSaltTrueReturn(ret) + @skipIf(LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION), 'Skip with npm >= 5.0.0 until #41770 is fixed') @destructiveTest def test_npm_cache_clean(self): ''' From 744bf954ff1d3cb95a9a53f0dbd9afa50da9cb01 Mon Sep 17 00:00:00 2001 From: Robert James Hernandez Date: Thu, 10 Aug 2017 13:19:08 -0700 Subject: [PATCH 271/508] Adding missing output flags to salt cli --- doc/ref/cli/_includes/output-options.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/ref/cli/_includes/output-options.rst b/doc/ref/cli/_includes/output-options.rst index 0128a1cfb0..c679269e76 100644 --- a/doc/ref/cli/_includes/output-options.rst +++ b/doc/ref/cli/_includes/output-options.rst @@ -33,6 +33,10 @@ Output Options Write the output to the specified file. +.. option:: --out-file-append, --output-file-append + + Append the output to the specified file. + .. option:: --no-color Disable all colored output @@ -46,3 +50,14 @@ Output Options ``green`` denotes success, ``red`` denotes failure, ``blue`` denotes changes and success and ``yellow`` denotes a expected future change in configuration. + +.. option:: --state-output=STATE_OUTPUT, --state_output=STATE_OUTPUT + + Override the configured state_output value for minion + output. One of 'full', 'terse', 'mixed', 'changes' or + 'filter'. Default: 'none'. + +.. option:: --state-verbose=STATE_VERBOSE, --state_verbose=STATE_VERBOSE + + Override the configured state_verbose value for minion + output. Set to True or False. Default: none. From 3055e17ed5396c38ac2372de53f03e42e58d5558 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 16:40:11 -0400 Subject: [PATCH 272/508] Replace @mock_ec2 calls with @mock_ec2_deprecated calls moto versions >= 1.0.0 have changed the way the mocked connections through boto are handled with the @mock_ec2 decorator. They use the boto3 connection method. However, since we are still using boto in many places, we need to use the new @mock_ec2_deprecated decorator instead to handle the boto connection functions for the unit tests. Versions of moto < 1.0.0 are not Python 3 compatible, so salt-jenkins should be installing newer versions of moto for those tests. Unfortunately, we cannot install an older version of moto for Python2 that use the original @mock_ec2 call and also import the @mock_ec2_deprecated function for newer versions of moto simultaneously as the @mock_ec2_deprecated function doesn't exist in older versions of moto. --- tests/unit/modules/test_boto_elb.py | 26 +-- tests/unit/modules/test_boto_secgroup.py | 38 ++-- tests/unit/modules/test_boto_vpc.py | 252 +++++++++++------------ tests/unit/states/test_boto_vpc.py | 38 ++-- 4 files changed, 177 insertions(+), 177 deletions(-) diff --git a/tests/unit/modules/test_boto_elb.py b/tests/unit/modules/test_boto_elb.py index 4582155c6f..15a995ca4a 100644 --- a/tests/unit/modules/test_boto_elb.py +++ b/tests/unit/modules/test_boto_elb.py @@ -16,17 +16,17 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2, mock_elb + from moto import mock_ec2_deprecated, mock_elb HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -34,10 +34,10 @@ except ImportError: def mock_elb(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -114,7 +114,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): # __virtual__ must be caller in order for _get_conn to be injected boto_elb.__virtual__() - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_register_instances_valid_id_result_true(self): ''' @@ -133,7 +133,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(True, register_result) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_register_instances_valid_id_string(self): ''' @@ -156,7 +156,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): log.debug(load_balancer_refreshed.instances) self.assertEqual([reservations.instances[0].id], registered_instance_ids) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_result_true(self): ''' @@ -177,7 +177,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(True, deregister_result) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_string(self): ''' @@ -203,7 +203,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): load_balancer_refreshed.instances] self.assertEqual(actual_instances, expected_instances) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_list(self): ''' diff --git a/tests/unit/modules/test_boto_secgroup.py b/tests/unit/modules/test_boto_secgroup.py index 7a1d9c0197..51ced62319 100644 --- a/tests/unit/modules/test_boto_secgroup.py +++ b/tests/unit/modules/test_boto_secgroup.py @@ -29,17 +29,17 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_secgroup unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_secgroup unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -117,7 +117,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): {'to_port': 80, 'from_port': 80, 'ip_protocol': u'tcp', 'cidr_ip': u'0.0.0.0/0'}] self.assertEqual(boto_secgroup._split_rules(rules), split_rules) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_classic(self): ''' Test of creation of an EC2-Classic security group. The test ensures @@ -137,7 +137,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_created_group[0].vpc_id] self.assertEqual(expected_create_result, secgroup_create_result) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_vpc(self): ''' test of creation of an EC2-VPC security group. The test ensures that a @@ -155,7 +155,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_create_result = [secgroup_created_group[0].name, secgroup_created_group[0].description, secgroup_created_group[0].vpc_id] self.assertEqual(expected_create_result, secgroup_create_result) - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_classic(self): ''' tests that given a name of a group in EC2-Classic that the correct @@ -177,7 +177,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): @skipIf(True, 'test skipped because moto does not yet support group' ' filters https://github.com/spulec/moto/issues/154') - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_vpc(self): ''' tests that given a name of a group in EC2-VPC that the correct @@ -197,7 +197,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(group_vpc.id, retrieved_group_id) - @mock_ec2 + @mock_ec2_deprecated def test_get_config_single_rule_group_name(self): ''' tests return of 'config' when given group name. get_config returns an OrderedDict. @@ -221,7 +221,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_get_config_result = boto_secgroup.get_config(group_id=group.id, **conn_parameters) self.assertEqual(expected_get_config_result, secgroup_get_config_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_classic(self): ''' tests 'true' existence of a group in EC2-Classic when given name @@ -234,11 +234,11 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(name=group_name, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_classic(self): pass - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_vpc(self): ''' tests 'true' existence of a group in EC2-VPC when given name and vpc_id @@ -250,7 +250,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(name=group_name, vpc_id=vpc_id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_vpc(self): ''' tests 'false' existence of a group in vpc when given name and vpc_id @@ -259,7 +259,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_name, vpc_id=vpc_id, **conn_parameters) self.assertFalse(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_group_id(self): ''' tests 'true' existence of a group when given group_id @@ -271,7 +271,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_id=group.id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_group_id(self): ''' tests 'false' existence of a group when given group_id @@ -280,7 +280,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_id=group_id, **conn_parameters) self.assertFalse(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_ec2_classic(self): ''' test deletion of a group in EC2-Classic. Test does the following: @@ -306,11 +306,11 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): actual_groups = [group.id for group in conn.get_all_security_groups()] self.assertEqual(expected_groups, actual_groups) - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_name_ec2_vpc(self): pass - @mock_ec2 + @mock_ec2_deprecated def test__get_conn_true(self): ''' tests ensures that _get_conn returns an boto.ec2.connection.EC2Connection object. diff --git a/tests/unit/modules/test_boto_vpc.py b/tests/unit/modules/test_boto_vpc.py index 2f53edf5e5..e5b337c2d1 100644 --- a/tests/unit/modules/test_boto_vpc.py +++ b/tests/unit/modules/test_boto_vpc.py @@ -40,17 +40,17 @@ except ImportError: try: import moto - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -63,7 +63,7 @@ except ImportError: # which was added in boto 2.8.0 # https://github.com/boto/boto/commit/33ac26b416fbb48a60602542b4ce15dcc7029f12 required_boto_version = '2.8.0' -required_moto_version = '0.3.7' +required_moto_version = '1.0.0' region = 'us-east-1' access_key = 'GKTADJGHEIQSXMKKRBJ08H' @@ -272,7 +272,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.modules.boto_vpc module ''' - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via id when the vpc already exists @@ -283,7 +283,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -295,7 +295,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via name when vpc exists @@ -306,7 +306,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -318,7 +318,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via tag when vpc exists @@ -329,7 +329,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -341,7 +341,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via cidr when vpc exists @@ -352,7 +352,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -364,7 +364,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_vpc_exists_but_providing_no_filters_the_vpc_exists_method_raises_a_salt_invocation_error(self): ''' @@ -375,7 +375,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): 'cidr or tags.'): boto_vpc.exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_name(self): ''' Tests getting vpc id when filtering by name @@ -386,7 +386,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_name(self): ''' Tests getting vpc id when filtering by invalid name @@ -397,7 +397,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_cidr(self): ''' Tests getting vpc id when filtering by cidr @@ -408,7 +408,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_cidr(self): ''' Tests getting vpc id when filtering by invalid cidr @@ -419,7 +419,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_tags(self): ''' Tests getting vpc id when filtering by tags @@ -430,7 +430,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_tags(self): ''' Tests getting vpc id when filtering by invalid tags @@ -441,7 +441,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_get_vpc_id_method_when_not_providing_filters_raises_a_salt_invocation_error(self): ''' @@ -450,7 +450,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegex(SaltInvocationError, 'At least one of the following must be provided: vpc_id, vpc_name, cidr or tags.'): boto_vpc.get_id(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_more_than_one_vpc_is_matched_raises_a_salt_command_execution_error(self): ''' Tests getting vpc id but providing no filters @@ -461,7 +461,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegex(CommandExecutionError, 'Found more than one VPC matching the criteria.'): boto_vpc.get_id(cidr=u'10.0.0.0/24', **conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -470,7 +470,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_a_vpc_name_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -479,7 +479,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_tags_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -488,7 +488,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_vpc_fails_the_create_vpc_method_returns_false(self): ''' @@ -499,7 +499,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_creation_result['created']) self.assertTrue('error' in vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_vpc_the_delete_vpc_method_returns_true(self): ''' Tests deleting an existing vpc @@ -510,7 +510,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_vpc_the_delete_vpc_method_returns_false(self): ''' Tests deleting a non-existent vpc @@ -519,7 +519,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(delete_vpc_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_true(self): ''' Tests describing parameters via vpc id if vpc exist @@ -546,7 +546,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_vpc, {'vpc': vpc_properties}) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_false(self): ''' Tests describing parameters via vpc id if vpc does not exist @@ -557,7 +557,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(describe_vpc['vpc']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_describing_vpc_by_id_on_connection_error_it_returns_error(self): ''' @@ -570,7 +570,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): describe_result = boto_vpc.describe(vpc_id=vpc.id, **conn_parameters) self.assertTrue('error' in describe_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_but_providing_no_vpc_id_the_describe_method_raises_a_salt_invocation_error(self): ''' Tests describing vpc without vpc id @@ -588,7 +588,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version, _get_boto_version())) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_single_subnet(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -601,7 +601,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_same_vpc(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -614,7 +614,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_different_vpc(self): ''' tests that given multiple subnet ids in different VPCs that False is @@ -628,7 +628,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(set(subnet_association['vpc_ids']), set([vpc_a.id, vpc_b.id])) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully @@ -640,7 +640,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) self.assertTrue('id' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_a_name_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a name @@ -651,7 +651,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_tags_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a tag @@ -663,7 +663,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_subnet_fails_the_create_subnet_method_returns_error(self): ''' @@ -675,7 +675,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_creation_result = boto_vpc.create_subnet(vpc.id, '10.0.0.0/24', **conn_parameters) self.assertTrue('error' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_subnet_the_delete_subnet_method_returns_true(self): ''' Tests deleting an existing subnet @@ -687,7 +687,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_deletion_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_subnet_the_delete_vpc_method_returns_false(self): ''' Tests deleting a subnet that doesn't exist @@ -695,7 +695,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): delete_subnet_result = boto_vpc.delete_subnet(subnet_id='1234', **conn_parameters) self.assertTrue('error' in delete_subnet_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' Tests checking if a subnet exists when it does exist @@ -707,7 +707,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' Tests checking if a subnet exists which doesn't exist @@ -716,7 +716,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by name @@ -728,7 +728,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by name when it doesn't exist @@ -740,7 +740,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by tag @@ -752,7 +752,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by tag when subnet doesn't exist @@ -764,7 +764,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_subnet_exists_but_providing_no_filters_the_subnet_exists_method_raises_a_salt_invocation_error(self): ''' @@ -776,7 +776,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): boto_vpc.subnet_exists(**conn_parameters) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by id. @@ -791,7 +791,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -805,7 +805,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by name. @@ -820,7 +820,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -834,7 +834,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -852,7 +852,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): set(['id', 'cidr_block', 'availability_zone', 'tags'])) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_name_for_existing_subnets_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -869,7 +869,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnets'][0].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_create_subnet_passes_availability_zone(self): ''' Tests that the availability_zone kwarg is passed on to _create_resource @@ -890,7 +890,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway successfully (with no vpc id or name) @@ -901,7 +901,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_non_existent_vpc_the_create_internet_gateway_method_returns_an_error(self): ''' Tests that creating an internet gateway for a non-existent VPC fails. @@ -913,7 +913,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): vpc_name='non-existent-vpc') self.assertTrue('error' in igw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_name_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -928,7 +928,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_id_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -951,7 +951,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway successfully (with subnet_id specified) @@ -965,7 +965,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(ngw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_non_existent_subnet_the_create_nat_gateway_method_returns_an_error(self): ''' Tests that creating an nat gateway for a non-existent subnet fails. @@ -977,7 +977,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_name='non-existent-subnet') self.assertTrue('error' in ngw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_subnet_name_specified_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway with subnet name specified. @@ -1000,7 +1000,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_customer_gateway_the_create_customer_gateway_method_returns_true(self): ''' @@ -1010,7 +1010,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_creation_result = boto_vpc.create_customer_gateway('ipsec.1', '10.1.1.1', None) self.assertTrue(gw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' @@ -1021,7 +1021,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_exists_result = boto_vpc.customer_gateway_exists(customer_gateway_id=gw_creation_result['id']) self.assertTrue(gw_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' @@ -1039,7 +1039,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version, _get_boto_version())) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_succeeds_the_create_dhcp_options_method_returns_true(self): ''' Tests creating dhcp options successfully @@ -1048,7 +1048,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_dhcp_options_and_specifying_a_name_succeeds_the_create_dhcp_options_method_returns_true( self): @@ -1060,7 +1060,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_and_specifying_tags_succeeds_the_create_dhcp_options_method_returns_true( self): ''' @@ -1071,7 +1071,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_dhcp_options_fails_the_create_dhcp_options_method_returns_error(self): ''' @@ -1082,7 +1082,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = dhcp_options_creation_result = boto_vpc.create_dhcp_options(**dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_true( self): ''' @@ -1096,7 +1096,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_association_result['associated']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_error( self): ''' @@ -1108,7 +1108,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_a_non_existent_vpc_the_associate_dhcp_options_method_returns_false( self): ''' @@ -1121,7 +1121,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_an_existing_vpc_succeeds_the_associate_new_dhcp_options_method_returns_true( self): ''' @@ -1133,7 +1133,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_creating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception( self): @@ -1147,7 +1147,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_associating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception(self): ''' @@ -1160,7 +1160,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_a_non_existent_vpc_the_dhcp_options_the_associate_new_dhcp_options_method_returns_false( self): ''' @@ -1170,7 +1170,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.create_dhcp_options(vpc_name='fake', **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_exists_the_dhcp_options_exists_method_returns_true(self): ''' Tests existence of dhcp options successfully @@ -1181,7 +1181,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_do_not_exist_the_dhcp_options_exists_method_returns_false(self): ''' Tests existence of dhcp options failure @@ -1189,7 +1189,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.dhcp_options_exists('fake', **conn_parameters) self.assertFalse(r['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_dhcp_options_exists_but_providing_no_filters_the_dhcp_options_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1206,7 +1206,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_the_create_network_acl_method_returns_true(self): ''' Tests creation of network acl with existing vpc @@ -1217,7 +1217,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_a_name_the_create_network_acl_method_returns_true( self): ''' @@ -1229,7 +1229,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_tags_the_create_network_acl_method_returns_true( self): ''' @@ -1241,7 +1241,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_a_non_existent_vpc_the_create_network_acl_method_returns_an_error(self): ''' Tests creation of network acl with a non-existent vpc @@ -1250,7 +1250,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_network_acl_fails_the_create_network_acl_method_returns_false(self): ''' @@ -1264,7 +1264,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_network_acl_the_delete_network_acl_method_returns_true(self): ''' Tests deletion of existing network acl successfully @@ -1276,7 +1276,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_network_acl_the_delete_network_acl_method_returns_an_error(self): ''' Tests deleting a non-existent network acl @@ -1285,7 +1285,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_network_acl_exists_the_network_acl_exists_method_returns_true(self): ''' Tests existence of network acl @@ -1297,7 +1297,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_network_acl_does_not_exist_the_network_acl_exists_method_returns_false(self): ''' Tests checking network acl does not exist @@ -1306,7 +1306,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_deletion_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_network_acl_exists_but_providing_no_filters_the_network_acl_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1318,7 +1318,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_successfully_the_create_network_acl_entry_method_returns_true(self): ''' @@ -1333,7 +1333,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_for_a_non_existent_network_acl_the_create_network_acl_entry_method_returns_false( self): @@ -1345,7 +1345,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_successfully_the_replace_network_acl_entry_method_returns_true( self): @@ -1362,7 +1362,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_for_a_non_existent_network_acl_the_replace_network_acl_entry_method_returns_false( self): @@ -1373,7 +1373,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_an_existing_network_acl_entry_the_delete_network_acl_entry_method_returns_true(self): ''' @@ -1388,7 +1388,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_network_acl_entry_the_delete_network_acl_entry_method_returns_false( self): @@ -1400,7 +1400,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_true( self): @@ -1416,7 +1416,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_an_error( self): ''' @@ -1430,7 +1430,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_a_non_existent_subnet_the_associate_network_acl_method_returns_false( self): @@ -1445,7 +1445,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1460,7 +1460,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_a_name_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1476,7 +1476,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_tags_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1493,7 +1493,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_non_existent_subnet_the_associate_new_network_acl_to_subnet_method_returns_false( self): @@ -1507,7 +1507,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_network_acl_to_a_non_existent_vpc_the_associate_new_network_acl_to_subnet_method_returns_an_error( self): ''' @@ -1520,7 +1520,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_succeeds_the_disassociate_network_acl_method_should_return_true(self): ''' @@ -1533,7 +1533,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_vpc_the_disassociate_network_acl_method_should_return_false( self): @@ -1547,7 +1547,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_subnet_the_disassociate_network_acl_method_should_return_false( self): @@ -1568,7 +1568,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_succeeds_the_create_route_table_method_returns_true(self): ''' @@ -1580,7 +1580,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_on_a_non_existent_vpc_the_create_route_table_method_returns_false(self): ''' @@ -1590,7 +1590,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_table_succeeds_the_delete_route_table_method_returns_true(self): ''' @@ -1603,7 +1603,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_route_table_the_delete_route_table_method_returns_false(self): ''' @@ -1613,7 +1613,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_exists_the_route_table_exists_method_returns_true(self): ''' @@ -1626,7 +1626,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_does_not_exist_the_route_table_exists_method_returns_false(self): ''' @@ -1636,7 +1636,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_route_table_exists_but_providing_no_filters_the_route_table_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1648,7 +1648,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_succeeds_the_associate_route_table_method_should_return_the_association_id( self): @@ -1663,7 +1663,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_route_table_the_associate_route_table_method_should_return_false( self): @@ -1677,7 +1677,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_subnet_the_associate_route_table_method_should_return_false( self): @@ -1691,7 +1691,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_a_route_table_succeeds_the_disassociate_route_table_method_should_return_true( self): @@ -1708,7 +1708,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_succeeds_the_create_route_method_should_return_true(self): ''' @@ -1721,7 +1721,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_with_a_non_existent_route_table_the_create_route_method_should_return_false( self): @@ -1732,7 +1732,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_succeeds_the_delete_route_method_should_return_true(self): ''' @@ -1745,7 +1745,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_with_a_non_existent_route_table_the_delete_route_method_should_return_false( self): @@ -1756,7 +1756,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_succeeds_the_replace_route_method_should_return_true(self): ''' @@ -1769,7 +1769,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_replacing_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_with_a_non_existent_route_table_the_replace_route_method_should_return_false( self): @@ -1790,7 +1790,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_request_vpc_peering_connection(self): ''' Run with 2 vpc ids and returns a message @@ -1803,7 +1803,7 @@ class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): peer_vpc_id=other_vpc.id, **conn_parameters)) - @mock_ec2 + @mock_ec2_deprecated def test_raises_error_if_both_vpc_name_and_vpc_id_are_specified(self): ''' Must specify only one diff --git a/tests/unit/states/test_boto_vpc.py b/tests/unit/states/test_boto_vpc.py index e632b1ec2e..34b74bd068 100644 --- a/tests/unit/states/test_boto_vpc.py +++ b/tests/unit/states/test_boto_vpc.py @@ -34,18 +34,18 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -131,7 +131,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.states.boto_vpc state.module ''' - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_does_not_exist(self): ''' Tests present on a VPC that does not exist. @@ -142,14 +142,14 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes']['new']['vpc']['state'], 'available') - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_exists(self): vpc = self._create_vpc(name='test') vpc_present_result = self.salt_states['boto_vpc.present']('test', cidr_block) self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): with patch('moto.ec2.models.VPCBackend.create_vpc', side_effect=BotoServerError(400, 'Mocked error')): @@ -157,7 +157,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_present_result['result']) self.assertTrue('Mocked error' in vpc_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_does_not_exist(self): ''' Tests absent on a VPC that does not exist. @@ -167,7 +167,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_exists(self): vpc = self._create_vpc(name='test') with patch.dict(salt.utils.boto.__salt__, self.funcs): @@ -175,7 +175,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes']['new']['vpc'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -195,7 +195,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): _create = getattr(self, '_create_' + self.resource_type) _create(vpc_id=vpc_id, name=name, **self.extra_kwargs) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_does_not_exist(self): ''' Tests present on a resource that does not exist. @@ -210,7 +210,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = self.funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertTrue(exists) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_exists(self): vpc = self._create_vpc(name='test') resource = self._create_resource(vpc_id=vpc.id, name='test') @@ -220,7 +220,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_present_result['result']) self.assertEqual(resource_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): vpc = self._create_vpc(name='test') @@ -231,7 +231,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertFalse(resource_present_result['result']) self.assertTrue('Mocked error' in resource_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_does_not_exist(self): ''' Tests absent on a resource that does not exist. @@ -241,7 +241,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_absent_result['result']) self.assertEqual(resource_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_exists(self): vpc = self._create_vpc(name='test') self._create_resource(vpc_id=vpc.id, name='test') @@ -253,7 +253,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = self.funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertFalse(exists) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -301,7 +301,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas backend_create = 'RouteTableBackend.create_route_table' backend_delete = 'RouteTableBackend.delete_route_table' - @mock_ec2 + @mock_ec2_deprecated def test_present_with_subnets(self): vpc = self._create_vpc(name='test') subnet1 = self._create_subnet(vpc_id=vpc.id, name='test1') @@ -326,7 +326,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas new_subnets = changes['new']['subnets_associations'] self.assertEqual(new_subnets[0]['subnet_id'], subnet2.id) - @mock_ec2 + @mock_ec2_deprecated def test_present_with_routes(self): vpc = self._create_vpc(name='test') igw = self._create_internet_gateway(name='test', vpc_id=vpc.id) From c1f673eca4b55b8c70f81596eba8c8290acf63e2 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 10 Aug 2017 14:43:33 -0600 Subject: [PATCH 273/508] use older name if _create_unverified_context is unvailable --- .../unit/utils/vmware_test/test_connection.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/unit/utils/vmware_test/test_connection.py b/tests/unit/utils/vmware_test/test_connection.py index 812cb1231c..da5e5ae63f 100644 --- a/tests/unit/utils/vmware_test/test_connection.py +++ b/tests/unit/utils/vmware_test/test_connection.py @@ -40,6 +40,11 @@ if sys.version_info[:3] > (2, 7, 8): else: SSL_VALIDATION = False +if hasattr(ssl, '_create_unverified_context'): + ssl_context = 'ssl._create_unverified_context' +else: + ssl_context = 'ssl._create_stdlib_context' + # Get Logging Started log = logging.getLogger(__name__) @@ -337,14 +342,14 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_successful_connection(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' mock_sc = MagicMock(side_effect=[exc, None]) mock_ssl = MagicMock() with patch('salt.utils.vmware.SmartConnect', mock_sc): - with patch('ssl._create_unverified_context', + with patch(ssl_context, mock_ssl): salt.utils.vmware._get_service_instance( @@ -378,7 +383,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_successful_connection(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') @@ -387,9 +392,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): mock_ssl_context = MagicMock() with patch('salt.utils.vmware.SmartConnect', mock_sc): - with patch('ssl._create_unverified_context', - mock_ssl_unverif): - + with patch(ssl_context, mock_ssl_unverif): with patch('ssl.SSLContext', mock_ssl_context): salt.utils.vmware._get_service_instance( @@ -473,7 +476,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_unsuccsessful_connection_default_error(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('Exception') @@ -498,7 +501,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_unsuccsessful_connection_vim_fault(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = vim.fault.VimFault() @@ -523,7 +526,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_unsuccessful_connection_detault_error(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') @@ -548,7 +551,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_unsuccessful_connection_vim_fault(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') From 7c1d493fdd7c1f15e44c99ac20ba8af876c56ea7 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 17:15:45 -0400 Subject: [PATCH 274/508] @mock_elb needs to be changed to @mock_elb_deprecated as well --- tests/unit/modules/test_boto_elb.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unit/modules/test_boto_elb.py b/tests/unit/modules/test_boto_elb.py index 15a995ca4a..b577f586a7 100644 --- a/tests/unit/modules/test_boto_elb.py +++ b/tests/unit/modules/test_boto_elb.py @@ -16,7 +16,7 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2_deprecated, mock_elb + from moto import mock_ec2_deprecated, mock_elb_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False @@ -25,19 +25,19 @@ except ImportError: ''' if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + Allows boto_elb unit tests to use the @mock_ec2_deprecated decorator without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass return stub_function - def mock_elb(self): + def mock_elb_deprecated(self): ''' - if the mock_ec2_deprecated function is not available due to import failure + if the mock_elb_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator - without a "NameError: name 'mock_ec2_deprecated' is not defined" error. + Allows boto_elb unit tests to use the @mock_elb_deprecated decorator + without a "NameError: name 'mock_elb_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -115,7 +115,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): boto_elb.__virtual__() @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_register_instances_valid_id_result_true(self): ''' tests that given a valid instance id and valid ELB that @@ -134,7 +134,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(True, register_result) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_register_instances_valid_id_string(self): ''' tests that given a string containing a instance id and valid ELB that @@ -157,7 +157,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual([reservations.instances[0].id], registered_instance_ids) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_result_true(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -178,7 +178,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(True, deregister_result) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_string(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -204,7 +204,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(actual_instances, expected_instances) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_list(self): ''' tests that given an valid ids in the form of a list that the boto_elb From 7f46603e9cb895318f4d609e6a4be0ae6b37f18d Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 17:31:14 -0400 Subject: [PATCH 275/508] Update account id value in boto_secgroup module unit test This value was updated in moto 1.0.0 with the following commit: https://github.com/spulec/moto/commit/5f3fbff627029c0c260546a674a3b372cacdae82 --- tests/unit/modules/test_boto_secgroup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_boto_secgroup.py b/tests/unit/modules/test_boto_secgroup.py index 51ced62319..80afac0436 100644 --- a/tests/unit/modules/test_boto_secgroup.py +++ b/tests/unit/modules/test_boto_secgroup.py @@ -213,7 +213,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): group = conn.create_security_group(name=group_name, description=group_name) group.authorize(ip_protocol=ip_protocol, from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) # setup the expected get_config result - expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'111122223333'), + expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id),('owner_id', u'123456789012'), ('description', group.description), ('tags', {}), ('rules', [{'to_port': to_port, 'from_port': from_port, 'ip_protocol': ip_protocol, 'cidr_ip': cidr_ip}]), From 35e05c95153c9212022d7eca41701fbbffa53083 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 10 Aug 2017 16:47:47 -0500 Subject: [PATCH 276/508] Add note about git CLI requirement for GitPython to GitFS tutorial --- doc/topics/tutorials/gitfs.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/topics/tutorials/gitfs.rst b/doc/topics/tutorials/gitfs.rst index 160589199f..43bf9b4071 100644 --- a/doc/topics/tutorials/gitfs.rst +++ b/doc/topics/tutorials/gitfs.rst @@ -166,13 +166,15 @@ Ubuntu 14.04 LTS and Debian Wheezy (7.x) also have a compatible version packaged # apt-get install python-git -If your master is running an older version (such as Ubuntu 12.04 LTS or Debian -Squeeze), then you will need to install GitPython using either pip_ or -easy_install (it is recommended to use pip). Version 0.3.2.RC1 is now marked as -the stable release in PyPI, so it should be a simple matter of running ``pip -install GitPython`` (or ``easy_install GitPython``) as root. +GitPython_ requires the ``git`` CLI utility to work. If installed from a system +package, then git should already be installed, but if installed via pip_ then +it may still be necessary to install git separately. For MacOS users, +GitPython_ comes bundled in with the Salt installer, but git must still be +installed for it to work properly. Git can be installed in several ways, +including by installing XCode_. -.. _`pip`: http://www.pip-installer.org/ +.. _pip: http://www.pip-installer.org/ +.. _XCode: https://developer.apple.com/xcode/ .. warning:: From 5f85a03636755ab6c7dfbb63060ff0a267e5ef1d Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Thu, 10 Aug 2017 16:35:45 -0500 Subject: [PATCH 277/508] hash_and_stat_file should return a 2-tuple Callers of `hash_and_stat_file` expect a 2-tuple and an exception will be raised if only a single value is returned. Signed-off-by: Sergey Kizunov --- salt/fileclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index a3faa10cc1..5f02723ef9 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -1257,7 +1257,7 @@ class RemoteClient(Client): if not os.path.isfile(path): msg = 'specified file {0} is not present to generate hash: {1}' log.warning(msg.format(path, err)) - return {} + return {}, None else: ret = {} hash_type = self.opts.get('hash_type', 'md5') From 0d3789f0c6624bad4b3890e55cad019fd2b39b86 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 10 Aug 2017 16:36:12 -0600 Subject: [PATCH 278/508] Fix pkg.install salt-minion using salt-call When executing `pkg.install salt-minion` using salt-call there is still a python process running maintaining locks on files that the installer is trying to overwrite. This is because the `install` function is using `task.run_wait` which waits for the task to finish before returning success. This is fine for standard programs that install using the schedular, but bad for salt. This change will use `task.run` and check that the task is actually running and then return that the task was started. This will apply only if the task was scheduled for `salt-minion` or `salt-minion-py3`. --- salt/modules/win_pkg.py | 71 ++++++++++++++++++++++++++-------------- salt/modules/win_task.py | 2 +- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 9872f9891d..f1b51af308 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1211,21 +1211,22 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): #Compute msiexec string use_msiexec, msiexec = _get_msiexec(pkginfo[version_num].get('msiexec', False)) + # Build cmd and arguments + # cmd and arguments must be seperated for use with the task scheduler + if use_msiexec: + cmd = msiexec + arguments = ['/i', cached_pkg] + if pkginfo['version_num'].get('allusers', True): + arguments.append('ALLUSERS="1"') + arguments.extend(salt.utils.shlex_split(install_flags)) + else: + cmd = cached_pkg + arguments = salt.utils.shlex_split(install_flags) + # Install the software # Check Use Scheduler Option if pkginfo[version_num].get('use_scheduler', False): - # Build Scheduled Task Parameters - if use_msiexec: - cmd = msiexec - arguments = ['/i', cached_pkg] - if pkginfo['version_num'].get('allusers', True): - arguments.append('ALLUSERS="1"') - arguments.extend(salt.utils.shlex_split(install_flags)) - else: - cmd = cached_pkg - arguments = salt.utils.shlex_split(install_flags) - # Create Scheduled Task __salt__['task.create_task'](name='update-salt-software', user_name='System', @@ -1239,21 +1240,43 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): start_time='01:00', ac_only=False, stop_if_on_batteries=False) + # Run Scheduled Task - if not __salt__['task.run_wait'](name='update-salt-software'): - log.error('Failed to install {0}'.format(pkg_name)) - log.error('Scheduled Task failed to run') - ret[pkg_name] = {'install status': 'failed'} - else: - # Build the install command - cmd = [] - if use_msiexec: - cmd.extend([msiexec, '/i', cached_pkg]) - if pkginfo[version_num].get('allusers', True): - cmd.append('ALLUSERS="1"') + # Special handling for installing salt + if pkg_name in ['salt-minion', 'salt-minion-py3']: + ret[pkg_name] = {'install status': 'task started'} + if not __salt__['task.run'](name='update-salt-software'): + log.error('Failed to install {0}'.format(pkg_name)) + log.error('Scheduled Task failed to run') + ret[pkg_name] = {'install status': 'failed'} + else: + + # Make sure the task is running, try for 5 secs + from time import time + t_end = time() + 5 + while time() < t_end: + task_running = __salt__['task.status']( + 'update-salt-software') == 'Running' + if task_running: + break + + if not task_running: + log.error( + 'Failed to install {0}'.format(pkg_name)) + log.error('Scheduled Task failed to run') + ret[pkg_name] = {'install status': 'failed'} + + # All other packages run with task scheduler else: - cmd.append(cached_pkg) - cmd.extend(salt.utils.shlex_split(install_flags)) + if not __salt__['task.run_wait'](name='update-salt-software'): + log.error('Failed to install {0}'.format(pkg_name)) + log.error('Scheduled Task failed to run') + ret[pkg_name] = {'install status': 'failed'} + else: + + # Combine cmd and arguments + cmd = [cmd].extend(arguments) + # Launch the command result = __salt__['cmd.run_all'](cmd, cache_path, diff --git a/salt/modules/win_task.py b/salt/modules/win_task.py index 22285223e4..e3179db4b4 100644 --- a/salt/modules/win_task.py +++ b/salt/modules/win_task.py @@ -1259,7 +1259,7 @@ def status(name, location='\\'): task_service = win32com.client.Dispatch("Schedule.Service") task_service.Connect() - # get the folder to delete the folder from + # get the folder where the task is defined task_folder = task_service.GetFolder(location) task = task_folder.GetTask(name) From 4b1f55da9c3219f2687c37dedc748b9b9845fbdc Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 10 Aug 2017 16:59:11 -0600 Subject: [PATCH 279/508] Make syndic_log_file respect root_dir setting --- salt/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 5417790f06..9f69900f65 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2136,7 +2136,7 @@ def syndic_config(master_config_path, 'pki_dir', 'cachedir', 'pidfile', 'sock_dir', 'extension_modules', 'autosign_file', 'autoreject_file', 'token_dir' ] - for config_key in ('log_file', 'key_logfile'): + for config_key in ('log_file', 'key_logfile', 'syndic_log_file'): # If this is not a URI and instead a local path if urlparse(opts.get(config_key, '')).scheme == '': prepend_root_dirs.append(config_key) From 28053a84a63457713cabb55436e4c39a558dcd32 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 10 Aug 2017 17:39:58 -0600 Subject: [PATCH 280/508] Change GitPython version to 2.1.1 Version 2.1.4+ requires git to be installed, usually in the form of xcode command line tools. 2.1.3 has a memory leak problem, 2.1.2 is not available. So that leaves 2.1.1. --- pkg/osx/req.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osx/req.txt b/pkg/osx/req.txt index 2224830a5a..338454f456 100644 --- a/pkg/osx/req.txt +++ b/pkg/osx/req.txt @@ -7,7 +7,7 @@ CherryPy==11.0.0 click==6.7 enum34==1.1.6 gitdb==0.6.4 -GitPython==2.1.5 +GitPython==2.1.1 idna==2.5 ipaddress==1.0.18 Jinja2==2.9.6 From 2728e5d9777dd15c6d716d3e2d13707c9ba2d2ec Mon Sep 17 00:00:00 2001 From: Steven Joseph Date: Fri, 11 Aug 2017 14:59:57 +1000 Subject: [PATCH 281/508] Only include new tags in changes --- salt/states/git.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 7e0110b40c..4808c0dce3 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1322,10 +1322,9 @@ def latest(name, ]) if set(all_local_tags) != remote_tags: has_remote_rev = False - ret['changes']['tags'] = { - 'old': all_local_tags, - 'new': list(remote_tags) - } + ret['changes']['new_tags'] = remote_tags.symmetric_difference( + all_local_tags + ) if not has_remote_rev: try: From 71995505bc3aaf71bb205c7ea08f0abf9bdf2daf Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Fri, 11 Aug 2017 08:57:03 +0200 Subject: [PATCH 282/508] Not depending on specific K8s version anymore Kubernetes imports are no longer version specific. Seems like the API change was just a rename, so now we are importing either `V1beta1Deployment` or `AppsV1beta1Deployment`. Usage stays the same. --- salt/modules/kubernetes.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 10e10acef1..6ecfdd1b13 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -2,7 +2,7 @@ ''' Module for handling kubernetes calls. -:optdepends: - kubernetes Python client >= version 2.0.0 +:optdepends: - kubernetes Python client up to version 2.0.0 :configuration: The k8s API settings are provided either in a pillar, in the minion's config file, or in master's config file:: @@ -40,6 +40,14 @@ try: except ImportError: HAS_LIBS = False +try: + # There is an API change in Kubernetes >= 2.0.0. + from kubernetes.client import V1beta1Deployment as AppsV1beta1Deployment + from kubernetes.client import V1beta1DeploymentSpec as AppsV1beta1DeploymentSpec +except ImportError: + from kubernetes.client import AppsV1beta1Deployment + from kubernetes.client import AppsV1beta1DeploymentSpec + log = logging.getLogger(__name__) @@ -762,7 +770,7 @@ def create_deployment( ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.AppsV1beta1Deployment, + obj_class=AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1013,7 +1021,7 @@ def replace_deployment(name, ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.AppsV1beta1Deployment, + obj_class=AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1278,7 +1286,7 @@ def __dict_to_deployment_spec(spec): ''' Converts a dictionary into kubernetes AppsV1beta1DeploymentSpec instance. ''' - spec_obj = kubernetes.client.AppsV1beta1DeploymentSpec() + spec_obj = AppsV1beta1DeploymentSpec() for key, value in iteritems(spec): if hasattr(spec_obj, key): setattr(spec_obj, key, value) From 81674aa88a0254d5d510e9b803ee9975bc844a2a Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Fri, 11 Aug 2017 11:20:42 +0200 Subject: [PATCH 283/508] Version info in :optdepends: not needed anymore Since this should work with 1.x.x and 2.x.x we don't have to provide a ersion info. --- salt/modules/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 6ecfdd1b13..3a54e34fd3 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -2,7 +2,7 @@ ''' Module for handling kubernetes calls. -:optdepends: - kubernetes Python client up to version 2.0.0 +:optdepends: - kubernetes Python client :configuration: The k8s API settings are provided either in a pillar, in the minion's config file, or in master's config file:: From f1de196740d00803594a51f1bfac9ec6a91e73c5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 11 Aug 2017 08:32:19 -0500 Subject: [PATCH 284/508] Add virtual func for cron state module This prevents a traceback when you attempt to use a cron state without cron being installed. --- salt/states/cron.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/states/cron.py b/salt/states/cron.py index ca8f69afb6..381bb173ab 100644 --- a/salt/states/cron.py +++ b/salt/states/cron.py @@ -150,6 +150,13 @@ from salt.modules.cron import ( ) +def __virtual__(): + if 'cron.list_tab' in __salt__: + return True + else: + return (False, 'cron module could not be loaded') + + def _check_cron(user, cmd, minute=None, From 05ecc6ac8dd43d01991c5143b64acb6231ce0b14 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 11 Aug 2017 09:59:23 -0600 Subject: [PATCH 285/508] fix vmware for python 3.4.2 in salt.utils.vmware --- salt/utils/vmware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/vmware.py b/salt/utils/vmware.py index 19292f2128..4b8f5a091a 100644 --- a/salt/utils/vmware.py +++ b/salt/utils/vmware.py @@ -238,7 +238,7 @@ def _get_service_instance(host, username, password, protocol, pwd=password, protocol=protocol, port=port, - sslContext=ssl._create_unverified_context(), + sslContext=getattr(ssl, '_create_unverified_context', getattr(ssl, '_create_stdlib_context'))(), b64token=token, mechanism=mechanism) else: From 80fd733c995f77cec90a875d78576a5b76a57e63 Mon Sep 17 00:00:00 2001 From: Giandom Date: Fri, 11 Aug 2017 18:37:52 +0200 Subject: [PATCH 286/508] Update slack.py --- salt/engines/slack.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 1709364832..024889ceb6 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -62,6 +62,7 @@ import logging import time import re import yaml +import ast try: import slackclient @@ -246,6 +247,10 @@ def start(token, tgt_type = kwargs['tgt_type'] del kwargs['tgt_type'] + # Check for pillar string representation of dict and convert it to dict + if 'pillar' in kwargs: + kwargs.update(pillar=ast.literal_eval(kwargs['pillar'])) + ret = {} if cmd in runner_functions: @@ -255,7 +260,7 @@ def start(token, # Default to trying to run as a client module. else: local = salt.client.LocalClient() - ret = local.cmd('{0}'.format(target), cmd, args, kwargs, tgt_type='{0}'.format(tgt_type)) + ret = local.cmd('{0}'.format(target), cmd, arg=args, kwarg=kwargs, tgt_type='{0}'.format(tgt_type)) if ret: return_text = json.dumps(ret, sort_keys=True, indent=1) From 43643227c62b9f5fc6b94dfef457ae4257e90d4a Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 12:57:39 -0400 Subject: [PATCH 287/508] Skip 2 failing tests in Python 3 due to upstream bugs --- tests/unit/states/test_boto_vpc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/states/test_boto_vpc.py b/tests/unit/states/test_boto_vpc.py index 34b74bd068..00336d20d5 100644 --- a/tests/unit/states/test_boto_vpc.py +++ b/tests/unit/states/test_boto_vpc.py @@ -14,6 +14,7 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch import salt.config import salt.loader import salt.utils.boto +from salt.ext import six from salt.utils.versions import LooseVersion import salt.states.boto_vpc as boto_vpc @@ -291,6 +292,9 @@ class BotoVpcInternetGatewayTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTe @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(six.PY3, 'Disabled for Python 3 due to upstream bugs: ' + 'https://github.com/spulec/moto/issues/548 and ' + 'https://github.com/gabrielfalcao/HTTPretty/issues/325') @skipIf(HAS_BOTO is False, 'The boto module must be installed.') @skipIf(HAS_MOTO is False, 'The moto module must be installed.') @skipIf(_has_required_boto() is False, 'The boto module must be greater than' From da3402a53dbb77e1d1ed55bfe8c0ec890ed5330d Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 11 Aug 2017 10:58:00 -0600 Subject: [PATCH 288/508] make sure cmd is not run when npm isn't installed apparently the skipIf on the functions still get run, even if the function is going to be skipped based on a skipIf on the class. --- tests/integration/states/npm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/states/npm.py b/tests/integration/states/npm.py index 44f43a9a07..140eaa92ca 100644 --- a/tests/integration/states/npm.py +++ b/tests/integration/states/npm.py @@ -54,7 +54,8 @@ class NpmStateTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ret = self.run_state('npm.installed', name=None, pkgs=['pm2', 'grunt']) self.assertSaltTrueReturn(ret) - @skipIf(LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION), 'Skip with npm >= 5.0.0 until #41770 is fixed') + @skipIf(salt.utils.which('npm') and LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION), + 'Skip with npm >= 5.0.0 until #41770 is fixed') @destructiveTest def test_npm_cache_clean(self): ''' From 462d65308236e0fd81d5dc938d432031dc904966 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 11 Aug 2017 12:20:50 -0500 Subject: [PATCH 289/508] Move weird tearDown test to an actual tearDown Also catch KeyError when user doesn't exist --- tests/integration/shell/test_auth.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/integration/shell/test_auth.py b/tests/integration/shell/test_auth.py index 3c4c7000a2..94793af51d 100644 --- a/tests/integration/shell/test_auth.py +++ b/tests/integration/shell/test_auth.py @@ -54,7 +54,6 @@ class AuthTest(ShellCase): group = 'saltops' def setUp(self): - # This is a little wasteful but shouldn't be a problem for user in (self.userA, self.userB): try: pwd.getpwnam(user) @@ -68,6 +67,21 @@ class AuthTest(ShellCase): self.run_call('group.add {0}'.format(self.group)) self.run_call('user.chgroups {0} {1} True'.format(self.userB, self.group)) + def tearDown(self): + for user in (self.userA, self.userB): + try: + pwd.getpwnam(user) + except KeyError: + pass + else: + self.run_call('user.delete {0}'.format(user)) + try: + grp.getgrnam(self.group) + except KeyError: + pass + else: + self.run_call('group.delete {0}'.format(self.group)) + def test_pam_auth_valid_user(self): ''' test that pam auth mechanism works with a valid user @@ -121,10 +135,3 @@ class AuthTest(ShellCase): self.assertTrue( 'minion:' in resp ) - - def test_zzzz_tearDown(self): - for user in (self.userA, self.userB): - if pwd.getpwnam(user): - self.run_call('user.delete {0}'.format(user)) - if grp.getgrnam(self.group): - self.run_call('group.delete {0}'.format(self.group)) From 6a7bf998484afd437a71f48346e4915d8488b92d Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 14:19:15 -0400 Subject: [PATCH 290/508] Lint fix: add missing space --- tests/unit/modules/test_boto_secgroup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_boto_secgroup.py b/tests/unit/modules/test_boto_secgroup.py index 80afac0436..ef90958dec 100644 --- a/tests/unit/modules/test_boto_secgroup.py +++ b/tests/unit/modules/test_boto_secgroup.py @@ -213,7 +213,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): group = conn.create_security_group(name=group_name, description=group_name) group.authorize(ip_protocol=ip_protocol, from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) # setup the expected get_config result - expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id),('owner_id', u'123456789012'), + expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'123456789012'), ('description', group.description), ('tags', {}), ('rules', [{'to_port': to_port, 'from_port': from_port, 'ip_protocol': ip_protocol, 'cidr_ip': cidr_ip}]), From 885bee2a7d9326071416d1abc4248e526253f026 Mon Sep 17 00:00:00 2001 From: Carson Anderson Date: Fri, 11 Aug 2017 10:39:11 -0600 Subject: [PATCH 291/508] Stub out required functions for redis cache Without these functions the masters are not able to use the redis cache completely Signed-off-by: Carson Anderson --- salt/cache/redis_cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/cache/redis_cache.py b/salt/cache/redis_cache.py index 6678787efd..434e5f4d8c 100644 --- a/salt/cache/redis_cache.py +++ b/salt/cache/redis_cache.py @@ -115,7 +115,7 @@ from salt.exceptions import SaltCacheError __virtualname__ = 'redis' __func_alias__ = { - 'list_': 'list' + 'ls': 'list' } log = logging.getLogger(__file__) @@ -145,6 +145,9 @@ def __virtual__(): # helper functions -- will not be exported # ----------------------------------------------------------------------------- +def init_kwargs(kwargs): + return {} + def _get_redis_cache_opts(): ''' @@ -415,7 +418,7 @@ def flush(bank, key=None): return True -def list_(bank): +def ls(bank): ''' Lists entries stored in the specified bank. ''' From 71e7581a2d8d2b14dfaf32018053a0b9d065f202 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 14:31:17 -0400 Subject: [PATCH 292/508] Remove extraneous "deprecated" notation This deprecated notation was supposed to be on the "useradd_all" function, but somehow was moved to the "useradd" function. Fixes #42870 --- salt/modules/htpasswd.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/modules/htpasswd.py b/salt/modules/htpasswd.py index 02f48457c2..cbb09e99ea 100644 --- a/salt/modules/htpasswd.py +++ b/salt/modules/htpasswd.py @@ -35,8 +35,6 @@ def useradd(pwfile, user, password, opts='', runas=None): Add a user to htpasswd file using the htpasswd command. If the htpasswd file does not exist, it will be created. - .. deprecated:: 2016.3.0 - pwfile Path to htpasswd file From 999388680ca67d9d2aafa6c0fbc3acc5d8389208 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Fri, 11 Aug 2017 22:44:42 +0300 Subject: [PATCH 293/508] Make chunked mode in salt-cp optional (disabled by default). This reverts breaking of backward compatibility related to the cp.recv interface change. --- doc/ref/cli/salt-cp.rst | 15 ++++++++- salt/cli/cp.py | 71 +++++++++++++++++++++++++++++++++++++---- salt/modules/cp.py | 31 +++++++++++++++++- salt/utils/parsers.py | 11 +++++-- 4 files changed, 117 insertions(+), 11 deletions(-) diff --git a/doc/ref/cli/salt-cp.rst b/doc/ref/cli/salt-cp.rst index 0540c154b8..19afe9e396 100644 --- a/doc/ref/cli/salt-cp.rst +++ b/doc/ref/cli/salt-cp.rst @@ -39,6 +39,12 @@ specified target expression. desitination will be assumed to be a directory. Finally, recursion is now supported, allowing for entire directories to be copied. +.. versionchanged:: 2016.11.7,Nitrogen + Returned tha old mode when salt-cp copies all files in one command back. + The new chunked mode is available with the new ``-C``, ``--cunked`` argument. + The compression and directories copying and large files support is available + in the chunked mode only. + Options ======= @@ -56,9 +62,16 @@ Options .. include:: _includes/target-selection.rst +.. option:: -C, --chunked + + Use new chunked mode to copy files. This mode supports large files, recursive + directories copying and compression. + + .. versionadded:: 2016.11.7,Nitrogen + .. option:: -n, --no-compression - Disable gzip compression. + Disable gzip compression in chunked mode. .. versionadded:: 2016.3.7,2016.11.6,Nitrogen diff --git a/salt/cli/cp.py b/salt/cli/cp.py index 62f2596ec4..e51bdc7167 100644 --- a/salt/cli/cp.py +++ b/salt/cli/cp.py @@ -19,8 +19,9 @@ import sys # Import salt libs import salt.client import salt.utils.gzip_util +import salt.utils.itertools import salt.utils.minions -from salt.utils import parsers, to_bytes +from salt.utils import parsers, to_bytes, print_cli from salt.utils.verify import verify_log import salt.output @@ -100,10 +101,69 @@ class SaltCP(object): empty_dirs.update(empty_dirs_) return files, sorted(empty_dirs) + def _file_dict(self, fn_): + ''' + Take a path and return the contents of the file as a string + ''' + if not os.path.isfile(fn_): + err = 'The referenced file, {0} is not available.'.format(fn_) + sys.stderr.write(err + '\n') + sys.exit(42) + with salt.utils.fopen(fn_, 'r') as fp_: + data = fp_.read() + return {fn_: data} + + def _load_files(self): + ''' + Parse the files indicated in opts['src'] and load them into a python + object for transport + ''' + files = {} + for fn_ in self.opts['src']: + if os.path.isfile(fn_): + files.update(self._file_dict(fn_)) + elif os.path.isdir(fn_): + print_cli(fn_ + ' is a directory, only files are supported in non-chunked mode. ' + 'Use "--chunked" command line argument.') + sys.exit(1) + return files + def run(self): ''' Make the salt client call ''' + if self.opts['chunked']: + ret = self.run_chunked() + else: + ret = self.run_oldstyle() + + salt.output.display_output( + ret, + self.opts.get('output', 'nested'), + self.opts) + + def run_oldstyle(self): + ''' + Make the salt client call in old-style all-in-one call method + ''' + arg = [self._load_files(), self.opts['dest']] + local = salt.client.get_local_client(self.opts['conf_file']) + args = [self.opts['tgt'], + 'cp.recv', + arg, + self.opts['timeout'], + ] + + selected_target_option = self.opts.get('selected_target_option', None) + if selected_target_option is not None: + args.append(selected_target_option) + + return local.cmd(*args) + + def run_chunked(self): + ''' + Make the salt client call in the new fasion chunked multi-call way + ''' files, empty_dirs = self._list_files() dest = self.opts['dest'] gzip = self.opts['gzip'] @@ -165,7 +225,7 @@ class SaltCP(object): ) args = [ tgt, - 'cp.recv', + 'cp.recv_chunked', [remote_path, chunk, append, gzip, mode], timeout, ] @@ -211,14 +271,11 @@ class SaltCP(object): else '', tgt, ) - args = [tgt, 'cp.recv', [remote_path, None], timeout] + args = [tgt, 'cp.recv_chunked', [remote_path, None], timeout] if selected_target_option is not None: args.append(selected_target_option) for minion_id, minion_ret in six.iteritems(local.cmd(*args)): ret.setdefault(minion_id, {})[remote_path] = minion_ret - salt.output.display_output( - ret, - self.opts.get('output', 'nested'), - self.opts) + return ret diff --git a/salt/modules/cp.py b/salt/modules/cp.py index aac09a4047..3f3fe1a989 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -57,7 +57,36 @@ def _gather_pillar(pillarenv, pillar_override): return ret -def recv(dest, chunk, append=False, compressed=True, mode=None): +def recv(files, dest): + ''' + Used with salt-cp, pass the files dict, and the destination. + + This function receives small fast copy files from the master via salt-cp. + It does not work via the CLI. + ''' + ret = {} + for path, data in six.iteritems(files): + if os.path.basename(path) == os.path.basename(dest) \ + and not os.path.isdir(dest): + final = dest + elif os.path.isdir(dest): + final = os.path.join(dest, os.path.basename(path)) + elif os.path.isdir(os.path.dirname(dest)): + final = dest + else: + return 'Destination unavailable' + + try: + with salt.utils.fopen(final, 'w+') as fp_: + fp_.write(data) + ret[final] = True + except IOError: + ret[final] = False + + return ret + + +def recv_chunked(dest, chunk, append=False, compressed=True, mode=None): ''' This function receives files copied to the minion using ``salt-cp`` and is not intended to be used directly on the CLI. diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 34565da0d6..b82b8a6d27 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -2110,10 +2110,18 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta, def _mixin_setup(self): file_opts_group = optparse.OptionGroup(self, 'File Options') + file_opts_group.add_option( + '-C', '--chunked', + default=False, + dest='chunked', + action='store_true', + help='Use chunked files transfer. Supports big files, recursive ' + 'lookup and directories creation.' + ) file_opts_group.add_option( '-n', '--no-compression', default=True, - dest='compression', + dest='gzip', action='store_false', help='Disable gzip compression.' ) @@ -2134,7 +2142,6 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta, self.config['tgt'] = self.args[0] self.config['src'] = [os.path.realpath(x) for x in self.args[1:-1]] self.config['dest'] = self.args[-1] - self.config['gzip'] = True def setup_config(self): return config.master_config(self.get_config_file_path()) From fb7117f2acdb54560048d319235b5f3d35ae989b Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 16:31:24 -0400 Subject: [PATCH 294/508] Use salt.utils.versions.LooseVersion instead of distutils --- tests/integration/states/test_npm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/states/test_npm.py b/tests/integration/states/test_npm.py index e5f298930e..42bc94e4b0 100644 --- a/tests/integration/states/test_npm.py +++ b/tests/integration/states/test_npm.py @@ -6,7 +6,6 @@ ''' # Import Python libs from __future__ import absolute_import -from distutils.version import LooseVersion # Import Salt Testing libs from tests.support.case import ModuleCase @@ -17,6 +16,7 @@ from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs import salt.utils import salt.modules.cmdmod as cmd +from salt.utils.versions import LooseVersion MAX_NPM_VERSION = '5.0.0' From 86ce7004a2cb6b4bba4739a1a11c3b3868aceb1e Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 9 Aug 2017 22:35:35 +0000 Subject: [PATCH 295/508] Backport salt.utils.versions from develop to 2016.11 This makes it easier to backport boto* and other modules from develop that are using the salt.utils.versions module. Additionally, because the `blacklisted-module` Salt pylint module is not available in 2016.11, remove references to it. --- salt/utils/versions.py | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 salt/utils/versions.py diff --git a/salt/utils/versions.py b/salt/utils/versions.py new file mode 100644 index 0000000000..f1533a84ee --- /dev/null +++ b/salt/utils/versions.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +''' + :copyright: Copyright 2017 by the SaltStack Team, see AUTHORS for more details. + :license: Apache 2.0, see LICENSE for more details. + + + salt.utils.versions + ~~~~~~~~~~~~~~~~~~~ + + Version parsing based on distutils.version which works under python 3 + because on python 3 you can no longer compare strings against integers. +''' + +# Import python libs +from __future__ import absolute_import +from distutils.version import StrictVersion as _StrictVersion +from distutils.version import LooseVersion as _LooseVersion + +# Import 3rd-party libs +from salt.ext import six + + +class StrictVersion(_StrictVersion): + def parse(self, vstring): + _StrictVersion.parse(self, vstring) + + def _cmp(self, other): + if isinstance(other, six.string_types): + other = StrictVersion(other) + return _StrictVersion._cmp(self, other) + + +class LooseVersion(_LooseVersion): + + def parse(self, vstring): + _LooseVersion.parse(self, vstring) + + if six.PY3: + # Convert every part of the version to string in order to be able to compare + self._str_version = [ + str(vp).zfill(8) if isinstance(vp, int) else vp for vp in self.version] + + if six.PY3: + def _cmp(self, other): + if isinstance(other, six.string_types): + other = LooseVersion(other) + + string_in_version = False + for part in self.version + other.version: + if not isinstance(part, int): + string_in_version = True + break + + if string_in_version is False: + return _LooseVersion._cmp(self, other) + + # If we reached this far, it means at least a part of the version contains a string + # In python 3, strings and integers are not comparable + if self._str_version == other._str_version: + return 0 + if self._str_version < other._str_version: + return -1 + if self._str_version > other._str_version: + return 1 From f903e7bc394242703d990cd3a7450838d9c0629d Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sat, 12 Aug 2017 23:11:25 +0100 Subject: [PATCH 296/508] Minor eos doc correction --- doc/topics/installation/eos.rst | 61 ++++++++++++++++----------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/doc/topics/installation/eos.rst b/doc/topics/installation/eos.rst index f7537c334b..c8c7787be7 100644 --- a/doc/topics/installation/eos.rst +++ b/doc/topics/installation/eos.rst @@ -1,4 +1,3 @@ - ========================================= Arista EOS Salt minion installation guide ========================================= @@ -122,34 +121,34 @@ Additional Information This SWIX extension contains the following RPM packages: -.. code-block:: +.. code-block:: text - libsodium-1.0.11-1.fc25.i686.rpm - libstdc++-6.2.1-2.fc25.i686.rpm - openpgm-5.2.122-6.fc24.i686.rpm - python-Jinja2-2.8-0.i686.rpm - python-PyYAML-3.12-0.i686.rpm - python-babel-0.9.6-5.fc18.noarch.rpm - python-backports-1.0-3.fc18.i686.rpm - python-backports-ssl_match_hostname-3.4.0.2-1.fc18.noarch.rpm - python-backports_abc-0.5-0.i686.rpm - python-certifi-2016.9.26-0.i686.rpm - python-chardet-2.0.1-5.fc18.noarch.rpm - python-crypto-1.4.1-1.noarch.rpm - python-crypto-2.6.1-1.fc18.i686.rpm - python-futures-3.1.1-1.noarch.rpm - python-jtextfsm-0.3.1-0.noarch.rpm - python-kitchen-1.1.1-2.fc18.noarch.rpm - python-markupsafe-0.18-1.fc18.i686.rpm - python-msgpack-python-0.4.8-0.i686.rpm - python-napalm-base-0.24.3-1.noarch.rpm - python-napalm-eos-0.6.0-1.noarch.rpm - python-netaddr-0.7.18-0.noarch.rpm - python-pyeapi-0.7.0-0.noarch.rpm - python-salt-2017.7.0_1414_g2fb986f-1.noarch.rpm - python-singledispatch-3.4.0.3-0.i686.rpm - python-six-1.10.0-0.i686.rpm - python-tornado-4.4.2-0.i686.rpm - python-urllib3-1.5-7.fc18.noarch.rpm - python2-zmq-15.3.0-2.fc25.i686.rpm - zeromq-4.1.4-5.fc25.i686.rpm + libsodium-1.0.11-1.fc25.i686.rpm + libstdc++-6.2.1-2.fc25.i686.rpm + openpgm-5.2.122-6.fc24.i686.rpm + python-Jinja2-2.8-0.i686.rpm + python-PyYAML-3.12-0.i686.rpm + python-babel-0.9.6-5.fc18.noarch.rpm + python-backports-1.0-3.fc18.i686.rpm + python-backports-ssl_match_hostname-3.4.0.2-1.fc18.noarch.rpm + python-backports_abc-0.5-0.i686.rpm + python-certifi-2016.9.26-0.i686.rpm + python-chardet-2.0.1-5.fc18.noarch.rpm + python-crypto-1.4.1-1.noarch.rpm + python-crypto-2.6.1-1.fc18.i686.rpm + python-futures-3.1.1-1.noarch.rpm + python-jtextfsm-0.3.1-0.noarch.rpm + python-kitchen-1.1.1-2.fc18.noarch.rpm + python-markupsafe-0.18-1.fc18.i686.rpm + python-msgpack-python-0.4.8-0.i686.rpm + python-napalm-base-0.24.3-1.noarch.rpm + python-napalm-eos-0.6.0-1.noarch.rpm + python-netaddr-0.7.18-0.noarch.rpm + python-pyeapi-0.7.0-0.noarch.rpm + python-salt-2017.7.0_1414_g2fb986f-1.noarch.rpm + python-singledispatch-3.4.0.3-0.i686.rpm + python-six-1.10.0-0.i686.rpm + python-tornado-4.4.2-0.i686.rpm + python-urllib3-1.5-7.fc18.noarch.rpm + python2-zmq-15.3.0-2.fc25.i686.rpm + zeromq-4.1.4-5.fc25.i686.rpm From 93be79a1352ea048c3987d3631d560c904d60b45 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sat, 12 Aug 2017 23:15:11 +0100 Subject: [PATCH 297/508] Index eos under the installation instructions list --- doc/topics/installation/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index 0ee4b38d7a..2f96830c4e 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -46,6 +46,7 @@ These guides go into detail how to install Salt on a given platform. arch debian + eos fedora freebsd gentoo From 9fedf6012ef9a2aeebf196a47518761364a7cbed Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 14 Aug 2017 10:54:16 +0300 Subject: [PATCH 298/508] Fixed 'test_valid_docs' test. --- tests/integration/files/file/base/_modules/runtests_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/files/file/base/_modules/runtests_helpers.py b/tests/integration/files/file/base/_modules/runtests_helpers.py index c725e02814..d95526250d 100644 --- a/tests/integration/files/file/base/_modules/runtests_helpers.py +++ b/tests/integration/files/file/base/_modules/runtests_helpers.py @@ -50,6 +50,7 @@ def get_invalid_docs(): allow_failure = ( 'cmd.win_runas', 'cp.recv', + 'cp.recv_chunked', 'glance.warn_until', 'ipset.long_range', 'libcloud_dns.get_driver', From afedd3b654978e9a15145a8b0f19b9570e4ef6b7 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 14 Aug 2017 18:08:40 +0300 Subject: [PATCH 299/508] Typos and version fixes in the doc. --- doc/ref/cli/salt-cp.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ref/cli/salt-cp.rst b/doc/ref/cli/salt-cp.rst index 19afe9e396..b3a9a1a6ae 100644 --- a/doc/ref/cli/salt-cp.rst +++ b/doc/ref/cli/salt-cp.rst @@ -39,9 +39,9 @@ specified target expression. desitination will be assumed to be a directory. Finally, recursion is now supported, allowing for entire directories to be copied. -.. versionchanged:: 2016.11.7,Nitrogen - Returned tha old mode when salt-cp copies all files in one command back. - The new chunked mode is available with the new ``-C``, ``--cunked`` argument. +.. versionchanged:: 2016.11.7,2017.7.2 + Returned the old mode when salt-cp copies all files in one command back. + The new chunked mode is available with the new ``-C``, ``--chunked`` argument. The compression and directories copying and large files support is available in the chunked mode only. @@ -67,7 +67,7 @@ Options Use new chunked mode to copy files. This mode supports large files, recursive directories copying and compression. - .. versionadded:: 2016.11.7,Nitrogen + .. versionadded:: 2016.11.7,2017.7.2 .. option:: -n, --no-compression From 99046b441f890bf36f0cff72c4a591da8df7553e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 14 Aug 2017 10:26:21 -0600 Subject: [PATCH 300/508] cloud driver isn't a provider This will not have a ':' in it at all. --- salt/utils/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index f4ee061958..4677fde631 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -312,7 +312,7 @@ def bootstrap(vm_, opts): } } - if vm_.get('driver', 'none:none').split(':')[1] == 'saltify': + if vm_.get('driver') == 'saltify': saltify_driver = True else: saltify_driver = False From a3becf8342c46173b8522190c0afb3082b3004dd Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 14 Aug 2017 11:19:09 -0600 Subject: [PATCH 301/508] Change service shutdown timeouts Windows default for stopping a service is 30 seconds. This changes the default timeouts for the salt-minion service to more closely resemble how it is handled in Windows. This gives Python a chance to cleanly exit before being forcibly closed. --- pkg/windows/installer/Salt-Minion-Setup.nsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index b3566583e3..dbb795e89b 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -383,6 +383,8 @@ Section -Post nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion Start SERVICE_AUTO_START" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" + nsExec::Exec "nssm.exe set salt-minion AppStopMethodConsole 24000" + nsExec::Exec "nssm.exe set salt-minion AppStopMethodWindow 2000" RMDir /R "$INSTDIR\var\cache\salt" ; removing cache from old version From ffb23fbe47f5b16bd6b61c589f1e08bda1f3de49 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 14 Aug 2017 12:26:27 -0600 Subject: [PATCH 302/508] Remove the line that wipes out the cache --- pkg/windows/installer/Salt-Minion-Setup.nsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index dbb795e89b..46fb821fb8 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -386,8 +386,6 @@ Section -Post nsExec::Exec "nssm.exe set salt-minion AppStopMethodConsole 24000" nsExec::Exec "nssm.exe set salt-minion AppStopMethodWindow 2000" - RMDir /R "$INSTDIR\var\cache\salt" ; removing cache from old version - Call updateMinionConfig Push "C:\salt" From 62eca9b00b1f916f9e8debd7af47f8a1a23a3750 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 10 Aug 2017 17:55:26 +0300 Subject: [PATCH 303/508] Execute fire_master asynchronously in the main minion thread. In another case it will block minion execution if master is not responding. This is actual for MultiMaster configuration because blocks minion to respond to the active master requests if another one is down. --- salt/minion.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index ab3371cf45..bc34a54c70 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1263,7 +1263,7 @@ class Minion(MinionBase): ret = yield channel.send(load, timeout=timeout) raise tornado.gen.Return(ret) - def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True): + def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True, timeout_handler=None): ''' Fire an event on the master, or drop message if unable to send. ''' @@ -1282,9 +1282,10 @@ class Minion(MinionBase): else: return - def timeout_handler(*_): - log.info('fire_master failed: master could not be contacted. Request timed out.') - return True + if timeout_handler is None: + def timeout_handler(*_): + log.info('fire_master failed: master could not be contacted. Request timed out.') + return True if sync: try: @@ -2216,13 +2217,15 @@ class Minion(MinionBase): if ping_interval > 0 and self.connected: def ping_master(): try: - if not self._fire_master('ping', 'minion_ping'): + def ping_timeout_handler(*_): if not self.opts.get('auth_safemode', True): log.error('** Master Ping failed. Attempting to restart minion**') delay = self.opts.get('random_reauth_delay', 5) log.info('delaying random_reauth_delay {0}s'.format(delay)) # regular sys.exit raises an exception -- which isn't sufficient in a thread os._exit(salt.defaults.exitcodes.SALT_KEEPALIVE) + + self._fire_master('ping', 'minion_ping', sync=False, timeout_handler=ping_timeout_handler) except Exception: log.warning('Attempt to ping master failed.', exc_on_loglevel=logging.DEBUG) self.periodic_callbacks['ping'] = tornado.ioloop.PeriodicCallback(ping_master, ping_interval * 1000, io_loop=self.io_loop) @@ -2237,7 +2240,7 @@ class Minion(MinionBase): except Exception: log.critical('The beacon errored: ', exc_info=True) if beacons and self.connected: - self._fire_master(events=beacons) + self._fire_master(events=beacons, sync=False) self.periodic_callbacks['beacons'] = tornado.ioloop.PeriodicCallback(handle_beacons, loop_interval * 1000, io_loop=self.io_loop) From cdb48126f73b5d512819c58f124745ee0f5c2c85 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 10 Aug 2017 18:54:46 +0300 Subject: [PATCH 304/508] Make lint happier. --- salt/minion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index bc34a54c70..afd763716b 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1282,11 +1282,6 @@ class Minion(MinionBase): else: return - if timeout_handler is None: - def timeout_handler(*_): - log.info('fire_master failed: master could not be contacted. Request timed out.') - return True - if sync: try: self._send_req_sync(load, timeout) @@ -1297,6 +1292,12 @@ class Minion(MinionBase): log.info('fire_master failed: {0}'.format(traceback.format_exc())) return False else: + if timeout_handler is None: + def handle_timeout(*_): + log.info('fire_master failed: master could not be contacted. Request timed out.') + return True + timeout_handler = handle_timeout + with tornado.stack_context.ExceptionStackContext(timeout_handler): self._send_req_async(load, timeout, callback=lambda f: None) # pylint: disable=unexpected-keyword-arg return True From 4e46c968e6f6f792eb0dcf1f0924005e0c003ffd Mon Sep 17 00:00:00 2001 From: amalleo25 Date: Thu, 10 Aug 2017 23:08:38 -0400 Subject: [PATCH 305/508] Update joyent.rst Missing colon in example cloud.profiles.d config file. --- doc/topics/cloud/joyent.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/cloud/joyent.rst b/doc/topics/cloud/joyent.rst index 8d4ca2c100..a56d533840 100644 --- a/doc/topics/cloud/joyent.rst +++ b/doc/topics/cloud/joyent.rst @@ -49,7 +49,7 @@ Set up an initial profile at ``/etc/salt/cloud.profiles`` or in the .. code-block:: yaml - joyent_512 + joyent_512: provider: my-joyent-config size: g4-highcpu-512M image: ubuntu-16.04 From 5e930b8cbda8fddec48770b4d4c60050e5a7eecb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 14 Aug 2017 12:23:27 -0700 Subject: [PATCH 306/508] If we catch the pid file in a transistory state, return None --- salt/utils/process.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/utils/process.py b/salt/utils/process.py index aefe8731ee..fc47877d92 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -130,8 +130,10 @@ def get_pidfile(pidfile): ''' with salt.utils.fopen(pidfile) as pdf: pid = pdf.read() - - return int(pid) + if pid: + return int(pid) + else: + return def clean_proc(proc, wait_for_kill=10): From 8165f46165d2aeeebbeedc0cb89de816601ddc48 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 14 Aug 2017 16:25:13 -0500 Subject: [PATCH 307/508] Add debug logging to troubleshoot test failures This adds logging to troubleshoot https://github.com/saltstack/salt-jenkins/issues/477 --- tests/integration/shell/test_auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/shell/test_auth.py b/tests/integration/shell/test_auth.py index 94793af51d..fc80c7ac05 100644 --- a/tests/integration/shell/test_auth.py +++ b/tests/integration/shell/test_auth.py @@ -6,6 +6,7 @@ # Import python libs from __future__ import absolute_import +import logging import pwd import grp import random @@ -21,6 +22,8 @@ from salt.utils.pycrypto import gen_hash # Import 3rd-party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin +log = logging.getLogger(__name__) + def gen_password(): ''' @@ -99,6 +102,7 @@ class AuthTest(ShellCase): cmd = ('-a pam "*" test.ping ' '--username {0} --password {1}'.format(self.userA, password)) resp = self.run_salt(cmd) + log.debug('resp = %s', resp) self.assertTrue( 'minion:' in resp ) From 5ac41f496dbd6ad39ce83dd9eb92056668c6e1bf Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 14 Aug 2017 20:47:43 -0700 Subject: [PATCH 308/508] When running osquery commands through cmd.run we should pass python_shell=True to ensure everything is formatted right. #42873 --- salt/modules/osquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/osquery.py b/salt/modules/osquery.py index 137125f070..99fd89c330 100644 --- a/salt/modules/osquery.py +++ b/salt/modules/osquery.py @@ -56,7 +56,7 @@ def _osquery(sql, format='json'): } cmd = 'osqueryi --json "{0}"'.format(sql) - res = __salt__['cmd.run_all'](cmd) + res = __salt__['cmd.run_all'](cmd, python_shell=True) if res['retcode'] == 0: ret['data'] = json.loads(res['stdout']) else: From bd63074e7a2982faf8ecd7bbbf0d83689d408a57 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 15 Aug 2017 08:21:25 -0600 Subject: [PATCH 309/508] create new ip address before checking list of allocated ips This will help prevent a race condition of all cloud servers being built in parallel trying to build with the same floating ip address. --- salt/cloud/clouds/nova.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py index 6e7797c121..bf1c917d7d 100644 --- a/salt/cloud/clouds/nova.py +++ b/salt/cloud/clouds/nova.py @@ -726,12 +726,18 @@ def request_instance(vm_=None, call=None): else: pool = floating_ip_conf.get('pool', 'public') - for fl_ip, opts in six.iteritems(conn.floating_ip_list()): - if opts['fixed_ip'] is None and opts['pool'] == pool: - floating_ip = fl_ip - break - if floating_ip is None: + try: floating_ip = conn.floating_ip_create(pool)['ip'] + except Exception: + log.info('A new ip address was unable to be allocated. ' + 'An ip address will be pulled from the already allocated list, ' + 'This will cause a race condition when building in parallel.') + for fl_ip, opts in six.iteritems(conn.floating_ip_list()): + if opts['fixed_ip'] is None and opts['pool'] == pool: + floating_ip = fl_ip + break + if floating_ip is None: + log.error('No ip addresses available to allocate for this server: {0}'.format(vm_['name'])) def __query_node_data(vm_): try: From a0118bcecec895b8247245b44611bc003d71ea43 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 16:33:42 -0500 Subject: [PATCH 310/508] Remove bytestrings and use textwrap.dedent for readability PyYAML works with and without bytestrings on PY3, and Python 3 can read all of the test data as regular strings, so bytestrings are unnecessary here. This also adds use of textwrap.dedent to make the YAML easier to read, so it can be indented away from the far left side of the line. --- tests/unit/utils/test_yamlloader.py | 69 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/tests/unit/utils/test_yamlloader.py b/tests/unit/utils/test_yamlloader.py index 69a2724f5d..2f90f29ff9 100644 --- a/tests/unit/utils/test_yamlloader.py +++ b/tests/unit/utils/test_yamlloader.py @@ -5,6 +5,7 @@ # Import python libs from __future__ import absolute_import +import textwrap # Import Salt Libs from yaml.constructor import ConstructorError @@ -36,12 +37,11 @@ class YamlLoaderTestCase(TestCase): ''' Test parsing an ordinary path ''' - self.assertEqual( - self._render_yaml(b''' -p1: - - alpha - - beta'''), + self._render_yaml(textwrap.dedent('''\ + p1: + - alpha + - beta''')), {'p1': ['alpha', 'beta']} ) @@ -49,38 +49,37 @@ p1: ''' Test YAML anchors ''' - # Simple merge test self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v2: beta'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v2: beta''')), {'p1': {'v1': 'alpha'}, 'p2': {'v1': 'alpha', 'v2': 'beta'}} ) # Test that keys/nodes are overwritten self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v1: new_alpha'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v1: new_alpha''')), {'p1': {'v1': 'alpha'}, 'p2': {'v1': 'new_alpha'}} ) # Test merging of lists self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: &v1 - - t1 - - t2 -p2: - v2: *v1'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: &v1 + - t1 + - t2 + p2: + v2: *v1''')), {"p2": {"v2": ["t1", "t2"]}, "p1": {"v1": ["t1", "t2"]}} ) @@ -89,15 +88,15 @@ p2: Test that duplicates still throw an error ''' with self.assertRaises(ConstructorError): - self._render_yaml(b''' -p1: alpha -p1: beta''') + self._render_yaml(textwrap.dedent('''\ + p1: alpha + p1: beta''')) with self.assertRaises(ConstructorError): - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v2: beta - v2: betabeta''') + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v2: beta + v2: betabeta''')) From 9d8486a89484b772d728441d734b3f4be6db525d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 16:48:43 -0500 Subject: [PATCH 311/508] Add test for custom YAML loader with unicode literal strings --- tests/unit/utils/test_yamlloader.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/utils/test_yamlloader.py b/tests/unit/utils/test_yamlloader.py index 2f90f29ff9..11228ff7e0 100644 --- a/tests/unit/utils/test_yamlloader.py +++ b/tests/unit/utils/test_yamlloader.py @@ -100,3 +100,15 @@ class YamlLoaderTestCase(TestCase): <<: *p1 v2: beta v2: betabeta''')) + + def test_yaml_with_unicode_literals(self): + ''' + Test proper loading of unicode literals + ''' + self.assertEqual( + self._render_yaml(textwrap.dedent('''\ + foo: + a: Д + b: {'a': u'\\u0414'}''')), + {'foo': {'a': u'\u0414', 'b': {'a': u'\u0414'}}} + ) From 7a4cddcd95df473d5664fdbdb9b6e4611d743968 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 31 Jul 2017 11:43:49 -0400 Subject: [PATCH 312/508] Add clean_id function to salt.utils.verify.py --- salt/utils/verify.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 6c3e606f72..9cc31201b8 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -485,12 +485,21 @@ def clean_path(root, path, subdir=False): return '' +def clean_id(id_): + ''' + Returns if the passed id is clean. + ''' + if re.search(r'\.\.{sep}'.format(sep=os.sep), id_): + return False + return True + + def valid_id(opts, id_): ''' Returns if the passed id is valid ''' try: - return bool(clean_path(opts['pki_dir'], id_)) + return bool(clean_path(opts['pki_dir'], id_)) and clean_id(id_) except (AttributeError, KeyError) as e: return False From 73f9135340843325e7ed3eb3dd0bdd18935cfbbc Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Tue, 15 Aug 2017 16:08:58 +0000 Subject: [PATCH 313/508] extension_modules should default to /proxy/extmods Additionally, append it to the append_minionid_config_dirs list, so each proxy caches its extension modules separately. --- salt/config/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 4feaff3293..0f06f9ccca 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1633,7 +1633,8 @@ DEFAULT_PROXY_MINION_OPTS = { 'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'proxy'), 'add_proxymodule_to_opts': False, 'proxy_merge_grains_in_module': True, - 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include'], + 'extension_modules': os.path.join(salt.syspaths.CACHE_DIR, 'proxy', 'extmods'), + 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include', 'extension_modules'], 'default_include': 'proxy.d/*.conf', # By default, proxies will preserve the connection. From 149633fdca8001f3246206a99010623a1d1dc45e Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 31 Jul 2017 11:46:43 -0400 Subject: [PATCH 314/508] Add release notes for 2016.3.7 release --- doc/topics/releases/2016.3.7.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/topics/releases/2016.3.7.rst b/doc/topics/releases/2016.3.7.rst index 3c47df7eda..4efb507c46 100644 --- a/doc/topics/releases/2016.3.7.rst +++ b/doc/topics/releases/2016.3.7.rst @@ -4,6 +4,9 @@ Salt 2016.3.7 Release Notes Version 2016.3.7 is a bugfix release for :ref:`2016.3.0 `. +Changes for v2016.3.6..v2016.3.7 +-------------------------------- + New master configuration option `allow_minion_key_revoke`, defaults to True. This option controls whether a minion can request that the master revoke its key. When True, a minion can request a key revocation and the master will comply. If it is False, the key will not @@ -24,3 +27,10 @@ New minion configuration option `minion_sign_messages` Causes the minion to cryptographically sign the payload of messages it places on the event bus for the master. The payloads are signed with the minion's private key so the master can verify the signature with its public key. + +Security Fix +============ + +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master + +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com From 8a07d952129af4f253c799c329c0b490a78ce416 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 11:53:03 -0400 Subject: [PATCH 315/508] update release notes with cve number --- doc/topics/releases/2016.3.7.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/topics/releases/2016.3.7.rst b/doc/topics/releases/2016.3.7.rst index 4efb507c46..f48f567786 100644 --- a/doc/topics/releases/2016.3.7.rst +++ b/doc/topics/releases/2016.3.7.rst @@ -7,6 +7,7 @@ Version 2016.3.7 is a bugfix release for :ref:`2016.3.0 `. Changes for v2016.3.6..v2016.3.7 -------------------------------- +<<<<<<< HEAD New master configuration option `allow_minion_key_revoke`, defaults to True. This option controls whether a minion can request that the master revoke its key. When True, a minion can request a key revocation and the master will comply. If it is False, the key will not From 168604ba6be105c89435df554d249dcc085fcf2c Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:12:40 -0400 Subject: [PATCH 316/508] remove merge conflict --- doc/topics/releases/2016.3.7.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/topics/releases/2016.3.7.rst b/doc/topics/releases/2016.3.7.rst index f48f567786..4efb507c46 100644 --- a/doc/topics/releases/2016.3.7.rst +++ b/doc/topics/releases/2016.3.7.rst @@ -7,7 +7,6 @@ Version 2016.3.7 is a bugfix release for :ref:`2016.3.0 `. Changes for v2016.3.6..v2016.3.7 -------------------------------- -<<<<<<< HEAD New master configuration option `allow_minion_key_revoke`, defaults to True. This option controls whether a minion can request that the master revoke its key. When True, a minion can request a key revocation and the master will comply. If it is False, the key will not From f281e1795fd1387caa6dbb96f49c5e9787bfedfb Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:16:32 -0400 Subject: [PATCH 317/508] move additional minion config options to 2016.3.8 release notes --- doc/topics/releases/2016.3.7.rst | 21 --------------------- doc/topics/releases/2016.3.8.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 21 deletions(-) create mode 100644 doc/topics/releases/2016.3.8.rst diff --git a/doc/topics/releases/2016.3.7.rst b/doc/topics/releases/2016.3.7.rst index 4efb507c46..0fca807661 100644 --- a/doc/topics/releases/2016.3.7.rst +++ b/doc/topics/releases/2016.3.7.rst @@ -7,27 +7,6 @@ Version 2016.3.7 is a bugfix release for :ref:`2016.3.0 `. Changes for v2016.3.6..v2016.3.7 -------------------------------- -New master configuration option `allow_minion_key_revoke`, defaults to True. This option -controls whether a minion can request that the master revoke its key. When True, a minion -can request a key revocation and the master will comply. If it is False, the key will not -be revoked by the msater. - -New master configuration option `require_minion_sign_messages` -This requires that minions cryptographically sign the messages they -publish to the master. If minions are not signing, then log this information -at loglevel 'INFO' and drop the message without acting on it. - -New master configuration option `drop_messages_signature_fail` -Drop messages from minions when their signatures do not validate. -Note that when this option is False but `require_minion_sign_messages` is True -minions MUST sign their messages but the validity of their signatures -is ignored. - -New minion configuration option `minion_sign_messages` -Causes the minion to cryptographically sign the payload of messages it places -on the event bus for the master. The payloads are signed with the minion's -private key so the master can verify the signature with its public key. - Security Fix ============ diff --git a/doc/topics/releases/2016.3.8.rst b/doc/topics/releases/2016.3.8.rst new file mode 100644 index 0000000000..c5f0c01da8 --- /dev/null +++ b/doc/topics/releases/2016.3.8.rst @@ -0,0 +1,29 @@ +=========================== +Salt 2016.3.8 Release Notes +=========================== + +Version 2016.3.8 is a bugfix release for :ref:`2016.3.0 `. + +Changes for v2016.3.7..v2016.3.8 +-------------------------------- + +New master configuration option `allow_minion_key_revoke`, defaults to True. This option +controls whether a minion can request that the master revoke its key. When True, a minion +can request a key revocation and the master will comply. If it is False, the key will not +be revoked by the msater. + +New master configuration option `require_minion_sign_messages` +This requires that minions cryptographically sign the messages they +publish to the master. If minions are not signing, then log this information +at loglevel 'INFO' and drop the message without acting on it. + +New master configuration option `drop_messages_signature_fail` +Drop messages from minions when their signatures do not validate. +Note that when this option is False but `require_minion_sign_messages` is True +minions MUST sign their messages but the validity of their signatures +is ignored. + +New minion configuration option `minion_sign_messages` +Causes the minion to cryptographically sign the payload of messages it places +on the event bus for the master. The payloads are signed with the minion's +private key so the master can verify the signature with its public key. From 63823f8c3ed02e7809fc8e61972d289bf233a9ed Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 31 Jul 2017 11:50:21 -0400 Subject: [PATCH 318/508] Add clean_id function to salt.utils.verify.py --- salt/utils/verify.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 5e320c3b59..db513ba675 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -481,12 +481,21 @@ def clean_path(root, path, subdir=False): return '' +def clean_id(id_): + ''' + Returns if the passed id is clean. + ''' + if re.search(r'\.\.{sep}'.format(sep=os.sep), id_): + return False + return True + + def valid_id(opts, id_): ''' Returns if the passed id is valid ''' try: - return bool(clean_path(opts['pki_dir'], id_)) + return bool(clean_path(opts['pki_dir'], id_)) and clean_id(id_) except (AttributeError, KeyError) as e: return False From 7e0a20afca6fad7ab0927b0337eb2f8e4e5ef099 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 31 Jul 2017 11:51:52 -0400 Subject: [PATCH 319/508] Add release notes for 2016.11.7 release --- doc/topics/releases/2016.11.7.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/topics/releases/2016.11.7.rst b/doc/topics/releases/2016.11.7.rst index 6ef9587133..2ad821ff22 100644 --- a/doc/topics/releases/2016.11.7.rst +++ b/doc/topics/releases/2016.11.7.rst @@ -3,3 +3,13 @@ Salt 2016.11.7 Release Notes ============================ Version 2016.11.7 is a bugfix release for :ref:`2016.11.0 `. + +Changes for v2016.11.6..v2016.11.7 +---------------------------------- + +Security Fix +============ + +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master + +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com From d75d3741f8cfc92f66cbcd30ec2a8429c9bcc659 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:33:27 -0400 Subject: [PATCH 320/508] Add Security Notice to 2016.3.7 Release Notes --- doc/topics/releases/2016.3.7.rst | 23 ++++++----------------- doc/topics/releases/2016.3.8.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 doc/topics/releases/2016.3.8.rst diff --git a/doc/topics/releases/2016.3.7.rst b/doc/topics/releases/2016.3.7.rst index 3c47df7eda..0fca807661 100644 --- a/doc/topics/releases/2016.3.7.rst +++ b/doc/topics/releases/2016.3.7.rst @@ -4,23 +4,12 @@ Salt 2016.3.7 Release Notes Version 2016.3.7 is a bugfix release for :ref:`2016.3.0 `. -New master configuration option `allow_minion_key_revoke`, defaults to True. This option -controls whether a minion can request that the master revoke its key. When True, a minion -can request a key revocation and the master will comply. If it is False, the key will not -be revoked by the msater. +Changes for v2016.3.6..v2016.3.7 +-------------------------------- -New master configuration option `require_minion_sign_messages` -This requires that minions cryptographically sign the messages they -publish to the master. If minions are not signing, then log this information -at loglevel 'INFO' and drop the message without acting on it. +Security Fix +============ -New master configuration option `drop_messages_signature_fail` -Drop messages from minions when their signatures do not validate. -Note that when this option is False but `require_minion_sign_messages` is True -minions MUST sign their messages but the validity of their signatures -is ignored. +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master -New minion configuration option `minion_sign_messages` -Causes the minion to cryptographically sign the payload of messages it places -on the event bus for the master. The payloads are signed with the minion's -private key so the master can verify the signature with its public key. +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com diff --git a/doc/topics/releases/2016.3.8.rst b/doc/topics/releases/2016.3.8.rst new file mode 100644 index 0000000000..c5f0c01da8 --- /dev/null +++ b/doc/topics/releases/2016.3.8.rst @@ -0,0 +1,29 @@ +=========================== +Salt 2016.3.8 Release Notes +=========================== + +Version 2016.3.8 is a bugfix release for :ref:`2016.3.0 `. + +Changes for v2016.3.7..v2016.3.8 +-------------------------------- + +New master configuration option `allow_minion_key_revoke`, defaults to True. This option +controls whether a minion can request that the master revoke its key. When True, a minion +can request a key revocation and the master will comply. If it is False, the key will not +be revoked by the msater. + +New master configuration option `require_minion_sign_messages` +This requires that minions cryptographically sign the messages they +publish to the master. If minions are not signing, then log this information +at loglevel 'INFO' and drop the message without acting on it. + +New master configuration option `drop_messages_signature_fail` +Drop messages from minions when their signatures do not validate. +Note that when this option is False but `require_minion_sign_messages` is True +minions MUST sign their messages but the validity of their signatures +is ignored. + +New minion configuration option `minion_sign_messages` +Causes the minion to cryptographically sign the payload of messages it places +on the event bus for the master. The payloads are signed with the minion's +private key so the master can verify the signature with its public key. From a6f902db40e7eef24c15ce7c475d155f4eba58b5 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:35:37 -0400 Subject: [PATCH 321/508] Add Security Notice to 2016.11.77 Release Notes --- doc/topics/releases/2016.11.7.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/topics/releases/2016.11.7.rst b/doc/topics/releases/2016.11.7.rst index 6ef9587133..2ad821ff22 100644 --- a/doc/topics/releases/2016.11.7.rst +++ b/doc/topics/releases/2016.11.7.rst @@ -3,3 +3,13 @@ Salt 2016.11.7 Release Notes ============================ Version 2016.11.7 is a bugfix release for :ref:`2016.11.0 `. + +Changes for v2016.11.6..v2016.11.7 +---------------------------------- + +Security Fix +============ + +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master + +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com From 1d8f827c5840efe4680fe4c33598000e032812b8 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:37:50 -0400 Subject: [PATCH 322/508] Add Security Notice to 2017.7.1 Release Notes --- doc/topics/releases/2017.7.1.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/topics/releases/2017.7.1.rst b/doc/topics/releases/2017.7.1.rst index 69f7a9aa79..7cc616c94b 100644 --- a/doc/topics/releases/2017.7.1.rst +++ b/doc/topics/releases/2017.7.1.rst @@ -4,6 +4,13 @@ Salt 2017.7.1 Release Notes Version 2017.7.1 is a bugfix release for :ref:`2017.7.0 `. +Security Fix +============ + +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master + +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com + Changes for v2017.7.0..v2017.7.1 -------------------------------- From b551e66744f1fa40f4c555111a8b2ea6b43fb1c4 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:53:16 -0400 Subject: [PATCH 323/508] [2016.3] Bump latest and previous versions --- doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 210bca239e..a55bb0362a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -213,8 +213,8 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2017.7.0' # latest release -previous_release = '2016.3.6' # latest release from previous branch +latest_release = '2017.7.1' # latest release +previous_release = '2016.3.7' # latest release from previous branch previous_release_dir = '2016.3' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch From 74e7055d54116542765b37934a8a6d8a943339b8 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:54:12 -0400 Subject: [PATCH 324/508] [2016.11] Bump latest and previous versions --- doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index e891986dc6..0510bf8d50 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -240,8 +240,8 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2017.7.0' # latest release -previous_release = '2016.11.6' # latest release from previous branch +latest_release = '2017.7.1' # latest release +previous_release = '2016.11.7' # latest release from previous branch previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch From cbecf658230fedd8bb8135e24d3b9c522cbee341 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:55:02 -0400 Subject: [PATCH 325/508] [2017.7] Bump latest and previous versions --- doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 6e828afc77..de8db8ea90 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -245,8 +245,8 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2017.7.0' # latest release -previous_release = '2016.11.6' # latest release from previous branch +latest_release = '2017.7.1' # latest release +previous_release = '2016.11.7' # latest release from previous branch previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch From fd6874668b228c744c6ac3ffce8a97ac6bb8969a Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 15 Aug 2017 12:00:38 -0600 Subject: [PATCH 326/508] runit module should also be loaded as runit This will allow the use of both systemd and runit at the same time --- salt/modules/runit.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/modules/runit.py b/salt/modules/runit.py index 083ac86917..e743ed4636 100644 --- a/salt/modules/runit.py +++ b/salt/modules/runit.py @@ -80,7 +80,8 @@ for service_dir in VALID_SERVICE_DIRS: AVAIL_SVR_DIRS = [] # Define the module's virtual name -__virtualname__ = 'service' +__virtualname__ = 'runit' +__virtual_aliases__ = ('runit',) def __virtual__(): @@ -91,8 +92,12 @@ def __virtual__(): if __grains__.get('init') == 'runit': if __grains__['os'] == 'Void': add_svc_avail_path('/etc/sv') + global __virtualname__ + __virtualname__ = 'service' return __virtualname__ - return False + if salt.utils.which('sv'): + return __virtualname__ + return (False, 'Runit not available. Please install sv') def _service_path(name): From 6ae11112957dab7e289668341c85dffdaa1063ef Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 16:40:11 -0400 Subject: [PATCH 327/508] Replace @mock_ec2 calls with @mock_ec2_deprecated calls moto versions >= 1.0.0 have changed the way the mocked connections through boto are handled with the @mock_ec2 decorator. They use the boto3 connection method. However, since we are still using boto in many places, we need to use the new @mock_ec2_deprecated decorator instead to handle the boto connection functions for the unit tests. Versions of moto < 1.0.0 are not Python 3 compatible, so salt-jenkins should be installing newer versions of moto for those tests. Unfortunately, we cannot install an older version of moto for Python2 that use the original @mock_ec2 call and also import the @mock_ec2_deprecated function for newer versions of moto simultaneously as the @mock_ec2_deprecated function doesn't exist in older versions of moto. --- tests/unit/modules/boto_elb_test.py | 26 +-- tests/unit/modules/boto_secgroup_test.py | 38 ++-- tests/unit/modules/boto_vpc_test.py | 262 +++++++++++------------ tests/unit/states/boto_vpc_test.py | 38 ++-- 4 files changed, 177 insertions(+), 187 deletions(-) diff --git a/tests/unit/modules/boto_elb_test.py b/tests/unit/modules/boto_elb_test.py index 20912199e7..5a8adc0efc 100644 --- a/tests/unit/modules/boto_elb_test.py +++ b/tests/unit/modules/boto_elb_test.py @@ -15,17 +15,17 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2, mock_elb + from moto import mock_ec2_deprecated, mock_elb HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -33,10 +33,10 @@ except ImportError: def mock_elb(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -83,7 +83,7 @@ class BotoElbTestCase(TestCase): ''' TestCase for salt.modules.boto_elb module ''' - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_register_instances_valid_id_result_true(self): ''' @@ -102,7 +102,7 @@ class BotoElbTestCase(TestCase): **conn_parameters) self.assertEqual(True, register_result) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_register_instances_valid_id_string(self): ''' @@ -125,7 +125,7 @@ class BotoElbTestCase(TestCase): log.debug(load_balancer_refreshed.instances) self.assertEqual([reservations.instances[0].id], registered_instance_ids) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_result_true(self): ''' @@ -146,7 +146,7 @@ class BotoElbTestCase(TestCase): **conn_parameters) self.assertEqual(True, deregister_result) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_string(self): ''' @@ -172,7 +172,7 @@ class BotoElbTestCase(TestCase): load_balancer_refreshed.instances] self.assertEqual(actual_instances, expected_instances) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_list(self): ''' diff --git a/tests/unit/modules/boto_secgroup_test.py b/tests/unit/modules/boto_secgroup_test.py index 7fd51adf01..14bbf08699 100644 --- a/tests/unit/modules/boto_secgroup_test.py +++ b/tests/unit/modules/boto_secgroup_test.py @@ -29,17 +29,17 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_secgroup unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_secgroup unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -111,7 +111,7 @@ class BotoSecgroupTestCase(TestCase): {'to_port': 80, 'from_port': 80, 'ip_protocol': u'tcp', 'cidr_ip': u'0.0.0.0/0'}] self.assertEqual(boto_secgroup._split_rules(rules), split_rules) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_classic(self): ''' Test of creation of an EC2-Classic security group. The test ensures @@ -131,7 +131,7 @@ class BotoSecgroupTestCase(TestCase): secgroup_created_group[0].vpc_id] self.assertEqual(expected_create_result, secgroup_create_result) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_vpc(self): ''' test of creation of an EC2-VPC security group. The test ensures that a @@ -151,7 +151,7 @@ class BotoSecgroupTestCase(TestCase): @skipIf(True, 'test skipped due to error in moto return - fixed in' ' https://github.com/spulec/moto/commit/cc0166964371f7b5247a49d45637a8f936ccbe6f') - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_classic(self): ''' tests that given a name of a group in EC2-Classic that the correct @@ -173,7 +173,7 @@ class BotoSecgroupTestCase(TestCase): @skipIf(True, 'test skipped because moto does not yet support group' ' filters https://github.com/spulec/moto/issues/154') - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_vpc(self): ''' tests that given a name of a group in EC2-VPC that the correct @@ -193,7 +193,7 @@ class BotoSecgroupTestCase(TestCase): **conn_parameters) self.assertEqual(group_vpc.id, retrieved_group_id) - @mock_ec2 + @mock_ec2_deprecated def test_get_config_single_rule_group_name(self): ''' tests return of 'config' when given group name. get_config returns an OrderedDict. @@ -219,7 +219,7 @@ class BotoSecgroupTestCase(TestCase): @skipIf(True, 'test skipped due to error in moto return - fixed in ' 'https://github.com/spulec/moto/commit/cc0166964371f7b5247a49d45637a8f936ccbe6f') - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_classic(self): ''' tests 'true' existence of a group in EC2-Classic when given name @@ -234,11 +234,11 @@ class BotoSecgroupTestCase(TestCase): @skipIf(True, 'test skipped because moto does not yet support group' ' filters https://github.com/spulec/moto/issues/154') - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_classic(self): pass - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_vpc(self): ''' tests 'true' existence of a group in EC2-VPC when given name and vpc_id @@ -250,7 +250,7 @@ class BotoSecgroupTestCase(TestCase): salt_exists_result = boto_secgroup.exists(name=group_name, vpc_id=vpc_id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_vpc(self): ''' tests 'false' existence of a group in vpc when given name and vpc_id @@ -259,7 +259,7 @@ class BotoSecgroupTestCase(TestCase): salt_exists_result = boto_secgroup.exists(group_name, vpc_id=vpc_id, **conn_parameters) self.assertFalse(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_group_id(self): ''' tests 'true' existence of a group when given group_id @@ -271,7 +271,7 @@ class BotoSecgroupTestCase(TestCase): salt_exists_result = boto_secgroup.exists(group_id=group.id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_group_id(self): ''' tests 'false' existence of a group when given group_id @@ -282,7 +282,7 @@ class BotoSecgroupTestCase(TestCase): @skipIf(True, 'test skipped due to error in moto return - fixed in' ' https://github.com/spulec/moto/commit/cc0166964371f7b5247a49d45637a8f936ccbe6f') - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_ec2_classic(self): ''' test deletion of a group in EC2-Classic. Test does the following: @@ -310,11 +310,11 @@ class BotoSecgroupTestCase(TestCase): @skipIf(True, 'test skipped because moto does not yet support group' ' filters https://github.com/spulec/moto/issues/154') - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_name_ec2_vpc(self): pass - @mock_ec2 + @mock_ec2_deprecated def test__get_conn_true(self): ''' tests ensures that _get_conn returns an boto.ec2.connection.EC2Connection object. diff --git a/tests/unit/modules/boto_vpc_test.py b/tests/unit/modules/boto_vpc_test.py index 54502b0edd..9752f680e2 100644 --- a/tests/unit/modules/boto_vpc_test.py +++ b/tests/unit/modules/boto_vpc_test.py @@ -39,17 +39,17 @@ except ImportError: try: import moto - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -62,7 +62,7 @@ except ImportError: # which was added in boto 2.8.0 # https://github.com/boto/boto/commit/33ac26b416fbb48a60602542b4ce15dcc7029f12 required_boto_version = '2.8.0' -required_moto_version = '0.3.7' +required_moto_version = '1.0.0' region = 'us-east-1' access_key = 'GKTADJGHEIQSXMKKRBJ08H' @@ -265,7 +265,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.modules.boto_vpc module ''' - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via id when the vpc already exists @@ -276,7 +276,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -288,7 +288,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via name when vpc exists @@ -299,7 +299,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -311,7 +311,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via tag when vpc exists @@ -322,7 +322,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -334,7 +334,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via cidr when vpc exists @@ -345,7 +345,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -357,7 +357,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_vpc_exists_but_providing_no_filters_the_vpc_exists_method_raises_a_salt_invocation_error(self): ''' @@ -368,7 +368,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): 'cidr or tags.'): boto_vpc.exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_name(self): ''' Tests getting vpc id when filtering by name @@ -379,7 +379,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_name(self): ''' Tests getting vpc id when filtering by invalid name @@ -390,7 +390,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_cidr(self): ''' Tests getting vpc id when filtering by cidr @@ -401,7 +401,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_cidr(self): ''' Tests getting vpc id when filtering by invalid cidr @@ -412,7 +412,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_tags(self): ''' Tests getting vpc id when filtering by tags @@ -423,7 +423,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_tags(self): ''' Tests getting vpc id when filtering by invalid tags @@ -434,7 +434,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_get_vpc_id_method_when_not_providing_filters_raises_a_salt_invocation_error(self): ''' @@ -443,7 +443,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegexp(SaltInvocationError, 'At least one of the following must be provided: vpc_id, vpc_name, cidr or tags.'): boto_vpc.get_id(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_more_than_one_vpc_is_matched_raises_a_salt_command_execution_error(self): ''' Tests getting vpc id but providing no filters @@ -454,7 +454,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegexp(CommandExecutionError, 'Found more than one VPC matching the criteria.'): boto_vpc.get_id(cidr=u'10.0.0.0/24', **conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -463,7 +463,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_a_vpc_name_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -472,7 +472,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_tags_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -481,7 +481,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_vpc_fails_the_create_vpc_method_returns_false(self): ''' @@ -492,7 +492,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_creation_result['created']) self.assertTrue('error' in vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_vpc_the_delete_vpc_method_returns_true(self): ''' Tests deleting an existing vpc @@ -503,7 +503,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_vpc_the_delete_vpc_method_returns_false(self): ''' Tests deleting a non-existent vpc @@ -512,7 +512,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(delete_vpc_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_true(self): ''' Tests describing parameters via vpc id if vpc exist @@ -539,7 +539,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_vpc, {'vpc': vpc_properties}) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_false(self): ''' Tests describing parameters via vpc id if vpc does not exist @@ -550,7 +550,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(describe_vpc['vpc']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_describing_vpc_by_id_on_connection_error_it_returns_error(self): ''' @@ -563,7 +563,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): describe_result = boto_vpc.describe(vpc_id=vpc.id, **conn_parameters) self.assertTrue('error' in describe_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_but_providing_no_vpc_id_the_describe_method_raises_a_salt_invocation_error(self): ''' Tests describing vpc without vpc id @@ -581,7 +581,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version)) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_single_subnet(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -594,7 +594,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_same_vpc(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -607,7 +607,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_different_vpc(self): ''' tests that given multiple subnet ids in different VPCs that False is @@ -621,7 +621,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(set(subnet_association['vpc_ids']), set([vpc_a.id, vpc_b.id])) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully @@ -633,7 +633,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) self.assertTrue('id' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_a_name_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a name @@ -644,7 +644,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_tags_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a tag @@ -656,7 +656,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_subnet_fails_the_create_subnet_method_returns_error(self): ''' @@ -668,7 +668,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_creation_result = boto_vpc.create_subnet(vpc.id, '10.0.0.0/24', **conn_parameters) self.assertTrue('error' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_subnet_the_delete_subnet_method_returns_true(self): ''' Tests deleting an existing subnet @@ -680,7 +680,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_deletion_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_subnet_the_delete_vpc_method_returns_false(self): ''' Tests deleting a subnet that doesn't exist @@ -688,7 +688,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): delete_subnet_result = boto_vpc.delete_subnet(subnet_id='1234', **conn_parameters) self.assertTrue('error' in delete_subnet_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' Tests checking if a subnet exists when it does exist @@ -700,7 +700,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' Tests checking if a subnet exists which doesn't exist @@ -709,7 +709,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by name @@ -721,7 +721,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by name when it doesn't exist @@ -733,7 +733,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by tag @@ -745,7 +745,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by tag when subnet doesn't exist @@ -757,7 +757,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_subnet_exists_but_providing_no_filters_the_subnet_exists_method_raises_a_salt_invocation_error(self): ''' @@ -769,7 +769,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): boto_vpc.subnet_exists(**conn_parameters) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by id. @@ -784,7 +784,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -798,7 +798,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by name. @@ -813,7 +813,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -827,7 +827,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -845,7 +845,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): set(['id', 'cidr_block', 'availability_zone', 'tags'])) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_name_for_existing_subnets_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -862,7 +862,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnets'][0].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_create_subnet_passes_availability_zone(self): ''' Tests that the availability_zone kwarg is passed on to _create_resource @@ -883,7 +883,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {0}' .format(required_boto_version)) class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway successfully (with no vpc id or name) @@ -894,7 +894,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_non_existent_vpc_the_create_internet_gateway_method_returns_an_error(self): ''' Tests that creating an internet gateway for a non-existent VPC fails. @@ -906,7 +906,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): vpc_name='non-existent-vpc') self.assertTrue('error' in igw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_name_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -921,7 +921,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_id_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -944,7 +944,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {0}' .format(required_boto_version)) class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway successfully (with subnet_id specified) @@ -958,7 +958,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(ngw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_non_existent_subnet_the_create_nat_gateway_method_returns_an_error(self): ''' Tests that creating an nat gateway for a non-existent subnet fails. @@ -970,7 +970,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_name='non-existent-subnet') self.assertTrue('error' in ngw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_subnet_name_specified_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway with subnet name specified. @@ -993,7 +993,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {0}' .format(required_boto_version)) class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_customer_gateway_the_create_customer_gateway_method_returns_true(self): ''' @@ -1003,7 +1003,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_creation_result = boto_vpc.create_customer_gateway('ipsec.1', '10.1.1.1', None) self.assertTrue(gw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' @@ -1014,7 +1014,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_exists_result = boto_vpc.customer_gateway_exists(customer_gateway_id=gw_creation_result['id']) self.assertTrue(gw_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' @@ -1032,7 +1032,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version)) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_succeeds_the_create_dhcp_options_method_returns_true(self): ''' Tests creating dhcp options successfully @@ -1041,7 +1041,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_dhcp_options_and_specifying_a_name_succeeds_the_create_dhcp_options_method_returns_true( self): @@ -1053,7 +1053,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_and_specifying_tags_succeeds_the_create_dhcp_options_method_returns_true( self): ''' @@ -1064,7 +1064,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_dhcp_options_fails_the_create_dhcp_options_method_returns_error(self): ''' @@ -1075,7 +1075,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = dhcp_options_creation_result = boto_vpc.create_dhcp_options(**dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_true( self): ''' @@ -1089,7 +1089,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_association_result['associated']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_error( self): ''' @@ -1101,7 +1101,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_a_non_existent_vpc_the_associate_dhcp_options_method_returns_false( self): ''' @@ -1114,7 +1114,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_an_existing_vpc_succeeds_the_associate_new_dhcp_options_method_returns_true( self): ''' @@ -1126,7 +1126,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_creating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception( self): @@ -1140,7 +1140,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_associating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception(self): ''' @@ -1153,7 +1153,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_a_non_existent_vpc_the_dhcp_options_the_associate_new_dhcp_options_method_returns_false( self): ''' @@ -1163,7 +1163,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.create_dhcp_options(vpc_name='fake', **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_exists_the_dhcp_options_exists_method_returns_true(self): ''' Tests existence of dhcp options successfully @@ -1174,7 +1174,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_do_not_exist_the_dhcp_options_exists_method_returns_false(self): ''' Tests existence of dhcp options failure @@ -1182,7 +1182,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.dhcp_options_exists('fake', **conn_parameters) self.assertFalse(r['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_dhcp_options_exists_but_providing_no_filters_the_dhcp_options_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1199,8 +1199,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {0}' .format(required_boto_version)) class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_the_create_network_acl_method_returns_true(self): ''' Tests creation of network acl with existing vpc @@ -1211,8 +1210,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_a_name_the_create_network_acl_method_returns_true( self): ''' @@ -1224,8 +1222,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_tags_the_create_network_acl_method_returns_true( self): ''' @@ -1237,8 +1234,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_a_non_existent_vpc_the_create_network_acl_method_returns_an_error(self): ''' Tests creation of network acl with a non-existent vpc @@ -1247,7 +1243,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_network_acl_fails_the_create_network_acl_method_returns_false(self): ''' @@ -1261,8 +1257,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_deleting_an_existing_network_acl_the_delete_network_acl_method_returns_true(self): ''' Tests deletion of existing network acl successfully @@ -1274,8 +1269,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_network_acl_the_delete_network_acl_method_returns_an_error(self): ''' Tests deleting a non-existent network acl @@ -1284,8 +1278,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_deletion_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_a_network_acl_exists_the_network_acl_exists_method_returns_true(self): ''' Tests existence of network acl @@ -1297,8 +1290,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_a_network_acl_does_not_exist_the_network_acl_exists_method_returns_false(self): ''' Tests checking network acl does not exist @@ -1307,7 +1299,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_deletion_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_network_acl_exists_but_providing_no_filters_the_network_acl_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1319,7 +1311,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_successfully_the_create_network_acl_entry_method_returns_true(self): ''' @@ -1334,7 +1326,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_for_a_non_existent_network_acl_the_create_network_acl_entry_method_returns_false( self): @@ -1346,7 +1338,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_successfully_the_replace_network_acl_entry_method_returns_true( self): @@ -1363,7 +1355,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_for_a_non_existent_network_acl_the_replace_network_acl_entry_method_returns_false( self): @@ -1374,7 +1366,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_an_existing_network_acl_entry_the_delete_network_acl_entry_method_returns_true(self): ''' @@ -1389,7 +1381,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_network_acl_entry_the_delete_network_acl_entry_method_returns_false( self): @@ -1401,7 +1393,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_true( self): @@ -1417,8 +1409,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_association_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_an_error( self): ''' @@ -1432,7 +1423,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_a_non_existent_subnet_the_associate_network_acl_method_returns_false( self): @@ -1447,7 +1438,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1462,7 +1453,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_a_name_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1478,7 +1469,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_tags_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1495,7 +1486,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_non_existent_subnet_the_associate_new_network_acl_to_subnet_method_returns_false( self): @@ -1509,8 +1500,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_and_association_result) - @mock_ec2 - #@skipIf(True, 'Moto has not implemented this feature. Skipping for now.') + @mock_ec2_deprecated def test_that_when_creating_a_network_acl_to_a_non_existent_vpc_the_associate_new_network_acl_to_subnet_method_returns_an_error( self): ''' @@ -1523,7 +1513,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_succeeds_the_disassociate_network_acl_method_should_return_true(self): ''' @@ -1536,7 +1526,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_vpc_the_disassociate_network_acl_method_should_return_false( self): @@ -1550,7 +1540,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_subnet_the_disassociate_network_acl_method_should_return_false( self): @@ -1571,7 +1561,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {0}' .format(required_boto_version)) class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_succeeds_the_create_route_table_method_returns_true(self): ''' @@ -1583,7 +1573,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_on_a_non_existent_vpc_the_create_route_table_method_returns_false(self): ''' @@ -1593,7 +1583,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_table_succeeds_the_delete_route_table_method_returns_true(self): ''' @@ -1606,7 +1596,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_route_table_the_delete_route_table_method_returns_false(self): ''' @@ -1616,7 +1606,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_exists_the_route_table_exists_method_returns_true(self): ''' @@ -1629,7 +1619,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_does_not_exist_the_route_table_exists_method_returns_false(self): ''' @@ -1639,7 +1629,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_route_table_exists_but_providing_no_filters_the_route_table_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1651,7 +1641,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_succeeds_the_associate_route_table_method_should_return_the_association_id( self): @@ -1666,7 +1656,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_route_table_the_associate_route_table_method_should_return_false( self): @@ -1680,7 +1670,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_subnet_the_associate_route_table_method_should_return_false( self): @@ -1694,7 +1684,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_a_route_table_succeeds_the_disassociate_route_table_method_should_return_true( self): @@ -1711,7 +1701,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_succeeds_the_create_route_method_should_return_true(self): ''' @@ -1724,7 +1714,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_with_a_non_existent_route_table_the_create_route_method_should_return_false( self): @@ -1735,7 +1725,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_succeeds_the_delete_route_method_should_return_true(self): ''' @@ -1748,7 +1738,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_with_a_non_existent_route_table_the_delete_route_method_should_return_false( self): @@ -1759,7 +1749,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_succeeds_the_replace_route_method_should_return_true(self): ''' @@ -1772,7 +1762,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_replacing_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_with_a_non_existent_route_table_the_replace_route_method_should_return_false( self): @@ -1793,7 +1783,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_request_vpc_peering_connection(self): ''' Run with 2 vpc ids and returns a message @@ -1806,7 +1796,7 @@ class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): peer_vpc_id=other_vpc.id, **conn_parameters)) - @mock_ec2 + @mock_ec2_deprecated def test_raises_error_if_both_vpc_name_and_vpc_id_are_specified(self): ''' Must specify only one diff --git a/tests/unit/states/boto_vpc_test.py b/tests/unit/states/boto_vpc_test.py index 31be861915..94225e1993 100644 --- a/tests/unit/states/boto_vpc_test.py +++ b/tests/unit/states/boto_vpc_test.py @@ -34,18 +34,18 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -112,7 +112,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.states.boto_vpc state.module ''' - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_does_not_exist(self): ''' Tests present on a VPC that does not exist. @@ -123,14 +123,14 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes']['new']['vpc']['state'], 'available') - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_exists(self): vpc = self._create_vpc(name='test') vpc_present_result = salt_states['boto_vpc.present']('test', cidr_block) self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): with patch('moto.ec2.models.VPCBackend.create_vpc', side_effect=BotoServerError(400, 'Mocked error')): @@ -138,7 +138,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_present_result['result']) self.assertTrue('Mocked error' in vpc_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_does_not_exist(self): ''' Tests absent on a VPC that does not exist. @@ -148,7 +148,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_exists(self): vpc = self._create_vpc(name='test') with patch.dict('salt.utils.boto.__salt__', funcs): @@ -156,7 +156,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes']['new']['vpc'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -176,7 +176,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): _create = getattr(self, '_create_' + self.resource_type) _create(vpc_id=vpc_id, name=name, **self.extra_kwargs) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_does_not_exist(self): ''' Tests present on a resource that does not exist. @@ -191,7 +191,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertTrue(exists) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_exists(self): vpc = self._create_vpc(name='test') resource = self._create_resource(vpc_id=vpc.id, name='test') @@ -201,7 +201,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_present_result['result']) self.assertEqual(resource_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): vpc = self._create_vpc(name='test') @@ -212,7 +212,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertFalse(resource_present_result['result']) self.assertTrue('Mocked error' in resource_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_does_not_exist(self): ''' Tests absent on a resource that does not exist. @@ -222,7 +222,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_absent_result['result']) self.assertEqual(resource_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_exists(self): vpc = self._create_vpc(name='test') self._create_resource(vpc_id=vpc.id, name='test') @@ -234,7 +234,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertFalse(exists) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -282,7 +282,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas backend_create = 'RouteTableBackend.create_route_table' backend_delete = 'RouteTableBackend.delete_route_table' - @mock_ec2 + @mock_ec2_deprecated def test_present_with_subnets(self): vpc = self._create_vpc(name='test') subnet1 = self._create_subnet(vpc_id=vpc.id, name='test1') @@ -307,7 +307,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas new_subnets = changes['new']['subnets_associations'] self.assertEqual(new_subnets[0]['subnet_id'], subnet2.id) - @mock_ec2 + @mock_ec2_deprecated def test_present_with_routes(self): vpc = self._create_vpc(name='test') igw = self._create_internet_gateway(name='test', vpc_id=vpc.id) From 60b406e088e327a19d98b9e537f3d52b51722fa2 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 17:15:45 -0400 Subject: [PATCH 328/508] @mock_elb needs to be changed to @mock_elb_deprecated as well --- tests/unit/modules/boto_elb_test.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unit/modules/boto_elb_test.py b/tests/unit/modules/boto_elb_test.py index 5a8adc0efc..7f0479107b 100644 --- a/tests/unit/modules/boto_elb_test.py +++ b/tests/unit/modules/boto_elb_test.py @@ -15,7 +15,7 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2_deprecated, mock_elb + from moto import mock_ec2_deprecated, mock_elb_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False @@ -24,19 +24,19 @@ except ImportError: ''' if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + Allows boto_elb unit tests to use the @mock_ec2_deprecated decorator without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass return stub_function - def mock_elb(self): + def mock_elb_deprecated(self): ''' - if the mock_ec2_deprecated function is not available due to import failure + if the mock_elb_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator - without a "NameError: name 'mock_ec2_deprecated' is not defined" error. + Allows boto_elb unit tests to use the @mock_elb_deprecated decorator + without a "NameError: name 'mock_elb_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -84,7 +84,7 @@ class BotoElbTestCase(TestCase): TestCase for salt.modules.boto_elb module ''' @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_register_instances_valid_id_result_true(self): ''' tests that given a valid instance id and valid ELB that @@ -103,7 +103,7 @@ class BotoElbTestCase(TestCase): self.assertEqual(True, register_result) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_register_instances_valid_id_string(self): ''' tests that given a string containing a instance id and valid ELB that @@ -126,7 +126,7 @@ class BotoElbTestCase(TestCase): self.assertEqual([reservations.instances[0].id], registered_instance_ids) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_result_true(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -147,7 +147,7 @@ class BotoElbTestCase(TestCase): self.assertEqual(True, deregister_result) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_string(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -173,7 +173,7 @@ class BotoElbTestCase(TestCase): self.assertEqual(actual_instances, expected_instances) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_list(self): ''' tests that given an valid ids in the form of a list that the boto_elb From a0b19bdc27a3010fb9309f0f918dda07b73fca01 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 17:31:14 -0400 Subject: [PATCH 329/508] Update account id value in boto_secgroup module unit test This value was updated in moto 1.0.0 with the following commit: https://github.com/spulec/moto/commit/5f3fbff627029c0c260546a674a3b372cacdae82 --- tests/unit/modules/boto_secgroup_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/boto_secgroup_test.py b/tests/unit/modules/boto_secgroup_test.py index 14bbf08699..7bcf262477 100644 --- a/tests/unit/modules/boto_secgroup_test.py +++ b/tests/unit/modules/boto_secgroup_test.py @@ -209,7 +209,7 @@ class BotoSecgroupTestCase(TestCase): group = conn.create_security_group(name=group_name, description=group_name) group.authorize(ip_protocol=ip_protocol, from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) # setup the expected get_config result - expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'111122223333'), + expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id),('owner_id', u'123456789012'), ('description', group.description), ('tags', {}), ('rules', [{'to_port': to_port, 'from_port': from_port, 'ip_protocol': ip_protocol, 'cidr_ip': cidr_ip}]), From 5597b1a30ec963d302ddd72fd1ca73fcae61cccd Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 12:57:39 -0400 Subject: [PATCH 330/508] Skip 2 failing tests in Python 3 due to upstream bugs --- tests/unit/states/boto_vpc_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/states/boto_vpc_test.py b/tests/unit/states/boto_vpc_test.py index 94225e1993..2c85cc86cd 100644 --- a/tests/unit/states/boto_vpc_test.py +++ b/tests/unit/states/boto_vpc_test.py @@ -15,10 +15,10 @@ from salttesting.helpers import ensure_in_syspath ensure_in_syspath('../../') # Import Salt libs -import salt.config -import salt.loader import salt.utils.boto from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin +from salt.ext import six +from salt.utils.versions import LooseVersion # pylint: disable=import-error,unused-import from unit.modules.boto_vpc_test import BotoVpcTestCaseMixin @@ -272,6 +272,9 @@ class BotoVpcInternetGatewayTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTe @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(six.PY3, 'Disabled for Python 3 due to upstream bugs: ' + 'https://github.com/spulec/moto/issues/548 and ' + 'https://github.com/gabrielfalcao/HTTPretty/issues/325') @skipIf(HAS_BOTO is False, 'The boto module must be installed.') @skipIf(HAS_MOTO is False, 'The moto module must be installed.') @skipIf(_has_required_boto() is False, 'The boto module must be greater than' From c6b9ca4b9e6c33e162b254b054f890c765ccb534 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 14:19:15 -0400 Subject: [PATCH 331/508] Lint fix: add missing space --- tests/unit/modules/boto_secgroup_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/boto_secgroup_test.py b/tests/unit/modules/boto_secgroup_test.py index 7bcf262477..0b6ea63258 100644 --- a/tests/unit/modules/boto_secgroup_test.py +++ b/tests/unit/modules/boto_secgroup_test.py @@ -209,7 +209,7 @@ class BotoSecgroupTestCase(TestCase): group = conn.create_security_group(name=group_name, description=group_name) group.authorize(ip_protocol=ip_protocol, from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) # setup the expected get_config result - expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id),('owner_id', u'123456789012'), + expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'123456789012'), ('description', group.description), ('tags', {}), ('rules', [{'to_port': to_port, 'from_port': from_port, 'ip_protocol': ip_protocol, 'cidr_ip': cidr_ip}]), From 253e216a8d1498947b4ccd0e978f0af7bef917e5 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 15 Aug 2017 12:30:03 -0600 Subject: [PATCH 332/508] fix IP address spelling --- salt/cloud/clouds/nova.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py index bf1c917d7d..1905e28102 100644 --- a/salt/cloud/clouds/nova.py +++ b/salt/cloud/clouds/nova.py @@ -729,15 +729,15 @@ def request_instance(vm_=None, call=None): try: floating_ip = conn.floating_ip_create(pool)['ip'] except Exception: - log.info('A new ip address was unable to be allocated. ' - 'An ip address will be pulled from the already allocated list, ' + log.info('A new IP address was unable to be allocated. ' + 'An IP address will be pulled from the already allocated list, ' 'This will cause a race condition when building in parallel.') for fl_ip, opts in six.iteritems(conn.floating_ip_list()): if opts['fixed_ip'] is None and opts['pool'] == pool: floating_ip = fl_ip break if floating_ip is None: - log.error('No ip addresses available to allocate for this server: {0}'.format(vm_['name'])) + log.error('No IP addresses available to allocate for this server: {0}'.format(vm_['name'])) def __query_node_data(vm_): try: From 37029c1a16945fe407f4562fcf649e980bdf5fe2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 15 Aug 2017 15:30:13 -0600 Subject: [PATCH 333/508] Fix unit.test_doc test Use findstr instead of grep on Windows Use os.linesep for file paths --- tests/unit/test_doc.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_doc.py b/tests/unit/test_doc.py index 336833a46c..9e6583a099 100644 --- a/tests/unit/test_doc.py +++ b/tests/unit/test_doc.py @@ -6,6 +6,7 @@ # Import Python libs from __future__ import absolute_import +import os # Import Salt Testing libs from tests.support.unit import TestCase @@ -13,6 +14,7 @@ from tests.support.unit import TestCase # Import Salt libs import tests.integration as integration import salt.modules.cmdmod +import salt.utils class DocTestCase(TestCase): @@ -32,8 +34,15 @@ class DocTestCase(TestCase): https://github.com/saltstack/salt/issues/12788 ''' salt_dir = integration.CODE_DIR - salt_dir += '/' - cmd = 'grep -r :doc: ' + salt_dir + + if salt.utils.is_windows(): + # No grep in Windows, use findstr + # findstr in windows doesn't prepend 'Binary` to binary files, so + # use the '/P' switch to skip files with unprintable characters + cmd = 'findstr /C:":doc:" /S /P {0}\*'.format(salt_dir) + else: + salt_dir += '/' + cmd = 'grep -r :doc: ' + salt_dir grep_call = salt.modules.cmdmod.run_stdout(cmd=cmd).split('\n') @@ -43,25 +52,32 @@ class DocTestCase(TestCase): if line.startswith('Binary'): continue - key, val = line.split(':', 1) + if salt.utils.is_windows(): + # Need the space after the colon so it doesn't split the drive + # letter + key, val = line.split(': ', 1) + else: + key, val = line.split(':', 1) # Don't test man pages, this file, # the page that documents to not use ":doc:", or # the doc/conf.py file if 'man' in key \ or key.endswith('test_doc.py') \ - or key.endswith('doc/conf.py') \ - or key.endswith('/conventions/documentation.rst') \ - or key.endswith('doc/topics/releases/2016.11.2.rst') \ - or key.endswith('doc/topics/releases/2016.11.3.rst') \ - or key.endswith('doc/topics/releases/2016.3.5.rst'): + or key.endswith(os.sep.join(['doc', 'conf.py'])) \ + or key.endswith(os.sep.join(['conventions', 'documentation.rst'])) \ + or key.endswith(os.sep.join(['doc', 'topics', 'releases', '2016.11.2.rst'])) \ + or key.endswith(os.sep.join(['doc', 'topics', 'releases', '2016.11.3.rst'])) \ + or key.endswith(os.sep.join(['doc', 'topics', 'releases', '2016.3.5.rst'])): continue # Set up test return dict if test_ret.get(key) is None: - test_ret[key] = [val.lstrip()] + test_ret[key] = [val.strip()] else: - test_ret[key].append(val.lstrip()) + test_ret[key].append(val.strip()) + + print('*' * 68) # Allow test results to show files with :doc: ref, rather than truncating self.maxDiff = None From e9febe4893280b50cb56686ac4253b980567f245 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 15 Aug 2017 16:34:03 -0600 Subject: [PATCH 334/508] Fix unit.test_fileclient Use os.sep instead of hard-coded, unix-style paths --- tests/unit/test_fileclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_fileclient.py b/tests/unit/test_fileclient.py index 077630693a..b6bd2207e0 100644 --- a/tests/unit/test_fileclient.py +++ b/tests/unit/test_fileclient.py @@ -6,6 +6,7 @@ # Import Python libs from __future__ import absolute_import import errno +import os # Import Salt Testing libs from tests.support.mock import patch, Mock @@ -38,7 +39,7 @@ class FileclientTestCase(TestCase): for exists in range(2): with patch('os.makedirs', self._fake_makedir()): with Client(self.opts)._cache_loc('testfile') as c_ref_itr: - assert c_ref_itr == '/__test__/files/base/testfile' + assert c_ref_itr == os.sep + os.sep.join(['__test__', 'files', 'base', 'testfile']) def test_cache_raises_exception_on_non_eexist_ioerror(self): ''' From 121cd4ef818dbb55e6bce95ae2c159a0b42f299d Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 15 Aug 2017 16:53:28 -0600 Subject: [PATCH 335/508] Fix `salt.utils.recursive_copy` for Windows Use os.sep instead of hard-coded `/` --- salt/utils/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/files.py b/salt/utils/files.py index 12dd6987c6..d4893608a2 100644 --- a/salt/utils/files.py +++ b/salt/utils/files.py @@ -57,7 +57,7 @@ def recursive_copy(source, dest): (identical to cp -r on a unix machine) ''' for root, _, files in os.walk(source): - path_from_source = root.replace(source, '').lstrip('/') + path_from_source = root.replace(source, '').lstrip(os.sep) target_directory = os.path.join(dest, path_from_source) if not os.path.exists(target_directory): os.makedirs(target_directory) From e6a4619ec1a4c965d2a218ccc1b6d96942315d12 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 15 Aug 2017 16:01:32 -0700 Subject: [PATCH 336/508] Better approach without using python_shell=True. --- salt/modules/osquery.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/osquery.py b/salt/modules/osquery.py index 99fd89c330..a5cb1ee4e5 100644 --- a/salt/modules/osquery.py +++ b/salt/modules/osquery.py @@ -8,6 +8,7 @@ from __future__ import absolute_import # Import python libs import json +import shlex # Import Salt libs import salt.utils @@ -55,8 +56,8 @@ def _osquery(sql, format='json'): 'result': True, } - cmd = 'osqueryi --json "{0}"'.format(sql) - res = __salt__['cmd.run_all'](cmd, python_shell=True) + cmd = ['osqueryi'] + ['--json'] + [sql] + res = __salt__['cmd.run_all'](cmd) if res['retcode'] == 0: ret['data'] = json.loads(res['stdout']) else: From b8384608160a88b91cecc8d8bf90e0a09719875f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 15 Aug 2017 22:18:53 -0500 Subject: [PATCH 337/508] Fix bug in on_header callback when no Content-Type is found in headers 01d6ee176d347fd341e7d45236cc62871a6dda21 modified our header-parsing callback to parse the headers for the encoding. However, this did not take into account the cases in which there is a 30x redirect with no Content-Type header. In these cases, we fail to reset the values we use to track both a) the headers and b) whether or not we've parsed the HTTP status code, leading to a ValueError when the HTTP status line is passed to the callback after following the redirect. This commit fixes this case by detecting the blank line at the end of the HTTP headers, and if no Content-Type has been found, the tracking values are reset so that we properly process the headers post-redirect. --- salt/fileclient.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/fileclient.py b/salt/fileclient.py index 5f02723ef9..2b4484211c 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -621,6 +621,13 @@ class Client(object): def on_header(hdr): if write_body[1] is not False and write_body[2] is None: + if not hdr.strip() and 'Content-Type' not in write_body[1]: + # We've reached the end of the headers and not yet + # found the Content-Type. Reset the values we're + # tracking so that we properly follow the redirect. + write_body[0] = None + write_body[1] = False + return # Try to find out what content type encoding is used if # this is a text file write_body[1].parse_line(hdr) # pylint: disable=no-member From 44ed53b1dfd3f7d3a57cc15499c8ab26b5dc022a Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Wed, 16 Aug 2017 10:26:45 +0300 Subject: [PATCH 338/508] [DOCS] Fix link to Salt Cloud Feature Matrix --- doc/topics/cloud/action.rst | 6 +++--- doc/topics/cloud/function.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/topics/cloud/action.rst b/doc/topics/cloud/action.rst index 0e10a95653..f36211d01a 100644 --- a/doc/topics/cloud/action.rst +++ b/doc/topics/cloud/action.rst @@ -21,7 +21,7 @@ Or you may specify a map which includes all VMs to perform the action on: $ salt-cloud -a reboot -m /path/to/mapfile -The following is a list of actions currently supported by salt-cloud: +The following is an example list of actions currently supported by ``salt-cloud``: .. code-block:: yaml @@ -36,5 +36,5 @@ The following is a list of actions currently supported by salt-cloud: - start - stop -Another useful reference for viewing more salt-cloud actions is the -:ref:Salt Cloud Feature Matrix +Another useful reference for viewing more ``salt-cloud`` actions is the +:ref:`Salt Cloud Feature Matrix `. diff --git a/doc/topics/cloud/function.rst b/doc/topics/cloud/function.rst index 3368cd820b..c83487d36b 100644 --- a/doc/topics/cloud/function.rst +++ b/doc/topics/cloud/function.rst @@ -26,5 +26,5 @@ gathering information about instances on a provider basis: $ salt-cloud -f list_nodes_full linode $ salt-cloud -f list_nodes_select linode -Another useful reference for viewing salt-cloud functions is the +Another useful reference for viewing ``salt-cloud`` functions is the :ref:`Salt Cloud Feature Matrix `. From 3f2e96e561947b66ed0ea362bb46cf3e444614dd Mon Sep 17 00:00:00 2001 From: Steven Joseph Date: Wed, 16 Aug 2017 21:04:37 +1000 Subject: [PATCH 339/508] Convert set to list for serializer --- salt/states/git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 4808c0dce3..f12ae29c26 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1322,9 +1322,9 @@ def latest(name, ]) if set(all_local_tags) != remote_tags: has_remote_rev = False - ret['changes']['new_tags'] = remote_tags.symmetric_difference( + ret['changes']['new_tags'] = list(remote_tags.symmetric_difference( all_local_tags - ) + )) if not has_remote_rev: try: From c8e98c8d8a38b1263d4d7fc4371765bf3a1676ac Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Wed, 16 Aug 2017 14:51:16 +0200 Subject: [PATCH 340/508] Added unit tests for Kubernetes module Added unit tests for: * Node listing, * deployment listing, * service listing, * pod listing, * deployment deletion and * deployment creation. --- tests/unit/modules/kubernetes_test.py | 115 ++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 tests/unit/modules/kubernetes_test.py diff --git a/tests/unit/modules/kubernetes_test.py b/tests/unit/modules/kubernetes_test.py new file mode 100644 index 0000000000..5f2dcdc1dc --- /dev/null +++ b/tests/unit/modules/kubernetes_test.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Jochen Breuer ` +''' + +# Import Python Libs +from __future__ import absolute_import +import os + +# Import Salt Testing Libs +from salttesting import TestCase, skipIf +from salttesting.mock import ( + Mock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + +from salt.modules import kubernetes + +kubernetes.__salt__ = {} +kubernetes.__grains__ = {} +kubernetes.__context__ = {} + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class KubernetesTestCase(TestCase): + ''' + Test cases for salt.modules.kubernetes + ''' + def test_nodes(self): + ''' + Test node listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.CoreV1Api.return_value = Mock( + **{"list_node.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_node_name'}}]}} + ) + self.assertEqual(kubernetes.nodes(), ['mock_node_name']) + self.assertTrue(kubernetes.kubernetes.client.CoreV1Api().list_node().to_dict.called) + + def test_deployments(self): + ''' + Tests deployment listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.ExtensionsV1beta1Api.return_value = Mock( + **{"list_namespaced_deployment.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_deployment_name'}}]}} + ) + self.assertEqual(kubernetes.deployments(), ['mock_deployment_name']) + self.assertTrue( + kubernetes.kubernetes.client.ExtensionsV1beta1Api().list_namespaced_deployment().to_dict.called) + + def test_services(self): + ''' + Tests services listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.CoreV1Api.return_value = Mock( + **{"list_namespaced_service.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_service_name'}}]}} + ) + self.assertEqual(kubernetes.services(), ['mock_service_name']) + self.assertTrue(kubernetes.kubernetes.client.CoreV1Api().list_namespaced_service().to_dict.called) + + def test_pods(self): + ''' + Tests pods listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.CoreV1Api.return_value = Mock( + **{"list_namespaced_pod.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_pod_name'}}]}} + ) + self.assertEqual(kubernetes.pods(), ['mock_pod_name']) + self.assertTrue(kubernetes.kubernetes.client.CoreV1Api().list_namespaced_pod().to_dict.called) + + def test_delete_deployments(self): + ''' + Tests deployment creation. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.V1DeleteOptions = Mock(return_value="") + mock_kubernetes_lib.client.ExtensionsV1beta1Api.return_value = Mock( + **{"delete_namespaced_deployment.return_value.to_dict.return_value": {}} + ) + self.assertEqual(kubernetes.delete_deployment("test"), {}) + self.assertTrue( + kubernetes.kubernetes.client.ExtensionsV1beta1Api().delete_namespaced_deployment().to_dict.called) + + def test_create_deployments(self): + ''' + Tests deployment creation. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.ExtensionsV1beta1Api.return_value = Mock( + **{"create_namespaced_deployment.return_value.to_dict.return_value": {}} + ) + self.assertEqual(kubernetes.create_deployment("test", "default", {}, {}, None, None, None), {}) + self.assertTrue( + kubernetes.kubernetes.client.ExtensionsV1beta1Api().create_namespaced_deployment().to_dict.called) From 651b1bab096b9f45660b81baec30ebe7258e9874 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 16 Aug 2017 18:36:09 +0300 Subject: [PATCH 341/508] Properly handle `prereq` having lost requisites. Add the result to `self.pre` instead of to `running`. --- salt/state.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/salt/state.py b/salt/state.py index 82c180486b..729e740f5f 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2071,13 +2071,17 @@ class State(object): req_val = lreq[req_key] comment += \ '{0}{1}: {2}\n'.format(' ' * 23, req_key, req_val) - running[tag] = {'changes': {}, - 'result': False, - 'comment': comment, - '__run_num__': self.__run_num, - '__sls__': low['__sls__']} + if low.get('__prereq__'): + run_dict = self.pre + else: + run_dict = running + run_dict[tag] = {'changes': {}, + 'result': False, + 'comment': comment, + '__run_num__': self.__run_num, + '__sls__': low['__sls__']} self.__run_num += 1 - self.event(running[tag], len(chunks), fire_event=low.get('fire_event')) + self.event(run_dict[tag], len(chunks), fire_event=low.get('fire_event')) return running for chunk in reqs: # Check to see if the chunk has been run, only run it if From 74bc377eb4683d6a2776b65ac1a71feda694b3ef Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 16 Aug 2017 09:13:39 -0700 Subject: [PATCH 342/508] Updating the other function that uses cmd.run_all --- salt/modules/osquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/osquery.py b/salt/modules/osquery.py index a5cb1ee4e5..99c0093284 100644 --- a/salt/modules/osquery.py +++ b/salt/modules/osquery.py @@ -37,7 +37,7 @@ def _table_attrs(table): ''' Helper function to find valid table attributes ''' - cmd = 'osqueryi --json "pragma table_info({0})"'.format(table) + cmd = ['osqueryi'] + ['--json'] + ['pragma table_info{0}'.format(table)] res = __salt__['cmd.run_all'](cmd) if res['retcode'] == 0: attrs = [] From 8915e62bd94709127b20b7fcef9bf7c4b1ddb4a6 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 16 Aug 2017 09:22:54 -0700 Subject: [PATCH 343/508] Removing an import that is not needed. --- salt/modules/osquery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/osquery.py b/salt/modules/osquery.py index 99c0093284..842f63944d 100644 --- a/salt/modules/osquery.py +++ b/salt/modules/osquery.py @@ -8,7 +8,6 @@ from __future__ import absolute_import # Import python libs import json -import shlex # Import Salt libs import salt.utils From 201ceae4c45782f0bfbf2ee6d1e341880460a10b Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 16 Aug 2017 10:29:26 -0600 Subject: [PATCH 344/508] Fix lint, remove debug statement --- tests/unit/test_doc.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test_doc.py b/tests/unit/test_doc.py index 9e6583a099..52311d2f9e 100644 --- a/tests/unit/test_doc.py +++ b/tests/unit/test_doc.py @@ -39,7 +39,7 @@ class DocTestCase(TestCase): # No grep in Windows, use findstr # findstr in windows doesn't prepend 'Binary` to binary files, so # use the '/P' switch to skip files with unprintable characters - cmd = 'findstr /C:":doc:" /S /P {0}\*'.format(salt_dir) + cmd = 'findstr /C:":doc:" /S /P {0}\\*'.format(salt_dir) else: salt_dir += '/' cmd = 'grep -r :doc: ' + salt_dir @@ -77,8 +77,6 @@ class DocTestCase(TestCase): else: test_ret[key].append(val.strip()) - print('*' * 68) - # Allow test results to show files with :doc: ref, rather than truncating self.maxDiff = None From fe7554cfebe64a42cc962078cb087deec5ca918b Mon Sep 17 00:00:00 2001 From: Pablo Hernandez Date: Wed, 16 Aug 2017 15:56:49 -0400 Subject: [PATCH 345/508] Fixes ignored push flag for docker.push module issue #42992 --- salt/modules/dockermod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 4d4ca55bac..0d735f0f65 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -3949,7 +3949,7 @@ def save(name, ret['Size_Human'] = _size_fmt(ret['Size']) # Process push - if kwargs.get(push, False): + if kwargs.get('push', False): ret['Push'] = __salt__['cp.push'](path) return ret From 92dc3c0ece86f8d10532031392f8f0f84f53bce6 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 16 Aug 2017 15:59:10 -0600 Subject: [PATCH 346/508] Use os.sep for path --- tests/unit/test_stateconf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_stateconf.py b/tests/unit/test_stateconf.py index 5396d0dac0..9edb7d1eca 100644 --- a/tests/unit/test_stateconf.py +++ b/tests/unit/test_stateconf.py @@ -102,8 +102,9 @@ test: - name: echo sls_dir={{sls_dir}} - cwd: / ''', sls='path.to.sls') - self.assertEqual(result['test']['cmd.run'][0]['name'], - 'echo sls_dir=path/to') + self.assertEqual( + result['test']['cmd.run'][0]['name'], + 'echo sls_dir=path{0}to'.format(os.sep)) def test_states_declared_with_shorthand_no_args(self): result = self._render_sls(''' From 0ece2a8f0c2108c10218b63766ed38d73f5b5eb7 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 16 Aug 2017 16:28:05 -0700 Subject: [PATCH 347/508] Fixing a bug that prevented editing Slack messages and having the commands resent to the Slack engine. --- salt/engines/slack.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 1efd94fee0..668f3d952a 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -181,11 +181,20 @@ def start(token, if 'aliases' in groups[group]: aliases.update(groups[group]['aliases']) + if 'user' not in _m: + if 'message' in _m and 'user' in _m['message']: + log.debug('Message was edited, ' + 'so we look for user in ' + 'the original message.') + _user = _m['message']['user'] + else: + _user = _m['user'] + # Ensure the user is allowed to run commands if valid_users: - log.debug('{0} {1}'.format(all_users, _m['user'])) - if _m['user'] not in valid_users and all_users.get(_m['user'], None) not in valid_users: - channel.send_message('{0} not authorized to run Salt commands'.format(all_users[_m['user']])) + log.debug('{0} {1}'.format(all_users, _user)) + if _user not in valid_users and all_users.get(_user, None) not in valid_users: + channel.send_message('{0} not authorized to run Salt commands'.format(all_users[_user])) return # Trim the ! from the front @@ -219,7 +228,7 @@ def start(token, # Ensure the command is allowed if valid_commands: if cmd not in valid_commands: - channel.send_message('{0} is not allowed to use command {1}.'.format(all_users[_m['user']], cmd)) + channel.send_message('{0} is not allowed to use command {1}.'.format(all_users[_user], cmd)) return # Parse args and kwargs From 1b8729b3e786107f8dc95740420eb97189e4d237 Mon Sep 17 00:00:00 2001 From: SuperPommeDeTerre Date: Thu, 17 Aug 2017 11:56:23 +0200 Subject: [PATCH 348/508] Fix for #26995 --- salt/modules/artifactory.py | 77 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/salt/modules/artifactory.py b/salt/modules/artifactory.py index d521e786f3..169ffaff8b 100644 --- a/salt/modules/artifactory.py +++ b/salt/modules/artifactory.py @@ -202,45 +202,48 @@ def _get_snapshot_url(artifactory_url, repository, group_id, artifact_id, versio has_classifier = classifier is not None and classifier != "" if snapshot_version is None: - snapshot_version_metadata = _get_snapshot_version_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, headers=headers) + try: + snapshot_version_metadata = _get_snapshot_version_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, headers=headers) + if packaging not in snapshot_version_metadata['snapshot_versions']: + error_message = '''Cannot find requested packaging '{packaging}' in the snapshot version metadata. + artifactory_url: {artifactory_url} + repository: {repository} + group_id: {group_id} + artifact_id: {artifact_id} + packaging: {packaging} + classifier: {classifier} + version: {version}'''.format( + artifactory_url=artifactory_url, + repository=repository, + group_id=group_id, + artifact_id=artifact_id, + packaging=packaging, + classifier=classifier, + version=version) + raise ArtifactoryError(error_message) - if packaging not in snapshot_version_metadata['snapshot_versions']: - error_message = '''Cannot find requested packaging '{packaging}' in the snapshot version metadata. - artifactory_url: {artifactory_url} - repository: {repository} - group_id: {group_id} - artifact_id: {artifact_id} - packaging: {packaging} - classifier: {classifier} - version: {version}'''.format( - artifactory_url=artifactory_url, - repository=repository, - group_id=group_id, - artifact_id=artifact_id, - packaging=packaging, - classifier=classifier, - version=version) - raise ArtifactoryError(error_message) + if has_classifier and classifier not in snapshot_version_metadata['snapshot_versions']: + error_message = '''Cannot find requested classifier '{classifier}' in the snapshot version metadata. + artifactory_url: {artifactory_url} + repository: {repository} + group_id: {group_id} + artifact_id: {artifact_id} + packaging: {packaging} + classifier: {classifier} + version: {version}'''.format( + artifactory_url=artifactory_url, + repository=repository, + group_id=group_id, + artifact_id=artifact_id, + packaging=packaging, + classifier=classifier, + version=version) + raise ArtifactoryError(error_message) - if has_classifier and classifier not in snapshot_version_metadata['snapshot_versions']: - error_message = '''Cannot find requested classifier '{classifier}' in the snapshot version metadata. - artifactory_url: {artifactory_url} - repository: {repository} - group_id: {group_id} - artifact_id: {artifact_id} - packaging: {packaging} - classifier: {classifier} - version: {version}'''.format( - artifactory_url=artifactory_url, - repository=repository, - group_id=group_id, - artifact_id=artifact_id, - packaging=packaging, - classifier=classifier, - version=version) - raise ArtifactoryError(error_message) - - snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] + snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] + except CommandExecutionError as err: + log.error('Could not fetch maven-metadat.xml. Assuming snapshot_version=%s.', version) + snapshot_version = version group_url = __get_group_id_subpath(group_id) From 0b666e100bda9b71e39a193e609f302f43918d75 Mon Sep 17 00:00:00 2001 From: SuperPommeDeTerre Date: Thu, 17 Aug 2017 13:28:10 +0200 Subject: [PATCH 349/508] Fix typo. --- salt/modules/artifactory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/artifactory.py b/salt/modules/artifactory.py index 169ffaff8b..26065a8d37 100644 --- a/salt/modules/artifactory.py +++ b/salt/modules/artifactory.py @@ -242,7 +242,7 @@ def _get_snapshot_url(artifactory_url, repository, group_id, artifact_id, versio snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] except CommandExecutionError as err: - log.error('Could not fetch maven-metadat.xml. Assuming snapshot_version=%s.', version) + log.error('Could not fetch maven-metadata.xml. Assuming snapshot_version=%s.', version) snapshot_version = version group_url = __get_group_id_subpath(group_id) From cfddbf1c75bb8aad1e3ff8f6f422457b9de96172 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 17 Aug 2017 16:47:45 +0300 Subject: [PATCH 350/508] Apply code review: update the doc --- doc/ref/cli/salt-cp.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/ref/cli/salt-cp.rst b/doc/ref/cli/salt-cp.rst index b3a9a1a6ae..f32af68c77 100644 --- a/doc/ref/cli/salt-cp.rst +++ b/doc/ref/cli/salt-cp.rst @@ -40,10 +40,11 @@ specified target expression. supported, allowing for entire directories to be copied. .. versionchanged:: 2016.11.7,2017.7.2 - Returned the old mode when salt-cp copies all files in one command back. - The new chunked mode is available with the new ``-C``, ``--chunked`` argument. - The compression and directories copying and large files support is available - in the chunked mode only. + Reverted back to the old copy mode to preserve backward compatibility. The + new functionality added in 2016.6.6 and 2017.7.0 is now available using the + ``-C`` or ``--chunked`` CLI arguments. Note that compression, recursive + copying, and support for copying large files is only available in chunked + mode. Options ======= From b8eee4401e5ea478ddb1b7843a49c3f27d1c5657 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Thu, 17 Aug 2017 10:55:53 -0400 Subject: [PATCH 351/508] Change AF_INET6 family for mac in test_host_to_ips --- tests/unit/utils/network_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/network_test.py b/tests/unit/utils/network_test.py index 7a3d278f66..2884d6211e 100644 --- a/tests/unit/utils/network_test.py +++ b/tests/unit/utils/network_test.py @@ -12,6 +12,7 @@ ensure_in_syspath('../../') # Import salt libs from salt.utils import network +import salt.utils LINUX = '''\ eth0 Link encap:Ethernet HWaddr e0:3f:49:85:6a:af @@ -114,13 +115,14 @@ class NetworkTestCase(TestCase): ''' def _side_effect(host, *args): try: + ipv6_fam = 30 if salt.utils.is_darwin() else 10 return { 'github.com': [ (2, 1, 6, '', ('192.30.255.112', 0)), (2, 1, 6, '', ('192.30.255.113', 0)), ], 'ipv6host.foo': [ - (10, 1, 6, '', ('2001:a71::1', 0, 0, 0)), + (ipv6_fam, 1, 6, '', ('2001:a71::1', 0, 0, 0)), ], }[host] except KeyError: From 21c264fe5503ba91be8d50dfde68f59d1df449e4 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 17 Aug 2017 08:55:54 -0600 Subject: [PATCH 352/508] service should return false on exception --- salt/states/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/service.py b/salt/states/service.py index 7b68ee0040..ff5300df6a 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -893,7 +893,7 @@ def mod_watch(name, try: result = func(name, **func_kwargs) except CommandExecutionError as exc: - ret['result'] = True + ret['result'] = False ret['comment'] = exc.strerror return ret From f096917a0e313d713c96fc54e29298a84678376a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 10:23:37 -0500 Subject: [PATCH 353/508] Raise an exception if we fail to cache the config xml --- salt/modules/jenkins.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/salt/modules/jenkins.py b/salt/modules/jenkins.py index 2b68b8ec1a..4844bc313f 100644 --- a/salt/modules/jenkins.py +++ b/salt/modules/jenkins.py @@ -35,7 +35,7 @@ import salt.utils # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -from salt.exceptions import SaltInvocationError +from salt.exceptions import CommandExecutionError, SaltInvocationError # pylint: enable=import-error,no-name-in-module log = logging.getLogger(__name__) @@ -87,6 +87,19 @@ def _connect(): password=jenkins_password) +def _retrieve_config_xml(config_xml, saltenv): + ''' + Helper to cache the config XML and raise a CommandExecutionError if we fail + to do so. If we successfully cache the file, return the cached path. + ''' + ret = __salt__['cp.cache_file'](config_xml, saltenv) + + if not ret: + raise CommandExecutionError('Failed to retrieve {0}'.format(config_xml)) + + return ret + + def get_version(): ''' Return version of Jenkins @@ -240,7 +253,7 @@ def create_job(name=None, if not config_xml: config_xml = jenkins.EMPTY_CONFIG_XML else: - config_xml_file = __salt__['cp.cache_file'](config_xml, saltenv) + config_xml_file = _retrieve_config_xml(config_xml, saltenv) with salt.utils.fopen(config_xml_file) as _fp: config_xml = _fp.read() @@ -279,7 +292,7 @@ def update_job(name=None, if not config_xml: config_xml = jenkins.EMPTY_CONFIG_XML else: - config_xml_file = __salt__['cp.cache_file'](config_xml, saltenv) + config_xml_file = _retrieve_config_xml(config_xml, saltenv) with salt.utils.fopen(config_xml_file) as _fp: config_xml = _fp.read() From 91b583b493b29a1f20aa04519cec714987c1e35e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 10:38:50 -0500 Subject: [PATCH 354/508] Improve and correct execption raising SaltInvocationError is for invalid arguments, we should be using CommandExecutionError elsewhere. Also, we don't use backticks as quotes. Finally, this changes the message in the errors we raise when the Jenkins bindings raise an exception so that it's a bit more descriptive than "Something went wrong". --- salt/modules/jenkins.py | 62 ++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/salt/modules/jenkins.py b/salt/modules/jenkins.py index 4844bc313f..6c4f251a76 100644 --- a/salt/modules/jenkins.py +++ b/salt/modules/jenkins.py @@ -157,7 +157,7 @@ def job_exists(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if server.job_exists(name): @@ -181,12 +181,12 @@ def get_job_info(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exist.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) job_info = server.get_job_info(name) if job_info: @@ -210,17 +210,19 @@ def build_job(name=None, parameters=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exist.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist.'.format(name)) try: server.build_job(name, parameters) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error building job \'{0}\': {1}'.format(name, err) + ) return True @@ -245,10 +247,10 @@ def create_job(name=None, ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') if job_exists(name): - raise SaltInvocationError('Job `{0}` already exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' already exists'.format(name)) if not config_xml: config_xml = jenkins.EMPTY_CONFIG_XML @@ -262,7 +264,9 @@ def create_job(name=None, try: server.create_job(name, config_xml) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error creating job \'{0}\': {1}'.format(name, err) + ) return config_xml @@ -287,7 +291,7 @@ def update_job(name=None, ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') if not config_xml: config_xml = jenkins.EMPTY_CONFIG_XML @@ -301,7 +305,9 @@ def update_job(name=None, try: server.reconfig_job(name, config_xml) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error updating job \'{0}\': {1}'.format(name, err) + ) return config_xml @@ -320,17 +326,19 @@ def delete_job(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) try: server.delete_job(name) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error deleting job \'{0}\': {1}'.format(name, err) + ) return True @@ -349,17 +357,19 @@ def enable_job(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) try: server.enable_job(name) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error enabling job \'{0}\': {1}'.format(name, err) + ) return True @@ -379,17 +389,19 @@ def disable_job(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) try: server.disable_job(name) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error disabling job \'{0}\': {1}'.format(name, err) + ) return True @@ -409,12 +421,12 @@ def job_status(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) return server.get_job_info('empty')['buildable'] @@ -435,12 +447,12 @@ def get_job_config(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() - if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + job_exists(name) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) job_info = server.get_job_config(name) return job_info From 2f762b3a17e25fdad00ba1cc6bb1544f48e9e332 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 17 Aug 2017 12:31:25 -0400 Subject: [PATCH 355/508] Update bootstrap script to latest stable: v2017.08.17 --- salt/cloud/deploy/bootstrap-salt.sh | 1506 +++++++++++++-------------- 1 file changed, 727 insertions(+), 779 deletions(-) diff --git a/salt/cloud/deploy/bootstrap-salt.sh b/salt/cloud/deploy/bootstrap-salt.sh index ededd58508..ddda8e6398 100755 --- a/salt/cloud/deploy/bootstrap-salt.sh +++ b/salt/cloud/deploy/bootstrap-salt.sh @@ -18,7 +18,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2017.05.24" +__ScriptVersion="2017.08.17" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -121,8 +121,7 @@ __check_command_exists() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __check_pip_allowed -# DESCRIPTION: Simple function to let the users know that -P needs to be -# used. +# DESCRIPTION: Simple function to let the users know that -P needs to be used. #---------------------------------------------------------------------------------------------------------------------- __check_pip_allowed() { if [ $# -eq 1 ]; then @@ -155,10 +154,16 @@ __check_config_dir() { __fetch_url "/tmp/${CC_DIR_BASE}" "${CC_DIR_NAME}" CC_DIR_NAME="/tmp/${CC_DIR_BASE}" ;; + *://*) + echoerror "Unsupported URI scheme for $CC_DIR_NAME" + echo "null" + return + ;; *) if [ ! -e "${CC_DIR_NAME}" ]; then + echoerror "The configuration directory or archive $CC_DIR_NAME does not exist." echo "null" - return 0 + return fi ;; esac @@ -187,6 +192,28 @@ __check_config_dir() { echo "${CC_DIR_NAME}" } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_unparsed_options +# DESCRIPTION: Checks the placed after the install arguments +#---------------------------------------------------------------------------------------------------------------------- +__check_unparsed_options() { + shellopts="$1" + # grep alternative for SunOS + if [ -f /usr/xpg4/bin/grep ]; then + grep='/usr/xpg4/bin/grep' + else + grep='grep' + fi + unparsed_options=$( echo "$shellopts" | ${grep} -E '(^|[[:space:]])[-]+[[:alnum:]]' ) + if [ "$unparsed_options" != "" ]; then + __usage + echo + echoerror "options are only allowed before install arguments" + echo + exit 1 + fi +} + #---------------------------------------------------------------------------------------------------------------------- # Handle command line arguments @@ -243,6 +270,10 @@ _REPO_URL="repo.saltstack.com" _PY_EXE="" _INSTALL_PY="$BS_FALSE" +# Defaults for install arguments +ITYPE="stable" + + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __usage # DESCRIPTION: Display usage information. @@ -380,19 +411,7 @@ do v ) echo "$0 -- Version $__ScriptVersion"; exit 0 ;; n ) _COLORS=0; __detect_color_support ;; D ) _ECHO_DEBUG=$BS_TRUE ;; - - c ) _TEMP_CONFIG_DIR=$(__check_config_dir "$OPTARG") - # If the configuration directory does not exist, error out - if [ "$_TEMP_CONFIG_DIR" = "null" ]; then - echoerror "Unsupported URI scheme for $OPTARG" - exit 1 - fi - if [ ! -d "$_TEMP_CONFIG_DIR" ]; then - echoerror "The configuration directory ${_TEMP_CONFIG_DIR} does not exist." - exit 1 - fi - ;; - + c ) _TEMP_CONFIG_DIR="$OPTARG" ;; g ) _SALT_REPO_URL=$OPTARG ;; G ) echowarn "The '-G' option is DEPRECATED and will be removed in the future stable release!" @@ -401,14 +420,7 @@ do ;; w ) _DOWNSTREAM_PKG_REPO=$BS_TRUE ;; - - k ) _TEMP_KEYS_DIR="$OPTARG" - # If the configuration directory does not exist, error out - if [ ! -d "$_TEMP_KEYS_DIR" ]; then - echoerror "The pre-seed keys directory ${_TEMP_KEYS_DIR} does not exist." - exit 1 - fi - ;; + k ) _TEMP_KEYS_DIR="$OPTARG" ;; s ) _SLEEP=$OPTARG ;; M ) _INSTALL_MASTER=$BS_TRUE ;; S ) _INSTALL_SYNDIC=$BS_TRUE ;; @@ -451,205 +463,28 @@ done shift $((OPTIND-1)) -__check_unparsed_options() { - shellopts="$1" - # grep alternative for SunOS - if [ -f /usr/xpg4/bin/grep ]; then - grep='/usr/xpg4/bin/grep' - else - grep='grep' - fi - unparsed_options=$( echo "$shellopts" | ${grep} -E '(^|[[:space:]])[-]+[[:alnum:]]' ) - if [ "$unparsed_options" != "" ]; then - __usage - echo - echoerror "options are only allowed before install arguments" - echo - exit 1 - fi -} +# Define our logging file and pipe paths +LOGFILE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.log/g )" +LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )" - -# Check that we're actually installing one of minion/master/syndic -if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then - echowarn "Nothing to install or configure" - exit 0 -fi - -# Check that we're installing a minion if we're being passed a master address -if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MASTER_ADDRESS" != "null" ]; then - echoerror "Don't pass a master address (-A) if no minion is going to be bootstrapped." +# Create our logging pipe +# On FreeBSD we have to use mkfifo instead of mknod +mknod "$LOGPIPE" p >/dev/null 2>&1 || mkfifo "$LOGPIPE" >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echoerror "Failed to create the named pipe required to log" exit 1 fi -# Check that we're installing a minion if we're being passed a minion id -if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MINION_ID" != "null" ]; then - echoerror "Don't pass a minion id (-i) if no minion is going to be bootstrapped." - exit 1 -fi +# What ever is written to the logpipe gets written to the logfile +tee < "$LOGPIPE" "$LOGFILE" & -# Check that we're installing or configuring a master if we're being passed a master config json dict -if [ "$_CUSTOM_MASTER_CONFIG" != "null" ]; then - if [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then - echoerror "Don't pass a master config JSON dict (-J) if no master is going to be bootstrapped or configured." - exit 1 - fi -fi +# Close STDOUT, reopen it directing it to the logpipe +exec 1>&- +exec 1>"$LOGPIPE" +# Close STDERR, reopen it directing it to the logpipe +exec 2>&- +exec 2>"$LOGPIPE" -# Check that we're installing or configuring a minion if we're being passed a minion config json dict -if [ "$_CUSTOM_MINION_CONFIG" != "null" ]; then - if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then - echoerror "Don't pass a minion config JSON dict (-j) if no minion is going to be bootstrapped or configured." - exit 1 - fi -fi - -# Define installation type -if [ "$#" -eq 0 ];then - ITYPE="stable" -else - __check_unparsed_options "$*" - ITYPE=$1 - shift -fi - -# Check installation type -if [ "$(echo "$ITYPE" | egrep '(stable|testing|daily|git)')" = "" ]; then - echoerror "Installation type \"$ITYPE\" is not known..." - exit 1 -fi - -# If doing a git install, check what branch/tag/sha will be checked out -if [ "$ITYPE" = "git" ]; then - if [ "$#" -eq 0 ];then - GIT_REV="develop" - else - __check_unparsed_options "$*" - GIT_REV="$1" - shift - fi - - # Disable shell warning about unbound variable during git install - STABLE_REV="latest" - -# If doing stable install, check if version specified -elif [ "$ITYPE" = "stable" ]; then - if [ "$#" -eq 0 ];then - STABLE_REV="latest" - else - __check_unparsed_options "$*" - - if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11)$')" != "" ]; then - STABLE_REV="$1" - shift - elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then - STABLE_REV="archive/$1" - shift - else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, latest, \$MAJOR.\$MINOR.\$PATCH)" - exit 1 - fi - fi -fi - -# -a and -V only work from git -if [ "$ITYPE" != "git" ]; then - if [ $_PIP_ALL -eq $BS_TRUE ]; then - echoerror "Pip installing all python packages with -a is only possible when installing Salt via git" - exit 1 - fi - if [ "$_VIRTUALENV_DIR" != "null" ]; then - echoerror "Virtualenv installs via -V is only possible when installing Salt via git" - exit 1 - fi -fi - -# Set the _REPO_URL value based on if -R was passed or not. Defaults to repo.saltstack.com. -if [ "$_CUSTOM_REPO_URL" != "null" ]; then - _REPO_URL="$_CUSTOM_REPO_URL" - - # Check for -r since -R is being passed. Set -r with a warning. - if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then - echowarn "Detected -R option. No other repositories will be configured when -R is used. Setting -r option to True." - _DISABLE_REPOS=$BS_TRUE - fi -fi - -# Check for any unparsed arguments. Should be an error. -if [ "$#" -gt 0 ]; then - __check_unparsed_options "$*" - __usage - echo - echoerror "Too many arguments." - exit 1 -fi - -# Check the _DISABLE_SSL value and set HTTP or HTTPS. -if [ "$_DISABLE_SSL" -eq $BS_TRUE ]; then - HTTP_VAL="http" -else - HTTP_VAL="https" -fi - -# Check the _QUIET_GIT_INSTALLATION value and set SETUP_PY_INSTALL_ARGS. -if [ "$_QUIET_GIT_INSTALLATION" -eq $BS_TRUE ]; then - SETUP_PY_INSTALL_ARGS="-q" -else - SETUP_PY_INSTALL_ARGS="" -fi - -# whoami alternative for SunOS -if [ -f /usr/xpg4/bin/id ]; then - whoami='/usr/xpg4/bin/id -un' -else - whoami='whoami' -fi - -# Root permissions are required to run this script -if [ "$($whoami)" != "root" ]; then - echoerror "Salt requires root privileges to install. Please re-run this script as root." - exit 1 -fi - -# Export the http_proxy configuration to our current environment -if [ "${_HTTP_PROXY}" != "" ]; then - export http_proxy="$_HTTP_PROXY" - export https_proxy="$_HTTP_PROXY" -fi - -# Let's discover how we're being called -# shellcheck disable=SC2009 -CALLER=$(ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' ' | cut -d ' ' -f 3) - -if [ "${CALLER}x" = "${0}x" ]; then - CALLER="shell pipe" -fi - -# Work around for 'Docker + salt-bootstrap failure' https://github.com/saltstack/salt-bootstrap/issues/394 -if [ "${_DISABLE_SALT_CHECKS}" -eq $BS_FALSE ] && [ -f /tmp/disable_salt_checks ]; then - # shellcheck disable=SC2016 - echowarn 'Found file: /tmp/disable_salt_checks, setting _DISABLE_SALT_CHECKS=$BS_TRUE' - _DISABLE_SALT_CHECKS=$BS_TRUE -fi - -# Because -a can only be installed into virtualenv -if [ "${_PIP_ALL}" -eq $BS_TRUE ] && [ "${_VIRTUALENV_DIR}" = "null" ]; then - usage - # Could possibly set up a default virtualenv location when -a flag is passed - echoerror "Using -a requires -V because pip pkgs should be siloed from python system pkgs" - exit 1 -fi - -# Make sure virtualenv directory does not already exist -if [ -d "${_VIRTUALENV_DIR}" ]; then - echoerror "The directory ${_VIRTUALENV_DIR} for virtualenv already exists" - exit 1 -fi - -echoinfo "Running version: ${__ScriptVersion}" -echoinfo "Executed by: ${CALLER}" -echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'" -#echowarn "Running the unstable version of ${__ScriptName}" #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __exit_cleanup @@ -684,8 +519,10 @@ __exit_cleanup() { fi # Remove the logging pipe when the script exits - echodebug "Removing the logging pipe $LOGPIPE" - rm -f "$LOGPIPE" + if [ -p "$LOGPIPE" ]; then + echodebug "Removing the logging pipe $LOGPIPE" + rm -f "$LOGPIPE" + fi # Kill tee when exiting, CentOS, at least requires this # shellcheck disable=SC2009 @@ -713,28 +550,192 @@ __exit_cleanup() { trap "__exit_cleanup" EXIT INT -# Define our logging file and pipe paths -LOGFILE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.log/g )" -LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )" +# Let's discover how we're being called +# shellcheck disable=SC2009 +CALLER=$(ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' ' | cut -d ' ' -f 3) -# Create our logging pipe -# On FreeBSD we have to use mkfifo instead of mknod -mknod "$LOGPIPE" p >/dev/null 2>&1 || mkfifo "$LOGPIPE" >/dev/null 2>&1 -if [ $? -ne 0 ]; then - echoerror "Failed to create the named pipe required to log" +if [ "${CALLER}x" = "${0}x" ]; then + CALLER="shell pipe" +fi + +echoinfo "Running version: ${__ScriptVersion}" +echoinfo "Executed by: ${CALLER}" +echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'" +#echowarn "Running the unstable version of ${__ScriptName}" + +# Define installation type +if [ "$#" -gt 0 ];then + __check_unparsed_options "$*" + ITYPE=$1 + shift +fi + +# Check installation type +if [ "$(echo "$ITYPE" | egrep '(stable|testing|daily|git)')" = "" ]; then + echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi -# What ever is written to the logpipe gets written to the logfile -tee < "$LOGPIPE" "$LOGFILE" & +# If doing a git install, check what branch/tag/sha will be checked out +if [ "$ITYPE" = "git" ]; then + if [ "$#" -eq 0 ];then + GIT_REV="develop" + else + GIT_REV="$1" + shift + fi -# Close STDOUT, reopen it directing it to the logpipe -exec 1>&- -exec 1>"$LOGPIPE" -# Close STDERR, reopen it directing it to the logpipe -exec 2>&- -exec 2>"$LOGPIPE" + # Disable shell warning about unbound variable during git install + STABLE_REV="latest" +# If doing stable install, check if version specified +elif [ "$ITYPE" = "stable" ]; then + if [ "$#" -eq 0 ];then + STABLE_REV="latest" + else + if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7)$')" != "" ]; then + STABLE_REV="$1" + shift + elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then + STABLE_REV="archive/$1" + shift + else + echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, latest, \$MAJOR.\$MINOR.\$PATCH)" + exit 1 + fi + fi +fi + +# Check for any unparsed arguments. Should be an error. +if [ "$#" -gt 0 ]; then + __usage + echo + echoerror "Too many arguments." + exit 1 +fi + +# whoami alternative for SunOS +if [ -f /usr/xpg4/bin/id ]; then + whoami='/usr/xpg4/bin/id -un' +else + whoami='whoami' +fi + +# Root permissions are required to run this script +if [ "$($whoami)" != "root" ]; then + echoerror "Salt requires root privileges to install. Please re-run this script as root." + exit 1 +fi + +# Check that we're actually installing one of minion/master/syndic +if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then + echowarn "Nothing to install or configure" + exit 1 +fi + +# Check that we're installing a minion if we're being passed a master address +if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MASTER_ADDRESS" != "null" ]; then + echoerror "Don't pass a master address (-A) if no minion is going to be bootstrapped." + exit 1 +fi + +# Check that we're installing a minion if we're being passed a minion id +if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MINION_ID" != "null" ]; then + echoerror "Don't pass a minion id (-i) if no minion is going to be bootstrapped." + exit 1 +fi + +# Check that we're installing or configuring a master if we're being passed a master config json dict +if [ "$_CUSTOM_MASTER_CONFIG" != "null" ]; then + if [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then + echoerror "Don't pass a master config JSON dict (-J) if no master is going to be bootstrapped or configured." + exit 1 + fi +fi + +# Check that we're installing or configuring a minion if we're being passed a minion config json dict +if [ "$_CUSTOM_MINION_CONFIG" != "null" ]; then + if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then + echoerror "Don't pass a minion config JSON dict (-j) if no minion is going to be bootstrapped or configured." + exit 1 + fi +fi + +# If the configuration directory or archive does not exist, error out +if [ "$_TEMP_CONFIG_DIR" != "null" ]; then + _TEMP_CONFIG_DIR="$(__check_config_dir "$_TEMP_CONFIG_DIR")" + [ "$_TEMP_CONFIG_DIR" = "null" ] && exit 1 +fi + +# If the pre-seed keys directory does not exist, error out +if [ "$_TEMP_KEYS_DIR" != "null" ] && [ ! -d "$_TEMP_KEYS_DIR" ]; then + echoerror "The pre-seed keys directory ${_TEMP_KEYS_DIR} does not exist." + exit 1 +fi + +# -a and -V only work from git +if [ "$ITYPE" != "git" ]; then + if [ $_PIP_ALL -eq $BS_TRUE ]; then + echoerror "Pip installing all python packages with -a is only possible when installing Salt via git" + exit 1 + fi + if [ "$_VIRTUALENV_DIR" != "null" ]; then + echoerror "Virtualenv installs via -V is only possible when installing Salt via git" + exit 1 + fi +fi + +# Set the _REPO_URL value based on if -R was passed or not. Defaults to repo.saltstack.com. +if [ "$_CUSTOM_REPO_URL" != "null" ]; then + _REPO_URL="$_CUSTOM_REPO_URL" + + # Check for -r since -R is being passed. Set -r with a warning. + if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then + echowarn "Detected -R option. No other repositories will be configured when -R is used. Setting -r option to True." + _DISABLE_REPOS=$BS_TRUE + fi +fi + +# Check the _DISABLE_SSL value and set HTTP or HTTPS. +if [ "$_DISABLE_SSL" -eq $BS_TRUE ]; then + HTTP_VAL="http" +else + HTTP_VAL="https" +fi + +# Check the _QUIET_GIT_INSTALLATION value and set SETUP_PY_INSTALL_ARGS. +if [ "$_QUIET_GIT_INSTALLATION" -eq $BS_TRUE ]; then + SETUP_PY_INSTALL_ARGS="-q" +else + SETUP_PY_INSTALL_ARGS="" +fi + +# Export the http_proxy configuration to our current environment +if [ "${_HTTP_PROXY}" != "" ]; then + export http_proxy="$_HTTP_PROXY" + export https_proxy="$_HTTP_PROXY" +fi + +# Work around for 'Docker + salt-bootstrap failure' https://github.com/saltstack/salt-bootstrap/issues/394 +if [ "${_DISABLE_SALT_CHECKS}" -eq $BS_FALSE ] && [ -f /tmp/disable_salt_checks ]; then + # shellcheck disable=SC2016 + echowarn 'Found file: /tmp/disable_salt_checks, setting _DISABLE_SALT_CHECKS=$BS_TRUE' + _DISABLE_SALT_CHECKS=$BS_TRUE +fi + +# Because -a can only be installed into virtualenv +if [ "${_PIP_ALL}" -eq $BS_TRUE ] && [ "${_VIRTUALENV_DIR}" = "null" ]; then + usage + # Could possibly set up a default virtualenv location when -a flag is passed + echoerror "Using -a requires -V because pip pkgs should be siloed from python system pkgs" + exit 1 +fi + +# Make sure virtualenv directory does not already exist +if [ -d "${_VIRTUALENV_DIR}" ]; then + echoerror "The directory ${_VIRTUALENV_DIR} for virtualenv already exists" + exit 1 +fi # Handle the insecure flags if [ "$_INSECURE_DL" -eq $BS_TRUE ]; then @@ -856,8 +857,10 @@ __derive_debian_numeric_version() { elif [ "$INPUT_VERSION" = "jessie/sid" ]; then NUMERIC_VERSION=$(__parse_version_string "8.0") elif [ "$INPUT_VERSION" = "stretch/sid" ]; then - # Let's start detecting the upcoming Debian 9 (Stretch) NUMERIC_VERSION=$(__parse_version_string "9.0") + elif [ "$INPUT_VERSION" = "buster/sid" ]; then + # Let's start detecting the upcoming Debian 10 (Buster) release + NUMERIC_VERSION=$(__parse_version_string "10.0") else echowarn "Unable to parse the Debian Version (codename: '$INPUT_VERSION')" fi @@ -1089,11 +1092,11 @@ __gather_linux_system_info() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __install_python_and_deps() -# DESCRIPTION: Install a different version of python and its dependencies on a host. Currently this has only been -# tested on Centos 6 and is considered experimental. +# NAME: __install_python() +# DESCRIPTION: Install a different version of python on a host. Currently this has only been tested on CentOS 6 and +# is considered experimental. #---------------------------------------------------------------------------------------------------------------------- -__install_python_and_deps() { +__install_python() { if [ "$_PY_EXE" = "" ]; then echoerror "Must specify -x with -y to install a specific python version" exit 1 @@ -1103,7 +1106,7 @@ __install_python_and_deps() { __PACKAGES="${PY_PKG_V}" - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then + if [ ${_DISABLE_REPOS} -eq ${BS_FALSE} ]; then echoinfo "Attempting to install a repo to help provide a separate python package" echoinfo "$DISTRO_NAME_L" case "$DISTRO_NAME_L" in @@ -1112,7 +1115,7 @@ __install_python_and_deps() { ;; *) echoerror "Installing a repo to provide a python package is only supported on Redhat/CentOS. - If a repo is already available please try running script with -r" + If a repo is already available, please try running script with -r." exit 1 ;; esac @@ -1123,13 +1126,6 @@ __install_python_and_deps() { echoinfo "Installing ${__PACKAGES}" __yum_install_noinput "${__PACKAGES}" || return 1 - - _PIP_PACKAGES="tornado PyYAML msgpack-python jinja2 pycrypto zmq" - if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - _PIP_PACKAGES="${_PIP_PACKAGES} apache-libcloud" - fi - - __install_pip_pkgs "${_PIP_PACKAGES}" "${_PY_EXE}" || return 1 } @@ -1243,22 +1239,6 @@ __gather_system_info() { } -#--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __get_dpkg_architecture -# DESCRIPTION: Determine primary architecture for packages to install on Debian and derivatives -#---------------------------------------------------------------------------------------------------------------------- -__get_dpkg_architecture() { - if __check_command_exists dpkg; then - DPKG_ARCHITECTURE="$(dpkg --print-architecture)" - else - echoerror "dpkg: command not found." - return 1 - fi - - return 0 -} - - #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __ubuntu_derivatives_translation # DESCRIPTION: Map Ubuntu derivatives to their Ubuntu base versions. @@ -1305,6 +1285,63 @@ __ubuntu_derivatives_translation() { } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_dpkg_architecture +# DESCRIPTION: Determine the primary architecture for packages to install on Debian and derivatives +# and issue all necessary error messages. +#---------------------------------------------------------------------------------------------------------------------- +__check_dpkg_architecture() { + if __check_command_exists dpkg; then + DPKG_ARCHITECTURE="$(dpkg --print-architecture)" + else + echoerror "dpkg: command not found." + return 1 + fi + + __REPO_ARCH="$DPKG_ARCHITECTURE" + __return_code=0 + + case $DPKG_ARCHITECTURE in + "i386") + error_msg="$_REPO_URL likely doesn't have all required 32-bit packages for $DISTRO_NAME $DISTRO_MAJOR_VERSION." + # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location + __REPO_ARCH="amd64" + ;; + "amd64") + error_msg="" + ;; + "armhf") + if [ "$DISTRO_NAME_L" = "ubuntu" ] || [ "$DISTRO_MAJOR_VERSION" -lt 8 ]; then + error_msg="Support for armhf packages at $_REPO_URL is limited to Debian/Raspbian 8 platforms." + __return_code=1 + else + error_msg="" + fi + ;; + *) + error_msg="$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." + __return_code=1 + ;; + esac + + if [ "${error_msg}" != "" ]; then + echoerror "${error_msg}" + if [ "$ITYPE" != "git" ]; then + echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.11.5." + echoerror "It may be necessary to use git installation mode with pip and disable the SaltStack apt repository." + echoerror "For example:" + echoerror " sh ${__ScriptName} -r -P git v2016.11.5" + fi + fi + + if [ "${__return_code}" -eq 0 ]; then + return 0 + else + return 1 + fi +} + + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __ubuntu_codename_translation # DESCRIPTION: Map Ubuntu major versions to their corresponding codenames @@ -1403,18 +1440,172 @@ __debian_derivatives_translation() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __check_and_refresh_suse_pkg_repo -# DESCRIPTION: Check if zypper knows about systemsmanagement_saltstack repo yet. -# If it doesn't, add the repo and refresh with the SUSE_PKG_URL. +# NAME: __debian_codename_translation +# DESCRIPTION: Map Debian major versions to their corresponding code names #---------------------------------------------------------------------------------------------------------------------- -__check_and_refresh_suse_pkg_repo() { - # Check to see if systemsmanagement_saltstack exists - __zypper repos | grep systemsmanagement_saltstack >/dev/null 2>&1 +# shellcheck disable=SC2034 +__debian_codename_translation() { - if [ $? -eq 1 ]; then - # zypper does not yet know anything about systemsmanagement_saltstack - __zypper addrepo --refresh "${SUSE_PKG_URL}" || return 1 - fi + case $DISTRO_MAJOR_VERSION in + "7") + DISTRO_CODENAME="wheezy" + ;; + "8") + DISTRO_CODENAME="jessie" + ;; + "9") + DISTRO_CODENAME="stretch" + ;; + "10") + DISTRO_CODENAME="buster" + ;; + *) + DISTRO_CODENAME="jessie" + ;; + esac +} + + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_end_of_life_versions +# DESCRIPTION: Check for end of life distribution versions +#---------------------------------------------------------------------------------------------------------------------- +__check_end_of_life_versions() { + case "${DISTRO_NAME_L}" in + debian) + # Debian versions below 7 are not supported + if [ "$DISTRO_MAJOR_VERSION" -lt 7 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://wiki.debian.org/DebianReleases" + exit 1 + fi + ;; + + ubuntu) + # Ubuntu versions not supported + # + # < 14.04 + # = 14.10 + # = 15.04, 15.10 + if [ "$DISTRO_MAJOR_VERSION" -lt 14 ] || \ + [ "$DISTRO_MAJOR_VERSION" -eq 15 ] || \ + ([ "$DISTRO_MAJOR_VERSION" -lt 16 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]); then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://wiki.ubuntu.com/Releases" + exit 1 + fi + ;; + + opensuse) + # openSUSE versions not supported + # + # <= 12.1 + if ([ "$DISTRO_MAJOR_VERSION" -eq 12 ] && [ "$DISTRO_MINOR_VERSION" -eq 1 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 12 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " http://en.opensuse.org/Lifetime" + exit 1 + fi + ;; + + suse) + # SuSE versions not supported + # + # < 11 SP2 + SUSE_PATCHLEVEL=$(awk '/PATCHLEVEL/ {print $3}' /etc/SuSE-release ) + if [ "${SUSE_PATCHLEVEL}" = "" ]; then + SUSE_PATCHLEVEL="00" + fi + if ([ "$DISTRO_MAJOR_VERSION" -eq 11 ] && [ "$SUSE_PATCHLEVEL" -lt 02 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 11 ]; then + echoerror "Versions lower than SuSE 11 SP2 are not supported." + echoerror "Please consider upgrading to the next stable" + exit 1 + fi + ;; + + fedora) + # Fedora lower than 24 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 24 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://fedoraproject.org/wiki/Releases" + exit 1 + fi + ;; + + centos) + # CentOS versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " http://wiki.centos.org/Download" + exit 1 + fi + ;; + + red_hat*linux) + # Red Hat (Enterprise) Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://access.redhat.com/support/policy/updates/errata/" + exit 1 + fi + ;; + + oracle*linux) + # Oracle Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " http://www.oracle.com/us/support/library/elsp-lifetime-069338.pdf" + exit 1 + fi + ;; + + scientific*linux) + # Scientific Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://www.scientificlinux.org/downloads/sl-versions/" + exit 1 + fi + ;; + + cloud*linux) + # Cloud Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://docs.cloudlinux.com/index.html?cloudlinux_life-cycle.html" + exit 1 + fi + ;; + + amazon*linux*ami) + # Amazon Linux versions lower than 2012.0X no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 2012 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://aws.amazon.com/amazon-linux-ami/" + exit 1 + fi + ;; + + freebsd) + # FreeBSD versions lower than 9.1 are not supported. + if ([ "$DISTRO_MAJOR_VERSION" -eq 9 ] && [ "$DISTRO_MINOR_VERSION" -lt 01 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 9 ]; then + echoerror "Versions lower than FreeBSD 9.1 are not supported." + exit 1 + fi + ;; + + *) + ;; + esac } @@ -1429,6 +1620,37 @@ echoinfo " OS Version: ${OS_VERSION}" echoinfo " Distribution: ${DISTRO_NAME} ${DISTRO_VERSION}" echo +# Simplify distro name naming on functions +DISTRO_NAME_L=$(echo "$DISTRO_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-zA-Z0-9_ ]//g' | sed -re 's/([[:space:]])+/_/g') + +# Simplify version naming on functions +if [ "$DISTRO_VERSION" = "" ] || [ ${_SIMPLIFY_VERSION} -eq $BS_FALSE ]; then + DISTRO_MAJOR_VERSION="" + DISTRO_MINOR_VERSION="" + PREFIXED_DISTRO_MAJOR_VERSION="" + PREFIXED_DISTRO_MINOR_VERSION="" +else + DISTRO_MAJOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).*/\1/g') + DISTRO_MINOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).\([0-9]*\).*/\2/g') + PREFIXED_DISTRO_MAJOR_VERSION="_${DISTRO_MAJOR_VERSION}" + if [ "${PREFIXED_DISTRO_MAJOR_VERSION}" = "_" ]; then + PREFIXED_DISTRO_MAJOR_VERSION="" + fi + PREFIXED_DISTRO_MINOR_VERSION="_${DISTRO_MINOR_VERSION}" + if [ "${PREFIXED_DISTRO_MINOR_VERSION}" = "_" ]; then + PREFIXED_DISTRO_MINOR_VERSION="" + fi +fi + +# For Ubuntu derivatives, pretend to be their Ubuntu base version +__ubuntu_derivatives_translation + +# For Debian derivates, pretend to be their Debian base version +__debian_derivatives_translation + +# Fail soon for end of life versions +__check_end_of_life_versions + echodebug "Binaries will be searched using the following \$PATH: ${PATH}" # Let users know that we'll use a proxy @@ -1469,37 +1691,14 @@ if [ $_START_DAEMONS -eq $BS_FALSE ]; then echoinfo "Daemons will not be started" fi -# Simplify distro name naming on functions -DISTRO_NAME_L=$(echo "$DISTRO_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-zA-Z0-9_ ]//g' | sed -re 's/([[:space:]])+/_/g') - -# For Ubuntu derivatives, pretend to be their Ubuntu base version -__ubuntu_derivatives_translation - -# For Debian derivates, pretend to be their Debian base version -__debian_derivatives_translation - -# Simplify version naming on functions -if [ "$DISTRO_VERSION" = "" ] || [ ${_SIMPLIFY_VERSION} -eq $BS_FALSE ]; then - DISTRO_MAJOR_VERSION="" - DISTRO_MINOR_VERSION="" - PREFIXED_DISTRO_MAJOR_VERSION="" - PREFIXED_DISTRO_MINOR_VERSION="" -else - DISTRO_MAJOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).*/\1/g') - DISTRO_MINOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).\([0-9]*\).*/\2/g') - PREFIXED_DISTRO_MAJOR_VERSION="_${DISTRO_MAJOR_VERSION}" - if [ "${PREFIXED_DISTRO_MAJOR_VERSION}" = "_" ]; then - PREFIXED_DISTRO_MAJOR_VERSION="" - fi - PREFIXED_DISTRO_MINOR_VERSION="_${DISTRO_MINOR_VERSION}" - if [ "${PREFIXED_DISTRO_MINOR_VERSION}" = "_" ]; then - PREFIXED_DISTRO_MINOR_VERSION="" - fi +if [ "${DISTRO_NAME_L}" = "ubuntu" ]; then + # For ubuntu versions, obtain the codename from the release version + __ubuntu_codename_translation +elif [ "${DISTRO_NAME_L}" = "debian" ]; then + # For debian versions, obtain the codename from the release version + __debian_codename_translation fi -# For ubuntu versions, obtain the codename from the release version -__ubuntu_codename_translation - # Only Ubuntu has daily packages, let's let users know about that if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$ITYPE" = "daily" ]); then echoerror "${DISTRO_NAME} does not have daily packages support" @@ -1525,14 +1724,18 @@ if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$_VIRTUALENV_DIR" != "null" ]); the fi # Only Ubuntu has support for pip installing all packages -if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ $_PIP_ALL -eq $BS_TRUE ]);then +if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ $_PIP_ALL -eq $BS_TRUE ]); then echoerror "${DISTRO_NAME} does not have -a support" exit 1 fi -# Starting from Ubuntu 16.10, gnupg-curl has been renamed to gnupg1-curl. +# Starting from Debian 9 and Ubuntu 16.10, gnupg-curl has been renamed to gnupg1-curl. GNUPG_CURL="gnupg-curl" -if [ "${DISTRO_NAME_L}" = "ubuntu" ]; then +if [ "$DISTRO_NAME_L" = "debian" ]; then + if [ "$DISTRO_MAJOR_VERSION" -gt 8 ]; then + GNUPG_CURL="gnupg1-curl" + fi +elif [ "$DISTRO_NAME_L" = "ubuntu" ]; then if [ "${DISTRO_VERSION}" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then GNUPG_CURL="gnupg1-curl" fi @@ -1624,7 +1827,9 @@ __rpm_import_gpg() { __yum_install_noinput() { ENABLE_EPEL_CMD="" - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then + # Skip Amazon Linux for the first round, since EPEL is no longer required. + # See issue #724 + if [ $_DISABLE_REPOS -eq $BS_FALSE ] && [ "$DISTRO_NAME_L" != "amazon_linux_ami" ]; then ENABLE_EPEL_CMD="--enablerepo=${_EPEL_REPO}" fi @@ -1768,152 +1973,6 @@ __git_clone_and_checkout() { } -#--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __check_end_of_life_versions -# DESCRIPTION: Check for end of life distribution versions -#---------------------------------------------------------------------------------------------------------------------- -__check_end_of_life_versions() { - case "${DISTRO_NAME_L}" in - debian) - # Debian versions bellow 6 are not supported - if [ "$DISTRO_MAJOR_VERSION" -lt 7 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://wiki.debian.org/DebianReleases" - exit 1 - fi - ;; - - ubuntu) - # Ubuntu versions not supported - # - # < 12.04 - # 13.x, 15.x - # 12.10, 14.10 - if [ "$DISTRO_MAJOR_VERSION" -lt 12 ] || \ - [ "$DISTRO_MAJOR_VERSION" -eq 13 ] || \ - [ "$DISTRO_MAJOR_VERSION" -eq 15 ] || \ - ([ "$DISTRO_MAJOR_VERSION" -lt 16 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]); then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://wiki.ubuntu.com/Releases" - exit 1 - fi - ;; - - opensuse) - # openSUSE versions not supported - # - # <= 12.1 - if ([ "$DISTRO_MAJOR_VERSION" -eq 12 ] && [ "$DISTRO_MINOR_VERSION" -eq 1 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 12 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " http://en.opensuse.org/Lifetime" - exit 1 - fi - ;; - - suse) - # SuSE versions not supported - # - # < 11 SP2 - SUSE_PATCHLEVEL=$(awk '/PATCHLEVEL/ {print $3}' /etc/SuSE-release ) - if [ "${SUSE_PATCHLEVEL}" = "" ]; then - SUSE_PATCHLEVEL="00" - fi - if ([ "$DISTRO_MAJOR_VERSION" -eq 11 ] && [ "$SUSE_PATCHLEVEL" -lt 02 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 11 ]; then - echoerror "Versions lower than SuSE 11 SP2 are not supported." - echoerror "Please consider upgrading to the next stable" - exit 1 - fi - ;; - - fedora) - # Fedora lower than 18 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 23 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://fedoraproject.org/wiki/Releases" - exit 1 - fi - ;; - - centos) - # CentOS versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " http://wiki.centos.org/Download" - exit 1 - fi - ;; - - red_hat*linux) - # Red Hat (Enterprise) Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://access.redhat.com/support/policy/updates/errata/" - exit 1 - fi - ;; - - oracle*linux) - # Oracle Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " http://www.oracle.com/us/support/library/elsp-lifetime-069338.pdf" - exit 1 - fi - ;; - - scientific*linux) - # Scientific Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://www.scientificlinux.org/downloads/sl-versions/" - exit 1 - fi - ;; - - cloud*linux) - # Cloud Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://docs.cloudlinux.com/index.html?cloudlinux_life-cycle.html" - exit 1 - fi - ;; - - amazon*linux*ami) - # Amazon Linux versions lower than 2012.0X no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 2012 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://aws.amazon.com/amazon-linux-ami/" - exit 1 - fi - ;; - - freebsd) - # FreeBSD versions lower than 9.1 are not supported. - if ([ "$DISTRO_MAJOR_VERSION" -eq 9 ] && [ "$DISTRO_MINOR_VERSION" -lt 01 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 9 ]; then - echoerror "Versions lower than FreeBSD 9.1 are not supported." - exit 1 - fi - ;; - - *) - ;; - esac -} -# Fail soon for end of life versions -__check_end_of_life_versions - - #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __copyfile # DESCRIPTION: Simple function to copy files. Overrides if asked. @@ -2479,38 +2538,55 @@ __enable_universe_repository() { return 0 } -install_ubuntu_deps() { - if [ "$DISTRO_MAJOR_VERSION" -gt 12 ]; then - # Above Ubuntu 12.04 add-apt-repository is in a different package - __apt_get_install_noinput software-properties-common || return 1 +__install_saltstack_ubuntu_repository() { + + # Workaround for latest non-LTS ubuntu + if [ "$DISTRO_VERSION" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then + echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages from latest LTS release. You may experience problems." + UBUNTU_VERSION=16.04 + UBUNTU_CODENAME="xenial" else - __apt_get_install_noinput python-software-properties || return 1 + UBUNTU_VERSION=$DISTRO_VERSION + UBUNTU_CODENAME=$DISTRO_CODENAME fi + # SaltStack's stable Ubuntu repository: + SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/apt/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" + echo "deb $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/saltstack.list + + # Make sure https transport is available + if [ "$HTTP_VAL" = "https" ] ; then + __apt_get_install_noinput apt-transport-https ca-certificates || return 1 + fi + + __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 + + apt-get update +} + +install_ubuntu_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - __enable_universe_repository || return 1 - - # Versions starting with 2015.5.6 and 2015.8.1 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.5|2015\.8|2016\.3|2016\.11|latest|archive\/)')" = "" ]; then - if [ "$DISTRO_MAJOR_VERSION" -lt 14 ]; then - echoinfo "Installing Python Requests/Chardet from Chris Lea's PPA repository" - add-apt-repository -y "ppa:chris-lea/python-requests" || return 1 - add-apt-repository -y "ppa:chris-lea/python-chardet" || return 1 - add-apt-repository -y "ppa:chris-lea/python-urllib3" || return 1 - add-apt-repository -y "ppa:chris-lea/python-crypto" || return 1 - fi - + # Install add-apt-repository + if ! __check_command_exists add-apt-repository; then + __apt_get_install_noinput software-properties-common || return 1 fi + __enable_universe_repository || return 1 + apt-get update fi - # Minimal systems might not have upstart installed, install it - __PACKAGES="upstart" + __PACKAGES='' + + if [ "$DISTRO_MAJOR_VERSION" -lt 16 ]; then + # Minimal systems might not have upstart installed, install it + __PACKAGES="upstart" + fi if [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then __PACKAGES="${__PACKAGES} python2.7" fi + if [ "$_VIRTUALENV_DIR" != "null" ]; then __PACKAGES="${__PACKAGES} python-virtualenv" fi @@ -2564,59 +2640,10 @@ install_ubuntu_stable_deps() { __apt_get_upgrade_noinput || return 1 fi - if [ ${_DISABLE_REPOS} -eq $BS_FALSE ]; then - __get_dpkg_architecture || return 1 - __REPO_ARCH="$DPKG_ARCHITECTURE" + __check_dpkg_architecture || return 1 - if [ "$DPKG_ARCHITECTURE" = "i386" ]; then - echoerror "$_REPO_URL likely doesn't have all required 32-bit packages for Ubuntu $DISTRO_MAJOR_VERSION (yet?)." - - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - __REPO_ARCH="amd64" - elif [ "$DPKG_ARCHITECTURE" != "amd64" ]; then - echoerror "$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." - if [ "$ITYPE" != "git" ]; then - echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.3.1" - exit 1 - fi - fi - - # Versions starting with 2015.5.6, 2015.8.1 and 2016.3.0 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.5|2015\.8|2016\.3|2016\.11|latest|archive\/)')" != "" ]; then - # Workaround for latest non-LTS ubuntu - if [ "$DISTRO_VERSION" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then - echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages from latest LTS release. You may experience problems." - UBUNTU_VERSION=16.04 - UBUNTU_CODENAME="xenial" - else - UBUNTU_VERSION=$DISTRO_VERSION - UBUNTU_CODENAME=$DISTRO_CODENAME - fi - - # SaltStack's stable Ubuntu repository: - SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/apt/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/saltstack.list - - # Make sure https transport is available - if [ "$HTTP_VAL" = "https" ] ; then - __apt_get_install_noinput apt-transport-https ca-certificates || return 1 - fi - - __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 - else - # Alternate PPAs: salt16, salt17, salt2014-1, salt2014-7 - if [ ! "$(echo "$STABLE_REV" | egrep '^(1\.6|1\.7)$')" = "" ]; then - STABLE_PPA="saltstack/salt$(echo "$STABLE_REV" | tr -d .)" - elif [ ! "$(echo "$STABLE_REV" | egrep '^(2014\.1|2014\.7)$')" = "" ]; then - STABLE_PPA="saltstack/salt$(echo "$STABLE_REV" | tr . -)" - else - STABLE_PPA="saltstack/salt" - fi - - add-apt-repository -y "ppa:$STABLE_PPA" || return 1 - fi - - apt-get update + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_ubuntu_repository || return 1 fi install_ubuntu_deps || return 1 @@ -2625,13 +2652,6 @@ install_ubuntu_stable_deps() { install_ubuntu_daily_deps() { install_ubuntu_stable_deps || return 1 - if [ "$DISTRO_MAJOR_VERSION" -gt 12 ]; then - __apt_get_install_noinput software-properties-common || return 1 - else - # Ubuntu 12.04 needs python-software-properties to get add-apt-repository binary - __apt_get_install_noinput python-software-properties || return 1 - fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then __enable_universe_repository || return 1 @@ -2716,11 +2736,13 @@ install_ubuntu_stable() { # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 + return 0 } install_ubuntu_daily() { install_ubuntu_stable || return 1 + return 0 } @@ -2735,6 +2757,7 @@ install_ubuntu_git() { else python setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 fi + return 0 } @@ -2760,6 +2783,8 @@ install_ubuntu_stable_post() { update-rc.d salt-$fname defaults fi done + + return 0 } install_ubuntu_git_post() { @@ -2772,7 +2797,7 @@ install_ubuntu_git_post() { [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -2796,10 +2821,10 @@ install_ubuntu_git_post() { /sbin/initctl reload-configuration || return 1 fi # No upstart support in Ubuntu!? - elif [ -f "${_SALT_GIT_CHECKOUT_DIR}/debian/salt-${fname}.init" ]; then + elif [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" ]; then echodebug "There's NO upstart support!?" - echodebug "Copying ${_SALT_GIT_CHECKOUT_DIR}/debian/salt-${fname}.init to /etc/init.d/salt-$fname" - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/debian/salt-${fname}.init" "/etc/init.d/salt-$fname" + echodebug "Copying ${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init to /etc/init.d/salt-$fname" + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" "/etc/init.d/salt-$fname" chmod +x /etc/init.d/salt-$fname # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -2810,6 +2835,8 @@ install_ubuntu_git_post() { echoerror "Neither upstart nor init.d was setup for salt-$fname" fi done + + return 0 } install_ubuntu_restart_daemons() { @@ -2862,6 +2889,7 @@ install_ubuntu_restart_daemons() { /etc/init.d/salt-$fname stop > /dev/null 2>&1 /etc/init.d/salt-$fname start done + return 0 } @@ -2895,6 +2923,33 @@ install_ubuntu_check_services() { # # Debian Install Functions # +__install_saltstack_debian_repository() { + if [ "$DISTRO_MAJOR_VERSION" -eq 10 ]; then + # Packages for Debian 10 at repo.saltstack.com are not yet available + # Set up repository for Debian 9 for Debian 10 for now until support + # is available at repo.saltstack.com for Debian 10. + echowarn "Debian 10 distribution detected, but stable packages requested. Trying packages from Debian 9. You may experience problems." + DEBIAN_RELEASE="9" + DEBIAN_CODENAME="stretch" + else + DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" + DEBIAN_CODENAME="$DISTRO_CODENAME" + fi + + # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location + SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/apt/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" + echo "deb $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/saltstack.list" + + if [ "$HTTP_VAL" = "https" ] ; then + __apt_get_install_noinput apt-transport-https ca-certificates || return 1 + fi + + __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 + + apt-get update + +} + install_debian_deps() { if [ $_START_DAEMONS -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -2915,95 +2970,7 @@ install_debian_deps() { __apt_get_upgrade_noinput || return 1 fi - # Install procps and pciutils which allows for Docker bootstraps. See #366#issuecomment-39666813 - __PACKAGES="procps pciutils" - __PIP_PACKAGES="" - - # YAML module is used for generating custom master/minion configs - __PACKAGES="${__PACKAGES} python-yaml" - - # shellcheck disable=SC2086 - __apt_get_install_noinput ${__PACKAGES} || return 1 - - if [ "${_EXTRA_PACKAGES}" != "" ]; then - echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" - # shellcheck disable=SC2086 - __apt_get_install_noinput ${_EXTRA_PACKAGES} || return 1 - fi - - if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - # shellcheck disable=SC2089 - __PIP_PACKAGES="${__PIP_PACKAGES} 'apache-libcloud>=$_LIBCLOUD_MIN_VERSION'" - fi - - if [ "${__PIP_PACKAGES}" != "" ]; then - # shellcheck disable=SC2086,SC2090 - pip install -U ${__PIP_PACKAGES} || return 1 - fi - - return 0 -} - -install_debian_7_deps() { - if [ $_START_DAEMONS -eq $BS_FALSE ]; then - echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." - fi - - # No user interaction, libc6 restart services for example - export DEBIAN_FRONTEND=noninteractive - - apt-get update - - if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then - # Try to update GPG keys first if allowed - if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then - __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && - apt-key update && apt-get update - fi - - __apt_get_upgrade_noinput || return 1 - fi - - if [ "${_DISABLE_REPOS}" -eq $BS_FALSE ]; then - __get_dpkg_architecture || return 1 - - __REPO_ARCH="$DPKG_ARCHITECTURE" - - if [ "$DPKG_ARCHITECTURE" = "i386" ]; then - echoerror "$_REPO_URL likely doesn't have all required 32-bit packages for Debian $DISTRO_MAJOR_VERSION (yet?)." - - if [ "$ITYPE" != "git" ]; then - echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.3.1" - fi - - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - __REPO_ARCH="amd64" - elif [ "$DPKG_ARCHITECTURE" != "amd64" ]; then - echoerror "$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." - exit 1 - fi - - # Versions starting with 2015.8.7 and 2016.3.0 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.8|2016\.3|2016\.11|latest|archive\/201[5-6]\.)')" != "" ]; then - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/apt/debian/${DISTRO_MAJOR_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_DEBIAN_URL wheezy main" > "/etc/apt/sources.list.d/saltstack.list" - - if [ "$HTTP_VAL" = "https" ] ; then - __apt_get_install_noinput apt-transport-https ca-certificates || return 1 - fi - - __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 - elif [ -n "$STABLE_REV" ]; then - echoerror "Installation of Salt ${STABLE_REV#*/} packages not supported by ${__ScriptName} ${__ScriptVersion} on Debian $DISTRO_MAJOR_VERSION." - - return 1 - fi - - apt-get update - else - echowarn "Packages from $_REPO_URL are required to install Salt version 2015.8 or higher on Debian $DISTRO_MAJOR_VERSION." - fi + __check_dpkg_architecture || return 1 # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES='procps pciutils' @@ -3011,87 +2978,17 @@ install_debian_7_deps() { # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python-yaml" - # shellcheck disable=SC2086 - __apt_get_install_noinput ${__PACKAGES} || return 1 - - if [ "${_EXTRA_PACKAGES}" != "" ]; then - echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" - # shellcheck disable=SC2086 - __apt_get_install_noinput ${_EXTRA_PACKAGES} || return 1 + # Debian 9 needs the dirmgr package in order to import the GPG key later + if [ "$DISTRO_MAJOR_VERSION" -ge 9 ]; then + __PACKAGES="${__PACKAGES} dirmngr" fi - return 0 -} - -install_debian_8_deps() { - if [ $_START_DAEMONS -eq $BS_FALSE ]; then - echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." - fi - - # No user interaction, libc6 restart services for example - export DEBIAN_FRONTEND=noninteractive - - apt-get update - - if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then - # Try to update GPG keys first if allowed - if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then - __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && - apt-key update && apt-get update - fi - - __apt_get_upgrade_noinput || return 1 - fi - - if [ ${_DISABLE_REPOS} -eq $BS_FALSE ]; then - __get_dpkg_architecture || return 1 - - __REPO_ARCH="$DPKG_ARCHITECTURE" - - if [ "$DPKG_ARCHITECTURE" = "i386" ]; then - echoerror "$_REPO_URL likely doesn't have all required 32-bit packages for Debian $DISTRO_MAJOR_VERSION (yet?)." - - if [ "$ITYPE" != "git" ]; then - echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.3.1" - fi - - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - __REPO_ARCH="amd64" - elif [ "$DPKG_ARCHITECTURE" != "amd64" ] && [ "$DPKG_ARCHITECTURE" != "armhf" ]; then - echoerror "$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." - echoerror "Try git installation mode with pip and disable SaltStack apt repository, for example:" - echoerror " sh ${__ScriptName} -r -P git v2016.3.1" - - exit 1 - fi - - # Versions starting with 2015.5.6, 2015.8.1 and 2016.3.0 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.5|2015\.8|2016\.3|2016\.11|latest|archive\/201[5-6]\.)')" != "" ]; then - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/apt/debian/${DISTRO_MAJOR_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_DEBIAN_URL jessie main" > "/etc/apt/sources.list.d/saltstack.list" - - if [ "$HTTP_VAL" = "https" ] ; then - __apt_get_install_noinput apt-transport-https ca-certificates || return 1 - fi - - __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 - elif [ -n "$STABLE_REV" ]; then - echoerror "Installation of Salt ${STABLE_REV#*/} packages not supported by ${__ScriptName} ${__ScriptVersion} on Debian $DISTRO_MAJOR_VERSION." - - return 1 - fi - - apt-get update - fi - - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES='procps pciutils' - # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 - # YAML module is used for generating custom master/minion configs - __PACKAGES="${__PACKAGES} python-yaml" + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_debian_repository || return 1 + fi if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" @@ -3135,14 +3032,14 @@ install_debian_git_deps() { } install_debian_7_git_deps() { - install_debian_7_deps || return 1 + install_debian_deps || return 1 install_debian_git_deps || return 1 return 0 } install_debian_8_git_deps() { - install_debian_8_deps || return 1 + install_debian_deps || return 1 if ! __check_command_exists git; then __apt_get_install_noinput git || return 1 @@ -3204,6 +3101,45 @@ install_debian_8_git_deps() { return 0 } +install_debian_9_git_deps() { + install_debian_deps || return 1 + + if ! __check_command_exists git; then + __apt_get_install_noinput git || return 1 + fi + + if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then + __apt_get_install_noinput ca-certificates + fi + + __git_clone_and_checkout || return 1 + + __PACKAGES="libzmq5 lsb-release python-apt python-backports-abc python-crypto" + __PACKAGES="${__PACKAGES} python-jinja2 python-msgpack python-requests python-systemd" + __PACKAGES="${__PACKAGES} python-tornado python-yaml python-zmq" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + # Install python-libcloud if asked to + __PACKAGES="${__PACKAGES} python-libcloud" + fi + + # shellcheck disable=SC2086 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + # Let's trigger config_salt() + if [ "$_TEMP_CONFIG_DIR" = "null" ]; then + _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_debian_10_git_deps() { + install_debian_9_git_deps || return 1 + return 0 +} + install_debian_stable() { __PACKAGES="" @@ -3236,9 +3172,14 @@ install_debian_8_stable() { return 0 } +install_debian_9_stable() { + install_debian_stable || return 1 + return 0 +} + install_debian_git() { if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then - python setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + python setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 else python setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 fi @@ -3254,6 +3195,11 @@ install_debian_8_git() { return 0 } +install_debian_9_git() { + install_debian_git || return 1 + return 0 +} + install_debian_git_post() { for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3267,9 +3213,9 @@ install_debian_git_post() { if [ -f /bin/systemctl ]; then if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ ([ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" /lib/systemd/system - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.environment" "/etc/default/salt-${fname}" + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" ]; then + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.environment" "/etc/default/salt-${fname}" else # workaround before adding Debian-specific unit files to the Salt main repo __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system @@ -3286,9 +3232,9 @@ install_debian_git_post() { # Install initscripts for Debian 7 "Wheezy" elif [ ! -f "/etc/init.d/salt-$fname" ] || \ ([ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]); then - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-$fname.init" ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.init" "/etc/init.d/salt-${fname}" - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.environment" "/etc/default/salt-${fname}" + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.init" ]; then + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" "/etc/init.d/salt-${fname}" + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.environment" "/etc/default/salt-${fname}" else # Make sure wget is available __check_command_exists wget || __apt_get_install_noinput wget || return 1 @@ -3442,23 +3388,7 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="systemd-python" - __PIP_PACKAGES="" - - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then - # We're on the develop branch, install whichever tornado is on the requirements file - __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" - if [ "${__REQUIRED_TORNADO}" != "" ]; then - __check_pip_allowed "You need to allow pip based installations (-P) in order to install tornado" - - # Install pip and pip dependencies - if ! __check_command_exists pip; then - __PACKAGES="${__PACKAGES} python-setuptools python-pip gcc python-devel" - fi - - __PIP_PACKAGES="${__PIP_PACKAGES} tornado" - fi - fi + __PACKAGES="python2-tornado systemd-python" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then __PACKAGES="${__PACKAGES} python-libcloud python-netaddr" @@ -3467,11 +3397,6 @@ install_fedora_git_deps() { # shellcheck disable=SC2086 dnf install -y ${__PACKAGES} || return 1 - if [ "${__PIP_PACKAGES}" != "" ]; then - # shellcheck disable=SC2086,SC2090 - pip install -U ${__PIP_PACKAGES} || return 1 - fi - # Let's trigger config_salt() if [ "$_TEMP_CONFIG_DIR" = "null" ]; then _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" @@ -3563,13 +3488,6 @@ __install_epel_repository() { return 0 fi - # Check if epel-release is already installed and flag it accordingly - rpm --nodigest --nosignature -q epel-release > /dev/null 2>&1 - if [ $? -eq 0 ]; then - _EPEL_REPOS_INSTALLED=$BS_TRUE - return 0 - fi - # Download latest 'epel-release' package for the distro version directly epel_repo_url="${HTTP_VAL}://dl.fedoraproject.org/pub/epel/epel-release-latest-${DISTRO_MAJOR_VERSION}.noarch.rpm" rpm -Uvh --force "$epel_repo_url" || return 1 @@ -3754,8 +3672,29 @@ install_centos_git_deps() { __PACKAGES="${__PACKAGES} python-libcloud" fi - if [ "${_INSTALL_PY}" = "${BS_TRUE}" ]; then - __install_python_and_deps || return 1 + if [ "${_INSTALL_PY}" -eq "${BS_TRUE}" ]; then + # Install Python if "-y" was passed in. + __install_python || return 1 + fi + + if [ "${_PY_EXE}" != "" ]; then + # If "-x" is defined, install dependencies with pip based on the Python version given. + _PIP_PACKAGES="jinja2 msgpack-python pycrypto PyYAML tornado zmq" + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then + for SINGLE_PACKAGE in $_PIP_PACKAGES; do + __REQUIRED_VERSION="$(grep "${SINGLE_PACKAGE}" "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" + if [ "${__REQUIRED_VERSION}" != "" ]; then + _PIP_PACKAGES=$(echo "$_PIP_PACKAGES" | sed "s/${SINGLE_PACKAGE}/${__REQUIRED_VERSION}/") + fi + done + fi + + if [ "$_INSTALL_CLOUD" -eq "${BS_TRUE}" ]; then + _PIP_PACKAGES="${_PIP_PACKAGES} apache-libcloud" + fi + + __install_pip_pkgs "${_PIP_PACKAGES}" "${_PY_EXE}" || return 1 else # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4460,44 +4399,37 @@ daemons_running_alpine_linux() { install_amazon_linux_ami_deps() { # Shim to figure out if we're using old (rhel) or new (aws) rpms. _USEAWS=$BS_FALSE + pkg_append="python" repo_rev="$(echo "${STABLE_REV}" | sed 's|.*\/||g')" - if echo "$repo_rev" | egrep -q '^(latest|2016\.11)$'; then - _USEAWS=$BS_TRUE - elif echo "$repo_rev" | egrep -q '^[0-9]+$' && [ "$(echo "$repo_rev" | cut -c1-4)" -gt 2016 ]; then + if echo "$repo_rev" | egrep -q '^(latest|2016\.11)$' || \ + ( echo "$repo_rev" | egrep -q '^[0-9]+$' && [ "$(echo "$repo_rev" | cut -c1-4)" -gt 2016 ] ); then _USEAWS=$BS_TRUE + pkg_append="python27" fi # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - yum -y install yum-utils + __yum_install_noinput yum-utils - ENABLE_EPEL_CMD="" - if [ $_DISABLE_REPOS -eq $BS_TRUE ]; then - ENABLE_EPEL_CMD="--enablerepo=${_EPEL_REPO}" + # Do upgrade early + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + yum -y update || return 1 fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - # enable the EPEL repo - /usr/bin/yum-config-manager --enable epel || return 1 - - # exclude Salt and ZeroMQ packages from EPEL - /usr/bin/yum-config-manager epel --setopt "epel.exclude=zeromq* salt* python-zmq*" --save || return 1 - + if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then __REPO_FILENAME="saltstack-repo.repo" # Set a few vars to make life easier. if [ $_USEAWS -eq $BS_TRUE ]; then - base_url="$HTTP_VAL://repo.saltstack.com/yum/amazon/latest/\$basearch/$repo_rev/" + base_url="$HTTP_VAL://${_REPO_URL}/yum/amazon/latest/\$basearch/$repo_rev/" gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" repo_name="SaltStack repo for Amazon Linux" - pkg_append="python27" else - base_url="$HTTP_VAL://repo.saltstack.com/yum/redhat/6/\$basearch/$repo_rev/" + base_url="$HTTP_VAL://${_REPO_URL}/yum/redhat/6/\$basearch/$repo_rev/" gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" repo_name="SaltStack repo for RHEL/CentOS 6" - pkg_append="python" fi # This should prob be refactored to use __install_saltstack_rhel_repository() @@ -4515,21 +4447,19 @@ baseurl=$base_url _eof fi - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - yum -y update || return 1 - fi fi - #ordereddict removed. - #Package python-ordereddict-1.1-2.el6.noarch is obsoleted by python26-2.6.9-2.88.amzn1.x86_64 which is already installed + + # Package python-ordereddict-1.1-2.el6.noarch is obsoleted by python26-2.6.9-2.88.amzn1.x86_64 + # which is already installed __PACKAGES="${pkg_append}-PyYAML ${pkg_append}-crypto ${pkg_append}-msgpack ${pkg_append}-zmq ${pkg_append}-jinja2 ${pkg_append}-requests" # shellcheck disable=SC2086 - yum -y install ${__PACKAGES} ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput ${__PACKAGES} || return 1 if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" # shellcheck disable=SC2086 - yum install -y ${_EXTRA_PACKAGES} ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput ${_EXTRA_PACKAGES} || return 1 fi } @@ -4549,13 +4479,8 @@ install_amazon_linux_ami_git_deps() { install_amazon_linux_ami_deps || return 1 - ENABLE_EPEL_CMD="" - if [ $_DISABLE_REPOS -eq $BS_TRUE ]; then - ENABLE_EPEL_CMD="--enablerepo=${_EPEL_REPO}" - fi - if ! __check_command_exists git; then - yum -y install git ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput git || return 1 fi __git_clone_and_checkout || return 1 @@ -4579,7 +4504,7 @@ install_amazon_linux_ami_git_deps() { if [ "${__PACKAGES}" != "" ]; then # shellcheck disable=SC2086 - yum -y install ${__PACKAGES} ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput ${__PACKAGES} || return 1 fi if [ "${__PIP_PACKAGES}" != "" ]; then @@ -4660,7 +4585,7 @@ install_arch_linux_stable_deps() { pacman -Su --noconfirm --needed python2-yaml if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - pacman -Su --noconfirm --needed apache-libcloud || return 1 + pacman -Su --noconfirm --needed python2-apache-libcloud || return 1 fi if [ "${_EXTRA_PACKAGES}" != "" ]; then @@ -5458,6 +5383,16 @@ install_smartos_restart_daemons() { # __ZYPPER_REQUIRES_REPLACE_FILES=-1 +__check_and_refresh_suse_pkg_repo() { + # Check to see if systemsmanagement_saltstack exists + __zypper repos | grep systemsmanagement_saltstack >/dev/null 2>&1 + + if [ $? -eq 1 ]; then + # zypper does not yet know anything about systemsmanagement_saltstack + __zypper addrepo --refresh "${SUSE_PKG_URL}" || return 1 + fi +} + __set_suse_pkg_repo() { suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" if [ "$_DOWNSTREAM_PKG_REPO" -eq $BS_TRUE ]; then @@ -6151,11 +6086,15 @@ install_suse_check_services() { # # Gentoo Install Functions. # +__autounmask() { + emerge --autounmask-write --autounmask-only "${@}"; return $? +} + __emerge() { if [ "$_GENTOO_USE_BINHOST" -eq $BS_TRUE ]; then - emerge --autounmask-write --getbinpkg "${@}"; return $? + emerge --getbinpkg "${@}"; return $? fi - emerge --autounmask-write "${@}"; return $? + emerge "${@}"; return $? } __gentoo_config_protection() { @@ -6165,6 +6104,9 @@ __gentoo_config_protection() { # cfg-update and then restart the bootstrapping script, so instead we allow # at this point to modify certain config files directly export CONFIG_PROTECT_MASK="${CONFIG_PROTECT_MASK:-} /etc/portage/package.accept_keywords /etc/portage/package.keywords /etc/portage/package.license /etc/portage/package.unmask /etc/portage/package.use" + + # emerge currently won't write to files that aren't there, so we need to ensure their presence + touch /etc/portage/package.accept_keywords /etc/portage/package.keywords /etc/portage/package.license /etc/portage/package.unmask /etc/portage/package.use } __gentoo_pre_dep() { @@ -6193,15 +6135,21 @@ __gentoo_post_dep() { __gentoo_config_protection if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + __autounmask 'dev-python/libcloud' __emerge -v 'dev-python/libcloud' fi + __autounmask 'dev-python/requests' + __autounmask 'app-admin/salt' + __emerge -vo 'dev-python/requests' __emerge -vo 'app-admin/salt' if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" # shellcheck disable=SC2086 + __autounmask ${_EXTRA_PACKAGES} || return 1 + # shellcheck disable=SC2086 __emerge -v ${_EXTRA_PACKAGES} || return 1 fi } @@ -6437,12 +6385,12 @@ config_salt() { # Copy the minion's keys if found if [ -f "$_TEMP_CONFIG_DIR/minion.pem" ]; then - __movefile "$_TEMP_CONFIG_DIR/minion.pem" "$_PKI_DIR/minion/" "$_CONFIG_ONLY" || return 1 + __movefile "$_TEMP_CONFIG_DIR/minion.pem" "$_PKI_DIR/minion/" "$_FORCE_OVERWRITE" || return 1 chmod 400 "$_PKI_DIR/minion/minion.pem" || return 1 CONFIGURED_ANYTHING=$BS_TRUE fi if [ -f "$_TEMP_CONFIG_DIR/minion.pub" ]; then - __movefile "$_TEMP_CONFIG_DIR/minion.pub" "$_PKI_DIR/minion/" "$_CONFIG_ONLY" || return 1 + __movefile "$_TEMP_CONFIG_DIR/minion.pub" "$_PKI_DIR/minion/" "$_FORCE_OVERWRITE" || return 1 chmod 664 "$_PKI_DIR/minion/minion.pub" || return 1 CONFIGURED_ANYTHING=$BS_TRUE fi From 0a1f40a664c9053de3ec082e0a36d4028424c37d Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 17 Aug 2017 10:38:32 -0600 Subject: [PATCH 356/508] test with gem that appears to be abandoned --- tests/integration/modules/test_gem.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/modules/test_gem.py b/tests/integration/modules/test_gem.py index da902c9f66..bd1e013c24 100644 --- a/tests/integration/modules/test_gem.py +++ b/tests/integration/modules/test_gem.py @@ -19,8 +19,9 @@ from tornado.httpclient import HTTPClient GEM = 'tidy' GEM_VER = '1.1.2' -OLD_GEM = 'thor' -OLD_VERSION = '0.17.0' +OLD_GEM = 'test' +OLD_VERSION = '0.2.0' +NEW_VERSION = '1.0.0' GEM_LIST = [GEM, OLD_GEM] @@ -129,18 +130,18 @@ class GemModuleTest(ModuleCase): self.run_function('gem.install', [OLD_GEM], version=OLD_VERSION) gem_list = self.run_function('gem.list', [OLD_GEM]) - self.assertEqual({'thor': ['0.17.0']}, gem_list) + self.assertEqual({OLD_GEM: [OLD_VERSION]}, gem_list) self.run_function('gem.update', [OLD_GEM]) gem_list = self.run_function('gem.list', [OLD_GEM]) - self.assertEqual({'thor': ['0.19.4', '0.17.0']}, gem_list) + self.assertEqual({OLD_GEM: [NEW_VERSION, OLD_VERSION]}, gem_list) self.run_function('gem.uninstall', [OLD_GEM]) self.assertFalse(self.run_function('gem.list', [OLD_GEM])) - def test_udpate_system(self): + def test_update_system(self): ''' - gem.udpate_system + gem.update_system ''' ret = self.run_function('gem.update_system') self.assertTrue(ret) From 4089b7b1bc11d11c4ba3fc893caaf8a5b3366859 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 12:12:27 -0500 Subject: [PATCH 357/508] Use socket.AF_INET6 to get the correct value instead of doing an OS check This builds on #43014 with a more future-proof solution. --- tests/unit/utils/network_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/utils/network_test.py b/tests/unit/utils/network_test.py index 2884d6211e..0fab31024d 100644 --- a/tests/unit/utils/network_test.py +++ b/tests/unit/utils/network_test.py @@ -12,7 +12,6 @@ ensure_in_syspath('../../') # Import salt libs from salt.utils import network -import salt.utils LINUX = '''\ eth0 Link encap:Ethernet HWaddr e0:3f:49:85:6a:af @@ -115,14 +114,13 @@ class NetworkTestCase(TestCase): ''' def _side_effect(host, *args): try: - ipv6_fam = 30 if salt.utils.is_darwin() else 10 return { 'github.com': [ (2, 1, 6, '', ('192.30.255.112', 0)), (2, 1, 6, '', ('192.30.255.113', 0)), ], 'ipv6host.foo': [ - (ipv6_fam, 1, 6, '', ('2001:a71::1', 0, 0, 0)), + (socket.AF_INET6, 1, 6, '', ('2001:a71::1', 0, 0, 0)), ], }[host] except KeyError: From b04d1a2f18a4d0ec5459f72f35a313b2496c482b Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 17 Aug 2017 11:55:00 -0600 Subject: [PATCH 358/508] Fix `unit.utils.test_find` for Windows Fix skipIf statements Use sys.maxsize instead of sys.maxint --- tests/unit/utils/test_find.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tests/unit/utils/test_find.py b/tests/unit/utils/test_find.py index 86af790b0e..7ba487f8d2 100644 --- a/tests/unit/utils/test_find.py +++ b/tests/unit/utils/test_find.py @@ -121,19 +121,11 @@ class TestFind(TestCase): min_size, max_size = salt.utils.find._parse_size('+1m') self.assertEqual(min_size, 1048576) - # sys.maxint has been removed in Python3. Use maxsize instead. - if six.PY3: - self.assertEqual(max_size, sys.maxsize) - else: - self.assertEqual(max_size, sys.maxint) + self.assertEqual(max_size, sys.maxsize) min_size, max_size = salt.utils.find._parse_size('+1M') self.assertEqual(min_size, 1048576) - # sys.maxint has been removed in Python3. Use maxsize instead. - if six.PY3: - self.assertEqual(max_size, sys.maxsize) - else: - self.assertEqual(max_size, sys.maxint) + self.assertEqual(max_size, sys.maxsize) def test_option_requires(self): option = salt.utils.find.Option() @@ -217,7 +209,7 @@ class TestFind(TestCase): option = salt.utils.find.TypeOption('type', 's') self.assertEqual(option.match('', '', [stat.S_IFSOCK]), True) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'pwd not available on Windows') def test_owner_option_requires(self): self.assertRaises( ValueError, salt.utils.find.OwnerOption, 'owner', 'notexist' @@ -226,7 +218,7 @@ class TestFind(TestCase): option = salt.utils.find.OwnerOption('owner', 'root') self.assertEqual(option.requires(), salt.utils.find._REQUIRES_STAT) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'pwd not available on Windows') def test_owner_option_match(self): option = salt.utils.find.OwnerOption('owner', 'root') self.assertEqual(option.match('', '', [0] * 5), True) @@ -234,7 +226,7 @@ class TestFind(TestCase): option = salt.utils.find.OwnerOption('owner', '500') self.assertEqual(option.match('', '', [500] * 5), True) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'grp not available on Windows') def test_group_option_requires(self): self.assertRaises( ValueError, salt.utils.find.GroupOption, 'group', 'notexist' @@ -247,7 +239,7 @@ class TestFind(TestCase): option = salt.utils.find.GroupOption('group', group_name) self.assertEqual(option.requires(), salt.utils.find._REQUIRES_STAT) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'grp not available on Windows') def test_group_option_match(self): if sys.platform.startswith(('darwin', 'freebsd', 'openbsd')): group_name = 'wheel' @@ -412,7 +404,7 @@ class TestPrintOption(TestCase): option.execute('test_name', [0] * 9), [0, 'test_name'] ) - @skipIf(sys.platform.startswith('Windows'), "no /dev/null on windows") + @skipIf(sys.platform.startswith('win'), "pwd not available on Windows") def test_print_user(self): option = salt.utils.find.PrintOption('print', 'user') self.assertEqual(option.execute('', [0] * 10), 'root') @@ -420,7 +412,7 @@ class TestPrintOption(TestCase): option = salt.utils.find.PrintOption('print', 'user') self.assertEqual(option.execute('', [2 ** 31] * 10), 2 ** 31) - @skipIf(sys.platform.startswith('Windows'), "no /dev/null on windows") + @skipIf(sys.platform.startswith('win'), "grp not available on Windows") def test_print_group(self): option = salt.utils.find.PrintOption('print', 'group') if sys.platform.startswith(('darwin', 'freebsd', 'openbsd')): @@ -433,7 +425,7 @@ class TestPrintOption(TestCase): #option = salt.utils.find.PrintOption('print', 'group') #self.assertEqual(option.execute('', [2 ** 31] * 10), 2 ** 31) - @skipIf(sys.platform.startswith('Windows'), "no /dev/null on windows") + @skipIf(sys.platform.startswith('win'), "no /dev/null on windows") def test_print_md5(self): option = salt.utils.find.PrintOption('print', 'md5') self.assertEqual(option.execute('/dev/null', os.stat('/dev/null')), '') From d0fd949f85757f9359c9720fc34fe8d101e5ca0e Mon Sep 17 00:00:00 2001 From: Pablo Hernandez Date: Thu, 17 Aug 2017 14:24:48 -0400 Subject: [PATCH 359/508] Fixes ignore push flag for docker.push module issue #42992 --- salt/modules/dockerng.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/dockerng.py b/salt/modules/dockerng.py index 0cf393c4db..03ec146cb9 100644 --- a/salt/modules/dockerng.py +++ b/salt/modules/dockerng.py @@ -4403,7 +4403,7 @@ def save(name, ret['Size_Human'] = _size_fmt(ret['Size']) # Process push - if kwargs.get(push, False): + if kwargs.get('push', False): ret['Push'] = __salt__['cp.push'](path) return ret From 0eb15a1f67e35c9574932c731f24dc6da28d717e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 17 Aug 2017 10:38:32 -0600 Subject: [PATCH 360/508] test with gem that appears to be abandoned --- tests/integration/modules/gem.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/modules/gem.py b/tests/integration/modules/gem.py index 6df4626392..da44c7a728 100644 --- a/tests/integration/modules/gem.py +++ b/tests/integration/modules/gem.py @@ -18,8 +18,9 @@ import salt.utils.http GEM = 'tidy' GEM_VER = '1.1.2' -OLD_GEM = 'thor' -OLD_VERSION = '0.17.0' +OLD_GEM = 'test' +OLD_VERSION = '0.2.0' +NEW_VERSION = '1.0.0' GEM_LIST = [GEM, OLD_GEM] @@ -123,18 +124,18 @@ class GemModuleTest(integration.ModuleCase): self.run_function('gem.install', [OLD_GEM], version=OLD_VERSION) gem_list = self.run_function('gem.list', [OLD_GEM]) - self.assertEqual({'thor': ['0.17.0']}, gem_list) + self.assertEqual({OLD_GEM: [OLD_VERSION]}, gem_list) self.run_function('gem.update', [OLD_GEM]) gem_list = self.run_function('gem.list', [OLD_GEM]) - self.assertEqual({'thor': ['0.19.4', '0.17.0']}, gem_list) + self.assertEqual({OLD_GEM: [NEW_VERSION, OLD_VERSION]}, gem_list) self.run_function('gem.uninstall', [OLD_GEM]) self.assertFalse(self.run_function('gem.list', [OLD_GEM])) - def test_udpate_system(self): + def test_update_system(self): ''' - gem.udpate_system + gem.update_system ''' ret = self.run_function('gem.update_system') self.assertTrue(ret) From 822eabcc818387098772a47491f3950ca0b94ad9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 12:32:39 -0500 Subject: [PATCH 361/508] Catch exceptions raised when making changes to jenkins --- salt/modules/jenkins.py | 2 +- salt/states/jenkins.py | 53 +++++++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/salt/modules/jenkins.py b/salt/modules/jenkins.py index 6c4f251a76..77ae100b88 100644 --- a/salt/modules/jenkins.py +++ b/salt/modules/jenkins.py @@ -451,7 +451,7 @@ def get_job_config(name=None): server = _connect() - job_exists(name) + if not job_exists(name) raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) job_info = server.get_job_config(name) diff --git a/salt/states/jenkins.py b/salt/states/jenkins.py index ca83ac0b98..421dcda892 100644 --- a/salt/states/jenkins.py +++ b/salt/states/jenkins.py @@ -15,23 +15,28 @@ import logging # Import Salt libs import salt.ext.six as six import salt.utils +from salt.exceptions import CommandExecutionError, SaltInvocationError log = logging.getLogger(__name__) +def _fail(ret, msg): + ret['comment'] = msg + ret['result'] = False + return ret + + def present(name, config=None, **kwargs): ''' - Ensure the job is present in the Jenkins - configured jobs + Ensure the job is present in the Jenkins configured jobs name The unique name for the Jenkins job config - The Salt URL for the file to use for - configuring the job. + The Salt URL for the file to use for configuring the job ''' ret = {'name': name, @@ -39,9 +44,8 @@ def present(name, 'changes': {}, 'comment': ['Job {0} is up to date.'.format(name)]} - _job_exists = __salt__['jenkins.job_exists'](name) - if _job_exists: + if __salt__['jenkins.job_exists'](name): _current_job_config = __salt__['jenkins.get_job_config'](name) buf = six.moves.StringIO(_current_job_config) _current_job_config = buf.readlines() @@ -52,23 +56,30 @@ def present(name, if _current_job_config != new_config_xml: diff = difflib.unified_diff(_current_job_config, new_config_xml, lineterm='') - __salt__['jenkins.update_job'](name, config, __env__) - ret['changes'] = ''.join(diff) - ret['comment'].append('Job {0} updated.'.format(name)) + try: + __salt__['jenkins.update_job'](name, config, __env__) + except CommandExecutionError as exc: + return _fail(ret, exc.strerror) + else: + ret['changes'] = ''.join(diff) + ret['comment'].append('Job \'{0}\' updated.'.format(name)) else: cached_source_path = __salt__['cp.cache_file'](config, __env__) with salt.utils.fopen(cached_source_path) as _fp: new_config_xml = _fp.read() - __salt__['jenkins.create_job'](name, config, __env__) + try: + __salt__['jenkins.create_job'](name, config, __env__) + except CommandExecutionError as exc: + return _fail(ret, exc.strerror) buf = six.moves.StringIO(new_config_xml) _current_job_config = buf.readlines() diff = difflib.unified_diff('', buf, lineterm='') ret['changes'] = ''.join(diff) - ret['comment'].append('Job {0} added.'.format(name)) + ret['comment'].append('Job \'{0}\' added.'.format(name)) ret['comment'] = '\n'.join(ret['comment']) return ret @@ -77,24 +88,24 @@ def present(name, def absent(name, **kwargs): ''' - Ensure the job is present in the Jenkins - configured jobs + Ensure the job is absent from the Jenkins configured jobs name - The name of the Jenkins job to remove. - + The name of the Jenkins job to remove ''' - ret = {'name': name, 'result': True, 'changes': {}, 'comment': []} - _job_exists = __salt__['jenkins.job_exists'](name) - if _job_exists: - __salt__['jenkins.delete_job'](name) - ret['comment'] = 'Job {0} deleted.'.format(name) + if __salt__['jenkins.job_exists'](name): + try: + __salt__['jenkins.delete_job'](name) + except CommandExecutionError as exc: + return _fail(ret, exc.strerror) + else: + ret['comment'] = 'Job \'{0}\' deleted.'.format(name) else: - ret['comment'] = 'Job {0} already absent.'.format(name) + ret['comment'] = 'Job \'{0}\' already absent.'.format(name) return ret From ba80a7d4b546923f5a7e7529a9e40e7bfe69cdda Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 17 Aug 2017 13:45:10 -0600 Subject: [PATCH 362/508] use a ruby gem that doesn't have dependencies --- tests/integration/modules/test_gem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/modules/test_gem.py b/tests/integration/modules/test_gem.py index bd1e013c24..f0a6d048aa 100644 --- a/tests/integration/modules/test_gem.py +++ b/tests/integration/modules/test_gem.py @@ -19,9 +19,9 @@ from tornado.httpclient import HTTPClient GEM = 'tidy' GEM_VER = '1.1.2' -OLD_GEM = 'test' -OLD_VERSION = '0.2.0' -NEW_VERSION = '1.0.0' +OLD_GEM = 'brass' +OLD_VERSION = '1.0.0' +NEW_VERSION = '1.2.1' GEM_LIST = [GEM, OLD_GEM] From ee59d127e8945180e71829b40960b7b835ba26d5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 14:04:30 -0500 Subject: [PATCH 363/508] Normalize the salt caching API https://github.com/saltstack/salt/pull/40429 changed the attribute used in `salt/cache/__init__.py` without considering the usage of the cache system elsewhere in the codebase. This led to a cascade of additional changes to try to make everything work correctly. This commit does 2 things: 1) Reverts the calls to the list function to use `list` instead of `ls` to conform with the established API used elsewhere in the codebase. 2) Modifies the cache modules so that they now use `list_` functions, aliased to `list` so that calls to cache loader instances' `*.list` functions elsewhere in the codebase will work. --- salt/cache/__init__.py | 6 ++---- salt/cache/consul.py | 4 ++-- salt/cache/localfs.py | 4 ++-- salt/cache/redis_cache.py | 6 ++---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/salt/cache/__init__.py b/salt/cache/__init__.py index 4c66dd3d64..94d7a36f1e 100644 --- a/salt/cache/__init__.py +++ b/salt/cache/__init__.py @@ -224,7 +224,7 @@ class Cache(object): fun = '{0}.flush'.format(self.driver) return self.modules[fun](bank, key=key, **self._kwargs) - def ls(self, bank): + def list(self, bank): ''' Lists entries stored in the specified bank. @@ -240,11 +240,9 @@ class Cache(object): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.ls'.format(self.driver) + fun = '{0}.list'.format(self.driver) return self.modules[fun](bank, **self._kwargs) - list = ls - def contains(self, bank, key=None): ''' Checks if the specified bank contains the specified key. diff --git a/salt/cache/consul.py b/salt/cache/consul.py index b545c96ead..d8f1b32a74 100644 --- a/salt/cache/consul.py +++ b/salt/cache/consul.py @@ -61,7 +61,7 @@ api = None # Define the module's virtual name __virtualname__ = 'consul' -__func_alias__ = {'list': 'ls'} +__func_alias__ = {'list_': 'list'} def __virtual__(): @@ -139,7 +139,7 @@ def flush(bank, key=None): ) -def ls(bank): +def list_(bank): ''' Return an iterable object containing all entries stored in the specified bank. ''' diff --git a/salt/cache/localfs.py b/salt/cache/localfs.py index 68cd5aee5c..45dfa6d086 100644 --- a/salt/cache/localfs.py +++ b/salt/cache/localfs.py @@ -23,7 +23,7 @@ import salt.utils.atomicfile log = logging.getLogger(__name__) -__func_alias__ = {'list': 'ls'} +__func_alias__ = {'list_': 'list'} def __cachedir(kwargs=None): @@ -143,7 +143,7 @@ def flush(bank, key=None, cachedir=None): return True -def ls(bank, cachedir): +def list_(bank, cachedir): ''' Return an iterable object containing all entries stored in the specified bank. ''' diff --git a/salt/cache/redis_cache.py b/salt/cache/redis_cache.py index 65a33f5055..b02a0851e5 100644 --- a/salt/cache/redis_cache.py +++ b/salt/cache/redis_cache.py @@ -114,9 +114,7 @@ from salt.exceptions import SaltCacheError # ----------------------------------------------------------------------------- __virtualname__ = 'redis' -__func_alias__ = { - 'ls': 'list' -} +__func_alias__ = {'list_': 'list'} log = logging.getLogger(__file__) @@ -418,7 +416,7 @@ def flush(bank, key=None): return True -def ls(bank): +def list_(bank): ''' Lists entries stored in the specified bank. ''' From c4ae79b2299e2717a9080b306faf64c4a18bc224 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 15:06:09 -0500 Subject: [PATCH 364/508] Rename other refs to cache.ls with cache.list --- salt/key.py | 4 ++-- salt/thorium/__init__.py | 2 +- salt/utils/master.py | 6 +++--- salt/utils/minions.py | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/salt/key.py b/salt/key.py index 88e2acf0e4..73a1d639fb 100644 --- a/salt/key.py +++ b/salt/key.py @@ -496,7 +496,7 @@ class Key(object): if minion not in minions and minion not in preserve_minions: shutil.rmtree(os.path.join(m_cache, minion)) cache = salt.cache.factory(self.opts) - clist = cache.ls(self.ACC) + clist = cache.list(self.ACC) if clist: for minion in clist: if minion not in minions and minion not in preserve_minions: @@ -974,7 +974,7 @@ class RaetKey(Key): if minion not in minions: shutil.rmtree(os.path.join(m_cache, minion)) cache = salt.cache.factory(self.opts) - clist = cache.ls(self.ACC) + clist = cache.list(self.ACC) if clist: for minion in clist: if minion not in minions and minion not in preserve_minions: diff --git a/salt/thorium/__init__.py b/salt/thorium/__init__.py index 72e854a36b..5363733d1a 100644 --- a/salt/thorium/__init__.py +++ b/salt/thorium/__init__.py @@ -69,7 +69,7 @@ class ThorState(salt.state.HighState): cache = {'grains': {}, 'pillar': {}} if self.grains or self.pillar: if self.opts.get('minion_data_cache'): - minions = self.cache.ls('minions') + minions = self.cache.list('minions') if not minions: return cache for minion in minions: diff --git a/salt/utils/master.py b/salt/utils/master.py index c48137ce4c..16e95a40aa 100644 --- a/salt/utils/master.py +++ b/salt/utils/master.py @@ -122,7 +122,7 @@ class MasterPillarUtil(object): 'and enfore_mine_cache are both disabled.') return mine_data if not minion_ids: - minion_ids = self.cache.ls('minions') + minion_ids = self.cache.list('minions') for minion_id in minion_ids: if not salt.utils.verify.valid_id(self.opts, minion_id): continue @@ -141,7 +141,7 @@ class MasterPillarUtil(object): 'enabled.') return grains, pillars if not minion_ids: - minion_ids = self.cache.ls('minions') + minion_ids = self.cache.list('minions') for minion_id in minion_ids: if not salt.utils.verify.valid_id(self.opts, minion_id): continue @@ -364,7 +364,7 @@ class MasterPillarUtil(object): # in the same file, 'data.p' grains, pillars = self._get_cached_minion_data(*minion_ids) try: - c_minions = self.cache.ls('minions') + c_minions = self.cache.list('minions') for minion_id in minion_ids: if not salt.utils.verify.valid_id(self.opts, minion_id): continue diff --git a/salt/utils/minions.py b/salt/utils/minions.py index f4de4e92d9..8afa41698c 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -75,7 +75,7 @@ def get_minion_data(minion, opts): if opts.get('minion_data_cache', False): cache = salt.cache.factory(opts) if minion is None: - for id_ in cache.ls('minions'): + for id_ in cache.list('minions'): data = cache.fetch('minions/{0}'.format(id_), 'data') if data is None: continue @@ -342,13 +342,13 @@ class CkMinions(object): if greedy: minions = self._pki_minions() elif cache_enabled: - minions = self.cache.ls('minions') + minions = self.cache.list('minions') else: return [] if cache_enabled: if greedy: - cminions = self.cache.ls('minions') + cminions = self.cache.list('minions') else: cminions = minions if cminions is None: @@ -412,7 +412,7 @@ class CkMinions(object): mlist.append(fn_) return mlist elif cache_enabled: - return self.cache.ls('minions') + return self.cache.list('minions') else: return list() @@ -574,7 +574,7 @@ class CkMinions(object): ''' minions = set() if self.opts.get('minion_data_cache', False): - search = self.cache.ls('minions') + search = self.cache.list('minions') if search is None: return minions addrs = salt.utils.network.local_port_tcp(int(self.opts['publish_port'])) From 7f5ee55f5746e256e013fe351e59d8d976cf2855 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 17 Aug 2017 14:20:25 -0600 Subject: [PATCH 365/508] Fix `unit.utils.test_url` for Windows Detect escaped urls in Windows Unescape urls in Windows Fix tests to deal with sanitized Windows paths --- salt/utils/url.py | 16 ++++++++-------- tests/unit/utils/test_url.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/salt/utils/url.py b/salt/utils/url.py index 0998ee5a0c..df6e64443b 100644 --- a/salt/utils/url.py +++ b/salt/utils/url.py @@ -60,15 +60,15 @@ def is_escaped(url): ''' test whether `url` is escaped with `|` ''' - if salt.utils.is_windows(): - return False - scheme = urlparse(url).scheme if not scheme: return url.startswith('|') elif scheme == 'salt': path, saltenv = parse(url) - return path.startswith('|') + if salt.utils.is_windows() and '|' in url: + return path.startswith('_') + else: + return path.startswith('|') else: return False @@ -100,15 +100,15 @@ def unescape(url): ''' remove escape character `|` from `url` ''' - if salt.utils.is_windows(): - return url - scheme = urlparse(url).scheme if not scheme: return url.lstrip('|') elif scheme == 'salt': path, saltenv = parse(url) - return create(path.lstrip('|'), saltenv) + if salt.utils.is_windows() and '|' in url: + return create(path.lstrip('_'), saltenv) + else: + return create(path.lstrip('|'), saltenv) else: return url diff --git a/tests/unit/utils/test_url.py b/tests/unit/utils/test_url.py index 95caa0253e..af7efba323 100644 --- a/tests/unit/utils/test_url.py +++ b/tests/unit/utils/test_url.py @@ -38,6 +38,8 @@ class UrlTestCase(TestCase): ''' path = '?funny/path with {interesting|chars}' url = 'salt://' + path + if salt.utils.is_windows(): + path = '_funny/path with {interesting_chars}' self.assertEqual(salt.utils.url.parse(url), (path, None)) @@ -48,6 +50,8 @@ class UrlTestCase(TestCase): saltenv = 'ambience' path = '?funny/path&with {interesting|chars}' url = 'salt://' + path + '?saltenv=' + saltenv + if salt.utils.is_windows(): + path = '_funny/path&with {interesting_chars}' self.assertEqual(salt.utils.url.parse(url), (path, saltenv)) @@ -59,6 +63,8 @@ class UrlTestCase(TestCase): ''' path = '? interesting/&path.filetype' url = 'salt://' + path + if salt.utils.is_windows(): + url = 'salt://_ interesting/&path.filetype' self.assertEqual(salt.utils.url.create(path), url) @@ -68,6 +74,8 @@ class UrlTestCase(TestCase): ''' saltenv = 'raumklang' path = '? interesting/&path.filetype' + if salt.utils.is_windows(): + path = '_ interesting/&path.filetype' url = 'salt://' + path + '?saltenv=' + saltenv @@ -149,6 +157,8 @@ class UrlTestCase(TestCase): ''' path = 'dir/file.conf' escaped_path = '|' + path + if salt.utils.is_windows(): + escaped_path = path self.assertEqual(salt.utils.url.escape(path), escaped_path) @@ -167,6 +177,8 @@ class UrlTestCase(TestCase): path = 'dir/file.conf' url = 'salt://' + path escaped_url = 'salt://|' + path + if salt.utils.is_windows(): + escaped_url = url self.assertEqual(salt.utils.url.escape(url), escaped_url) From d6a5e85632245a6624dfc2fa99623697fc3cec2d Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 17 Aug 2017 16:30:46 -0400 Subject: [PATCH 366/508] Small cleanup to dockermod.save Removes the call to get ``compression`` out of the kwargs dict as ``compression`` is already listed in the list of positional kwargs. --- salt/modules/dockermod.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 0d735f0f65..d796e49913 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -3843,7 +3843,6 @@ def save(name, if os.path.exists(path) and not overwrite: raise CommandExecutionError('{0} already exists'.format(path)) - compression = kwargs.get('compression') if compression is None: if path.endswith('.tar.gz') or path.endswith('.tgz'): compression = 'gzip' From ea4d7f41765ddb510de91eb3e73c04188645f35c Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Tue, 25 Jul 2017 15:00:55 -0400 Subject: [PATCH 367/508] Updated testinfra modules to work with more recent versions The TestInfra library changed the package layout recently which caused the extension module to stop working. This patch addresses those updates and will work across all versions of the library. --- salt/modules/testinframod.py | 10 +++++++--- salt/states/testinframod.py | 8 ++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/salt/modules/testinframod.py b/salt/modules/testinframod.py index 4be4357d21..d705a3a2c9 100644 --- a/salt/modules/testinframod.py +++ b/salt/modules/testinframod.py @@ -285,13 +285,17 @@ def _register_functions(): functions, and then register them in the module namespace so that they can be called via salt. """ - for module_ in modules.__all__: + try: + modules_ = [_to_snake_case(module_) for module_ in modules.__all__] + except AttributeError: + modules_ = [module_ for module_ in modules.modules] + + for mod_name in modules_: mod_name = _to_snake_case(module_) mod_func = _copy_function(mod_name, str(mod_name)) - mod_func.__doc__ = _build_doc(module_) + mod_func.__doc__ = _build_doc(mod_name) __all__.append(mod_name) globals()[mod_name] = mod_func - if TESTINFRA_PRESENT: _register_functions() diff --git a/salt/states/testinframod.py b/salt/states/testinframod.py index 0350f4d385..a31a72c3e0 100644 --- a/salt/states/testinframod.py +++ b/salt/states/testinframod.py @@ -52,8 +52,12 @@ def _to_snake_case(pascal_case): def _generate_functions(): - for module in modules.__all__: - module_name = _to_snake_case(module) + try: + modules_ = [_to_snake_case(module_) for module_ in modules.__all__] + except AttributeError: + modules_ = [module_ for module_ in modules.modules] + + for module_name in modules_: func_name = 'testinfra.{0}'.format(module_name) __all__.append(module_name) log.debug('Generating state for module %s as function %s', From 3e3f7f5d8e093015fae639ce4c7dfb5e3d27c00c Mon Sep 17 00:00:00 2001 From: Aljosha Friemann Date: Sat, 5 Aug 2017 03:53:48 +0200 Subject: [PATCH 368/508] Catch TypeError thrown by m2crypto when parsing missing subjects in certificate files. --- salt/modules/x509.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/modules/x509.py b/salt/modules/x509.py index 53dac32d31..ae5f8c7723 100644 --- a/salt/modules/x509.py +++ b/salt/modules/x509.py @@ -330,10 +330,14 @@ def _parse_subject(subject): for nid_name, nid_num in six.iteritems(subject.nid): if nid_num in nids: continue - val = getattr(subject, nid_name) - if val: - ret[nid_name] = val - nids.append(nid_num) + try: + val = getattr(subject, nid_name) + if val: + ret[nid_name] = val + nids.append(nid_num) + except TypeError as e: + if e.args and e.args[0] == 'No string argument provided': + pass return ret From 350c0767dcb06a2c4f21991ce71af0b8c58b0060 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Thu, 17 Aug 2017 09:46:35 +0200 Subject: [PATCH 369/508] Try to fix #42989 by doing sslVerify and refspecs for origin remote only if there is no remotes --- salt/utils/gitfs.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index f9bc9b9b01..bad3cc883e 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -976,17 +976,17 @@ class GitPython(GitProvider): else: new = True - try: - ssl_verify = self.repo.git.config('--get', 'http.sslVerify') - except git.exc.GitCommandError: - ssl_verify = '' - desired_ssl_verify = str(self.ssl_verify).lower() - if ssl_verify != desired_ssl_verify: - self.repo.git.config('http.sslVerify', desired_ssl_verify) + try: + ssl_verify = self.repo.git.config('--get', 'http.sslVerify') + except git.exc.GitCommandError: + ssl_verify = '' + desired_ssl_verify = str(self.ssl_verify).lower() + if ssl_verify != desired_ssl_verify: + self.repo.git.config('http.sslVerify', desired_ssl_verify) - # Ensure that refspecs for the "origin" remote are set up as configured - if hasattr(self, 'refspecs'): - self.configure_refspecs() + # Ensure that refspecs for the "origin" remote are set up as configured + if hasattr(self, 'refspecs'): + self.configure_refspecs() return new @@ -1454,18 +1454,18 @@ class Pygit2(GitProvider): else: new = True - try: - ssl_verify = self.repo.config.get_bool('http.sslVerify') - except KeyError: - ssl_verify = None - if ssl_verify != self.ssl_verify: - self.repo.config.set_multivar('http.sslVerify', - '', - str(self.ssl_verify).lower()) + try: + ssl_verify = self.repo.config.get_bool('http.sslVerify') + except KeyError: + ssl_verify = None + if ssl_verify != self.ssl_verify: + self.repo.config.set_multivar('http.sslVerify', + '', + str(self.ssl_verify).lower()) - # Ensure that refspecs for the "origin" remote are set up as configured - if hasattr(self, 'refspecs'): - self.configure_refspecs() + # Ensure that refspecs for the "origin" remote are set up as configured + if hasattr(self, 'refspecs'): + self.configure_refspecs() return new From 73315f0cf03f28a8efa30c91d463e4a33fb8fe46 Mon Sep 17 00:00:00 2001 From: matt Date: Thu, 17 Aug 2017 17:06:17 -0400 Subject: [PATCH 370/508] Issue #43036 Bhyve virtual grain in Linux VMs --- salt/grains/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index 248984edf3..57937f2035 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -764,6 +764,8 @@ def _virtual(osdata): grains['virtual_subtype'] = 'ovirt' elif 'Google' in output: grains['virtual'] = 'gce' + elif 'BHYVE' in output: + grains['virtual'] = 'bhyve' except IOError: pass elif osdata['kernel'] == 'FreeBSD': From 7e269cb36885ddd3d5ab2555e287f284f4c7b588 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 17 Aug 2017 15:46:10 -0600 Subject: [PATCH 371/508] catch ImportError for kubernetes.client import --- salt/modules/kubernetes.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 3a54e34fd3..2e2056fb43 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -35,20 +35,18 @@ try: import kubernetes.client from kubernetes.client.rest import ApiException from urllib3.exceptions import HTTPError + try: + # There is an API change in Kubernetes >= 2.0.0. + from kubernetes.client import V1beta1Deployment as AppsV1beta1Deployment + from kubernetes.client import V1beta1DeploymentSpec as AppsV1beta1DeploymentSpec + except ImportError: + from kubernetes.client import AppsV1beta1Deployment + from kubernetes.client import AppsV1beta1DeploymentSpec HAS_LIBS = True except ImportError: HAS_LIBS = False -try: - # There is an API change in Kubernetes >= 2.0.0. - from kubernetes.client import V1beta1Deployment as AppsV1beta1Deployment - from kubernetes.client import V1beta1DeploymentSpec as AppsV1beta1DeploymentSpec -except ImportError: - from kubernetes.client import AppsV1beta1Deployment - from kubernetes.client import AppsV1beta1DeploymentSpec - - log = logging.getLogger(__name__) __virtualname__ = 'kubernetes' From fbe54c9a33da2ef69ed6da4f8debffcb390f6ab4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 17 Aug 2017 15:54:38 -0600 Subject: [PATCH 372/508] Remove unused import six (lint) --- tests/unit/utils/test_find.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/utils/test_find.py b/tests/unit/utils/test_find.py index 7ba487f8d2..a6a48fb473 100644 --- a/tests/unit/utils/test_find.py +++ b/tests/unit/utils/test_find.py @@ -13,7 +13,6 @@ from tests.support.unit import skipIf, TestCase from tests.support.paths import TMP # Import salt libs -import salt.ext.six as six import salt.utils import salt.utils.find From fc306fc8c3f46e33ac4ee281cd10aa32186b3ee0 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 17 Aug 2017 15:57:42 -0600 Subject: [PATCH 373/508] Add missing colon in `if` statement --- salt/modules/jenkins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/jenkins.py b/salt/modules/jenkins.py index 77ae100b88..86c6948810 100644 --- a/salt/modules/jenkins.py +++ b/salt/modules/jenkins.py @@ -451,7 +451,7 @@ def get_job_config(name=None): server = _connect() - if not job_exists(name) + if not job_exists(name): raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) job_info = server.get_job_config(name) From f8f74a310c12d95fc090a1b910a0b4848831f22b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 17:19:34 -0500 Subject: [PATCH 374/508] Update localfs cache tests to reflect changes to func naming --- tests/unit/cache/test_localfs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/cache/test_localfs.py b/tests/unit/cache/test_localfs.py index 3aa2608a79..e1aa8ecc32 100644 --- a/tests/unit/cache/test_localfs.py +++ b/tests/unit/cache/test_localfs.py @@ -209,26 +209,26 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): with patch('os.remove', MagicMock(side_effect=OSError)): self.assertRaises(SaltCacheError, localfs.flush, bank='', key='key', cachedir='/var/cache/salt') - # 'ls' function tests: 3 + # 'list' function tests: 3 - def test_ls_no_base_dir(self): + def test_list_no_base_dir(self): ''' Tests that the ls function returns an empty list if the bank directory doesn't exist. ''' with patch('os.path.isdir', MagicMock(return_value=False)): - self.assertEqual(localfs.ls(bank='', cachedir=''), []) + self.assertEqual(localfs.list_(bank='', cachedir=''), []) - def test_ls_error_raised_no_bank_directory_access(self): + def test_list_error_raised_no_bank_directory_access(self): ''' Tests that a SaltCacheError is raised when there is a problem accessing the cache bank directory. ''' with patch('os.path.isdir', MagicMock(return_value=True)): with patch('os.listdir', MagicMock(side_effect=OSError)): - self.assertRaises(SaltCacheError, localfs.ls, bank='', cachedir='') + self.assertRaises(SaltCacheError, localfs.list_, bank='', cachedir='') - def test_ls_success(self): + def test_list_success(self): ''' Tests the return of the ls function containing bank entries. ''' @@ -240,7 +240,7 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): # Now test the return of the ls function with patch.dict(localfs.__opts__, {'cachedir': tmp_dir}): - self.assertEqual(localfs.ls(bank='bank', cachedir=tmp_dir), ['key']) + self.assertEqual(localfs.list_(bank='bank', cachedir=tmp_dir), ['key']) # 'contains' function tests: 1 From 33fd8ff93989018ce045a50dad356fff3216b3fc Mon Sep 17 00:00:00 2001 From: garethgreenaway Date: Thu, 17 Aug 2017 15:22:31 -0700 Subject: [PATCH 375/508] Update jenkins.py --- salt/states/jenkins.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/states/jenkins.py b/salt/states/jenkins.py index 421dcda892..b194c14b9f 100644 --- a/salt/states/jenkins.py +++ b/salt/states/jenkins.py @@ -15,7 +15,7 @@ import logging # Import Salt libs import salt.ext.six as six import salt.utils -from salt.exceptions import CommandExecutionError, SaltInvocationError +from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -44,7 +44,6 @@ def present(name, 'changes': {}, 'comment': ['Job {0} is up to date.'.format(name)]} - if __salt__['jenkins.job_exists'](name): _current_job_config = __salt__['jenkins.get_job_config'](name) buf = six.moves.StringIO(_current_job_config) @@ -98,7 +97,6 @@ def absent(name, 'changes': {}, 'comment': []} - if __salt__['jenkins.job_exists'](name): try: __salt__['jenkins.delete_job'](name) From 2b5af5b59df98f1f1270d68a5afa10a80aa9cff0 Mon Sep 17 00:00:00 2001 From: Steven Joseph Date: Fri, 18 Aug 2017 18:24:31 +1000 Subject: [PATCH 376/508] Remove refs/tags prefix from remote tags --- salt/states/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/git.py b/salt/states/git.py index f12ae29c26..d6f8daaa23 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1309,7 +1309,7 @@ def latest(name, comments ) remote_tags = set([ - x.split('/')[-1] for x in __salt__['git.ls_remote']( + x.replace('refs/tags/', '') for x in __salt__['git.ls_remote']( cwd=target, remote=remote, opts="--tags", From 35e45049e269b46940a812c2f70cafb90439e0d8 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 17 Aug 2017 13:45:10 -0600 Subject: [PATCH 377/508] use a ruby gem that doesn't have dependencies --- tests/integration/modules/gem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/modules/gem.py b/tests/integration/modules/gem.py index da44c7a728..b65c54812e 100644 --- a/tests/integration/modules/gem.py +++ b/tests/integration/modules/gem.py @@ -18,9 +18,9 @@ import salt.utils.http GEM = 'tidy' GEM_VER = '1.1.2' -OLD_GEM = 'test' -OLD_VERSION = '0.2.0' -NEW_VERSION = '1.0.0' +OLD_GEM = 'brass' +OLD_VERSION = '1.0.0' +NEW_VERSION = '1.2.1' GEM_LIST = [GEM, OLD_GEM] From caf78d206d5b58c8f2a5c4cc15af6c165cef0f94 Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Fri, 18 Aug 2017 15:15:20 +0200 Subject: [PATCH 378/508] Fixed imports for pytest Imports now match the new test-suite introduced in 2017.7. --- tests/unit/modules/kubernetes_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/modules/kubernetes_test.py b/tests/unit/modules/kubernetes_test.py index 5f2dcdc1dc..6efc4d790d 100644 --- a/tests/unit/modules/kubernetes_test.py +++ b/tests/unit/modules/kubernetes_test.py @@ -5,11 +5,10 @@ # Import Python Libs from __future__ import absolute_import -import os # Import Salt Testing Libs -from salttesting import TestCase, skipIf -from salttesting.mock import ( +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( Mock, patch, NO_MOCK, From 3c99e61637f91f5e4210277a81e95be6904fe92f Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Fri, 18 Aug 2017 15:19:34 +0200 Subject: [PATCH 379/508] Renamed test to match new convention --- tests/unit/modules/{kubernetes_test.py => test_kubernetes.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/unit/modules/{kubernetes_test.py => test_kubernetes.py} (100%) diff --git a/tests/unit/modules/kubernetes_test.py b/tests/unit/modules/test_kubernetes.py similarity index 100% rename from tests/unit/modules/kubernetes_test.py rename to tests/unit/modules/test_kubernetes.py From d7eef70df063c9cdf1aa591901e69a906617f263 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 18 Aug 2017 09:44:22 -0400 Subject: [PATCH 380/508] Update release version number for jenkins.run function This function was added for the 2017.7.0 release, not Carbon. --- salt/modules/jenkins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/jenkins.py b/salt/modules/jenkins.py index d327d09292..7e7d194853 100644 --- a/salt/modules/jenkins.py +++ b/salt/modules/jenkins.py @@ -91,7 +91,7 @@ def _connect(): def run(script): ''' - .. versionadded:: Carbon + .. versionadded:: 2017.7.0 Execute a groovy script on the jenkins master From 153a463b864f01e3a67dabc29e4e5531c0da670a Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 18 Aug 2017 10:59:12 -0400 Subject: [PATCH 381/508] Lint: Add missing blank line --- salt/states/jenkins.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/states/jenkins.py b/salt/states/jenkins.py index 89c268b048..047ee8d6fe 100644 --- a/salt/states/jenkins.py +++ b/salt/states/jenkins.py @@ -36,6 +36,7 @@ def _elements_equal(e1, e2): return False return all(_elements_equal(c1, c2) for c1, c2 in zip(e1, e2)) + def _fail(ret, msg): ret['comment'] = msg ret['result'] = False From 964cebd954d341d091008fbb960f18cdd49a3fd7 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Sat, 19 Aug 2017 02:23:26 +1000 Subject: [PATCH 382/508] safe_filename_leaf(file_basename) and safe_filepath(file_path_name) fe_filename_leaf input the basename of a file, without the directory tree, and returns a safe name to use i.e. only the required characters are converted by urllib.quote If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode. For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible windows is \ / : * ? " < > | posix is / safe_filepath input the full path and filename, splits on directory separator and calls safe_filename_leaf for each part of the path. --- salt/utils/__init__.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index ec018f1ad7..04fcf017e5 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -163,6 +163,37 @@ def is_empty(filename): return False +def safe_filename_leaf(file_basename): + ''' + input the basename of a file, without the directory tree, and returns a safe name to use + i.e. only the required characters are converted by urllib.quote + If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode. + For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible + windows is \ / : * ? " < > | posix is / + ''' + def _replace(re_obj): + return urllib.quote(re_obj.group(0), safe=u'') + if not isinstance(file_basename, six.text_type): + # the following string is not prefixed with u + return re.sub('[\\\/:*?"<>|]', + _replace,six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace')) + # the following string is prefixed with u + return re.sub(u'[\\\/:*?"<>|]', _replace, file_basename, flags=re.UNICODE) + + +def safe_filepath(file_path_name): + ''' + input the full path and filename, splits on directory separator and calls safe_filename_leaf for + each part of the path. + ''' + (drive,path) = os.path.splitdrive(file_path_name) + path = os.sep.join([safe_filename_leaf(file_section) for file_section in file_path_name.rsplit(os.sep)]) + if drive: + return os.sep.join([drive, path]) + else: + return path + + def is_hex(value): ''' Returns True if value is a hexidecimal string, otherwise returns False From 8c864f02c7f7ef72abf1a754c07be27d69832e8c Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Sat, 19 Aug 2017 02:35:37 +1000 Subject: [PATCH 383/508] fix missing imports --- salt/utils/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 04fcf017e5..cbdf4ae138 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -16,6 +16,7 @@ import json import logging import numbers import os +import os.path import posixpath import random import re @@ -32,6 +33,7 @@ import warnings import string import subprocess import getpass +import urllib # Import 3rd-party libs from salt.ext import six From ebdca3a0f54b2897fb1e0fca2eb222c346633a51 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 18 Aug 2017 11:25:32 -0600 Subject: [PATCH 384/508] Update pkg-scripts Improves logging Removes /opt/salt/bin directory before install Removes symlink to salt-config before install --- pkg/osx/pkg-scripts/postinstall | 124 ++++++++++++++++++++++++-------- pkg/osx/pkg-scripts/preinstall | 90 ++++++++++++++++++++--- 2 files changed, 173 insertions(+), 41 deletions(-) diff --git a/pkg/osx/pkg-scripts/postinstall b/pkg/osx/pkg-scripts/postinstall index ed8ee7c142..f521666a6f 100755 --- a/pkg/osx/pkg-scripts/postinstall +++ b/pkg/osx/pkg-scripts/postinstall @@ -15,66 +15,130 @@ # This script is run as a part of the macOS Salt Installation # ############################################################################### -echo "Post install started on:" > /tmp/postinstall.txt -date >> /tmp/postinstall.txt + +############################################################################### +# Define Variables +############################################################################### +# Get Minor Version +OSX_VERSION=$(sw_vers | grep ProductVersion | cut -f 2 -d: | tr -d '[:space:]') +MINOR=$(echo ${OSX_VERSION} | cut -f 2 -d.) +# Path Variables +INSTALL_DIR="/opt/salt" +BIN_DIR="$INSTALL_DIR/bin" +CONFIG_DIR="/etc/salt" +TEMP_DIR="/tmp" +SBIN_DIR="/usr/local/sbin" + +############################################################################### +# Set up logging and error handling +############################################################################### +echo "Post install script started on:" > "$TEMP_DIR/postinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/postinstall.txt" trap 'quit_on_error $LINENO $BASH_COMMAND' ERR quit_on_error() { - echo "$(basename $0) caught error on line : $1 command was: $2" >> /tmp/postinstall.txt + echo "$(basename $0) caught error on line : $1 command was: $2" >> "$TEMP_DIR/postinstall.txt" exit -1 } ############################################################################### # Check for existing minion config, copy if it doesn't exist ############################################################################### -if [ ! -f /etc/salt/minion ]; then - echo "Config copy: Started..." >> /tmp/postinstall.txt - cp /etc/salt/minion.dist /etc/salt/minion - echo "Config copy: Successful" >> /tmp/postinstall.txt +if [ ! -f "$CONFIG_DIR/minion" ]; then + echo "Config: Copy Started..." >> "$TEMP_DIR/postinstall.txt" + cp "$CONFIG_DIR/minion.dist" "$CONFIG_DIR/minion" + echo "Config: Copied Successfully" >> "$TEMP_DIR/postinstall.txt" fi ############################################################################### # Create symlink to salt-config.sh ############################################################################### -# echo "Symlink: Creating symlink for salt-config..." >> /tmp/postinstall.txt -if [ ! -d "/usr/local/sbin" ]; then - mkdir /usr/local/sbin +if [ ! -d "$SBIN_DIR" ]; then + echo "Symlink: Creating $SBIN_DIR..." >> "$TEMP_DIR/postinstall.txt" + mkdir "$SBIN_DIR" + echo "Symlink: Created Successfully" >> "$TEMP_DIR/postinstall.txt" fi -ln -sf /opt/salt/bin/salt-config.sh /usr/local/sbin/salt-config +echo "Symlink: Creating symlink for salt-config..." >> "$TEMP_DIR/postinstall.txt" +ln -sf "$BIN_DIR/salt-config.sh" "$SBIN_DIR/salt-config" +echo "Symlink: Created Successfully" >> "$TEMP_DIR/postinstall.txt" ############################################################################### # Add salt to paths.d ############################################################################### -# echo "Path: Adding salt to the path..." >> /tmp/postinstall.txt if [ ! -d "/etc/paths.d" ]; then + echo "Path: Creating paths.d directory..." >> "$TEMP_DIR/postinstall.txt" mkdir /etc/paths.d + echo "Path: Created Successfully" >> "$TEMP_DIR/postinstall.txt" fi -sh -c 'echo "/opt/salt/bin" > /etc/paths.d/salt' -sh -c 'echo "/usr/local/sbin" >> /etc/paths.d/salt' +echo "Path: Adding salt to the path..." >> "$TEMP_DIR/postinstall.txt" +sh -c "echo \"$BIN_DIR\" > /etc/paths.d/salt" +sh -c "echo \"$SBIN_DIR\" >> /etc/paths.d/salt" +echo "Path: Added Successfully" >> "$TEMP_DIR/postinstall.txt" ############################################################################### # Register Salt as a service ############################################################################### -echo "Service start: Enabling service..." >> /tmp/postinstall.txt -launchctl enable system/com.saltstack.salt.minion -echo "Service start: Bootstrapping service..." >> /tmp/postinstall.txt -launchctl bootstrap system /Library/LaunchDaemons/com.saltstack.salt.minion.plist +setup_services_maverick() { + echo "Service: Using old (< 10.10) launchctl interface" >> "$TEMP_DIR/postinstall.txt" + if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then + echo "Service: Stopping salt-minion..." >> "$TEMP_DIR/postinstall.txt" + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/postinstall.txt" + fi; + echo "Service: Starting salt-minion..." >> "$TEMP_DIR/postinstall.txt" + launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + echo "Service: Started Successfully" >> "$TEMP_DIR/postinstall.txt" -if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Service is running" >> /tmp/postinstall.txt -else - echo "Service start: Kickstarting service..." >> /tmp/postinstall.txt - launchctl kickstart -kp system/com.saltstack.salt.minion -fi + echo "Service: Disabling Master, Syndic, and API services..." >> "$TEMP_DIR/postinstall.txt" + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.api.plist + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.syndic.plist + echo "Service: Disabled Successfully" >> "$TEMP_DIR/postinstall.txt" -echo "Service start: Successful" >> /tmp/postinstall.txt + return 0 +} -echo "Service disable: Disabling Master, Syndic, and API" >> /tmp/postinstall.txt +setup_services_yosemite_and_later() { + echo "Service: Using new (>= 10.10) launchctl interface" >> "$TEMP_DIR/postinstall.txt" + echo "Service: Enabling salt-minion..." >> "$TEMP_DIR/postinstall.txt" + launchctl enable system/com.saltstack.salt.minion + echo "Service: Enabled Successfully" >> "$TEMP_DIR/postinstall.txt" -launchctl disable system/com.saltstack.salt.master -launchctl disable system/com.saltstack.salt.syndic -launchctl disable system/com.saltstack.salt.api + echo "Service: Bootstrapping salt-minion..." >> "$TEMP_DIR/postinstall.txt" + launchctl bootstrap system /Library/LaunchDaemons/com.saltstack.salt.minion.plist + echo "Service: Bootstrapped Successfully" >> "$TEMP_DIR/postinstall.txt" -echo "Post install completed successfully" >> /tmp/postinstall.txt + if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then + echo "Service: Service Running" >> "$TEMP_DIR/postinstall.txt" + else + echo "Service: Kickstarting Service..." >> "$TEMP_DIR/postinstall.txt" + launchctl kickstart -kp system/com.saltstack.salt.minion + echo "Service: Kickstarted Successfully" >> "$TEMP_DIR/postinstall.txt" + fi + + echo "Service: Started Successfully" >> "$TEMP_DIR/postinstall.txt" + + echo "Service: Disabling Master, Syndic, and API services" >> "$TEMP_DIR/postinstall.txt" + launchctl disable system/com.saltstack.salt.master + launchctl disable system/com.saltstack.salt.syndic + launchctl disable system/com.saltstack.salt.api + echo "Service: Disabled Successfully" >> "$TEMP_DIR/postinstall.txt" + + return 0 +} + +echo "Service: Configuring..." >> "$TEMP_DIR/postinstall.txt" +case $MINOR in + 9 ) + setup_services_maverick; + ;; + * ) + setup_services_yosemite_and_later; + ;; +esac +echo "Service: Configured Successfully" >> "$TEMP_DIR/postinstall.txt" + +echo "Post install completed successfully on:" >> "$TEMP_DIR/postinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/postinstall.txt" exit 0 diff --git a/pkg/osx/pkg-scripts/preinstall b/pkg/osx/pkg-scripts/preinstall index 4112e07f5f..c29d07c6b7 100755 --- a/pkg/osx/pkg-scripts/preinstall +++ b/pkg/osx/pkg-scripts/preinstall @@ -6,7 +6,8 @@ # Date: December 2015 # # Description: This script stops the salt minion service before attempting to -# install Salt on macOS +# install Salt on macOS. It also removes the /opt/salt/bin +# directory. # # Requirements: # - None @@ -15,26 +16,93 @@ # This script is run as a part of the macOS Salt Installation # ############################################################################### -echo "Preinstall started on:" > /tmp/preinstall.txt -date >> /tmp/preinstall.txt + +############################################################################### +# Define Variables +############################################################################### +# Get Minor Version +OSX_VERSION=$(sw_vers | grep ProductVersion | cut -f 2 -d: | tr -d '[:space:]') +MINOR=$(echo ${OSX_VERSION} | cut -f 2 -d.) +# Path Variables +INSTALL_DIR="/opt/salt" +BIN_DIR="$INSTALL_DIR/bin" +CONFIG_DIR="/etc/salt" +TEMP_DIR="/tmp" +SBIN_DIR="/usr/local/sbin" + +############################################################################### +# Set up logging and error handling +############################################################################### +echo "Preinstall started on:" > "$TEMP_DIR/preinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/preinstall.txt" trap 'quit_on_error $LINENO $BASH_COMMAND' ERR quit_on_error() { - echo "$(basename $0) caught error on line : $1 command was: $2" >> /tmp/preinstall.txt + echo "$(basename $0) caught error on line : $1 command was: $2" >> "$TEMP_DIR/preinstall.txt" exit -1 } ############################################################################### # Stop the service ############################################################################### -if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Stop service: Started..." >> /tmp/preinstall.txt -# /bin/launchctl unload "/Library/LaunchDaemons/com.saltstack.salt.minion.plist" - launchctl disable system/com.saltstack.salt.minion - launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.minion.plist - echo "Stop service: Successful" >> /tmp/preinstall.txt +stop_service_maverick() { + echo "Service: Using old (< 10.10) launchctl interface" >> "$TEMP_DIR/preinstall.txt" + if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then + echo "Service: Unloading..." >> "$TEMP_DIR/preinstall.txt" + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.api.plist + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.syndic.plist + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" + fi +} + +stop_service_yosemite_and_later() { + echo "Service: Using new (>= 10.10) launchctl interface" >> "$TEMP_DIR/preinstall.txt" + if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then + echo "Service: Stopping..." >> "$TEMP_DIR/preinstall.txt" + launchctl disable system/com.saltstack.salt.minion + launchctl disable system/com.saltstack.salt.master + launchctl disable system/com.saltstack.salt.syndic + launchctl disable system/com.saltstack.salt.api + launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.minion.plist + launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.master.plist + launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.syndic.plist + launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.api.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" + fi +} + +echo "Service: Configuring..." >> "$TEMP_DIR/preinstall.txt" +case $MINOR in + 9 ) + stop_service_maverick; + ;; + * ) + stop_service_yosemite_and_later; + ;; +esac +echo "Service: Configured Successfully" >> "$TEMP_DIR/preinstall.txt" + +############################################################################### +# Remove the Symlink to salt-config.sh +############################################################################### +if [ -L "$SBIN_DIR/salt-config" ]; then + echo "Cleanup: Removing Symlink $BIN_DIR/salt-config" >> "$TEMP_DIR/preinstall.txt" + rm "$SBIN_DIR/salt-config" + echo "Cleanup: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" fi -echo "Preinstall Completed Successfully" >> /tmp/preinstall.txt +############################################################################### +# Remove the $BIN_DIR directory +############################################################################### +if [ -d "$BIN_DIR" ]; then + echo "Cleanup: Removing $BIN_DIR" >> "$TEMP_DIR/preinstall.txt" + rm -rf "$BIN_DIR" + echo "Cleanup: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" +fi + +echo "Preinstall Completed Successfully on:" >> "$TEMP_DIR/preinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/preinstall.txt" exit 0 From 3b62bf953cff57ac83db27eb2bb4c5e9329f4322 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 18 Aug 2017 11:44:35 -0600 Subject: [PATCH 385/508] Remove salt from the path --- pkg/osx/pkg-scripts/preinstall | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/osx/pkg-scripts/preinstall b/pkg/osx/pkg-scripts/preinstall index c29d07c6b7..e4ae38baa5 100755 --- a/pkg/osx/pkg-scripts/preinstall +++ b/pkg/osx/pkg-scripts/preinstall @@ -102,6 +102,15 @@ if [ -d "$BIN_DIR" ]; then echo "Cleanup: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" fi +############################################################################### +# Remove the salt from the paths.d +############################################################################### +if [ ! -f "/etc/paths.d/salt" ]; then + echo "Path: Removing salt from the path..." >> "$TEMP_DIR/preinstall.txt" + rm "/etc/paths.d/salt" + echo "Path: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" +fi + echo "Preinstall Completed Successfully on:" >> "$TEMP_DIR/preinstall.txt" date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/preinstall.txt" From f44f5b70dc5955552f60cb2234f8cd06873f0c9b Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 18 Aug 2017 11:52:34 -0600 Subject: [PATCH 386/508] Only stop services if they are running Otherwise it will cause an error and the installation will fail --- pkg/osx/pkg-scripts/preinstall | 36 ++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/pkg/osx/pkg-scripts/preinstall b/pkg/osx/pkg-scripts/preinstall index e4ae38baa5..bdd9a27341 100755 --- a/pkg/osx/pkg-scripts/preinstall +++ b/pkg/osx/pkg-scripts/preinstall @@ -48,26 +48,50 @@ quit_on_error() { stop_service_maverick() { echo "Service: Using old (< 10.10) launchctl interface" >> "$TEMP_DIR/preinstall.txt" if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Service: Unloading..." >> "$TEMP_DIR/preinstall.txt" + echo "Service: Unloading minion..." >> "$TEMP_DIR/preinstall.txt" launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist - launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.api.plist + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.master" &> /dev/null; then + echo "Service: Unloading master..." >> "$TEMP_DIR/preinstall.txt" launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.syndic" &> /dev/null; then + echo "Service: Unloading syndic..." >> "$TEMP_DIR/preinstall.txt" launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.syndic.plist echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" fi + if /bin/launchctl list "com.saltstack.salt.api" &> /dev/null; then + echo "Service: Unloading api..." >> "$TEMP_DIR/preinstall.txt" + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.api.plist + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" + fi } stop_service_yosemite_and_later() { echo "Service: Using new (>= 10.10) launchctl interface" >> "$TEMP_DIR/preinstall.txt" if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Service: Stopping..." >> "$TEMP_DIR/preinstall.txt" + echo "Service: Stopping minion..." >> "$TEMP_DIR/preinstall.txt" launchctl disable system/com.saltstack.salt.minion - launchctl disable system/com.saltstack.salt.master - launchctl disable system/com.saltstack.salt.syndic - launchctl disable system/com.saltstack.salt.api launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.minion.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.master" &> /dev/null; then + echo "Service: Stopping master..." >> "$TEMP_DIR/preinstall.txt" + launchctl disable system/com.saltstack.salt.master launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.master.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.syndic" &> /dev/null; then + echo "Service: Stopping syndic..." >> "$TEMP_DIR/preinstall.txt" + launchctl disable system/com.saltstack.salt.syndic launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.syndic.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.api" &> /dev/null; then + echo "Service: Stopping api..." >> "$TEMP_DIR/preinstall.txt" + launchctl disable system/com.saltstack.salt.api launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.api.plist echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" fi From 2dd62aa1daef85be8046170f3ad4ae7feac436cd Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 18 Aug 2017 12:09:08 -0600 Subject: [PATCH 387/508] Add more information to the description --- pkg/osx/pkg-scripts/preinstall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osx/pkg-scripts/preinstall b/pkg/osx/pkg-scripts/preinstall index bdd9a27341..c28a5e9e43 100755 --- a/pkg/osx/pkg-scripts/preinstall +++ b/pkg/osx/pkg-scripts/preinstall @@ -7,7 +7,7 @@ # # Description: This script stops the salt minion service before attempting to # install Salt on macOS. It also removes the /opt/salt/bin -# directory. +# directory, symlink to salt-config, and salt from paths.d. # # Requirements: # - None From ef8a14cdf93de26e272e4e4e73a01e14a1aca2b7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 18 Aug 2017 12:19:04 -0600 Subject: [PATCH 388/508] Remove /opt/salt instead of /opt/salt/bin --- pkg/osx/pkg-scripts/preinstall | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/osx/pkg-scripts/preinstall b/pkg/osx/pkg-scripts/preinstall index c28a5e9e43..c919cafcb1 100755 --- a/pkg/osx/pkg-scripts/preinstall +++ b/pkg/osx/pkg-scripts/preinstall @@ -118,11 +118,11 @@ if [ -L "$SBIN_DIR/salt-config" ]; then fi ############################################################################### -# Remove the $BIN_DIR directory +# Remove the $INSTALL_DIR directory ############################################################################### -if [ -d "$BIN_DIR" ]; then - echo "Cleanup: Removing $BIN_DIR" >> "$TEMP_DIR/preinstall.txt" - rm -rf "$BIN_DIR" +if [ -d "$INSTALL_DIR" ]; then + echo "Cleanup: Removing $INSTALL_DIR" >> "$TEMP_DIR/preinstall.txt" + rm -rf "$INSTALL_DIR" echo "Cleanup: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" fi From 0ffc57d1df97b5c6de5d6cdd5257d022e56bf6e0 Mon Sep 17 00:00:00 2001 From: Pablo Hernandez Date: Fri, 18 Aug 2017 14:29:19 -0400 Subject: [PATCH 389/508] Have docker.save use the image name when valid if not use image id, issue when loading and image is savid with id issue #43043 --- salt/modules/dockermod.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index d796e49913..ef33254f9f 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -3880,8 +3880,9 @@ def save(name, saved_path = salt.utils.files.mkstemp() else: saved_path = path - - cmd = ['docker', 'save', '-o', saved_path, inspect_image(name)['Id']] + # use the image name if its valid if not use the image id + image_to_save = name if name in inspect_image(name)['RepoTags'] else inspect_image(name)['Id'] + cmd = ['docker', 'save', '-o', saved_path, image_to_save] time_started = time.time() result = __salt__['cmd.run_all'](cmd, python_shell=False) if result['retcode'] != 0: From ca1b1bb6334da6597b97ae411d65b98844c8f661 Mon Sep 17 00:00:00 2001 From: Arount Date: Fri, 30 Jun 2017 14:54:54 +0200 Subject: [PATCH 390/508] use configparser to parse yum repo file --- salt/modules/yumpkg.py | 46 +++++++++++++----------------------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index b0a183d70a..9d98322cc8 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -2524,41 +2524,23 @@ def _parse_repo_file(filename): ''' Turn a single repo file into a dict ''' - repos = {} - header = '' - repo = '' - with salt.utils.fopen(filename, 'r') as rfile: - for line in rfile: - if line.startswith('['): - repo = line.strip().replace('[', '').replace(']', '') - repos[repo] = {} + parsed = configparser.ConfigParser() + parsed.read(filename) + config = {} - # Even though these are essentially uselss, I want to allow the - # user to maintain their own comments, etc - if not line: - if not repo: - header += line - if line.startswith('#'): - if not repo: - header += line - else: - if 'comments' not in repos[repo]: - repos[repo]['comments'] = [] - repos[repo]['comments'].append(line.strip()) - continue + for section in parsed._sections: + section_dict = dict(parsed._sections[section]) + section_dict.pop('__name__') + config[section] = section_dict - # These are the actual configuration lines that matter - if '=' in line: - try: - comps = line.strip().split('=') - repos[repo][comps[0].strip()] = '='.join(comps[1:]) - except KeyError: - log.error( - 'Failed to parse line in %s, offending line was ' - '\'%s\'', filename, line.rstrip() - ) + # Try to extract leading comments + headers = '' + with salt.utils.fopen(filename, 'r') as rawfile: + for line in rawfile: + if line.strip().startswith('#'): + headers += '{0}\n'.format(line.strip()) - return (header, repos) + return (headers, config) def file_list(*packages): From d7f65dc7a7447a7aa7db401430dafef6511d2783 Mon Sep 17 00:00:00 2001 From: Arount Date: Fri, 30 Jun 2017 15:28:52 +0200 Subject: [PATCH 391/508] fix configparser import & log if error was raised --- salt/modules/yumpkg.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 9d98322cc8..40cde4196e 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -35,8 +35,10 @@ try: import yum HAS_YUM = True except ImportError: - from salt.ext.six.moves import configparser HAS_YUM = False + +from salt.ext.six.moves import configparser + # pylint: enable=import-error,redefined-builtin # Import salt libs @@ -2525,9 +2527,16 @@ def _parse_repo_file(filename): Turn a single repo file into a dict ''' parsed = configparser.ConfigParser() - parsed.read(filename) config = {} + try: + parsed.read(filename) + except configparser.MissingSectionHeaderError as err: + log.error( + 'Failed to parser file {0}, error: {1}'.format(filename, err.message) + ) + return ('', {}) + for section in parsed._sections: section_dict = dict(parsed._sections[section]) section_dict.pop('__name__') From 38add0e4a25218b78ae13ce0bee659122c54f2ed Mon Sep 17 00:00:00 2001 From: Arount Date: Sat, 1 Jul 2017 00:02:16 +0200 Subject: [PATCH 392/508] break if leading comments are all fetched --- salt/modules/yumpkg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 40cde4196e..1df90f04bf 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -2548,6 +2548,8 @@ def _parse_repo_file(filename): for line in rawfile: if line.strip().startswith('#'): headers += '{0}\n'.format(line.strip()) + else: + break return (headers, config) From 3b2cb81a72b9e89e6536b5f58abaef83495f00fc Mon Sep 17 00:00:00 2001 From: Arount Date: Sat, 1 Jul 2017 01:11:18 +0200 Subject: [PATCH 393/508] fix typo in salt.modules.yumpkg --- salt/modules/yumpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 1df90f04bf..14cdf0d899 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -2533,7 +2533,7 @@ def _parse_repo_file(filename): parsed.read(filename) except configparser.MissingSectionHeaderError as err: log.error( - 'Failed to parser file {0}, error: {1}'.format(filename, err.message) + 'Failed to parse file {0}, error: {1}'.format(filename, err.message) ) return ('', {}) From 093c0c2f7767b8f9bdd66671a2407a9e53b858b3 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 18 Aug 2017 16:19:40 -0500 Subject: [PATCH 394/508] Fix race condition in git.latest The git.latest state runs a `git ls-remote` on the remote repo to discover which SHA it should end up at, and whether or not it needs to fetch from the repo to get the commit it needs. However, since we fast-forward using a `git merge` to the branch specified in the `rev` argument, this leaves the state susceptible to a race condition when someone pushes to the remote repo between when we run the `git ls-remote` and when we fetch the remote repo. We will successfully fast-forward to the head of the branch, but that branch will be pointing to a different commit than the one identified in the `git ls-remote`, so we will report the state as having failed. This commit fixes that race condition by fast-forwarding to the commit identified in the `git ls-remote`, rather than to the branch named by `rev`. NOTE: This means that in these edge cases, we will report a `True` result despite the head of the branch having advanced past the commit to which we fast-forwarded, but that seems like a reasonable trade-off for avoiding a race condition. If we repeated the `git ls-remote` after fetching, we'd just be opening up a window (albeit a smaller one) for another race condition. --- salt/states/git.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index caf75399de..0654d46d19 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1472,8 +1472,6 @@ def latest(name, user=user, password=password, ignore_retcode=True): - merge_rev = remote_rev if rev == 'HEAD' \ - else desired_upstream if git_ver >= _LooseVersion('1.8.1.6'): # --ff-only added in version 1.8.1.6. It's not @@ -1490,7 +1488,7 @@ def latest(name, __salt__['git.merge']( target, - rev=merge_rev, + rev=remote_rev, opts=merge_opts, user=user, password=password) From ee41171c9f6a517652028ee2cc77fefe908185c3 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Sat, 19 Aug 2017 11:39:41 +1000 Subject: [PATCH 395/508] lint fixes --- salt/utils/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index cbdf4ae138..1cceb88d52 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -177,8 +177,9 @@ def safe_filename_leaf(file_basename): return urllib.quote(re_obj.group(0), safe=u'') if not isinstance(file_basename, six.text_type): # the following string is not prefixed with u - return re.sub('[\\\/:*?"<>|]', - _replace,six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace')) + return re.sub('[\\\/:*?"<>|]', + _replace, + six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace')) # the following string is prefixed with u return re.sub(u'[\\\/:*?"<>|]', _replace, file_basename, flags=re.UNICODE) @@ -188,7 +189,7 @@ def safe_filepath(file_path_name): input the full path and filename, splits on directory separator and calls safe_filename_leaf for each part of the path. ''' - (drive,path) = os.path.splitdrive(file_path_name) + (drive, path) = os.path.splitdrive(file_path_name) path = os.sep.join([safe_filename_leaf(file_section) for file_section in file_path_name.rsplit(os.sep)]) if drive: return os.sep.join([drive, path]) From 6e9c0957fbb90a986d17210c7fb7dc2ce52d9915 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Sat, 19 Aug 2017 15:21:48 +1000 Subject: [PATCH 396/508] fix typo --- salt/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 1cceb88d52..4a9938dc9f 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -177,11 +177,11 @@ def safe_filename_leaf(file_basename): return urllib.quote(re_obj.group(0), safe=u'') if not isinstance(file_basename, six.text_type): # the following string is not prefixed with u - return re.sub('[\\\/:*?"<>|]', + return re.sub('[\\\\:/*?"<>|]', _replace, six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace')) # the following string is prefixed with u - return re.sub(u'[\\\/:*?"<>|]', _replace, file_basename, flags=re.UNICODE) + return re.sub(u'[\\\\:/*?"<>|]', _replace, file_basename, flags=re.UNICODE) def safe_filepath(file_path_name): From 08ded1546e01da3a31a7e0717a8fbcf8296ea699 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Sat, 19 Aug 2017 17:07:28 +1000 Subject: [PATCH 397/508] more lint --- salt/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 4a9938dc9f..c03e0a6d39 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -171,7 +171,7 @@ def safe_filename_leaf(file_basename): i.e. only the required characters are converted by urllib.quote If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode. For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible - windows is \ / : * ? " < > | posix is / + windows is \\ / : * ? " < > | posix is / ''' def _replace(re_obj): return urllib.quote(re_obj.group(0), safe=u'') From 13404a47b551f1cc1b16f662d5d50b5843ffcbdb Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Sun, 20 Aug 2017 18:03:16 +0300 Subject: [PATCH 398/508] Fix bug #42936 - win_iis module Fix set_container_setting by adding map from string to numeric and vice versa. --- salt/modules/win_iis.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index 9309873ac4..bc8abbbbe7 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -1255,6 +1255,9 @@ def set_container_setting(name, container, settings): salt '*' win_iis.set_container_setting name='MyTestPool' container='AppPools' settings="{'managedPipeLineMode': 'Integrated'}" ''' + + identityType_map2string = {'0': 'LocalSystem', '1': 'LocalService', '2': 'NetworkService', '3': 'SpecificUser', '4': 'ApplicationPoolIdentity'} + identityType_map2numeric = {'LocalSystem': '0', 'LocalService': '1', 'NetworkService': '2', 'SpecificUser': '3', 'ApplicationPoolIdentity': '4'} ps_cmd = list() container_path = r"IIS:\{0}\{1}".format(container, name) @@ -1281,6 +1284,10 @@ def set_container_setting(name, container, settings): except ValueError: value = "'{0}'".format(settings[setting]) + # Map to numeric to support server 2008 + if (setting == 'processModel.identityType' and settings[setting] in identityType_map2numeric.keys()): + value = identityType_map2numeric[settings[setting]] + ps_cmd.extend(['Set-ItemProperty', '-Path', "'{0}'".format(container_path), '-Name', "'{0}'".format(setting), @@ -1300,6 +1307,10 @@ def set_container_setting(name, container, settings): failed_settings = dict() for setting in settings: + # map identity type from numeric to string for comparing + if (setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys()): + settings[setting] = identityType_map2string[settings[setting]] + if str(settings[setting]) != str(new_settings[setting]): failed_settings[setting] = settings[setting] From dc793f9a05596c526eeb22fa7b18eb10383a4410 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Sun, 20 Aug 2017 18:06:19 +0300 Subject: [PATCH 399/508] Fix bug #42936 - win_iis state Fix container_setting by adding map from string to numeric to support server 2008 --- salt/states/win_iis.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/states/win_iis.py b/salt/states/win_iis.py index 69d35e5c4a..b9940b5dd7 100644 --- a/salt/states/win_iis.py +++ b/salt/states/win_iis.py @@ -481,7 +481,7 @@ def container_setting(name, container, settings=None): :param str container: The type of IIS container. The container types are: AppPools, Sites, SslBindings :param str settings: A dictionary of the setting names and their values. - + Example of usage for the ``AppPools`` container: .. code-block:: yaml @@ -496,7 +496,8 @@ def container_setting(name, container, settings=None): processModel.userName: TestUser processModel.password: TestPassword processModel.identityType: SpecificUser - + + Example of usage for the ``Sites`` container: .. code-block:: yaml @@ -510,6 +511,9 @@ def container_setting(name, container, settings=None): logFile.period: Daily limits.maxUrlSegments: 32 ''' + + identityType_map2string = {'0': 'LocalSystem', '1': 'LocalService', '2': 'NetworkService', '3': 'SpecificUser', '4': 'ApplicationPoolIdentity'} + ret = {'name': name, 'changes': {}, 'comment': str(), @@ -529,6 +533,10 @@ def container_setting(name, container, settings=None): container=container, settings=settings.keys()) for setting in settings: + # map identity type from numeric to string for comparing + if (setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys()): + settings[setting] = identityType_map2string[settings[setting]] + if str(settings[setting]) != str(current_settings[setting]): ret_settings['changes'][setting] = {'old': current_settings[setting], 'new': settings[setting]} @@ -541,8 +549,8 @@ def container_setting(name, container, settings=None): ret['changes'] = ret_settings return ret - __salt__['win_iis.set_container_setting'](name=name, container=container, - settings=settings) + __salt__['win_iis.set_container_setting'](name=name, container=container, settings=settings) + new_settings = __salt__['win_iis.get_container_setting'](name=name, container=container, settings=settings.keys()) From cf6563645b3cd297478cbe34bcaf7f11054b2d6c Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Fri, 30 Jun 2017 14:00:04 +0200 Subject: [PATCH 400/508] add support for certificate authentication to kubernetes module --- salt/modules/kubernetes.py | 148 +++++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 7 deletions(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 2e2056fb43..2e17b11444 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -9,9 +9,23 @@ Module for handling kubernetes calls. kubernetes.user: admin kubernetes.password: verybadpass kubernetes.api_url: 'http://127.0.0.1:8080' + kubernetes.certificate-authority-data: '...' + kubernetes.client-certificate-data: '....n + kubernetes.client-key-data: '...' + kubernetes.certificate-authority-file: '/path/to/ca.crt' + kubernetes.client-certificate-file: '/path/to/client.crt' + kubernetes.client-key-file: '/path/to/client.key' + These settings can be also overrided by adding `api_url`, `api_user`, -or `api_password` parameters when calling a function: +`api_password`, `api_certificate_authority_file`, `api_client_certificate_file` +or `api_client_key_file` parameters when calling a function: + +The data format for `kubernetes.*-data` values is the same as provided in `kubeconfig`. +It's base64 encoded certificates/keys in one line. + +For an item only one field should be provided. Either a `data` or a `file` entry. +In case both are provided the `file` entry is prefered. .. code-block:: bash salt '*' kubernetes.nodes api_url=http://k8s-api-server:port api_user=myuser api_password=pass @@ -21,9 +35,11 @@ or `api_password` parameters when calling a function: # Import Python Futures from __future__ import absolute_import +import os.path import base64 import logging import yaml +import tempfile from salt.exceptions import CommandExecutionError from salt.ext.six import iteritems @@ -71,16 +87,31 @@ def _setup_conn(**kwargs): 'http://localhost:8080') username = __salt__['config.option']('kubernetes.user') password = __salt__['config.option']('kubernetes.password') + ca_cert = __salt__['config.option']('kubernetes.certificate-authority-data') + client_cert = __salt__['config.option']('kubernetes.client-certificate-data') + client_key = __salt__['config.option']('kubernetes.client-key-data') + ca_cert_file = __salt__['config.option']('kubernetes.certificate-authority-file') + client_cert_file = __salt__['config.option']('kubernetes.client-certificate-file') + client_key_file = __salt__['config.option']('kubernetes.client-key-file') # Override default API settings when settings are provided - if kwargs.get('api_url'): - host = kwargs['api_url'] + if 'api_url' in kwargs: + host = kwargs.get('api_url') - if kwargs.get('api_user'): - username = kwargs['api_user'] + if 'api_user' in kwargs: + username = kwargs.get('api_user') - if kwargs.get('api_password'): - password = kwargs['api_password'] + if 'api_password' in kwargs: + password = kwargs.get('api_password') + + if 'api_certificate_authority_file' in kwargs: + ca_cert_file = kwargs.get('api_certificate_authority_file') + + if 'api_client_certificate_file' in kwargs: + client_cert_file = kwargs.get('api_client_certificate_file') + + if 'api_client_key_file' in kwargs: + client_key_file = kwargs.get('api_client_key_file') if ( kubernetes.client.configuration.host != host or @@ -93,6 +124,45 @@ def _setup_conn(**kwargs): kubernetes.client.configuration.user = username kubernetes.client.configuration.passwd = password + if ca_cert_file: + kubernetes.client.configuration.ssl_ca_cert = ca_cert_file + elif ca_cert: + with tempfile.NamedTemporaryFile(prefix='salt-kube-', delete=False) as ca: + ca.write(base64.b64decode(ca_cert)) + kubernetes.client.configuration.ssl_ca_cert = ca.name + else: + kubernetes.client.configuration.ssl_ca_cert = None + + if client_cert_file: + kubernetes.client.configuration.cert_file = client_cert_file + elif client_cert: + with tempfile.NamedTemporaryFile(prefix='salt-kube-', delete=False) as c: + c.write(base64.b64decode(client_cert)) + kubernetes.client.configuration.cert_file = c.name + else: + kubernetes.client.configuration.cert_file = None + + if client_key_file: + kubernetes.client.configuration.key_file = client_key_file + if client_key: + with tempfile.NamedTemporaryFile(prefix='salt-kube-', delete=False) as k: + k.write(base64.b64decode(client_key)) + kubernetes.client.configuration.key_file = k.name + else: + kubernetes.client.configuration.key_file = None + + +def _cleanup(**kwargs): + ca = kubernetes.client.configuration.ssl_ca_cert + cert = kubernetes.client.configuration.cert_file + key = kubernetes.client.configuration.key_file + if cert and os.path.exists(cert) and os.path.basename(cert).startswith('salt-kube-'): + salt.utils.safe_rm(cert) + if key and os.path.exists(key) and os.path.basename(key).startswith('salt-kube-'): + salt.utils.safe_rm(key) + if ca and os.path.exists(ca) and os.path.basename(ca).startswith('salt-kube-'): + salt.utils.safe_rm(ca) + def ping(**kwargs): ''' @@ -134,6 +204,8 @@ def nodes(**kwargs): 'Exception when calling CoreV1Api->list_node: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def node(name, **kwargs): @@ -156,6 +228,8 @@ def node(name, **kwargs): 'Exception when calling CoreV1Api->list_node: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() for k8s_node in api_response.items: if k8s_node.metadata.name == name: @@ -210,6 +284,8 @@ def node_add_label(node_name, label_name, label_value, **kwargs): 'Exception when calling CoreV1Api->patch_node: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() return None @@ -243,6 +319,8 @@ def node_remove_label(node_name, label_name, **kwargs): 'Exception when calling CoreV1Api->patch_node: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() return None @@ -271,6 +349,8 @@ def namespaces(**kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def deployments(namespace='default', **kwargs): @@ -298,6 +378,8 @@ def deployments(namespace='default', **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def services(namespace='default', **kwargs): @@ -324,6 +406,8 @@ def services(namespace='default', **kwargs): 'CoreV1Api->list_namespaced_service: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def pods(namespace='default', **kwargs): @@ -350,6 +434,8 @@ def pods(namespace='default', **kwargs): 'CoreV1Api->list_namespaced_pod: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def secrets(namespace='default', **kwargs): @@ -376,6 +462,8 @@ def secrets(namespace='default', **kwargs): 'CoreV1Api->list_namespaced_secret: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def configmaps(namespace='default', **kwargs): @@ -402,6 +490,8 @@ def configmaps(namespace='default', **kwargs): 'CoreV1Api->list_namespaced_config_map: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def show_deployment(name, namespace='default', **kwargs): @@ -429,6 +519,8 @@ def show_deployment(name, namespace='default', **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def show_service(name, namespace='default', **kwargs): @@ -455,6 +547,8 @@ def show_service(name, namespace='default', **kwargs): 'CoreV1Api->read_namespaced_service: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def show_pod(name, namespace='default', **kwargs): @@ -481,6 +575,8 @@ def show_pod(name, namespace='default', **kwargs): 'CoreV1Api->read_namespaced_pod: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def show_namespace(name, **kwargs): @@ -506,6 +602,8 @@ def show_namespace(name, **kwargs): 'CoreV1Api->read_namespace: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def show_secret(name, namespace='default', decode=False, **kwargs): @@ -541,6 +639,8 @@ def show_secret(name, namespace='default', decode=False, **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def show_configmap(name, namespace='default', **kwargs): @@ -570,6 +670,8 @@ def show_configmap(name, namespace='default', **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def delete_deployment(name, namespace='default', **kwargs): @@ -601,6 +703,8 @@ def delete_deployment(name, namespace='default', **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def delete_service(name, namespace='default', **kwargs): @@ -630,6 +734,8 @@ def delete_service(name, namespace='default', **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def delete_pod(name, namespace='default', **kwargs): @@ -661,6 +767,8 @@ def delete_pod(name, namespace='default', **kwargs): 'CoreV1Api->delete_namespaced_pod: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def delete_namespace(name, **kwargs): @@ -689,6 +797,8 @@ def delete_namespace(name, **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def delete_secret(name, namespace='default', **kwargs): @@ -720,6 +830,8 @@ def delete_secret(name, namespace='default', **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def delete_configmap(name, namespace='default', **kwargs): @@ -752,6 +864,8 @@ def delete_configmap(name, namespace='default', **kwargs): '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def create_deployment( @@ -796,6 +910,8 @@ def create_deployment( '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def create_pod( @@ -840,6 +956,8 @@ def create_pod( '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def create_service( @@ -883,6 +1001,8 @@ def create_service( 'CoreV1Api->create_namespaced_service: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def create_secret( @@ -928,6 +1048,8 @@ def create_secret( 'CoreV1Api->create_namespaced_secret: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def create_configmap( @@ -969,6 +1091,8 @@ def create_configmap( 'CoreV1Api->create_namespaced_config_map: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def create_namespace( @@ -1003,6 +1127,8 @@ def create_namespace( '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def replace_deployment(name, @@ -1047,6 +1173,8 @@ def replace_deployment(name, '{0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def replace_service(name, @@ -1096,6 +1224,8 @@ def replace_service(name, 'CoreV1Api->replace_namespaced_service: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def replace_secret(name, @@ -1141,6 +1271,8 @@ def replace_secret(name, 'CoreV1Api->replace_namespaced_secret: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def replace_configmap(name, @@ -1180,6 +1312,8 @@ def replace_configmap(name, 'CoreV1Api->replace_namespaced_configmap: {0}'.format(exc) ) raise CommandExecutionError(exc) + finally: + _cleanup() def __create_object_body(kind, From b502560e61fe1452f33784dcc1d7b39c14129012 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Mon, 21 Aug 2017 13:29:00 -0400 Subject: [PATCH 401/508] Fixed issue with silently passing all tests in Testinfra module The Testinfra module had a line where the collection of passed arguments was silently overwritten so that it would not actually perform any assertions. Updated the variable names to address the issue of the parameters being clobbered so that asertions are performed properly. --- salt/modules/testinframod.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/modules/testinframod.py b/salt/modules/testinframod.py index b46e879dfa..a384de9e7b 100644 --- a/salt/modules/testinframod.py +++ b/salt/modules/testinframod.py @@ -242,6 +242,8 @@ def _copy_function(module_name, name=None): elif hasattr(mod, '__call__'): mod_sig = inspect.getargspec(mod.__call__) parameters = mod_sig.args + log.debug('Parameters accepted by module {0}: {1}'.format(module_name, + parameters)) additional_args = {} for arg in set(parameters).intersection(set(methods)): additional_args[arg] = methods.pop(arg) @@ -251,12 +253,15 @@ def _copy_function(module_name, name=None): else: modinstance = mod() except TypeError: - modinstance = None - methods = {} + log.exception('Module failed to instantiate') + raise + valid_methods = {} + log.debug('Called methods are: {0}'.format(methods)) for meth_name in methods: if not meth_name.startswith('_'): - methods[meth_name] = methods[meth_name] - for meth, arg in methods.items(): + valid_methods[meth_name] = methods[meth_name] + log.debug('Valid methods are: {0}'.format(valid_methods)) + for meth, arg in valid_methods.items(): result = _get_method_result(mod, modinstance, meth, arg) assertion_result = _apply_assertion(arg, result) if not assertion_result: From d4b113acdf8378d71106d0f669735a56062a70b0 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Mon, 21 Aug 2017 13:29:00 -0400 Subject: [PATCH 402/508] Fixed issue with silently passing all tests in Testinfra module The Testinfra module had a line where the collection of passed arguments was silently overwritten so that it would not actually perform any assertions. Updated the variable names to address the issue of the parameters being clobbered so that asertions are performed properly. --- salt/modules/testinframod.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/modules/testinframod.py b/salt/modules/testinframod.py index d705a3a2c9..265f7ad9b0 100644 --- a/salt/modules/testinframod.py +++ b/salt/modules/testinframod.py @@ -242,6 +242,8 @@ def _copy_function(module_name, name=None): elif hasattr(mod, '__call__'): mod_sig = inspect.getargspec(mod.__call__) parameters = mod_sig.args + log.debug('Parameters accepted by module {0}: {1}'.format(module_name, + parameters)) additional_args = {} for arg in set(parameters).intersection(set(methods.keys())): additional_args[arg] = methods.pop(arg) @@ -251,12 +253,15 @@ def _copy_function(module_name, name=None): else: modinstance = mod() except TypeError: - modinstance = None - methods = {} + log.exception('Module failed to instantiate') + raise + valid_methods = {} + log.debug('Called methods are: {0}'.format(methods)) for meth_name in methods: if not meth_name.startswith('_'): - methods[meth_name] = methods[meth_name] - for meth, arg in methods.items(): + valid_methods[meth_name] = methods[meth_name] + log.debug('Valid methods are: {0}'.format(valid_methods)) + for meth, arg in valid_methods.items(): result = _get_method_result(mod, modinstance, meth, arg) assertion_result = _apply_assertion(arg, result) if not assertion_result: From b7283bcc6f929ef9f82ef91c7166056a91c13b18 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 21 Aug 2017 11:26:02 -0600 Subject: [PATCH 403/508] _vm_provider_driver isn't needed anymore We deprecated some of these names in 2017.7, and this function is no longer needed --- salt/cloud/clouds/ec2.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index c665d335cc..88f766f9aa 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -3432,7 +3432,7 @@ def list_nodes_full(location=None, call=None): ret = {} locations = set( get_location(vm_) for vm_ in six.itervalues(__opts__['profiles']) - if _vm_provider_driver(vm_) + if vm_.get('driver') == 'ec2' ) # If there aren't any profiles defined for EC2, check @@ -3447,17 +3447,6 @@ def list_nodes_full(location=None, call=None): return _list_nodes_full(location) -def _vm_provider_driver(vm_): - alias, driver = vm_['driver'].split(':') - if alias not in __opts__['providers']: - return None - - if driver not in __opts__['providers'][alias]: - return None - - return driver == 'ec2' - - def _extract_name_tag(item): if 'tagSet' in item and item['tagSet'] is not None: tagset = item['tagSet'] From c7cffb5a04d4e6b4d674d301b8bd9cb2e4daee4a Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 21 Aug 2017 11:45:11 -0600 Subject: [PATCH 404/508] This block isn't necessary If a location isn't passed to list_nodes_full, we can just use get_location(). Of no location is set in the provider, it will just use get_location(). The problem with the lookup if there are profiles, is the way that the provider/driver was changed in the last release, if it just uses the profile directly, it will fail when trying to do a full list. Then for multiple locations being called, each provider has to have a location in it, and will be called once, as it stands now, if you have multiple providers, minions from each provider could be doubled up with this lookup. This change should meet all the needs that this if statement was added for, and not fail in 2017.7 --- salt/cloud/clouds/ec2.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index 88f766f9aa..865806eeba 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -3428,23 +3428,7 @@ def list_nodes_full(location=None, call=None): 'or --function.' ) - if not location: - ret = {} - locations = set( - get_location(vm_) for vm_ in six.itervalues(__opts__['profiles']) - if vm_.get('driver') == 'ec2' - ) - - # If there aren't any profiles defined for EC2, check - # the provider config file, or use the default location. - if not locations: - locations = [get_location()] - - for loc in locations: - ret.update(_list_nodes_full(loc)) - return ret - - return _list_nodes_full(location) + return _list_nodes_full(location or get_location()) def _extract_name_tag(item): From 373a9a0be489223a8ba39e09c0f846cf1dc7a0c7 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 21 Aug 2017 09:35:46 -0600 Subject: [PATCH 405/508] allow docker util to be reloaded with reload_modules If we reference the actual import, the utils module won't be reloaded, but if we use `__utils__` it can be reloaded on a pip install --- salt/modules/dockermod.py | 2 +- tests/unit/modules/test_dockermod.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index d796e49913..a6cd82ce8e 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -798,7 +798,7 @@ def get_client_args(): salt myminion docker.get_client_args ''' - return salt.utils.docker.get_client_args() + return __utils__['docker.get_client_args']() def _get_create_kwargs(image, diff --git a/tests/unit/modules/test_dockermod.py b/tests/unit/modules/test_dockermod.py index 2800bc92f2..8f22a0605c 100644 --- a/tests/unit/modules/test_dockermod.py +++ b/tests/unit/modules/test_dockermod.py @@ -136,10 +136,11 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(docker_mod.__salt__, {'mine.send': mine_send, 'container_resource.run': MagicMock(), - 'cp.cache_file': MagicMock(return_value=False), - 'docker.get_client_args': client_args_mock}): - with patch.object(docker_mod, '_get_client', client): - command('container', *args) + 'cp.cache_file': MagicMock(return_value=False)}): + with patch.dict(docker_mod.__utils__, + {'docker.get_client_args': client_args_mock}): + with patch.object(docker_mod, '_get_client', client): + command('container', *args) mine_send.assert_called_with('docker.ps', verbose=True, all=True, host=True) From 93390de88b71471731e9e8237f4c62e30d98f4a2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 16 Aug 2017 15:34:37 -0600 Subject: [PATCH 406/508] Fix malformed requisite for Windows --- salt/state.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/state.py b/salt/state.py index 8d1ce6dae0..b1bbf4d74b 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2122,11 +2122,14 @@ class State(object): reqs[r_state].append(chunk) continue try: - if (fnmatch.fnmatch(chunk['name'], req_val) or - fnmatch.fnmatch(chunk['__id__'], req_val)): - if req_key == 'id' or chunk['state'] == req_key: - found = True - reqs[r_state].append(chunk) + if isinstance(req_val, six.string_types): + if (fnmatch.fnmatch(chunk['name'], req_val) or + fnmatch.fnmatch(chunk['__id__'], req_val)): + if req_key == 'id' or chunk['state'] == req_key: + found = True + reqs[r_state].append(chunk) + else: + raise KeyError except KeyError as exc: raise SaltRenderError( 'Could not locate requisite of [{0}] present in state with name [{1}]'.format( From 4f4e34c79f25a1153d4620c2af74e2be22e19ff7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 21 Aug 2017 18:39:09 -0600 Subject: [PATCH 407/508] Fix group state for Windows group no longer fails when domain is not specified group.present now accepts group names without domains local groups are assumed if domain is not specified documentation improved moved fix_local_user function to salt.utils.win_functions now called get_sam_name --- salt/modules/win_groupadd.py | 125 ++++++++++++++++++++++++++--------- salt/states/group.py | 99 +++++++++++++++++++++------ salt/utils/win_functions.py | 19 ++++++ 3 files changed, 190 insertions(+), 53 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index d466380d70..4368bb0201 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -12,6 +12,7 @@ from __future__ import absolute_import # Import salt libs import salt.utils +import salt.utils.win_functions try: @@ -35,10 +36,18 @@ def __virtual__(): return (False, "Module win_groupadd: module only works on Windows systems") -def add(name, gid=None, system=False): +def add(name, **kwargs): ''' Add the specified group + Args: + + name (str): + The name of the group to add + + Returns: + dict: A dictionary of results + CLI Example: .. code-block:: bash @@ -57,21 +66,16 @@ def add(name, gid=None, system=False): compObj = nt.GetObject('', 'WinNT://.,computer') newGroup = compObj.Create('group', name) newGroup.SetInfo() - ret['changes'].append(( - 'Successfully created group {0}' - ).format(name)) + ret['changes'].append('Successfully created group {0}'.format(name)) except pywintypes.com_error as com_err: ret['result'] = False if len(com_err.excepinfo) >= 2: friendly_error = com_err.excepinfo[2].rstrip('\r\n') - ret['comment'] = ( - 'Failed to create group {0}. {1}' - ).format(name, friendly_error) + ret['comment'] = 'Failed to create group {0}. {1}' \ + ''.format(name, friendly_error) else: ret['result'] = None - ret['comment'] = ( - 'The group {0} already exists.' - ).format(name) + ret['comment'] = 'The group {0} already exists.'.format(name) return ret @@ -80,6 +84,14 @@ def delete(name): ''' Remove the named group + Args: + + name (str): + The name of the group to remove + + Returns: + dict: A dictionary of results + CLI Example: .. code-block:: bash @@ -118,6 +130,14 @@ def info(name): ''' Return information about a group + Args: + + name (str): + The name of the group for which to get information + + Returns: + dict: A dictionary of information about the group + CLI Example: .. code-block:: bash @@ -151,6 +171,17 @@ def getent(refresh=False): ''' Return info on all groups + Args: + + refresh (bool): + Refresh the info for all groups in ``__context__``. If False only + the groups in ``__context__`` wil be returned. If True the + ``__context__`` will be refreshed with current data and returned. + Default is False + + Returns: + A list of groups and their information + CLI Example: .. code-block:: bash @@ -184,14 +215,24 @@ def getent(refresh=False): def adduser(name, username): ''' - add a user to a group + Add a user to a group + + Args: + + name (str): + The name of the group to modify + + username (str): + The name of the user to add to the group + + Returns: + dict: A dictionary of results CLI Example: .. code-block:: bash salt '*' group.adduser foo username - ''' ret = {'name': name, @@ -209,7 +250,7 @@ def adduser(name, username): '/', '\\').encode('ascii', 'backslashreplace').lower()) try: - if __fixlocaluser(username.lower()) not in existingMembers: + if salt.utils.win_functions.get_sam_name(username) not in existingMembers: if not __opts__['test']: groupObj.Add('WinNT://' + username.replace('\\', '/')) @@ -233,14 +274,24 @@ def adduser(name, username): def deluser(name, username): ''' - remove a user from a group + Remove a user from a group + + Args: + + name (str): + The name of the group to modify + + username (str): + The name of the user to remove from the group + + Returns: + dict: A dictionary of results CLI Example: .. code-block:: bash salt '*' group.deluser foo username - ''' ret = {'name': name, @@ -258,7 +309,7 @@ def deluser(name, username): '/', '\\').encode('ascii', 'backslashreplace').lower()) try: - if __fixlocaluser(username.lower()) in existingMembers: + if salt.utils.win_functions.get_sam_name(username) in existingMembers: if not __opts__['test']: groupObj.Remove('WinNT://' + username.replace('\\', '/')) @@ -282,14 +333,25 @@ def deluser(name, username): def members(name, members_list): ''' - remove a user from a group + Ensure a group contains only the members in the list + + Args: + + name (str): + The name of the group to modify + + members_list (str): + A single user or a comma separated list of users. The group will + contain only the users specified in this list. + + Returns: + dict: A dictionary of results CLI Example: .. code-block:: bash salt '*' group.members foo 'user1,user2,user3' - ''' ret = {'name': name, @@ -297,7 +359,7 @@ def members(name, members_list): 'changes': {'Users Added': [], 'Users Removed': []}, 'comment': []} - members_list = [__fixlocaluser(thisMember) for thisMember in members_list.lower().split(",")] + members_list = [salt.utils.win_functions.get_sam_name(m) for m in members_list.split(",")] if not isinstance(members_list, list): ret['result'] = False ret['comment'].append('Members is not a list object') @@ -364,27 +426,26 @@ def members(name, members_list): return ret -def __fixlocaluser(username): - ''' - prefixes a username w/o a backslash with the computername - - i.e. __fixlocaluser('Administrator') would return 'computername\administrator' - ''' - if '\\' not in username: - username = ('{0}\\{1}').format(__salt__['grains.get']('host'), username) - - return username.lower() - - def list_groups(refresh=False): ''' Return a list of groups + Args: + + refresh (bool): + Refresh the info for all groups in ``__context__``. If False only + the groups in ``__context__`` wil be returned. If True, the + ``__context__`` will be refreshed with current data and returned. + Default is False + + Returns: + list: A list of groups on the machine + CLI Example: .. code-block:: bash - salt '*' group.getent + salt '*' group.list_groups ''' if 'group.list_groups' in __context__ and not refresh: return __context__['group.getent'] diff --git a/salt/states/group.py b/salt/states/group.py index 8218e415d6..78f3568c74 100644 --- a/salt/states/group.py +++ b/salt/states/group.py @@ -3,8 +3,13 @@ Management of user groups ========================= -The group module is used to create and manage unix group settings, groups -can be either present or absent: +The group module is used to create and manage group settings, groups can be +either present or absent. User/Group names can be passed to the ``adduser``, +``deluser``, and ``members`` parameters. ``adduser`` and ``deluser`` can be used +together but not with ``members``. + +In Windows, if no domain is specified in the user or group name (ie: +`DOMAIN\username``) the module will assume a local user or group. .. code-block:: yaml @@ -36,6 +41,10 @@ import sys # Import 3rd-party libs import salt.ext.six as six +# Import Salt libs +import salt.utils +import salt.utils.win_functions + def _changes(name, gid=None, @@ -50,6 +59,18 @@ def _changes(name, if not lgrp: return False + # User and Domain names are not case sensitive in Windows. Let's make them + # all lower case so we can compare properly + if salt.utils.is_windows(): + if lgrp['members']: + lgrp['members'] = [user.lower() for user in lgrp['members']] + if members: + members = [salt.utils.win_functions.get_sam_name(user) for user in members] + if addusers: + addusers = [salt.utils.win_functions.get_sam_name(user) for user in addusers] + if delusers: + delusers = [salt.utils.win_functions.get_sam_name(user) for user in delusers] + change = {} if gid: if lgrp['gid'] != gid: @@ -57,7 +78,7 @@ def _changes(name, if members: # -- if new member list if different than the current - if set(lgrp['members']) ^ set(members): + if set(lgrp['members']).symmetric_difference(members): change['members'] = members if addusers: @@ -82,28 +103,55 @@ def present(name, ''' Ensure that a group is present - name - The name of the group to manage + Args: - gid - The group id to assign to the named group; if left empty, then the next - available group id will be assigned + name (str): + The name of the group to manage - system - Whether or not the named group is a system group. This is essentially - the '-r' option of 'groupadd'. + gid (str): + The group id to assign to the named group; if left empty, then the + next available group id will be assigned. Ignored on Windows - addusers - List of additional users to be added as a group members. + system (bool): + Whether or not the named group is a system group. This is essentially + the '-r' option of 'groupadd'. Ignored on Windows - delusers - Ensure these user are removed from the group membership. + addusers (list): + List of additional users to be added as a group members. Cannot + conflict with names in delusers. Cannot be used in conjunction with + members. - members - Replace existing group members with a list of new members. + delusers (list): + Ensure these user are removed from the group membership. Cannot + conflict with names in addusers. Cannot be used in conjunction with + members. - Note: Options 'members' and 'addusers/delusers' are mutually exclusive and - can not be used together. + members (list): + Replace existing group members with a list of new members. Cannot be + used in conjunction with addusers or delusers. + + Example: + + .. code-block:: yaml + + # Adds DOMAIN\db_admins and Administrators to the local db_admin group + # Removes Users + db_admin: + group.present: + - addusers: + - DOMAIN\db_admins + - Administrators + - delusers: + - Users + + # Ensures only DOMAIN\domain_admins and the local Administrator are + # members of the local Administrators group. All other users are + # removed + Administrators: + group.present: + - members: + - DOMAIN\domain_admins + - Administrator ''' ret = {'name': name, 'changes': {}, @@ -233,8 +281,17 @@ def absent(name): ''' Ensure that the named group is absent - name - The name of the group to remove + Args: + name (str): + The name of the group to remove + + Example: + + .. code-block:: yaml + + # Removes the local group `db_admin` + db_admin: + group.absent ''' ret = {'name': name, 'changes': {}, diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 23ee3edf04..4e3ec9663c 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -4,6 +4,9 @@ Various functions to be used by windows during start up and to monkey patch missing functions in other modules ''' from __future__ import absolute_import +import platform + +# Import Salt Libs from salt.exceptions import CommandExecutionError # Import 3rd Party Libs @@ -138,3 +141,19 @@ def get_current_user(): return False return user_name + + +def get_sam_name(username): + ''' + Gets the SAM name for a user. It basically prefixes a username without a + backslash with the computer name. If the username contains a backslash, it + is returned as is. + + Everything is returned lower case + + i.e. salt.utils.fix_local_user('Administrator') would return 'computername\administrator' + ''' + if '\\' not in username: + username = '{0}\\{1}'.format(platform.node(), username) + + return username.lower() From 9ffe315d7d12ad853b4404681526e8c59770d859 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 21 Aug 2017 18:45:11 -0600 Subject: [PATCH 408/508] Add kwargs --- salt/modules/win_groupadd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index 4368bb0201..f584a0ab94 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -80,7 +80,7 @@ def add(name, **kwargs): return ret -def delete(name): +def delete(name, **kwargs): ''' Remove the named group @@ -213,7 +213,7 @@ def getent(refresh=False): return ret -def adduser(name, username): +def adduser(name, username, **kwargs): ''' Add a user to a group @@ -272,7 +272,7 @@ def adduser(name, username): return ret -def deluser(name, username): +def deluser(name, username, **kwargs): ''' Remove a user from a group @@ -331,7 +331,7 @@ def deluser(name, username): return ret -def members(name, members_list): +def members(name, members_list, **kwargs): ''' Ensure a group contains only the members in the list From af743ff6c34b9e03e8865d1fa065af218b0dfb77 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 22 Aug 2017 12:10:20 +0300 Subject: [PATCH 409/508] [DOCS] Add missing `utils` sub-dir listed for `extension_modules` --- conf/master | 9 ++++----- doc/ref/configuration/master.rst | 4 ++-- doc/topics/utils/index.rst | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/conf/master b/conf/master index 1fc76cb89a..8c1db0ae67 100644 --- a/conf/master +++ b/conf/master @@ -59,15 +59,14 @@ # Directory for custom modules. This directory can contain subdirectories for # each of Salt's module types such as "runners", "output", "wheel", "modules", -# "states", "returners", etc. -#extension_modules: +# "states", "returners", "engines", "utils", etc. +#extension_modules: /var/cache/salt/master/extmods # Directory for custom modules. This directory can contain subdirectories for # each of Salt's module types such as "runners", "output", "wheel", "modules", -# "states", "returners", "engines", etc. +# "states", "returners", "engines", "utils", etc. # Like 'extension_modules' but can take an array of paths -#module_dirs: -# - /var/cache/salt/minion/extmods +#module_dirs: [] # Verify and set permissions on configuration directories at startup: #verify_env: True diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index fd310773bf..815b8a6d97 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -180,8 +180,8 @@ The directory to store the pki authentication keys. Directory for custom modules. This directory can contain subdirectories for each of Salt's module types such as ``runners``, ``output``, ``wheel``, -``modules``, ``states``, ``returners``, ``engines``, etc. This path is appended to -:conf_master:`root_dir`. +``modules``, ``states``, ``returners``, ``engines``, ``utils``, etc. +This path is appended to :conf_master:`root_dir`. .. code-block:: yaml diff --git a/doc/topics/utils/index.rst b/doc/topics/utils/index.rst index 44380f3541..19a0974d29 100644 --- a/doc/topics/utils/index.rst +++ b/doc/topics/utils/index.rst @@ -87,8 +87,8 @@ Also you could even write your utility modules in object oriented fashion: # -*- coding: utf-8 -*- ''' - My utils module - --------------- + My OOP-style utils module + ------------------------- This module contains common functions for use in my other custom types. ''' From f1765472dddcbbe319da2d376ec2001e6776d51b Mon Sep 17 00:00:00 2001 From: Johannes Renner Date: Wed, 16 Aug 2017 16:54:41 +0200 Subject: [PATCH 410/508] Notify systemd synchronously (via NOTIFY_SOCKET) Forking the systemd-notify command is known to be unreliable at least with older versions of the kernel and/or systemd. When systemd receives the notification the systemd-notify process may have already exited causing an error in the logs while waiting for a (90 seconds) timeout. This patch instead notifies the systemd NOTIFY_SOCKET synchronously in case the systemd.daemon python library is not available. --- salt/utils/process.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/utils/process.py b/salt/utils/process.py index 1b83eb7d42..51681ec364 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -15,6 +15,7 @@ import contextlib import subprocess import multiprocessing import multiprocessing.util +import socket # Import salt libs @@ -55,7 +56,17 @@ def notify_systemd(): import systemd.daemon except ImportError: if salt.utils.which('systemd-notify') and systemd_notify_call('--booted'): - return systemd_notify_call('--ready') + # Notify systemd synchronously + notify_socket = os.getenv('NOTIFY_SOCKET') + if notify_socket: + # Handle abstract namespace socket + if notify_socket.startswith('@'): + notify_socket = '\0{0}'.format(notify_socket[1:]) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + sock.connect(notify_socket) + sock.sendall('READY=1'.encode()) + sock.close() + return True return False if systemd.daemon.booted(): From 1a987cb9481da365e6ad1a49c6526c970bdfe605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tarjei=20Hus=C3=B8y?= Date: Wed, 16 Aug 2017 10:30:45 -0700 Subject: [PATCH 411/508] Fix broken negation in iptables Introduced in 7c6ff77c and released with 2017.7. --- salt/modules/iptables.py | 5 ++++- tests/unit/modules/test_iptables.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/modules/iptables.py b/salt/modules/iptables.py index 37d5842eea..a96c975da7 100644 --- a/salt/modules/iptables.py +++ b/salt/modules/iptables.py @@ -493,8 +493,11 @@ def build_rule(table='filter', chain=None, command=None, position='', full=None, after_jump.append('--{0} {1}'.format(after_jump_argument, value)) del kwargs[after_jump_argument] - for key, value in kwargs.items(): + for key in kwargs: negation = maybe_add_negation(key) + # don't use .items() since maybe_add_negation removes the prefix from + # the value in the kwargs, thus we need to fetch it after that has run + value = kwargs[key] flag = '-' if len(key) == 1 else '--' value = '' if value in (None, '') else ' {0}'.format(value) rule.append('{0}{1}{2}{3}'.format(negation, flag, key, value)) diff --git a/tests/unit/modules/test_iptables.py b/tests/unit/modules/test_iptables.py index 1c4f34118f..6fe9e91285 100644 --- a/tests/unit/modules/test_iptables.py +++ b/tests/unit/modules/test_iptables.py @@ -60,6 +60,9 @@ class IptablesTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(iptables.build_rule(**{'if': 'not eth0'}), '! -i eth0') + self.assertEqual(iptables.build_rule(**{'proto': 'tcp', 'syn': '!'}), + '-p tcp ! --syn') + self.assertEqual(iptables.build_rule(dports=[80, 443], proto='tcp'), '-p tcp -m multiport --dports 80,443') From 79c53f3f8193c22566978ce0e627818bfed2f4fa Mon Sep 17 00:00:00 2001 From: Johannes Renner Date: Tue, 22 Aug 2017 15:56:17 +0200 Subject: [PATCH 412/508] Fallback to systemd_notify_call() in case of socket.error --- salt/utils/process.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/utils/process.py b/salt/utils/process.py index 51681ec364..e63c888100 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -62,10 +62,13 @@ def notify_systemd(): # Handle abstract namespace socket if notify_socket.startswith('@'): notify_socket = '\0{0}'.format(notify_socket[1:]) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - sock.connect(notify_socket) - sock.sendall('READY=1'.encode()) - sock.close() + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + sock.connect(notify_socket) + sock.sendall('READY=1'.encode()) + sock.close() + except socket.error: + return systemd_notify_call('--ready') return True return False From 009ef6686b49cd523f846eb7ec9282c9bf03e694 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Tue, 22 Aug 2017 17:27:55 +0300 Subject: [PATCH 413/508] Fix dictionary keys from string to int identity type values are int. --- salt/states/win_iis.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/states/win_iis.py b/salt/states/win_iis.py index b9940b5dd7..74507c683c 100644 --- a/salt/states/win_iis.py +++ b/salt/states/win_iis.py @@ -495,8 +495,7 @@ def container_setting(name, container, settings=None): processModel.maxProcesses: 1 processModel.userName: TestUser processModel.password: TestPassword - processModel.identityType: SpecificUser - + processModel.identityType: SpecificUser Example of usage for the ``Sites`` container: @@ -511,9 +510,9 @@ def container_setting(name, container, settings=None): logFile.period: Daily limits.maxUrlSegments: 32 ''' - - identityType_map2string = {'0': 'LocalSystem', '1': 'LocalService', '2': 'NetworkService', '3': 'SpecificUser', '4': 'ApplicationPoolIdentity'} - + + identityType_map2string = {0: 'LocalSystem', 1: 'LocalService', 2: 'NetworkService', 3: 'SpecificUser', 4: 'ApplicationPoolIdentity'} + ret = {'name': name, 'changes': {}, 'comment': str(), From ec20e9a19a2cf08c7d005a5ca1fe8d7c380a0c99 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Tue, 22 Aug 2017 17:52:20 +0300 Subject: [PATCH 414/508] Fix bug #43110 - win_iis module Fix func create_cert_binding. win 2008 uses the following format: ip!port and not ip!port! --- salt/modules/win_iis.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index bc8abbbbe7..80bba6c438 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -837,6 +837,11 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, # IIS 7.5 and earlier have different syntax for associating a certificate with a site # Modify IP spec to IIS 7.5 format iis7path = binding_path.replace(r"\*!", "\\0.0.0.0!") + + # win 2008 uses the following format: ip!port and not ip!port! + if iis7path.endswith("!"): + iis7path = iis7path[:-1] + ps_cmd = ['New-Item', '-Path', "'{0}'".format(iis7path), From 43b03607639d586da1d7de368f70d61015330185 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 22 Aug 2017 10:54:01 -0600 Subject: [PATCH 415/508] Fix lint --- salt/states/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/group.py b/salt/states/group.py index 78f3568c74..d280243e08 100644 --- a/salt/states/group.py +++ b/salt/states/group.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +r''' Management of user groups ========================= @@ -100,7 +100,7 @@ def present(name, addusers=None, delusers=None, members=None): - ''' + r''' Ensure that a group is present Args: From 9a5ae2bba14256537a03eed19557737878a14a01 Mon Sep 17 00:00:00 2001 From: Vitaliy Fuks Date: Sat, 1 Jul 2017 02:07:00 +0000 Subject: [PATCH 416/508] Removed several uses of name.split('.')[0] in SoftLayer driver. This code is breaking when server names with multiple periods are used - for example, name=server1.prod.dc1 domain=example.com. Simply "split and use first one isn't the reverse operation of "'.'.join([name, domain])" which is done when VM is created here: https://github.com/saltstack/salt/commit/9dcd11c1551975faf32de54bed913b566a7d12d0#diff-ac8112a3f8d2ebde0edf104797fe64d7 We've been running with these changes for a while and don't have any issues creating or destroying instances. The caveat that some operations (such as destroying) through SoftLayer API require hostname (without domain) is already documented in Salt and still applies. --- salt/cloud/__init__.py | 9 --------- salt/cloud/clouds/softlayer.py | 5 +---- salt/cloud/clouds/softlayer_hw.py | 3 --- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index d7f1be2fd4..bb9530f8e4 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -730,15 +730,6 @@ class Cloud(object): continue for vm_name, details in six.iteritems(vms): - # If VM was created with use_fqdn with either of the softlayer drivers, - # we need to strip the VM name and only search for the short hostname. - if driver == 'softlayer' or driver == 'softlayer_hw': - ret = [] - for name in names: - name = name.split('.')[0] - ret.append(name) - if vm_name not in ret: - continue # XXX: The logic below can be removed once the aws driver # is removed elif vm_name not in names: diff --git a/salt/cloud/clouds/softlayer.py b/salt/cloud/clouds/softlayer.py index 457447b431..9b9343a1e0 100644 --- a/salt/cloud/clouds/softlayer.py +++ b/salt/cloud/clouds/softlayer.py @@ -508,7 +508,7 @@ def list_nodes_full(mask='mask[id]', call=None): conn = get_conn(service='SoftLayer_Account') response = conn.getVirtualGuests() for node_id in response: - hostname = node_id['hostname'].split('.')[0] + hostname = node_id['hostname'] ret[hostname] = node_id __utils__['cloud.cache_node_list'](ret, __active_provider_name__.split(':')[0], __opts__) return ret @@ -594,9 +594,6 @@ def destroy(name, call=None): transport=__opts__['transport'] ) - # If the VM was created with use_fqdn, the short hostname will be used instead. - name = name.split('.')[0] - node = show_instance(name, call='action') conn = get_conn() response = conn.deleteObject(id=node['id']) diff --git a/salt/cloud/clouds/softlayer_hw.py b/salt/cloud/clouds/softlayer_hw.py index 34dae95cca..030391fe6d 100644 --- a/salt/cloud/clouds/softlayer_hw.py +++ b/salt/cloud/clouds/softlayer_hw.py @@ -526,9 +526,6 @@ def destroy(name, call=None): transport=__opts__['transport'] ) - # If the VM was created with use_fqdn, the short hostname will be used instead. - name = name.split('.')[0] - node = show_instance(name, call='action') conn = get_conn(service='SoftLayer_Ticket') response = conn.createCancelServerTicket( From 8a6ad0a9cfa8cddd9f828c83983ee9be25ac3208 Mon Sep 17 00:00:00 2001 From: Vitaliy Fuks Date: Sat, 1 Jul 2017 12:47:52 +0000 Subject: [PATCH 417/508] Fixed typo. --- salt/cloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index bb9530f8e4..73327724ef 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -732,7 +732,7 @@ class Cloud(object): for vm_name, details in six.iteritems(vms): # XXX: The logic below can be removed once the aws driver # is removed - elif vm_name not in names: + if vm_name not in names: continue elif driver == 'ec2' and 'aws' in handled_drivers and \ From 9f3047c42023acecd662fc329450c5f43c3c07f3 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Mon, 17 Jul 2017 14:55:30 -0500 Subject: [PATCH 418/508] add additional checks for ADM policies that have the same ADMX policy ID (#42279) --- salt/modules/win_lgpo.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 347ca742f9..e55403adbc 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -4590,11 +4590,35 @@ def _lookup_admin_template(policy_name, if adml_search_results: multiple_adml_entries = False suggested_policies = '' + adml_to_remove = [] if len(adml_search_results) > 1: multiple_adml_entries = True for adml_search_result in adml_search_results: if not getattr(adml_search_result, 'text', '').strip() == policy_name: - adml_search_results.remove(adml_search_result) + adml_to_remove.append(adml_search_result) + if hierarchy: + display_name_searchval = '$({0}.{1})'.format( + adml_search_result.tag.split('}')[1], + adml_search_result.attrib['id']) + policy_search_string = '//{0}:policy[@*[local-name() = "displayName"] = "{1}" and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = "{2}") ]'.format( + adml_search_result.prefix, + display_name_searchval, + policy_class) + # this should only be 1 result + admx_search_results = admx_policy_definitions.xpath(policy_search_string, namespaces=adml_search_result.nsmap) + for search_result in admx_search_results: + this_hierarchy = _build_parent_list(search_result, + admx_policy_definitions, + True, + adml_policy_resources) + this_hierarchy.reverse() + if hierarchy != this_hierarchy: + adml_to_remove.append(adml_search_result) + for adml in adml_to_remove: + if adml in adml_search_results: + adml_search_results.remove(adml) + if len(adml_search_results) == 1 and multiple_adml_entries: + multiple_adml_entries = False for adml_search_result in adml_search_results: dmsg = 'found an ADML entry matching the string! {0} -- {1}' log.debug(dmsg.format(adml_search_result.tag, From ecd446fd55f65be7c9f29689bf650eb6a8bc4143 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Tue, 22 Aug 2017 09:04:31 -0500 Subject: [PATCH 419/508] track xml namespace to ensure policies w/duplicate IDs or Names do not conflict --- salt/modules/win_lgpo.py | 827 +++++++++++++++++++++------------------ 1 file changed, 446 insertions(+), 381 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index e55403adbc..ee0dbbad6d 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -3503,22 +3503,31 @@ def _checkAllAdmxPolicies(policy_class, not_configured_policies.remove(policy_item) for not_configured_policy in not_configured_policies: - policy_vals[not_configured_policy.attrib['name']] = 'Not Configured' + not_configured_policy_namespace = not_configured_policy.nsmap[not_configured_policy.prefix] + if not_configured_policy_namespace not in policy_vals: + policy_vals[not_configured_policy_namespace] = {} + policy_vals[not_configured_policy_namespace][not_configured_policy.attrib['name']] = 'Not Configured' if return_full_policy_names: - full_names[not_configured_policy.attrib['name']] = _getFullPolicyName( + if not_configured_policy_namespace not in full_names: + full_names[not_configured_policy_namespace] = {} + full_names[not_configured_policy_namespace][not_configured_policy.attrib['name']] = _getFullPolicyName( not_configured_policy, not_configured_policy.attrib['name'], return_full_policy_names, adml_policy_resources) log.debug('building hierarchy for non-configured item {0}'.format(not_configured_policy.attrib['name'])) - hierarchy[not_configured_policy.attrib['name']] = _build_parent_list(not_configured_policy, - admx_policy_definitions, - return_full_policy_names, - adml_policy_resources) + if not_configured_policy_namespace not in hierarchy: + hierarchy[not_configured_policy_namespace] = {} + hierarchy[not_configured_policy_namespace][not_configured_policy.attrib['name']] = _build_parent_list( + not_configured_policy, + admx_policy_definitions, + return_full_policy_names, + adml_policy_resources) for admx_policy in admx_policies: this_key = None this_valuename = None this_policyname = None + this_policynamespace = None this_policy_setting = 'Not Configured' element_only_enabled_disabled = True explicit_enable_disable_value_setting = False @@ -3537,6 +3546,7 @@ def _checkAllAdmxPolicies(policy_class, log.error('policy item {0} does not have the required "name" ' 'attribute'.format(admx_policy.attrib)) break + this_policynamespace = admx_policy.nsmap[admx_policy.prefix] if ENABLED_VALUE_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True @@ -3548,7 +3558,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if DISABLED_VALUE_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True @@ -3560,21 +3572,27 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if ENABLED_LIST_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True if _checkListItem(admx_policy, this_policyname, this_key, ENABLED_LIST_XPATH, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if DISABLED_LIST_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True if _checkListItem(admx_policy, this_policyname, this_key, DISABLED_LIST_XPATH, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if not explicit_enable_disable_value_setting and this_valuename: # the policy has a key/valuename but no explicit enabled/Disabled @@ -3587,7 +3605,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting elif _regexSearchRegPolData(re.escape(_buildKnownDataSearchString(this_key, this_valuename, 'REG_DWORD', @@ -3596,7 +3616,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if ELEMENTS_XPATH(admx_policy): if element_only_enabled_disabled or this_policy_setting == 'Enabled': @@ -3794,65 +3816,84 @@ def _checkAllAdmxPolicies(policy_class, and len(configured_elements.keys()) == len(required_elements.keys()): if policy_disabled_elements == len(required_elements.keys()): log.debug('{0} is disabled by all enum elements'.format(this_policyname)) - policy_vals[this_policyname] = 'Disabled' + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = 'Disabled' else: - policy_vals[this_policyname] = configured_elements + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = configured_elements log.debug('{0} is enabled by enum elements'.format(this_policyname)) else: if this_policy_setting == 'Enabled': - policy_vals[this_policyname] = configured_elements - if return_full_policy_names and this_policyname in policy_vals: - full_names[this_policyname] = _getFullPolicyName( + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = configured_elements + if return_full_policy_names and this_policynamespace in policy_vals and this_policyname in policy_vals[this_policynamespace]: + if this_policynamespace not in full_names: + full_names[this_policynamespace] = {} + full_names[this_policynamespace][this_policyname] = _getFullPolicyName( admx_policy, admx_policy.attrib['name'], return_full_policy_names, adml_policy_resources) - if this_policyname in policy_vals: - hierarchy[this_policyname] = _build_parent_list(admx_policy, + if this_policynamespace in policy_vals and this_policyname in policy_vals[this_policynamespace]: + if this_policynamespace not in hierarchy: + hierarchy[this_policynamespace] = {} + hierarchy[this_policynamespace][this_policyname] = _build_parent_list(admx_policy, admx_policy_definitions, return_full_policy_names, adml_policy_resources) if policy_vals and return_full_policy_names and not hierarchical_return: unpathed_dict = {} pathed_dict = {} - for policy_item in list(policy_vals): - if full_names[policy_item] in policy_vals: - # add this item with the path'd full name - full_path_list = hierarchy[policy_item] + for policy_namespace in list(policy_vals): + for policy_item in list(policy_vals[policy_namespace]): + if full_names[policy_namespace][policy_item] in policy_vals[policy_namespace]: + # add this item with the path'd full name + full_path_list = hierarchy[policy_namespace][policy_item] + full_path_list.reverse() + full_path_list.append(full_names[policy_namespace][policy_item]) + policy_vals['\\'.join(full_path_list)] = policy_vals[policy_namespace].pop(policy_item) + pathed_dict[full_names[policy_namespace][policy_item]] = True + else: + policy_vals[policy_namespace][full_names[policy_namespace][policy_item]] = policy_vals[policy_namespace].pop(policy_item) + if policy_namespace not in unpathed_dict: + unpathed_dict[policy_namespace] = {} + unpathed_dict[policy_namespace][full_names[policy_namespace][policy_item]] = policy_item + # go back and remove any "unpathed" policies that need a full path + for path_needed in unpathed_dict[policy_namespace]: + # remove the item with the same full name and re-add it w/a path'd version + full_path_list = hierarchy[policy_namespace][unpathed_dict[policy_namespace][path_needed]] full_path_list.reverse() - full_path_list.append(full_names[policy_item]) - policy_vals['\\'.join(full_path_list)] = policy_vals.pop(policy_item) - pathed_dict[full_names[policy_item]] = True - else: - policy_vals[full_names[policy_item]] = policy_vals.pop(policy_item) - unpathed_dict[full_names[policy_item]] = policy_item - # go back and remove any "unpathed" policies that need a full path - for path_needed in unpathed_dict: - # remove the item with the same full name and re-add it w/a path'd version - full_path_list = hierarchy[unpathed_dict[path_needed]] - full_path_list.reverse() - full_path_list.append(path_needed) - log.debug('full_path_list == {0}'.format(full_path_list)) - policy_vals['\\'.join(full_path_list)] = policy_vals.pop(path_needed) + full_path_list.append(path_needed) + log.debug('full_path_list == {0}'.format(full_path_list)) + policy_vals['\\'.join(full_path_list)] = policy_vals[policy_namespace].pop(path_needed) + for policy_namespace in list(policy_vals): + if policy_vals[policy_namespace] == {}: + policy_vals.pop(policy_namespace) if policy_vals and hierarchical_return: if hierarchy: - for hierarchy_item in hierarchy: - if hierarchy_item in policy_vals: - tdict = {} - first_item = True - for item in hierarchy[hierarchy_item]: - newdict = {} - if first_item: - h_policy_name = hierarchy_item - if return_full_policy_names: - h_policy_name = full_names[hierarchy_item] - newdict[item] = {h_policy_name: policy_vals.pop(hierarchy_item)} - first_item = False - else: - newdict[item] = tdict - tdict = newdict - if tdict: - policy_vals = dictupdate.update(policy_vals, tdict) + for policy_namespace in hierarchy: + for hierarchy_item in hierarchy[policy_namespace]: + if hierarchy_item in policy_vals[policy_namespace]: + tdict = {} + first_item = True + for item in hierarchy[policy_namespace][hierarchy_item]: + newdict = {} + if first_item: + h_policy_name = hierarchy_item + if return_full_policy_names: + h_policy_name = full_names[policy_namespace][hierarchy_item] + newdict[item] = {h_policy_name: policy_vals[policy_namespace].pop(hierarchy_item)} + first_item = False + else: + newdict[item] = tdict + tdict = newdict + if tdict: + policy_vals = dictupdate.update(policy_vals, tdict) + if policy_namespace in policy_vals and policy_vals[policy_namespace] == {}: + policy_vals.pop(policy_namespace) policy_vals = { module_policy_data.admx_registry_classes[policy_class]['lgpo_section']: { 'Administrative Templates': policy_vals @@ -4116,6 +4157,7 @@ def _policyFileReplaceOrAppend(this_string, policy_data, append_only=False): def _writeAdminTemplateRegPolFile(admtemplate_data, + admtemplate_namespace_data, admx_policy_definitions=None, adml_policy_resources=None, display_language='en-US', @@ -4132,7 +4174,9 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, existing_data = '' base_policy_settings = {} policy_data = _policy_info() - policySearchXpath = etree.XPath('//*[@*[local-name() = "id"] = $id or @*[local-name() = "name"] = $id]') + #//{0}:policy[@displayName = "{1}" and (@class = "Both" or @class = "{2}") ] + #policySearchXpath = etree.XPath('//*[@ns1:id = $id or @ns1:name = $id]') + policySearchXpath = '//ns1:*[@id = "{0}" or @name = "{0}"]' try: if admx_policy_definitions is None or adml_policy_resources is None: admx_policy_definitions, adml_policy_resources = _processPolicyDefinitions( @@ -4144,298 +4188,305 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, hierarchical_return=False, return_not_configured=False) log.debug('preparing to loop through policies requested to be configured') - for adm_policy in admtemplate_data: - if str(admtemplate_data[adm_policy]).lower() == 'not configured': - if adm_policy in base_policy_settings: - base_policy_settings.pop(adm_policy) - else: - log.debug('adding {0} to base_policy_settings'.format(adm_policy)) - base_policy_settings[adm_policy] = admtemplate_data[adm_policy] - for admPolicy in base_policy_settings: - log.debug('working on admPolicy {0}'.format(admPolicy)) - explicit_enable_disable_value_setting = False - this_key = None - this_valuename = None - if str(base_policy_settings[admPolicy]).lower() == 'disabled': - log.debug('time to disable {0}'.format(admPolicy)) - this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy) - if this_policy: - this_policy = this_policy[0] - if 'class' in this_policy.attrib: - if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': - if 'key' in this_policy.attrib: - this_key = this_policy.attrib['key'] - else: - msg = 'policy item {0} does not have the required "key" attribute' - log.error(msg.format(this_policy.attrib)) - break - if 'valueName' in this_policy.attrib: - this_valuename = this_policy.attrib['valueName'] - if DISABLED_VALUE_XPATH(this_policy): - # set the disabled value in the registry.pol file - explicit_enable_disable_value_setting = True - disabled_value_string = _checkValueItemParent(this_policy, - admPolicy, - this_key, - this_valuename, - DISABLED_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend(disabled_value_string, - existing_data) - if DISABLED_LIST_XPATH(this_policy): - explicit_enable_disable_value_setting = True - disabled_list_strings = _checkListItem(this_policy, - admPolicy, - this_key, - DISABLED_LIST_XPATH, - None, - test_items=False) - log.debug('working with disabledList portion of {0}'.format(admPolicy)) - existing_data = _policyFileReplaceOrAppendList(disabled_list_strings, + for adm_namespace in admtemplate_data: + for adm_policy in admtemplate_data[adm_namespace]: + if str(admtemplate_data[adm_namespace][adm_policy]).lower() == 'not configured': + if adm_policy in base_policy_settings[adm_namespace]: + base_policy_settings[adm_namespace].pop(adm_policy) + else: + log.debug('adding {0} to base_policy_settings'.format(adm_policy)) + if adm_namespace not in base_policy_settings: + base_policy_settings[adm_namespace] = {} + base_policy_settings[adm_namespace][adm_policy] = admtemplate_data[adm_namespace][adm_policy] + for adm_namespace in base_policy_settings: + for admPolicy in base_policy_settings[adm_namespace]: + log.debug('working on admPolicy {0}'.format(admPolicy)) + explicit_enable_disable_value_setting = False + this_key = None + this_valuename = None + if str(base_policy_settings[adm_namespace][admPolicy]).lower() == 'disabled': + log.debug('time to disable {0}'.format(admPolicy)) + #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format('ns1', admPolicy), namespaces={'ns1': adm_namespace}) + if this_policy: + this_policy = this_policy[0] + if 'class' in this_policy.attrib: + if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': + if 'key' in this_policy.attrib: + this_key = this_policy.attrib['key'] + else: + msg = 'policy item {0} does not have the required "key" attribute' + log.error(msg.format(this_policy.attrib)) + break + if 'valueName' in this_policy.attrib: + this_valuename = this_policy.attrib['valueName'] + if DISABLED_VALUE_XPATH(this_policy): + # set the disabled value in the registry.pol file + explicit_enable_disable_value_setting = True + disabled_value_string = _checkValueItemParent(this_policy, + admPolicy, + this_key, + this_valuename, + DISABLED_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend(disabled_value_string, existing_data) - if not explicit_enable_disable_value_setting and this_valuename: - disabled_value_string = _buildKnownDataSearchString(this_key, - this_valuename, - 'REG_DWORD', - None, - check_deleted=True) - existing_data = _policyFileReplaceOrAppend(disabled_value_string, - existing_data) - if ELEMENTS_XPATH(this_policy): - log.debug('checking elements of {0}'.format(admPolicy)) - for elements_item in ELEMENTS_XPATH(this_policy): - for child_item in elements_item.getchildren(): - child_key = this_key - child_valuename = this_valuename - if 'key' in child_item.attrib: - child_key = child_item.attrib['key'] - if 'valueName' in child_item.attrib: - child_valuename = child_item.attrib['valueName'] - if etree.QName(child_item).localname == 'boolean' \ - and (TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): - # WARNING: no OOB adm files use true/falseList items - # this has not been fully vetted - temp_dict = {'trueList': TRUE_LIST_XPATH, 'falseList': FALSE_LIST_XPATH} - for this_list in temp_dict: - disabled_list_strings = _checkListItem( - child_item, - admPolicy, - child_key, - temp_dict[this_list], - None, - test_items=False) - log.debug('working with {1} portion of {0}'.format( - admPolicy, - this_list)) - existing_data = _policyFileReplaceOrAppendList( - disabled_list_strings, - existing_data) - elif etree.QName(child_item).localname == 'boolean' \ - or etree.QName(child_item).localname == 'decimal' \ - or etree.QName(child_item).localname == 'text' \ - or etree.QName(child_item).localname == 'longDecimal' \ - or etree.QName(child_item).localname == 'multiText' \ - or etree.QName(child_item).localname == 'enum': - disabled_value_string = _processValueItem(child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=True) - msg = 'I have disabled value string of {0}' - log.debug(msg.format(disabled_value_string)) - existing_data = _policyFileReplaceOrAppend( - disabled_value_string, - existing_data) - elif etree.QName(child_item).localname == 'list': - disabled_value_string = _processValueItem(child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=True) - msg = 'I have disabled value string of {0}' - log.debug(msg.format(disabled_value_string)) - existing_data = _policyFileReplaceOrAppend( - disabled_value_string, - existing_data) - else: - msg = 'policy {0} was found but it does not appear to be valid for the class {1}' - log.error(msg.format(admPolicy, registry_class)) - else: - msg = 'policy item {0} does not have the requried "class" attribute' - log.error(msg.format(this_policy.attrib)) - else: - log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) - this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy) - if this_policy: - this_policy = this_policy[0] - if 'class' in this_policy.attrib: - if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': - if 'key' in this_policy.attrib: - this_key = this_policy.attrib['key'] - else: - msg = 'policy item {0} does not have the required "key" attribute' - log.error(msg.format(this_policy.attrib)) - break - if 'valueName' in this_policy.attrib: - this_valuename = this_policy.attrib['valueName'] - - if ENABLED_VALUE_XPATH(this_policy): - explicit_enable_disable_value_setting = True - enabled_value_string = _checkValueItemParent(this_policy, - admPolicy, - this_key, - this_valuename, - ENABLED_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if ENABLED_LIST_XPATH(this_policy): - explicit_enable_disable_value_setting = True - enabled_list_strings = _checkListItem(this_policy, - admPolicy, - this_key, - ENABLED_LIST_XPATH, - None, - test_items=False) - log.debug('working with enabledList portion of {0}'.format(admPolicy)) - existing_data = _policyFileReplaceOrAppendList( - enabled_list_strings, - existing_data) - if not explicit_enable_disable_value_setting and this_valuename: - enabled_value_string = _buildKnownDataSearchString(this_key, - this_valuename, - 'REG_DWORD', - '1', - check_deleted=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if ELEMENTS_XPATH(this_policy): - for elements_item in ELEMENTS_XPATH(this_policy): - for child_item in elements_item.getchildren(): - child_key = this_key - child_valuename = this_valuename - if 'key' in child_item.attrib: - child_key = child_item.attrib['key'] - if 'valueName' in child_item.attrib: - child_valuename = child_item.attrib['valueName'] - if child_item.attrib['id'] in base_policy_settings[admPolicy]: - if etree.QName(child_item).localname == 'boolean' and ( - TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): - list_strings = [] - if base_policy_settings[admPolicy][child_item.attrib['id']]: - list_strings = _checkListItem(child_item, - admPolicy, - child_key, - TRUE_LIST_XPATH, - None, - test_items=False) - log.debug('working with trueList portion of {0}'.format(admPolicy)) - else: - list_strings = _checkListItem(child_item, - admPolicy, - child_key, - FALSE_LIST_XPATH, - None, - test_items=False) - existing_data = _policyFileReplaceOrAppendList( - list_strings, - existing_data) - if etree.QName(child_item).localname == 'boolean' and ( - TRUE_VALUE_XPATH(child_item) or FALSE_VALUE_XPATH(child_item)): - value_string = '' - if base_policy_settings[admPolicy][child_item.attrib['id']]: - value_string = _checkValueItemParent(child_item, - admPolicy, - child_key, - child_valuename, - TRUE_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - else: - value_string = _checkValueItemParent(child_item, - admPolicy, - child_key, - child_valuename, - FALSE_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - value_string, - existing_data) + if DISABLED_LIST_XPATH(this_policy): + explicit_enable_disable_value_setting = True + disabled_list_strings = _checkListItem(this_policy, + admPolicy, + this_key, + DISABLED_LIST_XPATH, + None, + test_items=False) + log.debug('working with disabledList portion of {0}'.format(admPolicy)) + existing_data = _policyFileReplaceOrAppendList(disabled_list_strings, + existing_data) + if not explicit_enable_disable_value_setting and this_valuename: + disabled_value_string = _buildKnownDataSearchString(this_key, + this_valuename, + 'REG_DWORD', + None, + check_deleted=True) + existing_data = _policyFileReplaceOrAppend(disabled_value_string, + existing_data) + if ELEMENTS_XPATH(this_policy): + log.debug('checking elements of {0}'.format(admPolicy)) + for elements_item in ELEMENTS_XPATH(this_policy): + for child_item in elements_item.getchildren(): + child_key = this_key + child_valuename = this_valuename + if 'key' in child_item.attrib: + child_key = child_item.attrib['key'] + if 'valueName' in child_item.attrib: + child_valuename = child_item.attrib['valueName'] if etree.QName(child_item).localname == 'boolean' \ + and (TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): + # WARNING: no OOB adm files use true/falseList items + # this has not been fully vetted + temp_dict = {'trueList': TRUE_LIST_XPATH, 'falseList': FALSE_LIST_XPATH} + for this_list in temp_dict: + disabled_list_strings = _checkListItem( + child_item, + admPolicy, + child_key, + temp_dict[this_list], + None, + test_items=False) + log.debug('working with {1} portion of {0}'.format( + admPolicy, + this_list)) + existing_data = _policyFileReplaceOrAppendList( + disabled_list_strings, + existing_data) + elif etree.QName(child_item).localname == 'boolean' \ or etree.QName(child_item).localname == 'decimal' \ or etree.QName(child_item).localname == 'text' \ or etree.QName(child_item).localname == 'longDecimal' \ - or etree.QName(child_item).localname == 'multiText': - enabled_value_string = _processValueItem( - child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=False, - this_element_value=base_policy_settings[admPolicy][child_item.attrib['id']]) - msg = 'I have enabled value string of {0}' - log.debug(msg.format([enabled_value_string])) + or etree.QName(child_item).localname == 'multiText' \ + or etree.QName(child_item).localname == 'enum': + disabled_value_string = _processValueItem(child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=True) + msg = 'I have disabled value string of {0}' + log.debug(msg.format(disabled_value_string)) existing_data = _policyFileReplaceOrAppend( - enabled_value_string, + disabled_value_string, existing_data) - elif etree.QName(child_item).localname == 'enum': - for enum_item in child_item.getchildren(): - if base_policy_settings[admPolicy][child_item.attrib['id']] == \ - _getAdmlDisplayName(adml_policy_resources, - enum_item.attrib['displayName'] - ).strip(): - enabled_value_string = _checkValueItemParent( - enum_item, - child_item.attrib['id'], - child_key, - child_valuename, - VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if VALUE_LIST_XPATH(enum_item): - enabled_list_strings = _checkListItem(enum_item, - admPolicy, - child_key, - VALUE_LIST_XPATH, - None, - test_items=False) - msg = 'working with valueList portion of {0}' - log.debug(msg.format(child_item.attrib['id'])) - existing_data = _policyFileReplaceOrAppendList( - enabled_list_strings, - existing_data) - break elif etree.QName(child_item).localname == 'list': - enabled_value_string = _processValueItem( - child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=False, - this_element_value=base_policy_settings[admPolicy][child_item.attrib['id']]) - msg = 'I have enabled value string of {0}' - log.debug(msg.format([enabled_value_string])) + disabled_value_string = _processValueItem(child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=True) + msg = 'I have disabled value string of {0}' + log.debug(msg.format(disabled_value_string)) existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data, - append_only=True) + disabled_value_string, + existing_data) + else: + msg = 'policy {0} was found but it does not appear to be valid for the class {1}' + log.error(msg.format(admPolicy, registry_class)) + else: + msg = 'policy item {0} does not have the requried "class" attribute' + log.error(msg.format(this_policy.attrib)) + else: + log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) + #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) + log.debug('found this_policy == {0}'.format(this_policy)) + if this_policy: + this_policy = this_policy[0] + if 'class' in this_policy.attrib: + if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': + if 'key' in this_policy.attrib: + this_key = this_policy.attrib['key'] + else: + msg = 'policy item {0} does not have the required "key" attribute' + log.error(msg.format(this_policy.attrib)) + break + if 'valueName' in this_policy.attrib: + this_valuename = this_policy.attrib['valueName'] + + if ENABLED_VALUE_XPATH(this_policy): + explicit_enable_disable_value_setting = True + enabled_value_string = _checkValueItemParent(this_policy, + admPolicy, + this_key, + this_valuename, + ENABLED_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if ENABLED_LIST_XPATH(this_policy): + explicit_enable_disable_value_setting = True + enabled_list_strings = _checkListItem(this_policy, + admPolicy, + this_key, + ENABLED_LIST_XPATH, + None, + test_items=False) + log.debug('working with enabledList portion of {0}'.format(admPolicy)) + existing_data = _policyFileReplaceOrAppendList( + enabled_list_strings, + existing_data) + if not explicit_enable_disable_value_setting and this_valuename: + enabled_value_string = _buildKnownDataSearchString(this_key, + this_valuename, + 'REG_DWORD', + '1', + check_deleted=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if ELEMENTS_XPATH(this_policy): + for elements_item in ELEMENTS_XPATH(this_policy): + for child_item in elements_item.getchildren(): + child_key = this_key + child_valuename = this_valuename + if 'key' in child_item.attrib: + child_key = child_item.attrib['key'] + if 'valueName' in child_item.attrib: + child_valuename = child_item.attrib['valueName'] + if child_item.attrib['id'] in base_policy_settings[adm_namespace][admPolicy]: + if etree.QName(child_item).localname == 'boolean' and ( + TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): + list_strings = [] + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]: + list_strings = _checkListItem(child_item, + admPolicy, + child_key, + TRUE_LIST_XPATH, + None, + test_items=False) + log.debug('working with trueList portion of {0}'.format(admPolicy)) + else: + list_strings = _checkListItem(child_item, + admPolicy, + child_key, + FALSE_LIST_XPATH, + None, + test_items=False) + existing_data = _policyFileReplaceOrAppendList( + list_strings, + existing_data) + elif etree.QName(child_item).localname == 'boolean' and ( + TRUE_VALUE_XPATH(child_item) or FALSE_VALUE_XPATH(child_item)): + value_string = '' + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]: + value_string = _checkValueItemParent(child_item, + admPolicy, + child_key, + child_valuename, + TRUE_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + else: + value_string = _checkValueItemParent(child_item, + admPolicy, + child_key, + child_valuename, + FALSE_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + value_string, + existing_data) + elif etree.QName(child_item).localname == 'boolean' \ + or etree.QName(child_item).localname == 'decimal' \ + or etree.QName(child_item).localname == 'text' \ + or etree.QName(child_item).localname == 'longDecimal' \ + or etree.QName(child_item).localname == 'multiText': + enabled_value_string = _processValueItem( + child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=False, + this_element_value=base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]) + msg = 'I have enabled value string of {0}' + log.debug(msg.format([enabled_value_string])) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + elif etree.QName(child_item).localname == 'enum': + for enum_item in child_item.getchildren(): + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']] == \ + _getAdmlDisplayName(adml_policy_resources, + enum_item.attrib['displayName'] + ).strip(): + enabled_value_string = _checkValueItemParent( + enum_item, + child_item.attrib['id'], + child_key, + child_valuename, + VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if VALUE_LIST_XPATH(enum_item): + enabled_list_strings = _checkListItem(enum_item, + admPolicy, + child_key, + VALUE_LIST_XPATH, + None, + test_items=False) + msg = 'working with valueList portion of {0}' + log.debug(msg.format(child_item.attrib['id'])) + existing_data = _policyFileReplaceOrAppendList( + enabled_list_strings, + existing_data) + break + elif etree.QName(child_item).localname == 'list': + enabled_value_string = _processValueItem( + child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=False, + this_element_value=base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]) + msg = 'I have enabled value string of {0}' + log.debug(msg.format([enabled_value_string])) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data, + append_only=True) _write_regpol_data(existing_data, policy_data.admx_registry_classes[registry_class]['policy_path'], policy_data.gpt_ini_path, @@ -4551,6 +4602,7 @@ def _lookup_admin_template(policy_name, if admx_policy_definitions is None or adml_policy_resources is None: admx_policy_definitions, adml_policy_resources = _processPolicyDefinitions( display_language=adml_language) + admx_search_results = [] admx_search_results = ADMX_SEARCH_XPATH(admx_policy_definitions, policy_name=policy_name, registry_class=policy_class) @@ -4596,24 +4648,31 @@ def _lookup_admin_template(policy_name, for adml_search_result in adml_search_results: if not getattr(adml_search_result, 'text', '').strip() == policy_name: adml_to_remove.append(adml_search_result) - if hierarchy: - display_name_searchval = '$({0}.{1})'.format( - adml_search_result.tag.split('}')[1], - adml_search_result.attrib['id']) - policy_search_string = '//{0}:policy[@*[local-name() = "displayName"] = "{1}" and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = "{2}") ]'.format( - adml_search_result.prefix, - display_name_searchval, - policy_class) - # this should only be 1 result - admx_search_results = admx_policy_definitions.xpath(policy_search_string, namespaces=adml_search_result.nsmap) - for search_result in admx_search_results: - this_hierarchy = _build_parent_list(search_result, - admx_policy_definitions, - True, - adml_policy_resources) - this_hierarchy.reverse() - if hierarchy != this_hierarchy: - adml_to_remove.append(adml_search_result) + else: + if hierarchy: + display_name_searchval = '$({0}.{1})'.format( + adml_search_result.tag.split('}')[1], + adml_search_result.attrib['id']) + #policy_search_string = '//{0}:policy[@*[local-name() = "displayName"] = "{1}" and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = "{2}") ]'.format( + policy_search_string = '//{0}:policy[@displayName = "{1}" and (@class = "Both" or @class = "{2}") ]'.format( + adml_search_result.prefix, + display_name_searchval, + policy_class) + admx_results = [] + admx_search_results = admx_policy_definitions.xpath(policy_search_string, namespaces=adml_search_result.nsmap) + for search_result in admx_search_results: + log.debug('policy_name == {0}'.format(policy_name)) + this_hierarchy = _build_parent_list(search_result, + admx_policy_definitions, + True, + adml_policy_resources) + this_hierarchy.reverse() + if hierarchy != this_hierarchy: + adml_to_remove.append(adml_search_result) + else: + admx_results.append(search_result) + if len(admx_results) == 1: + admx_search_results = admx_results for adml in adml_to_remove: if adml in adml_search_results: adml_search_results.remove(adml) @@ -4627,10 +4686,11 @@ def _lookup_admin_template(policy_name, adml_search_result.tag.split('}')[1], adml_search_result.attrib['id']) log.debug('searching for displayName == {0}'.format(display_name_searchval)) - admx_search_results = ADMX_DISPLAYNAME_SEARCH_XPATH( - admx_policy_definitions, - display_name=display_name_searchval, - registry_class=policy_class) + if not admx_search_results: + admx_search_results = ADMX_DISPLAYNAME_SEARCH_XPATH( + admx_policy_definitions, + display_name=display_name_searchval, + registry_class=policy_class) if admx_search_results: if len(admx_search_results) == 1 or hierarchy and not multiple_adml_entries: found = False @@ -4642,6 +4702,7 @@ def _lookup_admin_template(policy_name, True, adml_policy_resources) this_hierarchy.reverse() + log.debug('testing {0} == {1}'.format(hierarchy, this_hierarchy)) if hierarchy == this_hierarchy: found = True else: @@ -5100,6 +5161,7 @@ def set_(computer_policy=None, user_policy=None, if policies[p_class]: for policy_name in policies[p_class]: _pol = None + policy_namespace = None policy_key_name = policy_name if policy_name in _policydata.policies[p_class]['policies']: _pol = _policydata.policies[p_class]['policies'][policy_name] @@ -5149,16 +5211,19 @@ def set_(computer_policy=None, user_policy=None, adml_policy_resources=admlPolicyResources) if success: policy_name = the_policy.attrib['name'] - _admTemplateData[policy_name] = _value + policy_namespace = the_policy.nsmap[the_policy.prefix] + if policy_namespace not in _admTemplateData: + _admTemplateData[policy_namespace] = {} + _admTemplateData[policy_namespace][policy_name] = _value else: raise SaltInvocationError(msg) - if policy_name in _admTemplateData and the_policy is not None: - log.debug('setting == {0}'.format(_admTemplateData[policy_name]).lower()) - log.debug('{0}'.format(str(_admTemplateData[policy_name]).lower())) - if str(_admTemplateData[policy_name]).lower() != 'disabled' \ - and str(_admTemplateData[policy_name]).lower() != 'not configured': + if policy_namespace and policy_name in _admTemplateData[policy_namespace] and the_policy is not None: + log.debug('setting == {0}'.format(_admTemplateData[policy_namespace][policy_name]).lower()) + log.debug('{0}'.format(str(_admTemplateData[policy_namespace][policy_name]).lower())) + if str(_admTemplateData[policy_namespace][policy_name]).lower() != 'disabled' \ + and str(_admTemplateData[policy_namespace][policy_name]).lower() != 'not configured': if ELEMENTS_XPATH(the_policy): - if isinstance(_admTemplateData[policy_name], dict): + if isinstance(_admTemplateData[policy_namespace][policy_name], dict): for elements_item in ELEMENTS_XPATH(the_policy): for child_item in elements_item.getchildren(): # check each element @@ -5169,9 +5234,9 @@ def set_(computer_policy=None, user_policy=None, True, admlPolicyResources) log.debug('id attribute == "{0}" this_element_name == "{1}"'.format(child_item.attrib['id'], this_element_name)) - if this_element_name in _admTemplateData[policy_name]: + if this_element_name in _admTemplateData[policy_namespace][policy_name]: temp_element_name = this_element_name - elif child_item.attrib['id'] in _admTemplateData[policy_name]: + elif child_item.attrib['id'] in _admTemplateData[policy_namespace][policy_name]: temp_element_name = child_item.attrib['id'] else: msg = ('Element "{0}" must be included' @@ -5179,12 +5244,12 @@ def set_(computer_policy=None, user_policy=None, raise SaltInvocationError(msg.format(this_element_name, policy_name)) if 'required' in child_item.attrib \ and child_item.attrib['required'].lower() == 'true': - if not _admTemplateData[policy_name][temp_element_name]: + if not _admTemplateData[policy_namespace][policy_name][temp_element_name]: msg = 'Element "{0}" requires a value to be specified' raise SaltInvocationError(msg.format(temp_element_name)) if etree.QName(child_item).localname == 'boolean': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], bool): msg = 'Element {0} requires a boolean True or False' raise SaltInvocationError(msg.format(temp_element_name)) @@ -5196,9 +5261,9 @@ def set_(computer_policy=None, user_policy=None, min_val = int(child_item.attrib['minValue']) if 'maxValue' in child_item.attrib: max_val = int(child_item.attrib['maxValue']) - if int(_admTemplateData[policy_name][temp_element_name]) \ + if int(_admTemplateData[policy_namespace][policy_name][temp_element_name]) \ < min_val or \ - int(_admTemplateData[policy_name][temp_element_name]) \ + int(_admTemplateData[policy_namespace][policy_name][temp_element_name]) \ > max_val: msg = 'Element "{0}" value must be between {1} and {2}' raise SaltInvocationError(msg.format(temp_element_name, @@ -5208,7 +5273,7 @@ def set_(computer_policy=None, user_policy=None, # make sure the value is in the enumeration found = False for enum_item in child_item.getchildren(): - if _admTemplateData[policy_name][temp_element_name] == \ + if _admTemplateData[policy_namespace][policy_name][temp_element_name] == \ _getAdmlDisplayName( admlPolicyResources, enum_item.attrib['displayName']).strip(): @@ -5222,33 +5287,33 @@ def set_(computer_policy=None, user_policy=None, and child_item.attrib['explicitValue'].lower() == \ 'true': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], dict): msg = ('Each list item of element "{0}" ' 'requires a dict value') msg = msg.format(temp_element_name) raise SaltInvocationError(msg) elif not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], list): msg = 'Element "{0}" requires a list value' msg = msg.format(temp_element_name) raise SaltInvocationError(msg) elif etree.QName(child_item).localname == 'multiText': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], list): msg = 'Element "{0}" requires a list value' msg = msg.format(temp_element_name) raise SaltInvocationError(msg) - _admTemplateData[policy_name][child_item.attrib['id']] = \ - _admTemplateData[policy_name].pop(temp_element_name) + _admTemplateData[policy_namespace][policy_name][child_item.attrib['id']] = \ + _admTemplateData[policy_namespace][policy_name].pop(temp_element_name) else: msg = 'The policy "{0}" has elements which must be configured' msg = msg.format(policy_name) raise SaltInvocationError(msg) else: - if str(_admTemplateData[policy_name]).lower() != 'enabled': + if str(_admTemplateData[policy_namespace][policy_name]).lower() != 'enabled': msg = ('The policy {0} must either be "Enabled", ' '"Disabled", or "Not Configured"') msg = msg.format(policy_name) @@ -5336,4 +5401,4 @@ def set_(computer_policy=None, user_policy=None, return True else: msg = 'You have to specify something!' - raise SaltInvocationError(msg) + raise SaltInvocationError(msg) \ No newline at end of file From f74480f11e7907e3af85dc110b01c120769c5924 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Tue, 22 Aug 2017 09:53:07 -0500 Subject: [PATCH 420/508] lint fix --- salt/modules/win_lgpo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index ee0dbbad6d..766f9162be 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -5401,4 +5401,4 @@ def set_(computer_policy=None, user_policy=None, return True else: msg = 'You have to specify something!' - raise SaltInvocationError(msg) \ No newline at end of file + raise SaltInvocationError(msg) From ed97cff5f6b787c0895e71f842df1da9017f098d Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 22 Aug 2017 16:59:22 -0600 Subject: [PATCH 421/508] Fix `unit.utils.test_which` for Windows This test wasn't really written with Windows in mind. Uses PATHEXT that actually resembles a Windows environment. The test value has the correct path seperator for Windows. --- tests/unit/utils/test_which.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/unit/utils/test_which.py b/tests/unit/utils/test_which.py index 9ab674791d..6bb4cf6e1a 100644 --- a/tests/unit/utils/test_which.py +++ b/tests/unit/utils/test_which.py @@ -44,18 +44,21 @@ class TestWhich(TestCase): # The second, iterating through $PATH, should also return False, # still checking for Linux False, + # We will now also return False once so we get a .EXE back from + # the function, see PATHEXT below. + False, # Lastly return True, this is the windows check. True ] # Let's patch os.environ to provide a custom PATH variable - with patch.dict(os.environ, {'PATH': '/bin'}): + with patch.dict(os.environ, {'PATH': '/bin', + 'PATHEXT': '.COM;.EXE;.BAT;.CMD'}): # Let's also patch is_windows to return True with patch('salt.utils.is_windows', lambda: True): with patch('os.path.isfile', lambda x: True): self.assertEqual( salt.utils.which('this-binary-exists-under-windows'), - # The returned path should return the .exe suffix - '/bin/this-binary-exists-under-windows.EXE' + os.path.join('/bin', 'this-binary-exists-under-windows.EXE') ) def test_missing_binary_in_windows(self): @@ -106,6 +109,5 @@ class TestWhich(TestCase): with patch('os.path.isfile', lambda x: True): self.assertEqual( salt.utils.which('this-binary-exists-under-windows'), - # The returned path should return the .exe suffix - '/bin/this-binary-exists-under-windows.CMD' + os.path.join('/bin', 'this-binary-exists-under-windows.CMD') ) From f4f32421abd86fcc1b21c36dcd3d0f9d92abf4b2 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Wed, 23 Aug 2017 10:03:44 +0300 Subject: [PATCH 422/508] Fix plint errors Fix plint errors --- salt/states/win_iis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/states/win_iis.py b/salt/states/win_iis.py index 74507c683c..38d2ed9ae4 100644 --- a/salt/states/win_iis.py +++ b/salt/states/win_iis.py @@ -481,7 +481,6 @@ def container_setting(name, container, settings=None): :param str container: The type of IIS container. The container types are: AppPools, Sites, SslBindings :param str settings: A dictionary of the setting names and their values. - Example of usage for the ``AppPools`` container: .. code-block:: yaml @@ -533,7 +532,7 @@ def container_setting(name, container, settings=None): settings=settings.keys()) for setting in settings: # map identity type from numeric to string for comparing - if (setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys()): + if setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys(): settings[setting] = identityType_map2string[settings[setting]] if str(settings[setting]) != str(current_settings[setting]): From 290d7b54af1926f8fb977d4613897ed3e7081ce4 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Wed, 23 Aug 2017 10:05:17 +0300 Subject: [PATCH 423/508] Fix plint errors Fix plint errors --- salt/modules/win_iis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index 80bba6c438..38217b07a5 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -1290,7 +1290,7 @@ def set_container_setting(name, container, settings): value = "'{0}'".format(settings[setting]) # Map to numeric to support server 2008 - if (setting == 'processModel.identityType' and settings[setting] in identityType_map2numeric.keys()): + if setting == 'processModel.identityType' and settings[setting] in identityType_map2numeric.keys(): value = identityType_map2numeric[settings[setting]] ps_cmd.extend(['Set-ItemProperty', @@ -1313,7 +1313,7 @@ def set_container_setting(name, container, settings): for setting in settings: # map identity type from numeric to string for comparing - if (setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys()): + if setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys(): settings[setting] = identityType_map2string[settings[setting]] if str(settings[setting]) != str(new_settings[setting]): From efc1c8c506c47e9240acc4d928cd590e3d14f4f0 Mon Sep 17 00:00:00 2001 From: Andreas Thienemann Date: Sat, 19 Aug 2017 00:19:25 +0200 Subject: [PATCH 424/508] Mark selinux._filetype_id_to_string as public function _filetype_id_to_string is a private function in the selinux module. The selinux state module calls the function as filetype_id_to_string which fails of course. Rename the function from the private one to a public one to make the state call work. Resolves #42505. --- salt/modules/selinux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/selinux.py b/salt/modules/selinux.py index 8a060e418e..d227b12eb4 100644 --- a/salt/modules/selinux.py +++ b/salt/modules/selinux.py @@ -403,7 +403,7 @@ def _context_string_to_dict(context): return ret -def _filetype_id_to_string(filetype='a'): +def filetype_id_to_string(filetype='a'): ''' Translates SELinux filetype single-letter representation to a more human-readable version (which is also used in `semanage fcontext -l`). @@ -444,7 +444,7 @@ def fcontext_get_policy(name, filetype=None, sel_type=None, sel_user=None, sel_l 'sel_role': '[^:]+', # se_role for file context is always object_r 'sel_type': sel_type or '[^:]+', 'sel_level': sel_level or '[^:]+'} - cmd_kwargs['filetype'] = '[[:alpha:] ]+' if filetype is None else _filetype_id_to_string(filetype) + cmd_kwargs['filetype'] = '[[:alpha:] ]+' if filetype is None else filetype_id_to_string(filetype) cmd = 'semanage fcontext -l | egrep ' + \ "'^{filespec}{spacer}{filetype}{spacer}{sel_user}:{sel_role}:{sel_type}:{sel_level}$'".format(**cmd_kwargs) current_entry_text = __salt__['cmd.shell'](cmd) From c5841e2ade59c3c24bf8f9a8310f0c544373b9ac Mon Sep 17 00:00:00 2001 From: Ushmodin Nikolay Date: Wed, 23 Aug 2017 22:43:04 +0700 Subject: [PATCH 425/508] state.sls hangs on file.recurse with clean: True on windows --- salt/states/file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 55cec4604e..128fc7165a 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -446,7 +446,11 @@ def _clean_dir(root, keep, exclude_pat): while True: fn_ = os.path.dirname(fn_) real_keep.add(fn_) - if fn_ in ['/', ''.join([os.path.splitdrive(fn_)[0], '\\\\'])]: + if fn_ in [ + os.sep, + ''.join([os.path.splitdrive(fn_)[0], os.sep]), + ''.join([os.path.splitdrive(fn_)[0], os.sep, os.sep]) + ]: break def _delete_not_kept(nfn): From f232bed9f98cd9c9623f8be687f6824fade9e027 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Mon, 17 Jul 2017 14:55:30 -0500 Subject: [PATCH 426/508] add additional checks for ADM policies that have the same ADMX policy ID (#42279) --- salt/modules/win_lgpo.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 21f855e3c2..cbb92e99cc 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -4597,11 +4597,35 @@ def _lookup_admin_template(policy_name, if adml_search_results: multiple_adml_entries = False suggested_policies = '' + adml_to_remove = [] if len(adml_search_results) > 1: multiple_adml_entries = True for adml_search_result in adml_search_results: if not getattr(adml_search_result, 'text', '').strip() == policy_name: - adml_search_results.remove(adml_search_result) + adml_to_remove.append(adml_search_result) + if hierarchy: + display_name_searchval = '$({0}.{1})'.format( + adml_search_result.tag.split('}')[1], + adml_search_result.attrib['id']) + policy_search_string = '//{0}:policy[@*[local-name() = "displayName"] = "{1}" and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = "{2}") ]'.format( + adml_search_result.prefix, + display_name_searchval, + policy_class) + # this should only be 1 result + admx_search_results = admx_policy_definitions.xpath(policy_search_string, namespaces=adml_search_result.nsmap) + for search_result in admx_search_results: + this_hierarchy = _build_parent_list(search_result, + admx_policy_definitions, + True, + adml_policy_resources) + this_hierarchy.reverse() + if hierarchy != this_hierarchy: + adml_to_remove.append(adml_search_result) + for adml in adml_to_remove: + if adml in adml_search_results: + adml_search_results.remove(adml) + if len(adml_search_results) == 1 and multiple_adml_entries: + multiple_adml_entries = False for adml_search_result in adml_search_results: dmsg = 'found an ADML entry matching the string! {0} -- {1}' log.debug(dmsg.format(adml_search_result.tag, From 61bd12c0de5bba5b506c716d03927f820c3b7dad Mon Sep 17 00:00:00 2001 From: lomeroe Date: Tue, 22 Aug 2017 09:04:31 -0500 Subject: [PATCH 427/508] track xml namespace to ensure policies w/duplicate IDs or Names do not conflict --- salt/modules/win_lgpo.py | 827 +++++++++++++++++++++------------------ 1 file changed, 446 insertions(+), 381 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index cbb92e99cc..ed556c4fc5 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -3510,22 +3510,31 @@ def _checkAllAdmxPolicies(policy_class, not_configured_policies.remove(policy_item) for not_configured_policy in not_configured_policies: - policy_vals[not_configured_policy.attrib['name']] = 'Not Configured' + not_configured_policy_namespace = not_configured_policy.nsmap[not_configured_policy.prefix] + if not_configured_policy_namespace not in policy_vals: + policy_vals[not_configured_policy_namespace] = {} + policy_vals[not_configured_policy_namespace][not_configured_policy.attrib['name']] = 'Not Configured' if return_full_policy_names: - full_names[not_configured_policy.attrib['name']] = _getFullPolicyName( + if not_configured_policy_namespace not in full_names: + full_names[not_configured_policy_namespace] = {} + full_names[not_configured_policy_namespace][not_configured_policy.attrib['name']] = _getFullPolicyName( not_configured_policy, not_configured_policy.attrib['name'], return_full_policy_names, adml_policy_resources) log.debug('building hierarchy for non-configured item {0}'.format(not_configured_policy.attrib['name'])) - hierarchy[not_configured_policy.attrib['name']] = _build_parent_list(not_configured_policy, - admx_policy_definitions, - return_full_policy_names, - adml_policy_resources) + if not_configured_policy_namespace not in hierarchy: + hierarchy[not_configured_policy_namespace] = {} + hierarchy[not_configured_policy_namespace][not_configured_policy.attrib['name']] = _build_parent_list( + not_configured_policy, + admx_policy_definitions, + return_full_policy_names, + adml_policy_resources) for admx_policy in admx_policies: this_key = None this_valuename = None this_policyname = None + this_policynamespace = None this_policy_setting = 'Not Configured' element_only_enabled_disabled = True explicit_enable_disable_value_setting = False @@ -3544,6 +3553,7 @@ def _checkAllAdmxPolicies(policy_class, log.error('policy item {0} does not have the required "name" ' 'attribute'.format(admx_policy.attrib)) break + this_policynamespace = admx_policy.nsmap[admx_policy.prefix] if ENABLED_VALUE_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True @@ -3555,7 +3565,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if DISABLED_VALUE_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True @@ -3567,21 +3579,27 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if ENABLED_LIST_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True if _checkListItem(admx_policy, this_policyname, this_key, ENABLED_LIST_XPATH, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if DISABLED_LIST_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True if _checkListItem(admx_policy, this_policyname, this_key, DISABLED_LIST_XPATH, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if not explicit_enable_disable_value_setting and this_valuename: # the policy has a key/valuename but no explicit enabled/Disabled @@ -3594,7 +3612,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting elif _regexSearchRegPolData(re.escape(_buildKnownDataSearchString(this_key, this_valuename, 'REG_DWORD', @@ -3603,7 +3623,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if ELEMENTS_XPATH(admx_policy): if element_only_enabled_disabled or this_policy_setting == 'Enabled': @@ -3801,65 +3823,84 @@ def _checkAllAdmxPolicies(policy_class, and len(configured_elements.keys()) == len(required_elements.keys()): if policy_disabled_elements == len(required_elements.keys()): log.debug('{0} is disabled by all enum elements'.format(this_policyname)) - policy_vals[this_policyname] = 'Disabled' + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = 'Disabled' else: - policy_vals[this_policyname] = configured_elements + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = configured_elements log.debug('{0} is enabled by enum elements'.format(this_policyname)) else: if this_policy_setting == 'Enabled': - policy_vals[this_policyname] = configured_elements - if return_full_policy_names and this_policyname in policy_vals: - full_names[this_policyname] = _getFullPolicyName( + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = configured_elements + if return_full_policy_names and this_policynamespace in policy_vals and this_policyname in policy_vals[this_policynamespace]: + if this_policynamespace not in full_names: + full_names[this_policynamespace] = {} + full_names[this_policynamespace][this_policyname] = _getFullPolicyName( admx_policy, admx_policy.attrib['name'], return_full_policy_names, adml_policy_resources) - if this_policyname in policy_vals: - hierarchy[this_policyname] = _build_parent_list(admx_policy, + if this_policynamespace in policy_vals and this_policyname in policy_vals[this_policynamespace]: + if this_policynamespace not in hierarchy: + hierarchy[this_policynamespace] = {} + hierarchy[this_policynamespace][this_policyname] = _build_parent_list(admx_policy, admx_policy_definitions, return_full_policy_names, adml_policy_resources) if policy_vals and return_full_policy_names and not hierarchical_return: unpathed_dict = {} pathed_dict = {} - for policy_item in policy_vals.keys(): - if full_names[policy_item] in policy_vals: - # add this item with the path'd full name - full_path_list = hierarchy[policy_item] + for policy_namespace in list(policy_vals): + for policy_item in list(policy_vals[policy_namespace]): + if full_names[policy_namespace][policy_item] in policy_vals[policy_namespace]: + # add this item with the path'd full name + full_path_list = hierarchy[policy_namespace][policy_item] + full_path_list.reverse() + full_path_list.append(full_names[policy_namespace][policy_item]) + policy_vals['\\'.join(full_path_list)] = policy_vals[policy_namespace].pop(policy_item) + pathed_dict[full_names[policy_namespace][policy_item]] = True + else: + policy_vals[policy_namespace][full_names[policy_namespace][policy_item]] = policy_vals[policy_namespace].pop(policy_item) + if policy_namespace not in unpathed_dict: + unpathed_dict[policy_namespace] = {} + unpathed_dict[policy_namespace][full_names[policy_namespace][policy_item]] = policy_item + # go back and remove any "unpathed" policies that need a full path + for path_needed in unpathed_dict[policy_namespace]: + # remove the item with the same full name and re-add it w/a path'd version + full_path_list = hierarchy[policy_namespace][unpathed_dict[policy_namespace][path_needed]] full_path_list.reverse() - full_path_list.append(full_names[policy_item]) - policy_vals['\\'.join(full_path_list)] = policy_vals.pop(policy_item) - pathed_dict[full_names[policy_item]] = True - else: - policy_vals[full_names[policy_item]] = policy_vals.pop(policy_item) - unpathed_dict[full_names[policy_item]] = policy_item - # go back and remove any "unpathed" policies that need a full path - for path_needed in unpathed_dict.keys(): - # remove the item with the same full name and re-add it w/a path'd version - full_path_list = hierarchy[unpathed_dict[path_needed]] - full_path_list.reverse() - full_path_list.append(path_needed) - log.debug('full_path_list == {0}'.format(full_path_list)) - policy_vals['\\'.join(full_path_list)] = policy_vals.pop(path_needed) + full_path_list.append(path_needed) + log.debug('full_path_list == {0}'.format(full_path_list)) + policy_vals['\\'.join(full_path_list)] = policy_vals[policy_namespace].pop(path_needed) + for policy_namespace in list(policy_vals): + if policy_vals[policy_namespace] == {}: + policy_vals.pop(policy_namespace) if policy_vals and hierarchical_return: if hierarchy: - for hierarchy_item in hierarchy.keys(): - if hierarchy_item in policy_vals: - tdict = {} - first_item = True - for item in hierarchy[hierarchy_item]: - newdict = {} - if first_item: - h_policy_name = hierarchy_item - if return_full_policy_names: - h_policy_name = full_names[hierarchy_item] - newdict[item] = {h_policy_name: policy_vals.pop(hierarchy_item)} - first_item = False - else: - newdict[item] = tdict - tdict = newdict - if tdict: - policy_vals = dictupdate.update(policy_vals, tdict) + for policy_namespace in hierarchy: + for hierarchy_item in hierarchy[policy_namespace]: + if hierarchy_item in policy_vals[policy_namespace]: + tdict = {} + first_item = True + for item in hierarchy[policy_namespace][hierarchy_item]: + newdict = {} + if first_item: + h_policy_name = hierarchy_item + if return_full_policy_names: + h_policy_name = full_names[policy_namespace][hierarchy_item] + newdict[item] = {h_policy_name: policy_vals[policy_namespace].pop(hierarchy_item)} + first_item = False + else: + newdict[item] = tdict + tdict = newdict + if tdict: + policy_vals = dictupdate.update(policy_vals, tdict) + if policy_namespace in policy_vals and policy_vals[policy_namespace] == {}: + policy_vals.pop(policy_namespace) policy_vals = { module_policy_data.admx_registry_classes[policy_class]['lgpo_section']: { 'Administrative Templates': policy_vals @@ -4123,6 +4164,7 @@ def _policyFileReplaceOrAppend(this_string, policy_data, append_only=False): def _writeAdminTemplateRegPolFile(admtemplate_data, + admtemplate_namespace_data, admx_policy_definitions=None, adml_policy_resources=None, display_language='en-US', @@ -4139,7 +4181,9 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, existing_data = '' base_policy_settings = {} policy_data = _policy_info() - policySearchXpath = etree.XPath('//*[@*[local-name() = "id"] = $id or @*[local-name() = "name"] = $id]') + #//{0}:policy[@displayName = "{1}" and (@class = "Both" or @class = "{2}") ] + #policySearchXpath = etree.XPath('//*[@ns1:id = $id or @ns1:name = $id]') + policySearchXpath = '//ns1:*[@id = "{0}" or @name = "{0}"]' try: if admx_policy_definitions is None or adml_policy_resources is None: admx_policy_definitions, adml_policy_resources = _processPolicyDefinitions( @@ -4151,298 +4195,305 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, hierarchical_return=False, return_not_configured=False) log.debug('preparing to loop through policies requested to be configured') - for adm_policy in admtemplate_data.keys(): - if str(admtemplate_data[adm_policy]).lower() == 'not configured': - if adm_policy in base_policy_settings: - base_policy_settings.pop(adm_policy) - else: - log.debug('adding {0} to base_policy_settings'.format(adm_policy)) - base_policy_settings[adm_policy] = admtemplate_data[adm_policy] - for admPolicy in base_policy_settings.keys(): - log.debug('working on admPolicy {0}'.format(admPolicy)) - explicit_enable_disable_value_setting = False - this_key = None - this_valuename = None - if str(base_policy_settings[admPolicy]).lower() == 'disabled': - log.debug('time to disable {0}'.format(admPolicy)) - this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy) - if this_policy: - this_policy = this_policy[0] - if 'class' in this_policy.attrib: - if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': - if 'key' in this_policy.attrib: - this_key = this_policy.attrib['key'] - else: - msg = 'policy item {0} does not have the required "key" attribute' - log.error(msg.format(this_policy.attrib)) - break - if 'valueName' in this_policy.attrib: - this_valuename = this_policy.attrib['valueName'] - if DISABLED_VALUE_XPATH(this_policy): - # set the disabled value in the registry.pol file - explicit_enable_disable_value_setting = True - disabled_value_string = _checkValueItemParent(this_policy, - admPolicy, - this_key, - this_valuename, - DISABLED_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend(disabled_value_string, - existing_data) - if DISABLED_LIST_XPATH(this_policy): - explicit_enable_disable_value_setting = True - disabled_list_strings = _checkListItem(this_policy, - admPolicy, - this_key, - DISABLED_LIST_XPATH, - None, - test_items=False) - log.debug('working with disabledList portion of {0}'.format(admPolicy)) - existing_data = _policyFileReplaceOrAppendList(disabled_list_strings, + for adm_namespace in admtemplate_data: + for adm_policy in admtemplate_data[adm_namespace]: + if str(admtemplate_data[adm_namespace][adm_policy]).lower() == 'not configured': + if adm_policy in base_policy_settings[adm_namespace]: + base_policy_settings[adm_namespace].pop(adm_policy) + else: + log.debug('adding {0} to base_policy_settings'.format(adm_policy)) + if adm_namespace not in base_policy_settings: + base_policy_settings[adm_namespace] = {} + base_policy_settings[adm_namespace][adm_policy] = admtemplate_data[adm_namespace][adm_policy] + for adm_namespace in base_policy_settings: + for admPolicy in base_policy_settings[adm_namespace]: + log.debug('working on admPolicy {0}'.format(admPolicy)) + explicit_enable_disable_value_setting = False + this_key = None + this_valuename = None + if str(base_policy_settings[adm_namespace][admPolicy]).lower() == 'disabled': + log.debug('time to disable {0}'.format(admPolicy)) + #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format('ns1', admPolicy), namespaces={'ns1': adm_namespace}) + if this_policy: + this_policy = this_policy[0] + if 'class' in this_policy.attrib: + if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': + if 'key' in this_policy.attrib: + this_key = this_policy.attrib['key'] + else: + msg = 'policy item {0} does not have the required "key" attribute' + log.error(msg.format(this_policy.attrib)) + break + if 'valueName' in this_policy.attrib: + this_valuename = this_policy.attrib['valueName'] + if DISABLED_VALUE_XPATH(this_policy): + # set the disabled value in the registry.pol file + explicit_enable_disable_value_setting = True + disabled_value_string = _checkValueItemParent(this_policy, + admPolicy, + this_key, + this_valuename, + DISABLED_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend(disabled_value_string, existing_data) - if not explicit_enable_disable_value_setting and this_valuename: - disabled_value_string = _buildKnownDataSearchString(this_key, - this_valuename, - 'REG_DWORD', - None, - check_deleted=True) - existing_data = _policyFileReplaceOrAppend(disabled_value_string, - existing_data) - if ELEMENTS_XPATH(this_policy): - log.debug('checking elements of {0}'.format(admPolicy)) - for elements_item in ELEMENTS_XPATH(this_policy): - for child_item in elements_item.getchildren(): - child_key = this_key - child_valuename = this_valuename - if 'key' in child_item.attrib: - child_key = child_item.attrib['key'] - if 'valueName' in child_item.attrib: - child_valuename = child_item.attrib['valueName'] - if etree.QName(child_item).localname == 'boolean' \ - and (TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): - # WARNING: no OOB adm files use true/falseList items - # this has not been fully vetted - temp_dict = {'trueList': TRUE_LIST_XPATH, 'falseList': FALSE_LIST_XPATH} - for this_list in temp_dict.keys(): - disabled_list_strings = _checkListItem( - child_item, - admPolicy, - child_key, - temp_dict[this_list], - None, - test_items=False) - log.debug('working with {1} portion of {0}'.format( - admPolicy, - this_list)) - existing_data = _policyFileReplaceOrAppendList( - disabled_list_strings, - existing_data) - elif etree.QName(child_item).localname == 'boolean' \ - or etree.QName(child_item).localname == 'decimal' \ - or etree.QName(child_item).localname == 'text' \ - or etree.QName(child_item).localname == 'longDecimal' \ - or etree.QName(child_item).localname == 'multiText' \ - or etree.QName(child_item).localname == 'enum': - disabled_value_string = _processValueItem(child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=True) - msg = 'I have disabled value string of {0}' - log.debug(msg.format(disabled_value_string)) - existing_data = _policyFileReplaceOrAppend( - disabled_value_string, - existing_data) - elif etree.QName(child_item).localname == 'list': - disabled_value_string = _processValueItem(child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=True) - msg = 'I have disabled value string of {0}' - log.debug(msg.format(disabled_value_string)) - existing_data = _policyFileReplaceOrAppend( - disabled_value_string, - existing_data) - else: - msg = 'policy {0} was found but it does not appear to be valid for the class {1}' - log.error(msg.format(admPolicy, registry_class)) - else: - msg = 'policy item {0} does not have the requried "class" attribute' - log.error(msg.format(this_policy.attrib)) - else: - log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) - this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy) - if this_policy: - this_policy = this_policy[0] - if 'class' in this_policy.attrib: - if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': - if 'key' in this_policy.attrib: - this_key = this_policy.attrib['key'] - else: - msg = 'policy item {0} does not have the required "key" attribute' - log.error(msg.format(this_policy.attrib)) - break - if 'valueName' in this_policy.attrib: - this_valuename = this_policy.attrib['valueName'] - - if ENABLED_VALUE_XPATH(this_policy): - explicit_enable_disable_value_setting = True - enabled_value_string = _checkValueItemParent(this_policy, - admPolicy, - this_key, - this_valuename, - ENABLED_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if ENABLED_LIST_XPATH(this_policy): - explicit_enable_disable_value_setting = True - enabled_list_strings = _checkListItem(this_policy, - admPolicy, - this_key, - ENABLED_LIST_XPATH, - None, - test_items=False) - log.debug('working with enabledList portion of {0}'.format(admPolicy)) - existing_data = _policyFileReplaceOrAppendList( - enabled_list_strings, - existing_data) - if not explicit_enable_disable_value_setting and this_valuename: - enabled_value_string = _buildKnownDataSearchString(this_key, - this_valuename, - 'REG_DWORD', - '1', - check_deleted=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if ELEMENTS_XPATH(this_policy): - for elements_item in ELEMENTS_XPATH(this_policy): - for child_item in elements_item.getchildren(): - child_key = this_key - child_valuename = this_valuename - if 'key' in child_item.attrib: - child_key = child_item.attrib['key'] - if 'valueName' in child_item.attrib: - child_valuename = child_item.attrib['valueName'] - if child_item.attrib['id'] in base_policy_settings[admPolicy]: - if etree.QName(child_item).localname == 'boolean' and ( - TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): - list_strings = [] - if base_policy_settings[admPolicy][child_item.attrib['id']]: - list_strings = _checkListItem(child_item, - admPolicy, - child_key, - TRUE_LIST_XPATH, - None, - test_items=False) - log.debug('working with trueList portion of {0}'.format(admPolicy)) - else: - list_strings = _checkListItem(child_item, - admPolicy, - child_key, - FALSE_LIST_XPATH, - None, - test_items=False) - existing_data = _policyFileReplaceOrAppendList( - list_strings, - existing_data) - if etree.QName(child_item).localname == 'boolean' and ( - TRUE_VALUE_XPATH(child_item) or FALSE_VALUE_XPATH(child_item)): - value_string = '' - if base_policy_settings[admPolicy][child_item.attrib['id']]: - value_string = _checkValueItemParent(child_item, - admPolicy, - child_key, - child_valuename, - TRUE_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - else: - value_string = _checkValueItemParent(child_item, - admPolicy, - child_key, - child_valuename, - FALSE_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - value_string, - existing_data) + if DISABLED_LIST_XPATH(this_policy): + explicit_enable_disable_value_setting = True + disabled_list_strings = _checkListItem(this_policy, + admPolicy, + this_key, + DISABLED_LIST_XPATH, + None, + test_items=False) + log.debug('working with disabledList portion of {0}'.format(admPolicy)) + existing_data = _policyFileReplaceOrAppendList(disabled_list_strings, + existing_data) + if not explicit_enable_disable_value_setting and this_valuename: + disabled_value_string = _buildKnownDataSearchString(this_key, + this_valuename, + 'REG_DWORD', + None, + check_deleted=True) + existing_data = _policyFileReplaceOrAppend(disabled_value_string, + existing_data) + if ELEMENTS_XPATH(this_policy): + log.debug('checking elements of {0}'.format(admPolicy)) + for elements_item in ELEMENTS_XPATH(this_policy): + for child_item in elements_item.getchildren(): + child_key = this_key + child_valuename = this_valuename + if 'key' in child_item.attrib: + child_key = child_item.attrib['key'] + if 'valueName' in child_item.attrib: + child_valuename = child_item.attrib['valueName'] if etree.QName(child_item).localname == 'boolean' \ + and (TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): + # WARNING: no OOB adm files use true/falseList items + # this has not been fully vetted + temp_dict = {'trueList': TRUE_LIST_XPATH, 'falseList': FALSE_LIST_XPATH} + for this_list in temp_dict: + disabled_list_strings = _checkListItem( + child_item, + admPolicy, + child_key, + temp_dict[this_list], + None, + test_items=False) + log.debug('working with {1} portion of {0}'.format( + admPolicy, + this_list)) + existing_data = _policyFileReplaceOrAppendList( + disabled_list_strings, + existing_data) + elif etree.QName(child_item).localname == 'boolean' \ or etree.QName(child_item).localname == 'decimal' \ or etree.QName(child_item).localname == 'text' \ or etree.QName(child_item).localname == 'longDecimal' \ - or etree.QName(child_item).localname == 'multiText': - enabled_value_string = _processValueItem( - child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=False, - this_element_value=base_policy_settings[admPolicy][child_item.attrib['id']]) - msg = 'I have enabled value string of {0}' - log.debug(msg.format([enabled_value_string])) + or etree.QName(child_item).localname == 'multiText' \ + or etree.QName(child_item).localname == 'enum': + disabled_value_string = _processValueItem(child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=True) + msg = 'I have disabled value string of {0}' + log.debug(msg.format(disabled_value_string)) existing_data = _policyFileReplaceOrAppend( - enabled_value_string, + disabled_value_string, existing_data) - elif etree.QName(child_item).localname == 'enum': - for enum_item in child_item.getchildren(): - if base_policy_settings[admPolicy][child_item.attrib['id']] == \ - _getAdmlDisplayName(adml_policy_resources, - enum_item.attrib['displayName'] - ).strip(): - enabled_value_string = _checkValueItemParent( - enum_item, - child_item.attrib['id'], - child_key, - child_valuename, - VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if VALUE_LIST_XPATH(enum_item): - enabled_list_strings = _checkListItem(enum_item, - admPolicy, - child_key, - VALUE_LIST_XPATH, - None, - test_items=False) - msg = 'working with valueList portion of {0}' - log.debug(msg.format(child_item.attrib['id'])) - existing_data = _policyFileReplaceOrAppendList( - enabled_list_strings, - existing_data) - break elif etree.QName(child_item).localname == 'list': - enabled_value_string = _processValueItem( - child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=False, - this_element_value=base_policy_settings[admPolicy][child_item.attrib['id']]) - msg = 'I have enabled value string of {0}' - log.debug(msg.format([enabled_value_string])) + disabled_value_string = _processValueItem(child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=True) + msg = 'I have disabled value string of {0}' + log.debug(msg.format(disabled_value_string)) existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data, - append_only=True) + disabled_value_string, + existing_data) + else: + msg = 'policy {0} was found but it does not appear to be valid for the class {1}' + log.error(msg.format(admPolicy, registry_class)) + else: + msg = 'policy item {0} does not have the requried "class" attribute' + log.error(msg.format(this_policy.attrib)) + else: + log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) + #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) + log.debug('found this_policy == {0}'.format(this_policy)) + if this_policy: + this_policy = this_policy[0] + if 'class' in this_policy.attrib: + if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': + if 'key' in this_policy.attrib: + this_key = this_policy.attrib['key'] + else: + msg = 'policy item {0} does not have the required "key" attribute' + log.error(msg.format(this_policy.attrib)) + break + if 'valueName' in this_policy.attrib: + this_valuename = this_policy.attrib['valueName'] + + if ENABLED_VALUE_XPATH(this_policy): + explicit_enable_disable_value_setting = True + enabled_value_string = _checkValueItemParent(this_policy, + admPolicy, + this_key, + this_valuename, + ENABLED_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if ENABLED_LIST_XPATH(this_policy): + explicit_enable_disable_value_setting = True + enabled_list_strings = _checkListItem(this_policy, + admPolicy, + this_key, + ENABLED_LIST_XPATH, + None, + test_items=False) + log.debug('working with enabledList portion of {0}'.format(admPolicy)) + existing_data = _policyFileReplaceOrAppendList( + enabled_list_strings, + existing_data) + if not explicit_enable_disable_value_setting and this_valuename: + enabled_value_string = _buildKnownDataSearchString(this_key, + this_valuename, + 'REG_DWORD', + '1', + check_deleted=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if ELEMENTS_XPATH(this_policy): + for elements_item in ELEMENTS_XPATH(this_policy): + for child_item in elements_item.getchildren(): + child_key = this_key + child_valuename = this_valuename + if 'key' in child_item.attrib: + child_key = child_item.attrib['key'] + if 'valueName' in child_item.attrib: + child_valuename = child_item.attrib['valueName'] + if child_item.attrib['id'] in base_policy_settings[adm_namespace][admPolicy]: + if etree.QName(child_item).localname == 'boolean' and ( + TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): + list_strings = [] + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]: + list_strings = _checkListItem(child_item, + admPolicy, + child_key, + TRUE_LIST_XPATH, + None, + test_items=False) + log.debug('working with trueList portion of {0}'.format(admPolicy)) + else: + list_strings = _checkListItem(child_item, + admPolicy, + child_key, + FALSE_LIST_XPATH, + None, + test_items=False) + existing_data = _policyFileReplaceOrAppendList( + list_strings, + existing_data) + elif etree.QName(child_item).localname == 'boolean' and ( + TRUE_VALUE_XPATH(child_item) or FALSE_VALUE_XPATH(child_item)): + value_string = '' + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]: + value_string = _checkValueItemParent(child_item, + admPolicy, + child_key, + child_valuename, + TRUE_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + else: + value_string = _checkValueItemParent(child_item, + admPolicy, + child_key, + child_valuename, + FALSE_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + value_string, + existing_data) + elif etree.QName(child_item).localname == 'boolean' \ + or etree.QName(child_item).localname == 'decimal' \ + or etree.QName(child_item).localname == 'text' \ + or etree.QName(child_item).localname == 'longDecimal' \ + or etree.QName(child_item).localname == 'multiText': + enabled_value_string = _processValueItem( + child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=False, + this_element_value=base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]) + msg = 'I have enabled value string of {0}' + log.debug(msg.format([enabled_value_string])) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + elif etree.QName(child_item).localname == 'enum': + for enum_item in child_item.getchildren(): + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']] == \ + _getAdmlDisplayName(adml_policy_resources, + enum_item.attrib['displayName'] + ).strip(): + enabled_value_string = _checkValueItemParent( + enum_item, + child_item.attrib['id'], + child_key, + child_valuename, + VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if VALUE_LIST_XPATH(enum_item): + enabled_list_strings = _checkListItem(enum_item, + admPolicy, + child_key, + VALUE_LIST_XPATH, + None, + test_items=False) + msg = 'working with valueList portion of {0}' + log.debug(msg.format(child_item.attrib['id'])) + existing_data = _policyFileReplaceOrAppendList( + enabled_list_strings, + existing_data) + break + elif etree.QName(child_item).localname == 'list': + enabled_value_string = _processValueItem( + child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=False, + this_element_value=base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]) + msg = 'I have enabled value string of {0}' + log.debug(msg.format([enabled_value_string])) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data, + append_only=True) _write_regpol_data(existing_data, policy_data.admx_registry_classes[registry_class]['policy_path'], policy_data.gpt_ini_path, @@ -4558,6 +4609,7 @@ def _lookup_admin_template(policy_name, if admx_policy_definitions is None or adml_policy_resources is None: admx_policy_definitions, adml_policy_resources = _processPolicyDefinitions( display_language=adml_language) + admx_search_results = [] admx_search_results = ADMX_SEARCH_XPATH(admx_policy_definitions, policy_name=policy_name, registry_class=policy_class) @@ -4603,24 +4655,31 @@ def _lookup_admin_template(policy_name, for adml_search_result in adml_search_results: if not getattr(adml_search_result, 'text', '').strip() == policy_name: adml_to_remove.append(adml_search_result) - if hierarchy: - display_name_searchval = '$({0}.{1})'.format( - adml_search_result.tag.split('}')[1], - adml_search_result.attrib['id']) - policy_search_string = '//{0}:policy[@*[local-name() = "displayName"] = "{1}" and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = "{2}") ]'.format( - adml_search_result.prefix, - display_name_searchval, - policy_class) - # this should only be 1 result - admx_search_results = admx_policy_definitions.xpath(policy_search_string, namespaces=adml_search_result.nsmap) - for search_result in admx_search_results: - this_hierarchy = _build_parent_list(search_result, - admx_policy_definitions, - True, - adml_policy_resources) - this_hierarchy.reverse() - if hierarchy != this_hierarchy: - adml_to_remove.append(adml_search_result) + else: + if hierarchy: + display_name_searchval = '$({0}.{1})'.format( + adml_search_result.tag.split('}')[1], + adml_search_result.attrib['id']) + #policy_search_string = '//{0}:policy[@*[local-name() = "displayName"] = "{1}" and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = "{2}") ]'.format( + policy_search_string = '//{0}:policy[@displayName = "{1}" and (@class = "Both" or @class = "{2}") ]'.format( + adml_search_result.prefix, + display_name_searchval, + policy_class) + admx_results = [] + admx_search_results = admx_policy_definitions.xpath(policy_search_string, namespaces=adml_search_result.nsmap) + for search_result in admx_search_results: + log.debug('policy_name == {0}'.format(policy_name)) + this_hierarchy = _build_parent_list(search_result, + admx_policy_definitions, + True, + adml_policy_resources) + this_hierarchy.reverse() + if hierarchy != this_hierarchy: + adml_to_remove.append(adml_search_result) + else: + admx_results.append(search_result) + if len(admx_results) == 1: + admx_search_results = admx_results for adml in adml_to_remove: if adml in adml_search_results: adml_search_results.remove(adml) @@ -4634,10 +4693,11 @@ def _lookup_admin_template(policy_name, adml_search_result.tag.split('}')[1], adml_search_result.attrib['id']) log.debug('searching for displayName == {0}'.format(display_name_searchval)) - admx_search_results = ADMX_DISPLAYNAME_SEARCH_XPATH( - admx_policy_definitions, - display_name=display_name_searchval, - registry_class=policy_class) + if not admx_search_results: + admx_search_results = ADMX_DISPLAYNAME_SEARCH_XPATH( + admx_policy_definitions, + display_name=display_name_searchval, + registry_class=policy_class) if admx_search_results: if len(admx_search_results) == 1 or hierarchy and not multiple_adml_entries: found = False @@ -4649,6 +4709,7 @@ def _lookup_admin_template(policy_name, True, adml_policy_resources) this_hierarchy.reverse() + log.debug('testing {0} == {1}'.format(hierarchy, this_hierarchy)) if hierarchy == this_hierarchy: found = True else: @@ -5107,6 +5168,7 @@ def set_(computer_policy=None, user_policy=None, if policies[p_class]: for policy_name in policies[p_class].keys(): _pol = None + policy_namespace = None policy_key_name = policy_name if policy_name in _policydata.policies[p_class]['policies']: _pol = _policydata.policies[p_class]['policies'][policy_name] @@ -5156,16 +5218,19 @@ def set_(computer_policy=None, user_policy=None, adml_policy_resources=admlPolicyResources) if success: policy_name = the_policy.attrib['name'] - _admTemplateData[policy_name] = _value + policy_namespace = the_policy.nsmap[the_policy.prefix] + if policy_namespace not in _admTemplateData: + _admTemplateData[policy_namespace] = {} + _admTemplateData[policy_namespace][policy_name] = _value else: raise SaltInvocationError(msg) - if policy_name in _admTemplateData and the_policy is not None: - log.debug('setting == {0}'.format(_admTemplateData[policy_name]).lower()) - log.debug('{0}'.format(str(_admTemplateData[policy_name]).lower())) - if str(_admTemplateData[policy_name]).lower() != 'disabled' \ - and str(_admTemplateData[policy_name]).lower() != 'not configured': + if policy_namespace and policy_name in _admTemplateData[policy_namespace] and the_policy is not None: + log.debug('setting == {0}'.format(_admTemplateData[policy_namespace][policy_name]).lower()) + log.debug('{0}'.format(str(_admTemplateData[policy_namespace][policy_name]).lower())) + if str(_admTemplateData[policy_namespace][policy_name]).lower() != 'disabled' \ + and str(_admTemplateData[policy_namespace][policy_name]).lower() != 'not configured': if ELEMENTS_XPATH(the_policy): - if isinstance(_admTemplateData[policy_name], dict): + if isinstance(_admTemplateData[policy_namespace][policy_name], dict): for elements_item in ELEMENTS_XPATH(the_policy): for child_item in elements_item.getchildren(): # check each element @@ -5176,9 +5241,9 @@ def set_(computer_policy=None, user_policy=None, True, admlPolicyResources) log.debug('id attribute == "{0}" this_element_name == "{1}"'.format(child_item.attrib['id'], this_element_name)) - if this_element_name in _admTemplateData[policy_name]: + if this_element_name in _admTemplateData[policy_namespace][policy_name]: temp_element_name = this_element_name - elif child_item.attrib['id'] in _admTemplateData[policy_name]: + elif child_item.attrib['id'] in _admTemplateData[policy_namespace][policy_name]: temp_element_name = child_item.attrib['id'] else: msg = ('Element "{0}" must be included' @@ -5186,12 +5251,12 @@ def set_(computer_policy=None, user_policy=None, raise SaltInvocationError(msg.format(this_element_name, policy_name)) if 'required' in child_item.attrib \ and child_item.attrib['required'].lower() == 'true': - if not _admTemplateData[policy_name][temp_element_name]: + if not _admTemplateData[policy_namespace][policy_name][temp_element_name]: msg = 'Element "{0}" requires a value to be specified' raise SaltInvocationError(msg.format(temp_element_name)) if etree.QName(child_item).localname == 'boolean': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], bool): msg = 'Element {0} requires a boolean True or False' raise SaltInvocationError(msg.format(temp_element_name)) @@ -5203,9 +5268,9 @@ def set_(computer_policy=None, user_policy=None, min_val = int(child_item.attrib['minValue']) if 'maxValue' in child_item.attrib: max_val = int(child_item.attrib['maxValue']) - if int(_admTemplateData[policy_name][temp_element_name]) \ + if int(_admTemplateData[policy_namespace][policy_name][temp_element_name]) \ < min_val or \ - int(_admTemplateData[policy_name][temp_element_name]) \ + int(_admTemplateData[policy_namespace][policy_name][temp_element_name]) \ > max_val: msg = 'Element "{0}" value must be between {1} and {2}' raise SaltInvocationError(msg.format(temp_element_name, @@ -5215,7 +5280,7 @@ def set_(computer_policy=None, user_policy=None, # make sure the value is in the enumeration found = False for enum_item in child_item.getchildren(): - if _admTemplateData[policy_name][temp_element_name] == \ + if _admTemplateData[policy_namespace][policy_name][temp_element_name] == \ _getAdmlDisplayName( admlPolicyResources, enum_item.attrib['displayName']).strip(): @@ -5229,33 +5294,33 @@ def set_(computer_policy=None, user_policy=None, and child_item.attrib['explicitValue'].lower() == \ 'true': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], dict): msg = ('Each list item of element "{0}" ' 'requires a dict value') msg = msg.format(temp_element_name) raise SaltInvocationError(msg) elif not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], list): msg = 'Element "{0}" requires a list value' msg = msg.format(temp_element_name) raise SaltInvocationError(msg) elif etree.QName(child_item).localname == 'multiText': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], list): msg = 'Element "{0}" requires a list value' msg = msg.format(temp_element_name) raise SaltInvocationError(msg) - _admTemplateData[policy_name][child_item.attrib['id']] = \ - _admTemplateData[policy_name].pop(temp_element_name) + _admTemplateData[policy_namespace][policy_name][child_item.attrib['id']] = \ + _admTemplateData[policy_namespace][policy_name].pop(temp_element_name) else: msg = 'The policy "{0}" has elements which must be configured' msg = msg.format(policy_name) raise SaltInvocationError(msg) else: - if str(_admTemplateData[policy_name]).lower() != 'enabled': + if str(_admTemplateData[policy_namespace][policy_name]).lower() != 'enabled': msg = ('The policy {0} must either be "Enabled", ' '"Disabled", or "Not Configured"') msg = msg.format(policy_name) @@ -5343,4 +5408,4 @@ def set_(computer_policy=None, user_policy=None, return True else: msg = 'You have to specify something!' - raise SaltInvocationError(msg) + raise SaltInvocationError(msg) \ No newline at end of file From 2da1cdd1099bf12bb6309277d7bb970bb5076e6c Mon Sep 17 00:00:00 2001 From: lomeroe Date: Tue, 22 Aug 2017 09:53:07 -0500 Subject: [PATCH 428/508] lint fix --- salt/modules/win_lgpo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index ed556c4fc5..5cdd35c8fb 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -5408,4 +5408,4 @@ def set_(computer_policy=None, user_policy=None, return True else: msg = 'You have to specify something!' - raise SaltInvocationError(msg) \ No newline at end of file + raise SaltInvocationError(msg) From acc3d7ac82b4cb52e249172c1f7e34b44627271d Mon Sep 17 00:00:00 2001 From: lomeroe Date: Wed, 23 Aug 2017 11:09:12 -0500 Subject: [PATCH 429/508] correct fopen calls from salt.utils for 2016.11's utils function --- salt/modules/win_lgpo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 5cdd35c8fb..efd76192bd 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -4033,14 +4033,14 @@ def _write_regpol_data(data_to_write, reg_pol_header = u'\u5250\u6765\x01\x00' if not os.path.exists(policy_file_path): ret = __salt__['file.makedirs'](policy_file_path) - with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: + with salt.utils.fopen(policy_file_path, 'wb') as pol_file: if not data_to_write.startswith(reg_pol_header): pol_file.write(reg_pol_header.encode('utf-16-le')) pol_file.write(data_to_write.encode('utf-16-le')) try: gpt_ini_data = '' if os.path.exists(gpt_ini_path): - with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + with salt.utils.fopen(gpt_ini_path, 'rb') as gpt_file: gpt_ini_data = gpt_file.read() if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): gpt_ini_data = '[General]\r\n' + gpt_ini_data @@ -4095,7 +4095,7 @@ def _write_regpol_data(data_to_write, int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), gpt_ini_data[general_location.end():]) if gpt_ini_data: - with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: + with salt.utils.fopen(gpt_ini_path, 'wb') as gpt_file: gpt_file.write(gpt_ini_data) except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( From af181b3257e2ca553564df96ce764b699ecd6559 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Wed, 23 Aug 2017 11:11:08 -0500 Subject: [PATCH 430/508] correct fopen calls from salt.utils for 2017.7 --- salt/modules/win_lgpo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 766f9162be..5edb672c90 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -4026,14 +4026,14 @@ def _write_regpol_data(data_to_write, reg_pol_header = u'\u5250\u6765\x01\x00' if not os.path.exists(policy_file_path): ret = __salt__['file.makedirs'](policy_file_path) - with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: + with salt.utils.fopen(policy_file_path, 'wb') as pol_file: if not data_to_write.startswith(reg_pol_header): pol_file.write(reg_pol_header.encode('utf-16-le')) pol_file.write(data_to_write.encode('utf-16-le')) try: gpt_ini_data = '' if os.path.exists(gpt_ini_path): - with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + with salt.utils.fopen(gpt_ini_path, 'rb') as gpt_file: gpt_ini_data = gpt_file.read() if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): gpt_ini_data = '[General]\r\n' + gpt_ini_data @@ -4088,7 +4088,7 @@ def _write_regpol_data(data_to_write, int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), gpt_ini_data[general_location.end():]) if gpt_ini_data: - with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: + with salt.utils.fopen(gpt_ini_path, 'wb') as gpt_file: gpt_file.write(gpt_ini_data) except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( From d5b2a0be6821d39ea2945665bd1cfce4be2b10a8 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 23 Aug 2017 11:57:00 -0500 Subject: [PATCH 431/508] Resolve image ID during container comparison This fixes an issue where inspecting the container returns an image ID instead of an image name, resulting in a spurious report of a changed image. By resolving the image down to its ID for both the existing and new containers, we ensure we're comparing ID to ID. --- salt/modules/dockermod.py | 10 ++++++++++ tests/unit/modules/test_dockermod.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index a6cd82ce8e..242067a4b0 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -902,6 +902,11 @@ def compare_container(first, second, ignore=None): if item in ('OomKillDisable',): if bool(val1) != bool(val2): ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + elif item == 'Image': + image1 = inspect_image(val1)['Id'] + image2 = inspect_image(val2)['Id'] + if image1 != image2: + ret.setdefault(conf_dict, {})[item] = {'old': image1, 'new': image2} else: if item == 'Links': val1 = _scrub_links(val1, first) @@ -920,6 +925,11 @@ def compare_container(first, second, ignore=None): if item in ('OomKillDisable',): if bool(val1) != bool(val2): ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + elif item == 'Image': + image1 = inspect_image(val1)['Id'] + image2 = inspect_image(val2)['Id'] + if image1 != image2: + ret.setdefault(conf_dict, {})[item] = {'old': image1, 'new': image2} else: if item == 'Links': val1 = _scrub_links(val1, first) diff --git a/tests/unit/modules/test_dockermod.py b/tests/unit/modules/test_dockermod.py index 8f22a0605c..24a6d3a3df 100644 --- a/tests/unit/modules/test_dockermod.py +++ b/tests/unit/modules/test_dockermod.py @@ -697,3 +697,29 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): result = docker_mod.images() self.assertEqual(result, {'sha256:abcdefg': {'RepoTags': ['image:latest']}}) + + def test_compare_container_image_id_resolution(self): + ''' + Compare + ''' + def _inspect_container_effect(id_): + return { + 'container1': {'Config': {'Image': 'realimage:latest'}, + 'HostConfig':{}}, + 'container2': {'Config': {'Image': 'image_id'}, + 'HostConfig':{}}, + }[id_] + + def _inspect_image_effect(id_): + return { + 'realimage:latest': {'Id': 'image_id'}, + 'image_id': {'Id': 'image_id'}, + }[id_] + + inspect_container_mock = MagicMock(side_effect=_inspect_container_effect) + inspect_image_mock = MagicMock(side_effect=_inspect_image_effect) + + with patch.object(docker_mod, 'inspect_container', inspect_container_mock): + with patch.object(docker_mod, 'inspect_image', inspect_image_mock): + ret = docker_mod.compare_container('container1', 'container2') + self.assertEqual(ret, {}) From 1dcf167bb7befa0e1ebf8c7256437318961f0351 Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Thu, 17 Aug 2017 12:02:07 -0400 Subject: [PATCH 432/508] Update state.py create `_clean_tag()` method to safely encode tag names for file creation use `_clean_tag()` method for both create and read of state cache file --- salt/state.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 43d1df17a7..950cea70ed 100644 --- a/salt/state.py +++ b/salt/state.py @@ -25,6 +25,7 @@ import traceback import re import time import random +from urllib import quote # Import salt libs import salt.utils @@ -145,6 +146,11 @@ def _gen_tag(low): ''' return '{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(low) +def _clean_tag(tag): + ''' + urllib safe quote the tag value to avoid invalid chars in the filename + ''' + return quote(tag, safe='') def _l_tag(name, id_): low = {'name': 'listen_{0}'.format(name), @@ -1695,7 +1701,7 @@ class State(object): trb) } troot = os.path.join(self.opts['cachedir'], self.jid) - tfile = os.path.join(troot, tag) + tfile = os.path.join(troot, _clean_tag(tag)) if not os.path.isdir(troot): try: os.makedirs(troot) @@ -2047,7 +2053,7 @@ class State(object): proc = running[tag].get('proc') if proc: if not proc.is_alive(): - ret_cache = os.path.join(self.opts['cachedir'], self.jid, tag) + ret_cache = os.path.join(self.opts['cachedir'], self.jid, _clean_tag(tag)) if not os.path.isfile(ret_cache): ret = {'result': False, 'comment': 'Parallel process failed to return', From fb80e174000f306fb2fe26f6efe9726edf54bc2c Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Fri, 18 Aug 2017 13:05:56 -0400 Subject: [PATCH 433/508] state.py: fix import and utf8 encode before quote --- salt/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 950cea70ed..88a7ec1532 100644 --- a/salt/state.py +++ b/salt/state.py @@ -25,7 +25,7 @@ import traceback import re import time import random -from urllib import quote +from urllib # Import salt libs import salt.utils @@ -150,7 +150,7 @@ def _clean_tag(tag): ''' urllib safe quote the tag value to avoid invalid chars in the filename ''' - return quote(tag, safe='') + return urllib.quote(tag.encode('utf8'), safe='') def _l_tag(name, id_): low = {'name': 'listen_{0}'.format(name), From 446457d017c1791041df9c71781903b3cf2fd99b Mon Sep 17 00:00:00 2001 From: garethgreenaway Date: Mon, 21 Aug 2017 10:46:41 -0700 Subject: [PATCH 434/508] Swapping `from` for `import` --- salt/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 88a7ec1532..391dc2b5a9 100644 --- a/salt/state.py +++ b/salt/state.py @@ -25,7 +25,7 @@ import traceback import re import time import random -from urllib +import urllib # Import salt libs import salt.utils From b8ead879edea078aae91768651bd12d419ca15e6 Mon Sep 17 00:00:00 2001 From: garethgreenaway Date: Mon, 21 Aug 2017 12:44:55 -0700 Subject: [PATCH 435/508] Fixing lint issues --- salt/state.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/state.py b/salt/state.py index 391dc2b5a9..a0e22e5e63 100644 --- a/salt/state.py +++ b/salt/state.py @@ -146,12 +146,14 @@ def _gen_tag(low): ''' return '{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(low) + def _clean_tag(tag): ''' urllib safe quote the tag value to avoid invalid chars in the filename ''' return urllib.quote(tag.encode('utf8'), safe='') + def _l_tag(name, id_): low = {'name': 'listen_{0}'.format(name), '__id__': 'listen_{0}'.format(id_), From 4957268b371e5fab1d9c8a05a99ce812c549cc42 Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Wed, 23 Aug 2017 14:22:46 -0400 Subject: [PATCH 436/508] update state.py to use safe_filename_leaf instead of urllib.quote generate state cache filename by using safe_filename_leaf salt util method --- salt/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index a0e22e5e63..912f06a9ed 100644 --- a/salt/state.py +++ b/salt/state.py @@ -149,9 +149,9 @@ def _gen_tag(low): def _clean_tag(tag): ''' - urllib safe quote the tag value to avoid invalid chars in the filename + Make tag name safe for filenames ''' - return urllib.quote(tag.encode('utf8'), safe='') + return salt.utils.safe_filename_leaf(tag) def _l_tag(name, id_): From 42064883ea8067161a6b4003fabab11a341d0de3 Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Wed, 23 Aug 2017 14:28:16 -0400 Subject: [PATCH 437/508] state.py remove unused urllib import --- salt/state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 912f06a9ed..ae2ba0c6db 100644 --- a/salt/state.py +++ b/salt/state.py @@ -25,7 +25,6 @@ import traceback import re import time import random -import urllib # Import salt libs import salt.utils From 2722e9521daa5285815ec75cbfd6eb45bf53d2a7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 16 Aug 2017 16:17:24 -0600 Subject: [PATCH 438/508] Use os.path.join to create paths --- tests/unit/test_test_module_names.py | 50 ++++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/unit/test_test_module_names.py b/tests/unit/test_test_module_names.py index a7b2bf94ad..485f737685 100644 --- a/tests/unit/test_test_module_names.py +++ b/tests/unit/test_test_module_names.py @@ -13,33 +13,33 @@ from tests.support.unit import TestCase from tests.support.paths import CODE_DIR EXCLUDED_DIRS = [ - 'tests/pkg', - 'tests/perf', - 'tests/support', - 'tests/unit/utils/cache_mods', - 'tests/unit/modules/inspectlib', - 'tests/unit/modules/zypp/', - 'tests/unit/templates/files', - 'tests/integration/files/', - 'tests/integration/cloud/helpers', + os.path.join('tests', 'pkg'), + os.path.join('tests', 'perf'), + os.path.join('tests', 'support'), + os.path.join('tests', 'unit', 'utils', 'cache_mods'), + os.path.join('tests', 'unit', 'modules', 'inspectlib'), + os.path.join('tests', 'unit', 'modules', 'zypp'), + os.path.join('tests', 'unit', 'templates', 'files'), + os.path.join('tests', 'integration', 'files'), + os.path.join('tests', 'integration', 'cloud', 'helpers'), ] EXCLUDED_FILES = [ - 'tests/eventlisten.py', - 'tests/buildpackage.py', - 'tests/saltsh.py', - 'tests/minionswarm.py', - 'tests/wheeltest.py', - 'tests/runtests.py', - 'tests/jenkins.py', - 'tests/salt-tcpdump.py', - 'tests/conftest.py', - 'tests/packdump.py', - 'tests/consist.py', - 'tests/modparser.py', - 'tests/committer_parser.py', - 'tests/zypp_plugin.py', - 'tests/unit/transport/mixins.py', - 'tests/integration/utils/testprogram.py', + os.path.join('tests', 'eventlisten.py'), + os.path.join('tests', 'buildpackage.py'), + os.path.join('tests', 'saltsh.py'), + os.path.join('tests', 'minionswarm.py'), + os.path.join('tests', 'wheeltest.py'), + os.path.join('tests', 'runtests.py'), + os.path.join('tests', 'jenkins.py'), + os.path.join('tests', 'salt-tcpdump.py'), + os.path.join('tests', 'conftest.py'), + os.path.join('tests', 'packdump.py'), + os.path.join('tests', 'consist.py'), + os.path.join('tests', 'modparser.py'), + os.path.join('tests', 'committer_parser.py'), + os.path.join('tests', 'zypp_plugin.py'), + os.path.join('tests', 'unit', 'transport', 'mixins.py'), + os.path.join('tests', 'integration', 'utils', 'testprogram.py'), ] From d8612ae0063af82f24dba94aa1e8d9139810694d Mon Sep 17 00:00:00 2001 From: Alessandro -oggei- Ogier Date: Tue, 22 Aug 2017 11:48:08 +0200 Subject: [PATCH 439/508] fix debootstrap and enhance packages selection/deletion via cmdline --- salt/modules/genesis.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/salt/modules/genesis.py b/salt/modules/genesis.py index 5680f00829..ba0a0a399a 100644 --- a/salt/modules/genesis.py +++ b/salt/modules/genesis.py @@ -24,6 +24,8 @@ import salt.utils.kickstart import salt.syspaths from salt.exceptions import SaltInvocationError +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -178,9 +180,13 @@ def bootstrap( if pkgs is None: pkgs = [] + elif isinstance(pkgs, six.string_types): + pkgs = pkgs.split(',') if exclude_pkgs is None: exclude_pkgs = [] + elif isinstance(exclude_pkgs, six.string_types): + exclude_pkgs = exclude_pkgs.split(',') if platform in ('rpm', 'yum'): _bootstrap_yum( @@ -393,15 +399,22 @@ def _bootstrap_deb( if repo_url is None: repo_url = 'http://ftp.debian.org/debian/' + if not salt.utils.which('debootstrap'): + log.error('Required tool debootstrap is not installed.') + return False + deb_args = [ 'debootstrap', '--foreign', '--arch', - _cmd_quote(arch), - '--include', - ] + pkgs + [ - '--exclude', - ] + exclude_pkgs + [ + _cmd_quote(arch)] + + if pkgs: + deb_args += ['--include'] + pkgs + if exclude_pkgs: + deb_args += ['--exclude'] + exclude_pkgs + + deb_args += [ _cmd_quote(flavor), _cmd_quote(root), _cmd_quote(repo_url), From 216ced69e56bca31db2f24fe703387e5da9664f9 Mon Sep 17 00:00:00 2001 From: Alessandro -oggei- Ogier Date: Thu, 24 Aug 2017 12:00:36 +0200 Subject: [PATCH 440/508] allow comma-separated pkgs lists, quote args, test deb behaviour --- salt/modules/genesis.py | 21 +++++++---- tests/unit/modules/genesis_test.py | 57 ++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/salt/modules/genesis.py b/salt/modules/genesis.py index ba0a0a399a..eceaba5bb5 100644 --- a/salt/modules/genesis.py +++ b/salt/modules/genesis.py @@ -180,13 +180,9 @@ def bootstrap( if pkgs is None: pkgs = [] - elif isinstance(pkgs, six.string_types): - pkgs = pkgs.split(',') if exclude_pkgs is None: exclude_pkgs = [] - elif isinstance(exclude_pkgs, six.string_types): - exclude_pkgs = exclude_pkgs.split(',') if platform in ('rpm', 'yum'): _bootstrap_yum( @@ -331,6 +327,8 @@ def _bootstrap_yum( ''' if pkgs is None: pkgs = [] + elif isinstance(pkgs, six.string_types): + pkgs = pkgs.split(',') default_pkgs = ('yum', 'centos-release', 'iputils') for pkg in default_pkgs: @@ -339,6 +337,8 @@ def _bootstrap_yum( if exclude_pkgs is None: exclude_pkgs = [] + elif isinstance(exclude_pkgs, six.string_types): + exclude_pkgs = exclude_pkgs.split(',') for pkg in exclude_pkgs: pkgs.remove(pkg) @@ -403,6 +403,11 @@ def _bootstrap_deb( log.error('Required tool debootstrap is not installed.') return False + if isinstance(pkgs, (list, tuple)): + pkgs = ','.join(pkgs) + if isinstance(exclude_pkgs, (list, tuple)): + exclude_pkgs = ','.join(exclude_pkgs) + deb_args = [ 'debootstrap', '--foreign', @@ -410,9 +415,9 @@ def _bootstrap_deb( _cmd_quote(arch)] if pkgs: - deb_args += ['--include'] + pkgs + deb_args += ['--include', _cmd_quote(pkgs)] if exclude_pkgs: - deb_args += ['--exclude'] + exclude_pkgs + deb_args += ['--exclude', _cmd_quote(exclude_pkgs)] deb_args += [ _cmd_quote(flavor), @@ -482,6 +487,8 @@ def _bootstrap_pacman( if pkgs is None: pkgs = [] + elif isinstance(pkgs, six.string_types): + pkgs = pkgs.split(',') default_pkgs = ('pacman', 'linux', 'systemd-sysvcompat', 'grub') for pkg in default_pkgs: @@ -490,6 +497,8 @@ def _bootstrap_pacman( if exclude_pkgs is None: exclude_pkgs = [] + elif isinstance(exclude_pkgs, six.string_types): + exclude_pkgs = exclude_pkgs.split(',') for pkg in exclude_pkgs: pkgs.remove(pkg) diff --git a/tests/unit/modules/genesis_test.py b/tests/unit/modules/genesis_test.py index cbee47f09d..bab16d3f1e 100644 --- a/tests/unit/modules/genesis_test.py +++ b/tests/unit/modules/genesis_test.py @@ -49,12 +49,63 @@ class GenesisTestCase(TestCase): with patch.dict(genesis.__salt__, {'disk.blkid': MagicMock(return_value={})}): self.assertEqual(genesis.bootstrap('rpm', 'root', 'dir'), None) - with patch.object(genesis, '_bootstrap_deb', return_value='A'): + common_parms = {'platform': 'deb', + 'root': 'root', + 'img_format': 'dir', + 'arch': 'amd64', + 'flavor': 'stable', + 'static_qemu': 'qemu'} + + param_sets = [ + + {'params': {}, + 'commandlines': [ + ['debootstrap', '--foreign', '--arch', 'amd64', + 'stable', 'root', 'http://ftp.debian.org/debian/'], + ]}, + + {'params': {'pkgs': 'vim'}, + 'commandlines': [ + ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim', + 'stable', 'root', 'http://ftp.debian.org/debian/'], + ]}, + + {'params': {'pkgs': 'vim,emacs'}, + 'commandlines': [ + ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'], + ]}, + + {'params': {'pkgs': ['vim', 'emacs']}, + 'commandlines': [ + ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'], + ]}, + + {'params': {'pkgs': ['vim', 'emacs'], 'exclude_pkgs': ['vim', 'foo']}, + 'commandlines': [ + ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', '--exclude', 'vim,foo', + 'stable', 'root', 'http://ftp.debian.org/debian/'], + ]}, + + ] + + for param_set in param_sets: + with patch.dict(genesis.__salt__, {'mount.umount': MagicMock(), 'file.rmdir': MagicMock(), - 'file.directory_exists': MagicMock()}): + 'file.directory_exists': MagicMock(), + 'cmd.run': MagicMock()}): with patch.dict(genesis.__salt__, {'disk.blkid': MagicMock(return_value={})}): - self.assertEqual(genesis.bootstrap('deb', 'root', 'dir'), None) + param_set['params'].update(common_parms) + self.assertEqual(genesis.bootstrap(**param_set['params']), + None) + for commandline in param_set['commandlines']: + genesis.__salt__['cmd.run'].assert_any_call(commandline, python_shell=False) with patch.object(genesis, '_bootstrap_pacman', return_value='A') as pacman_patch: with patch.dict(genesis.__salt__, {'mount.umount': MagicMock(), From db11e1985b2e3f53ce258e2a266455584d4b5e12 Mon Sep 17 00:00:00 2001 From: SuperPommeDeTerre Date: Thu, 17 Aug 2017 11:56:23 +0200 Subject: [PATCH 441/508] Fix for #26995 --- salt/modules/artifactory.py | 77 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/salt/modules/artifactory.py b/salt/modules/artifactory.py index d521e786f3..169ffaff8b 100644 --- a/salt/modules/artifactory.py +++ b/salt/modules/artifactory.py @@ -202,45 +202,48 @@ def _get_snapshot_url(artifactory_url, repository, group_id, artifact_id, versio has_classifier = classifier is not None and classifier != "" if snapshot_version is None: - snapshot_version_metadata = _get_snapshot_version_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, headers=headers) + try: + snapshot_version_metadata = _get_snapshot_version_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, headers=headers) + if packaging not in snapshot_version_metadata['snapshot_versions']: + error_message = '''Cannot find requested packaging '{packaging}' in the snapshot version metadata. + artifactory_url: {artifactory_url} + repository: {repository} + group_id: {group_id} + artifact_id: {artifact_id} + packaging: {packaging} + classifier: {classifier} + version: {version}'''.format( + artifactory_url=artifactory_url, + repository=repository, + group_id=group_id, + artifact_id=artifact_id, + packaging=packaging, + classifier=classifier, + version=version) + raise ArtifactoryError(error_message) - if packaging not in snapshot_version_metadata['snapshot_versions']: - error_message = '''Cannot find requested packaging '{packaging}' in the snapshot version metadata. - artifactory_url: {artifactory_url} - repository: {repository} - group_id: {group_id} - artifact_id: {artifact_id} - packaging: {packaging} - classifier: {classifier} - version: {version}'''.format( - artifactory_url=artifactory_url, - repository=repository, - group_id=group_id, - artifact_id=artifact_id, - packaging=packaging, - classifier=classifier, - version=version) - raise ArtifactoryError(error_message) + if has_classifier and classifier not in snapshot_version_metadata['snapshot_versions']: + error_message = '''Cannot find requested classifier '{classifier}' in the snapshot version metadata. + artifactory_url: {artifactory_url} + repository: {repository} + group_id: {group_id} + artifact_id: {artifact_id} + packaging: {packaging} + classifier: {classifier} + version: {version}'''.format( + artifactory_url=artifactory_url, + repository=repository, + group_id=group_id, + artifact_id=artifact_id, + packaging=packaging, + classifier=classifier, + version=version) + raise ArtifactoryError(error_message) - if has_classifier and classifier not in snapshot_version_metadata['snapshot_versions']: - error_message = '''Cannot find requested classifier '{classifier}' in the snapshot version metadata. - artifactory_url: {artifactory_url} - repository: {repository} - group_id: {group_id} - artifact_id: {artifact_id} - packaging: {packaging} - classifier: {classifier} - version: {version}'''.format( - artifactory_url=artifactory_url, - repository=repository, - group_id=group_id, - artifact_id=artifact_id, - packaging=packaging, - classifier=classifier, - version=version) - raise ArtifactoryError(error_message) - - snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] + snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] + except CommandExecutionError as err: + log.error('Could not fetch maven-metadat.xml. Assuming snapshot_version=%s.', version) + snapshot_version = version group_url = __get_group_id_subpath(group_id) From e314102978a842efe9b99f629e7015e9a3e844d1 Mon Sep 17 00:00:00 2001 From: SuperPommeDeTerre Date: Thu, 17 Aug 2017 13:28:10 +0200 Subject: [PATCH 442/508] Fix typo. --- salt/modules/artifactory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/artifactory.py b/salt/modules/artifactory.py index 169ffaff8b..26065a8d37 100644 --- a/salt/modules/artifactory.py +++ b/salt/modules/artifactory.py @@ -242,7 +242,7 @@ def _get_snapshot_url(artifactory_url, repository, group_id, artifact_id, versio snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] except CommandExecutionError as err: - log.error('Could not fetch maven-metadat.xml. Assuming snapshot_version=%s.', version) + log.error('Could not fetch maven-metadata.xml. Assuming snapshot_version=%s.', version) snapshot_version = version group_url = __get_group_id_subpath(group_id) From 13e5997457d3bcf592f1b32149d297c09437a521 Mon Sep 17 00:00:00 2001 From: Alessandro -oggei- Ogier Date: Thu, 24 Aug 2017 12:49:39 +0200 Subject: [PATCH 443/508] lint --- tests/unit/modules/genesis_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/genesis_test.py b/tests/unit/modules/genesis_test.py index bab16d3f1e..44325ab711 100644 --- a/tests/unit/modules/genesis_test.py +++ b/tests/unit/modules/genesis_test.py @@ -66,14 +66,14 @@ class GenesisTestCase(TestCase): {'params': {'pkgs': 'vim'}, 'commandlines': [ - ['debootstrap', '--foreign', '--arch', 'amd64', + ['debootstrap', '--foreign', '--arch', 'amd64', '--include', 'vim', 'stable', 'root', 'http://ftp.debian.org/debian/'], ]}, {'params': {'pkgs': 'vim,emacs'}, 'commandlines': [ - ['debootstrap', '--foreign', '--arch', 'amd64', + ['debootstrap', '--foreign', '--arch', 'amd64', '--include', 'vim,emacs', 'stable', 'root', 'http://ftp.debian.org/debian/'], ]}, From 3634055e3462edf9b4c0fabc377c0288e9f3a15c Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 24 Aug 2017 11:05:32 +0000 Subject: [PATCH 444/508] Improve napalm state output in debug mode --- salt/utils/napalm.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index 0ca387cc68..721aa43d95 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -435,10 +435,8 @@ def default_ret(name): def loaded_ret(ret, loaded, test, debug): ''' Return the final state output. - ret The initial state output structure. - loaded The loaded dictionary. ''' @@ -447,9 +445,6 @@ def loaded_ret(ret, loaded, test, debug): 'comment': loaded.get('comment', '') }) pchanges = {} - if not loaded.get('result', False): - # Failure of some sort - return ret if debug: # Always check for debug pchanges.update({ @@ -458,6 +453,15 @@ def loaded_ret(ret, loaded, test, debug): ret.update({ "pchanges": pchanges }) + if not loaded.get('result', False): + # Failure of some sort + if debug: + ret['comment'] = '{base_err}\n\nLoaded config:\n\n{loaded_cfg}'.format(base_err=ret['comment'], + loaded_cfg=loaded['loaded_config']) + if loaded.get('diff'): + ret['comment'] = '{comment_base}\n\nConfiguration diff:\n\n{diff}'.format(comment_base=ret['comment'], + diff=loaded['diff']) + return ret if not loaded.get('already_configured', True): # We're making changes pchanges.update({ @@ -484,6 +488,7 @@ def loaded_ret(ret, loaded, test, debug): return ret # No changes ret.update({ - 'result': True + 'result': True, + 'changes': {} }) return ret From 3a906109bd9398f403fe0acf07f78c216a7ece2a Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 24 Aug 2017 12:26:46 +0000 Subject: [PATCH 445/508] Include compliance reports For the netyang state, the compliance report will be included by default in the pchanges dictionary. It can be also returned in the comment field, by enabling using the `compliance_report` option. --- salt/states/netyang.py | 10 +++++++++- salt/utils/napalm.py | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/salt/states/netyang.py b/salt/states/netyang.py index e39b09b3b9..b116cb8f6a 100644 --- a/salt/states/netyang.py +++ b/salt/states/netyang.py @@ -38,6 +38,7 @@ except ImportError: HAS_NAPALM_YANG = False # Import salt modules +import salt.output from salt.utils import fopen import salt.utils.napalm @@ -140,6 +141,7 @@ def managed(name, debug = kwargs.get('debug', False) or __opts__.get('debug', False) commit = kwargs.get('commit', True) or __opts__.get('commit', True) replace = kwargs.get('replace', False) or __opts__.get('replace', False) + return_compliance_report = kwargs.get('compliance_report', False) or __opts__.get('compliance_report', False) profiles = kwargs.get('profiles', []) temp_file = __salt__['temp.file']() log.debug('Creating temp file: {0}'.format(temp_file)) @@ -180,7 +182,13 @@ def managed(name, log.debug('Loaded config result:') log.debug(loaded_changes) __salt__['file.remove'](temp_file) - return salt.utils.napalm.loaded_ret(ret, loaded_changes, test, debug) + loaded_changes['compliance_report'] = compliance_report + return salt.utils.napalm.loaded_ret(ret, + loaded_changes, + test, + debug, + opts=__opts__, + compliance_report=return_compliance_report) def configured(name, diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index 721aa43d95..abaf731ffa 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -432,7 +432,7 @@ def default_ret(name): return ret -def loaded_ret(ret, loaded, test, debug): +def loaded_ret(ret, loaded, test, debug, compliance_report=False, opts=None): ''' Return the final state output. ret @@ -445,6 +445,8 @@ def loaded_ret(ret, loaded, test, debug): 'comment': loaded.get('comment', '') }) pchanges = {} + if 'compliance_report' in loaded: + pchanges['compliance_report'] = loaded['compliance_report'] if debug: # Always check for debug pchanges.update({ @@ -471,10 +473,17 @@ def loaded_ret(ret, loaded, test, debug): 'pchanges': pchanges }) if test: - for k, v in pchanges.items(): - ret.update({ - "comment": "{}:\n{}\n\n{}".format(k, v, ret.get("comment", '')) - }) + if pchanges.get('diff'): + ret['comment'] = '{comment_base}\n\nConfiguration diff:\n\n{diff}'.format(comment_base=ret['comment'], + diff=pchanges['diff']) + if pchanges.get('loaded_config'): + ret['comment'] = '{comment_base}\n\nLoaded config:\n\n{loaded_cfg}'.format( + comment_base=ret['comment'], + loaded_cfg=pchanges['loaded_config']) + if compliance_report and pchanges.get('compliance_report'): + ret['comment'] = '{comment_base}\n\nCompliance report:\n\n{compliance}'.format( + comment_base=ret['comment'], + compliance=salt.output.string_format(pchanges['compliance_report'], 'nested', opts=opts)) ret.update({ 'result': None, }) From 0bbea6b04c58bad3dcbac707f64c997804aa02f4 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 24 Aug 2017 12:33:27 +0000 Subject: [PATCH 446/508] Document the new compliance_report arg --- salt/states/netyang.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/states/netyang.py b/salt/states/netyang.py index b116cb8f6a..e47f4a9830 100644 --- a/salt/states/netyang.py +++ b/salt/states/netyang.py @@ -93,6 +93,13 @@ def managed(name, Use certain profiles to generate the config. If not specified, will use the platform default profile(s). + compliance_report: ``False`` + Return the compliance report in the comment. + The compliance report structured object can be found however + in the ``pchanges`` field of the output (not displayed on the CLI). + + .. versionadded:: 2017.7.3 + test: ``False`` Dry run? If set as ``True``, will apply the config, discard and return the changes. Default: ``False`` and will commit From 1cd33cbaa97fdc9eb1931df73a051716d4431325 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 24 Aug 2017 13:00:15 +0000 Subject: [PATCH 447/508] Simplify the loaded_ret logic --- salt/utils/napalm.py | 64 +++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index abaf731ffa..dc7443f088 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -441,58 +441,44 @@ def loaded_ret(ret, loaded, test, debug, compliance_report=False, opts=None): The loaded dictionary. ''' # Always get the comment - ret.update({ - 'comment': loaded.get('comment', '') - }) + changes = {} pchanges = {} + ret['comment'] = loaded['comment'] + if 'diff' in loaded: + changes['diff'] = loaded['diff'] + pchanges['diff'] = loaded['diff'] if 'compliance_report' in loaded: + if compliance_report: + changes['compliance_report'] = loaded['compliance_report'] pchanges['compliance_report'] = loaded['compliance_report'] - if debug: - # Always check for debug - pchanges.update({ - 'loaded_config': loaded.get('loaded_config', '') - }) - ret.update({ - "pchanges": pchanges - }) + if debug and 'loaded_config' in loaded: + changes['loaded_config'] = loaded['loaded_config'] + pchanges['loaded_config'] = loaded['loaded_config'] + ret['pchanges']= pchanges + if changes.get('diff'): + ret['comment'] = '{comment_base}\n\nConfiguration diff:\n\n{diff}'.format(comment_base=ret['comment'], + diff=pchanges['diff']) + if changes.get('loaded_config'): + ret['comment'] = '{comment_base}\n\nLoaded config:\n\n{loaded_cfg}'.format( + comment_base=ret['comment'], + loaded_cfg=changes['loaded_config']) + if changes.get('compliance_report'): + ret['comment'] = '{comment_base}\n\nCompliance report:\n\n{compliance}'.format( + comment_base=ret['comment'], + compliance=salt.output.string_format(changes['compliance_report'], 'nested', opts=opts)) if not loaded.get('result', False): # Failure of some sort - if debug: - ret['comment'] = '{base_err}\n\nLoaded config:\n\n{loaded_cfg}'.format(base_err=ret['comment'], - loaded_cfg=loaded['loaded_config']) - if loaded.get('diff'): - ret['comment'] = '{comment_base}\n\nConfiguration diff:\n\n{diff}'.format(comment_base=ret['comment'], - diff=loaded['diff']) return ret if not loaded.get('already_configured', True): # We're making changes - pchanges.update({ - "diff": loaded.get('diff', '') - }) - ret.update({ - 'pchanges': pchanges - }) if test: - if pchanges.get('diff'): - ret['comment'] = '{comment_base}\n\nConfiguration diff:\n\n{diff}'.format(comment_base=ret['comment'], - diff=pchanges['diff']) - if pchanges.get('loaded_config'): - ret['comment'] = '{comment_base}\n\nLoaded config:\n\n{loaded_cfg}'.format( - comment_base=ret['comment'], - loaded_cfg=pchanges['loaded_config']) - if compliance_report and pchanges.get('compliance_report'): - ret['comment'] = '{comment_base}\n\nCompliance report:\n\n{compliance}'.format( - comment_base=ret['comment'], - compliance=salt.output.string_format(pchanges['compliance_report'], 'nested', opts=opts)) - ret.update({ - 'result': None, - }) + ret['result'] = None return ret # Not test, changes were applied ret.update({ 'result': True, - 'changes': pchanges, - 'comment': "Configuration changed!\n{}".format(ret.get('comment', '')) + 'changes': changes, + 'comment': "Configuration changed!\n{}".format(loaded['comment']) }) return ret # No changes From e5cc667762616446383a0a454b5b6fde74ddb8dc Mon Sep 17 00:00:00 2001 From: Alessandro -oggei- Ogier Date: Thu, 24 Aug 2017 15:04:37 +0200 Subject: [PATCH 448/508] tests: fix a leftover and simplify some parts --- tests/unit/modules/genesis_test.py | 54 +++++++++++++----------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/tests/unit/modules/genesis_test.py b/tests/unit/modules/genesis_test.py index 44325ab711..31dccabad1 100644 --- a/tests/unit/modules/genesis_test.py +++ b/tests/unit/modules/genesis_test.py @@ -59,38 +59,33 @@ class GenesisTestCase(TestCase): param_sets = [ {'params': {}, - 'commandlines': [ - ['debootstrap', '--foreign', '--arch', 'amd64', - 'stable', 'root', 'http://ftp.debian.org/debian/'], - ]}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, {'params': {'pkgs': 'vim'}, - 'commandlines': [ - ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim', - 'stable', 'root', 'http://ftp.debian.org/debian/'], - ]}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, {'params': {'pkgs': 'vim,emacs'}, - 'commandlines': [ - ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim,emacs', - 'stable', 'root', 'http://ftp.debian.org/debian/'], - ]}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, {'params': {'pkgs': ['vim', 'emacs']}, - 'commandlines': [ - ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim,emacs', - 'stable', 'root', 'http://ftp.debian.org/debian/'], - ]}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, {'params': {'pkgs': ['vim', 'emacs'], 'exclude_pkgs': ['vim', 'foo']}, - 'commandlines': [ - ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim,emacs', '--exclude', 'vim,foo', - 'stable', 'root', 'http://ftp.debian.org/debian/'], - ]}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', '--exclude', 'vim,foo', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, ] @@ -99,13 +94,12 @@ class GenesisTestCase(TestCase): with patch.dict(genesis.__salt__, {'mount.umount': MagicMock(), 'file.rmdir': MagicMock(), 'file.directory_exists': MagicMock(), - 'cmd.run': MagicMock()}): - with patch.dict(genesis.__salt__, {'disk.blkid': MagicMock(return_value={})}): + 'cmd.run': MagicMock(), + 'disk.blkid': MagicMock(return_value={})}): + with patch('salt.modules.genesis.salt.utils.which', return_value=True): param_set['params'].update(common_parms) - self.assertEqual(genesis.bootstrap(**param_set['params']), - None) - for commandline in param_set['commandlines']: - genesis.__salt__['cmd.run'].assert_any_call(commandline, python_shell=False) + self.assertEqual(genesis.bootstrap(**param_set['params']), None) + genesis.__salt__['cmd.run'].assert_any_call(param_set['cmd'], python_shell=False) with patch.object(genesis, '_bootstrap_pacman', return_value='A') as pacman_patch: with patch.dict(genesis.__salt__, {'mount.umount': MagicMock(), From db94f3bb1c4806f474f33ad7c4d067770e1642a0 Mon Sep 17 00:00:00 2001 From: Alessandro -oggei- Ogier Date: Thu, 24 Aug 2017 15:07:08 +0200 Subject: [PATCH 449/508] better formatting --- tests/unit/modules/genesis_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/modules/genesis_test.py b/tests/unit/modules/genesis_test.py index 31dccabad1..784bb8ad84 100644 --- a/tests/unit/modules/genesis_test.py +++ b/tests/unit/modules/genesis_test.py @@ -60,31 +60,31 @@ class GenesisTestCase(TestCase): {'params': {}, 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', - 'stable', 'root', 'http://ftp.debian.org/debian/'] + 'stable', 'root', 'http://ftp.debian.org/debian/'] }, {'params': {'pkgs': 'vim'}, 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim', - 'stable', 'root', 'http://ftp.debian.org/debian/'] + '--include', 'vim', + 'stable', 'root', 'http://ftp.debian.org/debian/'] }, {'params': {'pkgs': 'vim,emacs'}, 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim,emacs', - 'stable', 'root', 'http://ftp.debian.org/debian/'] + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'] }, {'params': {'pkgs': ['vim', 'emacs']}, 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim,emacs', - 'stable', 'root', 'http://ftp.debian.org/debian/'] + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'] }, {'params': {'pkgs': ['vim', 'emacs'], 'exclude_pkgs': ['vim', 'foo']}, 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', - '--include', 'vim,emacs', '--exclude', 'vim,foo', - 'stable', 'root', 'http://ftp.debian.org/debian/'] + '--include', 'vim,emacs', '--exclude', 'vim,foo', + 'stable', 'root', 'http://ftp.debian.org/debian/'] }, ] From c10717dc898f26478a1ac219d9394f88371b0f9d Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 24 Aug 2017 13:20:45 +0000 Subject: [PATCH 450/508] Lint and fix --- salt/states/netyang.py | 1 - salt/utils/napalm.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/states/netyang.py b/salt/states/netyang.py index e47f4a9830..2db6208f57 100644 --- a/salt/states/netyang.py +++ b/salt/states/netyang.py @@ -38,7 +38,6 @@ except ImportError: HAS_NAPALM_YANG = False # Import salt modules -import salt.output from salt.utils import fopen import salt.utils.napalm diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index dc7443f088..0523e0a568 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -24,6 +24,7 @@ from functools import wraps log = logging.getLogger(__file__) import salt.utils +import salt.output # Import third party lib try: @@ -454,10 +455,10 @@ def loaded_ret(ret, loaded, test, debug, compliance_report=False, opts=None): if debug and 'loaded_config' in loaded: changes['loaded_config'] = loaded['loaded_config'] pchanges['loaded_config'] = loaded['loaded_config'] - ret['pchanges']= pchanges + ret['pchanges'] = pchanges if changes.get('diff'): ret['comment'] = '{comment_base}\n\nConfiguration diff:\n\n{diff}'.format(comment_base=ret['comment'], - diff=pchanges['diff']) + diff=changes['diff']) if changes.get('loaded_config'): ret['comment'] = '{comment_base}\n\nLoaded config:\n\n{loaded_cfg}'.format( comment_base=ret['comment'], From d010b74b8779b3b627761e1e6ee4c439d0922a11 Mon Sep 17 00:00:00 2001 From: Darren Demicoli Date: Fri, 18 Aug 2017 01:36:51 +0200 Subject: [PATCH 451/508] Do not try to match pillarenv with __env__ --- salt/pillar/git_pillar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py index 8b85d14ab3..ae1d337387 100644 --- a/salt/pillar/git_pillar.py +++ b/salt/pillar/git_pillar.py @@ -561,7 +561,7 @@ def ext_pillar(minion_id, repo, pillar_dirs): ) for pillar_dir, env in six.iteritems(pillar.pillar_dirs): # If pillarenv is set, only grab pillars with that match pillarenv - if opts['pillarenv'] and env != opts['pillarenv']: + if opts['pillarenv'] and env != opts['pillarenv'] and env != '__env__': log.debug( 'env \'%s\' for pillar dir \'%s\' does not match ' 'pillarenv \'%s\', skipping', From 7b5943a31a62910097d06a72d046053f723fc6c1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 24 Aug 2017 11:04:31 -0500 Subject: [PATCH 452/508] Add warning about adding new functions to salt/utils/__init__.py Since we are breaking this up, we don't want new functions added here. --- salt/utils/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 35e9172a5a..41f06fbb9a 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- ''' Some of the utils used by salt + +NOTE: The dev team is working on splitting up this file for the Oxygen release. +Please do not add any new functions to this file. New functions should be +organized in other files under salt/utils/. Please consult the dev team if you +are unsure where a new function should go. ''' # Import python libs From 5385c7901edc646ae90cd9f852e1ed82dd546a40 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 24 Aug 2017 12:07:44 -0400 Subject: [PATCH 453/508] Move new utils/__init__.py funcs to utils.files.py We're in the process of breaking up the utils/__init__.py file, so let's steer clear of adding new funcs here. This way we can avoid a deprecation process for these functions in develop, since the funcs in this location were never released. Refs #43056 --- salt/utils/__init__.py | 34 ---------------------------------- salt/utils/files.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index c03e0a6d39..ec018f1ad7 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -16,7 +16,6 @@ import json import logging import numbers import os -import os.path import posixpath import random import re @@ -33,7 +32,6 @@ import warnings import string import subprocess import getpass -import urllib # Import 3rd-party libs from salt.ext import six @@ -165,38 +163,6 @@ def is_empty(filename): return False -def safe_filename_leaf(file_basename): - ''' - input the basename of a file, without the directory tree, and returns a safe name to use - i.e. only the required characters are converted by urllib.quote - If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode. - For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible - windows is \\ / : * ? " < > | posix is / - ''' - def _replace(re_obj): - return urllib.quote(re_obj.group(0), safe=u'') - if not isinstance(file_basename, six.text_type): - # the following string is not prefixed with u - return re.sub('[\\\\:/*?"<>|]', - _replace, - six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace')) - # the following string is prefixed with u - return re.sub(u'[\\\\:/*?"<>|]', _replace, file_basename, flags=re.UNICODE) - - -def safe_filepath(file_path_name): - ''' - input the full path and filename, splits on directory separator and calls safe_filename_leaf for - each part of the path. - ''' - (drive, path) = os.path.splitdrive(file_path_name) - path = os.sep.join([safe_filename_leaf(file_section) for file_section in file_path_name.rsplit(os.sep)]) - if drive: - return os.sep.join([drive, path]) - else: - return path - - def is_hex(value): ''' Returns True if value is a hexidecimal string, otherwise returns False diff --git a/salt/utils/files.py b/salt/utils/files.py index d4893608a2..8d463756d9 100644 --- a/salt/utils/files.py +++ b/salt/utils/files.py @@ -7,9 +7,11 @@ import contextlib import errno import logging import os +import re import shutil import subprocess import time +import urllib # Import salt libs import salt.utils @@ -258,3 +260,39 @@ def set_umask(mask): yield finally: os.umask(orig_mask) + + +def safe_filename_leaf(file_basename): + ''' + Input the basename of a file, without the directory tree, and returns a safe name to use + i.e. only the required characters are converted by urllib.quote + If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode. + For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible + windows is \\ / : * ? " < > | posix is / + + .. versionadded:: 2017.7.2 + ''' + def _replace(re_obj): + return urllib.quote(re_obj.group(0), safe=u'') + if not isinstance(file_basename, six.text_type): + # the following string is not prefixed with u + return re.sub('[\\\\:/*?"<>|]', + _replace, + six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace')) + # the following string is prefixed with u + return re.sub(u'[\\\\:/*?"<>|]', _replace, file_basename, flags=re.UNICODE) + + +def safe_filepath(file_path_name): + ''' + Input the full path and filename, splits on directory separator and calls safe_filename_leaf for + each part of the path. + + .. versionadded:: 2017.7.2 + ''' + (drive, path) = os.path.splitdrive(file_path_name) + path = os.sep.join([safe_filename_leaf(file_section) for file_section in file_path_name.rsplit(os.sep)]) + if drive: + return os.sep.join([drive, path]) + else: + return path From 1b24244bd32149dec40644771b670538b546bed4 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Thu, 24 Aug 2017 12:51:00 -0400 Subject: [PATCH 454/508] Add New Release Branch Strategy to Contribution Docs --- doc/topics/development/contributing.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/topics/development/contributing.rst b/doc/topics/development/contributing.rst index 57e3b3c677..fd21d86a23 100644 --- a/doc/topics/development/contributing.rst +++ b/doc/topics/development/contributing.rst @@ -260,6 +260,13 @@ The Salt development team will back-port bug fixes made to ``develop`` to the current release branch if the contributor cannot create the pull request against that branch. +Release Branches +---------------- + +For each release a branch will be created when we are ready to tag. The branch will be the same name as the tag minus the v. For example, the v2017.7.1 release was created from the 2017.7.1 branch. This branching strategy will allow for more stability when there is a need for a re-tag during the testing phase of our releases. + +Once the branch is created, the fixes required for a given release, as determined by the SaltStack release team, will be added to this branch. All commits in this branch will be merged forward into the parent branch as well. + Keeping Salt Forks in Sync ========================== From a0bb654e4695868ffd970717ad7a482aa0887ebb Mon Sep 17 00:00:00 2001 From: garethgreenaway Date: Thu, 24 Aug 2017 12:09:29 -0700 Subject: [PATCH 455/508] Fixing lint issues --- tests/unit/modules/test_dockermod.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/test_dockermod.py b/tests/unit/modules/test_dockermod.py index 24a6d3a3df..6d078d23b7 100644 --- a/tests/unit/modules/test_dockermod.py +++ b/tests/unit/modules/test_dockermod.py @@ -705,9 +705,9 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): def _inspect_container_effect(id_): return { 'container1': {'Config': {'Image': 'realimage:latest'}, - 'HostConfig':{}}, + 'HostConfig': {}}, 'container2': {'Config': {'Image': 'image_id'}, - 'HostConfig':{}}, + 'HostConfig': {}}, }[id_] def _inspect_image_effect(id_): From 4f8e6c65e599d8bdb7cbfb1bf5990be6f9354ec3 Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Thu, 24 Aug 2017 16:06:39 -0400 Subject: [PATCH 456/508] access safe_filename_leaf through utils.files, changed in #43172 --- salt/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index ae2ba0c6db..0d35b98276 100644 --- a/salt/state.py +++ b/salt/state.py @@ -37,6 +37,7 @@ import salt.utils.dictupdate import salt.utils.event import salt.utils.url import salt.utils.process +import salt.utils.files import salt.syspaths as syspaths from salt.utils import immutabletypes from salt.template import compile_template, compile_template_str @@ -150,7 +151,7 @@ def _clean_tag(tag): ''' Make tag name safe for filenames ''' - return salt.utils.safe_filename_leaf(tag) + return salt.utils.files.safe_filename_leaf(tag) def _l_tag(name, id_): From 3adf8ad04be2c3cddef1b7ad71c3d0327afaa63e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 24 Aug 2017 15:26:55 -0500 Subject: [PATCH 457/508] Fix missed deprecation This removes an argument that was slated for deprecation in the Hydrogen release. --- salt/modules/aptpkg.py | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index c8a26a8e06..b044c4ead2 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -135,19 +135,6 @@ def _reconstruct_ppa_name(owner_name, ppa_name): return 'ppa:{0}/{1}'.format(owner_name, ppa_name) -def _get_repo(**kwargs): - ''' - Check the kwargs for either 'fromrepo' or 'repo' and return the value. - 'fromrepo' takes precedence over 'repo'. - ''' - for key in ('fromrepo', 'repo'): - try: - return kwargs[key] - except KeyError: - pass - return '' - - def _check_apt(): ''' Abort if python-apt is not installed @@ -242,18 +229,11 @@ def latest_version(*names, **kwargs): ''' refresh = salt.utils.is_true(kwargs.pop('refresh', True)) show_installed = salt.utils.is_true(kwargs.pop('show_installed', False)) - if 'repo' in kwargs: - # Remember to kill _get_repo() too when removing this warning. - salt.utils.warn_until( - 'Hydrogen', - 'The \'repo\' argument to apt.latest_version is deprecated, and ' - 'will be removed in Salt {version}. Please use \'fromrepo\' ' - 'instead.' + raise SaltInvocationError( + 'The \'repo\' argument is invalid, use \'fromrepo\' instead' ) - fromrepo = _get_repo(**kwargs) - kwargs.pop('fromrepo', None) - kwargs.pop('repo', None) + fromrepo = kwargs.pop('fromrepo', None) cache_valid_time = kwargs.pop('cache_valid_time', 0) if len(names) == 0: @@ -1380,9 +1360,10 @@ def _get_upgradable(dist_upgrade=True, **kwargs): cmd.append('dist-upgrade') else: cmd.append('upgrade') - fromrepo = _get_repo(**kwargs) - if fromrepo: - cmd.extend(['-o', 'APT::Default-Release={0}'.format(fromrepo)]) + try: + cmd.extend(['-o', 'APT::Default-Release={0}'.format(kwargs['fromrepo'])]) + except KeyError: + pass call = __salt__['cmd.run_all'](cmd, python_shell=False, From 2640833400835afa6b9598d818c8bdef7d043856 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 24 Aug 2017 15:00:26 -0500 Subject: [PATCH 458/508] git.detached: Fix traceback when rev is a SHA and is not present locally This catches the (expected) CommandExecutionError when git.describe is run on a commit that doesn't exist. This also renames the remote_ref_type variable to remote_rev_type. This change was made for the 2017.7 branch, but this fix is being applied to 2016.11, so by making the rename here as well we will avoid a potential bug from creeping in on a merge-forward. --- salt/states/git.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 0654d46d19..428d256b41 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -2221,11 +2221,11 @@ def detached(name, return ret # Determine if supplied ref is a hash - remote_ref_type = 'ref' + remote_rev_type = 'ref' if len(ref) <= 40 \ and all(x in string.hexdigits for x in ref): ref = ref.lower() - remote_ref_type = 'hash' + remote_rev_type = 'hash' comments = [] hash_exists_locally = False @@ -2238,13 +2238,18 @@ def detached(name, local_commit_id = _get_local_rev_and_branch(target, user, password)[0] - if remote_ref_type is 'hash' \ - and __salt__['git.describe'](target, - ref, - user=user, - password=password): - # The ref is a hash and it exists locally so skip to checkout - hash_exists_locally = True + if remote_rev_type is 'hash': + try: + __salt__['git.describe'](target, + ref, + user=user, + password=password, + ignore_retcode=True) + except CommandExecutionError: + hash_exists_locally = False + else: + # The rev is a hash and it exists locally so skip to checkout + hash_exists_locally = True else: # Check that remote is present and set to correct url remotes = __salt__['git.remotes'](target, @@ -2409,7 +2414,7 @@ def detached(name, #get refs and checkout checkout_commit_id = '' - if remote_ref_type is 'hash': + if remote_rev_type is 'hash': if __salt__['git.describe'](target, ref, user=user, password=password): checkout_commit_id = ref else: From 0186835ebfa8223ab2b2eef2f85f5266a9f4e268 Mon Sep 17 00:00:00 2001 From: garethgreenaway Date: Thu, 24 Aug 2017 12:09:29 -0700 Subject: [PATCH 459/508] Fix docstring in test --- tests/unit/modules/test_dockermod.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/modules/test_dockermod.py b/tests/unit/modules/test_dockermod.py index 6d078d23b7..a774e23d32 100644 --- a/tests/unit/modules/test_dockermod.py +++ b/tests/unit/modules/test_dockermod.py @@ -700,7 +700,8 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): def test_compare_container_image_id_resolution(self): ''' - Compare + Test comparing two containers when one's inspect output is an ID and + not formatted in image:tag notation. ''' def _inspect_container_effect(id_): return { From 081f42ad7148c5dcb441fea3d9970de18dfe0eb4 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 24 Aug 2017 17:27:58 -0500 Subject: [PATCH 460/508] docker.compare_container: Perform boolean comparison when one side's value is null/None --- salt/modules/dockermod.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index a6cd82ce8e..aa02b6452c 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -899,7 +899,7 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict][item] val2 = result2[conf_dict].get(item) - if item in ('OomKillDisable',): + if item in ('OomKillDisable',) or (val1 is None or val2 is None): if bool(val1) != bool(val2): ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} else: @@ -917,7 +917,7 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict].get(item) val2 = result2[conf_dict][item] - if item in ('OomKillDisable',): + if item in ('OomKillDisable',) or (val1 is None or val2 is None): if bool(val1) != bool(val2): ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} else: From 7279f98e9276f77bef074062e8f718c8065093ba Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 24 Aug 2017 14:26:28 -0500 Subject: [PATCH 461/508] docker_image states: Handle Hub images prefixed with "docker.io/" On some platforms, for reason which I do not yet grok, images pulled from the Hub are prefixed with "docker.io/". This causes the docker_image states to fail unless the user manually adds "docker.io/" before the image name. This commit adds a new function called "docker.resolve_tag" which disambiguates this variance and allows images to be specified without the "docker.io/" prefix. Resolves #42935. --- salt/modules/dockermod.py | 38 +++++++++++++++ salt/states/docker_image.py | 67 ++++++++++++-------------- tests/unit/modules/test_dockermod.py | 25 +++++++++- tests/unit/states/test_docker_image.py | 39 ++++++--------- 4 files changed, 108 insertions(+), 61 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 3a10b20068..fa29f623ba 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -232,6 +232,7 @@ except ImportError: # pylint: enable=import-error HAS_NSENTER = bool(salt.utils.which('nsenter')) +HUB_PREFIX = 'docker.io/' # Set up logging log = logging.getLogger(__name__) @@ -1486,6 +1487,43 @@ def list_tags(): return sorted(ret) +def resolve_tag(name, tags=None): + ''' + .. versionadded:: 2017.7.2,Oxygen + + Given an image tag, check the locally-pulled tags (using + :py:func:`docker.list_tags `) and return + the matching tag. This helps disambiguate differences on some platforms + where images from the Docker Hub are prefixed with ``docker.io/``. If an + image name with no tag is passed, a tag of ``latest`` is assumed. + + If the specified image is not pulled locally, this function will return + ``False``. + + tags + An optional Python list of tags to check against. If passed, then + :py:func:`docker.list_tags ` will not + be run to get a list of tags. This is useful when resolving a number of + tags at the same time. + + CLI Examples: + + .. code-block:: bash + + salt myminion docker.resolve_tag busybox + salt myminion docker.resolve_tag busybox:latest + ''' + tag_name = ':'.join(salt.utils.docker.get_repo_tag(name)) + if tags is None: + tags = list_tags() + if tag_name in tags: + return tag_name + full_name = HUB_PREFIX + tag_name + if not name.startswith(HUB_PREFIX) and full_name in tags: + return full_name + return False + + def logs(name): ''' Returns the logs for the container. Equivalent to running the ``docker diff --git a/salt/states/docker_image.py b/salt/states/docker_image.py index e3c1a37779..087ce8f693 100644 --- a/salt/states/docker_image.py +++ b/salt/states/docker_image.py @@ -135,13 +135,14 @@ def present(name, .. versionadded:: 2016.11.0 sls - Allow for building images with ``dockerng.sls_build`` by specify the - SLS files to build with. This can be a list or comma-seperated string. + Allow for building of image with :py:func:`docker.sls_build + ` by specifying the SLS files with + which to build. This can be a list or comma-seperated string. .. code-block:: yaml myuser/myimage:mytag: - dockerng.image_present: + docker_image.present: - sls: - webapp1 - webapp2 @@ -151,12 +152,14 @@ def present(name, .. versionadded: 2017.7.0 base - Base image with which to start ``dockerng.sls_build`` + Base image with which to start :py:func:`docker.sls_build + ` .. versionadded: 2017.7.0 saltenv - environment from which to pull sls files for ``dockerng.sls_build``. + Environment from which to pull SLS files for :py:func:`docker.sls_build + ` .. versionadded: 2017.7.0 ''' @@ -169,11 +172,14 @@ def present(name, ret['comment'] = 'Only one of \'build\' or \'load\' is permitted.' return ret - # Ensure that we have repo:tag notation image = ':'.join(salt.utils.docker.get_repo_tag(name)) - all_tags = __salt__['docker.list_tags']() + resolved_tag = __salt__['docker.resolve_tag'](image) - if image in all_tags: + if resolved_tag is False: + # Specified image is not present + image_info = None + else: + # Specified image is present if not force: ret['result'] = True ret['comment'] = 'Image \'{0}\' already present'.format(name) @@ -185,8 +191,6 @@ def present(name, ret['comment'] = \ 'Unable to get info for image \'{0}\': {1}'.format(name, exc) return ret - else: - image_info = None if build or sls: action = 'built' @@ -197,15 +201,15 @@ def present(name, if __opts__['test']: ret['result'] = None - if (image in all_tags and force) or image not in all_tags: + if (resolved_tag is not False and force) or resolved_tag is False: ret['comment'] = 'Image \'{0}\' will be {1}'.format(name, action) return ret if build: try: image_update = __salt__['docker.build'](path=build, - image=image, - dockerfile=dockerfile) + image=image, + dockerfile=dockerfile) except Exception as exc: ret['comment'] = ( 'Encountered error building {0} as {1}: {2}' @@ -219,10 +223,10 @@ def present(name, if isinstance(sls, list): sls = ','.join(sls) try: - image_update = __salt__['dockerng.sls_build'](name=image, - base=base, - mods=sls, - saltenv=saltenv) + image_update = __salt__['docker.sls_build'](name=image, + base=base, + mods=sls, + saltenv=saltenv) except Exception as exc: ret['comment'] = ( 'Encountered error using sls {0} for building {1}: {2}' @@ -252,10 +256,8 @@ def present(name, client_timeout=client_timeout ) except Exception as exc: - ret['comment'] = ( - 'Encountered error pulling {0}: {1}' - .format(image, exc) - ) + ret['comment'] = \ + 'Encountered error pulling {0}: {1}'.format(image, exc) return ret if (image_info is not None and image_info['Id'][:12] == image_update .get('Layers', {}) @@ -267,7 +269,7 @@ def present(name, # Only add to the changes dict if layers were pulled ret['changes'] = image_update - ret['result'] = image in __salt__['docker.list_tags']() + ret['result'] = bool(__salt__['docker.resolve_tag'](image)) if not ret['result']: # This shouldn't happen, failure to pull should be caught above @@ -345,23 +347,16 @@ def absent(name=None, images=None, force=False): ret['comment'] = 'One of \'name\' and \'images\' must be provided' return ret elif images is not None: - targets = [] - for target in images: - try: - targets.append(':'.join(salt.utils.docker.get_repo_tag(target))) - except TypeError: - # Don't stomp on images with unicode characters in Python 2, - # only force image to be a str if it wasn't already (which is - # very unlikely). - targets.append(':'.join(salt.utils.docker.get_repo_tag(str(target)))) + targets = images elif name: - try: - targets = [':'.join(salt.utils.docker.get_repo_tag(name))] - except TypeError: - targets = [':'.join(salt.utils.docker.get_repo_tag(str(name)))] + targets = [name] pre_tags = __salt__['docker.list_tags']() - to_delete = [x for x in targets if x in pre_tags] + to_delete = [] + for target in targets: + resolved_tag = __salt__['docker.resolve_tag'](target, tags=pre_tags) + if resolved_tag is not False: + to_delete.append(resolved_tag) log.debug('targets = {0}'.format(targets)) log.debug('to_delete = {0}'.format(to_delete)) diff --git a/tests/unit/modules/test_dockermod.py b/tests/unit/modules/test_dockermod.py index a774e23d32..3d0328c180 100644 --- a/tests/unit/modules/test_dockermod.py +++ b/tests/unit/modules/test_dockermod.py @@ -679,9 +679,9 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual({"retcode": 0, "comment": "container cmd"}, ret) def test_images_with_empty_tags(self): - """ + ''' docker 1.12 reports also images without tags with `null`. - """ + ''' client = Mock() client.api_version = '1.24' client.images = Mock( @@ -724,3 +724,24 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): with patch.object(docker_mod, 'inspect_image', inspect_image_mock): ret = docker_mod.compare_container('container1', 'container2') self.assertEqual(ret, {}) + + def test_resolve_tag(self): + ''' + Test the resolve_tag function + ''' + with_prefix = 'docker.io/foo:latest' + no_prefix = 'bar:latest' + with patch.object(docker_mod, + 'list_tags', + MagicMock(return_value=[with_prefix])): + self.assertEqual(docker_mod.resolve_tag('foo'), with_prefix) + self.assertEqual(docker_mod.resolve_tag('foo:latest'), with_prefix) + self.assertEqual(docker_mod.resolve_tag(with_prefix), with_prefix) + self.assertEqual(docker_mod.resolve_tag('foo:bar'), False) + + with patch.object(docker_mod, + 'list_tags', + MagicMock(return_value=[no_prefix])): + self.assertEqual(docker_mod.resolve_tag('bar'), no_prefix) + self.assertEqual(docker_mod.resolve_tag(no_prefix), no_prefix) + self.assertEqual(docker_mod.resolve_tag('bar:baz'), False) diff --git a/tests/unit/states/test_docker_image.py b/tests/unit/states/test_docker_image.py index 4d94c2e239..868925ba3d 100644 --- a/tests/unit/states/test_docker_image.py +++ b/tests/unit/states/test_docker_image.py @@ -10,7 +10,7 @@ from __future__ import absolute_import from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import ( - Mock, + MagicMock, NO_MOCK, NO_MOCK_REASON, patch @@ -50,21 +50,19 @@ class DockerImageTestCase(TestCase, LoaderModuleMockMixin): if ``image:latest`` is already downloaded locally the state should not report changes. ''' - docker_inspect_image = Mock( - return_value={'Id': 'abcdefghijk'}) - docker_pull = Mock( + docker_inspect_image = MagicMock(return_value={'Id': 'abcdefghijkl'}) + docker_pull = MagicMock( return_value={'Layers': - {'Already_Pulled': ['abcdefghijk'], + {'Already_Pulled': ['abcdefghijkl'], 'Pulled': []}, 'Status': 'Image is up to date for image:latest', 'Time_Elapsed': 1.1}) - docker_list_tags = Mock( - return_value=['image:latest'] - ) + docker_list_tags = MagicMock(return_value=['image:latest']) + docker_resolve_tag = MagicMock(return_value='image:latest') __salt__ = {'docker.list_tags': docker_list_tags, 'docker.pull': docker_pull, 'docker.inspect_image': docker_inspect_image, - } + 'docker.resolve_tag': docker_resolve_tag} with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): ret = docker_state.present('image:latest', force=True) @@ -89,29 +87,24 @@ class DockerImageTestCase(TestCase, LoaderModuleMockMixin): if ``image:latest`` is not downloaded and force is true should pull a new image successfuly. ''' - docker_inspect_image = Mock( - side_effect=CommandExecutionError( - 'Error 404: No such image/container: image:latest')) - docker_pull = Mock( + docker_inspect_image = MagicMock(return_value={'Id': '1234567890ab'}) + docker_pull = MagicMock( return_value={'Layers': - {'Already_Pulled': ['abcdefghijk'], - 'Pulled': ['abcdefghijk']}, - 'Status': "Image 'image:latest' was pulled", - 'Time_Elapsed': 1.1}) - docker_list_tags = Mock( - side_effect=[[], ['image:latest']] - ) + {'Pulled': ['abcdefghijkl']}, + 'Status': "Image 'image:latest' was pulled", + 'Time_Elapsed': 1.1}) + docker_list_tags = MagicMock(side_effect=[[], ['image:latest']]) + docker_resolve_tag = MagicMock(return_value='image:latest') __salt__ = {'docker.list_tags': docker_list_tags, 'docker.pull': docker_pull, 'docker.inspect_image': docker_inspect_image, - } + 'docker.resolve_tag': docker_resolve_tag} with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): ret = docker_state.present('image:latest', force=True) self.assertEqual(ret, {'changes': { - 'Layers': {'Already_Pulled': ['abcdefghijk'], - 'Pulled': ['abcdefghijk']}, + 'Layers': {'Pulled': ['abcdefghijkl']}, 'Status': "Image 'image:latest' was pulled", 'Time_Elapsed': 1.1}, 'result': True, From 41640479511006fd51fea46ebff35bc595a4a137 Mon Sep 17 00:00:00 2001 From: Viktor Krivak Date: Fri, 25 Aug 2017 15:08:57 +0200 Subject: [PATCH 462/508] Fix apache.config with multiple statement At this moment when you post more than one statement in config only last is used. Also file is rewrited multiple times until last statement is written. Example: salt '*' apache.config /etc/httpd/conf.d/ports.conf config="[{'Listen': '8080'}, {'Proxy': "Something"}]" Ends only with Proxy Something and ignore Listen 8080, This patch fix this issue. --- salt/modules/apache.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/salt/modules/apache.py b/salt/modules/apache.py index ad502df530..18bdab2726 100644 --- a/salt/modules/apache.py +++ b/salt/modules/apache.py @@ -446,11 +446,15 @@ def config(name, config, edit=True): salt '*' apache.config /etc/httpd/conf.d/ports.conf config="[{'Listen': '22'}]" ''' + configs = [] for entry in config: key = next(six.iterkeys(entry)) - configs = _parse_config(entry[key], key) - if edit: - with salt.utils.fopen(name, 'w') as configfile: - configfile.write('# This file is managed by Salt.\n') - configfile.write(configs) - return configs + configs.append(_parse_config(entry[key], key)) + + # Python auto-correct line endings + configstext = "\n".join(configs) + if edit: + with salt.utils.fopen(name, 'w') as configfile: + configfile.write('# This file is managed by Salt.\n') + configfile.write(configstext) + return configstext From ccd224177793dec66cb89a62e608f9c622a0ebf6 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 25 Aug 2017 10:23:35 -0600 Subject: [PATCH 463/508] Pin request install to version This fails on centos 6 because its node is too old to support the version of hawk bumped here https://github.com/request/request/pull/2751, we can still test the functionality. This will pull from github, and install a specific tag version, and we still do the uninstall using the github path. This should be more stable. --- tests/integration/states/npm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/states/npm.py b/tests/integration/states/npm.py index 140eaa92ca..adcd603cf8 100644 --- a/tests/integration/states/npm.py +++ b/tests/integration/states/npm.py @@ -40,7 +40,7 @@ class NpmStateTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ''' Determine if URL-referenced NPM module can be successfully installed. ''' - ret = self.run_state('npm.installed', name='git://github.com/request/request') + ret = self.run_state('npm.installed', name='request/request#v2.81.1') self.assertSaltTrueReturn(ret) ret = self.run_state('npm.removed', name='git://github.com/request/request') self.assertSaltTrueReturn(ret) From f00d3a9ddc8433e9e92bda7215cc2036e2a9afab Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Fri, 25 Aug 2017 13:29:40 -0400 Subject: [PATCH 464/508] Add `disk.format` alias for `disk.format_` --- salt/modules/disk.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/modules/disk.py b/salt/modules/disk.py index 4b7106a4a7..2ff06278c1 100644 --- a/salt/modules/disk.py +++ b/salt/modules/disk.py @@ -22,6 +22,10 @@ import salt.utils.decorators as decorators from salt.utils.decorators import depends from salt.exceptions import CommandExecutionError +__func_alias__ = { + 'format_': 'format' +} + log = logging.getLogger(__name__) HAS_HDPARM = salt.utils.which('hdparm') is not None From 33a30bac06d967b91a1976f3c54ec17ab4b58008 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Fri, 25 Aug 2017 12:31:12 -0500 Subject: [PATCH 465/508] correcting bad format statement in search for policy to be disabled --- salt/modules/win_lgpo.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index efd76192bd..69986e77c0 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -4181,8 +4181,6 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, existing_data = '' base_policy_settings = {} policy_data = _policy_info() - #//{0}:policy[@displayName = "{1}" and (@class = "Both" or @class = "{2}") ] - #policySearchXpath = etree.XPath('//*[@ns1:id = $id or @ns1:name = $id]') policySearchXpath = '//ns1:*[@id = "{0}" or @name = "{0}"]' try: if admx_policy_definitions is None or adml_policy_resources is None: @@ -4213,8 +4211,7 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, this_valuename = None if str(base_policy_settings[adm_namespace][admPolicy]).lower() == 'disabled': log.debug('time to disable {0}'.format(admPolicy)) - #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) - this_policy = admx_policy_definitions.xpath(policySearchXpath.format('ns1', admPolicy), namespaces={'ns1': adm_namespace}) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) if this_policy: this_policy = this_policy[0] if 'class' in this_policy.attrib: @@ -4325,7 +4322,6 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, log.error(msg.format(this_policy.attrib)) else: log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) - #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) log.debug('found this_policy == {0}'.format(this_policy)) if this_policy: From ef7e93eb3f4a7f4ae4c7ae182fd1a2a721640360 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 25 Aug 2017 12:52:26 -0700 Subject: [PATCH 466/508] Reverting this change due to it breaking other uses. --- salt/modules/augeas_cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/augeas_cfg.py b/salt/modules/augeas_cfg.py index 401a11eb1b..6b1f1e7b1b 100644 --- a/salt/modules/augeas_cfg.py +++ b/salt/modules/augeas_cfg.py @@ -199,7 +199,7 @@ def execute(context=None, lens=None, commands=(), load_path=None): method = METHOD_MAP[cmd] nargs = arg_map[method] - parts = salt.utils.shlex_split(arg, posix=False) + parts = salt.utils.shlex_split(arg) if len(parts) not in nargs: err = '{0} takes {1} args: {2}'.format(method, nargs, parts) From 1eba8c4b8e74e6abd5cce084b74aa8344efa3ce4 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Mon, 28 Aug 2017 10:11:36 +0300 Subject: [PATCH 467/508] Fix pylint errors Fix pylint errors --- salt/modules/win_iis.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index 38217b07a5..2e2e5113d2 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -836,12 +836,10 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, if _iisVersion() < 8: # IIS 7.5 and earlier have different syntax for associating a certificate with a site # Modify IP spec to IIS 7.5 format - iis7path = binding_path.replace(r"\*!", "\\0.0.0.0!") - + iis7path = binding_path.replace(r"\*!", "\\0.0.0.0!") # win 2008 uses the following format: ip!port and not ip!port! if iis7path.endswith("!"): - iis7path = iis7path[:-1] - + iis7path = iis7path[:-1] ps_cmd = ['New-Item', '-Path', "'{0}'".format(iis7path), From 25c8190e484cac4658c81d0b730509a5fb7ab815 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Mon, 28 Aug 2017 10:12:56 +0300 Subject: [PATCH 468/508] Fix pylint errors Fix pylint errors --- salt/states/win_iis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/states/win_iis.py b/salt/states/win_iis.py index 38d2ed9ae4..6407315c21 100644 --- a/salt/states/win_iis.py +++ b/salt/states/win_iis.py @@ -494,8 +494,8 @@ def container_setting(name, container, settings=None): processModel.maxProcesses: 1 processModel.userName: TestUser processModel.password: TestPassword - processModel.identityType: SpecificUser - + processModel.identityType: SpecificUser + Example of usage for the ``Sites`` container: .. code-block:: yaml @@ -509,9 +509,8 @@ def container_setting(name, container, settings=None): logFile.period: Daily limits.maxUrlSegments: 32 ''' - + identityType_map2string = {0: 'LocalSystem', 1: 'LocalService', 2: 'NetworkService', 3: 'SpecificUser', 4: 'ApplicationPoolIdentity'} - ret = {'name': name, 'changes': {}, 'comment': str(), From bd76a870ce2e3844cac5dd115de5fbb13adadba8 Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Mon, 28 Aug 2017 09:34:51 +0200 Subject: [PATCH 469/508] Dunder vars are now defined via setup_loader_modules --- tests/unit/modules/test_kubernetes.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/unit/modules/test_kubernetes.py b/tests/unit/modules/test_kubernetes.py index 6efc4d790d..eda8a0a91b 100644 --- a/tests/unit/modules/test_kubernetes.py +++ b/tests/unit/modules/test_kubernetes.py @@ -7,6 +7,7 @@ from __future__ import absolute_import # Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import ( Mock, @@ -17,16 +18,20 @@ from tests.support.mock import ( from salt.modules import kubernetes -kubernetes.__salt__ = {} -kubernetes.__grains__ = {} -kubernetes.__context__ = {} - @skipIf(NO_MOCK, NO_MOCK_REASON) -class KubernetesTestCase(TestCase): +class KubernetesTestCase(TestCase, LoaderModuleMockMixin): ''' Test cases for salt.modules.kubernetes ''' + + def setup_loader_modules(self): + return { + kubernetes: { + '__salt__': {}, + } + } + def test_nodes(self): ''' Test node listing. @@ -82,7 +87,8 @@ class KubernetesTestCase(TestCase): {'items': [{'metadata': {'name': 'mock_pod_name'}}]}} ) self.assertEqual(kubernetes.pods(), ['mock_pod_name']) - self.assertTrue(kubernetes.kubernetes.client.CoreV1Api().list_namespaced_pod().to_dict.called) + self.assertTrue(kubernetes.kubernetes.client.CoreV1Api(). + list_namespaced_pod().to_dict.called) def test_delete_deployments(self): ''' @@ -97,7 +103,8 @@ class KubernetesTestCase(TestCase): ) self.assertEqual(kubernetes.delete_deployment("test"), {}) self.assertTrue( - kubernetes.kubernetes.client.ExtensionsV1beta1Api().delete_namespaced_deployment().to_dict.called) + kubernetes.kubernetes.client.ExtensionsV1beta1Api(). + delete_namespaced_deployment().to_dict.called) def test_create_deployments(self): ''' @@ -109,6 +116,8 @@ class KubernetesTestCase(TestCase): mock_kubernetes_lib.client.ExtensionsV1beta1Api.return_value = Mock( **{"create_namespaced_deployment.return_value.to_dict.return_value": {}} ) - self.assertEqual(kubernetes.create_deployment("test", "default", {}, {}, None, None, None), {}) + self.assertEqual(kubernetes.create_deployment("test", "default", {}, {}, + None, None, None), {}) self.assertTrue( - kubernetes.kubernetes.client.ExtensionsV1beta1Api().create_namespaced_deployment().to_dict.called) + kubernetes.kubernetes.client.ExtensionsV1beta1Api(). + create_namespaced_deployment().to_dict.called) From df18a898368d887469acd9032f8888dedd865780 Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Mon, 28 Aug 2017 09:21:52 -0400 Subject: [PATCH 470/508] Lint: Remove unused import --- tests/unit/states/test_docker_image.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/states/test_docker_image.py b/tests/unit/states/test_docker_image.py index 868925ba3d..e96dbc9f9d 100644 --- a/tests/unit/states/test_docker_image.py +++ b/tests/unit/states/test_docker_image.py @@ -17,7 +17,6 @@ from tests.support.mock import ( ) # Import Salt Libs -from salt.exceptions import CommandExecutionError import salt.modules.dockermod as docker_mod import salt.states.docker_image as docker_state From 8f593b0b025d91e84d951701a7e3541ed0539234 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Mon, 28 Aug 2017 09:24:25 -0500 Subject: [PATCH 471/508] verify that files exist before trying to remove them, win_file.remove raises an exception if the file does not exist --- salt/modules/win_lgpo.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 69986e77c0..b0f9419fc8 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2832,7 +2832,8 @@ def _findOptionValueInSeceditFile(option): _reader = codecs.open(_tfile, 'r', encoding='utf-16') _secdata = _reader.readlines() _reader.close() - _ret = __salt__['file.remove'](_tfile) + if __salt__['file.file_exists'](_tfile): + _ret = __salt__['file.remove'](_tfile) for _line in _secdata: if _line.startswith(option): return True, _line.split('=')[1].strip() @@ -2853,16 +2854,20 @@ def _importSeceditConfig(infdata): _tInfFile = '{0}\\{1}'.format(__salt__['config.get']('cachedir'), 'salt-secedit-config-{0}.inf'.format(_d)) # make sure our temp files don't already exist - _ret = __salt__['file.remove'](_tSdbfile) - _ret = __salt__['file.remove'](_tInfFile) + if __salt__['file.file_exists'](_tSdbfile): + _ret = __salt__['file.remove'](_tSdbfile) + if __salt__['file.file_exists'](_tInfFile): + _ret = __salt__['file.remove'](_tInfFile) # add the inf data to the file, win_file sure could use the write() function _ret = __salt__['file.touch'](_tInfFile) _ret = __salt__['file.append'](_tInfFile, infdata) # run secedit to make the change _ret = __salt__['cmd.run']('secedit /configure /db {0} /cfg {1}'.format(_tSdbfile, _tInfFile)) # cleanup our temp files - _ret = __salt__['file.remove'](_tSdbfile) - _ret = __salt__['file.remove'](_tInfFile) + if __salt__['file.file_exists'](_tSdbfile): + _ret = __salt__['file.remove'](_tSdbfile) + if __salt__['file.file_exists'](_tInfFile): + _ret = __salt__['file.remove'](_tInfFile) return True except Exception as e: log.debug('error occurred while trying to import secedit data') From 324cfd8d1e03c23d5ee99b575f49d65b934857fd Mon Sep 17 00:00:00 2001 From: lomeroe Date: Mon, 28 Aug 2017 09:39:51 -0500 Subject: [PATCH 472/508] correcting bad format statement in search for policy to be disabled (fix for #43166) verify that file exists before attempting to remove (fix for commits from #39773) --- salt/modules/win_lgpo.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 5edb672c90..2b834227e9 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2835,7 +2835,8 @@ def _findOptionValueInSeceditFile(option): _reader = codecs.open(_tfile, 'r', encoding='utf-16') _secdata = _reader.readlines() _reader.close() - _ret = __salt__['file.remove'](_tfile) + if __salt__['file.file_exists'](_tfile): + _ret = __salt__['file.remove'](_tfile) for _line in _secdata: if _line.startswith(option): return True, _line.split('=')[1].strip() @@ -2856,16 +2857,20 @@ def _importSeceditConfig(infdata): _tInfFile = '{0}\\{1}'.format(__salt__['config.get']('cachedir'), 'salt-secedit-config-{0}.inf'.format(_d)) # make sure our temp files don't already exist - _ret = __salt__['file.remove'](_tSdbfile) - _ret = __salt__['file.remove'](_tInfFile) + if __salt__['file.file_exists'](_tSdbfile): + _ret = __salt__['file.remove'](_tSdbfile) + if __salt__['file.file_exists'](_tInfFile): + _ret = __salt__['file.remove'](_tInfFile) # add the inf data to the file, win_file sure could use the write() function _ret = __salt__['file.touch'](_tInfFile) _ret = __salt__['file.append'](_tInfFile, infdata) # run secedit to make the change _ret = __salt__['cmd.run']('secedit /configure /db {0} /cfg {1}'.format(_tSdbfile, _tInfFile)) # cleanup our temp files - _ret = __salt__['file.remove'](_tSdbfile) - _ret = __salt__['file.remove'](_tInfFile) + if __salt__['file.file_exists'](_tSdbfile): + _ret = __salt__['file.remove'](_tSdbfile) + if __salt__['file.file_exists'](_tInfFile): + _ret = __salt__['file.remove'](_tInfFile) return True except Exception as e: log.debug('error occurred while trying to import secedit data') @@ -4174,8 +4179,6 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, existing_data = '' base_policy_settings = {} policy_data = _policy_info() - #//{0}:policy[@displayName = "{1}" and (@class = "Both" or @class = "{2}") ] - #policySearchXpath = etree.XPath('//*[@ns1:id = $id or @ns1:name = $id]') policySearchXpath = '//ns1:*[@id = "{0}" or @name = "{0}"]' try: if admx_policy_definitions is None or adml_policy_resources is None: @@ -4206,8 +4209,7 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, this_valuename = None if str(base_policy_settings[adm_namespace][admPolicy]).lower() == 'disabled': log.debug('time to disable {0}'.format(admPolicy)) - #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) - this_policy = admx_policy_definitions.xpath(policySearchXpath.format('ns1', admPolicy), namespaces={'ns1': adm_namespace}) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) if this_policy: this_policy = this_policy[0] if 'class' in this_policy.attrib: @@ -4318,7 +4320,6 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, log.error(msg.format(this_policy.attrib)) else: log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) - #this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy, namespaces={'ns1': adm_namespace}) this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) log.debug('found this_policy == {0}'.format(this_policy)) if this_policy: From b1788b1e5f9ba7b9c0e188ef3fa369f757003a3b Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 28 Aug 2017 12:35:46 -0600 Subject: [PATCH 473/508] Bring changes from #43228 to 2017.7 --- salt/modules/win_pkg.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 43bd829831..03faa6ecfd 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -973,7 +973,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Version is ignored salt '*' pkg.install pkgs="['foo', 'bar']" version=1.2.3 - If passed with a comma seperated list in the ``name`` parameter, the + If passed with a comma separated list in the ``name`` parameter, the version will apply to all packages in the list. CLI Example: @@ -1282,7 +1282,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): use_msiexec, msiexec = _get_msiexec(pkginfo[version_num].get('msiexec', False)) # Build cmd and arguments - # cmd and arguments must be seperated for use with the task scheduler + # cmd and arguments must be separated for use with the task scheduler if use_msiexec: cmd = msiexec arguments = ['/i', cached_pkg] @@ -1313,7 +1313,9 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Run Scheduled Task # Special handling for installing salt - if pkg_name in ['salt-minion', 'salt-minion-py3']: + if re.search(r'salt[\s-]*minion', + pkg_name, + flags=re.IGNORECASE + re.UNICODE) is not None: ret[pkg_name] = {'install status': 'task started'} if not __salt__['task.run'](name='update-salt-software'): log.error('Failed to install {0}'.format(pkg_name)) @@ -1345,7 +1347,8 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): else: # Combine cmd and arguments - cmd = [cmd].extend(arguments) + cmd = [cmd] + cmd.extend(arguments) # Launch the command result = __salt__['cmd.run_all'](cmd, From b1a3d15b28e7682f4839f971a4eba177d28686b1 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 28 Aug 2017 13:52:23 -0600 Subject: [PATCH 474/508] Remove trailing whitespace for linter --- salt/modules/win_iis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index 2e2e5113d2..bf52b4f0d5 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -836,10 +836,10 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, if _iisVersion() < 8: # IIS 7.5 and earlier have different syntax for associating a certificate with a site # Modify IP spec to IIS 7.5 format - iis7path = binding_path.replace(r"\*!", "\\0.0.0.0!") + iis7path = binding_path.replace(r"\*!", "\\0.0.0.0!") # win 2008 uses the following format: ip!port and not ip!port! if iis7path.endswith("!"): - iis7path = iis7path[:-1] + iis7path = iis7path[:-1] ps_cmd = ['New-Item', '-Path', "'{0}'".format(iis7path), From 53bd3a3e236d93850e4f53cebd674767b001825f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 28 Aug 2017 17:30:25 -0500 Subject: [PATCH 475/508] Improve inheritance in salt.utils.gitfs This makes the following changes: 1. Renames the valid_providers param in GitBase to git_providers, allowing for a dictionary mapping provider names to their associated classes. This allows for alternate providers to be used with a GitBase subclass. 2. Renames the get_provider function to verify_provider to reduce confusion with git_providers. 3. Uses super() to run a parent class' dunder init instead of invoking the parent class directly. --- salt/utils/gitfs.py | 54 +++++++++++++++++++--------------- tests/unit/utils/test_gitfs.py | 2 +- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index bad3cc883e..ef8c2adf74 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -38,7 +38,6 @@ from salt.utils.versions import LooseVersion as _LooseVersion # Import third party libs import salt.ext.six as six -VALID_PROVIDERS = ('pygit2', 'gitpython') # Optional per-remote params that can only be used on a per-remote basis, and # thus do not have defaults in salt/config.py. PER_REMOTE_ONLY = ('name',) @@ -164,7 +163,7 @@ class GitProvider(object): directly. self.provider should be set in the sub-class' __init__ function before - invoking GitProvider.__init__(). + invoking the parent class' __init__. ''' def __init__(self, opts, remote, per_remote_defaults, per_remote_only, override_params, cache_root, role='gitfs'): @@ -857,8 +856,10 @@ class GitPython(GitProvider): def __init__(self, opts, remote, per_remote_defaults, per_remote_only, override_params, cache_root, role='gitfs'): self.provider = 'gitpython' - GitProvider.__init__(self, opts, remote, per_remote_defaults, - per_remote_only, override_params, cache_root, role) + super(GitPython, self).__init__( + opts, remote, per_remote_defaults, per_remote_only, + override_params, cache_root, role + ) def add_refspecs(self, *refspecs): ''' @@ -1192,8 +1193,10 @@ class Pygit2(GitProvider): def __init__(self, opts, remote, per_remote_defaults, per_remote_only, override_params, cache_root, role='gitfs'): self.provider = 'pygit2' - GitProvider.__init__(self, opts, remote, per_remote_defaults, - per_remote_only, override_params, cache_root, role) + super(Pygit2, self).__init__( + opts, remote, per_remote_defaults, per_remote_only, + override_params, cache_root, role + ) def add_refspecs(self, *refspecs): ''' @@ -1877,11 +1880,17 @@ class Pygit2(GitProvider): fp_.write(blob.data) +GIT_PROVIDERS = { + 'pygit2': Pygit2, + 'gitpython': GitPython, +} + + class GitBase(object): ''' Base class for gitfs/git_pillar ''' - def __init__(self, opts, valid_providers=VALID_PROVIDERS, cache_root=None): + def __init__(self, opts, git_providers=None, cache_root=None): ''' IMPORTANT: If specifying a cache_root, understand that this is also where the remotes will be cloned. A non-default cache_root is only @@ -1889,8 +1898,9 @@ class GitBase(object): out into the winrepo locations and not within the cachedir. ''' self.opts = opts - self.valid_providers = valid_providers - self.get_provider() + self.git_providers = git_providers if git_providers is not None \ + else GIT_PROVIDERS + self.verify_provider() if cache_root is not None: self.cache_root = self.remote_root = cache_root else: @@ -1948,7 +1958,7 @@ class GitBase(object): self.remotes = [] for remote in remotes: - repo_obj = self.provider_class( + repo_obj = self.git_providers[self.provider]( self.opts, remote, per_remote_defaults, @@ -2202,7 +2212,7 @@ class GitBase(object): # Hash file won't exist if no files have yet been served up pass - def get_provider(self): + def verify_provider(self): ''' Determine which provider to use ''' @@ -2223,12 +2233,12 @@ class GitBase(object): # Should only happen if someone does something silly like # set the provider to a numeric value. desired_provider = str(desired_provider).lower() - if desired_provider not in self.valid_providers: + if desired_provider not in self.git_providers: log.critical( 'Invalid {0}_provider \'{1}\'. Valid choices are: {2}' .format(self.role, desired_provider, - ', '.join(self.valid_providers)) + ', '.join(self.git_providers)) ) failhard(self.role) elif desired_provider == 'pygit2' and self.verify_pygit2(): @@ -2241,17 +2251,13 @@ class GitBase(object): .format(self.role) ) failhard(self.role) - if self.provider == 'pygit2': - self.provider_class = Pygit2 - elif self.provider == 'gitpython': - self.provider_class = GitPython def verify_gitpython(self, quiet=False): ''' Check if GitPython is available and at a compatible version (>= 0.3.0) ''' def _recommend(): - if HAS_PYGIT2 and 'pygit2' in self.valid_providers: + if HAS_PYGIT2 and 'pygit2' in self.git_providers: log.error(_RECOMMEND_PYGIT2.format(self.role)) if not HAS_GITPYTHON: @@ -2262,7 +2268,7 @@ class GitBase(object): ) _recommend() return False - elif 'gitpython' not in self.valid_providers: + elif 'gitpython' not in self.git_providers: return False # pylint: disable=no-member @@ -2302,7 +2308,7 @@ class GitBase(object): Pygit2 must be at least 0.20.3 and libgit2 must be at least 0.20.0. ''' def _recommend(): - if HAS_GITPYTHON and 'gitpython' in self.valid_providers: + if HAS_GITPYTHON and 'gitpython' in self.git_providers: log.error(_RECOMMEND_GITPYTHON.format(self.role)) if not HAS_PYGIT2: @@ -2313,7 +2319,7 @@ class GitBase(object): ) _recommend() return False - elif 'pygit2' not in self.valid_providers: + elif 'pygit2' not in self.git_providers: return False # pylint: disable=no-member @@ -2432,7 +2438,7 @@ class GitFS(GitBase): ''' def __init__(self, opts): self.role = 'gitfs' - GitBase.__init__(self, opts) + super(GitFS, self).__init__(opts) def dir_list(self, load): ''' @@ -2735,7 +2741,7 @@ class GitPillar(GitBase): ''' def __init__(self, opts): self.role = 'git_pillar' - GitBase.__init__(self, opts) + super(GitPillar, self).__init__(opts) def checkout(self): ''' @@ -2837,7 +2843,7 @@ class WinRepo(GitBase): ''' def __init__(self, opts, winrepo_dir): self.role = 'winrepo' - GitBase.__init__(self, opts, cache_root=winrepo_dir) + super(WinRepo, self).__init__(opts, cache_root=winrepo_dir) def checkout(self): ''' diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py index c8942e4695..070a46fe75 100644 --- a/tests/unit/utils/test_gitfs.py +++ b/tests/unit/utils/test_gitfs.py @@ -66,7 +66,7 @@ class TestGitFSProvider(TestCase): ('git_pillar', salt.utils.gitfs.GitPillar), ('winrepo', salt.utils.gitfs.WinRepo)): key = '{0}_provider'.format(role_name) - for provider in salt.utils.gitfs.VALID_PROVIDERS: + for provider in salt.utils.gitfs.GIT_PROVIDERS: verify = 'verify_gitpython' mock1 = _get_mock(verify, provider) with patch.object(role_class, verify, mock1): From 630a1db3ab392fa76190beea620a3dcf2d851c6f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 29 Aug 2017 12:07:19 +0100 Subject: [PATCH 476/508] Include the line number by default on the log file format --- salt/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 0f06f9ccca..bd632b4116 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -55,7 +55,7 @@ _DFLT_LOG_DATEFMT = '%H:%M:%S' _DFLT_LOG_DATEFMT_LOGFILE = '%Y-%m-%d %H:%M:%S' _DFLT_LOG_FMT_CONSOLE = '[%(levelname)-8s] %(message)s' _DFLT_LOG_FMT_LOGFILE = ( - '%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s][%(process)d] %(message)s' + '%(asctime)s,%(msecs)03d [%(name)-17s:%(lineno)-4d][%(levelname)-8s][%(process)d] %(message)s' ) _DFLT_REFSPECS = ['+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/tags/*'] From 3c1ddc9bde546fdad3564ef6dddf41dc2ffee34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Wed, 23 Aug 2017 16:40:11 +0200 Subject: [PATCH 477/508] modules: iptables: correctly parse `--nfmask`/`--ctmask` `iptables-save` can return rules like this one: ``` -A PREROUTING -m connmark ! --mark 0x0/0xffff0000 -j CONNMARK \ --restore-mark --nfmask 0xffff0000 --ctmask 0xffff0000 ``` which leads to the following behavior: ``` $ salt '*' iptables.get_rules minion: Minion did not return. [No response] ``` This commit fixes the behavior of `iptables.get_rules` in this case, which also fixes the `iptables.append` state when such a rule already exists on the minion. --- salt/modules/iptables.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/iptables.py b/salt/modules/iptables.py index a96c975da7..563713ac92 100644 --- a/salt/modules/iptables.py +++ b/salt/modules/iptables.py @@ -1455,6 +1455,8 @@ def _parser(): add_arg('--or-mark', dest='or-mark', action='append') add_arg('--xor-mark', dest='xor-mark', action='append') add_arg('--set-mark', dest='set-mark', action='append') + add_arg('--nfmask', dest='nfmask', action='append') + add_arg('--ctmask', dest='ctmask', action='append') ## CONNSECMARK add_arg('--save', dest='save', action='append') add_arg('--restore', dest='restore', action='append') From c227cb25ad7f7025e661c71b9bb368a9c5b57c4c Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Tue, 29 Aug 2017 17:12:25 +0200 Subject: [PATCH 478/508] Skipping test on ImportError This most probably means that the Kubernetes client lib is not installed. --- tests/unit/modules/test_kubernetes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/modules/test_kubernetes.py b/tests/unit/modules/test_kubernetes.py index eda8a0a91b..1de939f6b0 100644 --- a/tests/unit/modules/test_kubernetes.py +++ b/tests/unit/modules/test_kubernetes.py @@ -16,10 +16,15 @@ from tests.support.mock import ( NO_MOCK_REASON ) -from salt.modules import kubernetes +try: + from salt.modules import kubernetes +except ImportError: + kubernetes = False @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(kubernetes is False, "Probably Kubernetes client lib is not installed. \ + Skipping test_kubernetes.py") class KubernetesTestCase(TestCase, LoaderModuleMockMixin): ''' Test cases for salt.modules.kubernetes From 23ec47c74cfa794ab61b485eadd3ce4c8b0bfeca Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 09:16:21 -0600 Subject: [PATCH 479/508] Add _ to regex search --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 03faa6ecfd..10854ef1f6 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1313,7 +1313,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Run Scheduled Task # Special handling for installing salt - if re.search(r'salt[\s-]*minion', + if re.search(r'salt[\s-_]*minion', pkg_name, flags=re.IGNORECASE + re.UNICODE) is not None: ret[pkg_name] = {'install status': 'task started'} From 869e8cc603d1d4580ef6a5f024d455a3dcd590d0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 28 Aug 2017 11:01:40 -0600 Subject: [PATCH 480/508] Fix `unit.fileserver.test_gitfs` for Windows Put `import pwd` in a try/except block Set `os.environ['USERNAME']` in windows using win_functions Add error function for `shutil.rmtree` --- tests/unit/fileserver/test_gitfs.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index 1920448e66..b3154638b7 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -9,8 +9,12 @@ import os import shutil import tempfile import textwrap -import pwd import logging +import stat +try: + import pwd +except ImportError: + pass # Import 3rd-party libs import yaml @@ -189,7 +193,6 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): self.integration_base_files = os.path.join(FILES, 'file', 'base') # Create the dir if it doesn't already exist - try: shutil.copytree(self.integration_base_files, self.tmp_repo_dir + '/') except OSError: @@ -203,7 +206,11 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): if 'USERNAME' not in os.environ: try: - os.environ['USERNAME'] = pwd.getpwuid(os.geteuid()).pw_name + if salt.utils.is_windows(): + import salt.utils.win_functions + os.environ['USERNAME'] = salt.utils.win_functions.get_current_user() + else: + os.environ['USERNAME'] = pwd.getpwuid(os.geteuid()).pw_name except AttributeError: log.error('Unable to get effective username, falling back to ' '\'root\'.') @@ -219,14 +226,18 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): Remove the temporary git repository and gitfs cache directory to ensure a clean environment for each test. ''' - shutil.rmtree(self.tmp_repo_dir) - shutil.rmtree(self.tmp_cachedir) - shutil.rmtree(self.tmp_sock_dir) + shutil.rmtree(self.tmp_repo_dir, onerror=self._rmtree_error) + shutil.rmtree(self.tmp_cachedir, onerror=self._rmtree_error) + shutil.rmtree(self.tmp_sock_dir, onerror=self._rmtree_error) del self.tmp_repo_dir del self.tmp_cachedir del self.tmp_sock_dir del self.integration_base_files + def _rmtree_error(self, func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + def test_file_list(self): ret = gitfs.file_list(LOAD) self.assertIn('testfile', ret) From c956d242830b2918fc2609454fe00600dba16093 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 10:15:09 -0600 Subject: [PATCH 481/508] Fix is_windows detection when USERNAME missing --- tests/unit/fileserver/test_gitfs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index b3154638b7..48cce5cf5a 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -206,6 +206,7 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): if 'USERNAME' not in os.environ: try: + import salt.utils if salt.utils.is_windows(): import salt.utils.win_functions os.environ['USERNAME'] = salt.utils.win_functions.get_current_user() From e5daff495acca2862b1deeeecffad277a6ca50d2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 28 Aug 2017 12:10:55 -0600 Subject: [PATCH 482/508] Fix pkg.install I jacked this up previously convert cmd to a list use cmd.extend --- salt/modules/win_pkg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index f1b51af308..39c2e59192 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1212,7 +1212,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): use_msiexec, msiexec = _get_msiexec(pkginfo[version_num].get('msiexec', False)) # Build cmd and arguments - # cmd and arguments must be seperated for use with the task scheduler + # cmd and arguments must be separated for use with the task scheduler if use_msiexec: cmd = msiexec arguments = ['/i', cached_pkg] @@ -1275,7 +1275,8 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): else: # Combine cmd and arguments - cmd = [cmd].extend(arguments) + cmd = [cmd] + cmd.extend(arguments) # Launch the command result = __salt__['cmd.run_all'](cmd, From ed030a35a50663f9da122c42f91ef0f210388642 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 28 Aug 2017 12:22:11 -0600 Subject: [PATCH 483/508] Use regex to detect salt-minion install --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 39c2e59192..89ba749aba 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1243,7 +1243,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Run Scheduled Task # Special handling for installing salt - if pkg_name in ['salt-minion', 'salt-minion-py3']: + if re.search(r'salt[\s-]*minion', pkg_name, flags=re.IGNORECASE+re.UNICODE) is not None: ret[pkg_name] = {'install status': 'task started'} if not __salt__['task.run'](name='update-salt-software'): log.error('Failed to install {0}'.format(pkg_name)) From 3cf2b6575c1e2a709186067a9a84f7583ed1ab1d Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 28 Aug 2017 12:39:47 -0600 Subject: [PATCH 484/508] Fix spelling --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 89ba749aba..b5efec590c 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -903,7 +903,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Version is ignored salt '*' pkg.install pkgs="['foo', 'bar']" version=1.2.3 - If passed with a comma seperated list in the ``name`` parameter, the + If passed with a comma separated list in the ``name`` parameter, the version will apply to all packages in the list. CLI Example: From 31ff69f0addf85f5a949a6b462932d7803d50949 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 09:13:44 -0600 Subject: [PATCH 485/508] Add underscore to regex search --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index b5efec590c..50a67dddde 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1243,7 +1243,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Run Scheduled Task # Special handling for installing salt - if re.search(r'salt[\s-]*minion', pkg_name, flags=re.IGNORECASE+re.UNICODE) is not None: + if re.search(r'salt[\s-_]*minion', pkg_name, flags=re.IGNORECASE+re.UNICODE) is not None: ret[pkg_name] = {'install status': 'task started'} if not __salt__['task.run'](name='update-salt-software'): log.error('Failed to install {0}'.format(pkg_name)) From 13dfabb1ce0ab506f1f564df442db2c1eb14f7ca Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 13:27:18 -0600 Subject: [PATCH 486/508] Fix regex statement, add `.` --- salt/modules/win_pkg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 50a67dddde..8e18ecbae3 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1243,7 +1243,9 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Run Scheduled Task # Special handling for installing salt - if re.search(r'salt[\s-_]*minion', pkg_name, flags=re.IGNORECASE+re.UNICODE) is not None: + if re.search(r'salt[\s_.-]*minion', + pkg_name, + flags=re.IGNORECASE+re.UNICODE) is not None: ret[pkg_name] = {'install status': 'task started'} if not __salt__['task.run'](name='update-salt-software'): log.error('Failed to install {0}'.format(pkg_name)) From e007a1c26ed2db642c55f539ec7c2ba127144a1c Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 13:57:28 -0600 Subject: [PATCH 487/508] Fix regex, add `.` --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 10854ef1f6..d3434cc2b7 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1313,7 +1313,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Run Scheduled Task # Special handling for installing salt - if re.search(r'salt[\s-_]*minion', + if re.search(r'salt[\s_.-]*minion', pkg_name, flags=re.IGNORECASE + re.UNICODE) is not None: ret[pkg_name] = {'install status': 'task started'} From 5185071d5ad07a3e93f84064f66f09e49cb1b4d1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 14:29:50 -0600 Subject: [PATCH 488/508] Skips `unit.modules.test_groupadd` on Windows There is a test_win_groupadd modules for testing the win_groupadd module on Windows. --- tests/unit/modules/test_groupadd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/modules/test_groupadd.py b/tests/unit/modules/test_groupadd.py index b836bd8805..29dfd15ed7 100644 --- a/tests/unit/modules/test_groupadd.py +++ b/tests/unit/modules/test_groupadd.py @@ -5,7 +5,10 @@ # Import Python libs from __future__ import absolute_import -import grp +try: + import grp +except ImportError: + pass # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -13,10 +16,12 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON # Import Salt Libs +import salt.utils import salt.modules.groupadd as groupadd @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(salt.utils.is_windows(), "Module not available on Windows") class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for salt.modules.groupadd From b401340e6c2b359da97b00cb086fde2cbb7a2da7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 15:59:08 -0600 Subject: [PATCH 489/508] Fix `unit.modules.test_inspect_collector` on Windows Uses os.sep instead of unix-style paths in the test Uses salt.utils.path.islink() to detect symlinks instead of os.path.islink(). os.path.islink() does not correctly detect symlinks in Windows Put grp and pwd imports inside a try/except block --- salt/modules/inspectlib/collector.py | 3 +- salt/modules/inspectlib/kiwiproc.py | 7 +++- tests/unit/modules/test_inspect_collector.py | 39 ++++++++++++++++---- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/salt/modules/inspectlib/collector.py b/salt/modules/inspectlib/collector.py index b87a46b82f..b8ebe7e804 100644 --- a/salt/modules/inspectlib/collector.py +++ b/salt/modules/inspectlib/collector.py @@ -29,6 +29,7 @@ from salt.modules.inspectlib.entities import (AllowedDir, IgnoredDir, Package, PayloadFile, PackageCfgFile) import salt.utils +import salt.utils.path from salt.utils import fsutils from salt.utils import reinit_crypto from salt.exceptions import CommandExecutionError @@ -311,7 +312,7 @@ class Inspector(EnvLoader): continue if not valid or not os.path.exists(obj) or not os.access(obj, os.R_OK): continue - if os.path.islink(obj): + if salt.utils.path.islink(obj): links.append(obj) elif os.path.isdir(obj): dirs.append(obj) diff --git a/salt/modules/inspectlib/kiwiproc.py b/salt/modules/inspectlib/kiwiproc.py index 136cacf00c..40b4f9c0bf 100644 --- a/salt/modules/inspectlib/kiwiproc.py +++ b/salt/modules/inspectlib/kiwiproc.py @@ -17,11 +17,14 @@ # Import python libs from __future__ import absolute_import import os -import grp -import pwd from xml.dom import minidom import platform import socket +try: + import grp + import pwd +except ImportError: + pass # Import salt libs import salt.utils diff --git a/tests/unit/modules/test_inspect_collector.py b/tests/unit/modules/test_inspect_collector.py index cdcb689eb7..0d37519a9e 100644 --- a/tests/unit/modules/test_inspect_collector.py +++ b/tests/unit/modules/test_inspect_collector.py @@ -49,9 +49,15 @@ class InspectorCollectorTestCase(TestCase): :return: ''' - inspector = Inspector(cachedir='/foo/cache', piddir='/foo/pid', pidfilename='bar.pid') - self.assertEqual(inspector.dbfile, '/foo/cache/_minion_collector.db') - self.assertEqual(inspector.pidfile, '/foo/pid/bar.pid') + cachedir = os.sep + os.sep.join(['foo', 'cache']) + piddir = os.sep + os.sep.join(['foo', 'pid']) + inspector = Inspector(cachedir=cachedir, piddir=piddir, pidfilename='bar.pid') + self.assertEqual( + inspector.dbfile, + os.sep + os.sep.join(['foo', 'cache', '_minion_collector.db'])) + self.assertEqual( + inspector.pidfile, + os.sep + os.sep.join(['foo', 'pid', 'bar.pid'])) def test_file_tree(self): ''' @@ -60,12 +66,29 @@ class InspectorCollectorTestCase(TestCase): :return: ''' - inspector = Inspector(cachedir='/test', piddir='/test', pidfilename='bar.pid') + inspector = Inspector(cachedir=os.sep + 'test', + piddir=os.sep + 'test', + pidfilename='bar.pid') tree_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'inspectlib', 'tree_test') - expected_tree = (['/a/a/dummy.a', '/a/b/dummy.b', '/b/b.1', '/b/b.2', '/b/b.3'], - ['/a', '/a/a', '/a/b', '/a/c', '/b', '/c'], - ['/a/a/dummy.ln.a', '/a/b/dummy.ln.b', '/a/c/b.1', '/b/b.4', - '/b/b.5', '/c/b.1', '/c/b.2', '/c/b.3']) + expected_tree = ([os.sep + os.sep.join(['a', 'a', 'dummy.a']), + os.sep + os.sep.join(['a', 'b', 'dummy.b']), + os.sep + os.sep.join(['b', 'b.1']), + os.sep + os.sep.join(['b', 'b.2']), + os.sep + os.sep.join(['b', 'b.3'])], + [os.sep + 'a', + os.sep + os.sep.join(['a', 'a']), + os.sep + os.sep.join(['a', 'b']), + os.sep + os.sep.join(['a', 'c']), + os.sep + 'b', + os.sep + 'c'], + [os.sep + os.sep.join(['a', 'a', 'dummy.ln.a']), + os.sep + os.sep.join(['a', 'b', 'dummy.ln.b']), + os.sep + os.sep.join(['a', 'c', 'b.1']), + os.sep + os.sep.join(['b', 'b.4']), + os.sep + os.sep.join(['b', 'b.5']), + os.sep + os.sep.join(['c', 'b.1']), + os.sep + os.sep.join(['c', 'b.2']), + os.sep + os.sep.join(['c', 'b.3'])]) tree_result = [] for chunk in inspector._get_all_files(tree_root): buff = [] From cec627a60bdbcd8de8662b3ab70ef46c9544df86 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Aug 2017 16:28:12 -0600 Subject: [PATCH 490/508] Skip mac tests for user and group They use grp and pwd --- tests/unit/modules/test_mac_group.py | 9 +++++++-- tests/unit/modules/test_mac_user.py | 25 +++++++++++++++---------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/unit/modules/test_mac_group.py b/tests/unit/modules/test_mac_group.py index 2c03deb357..d69288ccb9 100644 --- a/tests/unit/modules/test_mac_group.py +++ b/tests/unit/modules/test_mac_group.py @@ -5,11 +5,15 @@ # Import python libs from __future__ import absolute_import -import grp +HAS_GRP = True +try: + import grp +except ImportError: + HAS_GRP = False # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin -from tests.support.unit import TestCase +from tests.support.unit import TestCase, skipIf from tests.support.mock import MagicMock, patch # Import Salt Libs @@ -17,6 +21,7 @@ import salt.modules.mac_group as mac_group from salt.exceptions import SaltInvocationError, CommandExecutionError +@skipIf(not HAS_GRP, "Missing required library 'grp'") class MacGroupTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for the salt.modules.mac_group module diff --git a/tests/unit/modules/test_mac_user.py b/tests/unit/modules/test_mac_user.py index 51402e6cd0..c639f022da 100644 --- a/tests/unit/modules/test_mac_user.py +++ b/tests/unit/modules/test_mac_user.py @@ -2,10 +2,13 @@ ''' :codeauthor: :email:`Nicole Thomas ` ''' - # Import python libs from __future__ import absolute_import -import pwd +HAS_PWD = True +try: + import pwd +except ImportError: + HAS_PWD = False # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -17,6 +20,7 @@ import salt.modules.mac_user as mac_user from salt.exceptions import SaltInvocationError, CommandExecutionError +@skipIf(not HAS_PWD, "Missing required library 'pwd'") @skipIf(NO_MOCK, NO_MOCK_REASON) class MacUserTestCase(TestCase, LoaderModuleMockMixin): ''' @@ -26,14 +30,15 @@ class MacUserTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): return {mac_user: {}} - mock_pwall = [pwd.struct_passwd(('_amavisd', '*', 83, 83, 'AMaViS Daemon', - '/var/virusmails', '/usr/bin/false')), - pwd.struct_passwd(('_appleevents', '*', 55, 55, - 'AppleEvents Daemon', - '/var/empty', '/usr/bin/false')), - pwd.struct_passwd(('_appowner', '*', 87, 87, - 'Application Owner', - '/var/empty', '/usr/bin/false'))] + if HAS_PWD: + mock_pwall = [pwd.struct_passwd(('_amavisd', '*', 83, 83, 'AMaViS Daemon', + '/var/virusmails', '/usr/bin/false')), + pwd.struct_passwd(('_appleevents', '*', 55, 55, + 'AppleEvents Daemon', + '/var/empty', '/usr/bin/false')), + pwd.struct_passwd(('_appowner', '*', 87, 87, + 'Application Owner', + '/var/empty', '/usr/bin/false'))] mock_info_ret = {'shell': '/bin/bash', 'name': 'test', 'gid': 4376, 'groups': ['TEST_GROUP'], 'home': '/Users/foo', 'fullname': 'TEST USER', 'uid': 4376} From 83b0bab34ba2b6e1fc37797511240caf2b552f17 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 29 Aug 2017 16:11:40 -0600 Subject: [PATCH 491/508] opt_args needs to be a dict For napalm grains, otherwise there is an exception during bootup --- salt/grains/napalm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/grains/napalm.py b/salt/grains/napalm.py index 6ac52464c8..fcfbdcfe9f 100644 --- a/salt/grains/napalm.py +++ b/salt/grains/napalm.py @@ -447,8 +447,8 @@ def optional_args(proxy=None): device2: True ''' - opt_args = _get_device_grain('optional_args', proxy=proxy) - if _FORBIDDEN_OPT_ARGS: + opt_args = _get_device_grain('optional_args', proxy=proxy) or {} + if opt_args and _FORBIDDEN_OPT_ARGS: for arg in _FORBIDDEN_OPT_ARGS: opt_args.pop(arg, None) return {'optional_args': opt_args} From 04dd8ebedb99a2215ecb920b5657c2a40cbd3c85 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 30 Aug 2017 09:00:42 -0600 Subject: [PATCH 492/508] make sure meta-data grains work on ec2 We need to add on '/' for the end of the directories under latest. Then we should load anything that appears that we can that is json. The try except block is slightly faster than checking if the line starts with '{' --- salt/grains/metadata.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/salt/grains/metadata.py b/salt/grains/metadata.py index 2372aac6c7..dabffa2052 100644 --- a/salt/grains/metadata.py +++ b/salt/grains/metadata.py @@ -17,6 +17,7 @@ metadata server set `metadata_server_grains: True`. from __future__ import absolute_import # Import python libs +import json import os import socket @@ -47,14 +48,28 @@ def _search(prefix="latest/"): Recursively look up all grains in the metadata server ''' ret = {} - for line in http.query(os.path.join(HOST, prefix))['body'].split('\n'): + linedata = http.query(os.path.join(HOST, prefix)) + if 'body' not in linedata: + return ret + for line in linedata['body'].split('\n'): if line.endswith('/'): ret[line[:-1]] = _search(prefix=os.path.join(prefix, line)) + elif prefix == 'latest/': + # (gtmanfred) The first level should have a forward slash since + # they have stuff underneath. This will not be doubled up though, + # because lines ending with a slash are checked first. + ret[line] = _search(prefix=os.path.join(prefix, line + '/')) elif '=' in line: key, value = line.split('=') ret[value] = _search(prefix=os.path.join(prefix, key)) else: - ret[line] = http.query(os.path.join(HOST, prefix, line))['body'] + retdata = http.query(os.path.join(HOST, prefix, line)).get('body', None) + # (gtmanfred) This try except block is slightly faster than + # checking if the string starts with a curly brace + try: + ret[line] = json.loads(retdata) + except ValueError: + ret[line] = retdata return ret From 047ad07da4f1eb62324a138fab8e1a0e43545ded Mon Sep 17 00:00:00 2001 From: Timur Date: Tue, 29 Aug 2017 14:05:38 +0300 Subject: [PATCH 493/508] .utils.aws.get_location() expects a dict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … as it's `provider` argument. --- salt/utils/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/aws.py b/salt/utils/aws.py index 9e1212e92e..011dd346d7 100644 --- a/salt/utils/aws.py +++ b/salt/utils/aws.py @@ -392,7 +392,7 @@ def query(params=None, setname=None, requesturl=None, location=None, service_url = prov_dict.get('service_url', 'amazonaws.com') if not location: - location = get_location(opts, provider) + location = get_location(opts, prov_dict) if endpoint is None: if not requesturl: From 382bf92de730210af1d5944fd7cb53df9f8f4de4 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 30 Aug 2017 09:36:36 -0600 Subject: [PATCH 494/508] switch virtualbox cloud driver to use __utils__ --- salt/cloud/clouds/virtualbox.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/salt/cloud/clouds/virtualbox.py b/salt/cloud/clouds/virtualbox.py index 903722fd39..4266ce5f0e 100644 --- a/salt/cloud/clouds/virtualbox.py +++ b/salt/cloud/clouds/virtualbox.py @@ -24,7 +24,6 @@ import logging # Import salt libs from salt.exceptions import SaltCloudSystemExit import salt.config as config -import salt.utils.cloud as cloud # Import Third Party Libs try: @@ -136,7 +135,7 @@ def create(vm_info): ) log.debug("Going to fire event: starting create") - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'starting create', 'salt/cloud/{0}/creating'.format(vm_info['name']), @@ -151,7 +150,7 @@ def create(vm_info): 'clone_from': vm_info['clonefrom'] } - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'requesting instance', 'salt/cloud/{0}/requesting'.format(vm_info['name']), @@ -174,10 +173,10 @@ def create(vm_info): vm_info['key_filename'] = key_filename vm_info['ssh_host'] = ip - res = cloud.bootstrap(vm_info, __opts__) + res = __utils__['cloud.bootstrap'](vm_info) vm_result.update(res) - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'created machine', 'salt/cloud/{0}/created'.format(vm_info['name']), @@ -269,7 +268,7 @@ def list_nodes(kwargs=None, call=None): "private_ips", "public_ips", ] - return cloud.list_nodes_select( + return __utils__['cloud.list_nodes_select']( list_nodes_full('function'), attributes, call, ) @@ -278,7 +277,7 @@ def list_nodes_select(call=None): """ Return a list of the VMs that are on the provider, with select fields """ - return cloud.list_nodes_select( + return __utils__['cloud.list_nodes_select']( list_nodes_full('function'), __opts__['query.selection'], call, ) @@ -306,7 +305,7 @@ def destroy(name, call=None): if not vb_machine_exists(name): return "{0} doesn't exist and can't be deleted".format(name) - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'destroying instance', 'salt/cloud/{0}/destroying'.format(name), @@ -317,7 +316,7 @@ def destroy(name, call=None): vb_destroy_machine(name) - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'destroyed instance', 'salt/cloud/{0}/destroyed'.format(name), From 91b062f5646289a5ae168250b0539e14ced04b34 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 30 Aug 2017 10:41:21 -0600 Subject: [PATCH 495/508] Fix formatting issue, spaces surrounding + --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 8e18ecbae3..f66bd762ee 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1245,7 +1245,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Special handling for installing salt if re.search(r'salt[\s_.-]*minion', pkg_name, - flags=re.IGNORECASE+re.UNICODE) is not None: + flags=re.IGNORECASE + re.UNICODE) is not None: ret[pkg_name] = {'install status': 'task started'} if not __salt__['task.run'](name='update-salt-software'): log.error('Failed to install {0}'.format(pkg_name)) From d533877743bb9a7411a035c1cb8cbcd866e27790 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 30 Aug 2017 14:35:04 -0500 Subject: [PATCH 496/508] Use six.integer_types instead of int This catches longs on PY2. --- salt/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 41f06fbb9a..26320b7258 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1963,7 +1963,7 @@ def is_true(value=None): pass # Now check for truthiness - if isinstance(value, (int, float)): + if isinstance(value, (six.integer_types, float)): return value > 0 elif isinstance(value, six.string_types): return str(value).lower() == 'true' @@ -2735,7 +2735,7 @@ def repack_dictlist(data, if val_cb is None: val_cb = lambda x, y: y - valid_non_dict = (six.string_types, int, float) + valid_non_dict = (six.string_types, six.integer_types, float) if isinstance(data, list): for element in data: if isinstance(element, valid_non_dict): From c4ae2de30ffda3a7e81802f508c2ac8cc8514ff7 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 30 Aug 2017 20:38:50 -0600 Subject: [PATCH 497/508] bootstrap can come from dunders --- salt/utils/cloud.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index 4677fde631..4b84bd59c4 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -293,12 +293,14 @@ def salt_config_to_yaml(configuration, line_break='\n'): Dumper=SafeOrderedDumper) -def bootstrap(vm_, opts): +def bootstrap(vm_, opts=None): ''' This is the primary entry point for logging into any system (POSIX or Windows) to install Salt. It will make the decision on its own as to which deploy function to call. ''' + if opts is None: + opts = __opts__ deploy_config = salt.config.get_cloud_config_value( 'deploy', vm_, opts, default=False) From 2b4da0f0e72aec43dc0dcf4fc5237297acf8fd46 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 30 Aug 2017 18:08:40 -0400 Subject: [PATCH 498/508] Add CODEOWNERS file --- .github/CODEOWNERS | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..29288c6efe --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,60 @@ +# SALTSTACK CODE OWNERS + +# See https://help.github.com/articles/about-codeowners/ +# for more info about CODEOWNERS file + +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. + +# See https://help.github.com/articles/about-codeowners/ +# for more info about the CODEOWNERS file + +# Team Boto +salt/**/*boto* @saltstack/team-boto + +# Team Core +salt/auth/ @saltstack/team-core +salt/cache/ @saltstack/team-core +salt/cli/ @saltstack/team-core +salt/client/* @saltstack/team-core +salt/config/* @saltstack/team-core +salt/daemons/ @saltstack/team-core +salt/pillar/ @saltstack/team-core +salt/loader.py @saltstack/team-core +salt/payload.py @saltstack/team-core +salt/**/master* @saltstack/team-core +salt/**/minion* @saltstack/team-core + +# Team Cloud +salt/cloud/ @saltstack/team-cloud +salt/utils/openstack/ @saltstack/team-cloud +salt/utils/aws.py @saltstack/team-cloud +salt/**/*cloud* @saltstack/team-cloud + +# Team NetAPI +salt/cli/api.py @saltstack/team-netapi +salt/client/netapi.py @saltstack/team-netapi +salt/netapi/ @saltstack/team-netapi + +# Team Network +salt/proxy/ @saltstack/team-proxy + +# Team SPM +salt/cli/spm.py @saltstack/team-spm +salt/spm/ @saltstack/team-spm + +# Team SSH +salt/cli/ssh.py @saltstack/team-ssh +salt/client/ssh/ @saltstack/team-ssh +salt/runners/ssh.py @saltstack/team-ssh +salt/**/thin.py @saltstack/team-ssh + +# Team State +salt/state.py @saltstack/team-state + +# Team Transport +salt/transport/ @saltstack/team-transport +salt/utils/zeromq.py @saltstack/team-transport + +# Team Windows +salt/**/*win* @saltstack/team-windows From 1f104cf85b33d414408b131c79f3f85f30143497 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 31 Aug 2017 16:50:22 +0300 Subject: [PATCH 499/508] Fix ldap token groups auth. --- salt/auth/__init__.py | 34 +++++++++++++++++++++------------- salt/auth/ldap.py | 4 ++-- salt/daemons/masterapi.py | 21 +++------------------ salt/master.py | 21 +++------------------ 4 files changed, 29 insertions(+), 51 deletions(-) diff --git a/salt/auth/__init__.py b/salt/auth/__init__.py index f90488e153..e39ecf8373 100644 --- a/salt/auth/__init__.py +++ b/salt/auth/__init__.py @@ -200,7 +200,7 @@ class LoadAuth(object): ''' if not self.authenticate_eauth(load): return {} - fstr = '{0}.auth'.format(load['eauth']) + hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5')) tok = str(hash_type(os.urandom(512)).hexdigest()) t_path = os.path.join(self.opts['token_dir'], tok) @@ -224,8 +224,9 @@ class LoadAuth(object): acl_ret = self.__get_acl(load) tdata['auth_list'] = acl_ret - if 'groups' in load: - tdata['groups'] = load['groups'] + groups = self.get_groups(load) + if groups: + tdata['groups'] = groups try: with salt.utils.files.set_umask(0o177): @@ -345,7 +346,7 @@ class LoadAuth(object): return False return True - def get_auth_list(self, load): + def get_auth_list(self, load, token=None): ''' Retrieve access list for the user specified in load. The list is built by eauth module or from master eauth configuration. @@ -353,30 +354,37 @@ class LoadAuth(object): list if the user has no rights to execute anything on this master and returns non-empty list if user is allowed to execute particular functions. ''' + # Get auth list from token + if token and self.opts['keep_acl_in_token'] and 'auth_list' in token: + return token['auth_list'] # Get acl from eauth module. auth_list = self.__get_acl(load) if auth_list is not None: return auth_list - if load['eauth'] not in self.opts['external_auth']: + eauth = token['eauth'] if token else load['eauth'] + if eauth not in self.opts['external_auth']: # No matching module is allowed in config log.warning('Authorization failure occurred.') return None - name = self.load_name(load) # The username we are attempting to auth with - groups = self.get_groups(load) # The groups this user belongs to - eauth_config = self.opts['external_auth'][load['eauth']] - if groups is None or groups is False: + if token: + name = token['name'] + groups = token['groups'] + else: + name = self.load_name(load) # The username we are attempting to auth with + groups = self.get_groups(load) # The groups this user belongs to + eauth_config = self.opts['external_auth'][eauth] + if not groups: groups = [] group_perm_keys = [item for item in eauth_config if item.endswith('%')] # The configured auth groups # First we need to know if the user is allowed to proceed via any of their group memberships. group_auth_match = False for group_config in group_perm_keys: - group_config = group_config.rstrip('%') - for group in groups: - if group == group_config: - group_auth_match = True + if group_config.rstrip('%') in groups: + group_auth_match = True + break # If a group_auth_match is set it means only that we have a # user which matches at least one or more of the groups defined # in the configuration file. diff --git a/salt/auth/ldap.py b/salt/auth/ldap.py index 396c1d00a2..3065429815 100644 --- a/salt/auth/ldap.py +++ b/salt/auth/ldap.py @@ -306,7 +306,7 @@ def groups(username, **kwargs): ''' group_list = [] - bind = _bind(username, kwargs['password'], + bind = _bind(username, kwargs.get('password'), anonymous=_config('anonymous', mandatory=False)) if bind: log.debug('ldap bind to determine group membership succeeded!') @@ -371,7 +371,7 @@ def groups(username, **kwargs): search_results = bind.search_s(search_base, ldap.SCOPE_SUBTREE, search_string, - [_config('accountattributename'), 'cn']) + [_config('accountattributename'), 'cn', _config('groupattribute')]) for _, entry in search_results: if username in entry[_config('accountattributename')]: group_list.append(entry['cn'][0]) diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 9ca6c582fb..d47a5c3aa6 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -1055,12 +1055,7 @@ class LocalFuncs(object): return dict(error=dict(name=err_name, message='Authentication failure of type "token" occurred.')) username = token['name'] - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - load['eauth'] = token['eauth'] - load['username'] = username - auth_list = self.loadauth.get_auth_list(load) + auth_list = self.loadauth.get_auth_list(load, token) else: auth_type = 'eauth' err_name = 'EauthAuthenticationError' @@ -1102,12 +1097,7 @@ class LocalFuncs(object): return dict(error=dict(name=err_name, message='Authentication failure of type "token" occurred.')) username = token['name'] - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - load['eauth'] = token['eauth'] - load['username'] = username - auth_list = self.loadauth.get_auth_list(load) + auth_list = self.loadauth.get_auth_list(load, token) elif 'eauth' in load: auth_type = 'eauth' err_name = 'EauthAuthenticationError' @@ -1217,12 +1207,7 @@ class LocalFuncs(object): return '' # Get acl from eauth module. - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - extra['eauth'] = token['eauth'] - extra['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(extra) + auth_list = self.loadauth.get_auth_list(extra, token) # Authorize the request if not self.ckminions.auth_check( diff --git a/salt/master.py b/salt/master.py index 649a89a072..b913aeb1e5 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1705,12 +1705,7 @@ class ClearFuncs(object): message='Authentication failure of type "token" occurred.')) # Authorize - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - clear_load['eauth'] = token['eauth'] - clear_load['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(clear_load) + auth_list = self.loadauth.get_auth_list(clear_load, token) if not self.ckminions.runner_check(auth_list, clear_load['fun']): return dict(error=dict(name='TokenAuthenticationError', @@ -1774,12 +1769,7 @@ class ClearFuncs(object): message='Authentication failure of type "token" occurred.')) # Authorize - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - clear_load['eauth'] = token['eauth'] - clear_load['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(clear_load) + auth_list = self.loadauth.get_auth_list(clear_load, token) if not self.ckminions.wheel_check(auth_list, clear_load['fun']): return dict(error=dict(name='TokenAuthenticationError', message=('Authentication failure of type "token" occurred for ' @@ -1900,12 +1890,7 @@ class ClearFuncs(object): return '' # Get acl - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - extra['eauth'] = token['eauth'] - extra['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(extra) + auth_list = self.loadauth.get_auth_list(extra, token) # Authorize the request if not self.ckminions.auth_check( From 3ad6911210d02b758c253e60a02312e167627396 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 31 Aug 2017 20:39:35 +0300 Subject: [PATCH 500/508] Fix for tests: don't require 'groups' in the eauth token. --- salt/auth/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/auth/__init__.py b/salt/auth/__init__.py index e39ecf8373..73e4c98f8a 100644 --- a/salt/auth/__init__.py +++ b/salt/auth/__init__.py @@ -370,7 +370,7 @@ class LoadAuth(object): if token: name = token['name'] - groups = token['groups'] + groups = token.get('groups') else: name = self.load_name(load) # The username we are attempting to auth with groups = self.get_groups(load) # The groups this user belongs to From a5d9f85db6b4665ab74d2fa1b24aff37fd26a816 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 30 Aug 2017 18:05:17 -0600 Subject: [PATCH 501/508] Modifications to build scripts Py3 installs to c:\python35 Quote the path to the python install directory Fix spellings in comments Move pywin32 dlls to site-packages/win32 instead of python root Remove pywin32 postinstall and testing scripts Add passive switch to py3 uninstall --- pkg/windows/build.bat | 2 +- pkg/windows/build_env_2.ps1 | 24 +++++++++++++++++++++--- pkg/windows/build_env_3.ps1 | 8 ++++++-- pkg/windows/build_pkg.bat | 2 +- pkg/windows/clean_env.bat | 6 +++--- pkg/windows/modules/get-settings.psm1 | 6 +++--- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index 0117718539..59fafde137 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -89,7 +89,7 @@ if Defined x ( if %Python%==2 ( Set "PyDir=C:\Python27" ) else ( - Set "PyDir=C:\Program Files\Python35" + Set "PyDir=C:\Python35" ) Set "PATH=%PATH%;%PyDir%;%PyDir%\Scripts" diff --git a/pkg/windows/build_env_2.ps1 b/pkg/windows/build_env_2.ps1 index 98a922ca3d..b186517812 100644 --- a/pkg/windows/build_env_2.ps1 +++ b/pkg/windows/build_env_2.ps1 @@ -175,7 +175,7 @@ If (Test-Path "$($ini['Settings']['Python2Dir'])\python.exe") { DownloadFileWithProgress $url $file Write-Output " - $script_name :: Installing $($ini[$bitPrograms]['Python2']) . . ." - $p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,SharedCRT,Extensions,pip_feature,PrependPath TARGETDIR=$($ini['Settings']['Python2Dir'])" -Wait -NoNewWindow -PassThru + $p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,SharedCRT,Extensions,pip_feature,PrependPath TARGETDIR=`"$($ini['Settings']['Python2Dir'])`"" -Wait -NoNewWindow -PassThru } #------------------------------------------------------------------------------ @@ -191,7 +191,7 @@ If (!($Path.ToLower().Contains("$($ini['Settings']['Scripts2Dir'])".ToLower()))) #============================================================================== # Update PIP and SetupTools -# caching depends on environmant variable SALT_PIP_LOCAL_CACHE +# caching depends on environment variable SALT_PIP_LOCAL_CACHE #============================================================================== Write-Output " ----------------------------------------------------------------" Write-Output " - $script_name :: Updating PIP and SetupTools . . ." @@ -212,7 +212,7 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) { #============================================================================== # Install pypi resources using pip -# caching depends on environmant variable SALT_REQ_LOCAL_CACHE +# caching depends on environment variable SALT_REQ_LOCAL_CACHE #============================================================================== Write-Output " ----------------------------------------------------------------" Write-Output " - $script_name :: Installing pypi resources using pip . . ." @@ -230,6 +230,24 @@ if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_2.txt" "pip install" } +#============================================================================== +# Move PyWin32 DLL's to site-packages\win32 +#============================================================================== +Write-Output " - $script_name :: Moving PyWin32 DLLs . . ." +Move-Item "$($ini['Settings']['SitePkgs2Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['SitePkgs2Dir'])\win32" -Force + +# Remove pywin32_system32 directory +Write-Output " - $script_name :: Removing pywin32_system32 Directory . . ." +Remove-Item "$($ini['Settings']['SitePkgs2Dir'])\pywin32_system32" + +# Remove pythonwin directory +Write-Output " - $script_name :: Removing pythonwin Directory . . ." +Remove-Item "$($ini['Settings']['SitePkgs2Dir'])\pythonwin" -Force -Recurse + +# Remove PyWin32 PostInstall and testall Scripts +Write-Output " - $script_name :: Removing PyWin32 scripts . . ." +Remove-Item "$($ini['Settings']['Scripts2Dir'])\pywin32_*" -Force -Recurse + #============================================================================== # Install PyYAML with CLoader # This has to be a compiled binary to get the CLoader diff --git a/pkg/windows/build_env_3.ps1 b/pkg/windows/build_env_3.ps1 index 33f95871ae..0dcbafd996 100644 --- a/pkg/windows/build_env_3.ps1 +++ b/pkg/windows/build_env_3.ps1 @@ -175,7 +175,7 @@ If (Test-Path "$($ini['Settings']['Python3Dir'])\python.exe") { DownloadFileWithProgress $url $file Write-Output " - $script_name :: Installing $($ini[$bitPrograms]['Python3']) . . ." - $p = Start-Process $file -ArgumentList '/passive InstallAllUsers=1 TargetDir="C:\Program Files\Python35" Include_doc=0 Include_tcltk=0 Include_test=0 Include_launcher=0 PrependPath=1 Shortcuts=0' -Wait -NoNewWindow -PassThru + $p = Start-Process $file -ArgumentList "/passive InstallAllUsers=1 TargetDir=`"$($ini['Settings']['Python3Dir'])`" Include_doc=0 Include_tcltk=0 Include_test=0 Include_launcher=0 PrependPath=1 Shortcuts=0" -Wait -NoNewWindow -PassThru } #------------------------------------------------------------------------------ @@ -247,7 +247,7 @@ Start_Process_and_test_exitcode "$($ini['Settings']['Scripts3Dir'])\pip.exe" "i # Move DLL's to Python Root Write-Output " - $script_name :: Moving PyWin32 DLLs . . ." -Move-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['Python3Dir'])" -Force +Move-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['SitePkgs3Dir'])\win32" -Force # Remove pywin32_system32 directory Write-Output " - $script_name :: Removing pywin32_system32 Directory . . ." @@ -257,6 +257,10 @@ Remove-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32" Write-Output " - $script_name :: Removing pythonwin Directory . . ." Remove-Item "$($ini['Settings']['SitePkgs3Dir'])\pythonwin" -Force -Recurse +# Remove PyWin32 PostInstall and testall Scripts +Write-Output " - $script_name :: Removing PyWin32 scripts . . ." +Remove-Item "$($ini['Settings']['Scripts3Dir'])\pywin32_*" -Force -Recurse + #============================================================================== # Fix PyCrypto #============================================================================== diff --git a/pkg/windows/build_pkg.bat b/pkg/windows/build_pkg.bat index 0d30f047ac..95b185bfa7 100644 --- a/pkg/windows/build_pkg.bat +++ b/pkg/windows/build_pkg.bat @@ -56,7 +56,7 @@ if %Python%==2 ( Set "PyVerMajor=2" Set "PyVerMinor=7" ) else ( - Set "PyDir=C:\Program Files\Python35" + Set "PyDir=C:\Python35" Set "PyVerMajor=3" Set "PyVerMinor=5" ) diff --git a/pkg/windows/clean_env.bat b/pkg/windows/clean_env.bat index fb6e63a661..d474264a53 100644 --- a/pkg/windows/clean_env.bat +++ b/pkg/windows/clean_env.bat @@ -17,7 +17,7 @@ if %errorLevel%==0 ( echo. if exist "\Python27" goto RemovePython2 -if exist "\Program Files\Python35" goto RemovePython3 +if exist "\Python35" goto RemovePython3 goto eof :RemovePython2 @@ -53,13 +53,13 @@ goto eof :: 64 bit if exist "%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}" ( echo %0 :: - 3.5.3 64bit - "%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}\python-3.5.3-amd64.exe" /uninstall + "%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}\python-3.5.3-amd64.exe" /uninstall /passive ) :: 32 bit if exist "%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}" ( echo %0 :: - 3.5.3 32bit - "%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}\python-3.5.3" /uninstall + "%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}\python-3.5.3" /uninstall /passive ) rem wipe the Python directory diff --git a/pkg/windows/modules/get-settings.psm1 b/pkg/windows/modules/get-settings.psm1 index 292732cb83..5c57738fd3 100644 --- a/pkg/windows/modules/get-settings.psm1 +++ b/pkg/windows/modules/get-settings.psm1 @@ -19,9 +19,9 @@ Function Get-Settings { "Python2Dir" = "C:\Python27" "Scripts2Dir" = "C:\Python27\Scripts" "SitePkgs2Dir" = "C:\Python27\Lib\site-packages" - "Python3Dir" = "C:\Program Files\Python35" - "Scripts3Dir" = "C:\Program Files\Python35\Scripts" - "SitePkgs3Dir" = "C:\Program Files\Python35\Lib\site-packages" + "Python3Dir" = "C:\Python35" + "Scripts3Dir" = "C:\Python35\Scripts" + "SitePkgs3Dir" = "C:\Python35\Lib\site-packages" "DownloadDir" = "$env:Temp\DevSalt" } # The script deletes the DownLoadDir (above) for each install. From d4214ca283cf7b69548816ea6bd5513977b47505 Mon Sep 17 00:00:00 2001 From: Nathan Fish Date: Thu, 31 Aug 2017 13:44:12 -0500 Subject: [PATCH 502/508] file.py docs: specify absolute paths --- salt/states/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 128fc7165a..3acea8f129 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -1154,7 +1154,7 @@ def managed(name, the salt master and potentially run through a templating system. name - The location of the file to manage + The location of the file to manage, as an absolute path. source The source file to download to the minion, this source file can be @@ -2041,7 +2041,7 @@ def directory(name, Ensure that a named directory is present and has the right perms name - The location to create or manage a directory + The location to create or manage a directory, as an absolute path user The user to own the directory; this defaults to the user salt is From 14a45918549cc17b6c4edc938448c33a5045fbf5 Mon Sep 17 00:00:00 2001 From: Nathan Fish Date: Thu, 31 Aug 2017 13:59:07 -0500 Subject: [PATCH 503/508] file.py docs: correct group and mode --- salt/states/file.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 3acea8f129..ffc6f51c35 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -1324,13 +1324,15 @@ def managed(name, group The group ownership set for the file, this defaults to the group salt - is running as on the minion On Windows, this is ignored + is running as on the minion. On Windows, this is ignored mode - The permissions to set on this file, e.g. ``644``, ``0775``, or ``4664``. + The permissions to set on this file, e.g. ``644``, ``0775``, or + ``4664``. - The default mode for new files and directories corresponds umask of salt - process. For existing files and directories it's not enforced. + The default mode for new files and directories corresponds to the + umask of the salt process. The mode of existing files and directories + will only be changed if ``mode`` is specified. .. note:: This option is **not** supported on Windows. From 9979ccb613f32a3277caed5f21437c9a8eac5e88 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 31 Aug 2017 14:15:06 -0600 Subject: [PATCH 504/508] Remove Py2 and Py3 in the same run --- pkg/windows/clean_env.bat | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/windows/clean_env.bat b/pkg/windows/clean_env.bat index d474264a53..7c5e497802 100644 --- a/pkg/windows/clean_env.bat +++ b/pkg/windows/clean_env.bat @@ -16,9 +16,10 @@ if %errorLevel%==0 ( ) echo. +:CheckPython2 if exist "\Python27" goto RemovePython2 -if exist "\Python35" goto RemovePython3 -goto eof + +goto CheckPython3 :RemovePython2 rem Uninstall Python 2.7 @@ -47,6 +48,11 @@ goto eof goto eof +:CheckPython3 +if exist "\Python35" goto RemovePython3 + +goto eof + :RemovePython3 echo %0 :: Uninstalling Python 3 ... echo --------------------------------------------------------------------- @@ -63,9 +69,9 @@ goto eof ) rem wipe the Python directory - echo %0 :: Removing the C:\Program Files\Python35 Directory ... + echo %0 :: Removing the C:\Python35 Directory ... echo --------------------------------------------------------------------- - rd /s /q "C:\Program Files\Python35" + rd /s /q "C:\Python35" if %errorLevel%==0 ( echo Successful ) else ( From 3fbf24b91a1f2d4613fe951b05f14ec5552873a8 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 31 Aug 2017 16:56:58 -0600 Subject: [PATCH 505/508] Use os.sep instead of '/' --- salt/netapi/rest_cherrypy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/netapi/rest_cherrypy/__init__.py b/salt/netapi/rest_cherrypy/__init__.py index 6285de289c..d974cda6c1 100644 --- a/salt/netapi/rest_cherrypy/__init__.py +++ b/salt/netapi/rest_cherrypy/__init__.py @@ -20,7 +20,7 @@ try: except ImportError as exc: cpy_error = exc -__virtualname__ = os.path.abspath(__file__).rsplit('/')[-2] or 'rest_cherrypy' +__virtualname__ = os.path.abspath(__file__).rsplit(os.sep)[-2] or 'rest_cherrypy' logger = logging.getLogger(__virtualname__) cpy_min = '3.2.2' From c93d2ed386182dbe85d6d62ae17d985930db4858 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 31 Aug 2017 17:01:14 -0600 Subject: [PATCH 506/508] Use os.sep instead of '/' --- salt/netapi/rest_tornado/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py index fc547a02a3..2437714073 100644 --- a/salt/netapi/rest_tornado/__init__.py +++ b/salt/netapi/rest_tornado/__init__.py @@ -10,7 +10,7 @@ import os import salt.auth from salt.utils.versions import StrictVersion as _StrictVersion -__virtualname__ = os.path.abspath(__file__).rsplit('/')[-2] or 'rest_tornado' +__virtualname__ = os.path.abspath(__file__).rsplit(os.sep)[-2] or 'rest_tornado' logger = logging.getLogger(__virtualname__) From ec94a137506f0289bbf0d9c2db4352dc57cb7d2b Mon Sep 17 00:00:00 2001 From: Nathan Fish Date: Fri, 1 Sep 2017 10:20:46 -0500 Subject: [PATCH 507/508] cron docs: Remind user to use quotes for special strings Otherwise @ signs are stripped. https://github.com/saltstack/salt/issues/2896 --- salt/states/cron.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/states/cron.py b/salt/states/cron.py index 381bb173ab..efd1039baf 100644 --- a/salt/states/cron.py +++ b/salt/states/cron.py @@ -116,7 +116,7 @@ entry on the minion already contains a numeric value, then using the ``random`` keyword will not modify it. Added the opportunity to set a job with a special keyword like '@reboot' or -'@hourly'. +'@hourly'. Quotes must be used, otherwise PyYAML will strip the '@' sign. .. code-block:: yaml @@ -302,7 +302,8 @@ def present(name, edits. This defaults to the state id special - A special keyword to specify periodicity (eg. @reboot, @hourly...) + A special keyword to specify periodicity (eg. @reboot, @hourly...). + Quotes must be used, otherwise PyYAML will strip the '@' sign. .. versionadded:: 2016.3.0 ''' @@ -388,7 +389,8 @@ def absent(name, edits. This defaults to the state id special - The special keyword used in the job (eg. @reboot, @hourly...) + The special keyword used in the job (eg. @reboot, @hourly...). + Quotes must be used, otherwise PyYAML will strip the '@' sign. ''' ### NOTE: The keyword arguments in **kwargs are ignored in this state, but ### cannot be removed from the function definition, otherwise the use From 5bd5ea042a83a265b77f3ac8073c3a6e53137e73 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 1 Sep 2017 14:49:50 -0600 Subject: [PATCH 508/508] Fix `unit.modules.test_chef` for Windows Mocks the __opts__ to contain cachedir --- tests/unit/modules/test_chef.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/test_chef.py b/tests/unit/modules/test_chef.py index 0899ce3a47..03a892a241 100644 --- a/tests/unit/modules/test_chef.py +++ b/tests/unit/modules/test_chef.py @@ -36,7 +36,8 @@ class ChefTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it execute a chef client run and return a dict ''' - self.assertDictEqual(chef.client(), {}) + with patch.dict(chef.__opts__, {'cachedir': r'c:\salt\var\cache\salt\minion'}): + self.assertDictEqual(chef.client(), {}) # 'solo' function tests: 1 @@ -44,4 +45,5 @@ class ChefTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it execute a chef solo run and return a dict ''' - self.assertDictEqual(chef.solo('/dev/sda1'), {}) + with patch.dict(chef.__opts__, {'cachedir': r'c:\salt\var\cache\salt\minion'}): + self.assertDictEqual(chef.solo('/dev/sda1'), {})