From 3602af1e1bd7a2badae4365f2ae9ba57891a566a Mon Sep 17 00:00:00 2001 From: Andreas Ulm Date: Wed, 28 Jun 2017 00:32:20 +0200 Subject: [PATCH 001/300] Fix rabbitmqctl output handler for 3.6.10 Fix the output sanatizer for rabbitmqctl output generated by rabbitmq-server 3.6.10 and later. #41955 Signed-off-by: Andreas Ulm --- salt/modules/rabbitmq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index 801237789d..4cfdeb4d3d 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -72,6 +72,7 @@ def _safe_output(line): ''' return not any([ line.startswith('Listing') and line.endswith('...'), + line.startswith('Listing') and '\t' not in line, '...done' in line, line.startswith('WARNING:') ]) From 76fd941d915c518f3565c19f05037c767af8b04f Mon Sep 17 00:00:00 2001 From: Andreas Ulm Date: Wed, 28 Jun 2017 01:13:30 +0200 Subject: [PATCH 002/300] added tests for rabbitmq 3.6.10 output handler Signed-off-by: Andreas Ulm --- tests/unit/modules/rabbitmq_test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/modules/rabbitmq_test.py b/tests/unit/modules/rabbitmq_test.py index b0a665fc11..661a4cd9c8 100644 --- a/tests/unit/modules/rabbitmq_test.py +++ b/tests/unit/modules/rabbitmq_test.py @@ -52,6 +52,17 @@ class RabbitmqTestCase(TestCase): with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}): self.assertDictEqual(rabbitmq.list_users(), {'guest': ['administrator', 'user']}) + # 'list_users_rabbitmq4' function tests: 1 + + def test_list_users_rabbitmq4(self): + ''' + Test if it return a list of users based off of rabbitmqctl user_list. + Output changed in rabbitmq-server version 3.6.10 + ''' + mock_run = MagicMock(return_value='Listing users\nguest\t[administrator user]\n') + with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}): + self.assertDictEqual(rabbitmq.list_users(), {'guest': ['administrator', 'user']}) + # 'list_users_with_warning_rabbitmq2' function tests: 1 def test_list_users_with_warning_rabbitmq2(self): @@ -220,6 +231,19 @@ class RabbitmqTestCase(TestCase): self.assertDictEqual(rabbitmq.list_user_permissions('myuser'), {'saltstack': ['saltstack']}) + # 'list_user_permissions2' function tests: 1 + + def test_list_user_permissions2(self): + ''' + Test if it list permissions for a user + via rabbitmqctl list_user_permissions. + Output changed in rabbitmq-server version 3.6.10 + ''' + mock_run = MagicMock(return_value='Listing stuff\nsaltstack\tsaltstack\n...done') + with patch.dict(rabbitmq.__salt__, {'cmd.run': mock_run}): + self.assertDictEqual(rabbitmq.list_user_permissions('myuser'), + {'saltstack': ['saltstack']}) + # 'set_user_tags' function tests: 1 def test_set_user_tags(self): From 1cc2aa503a8e451e3c7f2be382593bd9298c2372 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Wed, 28 Jun 2017 15:59:17 +0100 Subject: [PATCH 003/300] Fix dockerng.network_* ignoring of tests=True Fixes #41976 --- salt/states/dockerng.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 3826cce670..78e6483b9f 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2123,6 +2123,11 @@ def network_present(name, driver=None, containers=None): ret['result'] = result else: + if __opts__['test']: + ret['result'] = None + ret['comment'] = ('The network \'{0}\' will be created'.format(name)) + return ret + try: ret['changes']['created'] = __salt__['dockerng.create_network']( name, driver=driver) @@ -2169,6 +2174,11 @@ def network_absent(name, driver=None): ret['comment'] = 'Network \'{0}\' already absent'.format(name) return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = ('The network \'{0}\' will be removed'.format(name)) + return ret + for container in networks[0]['Containers']: try: ret['changes']['disconnected'] = __salt__['dockerng.disconnect_container_from_network'](container, name) From 515c612808485f9a089ebb185d96441416e7922c Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Wed, 28 Jun 2017 17:28:22 +0100 Subject: [PATCH 004/300] Fix dockerng.network_* name matching Updating the states so that they loop through the list of networks returned from `dockerng.networks` to check that one of the networks returned fully matches the name of the network that needs to be ensured to be present or absent. Another option would be to add an argument to the `dockerng.networks` execution module to enable strict matching. This would probably be preferable to repeating the logic in the states, but I didn't want to meddle too much as the execution module is currently just wrapping the docker-py library. Fixes #41982 --- salt/states/dockerng.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 17559800f7..124e45f3e1 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2242,8 +2242,17 @@ def network_present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['dockerng.inspect_container'](c)['Id'] for c in containers] networks = __salt__['dockerng.networks'](names=[name]) + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None if networks: - network = networks[0] # we expect network's name to be unique + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is not None: if all(c in network['Containers'] for c in containers): ret['result'] = True ret['comment'] = 'Network \'{0}\' already exists.'.format(name) @@ -2302,7 +2311,17 @@ def network_absent(name, driver=None): 'comment': ''} networks = __salt__['dockerng.networks'](names=[name]) - if not networks: + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None + if networks: + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is None: ret['result'] = True ret['comment'] = 'Network \'{0}\' already absent'.format(name) return ret From 8c00c63b55006887861207b6e4716db0f69f8ed4 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Thu, 29 Jun 2017 13:36:29 +0100 Subject: [PATCH 005/300] Fix dockerng.network_* name matching Updating the states so that they loop through the list of networks returned from `dockerng.networks` to check that one of the networks returned fully matches the name of the network that needs to be ensured to be present or absent. Another option would be to add an argument to the `dockerng.networks` execution module to enable strict matching. This would probably be preferable to repeating the logic in the states, but I didn't want to meddle too much as the execution module is currently just wrapping the docker-py library. Fixes #41982 --- salt/states/docker_network.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/salt/states/docker_network.py b/salt/states/docker_network.py index 9d78d66b21..a10162ba9d 100644 --- a/salt/states/docker_network.py +++ b/salt/states/docker_network.py @@ -90,8 +90,17 @@ def present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['docker.inspect_container'](c)['Id'] for c in containers] networks = __salt__['docker.networks'](names=[name]) + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None if networks: - network = networks[0] # we expect network's name to be unique + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is not None: if all(c in network['Containers'] for c in containers): ret['result'] = True ret['comment'] = 'Network \'{0}\' already exists.'.format(name) @@ -150,7 +159,17 @@ def absent(name, driver=None): 'comment': ''} networks = __salt__['docker.networks'](names=[name]) - if not networks: + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None + if networks: + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is None: ret['result'] = True ret['comment'] = 'Network \'{0}\' already absent'.format(name) return ret From 494765e93950b2c8d41f63f8461a36943481ef44 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 26 Jun 2017 12:26:50 -0700 Subject: [PATCH 006/300] Updating the git module to allow an identity file to be used when passing the user parameter --- salt/modules/git.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index d164cd6cef..7cc2ddcc6c 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -170,13 +170,24 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, identity = [identity] # try each of the identities, independently + tmp_identity_file = None for id_file in identity: if 'salt://' in id_file: + tmp_identity_file = salt.utils.files.mkstemp() _id_file = id_file - id_file = __salt__['cp.cache_file'](id_file, saltenv) + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) if not id_file: log.error('identity {0} does not exist.'.format(_id_file)) + __salt__['file.remove'](tmp_identity_file) continue + else: + __salt__['file.set_mode'](id_file, '0600') + if user: + os.chown(id_file, + __salt__['file.user_to_uid'](user), + -1) else: if not __salt__['file.file_exists'](id_file): missing_keys.append(id_file) @@ -247,6 +258,11 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, if not salt.utils.is_windows() and 'GIT_SSH' in env: os.remove(env['GIT_SSH']) + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) + # If the command was successful, no need to try additional IDs if result['retcode'] == 0: return result From 8faa9f6d92bf7957c94d3e87d5604e804330291d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 26 Jun 2017 14:49:54 -0700 Subject: [PATCH 007/300] Updating PR with requested changes. --- salt/modules/git.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 7cc2ddcc6c..c50e61afa9 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,11 +169,15 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] + # Override umask + if not salt.utils.is_windows(): + _old_umask = os.umask(0o077) + # try each of the identities, independently tmp_identity_file = None for id_file in identity: if 'salt://' in id_file: - tmp_identity_file = salt.utils.files.mkstemp() + tmp_identity_file = salt.utils.mkstemp() _id_file = id_file id_file = __salt__['cp.get_file'](id_file, tmp_identity_file, @@ -183,7 +187,6 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, __salt__['file.remove'](tmp_identity_file) continue else: - __salt__['file.set_mode'](id_file, '0600') if user: os.chown(id_file, __salt__['file.user_to_uid'](user), @@ -255,13 +258,17 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, redirect_stderr=redirect_stderr, **kwargs) finally: - if not salt.utils.is_windows() and 'GIT_SSH' in env: - os.remove(env['GIT_SSH']) + if not salt.utils.is_windows(): + # Reset the umask + os.umask(_old_umask) - # Cleanup the temporary identify file - if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) + if 'GIT_SSH' in env: + os.remove(env['GIT_SSH']) + + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From 4949bf3ff322aa27028aa0af2b16cb0748ae0117 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 29 Jun 2017 10:08:58 -0700 Subject: [PATCH 008/300] Updating to swap on the new salt.utils.files.set_umask context_manager --- salt/modules/git.py | 189 +++++++++++++++++++++----------------------- 1 file changed, 91 insertions(+), 98 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index c50e61afa9..161de3d1d5 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,114 +169,107 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] - # Override umask - if not salt.utils.is_windows(): - _old_umask = os.umask(0o077) - - # try each of the identities, independently - tmp_identity_file = None - for id_file in identity: - if 'salt://' in id_file: - tmp_identity_file = salt.utils.mkstemp() - _id_file = id_file - id_file = __salt__['cp.get_file'](id_file, - tmp_identity_file, - saltenv) - if not id_file: - log.error('identity {0} does not exist.'.format(_id_file)) - __salt__['file.remove'](tmp_identity_file) - continue + with salt.utils.files.set_umask(0o177): + # try each of the identities, independently + tmp_identity_file = None + for id_file in identity: + if 'salt://' in id_file: + tmp_identity_file = salt.utils.mkstemp() + _id_file = id_file + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) + if not id_file: + log.error('identity {0} does not exist.'.format(_id_file)) + __salt__['file.remove'](tmp_identity_file) + continue + else: + if user: + os.chown(id_file, + __salt__['file.user_to_uid'](user), + -1) else: - if user: - os.chown(id_file, - __salt__['file.user_to_uid'](user), - -1) - else: - if not __salt__['file.file_exists'](id_file): - missing_keys.append(id_file) - log.error('identity {0} does not exist.'.format(id_file)) - continue + if not __salt__['file.file_exists'](id_file): + missing_keys.append(id_file) + log.error('identity {0} does not exist.'.format(id_file)) + continue - env = { - 'GIT_IDENTITY': id_file - } + env = { + 'GIT_IDENTITY': id_file + } - # copy wrapper to area accessible by ``runas`` user - # currently no suppport in windows for wrapping git ssh - ssh_id_wrapper = os.path.join( - salt.utils.templates.TEMPLATE_DIRNAME, - 'git/ssh-id-wrapper' - ) - if salt.utils.is_windows(): - for suffix in ('', ' (x86)'): - ssh_exe = ( - 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' - .format(suffix) - ) - if os.path.isfile(ssh_exe): - env['GIT_SSH_EXE'] = ssh_exe - break - else: - raise CommandExecutionError( - 'Failed to find ssh.exe, unable to use identity file' - ) - # Use the windows batch file instead of the bourne shell script - ssh_id_wrapper += '.bat' - env['GIT_SSH'] = ssh_id_wrapper - else: - tmp_file = salt.utils.mkstemp() - salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) - os.chmod(tmp_file, 0o500) - os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) - env['GIT_SSH'] = tmp_file - - if 'salt-call' not in _salt_cli \ - and __salt__['ssh.key_is_encrypted'](id_file): - errors.append( - 'Identity file {0} is passphrase-protected and cannot be ' - 'used in a non-interactive command. Using salt-call from ' - 'the minion will allow a passphrase-protected key to be ' - 'used.'.format(id_file) + # copy wrapper to area accessible by ``runas`` user + # currently no suppport in windows for wrapping git ssh + ssh_id_wrapper = os.path.join( + salt.utils.templates.TEMPLATE_DIRNAME, + 'git/ssh-id-wrapper' ) - continue + if salt.utils.is_windows(): + for suffix in ('', ' (x86)'): + ssh_exe = ( + 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' + .format(suffix) + ) + if os.path.isfile(ssh_exe): + env['GIT_SSH_EXE'] = ssh_exe + break + else: + raise CommandExecutionError( + 'Failed to find ssh.exe, unable to use identity file' + ) + # Use the windows batch file instead of the bourne shell script + ssh_id_wrapper += '.bat' + env['GIT_SSH'] = ssh_id_wrapper + else: + tmp_file = salt.utils.mkstemp() + salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) + os.chmod(tmp_file, 0o500) + os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) + env['GIT_SSH'] = tmp_file - log.info( - 'Attempting git authentication using identity file {0}' - .format(id_file) - ) + if 'salt-call' not in _salt_cli \ + and __salt__['ssh.key_is_encrypted'](id_file): + errors.append( + 'Identity file {0} is passphrase-protected and cannot be ' + 'used in a non-interactive command. Using salt-call from ' + 'the minion will allow a passphrase-protected key to be ' + 'used.'.format(id_file) + ) + continue - try: - result = __salt__['cmd.run_all']( - command, - cwd=cwd, - runas=user, - password=password, - env=env, - python_shell=False, - log_callback=salt.utils.url.redact_http_basic_auth, - ignore_retcode=ignore_retcode, - redirect_stderr=redirect_stderr, - **kwargs) - finally: - if not salt.utils.is_windows(): - # Reset the umask - os.umask(_old_umask) + log.info( + 'Attempting git authentication using identity file {0}' + .format(id_file) + ) - if 'GIT_SSH' in env: + try: + result = __salt__['cmd.run_all']( + command, + cwd=cwd, + runas=user, + password=password, + env=env, + python_shell=False, + log_callback=salt.utils.url.redact_http_basic_auth, + ignore_retcode=ignore_retcode, + redirect_stderr=redirect_stderr, + **kwargs) + finally: + if not salt.utils.is_windows() and 'GIT_SSH' in env: os.remove(env['GIT_SSH']) - # Cleanup the temporary identify file - if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) - # If the command was successful, no need to try additional IDs - if result['retcode'] == 0: - return result - else: - err = result['stdout' if redirect_stderr else 'stderr'] - if err: - errors.append(salt.utils.url.redact_http_basic_auth(err)) + # If the command was successful, no need to try additional IDs + if result['retcode'] == 0: + return result + else: + err = result['stdout' if redirect_stderr else 'stderr'] + if err: + errors.append(salt.utils.url.redact_http_basic_auth(err)) # We've tried all IDs and still haven't passed, so error out if failhard: From 68549f3496d5784a762b3aef4b0923ae059ec272 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 29 Jun 2017 10:15:50 -0700 Subject: [PATCH 009/300] Fixing umask to we can set files as executable. --- salt/modules/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 161de3d1d5..53bea307d1 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,7 +169,7 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] - with salt.utils.files.set_umask(0o177): + with salt.utils.files.set_umask(0o077): # try each of the identities, independently tmp_identity_file = None for id_file in identity: From 26beb18aa505bcc196fbd303c895c95d8f8a4680 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 29 Jun 2017 11:35:28 -0600 Subject: [PATCH 010/300] Set concurrent to True when running states with sudo Fixes #25842 --- salt/executors/sudo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/executors/sudo.py b/salt/executors/sudo.py index d3321847a5..825fee979a 100644 --- a/salt/executors/sudo.py +++ b/salt/executors/sudo.py @@ -73,7 +73,7 @@ class SudoExecutor(ModuleExecutorBase): '-c', salt.syspaths.CONFIG_DIR, '--', data.get('fun')] - if data['fun'] == 'state.sls': + if data['fun'] in ('state.sls', 'state.highstate', 'state.apply'): kwargs['concurrent'] = True for arg in args: self.cmd.append(_cmd_quote(str(arg))) From 1b6026177c09e2bf852c4c8cda1dc3da1653c7f2 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 29 Jun 2017 13:04:33 -0500 Subject: [PATCH 011/300] Restrict set_umask to mkstemp call only --- salt/modules/git.py | 190 ++++++++++++++++++++++---------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 53bea307d1..7b0f37f3f4 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -169,107 +169,107 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # force it into a list identity = [identity] - with salt.utils.files.set_umask(0o077): - # try each of the identities, independently - tmp_identity_file = None - for id_file in identity: - if 'salt://' in id_file: + # try each of the identities, independently + tmp_identity_file = None + for id_file in identity: + if 'salt://' in id_file: + with salt.utils.files.set_umask(0o077): tmp_identity_file = salt.utils.mkstemp() - _id_file = id_file - id_file = __salt__['cp.get_file'](id_file, - tmp_identity_file, - saltenv) - if not id_file: - log.error('identity {0} does not exist.'.format(_id_file)) - __salt__['file.remove'](tmp_identity_file) - continue - else: - if user: - os.chown(id_file, - __salt__['file.user_to_uid'](user), - -1) + _id_file = id_file + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) + if not id_file: + log.error('identity {0} does not exist.'.format(_id_file)) + __salt__['file.remove'](tmp_identity_file) + continue else: - if not __salt__['file.file_exists'](id_file): - missing_keys.append(id_file) - log.error('identity {0} does not exist.'.format(id_file)) - continue - - env = { - 'GIT_IDENTITY': id_file - } - - # copy wrapper to area accessible by ``runas`` user - # currently no suppport in windows for wrapping git ssh - ssh_id_wrapper = os.path.join( - salt.utils.templates.TEMPLATE_DIRNAME, - 'git/ssh-id-wrapper' - ) - if salt.utils.is_windows(): - for suffix in ('', ' (x86)'): - ssh_exe = ( - 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' - .format(suffix) - ) - if os.path.isfile(ssh_exe): - env['GIT_SSH_EXE'] = ssh_exe - break - else: - raise CommandExecutionError( - 'Failed to find ssh.exe, unable to use identity file' - ) - # Use the windows batch file instead of the bourne shell script - ssh_id_wrapper += '.bat' - env['GIT_SSH'] = ssh_id_wrapper - else: - tmp_file = salt.utils.mkstemp() - salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) - os.chmod(tmp_file, 0o500) - os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) - env['GIT_SSH'] = tmp_file - - if 'salt-call' not in _salt_cli \ - and __salt__['ssh.key_is_encrypted'](id_file): - errors.append( - 'Identity file {0} is passphrase-protected and cannot be ' - 'used in a non-interactive command. Using salt-call from ' - 'the minion will allow a passphrase-protected key to be ' - 'used.'.format(id_file) - ) + if user: + os.chown(id_file, + __salt__['file.user_to_uid'](user), + -1) + else: + if not __salt__['file.file_exists'](id_file): + missing_keys.append(id_file) + log.error('identity {0} does not exist.'.format(id_file)) continue - log.info( - 'Attempting git authentication using identity file {0}' - .format(id_file) - ) + env = { + 'GIT_IDENTITY': id_file + } - try: - result = __salt__['cmd.run_all']( - command, - cwd=cwd, - runas=user, - password=password, - env=env, - python_shell=False, - log_callback=salt.utils.url.redact_http_basic_auth, - ignore_retcode=ignore_retcode, - redirect_stderr=redirect_stderr, - **kwargs) - finally: - if not salt.utils.is_windows() and 'GIT_SSH' in env: - os.remove(env['GIT_SSH']) - - # Cleanup the temporary identify file - if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) - - # If the command was successful, no need to try additional IDs - if result['retcode'] == 0: - return result + # copy wrapper to area accessible by ``runas`` user + # currently no suppport in windows for wrapping git ssh + ssh_id_wrapper = os.path.join( + salt.utils.templates.TEMPLATE_DIRNAME, + 'git/ssh-id-wrapper' + ) + if salt.utils.is_windows(): + for suffix in ('', ' (x86)'): + ssh_exe = ( + 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' + .format(suffix) + ) + if os.path.isfile(ssh_exe): + env['GIT_SSH_EXE'] = ssh_exe + break else: - err = result['stdout' if redirect_stderr else 'stderr'] - if err: - errors.append(salt.utils.url.redact_http_basic_auth(err)) + raise CommandExecutionError( + 'Failed to find ssh.exe, unable to use identity file' + ) + # Use the windows batch file instead of the bourne shell script + ssh_id_wrapper += '.bat' + env['GIT_SSH'] = ssh_id_wrapper + else: + tmp_file = salt.utils.mkstemp() + salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) + os.chmod(tmp_file, 0o500) + os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) + env['GIT_SSH'] = tmp_file + + if 'salt-call' not in _salt_cli \ + and __salt__['ssh.key_is_encrypted'](id_file): + errors.append( + 'Identity file {0} is passphrase-protected and cannot be ' + 'used in a non-interactive command. Using salt-call from ' + 'the minion will allow a passphrase-protected key to be ' + 'used.'.format(id_file) + ) + continue + + log.info( + 'Attempting git authentication using identity file {0}' + .format(id_file) + ) + + try: + result = __salt__['cmd.run_all']( + command, + cwd=cwd, + runas=user, + password=password, + env=env, + python_shell=False, + log_callback=salt.utils.url.redact_http_basic_auth, + ignore_retcode=ignore_retcode, + redirect_stderr=redirect_stderr, + **kwargs) + finally: + if not salt.utils.is_windows() and 'GIT_SSH' in env: + os.remove(env['GIT_SSH']) + + # Cleanup the temporary identify file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identify file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) + + # If the command was successful, no need to try additional IDs + if result['retcode'] == 0: + return result + else: + err = result['stdout' if redirect_stderr else 'stderr'] + if err: + errors.append(salt.utils.url.redact_http_basic_auth(err)) # We've tried all IDs and still haven't passed, so error out if failhard: From 44841e5626defb0e330372d344554dab2b2c432d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 29 Jun 2017 11:22:34 -0700 Subject: [PATCH 012/300] Moving the call to cp.get_file inside the with block to ensure the umask is preserved when we grab the file. --- salt/modules/git.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 7b0f37f3f4..4f1101b755 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -175,10 +175,10 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, if 'salt://' in id_file: with salt.utils.files.set_umask(0o077): tmp_identity_file = salt.utils.mkstemp() - _id_file = id_file - id_file = __salt__['cp.get_file'](id_file, - tmp_identity_file, - saltenv) + _id_file = id_file + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) if not id_file: log.error('identity {0} does not exist.'.format(_id_file)) __salt__['file.remove'](tmp_identity_file) @@ -261,7 +261,7 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # Cleanup the temporary identify file if tmp_identity_file and os.path.exists(tmp_identity_file): log.debug('Removing identify file {0}'.format(tmp_identity_file)) - __salt__['file.remove'](tmp_identity_file) + #__salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From 26f848e1115fdcdd471ad863ffc604678dd6c304 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 29 Jun 2017 14:49:39 -0500 Subject: [PATCH 013/300] Mock socket.getaddrinfo in unit.utils.network_test.NetworkTestCase.test_host_to_ips This will keep future IP address changes from impacting this test. It also tests a host which returns an IPv6 address, and a host which cannot be resolved. --- tests/unit/utils/network_test.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils/network_test.py b/tests/unit/utils/network_test.py index a02675c92f..7a3d278f66 100644 --- a/tests/unit/utils/network_test.py +++ b/tests/unit/utils/network_test.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Import Python libs from __future__ import absolute_import +import socket # Import Salt Testing libs from salttesting import skipIf @@ -111,8 +112,31 @@ class NetworkTestCase(TestCase): changed. In these cases, we just need to update the IP address in the assertion. ''' - ret = network.host_to_ips('www.saltstack.com') - self.assertEqual(ret, ['104.197.168.128']) + def _side_effect(host, *args): + try: + return { + 'github.com': [ + (2, 1, 6, '', ('192.30.255.112', 0)), + (2, 1, 6, '', ('192.30.255.113', 0)), + ], + 'ipv6host.foo': [ + (10, 1, 6, '', ('2001:a71::1', 0, 0, 0)), + ], + }[host] + except KeyError: + raise socket.gaierror(-2, 'Name or service not known') + + getaddrinfo_mock = MagicMock(side_effect=_side_effect) + with patch.object(socket, 'getaddrinfo', getaddrinfo_mock): + # Test host that can be resolved + ret = network.host_to_ips('github.com') + self.assertEqual(ret, ['192.30.255.112', '192.30.255.113']) + # Test ipv6 + ret = network.host_to_ips('ipv6host.foo') + self.assertEqual(ret, ['2001:a71::1']) + # Test host that can't be resolved + ret = network.host_to_ips('someothersite.com') + self.assertEqual(ret, None) def test_generate_minion_id(self): self.assertTrue(network.generate_minion_id()) From 0ef6cf634c43d2d8265fe76bda3cc434532a519d Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 30 Jun 2017 10:57:47 +0100 Subject: [PATCH 014/300] Add trace logging of dockerng.networks result Added during debugging of failing test to determine what dockerng.networks is expected to return, keeping as it seems useful. --- salt/states/dockerng.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 124e45f3e1..2facade3ab 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2242,6 +2242,9 @@ def network_present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['dockerng.inspect_container'](c)['Id'] for c in containers] networks = __salt__['dockerng.networks'](names=[name]) + log.trace( + 'dockerng.network_present: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. @@ -2311,6 +2314,9 @@ def network_absent(name, driver=None): 'comment': ''} networks = __salt__['dockerng.networks'](names=[name]) + log.trace( + 'dockerng.network_present: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. From 3369f0072ff9afb22b083a66b42cbcafd6051895 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 30 Jun 2017 11:01:10 +0100 Subject: [PATCH 015/300] Fix broken unit test test_network_absent This started failing following commit 515c612, which relied on the 'Name' key being present in the return value of dockerng.networks - as the mock didn't have this set the test started failing. --- tests/unit/states/dockerng_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/dockerng_test.py b/tests/unit/states/dockerng_test.py index 884776bfb8..c8d5dddef0 100644 --- a/tests/unit/states/dockerng_test.py +++ b/tests/unit/states/dockerng_test.py @@ -724,9 +724,13 @@ class DockerngTestCase(TestCase): ''' dockerng_remove_network = Mock(return_value='removed') dockerng_disconnect_container_from_network = Mock(return_value='disconnected') + dockerng_networks = Mock(return_value=[{ + 'Name': 'network_foo', + 'Containers': {'container': {}} + }]) __salt__ = {'dockerng.remove_network': dockerng_remove_network, 'dockerng.disconnect_container_from_network': dockerng_disconnect_container_from_network, - 'dockerng.networks': Mock(return_value=[{'Containers': {'container': {}}}]), + 'dockerng.networks': dockerng_networks, } with patch.dict(dockerng_state.__dict__, {'__salt__': __salt__}): From 9eea796da8bdab8e25b141c7d3aa7041d422e387 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 30 Jun 2017 11:03:45 +0100 Subject: [PATCH 016/300] Add regression tests for #41982 These test the scenarios where another network with a similar name already exists, verifying that network_absent doesn't attempt to remove a network which isn't specified, and network_present still attempts to create the specified network despite a similarly named network already being present. --- tests/unit/states/dockerng_test.py | 40 +++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/dockerng_test.py b/tests/unit/states/dockerng_test.py index c8d5dddef0..d8e409cefa 100644 --- a/tests/unit/states/dockerng_test.py +++ b/tests/unit/states/dockerng_test.py @@ -698,10 +698,18 @@ class DockerngTestCase(TestCase): dockerng_create_network = Mock(return_value='created') dockerng_connect_container_to_network = Mock(return_value='connected') dockerng_inspect_container = Mock(return_value={'Id': 'abcd'}) + # Get dockerng.networks to return a network with a name which is a superset of the name of + # the network which is to be created, despite this network existing we should still expect + # that the new network will be created. + # Regression test for #41982. + dockerng_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) __salt__ = {'dockerng.create_network': dockerng_create_network, 'dockerng.inspect_container': dockerng_inspect_container, 'dockerng.connect_container_to_network': dockerng_connect_container_to_network, - 'dockerng.networks': Mock(return_value=[]), + 'dockerng.networks': dockerng_networks, } with patch.dict(dockerng_state.__dict__, {'__salt__': __salt__}): @@ -746,6 +754,36 @@ class DockerngTestCase(TestCase): 'removed': 'removed'}, 'result': True}) + def test_network_absent_with_matching_network(self): + ''' + Test dockerng.network_absent when the specified network does not exist, + but another network with a name which is a superset of the specified + name does exist. In this case we expect there to be no attempt to remove + any network. + Regression test for #41982. + ''' + dockerng_remove_network = Mock(return_value='removed') + dockerng_disconnect_container_from_network = Mock(return_value='disconnected') + dockerng_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) + __salt__ = {'dockerng.remove_network': dockerng_remove_network, + 'dockerng.disconnect_container_from_network': dockerng_disconnect_container_from_network, + 'dockerng.networks': dockerng_networks, + } + with patch.dict(dockerng_state.__dict__, + {'__salt__': __salt__}): + ret = dockerng_state.network_absent( + 'network_foo', + ) + dockerng_disconnect_container_from_network.assert_not_called() + dockerng_remove_network.assert_not_called() + self.assertEqual(ret, {'name': 'network_foo', + 'comment': 'Network \'network_foo\' already absent', + 'changes': {}, + 'result': True}) + def test_volume_present(self): ''' Test dockerng.volume_present From 1de5e008a06f2c73b73f5c7fde7200920dba7348 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 30 Jun 2017 11:32:10 -0400 Subject: [PATCH 017/300] Add initial 2016.11.7 Release Notes --- doc/topics/releases/2016.11.7.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/topics/releases/2016.11.7.rst diff --git a/doc/topics/releases/2016.11.7.rst b/doc/topics/releases/2016.11.7.rst new file mode 100644 index 0000000000..6ef9587133 --- /dev/null +++ b/doc/topics/releases/2016.11.7.rst @@ -0,0 +1,5 @@ +============================ +Salt 2016.11.7 Release Notes +============================ + +Version 2016.11.7 is a bugfix release for :ref:`2016.11.0 `. From e20cea635079ee1b66d28996b3502dca40ba1f88 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Fri, 30 Jun 2017 11:21:49 -0600 Subject: [PATCH 018/300] Upgrade support for gnupg v2.1 and higher --- salt/modules/debbuild.py | 115 ++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/salt/modules/debbuild.py b/salt/modules/debbuild.py index 2cb9deed2a..2124cbf259 100644 --- a/salt/modules/debbuild.py +++ b/salt/modules/debbuild.py @@ -60,17 +60,22 @@ def __virtual__(): return (False, 'The debbuild module could not be loaded: unsupported OS family') -def _check_repo_sign_utils_support(): - util_name = 'debsign' - if salt.utils.which(util_name): +def _check_repo_sign_utils_support(name): + ''' + Check for specified command name in search path + ''' + if salt.utils.which(name): return True else: raise CommandExecutionError( - 'utility \'{0}\' needs to be installed'.format(util_name) + 'utility \'{0}\' needs to be installed or made available in search path'.format(name) ) def _check_repo_gpg_phrase_utils_support(): + ''' + Check for /usr/lib/gnupg2/gpg-preset-passphrase is installed + ''' util_name = '/usr/lib/gnupg2/gpg-preset-passphrase' if __salt__['file.file_exists'](util_name): return True @@ -170,8 +175,8 @@ def _get_repo_dists_env(env): 'ORIGIN': ('O', 'Origin', 'SaltStack'), 'LABEL': ('O', 'Label', 'salt_debian'), 'SUITE': ('O', 'Suite', 'stable'), - 'VERSION': ('O', 'Version', '8.1'), - 'CODENAME': ('M', 'Codename', 'jessie'), + 'VERSION': ('O', 'Version', '9.0'), + 'CODENAME': ('M', 'Codename', 'stretch'), 'ARCHS': ('M', 'Architectures', 'i386 amd64 source'), 'COMPONENTS': ('M', 'Components', 'main'), 'DESCRIPTION': ('O', 'Description', 'SaltStack debian package repo'), @@ -205,7 +210,7 @@ def _get_repo_dists_env(env): else: env_dists += '{0}: {1}\n'.format(key, value) - ## ensure mandatories are included + # ensure mandatories are included env_keys = list(env.keys()) for key in env_keys: if key in dflts_keys and dflts_dict[key][0] == 'M' and key not in env_man_seen: @@ -312,7 +317,7 @@ def make_src_pkg(dest_dir, spec, sources, env=None, template=None, saltenv='base for src in sources: _get_src(tree_base, src, saltenv) - #.dsc then assumes sources already build + # .dsc then assumes sources already build if spec_pathfile.endswith('.dsc'): for efile in os.listdir(tree_base): full = os.path.join(tree_base, efile) @@ -578,7 +583,8 @@ def make_repo(repodir, with salt.utils.fopen(repoconfopts, 'w') as fow: fow.write('{0}'.format(repocfg_opts)) - local_fingerprint = None + local_keygrip_to_use = None + local_key_fingerprint = None local_keyid = None phrase = '' @@ -587,17 +593,14 @@ def make_repo(repodir, gpg_tty_info_file = '{0}/gpg-tty-info-salt'.format(gnupghome) gpg_tty_info_dict = {} - # test if using older than gnupg 2.1, env file exists + # if using older than gnupg 2.1, then env file exists older_gnupg = __salt__['file.file_exists'](gpg_info_file) - # interval of 0.125 is really too fast on some systems - interval = 0.5 - if keyid is not None: with salt.utils.fopen(repoconfdist, 'a') as fow: fow.write('SignWith: {0}\n'.format(keyid)) - ## import_keys + # import_keys pkg_pub_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_pub_keyname', None)) pkg_priv_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_priv_keyname', None)) @@ -621,21 +624,37 @@ def make_repo(repodir, local_keys = __salt__['gpg.list_keys'](user=runas, gnupghome=gnupghome) for gpg_key in local_keys: if keyid == gpg_key['keyid'][8:]: - local_fingerprint = gpg_key['fingerprint'] + local_keygrip_to_use = gpg_key['fingerprint'] + local_key_fingerprint = gpg_key['fingerprint'] local_keyid = gpg_key['keyid'] break + if not older_gnupg: + _check_repo_sign_utils_support('gpg2') + cmd = '{0} --with-keygrip --list-secret-keys'.format(salt.utils.which('gpg2')) + local_keys2_keygrip = __salt__['cmd.run'](cmd, runas=runas) + local_keys2 = iter(local_keys2_keygrip.splitlines()) + try: + for line in local_keys2: + if line.startswith('sec'): + line_fingerprint = next(local_keys2).lstrip().rstrip() + if local_key_fingerprint == line_fingerprint: + lkeygrip = next(local_keys2).split('=') + local_keygrip_to_use = lkeygrip[1].lstrip().rstrip() + break + except StopIteration: + raise SaltInvocationError( + 'unable to find keygrip associated with fingerprint \'{0}\' for keyid \'{1}\'' + .format(local_key_fingerprint, local_keyid) + ) + if local_keyid is None: raise SaltInvocationError( 'The key ID \'{0}\' was not found in GnuPG keyring at \'{1}\'' .format(keyid, gnupghome) ) - _check_repo_sign_utils_support() - - if use_passphrase: - _check_repo_gpg_phrase_utils_support() - phrase = __salt__['pillar.get']('gpg_passphrase') + _check_repo_sign_utils_support('debsign') if older_gnupg: with salt.utils.fopen(gpg_info_file, 'r') as fow: @@ -656,10 +675,30 @@ def make_repo(repodir, __salt__['environ.setenv'](gpg_tty_info_dict) break - ## sign_it_here - for file in os.listdir(repodir): - if file.endswith('.dsc'): - abs_file = os.path.join(repodir, file) + if use_passphrase: + _check_repo_gpg_phrase_utils_support() + phrase = __salt__['pillar.get']('gpg_passphrase') + cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --preset --passphrase "{0}" {1}'.format(phrase, local_keygrip_to_use) + __salt__['cmd.run'](cmd, runas=runas) + + for debfile in os.listdir(repodir): + abs_file = os.path.join(repodir, debfile) + if debfile.endswith('.changes'): + os.remove(abs_file) + + if debfile.endswith('.dsc'): + # sign_it_here + if older_gnupg: + if local_keyid is not None: + cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file) + __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) + + cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(codename, abs_file) + __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) + else: + # interval of 0.125 is really too fast on some systems + interval = 0.5 + if local_keyid is not None: number_retries = timeout / interval times_looped = 0 error_msg = 'Failed to debsign file {0}'.format(abs_file) @@ -702,27 +741,6 @@ def make_repo(repodir, finally: proc.close(terminate=True, kill=True) - if use_passphrase: - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --forget {0}'.format(local_fingerprint) - __salt__['cmd.run'](cmd, runas=runas) - - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --preset --passphrase "{0}" {1}'.format(phrase, local_fingerprint) - __salt__['cmd.run'](cmd, runas=runas) - - for debfile in os.listdir(repodir): - abs_file = os.path.join(repodir, debfile) - if debfile.endswith('.changes'): - os.remove(abs_file) - - if debfile.endswith('.dsc'): - if older_gnupg: - if local_keyid is not None: - cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file) - __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) - - cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(codename, abs_file) - __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) - else: number_retries = timeout / interval times_looped = 0 error_msg = 'Failed to reprepro includedsc file {0}'.format(abs_file) @@ -747,8 +765,7 @@ def make_repo(repodir, if times_looped > number_retries: raise SaltInvocationError( - 'Attemping to reprepro includedsc for file {0} failed, timed out after {1} loops' - .format(abs_file, int(times_looped * interval)) + 'Attemping to reprepro includedsc for file {0} failed, timed out after {1} loops'.format(abs_file, times_looped) ) time.sleep(interval) @@ -770,8 +787,4 @@ def make_repo(repodir, cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedeb {0} {1}'.format(codename, abs_file) res = __salt__['cmd.run_all'](cmd, cwd=repodir, use_vt=True) - if use_passphrase and local_keyid is not None: - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --forget {0}'.format(local_fingerprint) - res = __salt__['cmd.run_all'](cmd, runas=runas) - return res From fd4458b6c7c1878946d4572003311bb29535da0c Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 29 Jun 2017 10:19:08 -0600 Subject: [PATCH 019/300] import salt.minion for EventReturn for Windows Because of the way our multiprocessing works on Windows, this module is not available. In Linux, the namespace is copied over, and the module is available, so we only need the import on Windows. --- salt/utils/event.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/utils/event.py b/salt/utils/event.py index 0b64ee1833..3e0ad6a95f 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -59,6 +59,7 @@ import fnmatch import hashlib import logging import datetime +import sys from collections import MutableMapping from multiprocessing.util import Finalize from salt.ext.six.moves import range @@ -1159,6 +1160,16 @@ class EventReturn(salt.utils.process.SignalHandlingMultiprocessingProcess): A dedicated process which listens to the master event bus and queues and forwards events to the specified returner. ''' + def __new__(cls, *args, **kwargs): + if sys.platform.startswith('win'): + # This is required for Windows. On Linux, when a process is + # forked, the module namespace is copied and the current process + # gets all of sys.modules from where the fork happens. This is not + # the case for Windows. + import salt.minion + instance = super(EventReturn, cls).__new__(cls, *args, **kwargs) + return instance + def __init__(self, opts, log_queue=None): ''' Initialize the EventReturn system From b1960cea442c5d00ec55c6f4187ab1010e54f801 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Sun, 2 Jul 2017 17:16:22 +0300 Subject: [PATCH 020/300] Fix scheduled job run on Master if `when` parameter is a list --- salt/utils/schedule.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index a69e5ca418..08fcf36edb 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1090,23 +1090,35 @@ class Schedule(object): log.error('Invalid date string {0}. ' 'Ignoring job {1}.'.format(i, job)) continue - when = int(time.mktime(when__.timetuple())) - if when >= now: - _when.append(when) + _when.append(int(time.mktime(when__.timetuple()))) + + # Sort the list of "whens" from earlier to later schedules _when.sort() + + for i in _when: + if i < now and len(_when) > 1: + # Remove all missed schedules except the latest one. + # We need it to detect if it was triggered previously. + _when.remove(i) + if _when: - # Grab the first element - # which is the next run time + # Grab the first element, which is the next run time or + # last scheduled time in the past. when = _when[0] # If we're switching to the next run in a list - # ensure the job can run - if '_when' in data and data['_when'] != when: + # ensure the job can run. + if '_when' in data and \ + (data['_when'] != when or len(_when) == 1): data['_when_run'] = True - data['_when'] = when - seconds = when - now + # Calculate time to previously scheduled job + # (which has not been run yet), or it should be + # the last time definition in the list. + seconds = data['_when'] - now + else: + seconds = when - now - # scheduled time is in the past and the run was not triggered before + # Scheduled time is in the past and the run was not triggered before if seconds < 0 and not data.get('_when_run', False): continue @@ -1121,6 +1133,9 @@ class Schedule(object): if when > data['_when']: data['_when'] = when data['_when_run'] = True + # At last scheduled time, we disable all subsequent job runs + elif len(_when) == 1: + del data['_when'] else: continue From cf55c3361c1c0db95e49fcc0085548c786b6524b Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Mon, 3 Jul 2017 19:27:00 +1000 Subject: [PATCH 021/300] pkg.install and pkg.remove on the command line take number version numbers, store them within a float. However version is a string, to support versions numbers like 1.3.4 The following for example would not work as version would be a float, and a float does not match a string a dict with key of strings. c:\salt\salt-call -l debug pkg.install name=softwarename version=3.1 --- salt/modules/win_pkg.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 8451a6722d..8f8b962444 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1020,10 +1020,15 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ret[pkg_name] = 'Unable to locate package {0}'.format(pkg_name) continue - # Get the version number passed or the latest available + # Get the version number passed or the latest available (must be a string) version_num = '' if options: - version_num = options.get('version', False) + version_num = options.get('version', '') + # The user user salt cmdline with version=5.3 might be interpreted + # as a float it must be converted to a string in order for + # string matching to work. + if not isinstance(version, six.string_types): + version_num = str(version_num) if not version_num: version_num = _get_latest_pkg_version(pkginfo) @@ -1355,6 +1360,11 @@ def remove(name=None, pkgs=None, version=None, **kwargs): continue if version_num is not None: + # The user user salt cmdline with version=5.3 might be interpreted + # as a float it must be converted to a string in order for + # string matching to work. + if not isinstance(version, six.string_types): + version_num = str(version_num) if version_num not in pkginfo and 'latest' in pkginfo: version_num = 'latest' elif 'latest' in pkginfo: From 8d549685a7a4e71819b3701654bfb1522c5384f1 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Mon, 3 Jul 2017 16:08:16 +0100 Subject: [PATCH 022/300] Make result=true if Docker volume already exists In other states the result is always returned as `True` if no change is required, whether `test=True` is set or not. Fixes #42076 --- salt/states/dockerng.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 17559800f7..2f10abd444 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -2442,7 +2442,7 @@ def volume_present(name, driver=None, driver_opts=None, force=False): ret['result'] = result return ret - ret['result'] = None if __opts__['test'] else True + ret['result'] = True ret['comment'] = 'Volume \'{0}\' already exists.'.format(name) return ret From 2e1dc95500bc56262208402b0103b8fa01b0ab0d Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Mon, 3 Jul 2017 16:19:04 +0100 Subject: [PATCH 023/300] Make result=true if Docker volume already exists In other states the result is always returned as `True` if no change is required, whether `test=True` is set or not. Fixes #42076 --- salt/states/docker_volume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/docker_volume.py b/salt/states/docker_volume.py index 84a4f2154d..209b3839ac 100644 --- a/salt/states/docker_volume.py +++ b/salt/states/docker_volume.py @@ -187,7 +187,7 @@ def present(name, driver=None, driver_opts=None, force=False): ret['result'] = result return ret - ret['result'] = None if __opts__['test'] else True + ret['result'] = True ret['comment'] = 'Volume \'{0}\' already exists.'.format(name) return ret From 4fb2bb1856f527b27d5b1eedbc3969fc9a40decf Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Tue, 4 Jul 2017 03:08:08 +1000 Subject: [PATCH 024/300] Fix typo --- salt/modules/win_pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 8f8b962444..1c0be8a941 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1027,7 +1027,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version, six.string_types): + if not isinstance(version_num, six.string_types): version_num = str(version_num) if not version_num: @@ -1363,7 +1363,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version, six.string_types): + if not isinstance(version_num, six.string_types): version_num = str(version_num) if version_num not in pkginfo and 'latest' in pkginfo: version_num = 'latest' From 71675494251ef1d40a992b4068cb5fc45ec59035 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Tue, 4 Jul 2017 03:30:44 +1000 Subject: [PATCH 025/300] Handle version=None when converted to a string it becomes 'None' parm should default to empty string rather than None, it would fix better with existing code. --- salt/modules/win_pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 1c0be8a941..e13baed361 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1027,7 +1027,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version_num, six.string_types): + if not isinstance(version_num, six.string_types) and version_num is not None: version_num = str(version_num) if not version_num: @@ -1363,7 +1363,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # The user user salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. - if not isinstance(version_num, six.string_types): + if not isinstance(version_num, six.string_types) and version_num is not None: version_num = str(version_num) if version_num not in pkginfo and 'latest' in pkginfo: version_num = 'latest' From 09d37dd8922a5b3df4c14c119ef3aa5c22796529 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Tue, 4 Jul 2017 03:48:04 +1000 Subject: [PATCH 026/300] Fix comment typo --- salt/modules/win_pkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index e13baed361..b0d4052496 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1024,7 +1024,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): version_num = '' if options: version_num = options.get('version', '') - # The user user salt cmdline with version=5.3 might be interpreted + # Using the salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. if not isinstance(version_num, six.string_types) and version_num is not None: @@ -1360,7 +1360,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): continue if version_num is not None: - # The user user salt cmdline with version=5.3 might be interpreted + # Using the salt cmdline with version=5.3 might be interpreted # as a float it must be converted to a string in order for # string matching to work. if not isinstance(version_num, six.string_types) and version_num is not None: From 47d61f4edf800602d913933c5d6a9eb5fd0fe577 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 3 Jul 2017 13:16:57 -0500 Subject: [PATCH 027/300] Prevent command from showing in exception when output_loglevel=quiet When a command is being executed using salt.utils.timed_subprocess.TimedProc, and that command does not exist in the PATH, it will result in an OSError/IOError being raised. We catch this and then raise a CommandExecutionError with some information, including the command being run. However, when output_loglevel=quiet, we don't want any details of the command to be visible. This commit redacts the command details in these corner cases. --- salt/modules/cmdmod.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 82898edfeb..65b6767e33 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -526,10 +526,25 @@ def _run(cmd, try: proc = salt.utils.timed_subprocess.TimedProc(cmd, **kwargs) except (OSError, IOError) as exc: - raise CommandExecutionError( + msg = ( 'Unable to run command \'{0}\' with the context \'{1}\', ' - 'reason: {2}'.format(cmd, kwargs, exc) + 'reason: '.format( + cmd if _check_loglevel(output_loglevel) is not None + else 'REDACTED', + kwargs + ) ) + try: + if exc.filename is None: + msg += 'command not found' + else: + msg += '{0}: {1}'.format(exc, exc.filename) + except AttributeError: + # Both IOError and OSError have the filename attribute, so this + # is a precaution in case the exception classes in the previous + # try/except are changed. + msg += 'unknown' + raise CommandExecutionError(msg) try: proc.run() From bd27870a71300113a327d98e1abd2c2b1c424847 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 3 Jul 2017 13:30:01 -0500 Subject: [PATCH 028/300] Add debug logging to dockerng.login This can be used with debug logging enabled to troubleshoot which registries this function is attempting to authenticate. --- salt/modules/dockerng.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/modules/dockerng.py b/salt/modules/dockerng.py index 22aa4146ac..0cf393c4db 100644 --- a/salt/modules/dockerng.py +++ b/salt/modules/dockerng.py @@ -1899,6 +1899,10 @@ def login(*registries): cmd = ['docker', 'login', '-u', username, '-p', password] if registry.lower() != 'hub': cmd.append(registry) + log.debug( + 'Attempting to login to docker registry \'%s\' as user \'%s\'', + registry, username + ) login_cmd = __salt__['cmd.run_all']( cmd, python_shell=False, From c2822e05adde419c7ffedcab56b046af13a2833e Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 3 Jul 2017 14:47:36 -0600 Subject: [PATCH 029/300] Remove references in docs to pip install salt-cloud Instead, provide the option to use "-L" with the Bootstrap script. Fixes #41885 --- doc/topics/cloud/index.rst | 4 +++- doc/topics/cloud/qs.rst | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/topics/cloud/index.rst b/doc/topics/cloud/index.rst index e3f2959028..3dbb0c3aaa 100644 --- a/doc/topics/cloud/index.rst +++ b/doc/topics/cloud/index.rst @@ -64,7 +64,9 @@ automatically installed salt-cloud for you. Use your distribution's package manager to install the ``salt-cloud`` package from the same repo that you used to install Salt. These repos will automatically be setup by Salt Bootstrap. -If there is no salt-cloud package, install with ``pip install salt-cloud``. +Alternatively, the ``-L`` option can be passed to the `Salt Bootstrap`_ script when +installing Salt. The ``-L`` option will install ``salt-cloud`` and the required +``libcloud`` package. .. _`Salt Bootstrap`: https://github.com/saltstack/salt-bootstrap diff --git a/doc/topics/cloud/qs.rst b/doc/topics/cloud/qs.rst index 8cac85fe4f..3e6b40d37b 100644 --- a/doc/topics/cloud/qs.rst +++ b/doc/topics/cloud/qs.rst @@ -12,7 +12,9 @@ automatically installed salt-cloud for you. Use your distribution's package manager to install the ``salt-cloud`` package from the same repo that you used to install Salt. These repos will automatically be setup by Salt Bootstrap. -If there is no salt-cloud package, install with ``pip install salt-cloud``. +Alternatively, the ``-L`` option can be passed to the `Salt Bootstrap`_ script when +installing Salt. The ``-L`` option will install ``salt-cloud`` and the required +``libcloud`` package. .. _`Salt Bootstrap`: https://github.com/saltstack/salt-bootstrap From d4e7b91608cef279da68dc9e29065876d161716a Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 3 Jul 2017 23:06:54 -0400 Subject: [PATCH 030/300] Update releasecanddiate doc with new 2017.7.0rc1 Release --- doc/topics/releases/releasecandidate.rst | 34 +++++------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/doc/topics/releases/releasecandidate.rst b/doc/topics/releases/releasecandidate.rst index 0b042b86da..e2a628920a 100644 --- a/doc/topics/releases/releasecandidate.rst +++ b/doc/topics/releases/releasecandidate.rst @@ -8,7 +8,7 @@ Installing/Testing a Salt Release Candidate It's time for a new feature release of Salt! Follow the instructions below to install the latest release candidate of Salt, and try :ref:`all the shiny new -features `! Be sure to report any bugs you find on `Github +features `! Be sure to report any bugs you find on `Github `_. Installing Using Packages @@ -32,32 +32,10 @@ Builds for a few platforms are available as part of the RC at https://repo.salts Available builds: -- Amazon Linux -- Debian 8 -- macOS -- RHEL 7 -- SmartOS (see below) -- Ubuntu 16.04 - Windows .. FreeBSD -SmartOS -------- -Release candidate builds for SmartOS are available at http://pkg.blackdot.be/extras/salt-2016.11rc/. - -On a base64 2015Q4-x86_64 based native zone the package can be installed by the following: - -.. code-block:: bash - - pfexec pkg_add -U https://pkg.blackdot.be/extras/salt-2016.11rc/salt-2016.11.0rc2_2015Q4_x86_64.tgz - -When using the 2016Q2-tools release on the global zone by the following: - -.. code-block:: bash - - pfexec pkg_add -U https://pkg.blackdot.be/extras/salt-2016.11rc/salt-2016.11.0rc2_2016Q2_TOOLS.tgz - Installing Using Bootstrap ========================== @@ -67,14 +45,14 @@ You can install a release candidate of Salt using `Salt Bootstrap .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P git v2016.11.0rc2 + sudo sh install_salt.sh -P git v2017.7.0rc1 If you want to also install a master using Salt Bootstrap, use the ``-M`` flag: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M git v2016.11.0rc2 + sudo sh install_salt.sh -P -M git v2017.7.0rc1 If you want to install only a master and not a minion using Salt Bootstrap, use the ``-M`` and ``-N`` flags: @@ -82,13 +60,13 @@ the ``-M`` and ``-N`` flags: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M -N git v2016.11.0rc2 + sudo sh install_salt.sh -P -M -N git v2017.7.0rc1 Installing Using PyPI ===================== Installing from the `source archive -`_ on +`_ on `PyPI `_ is fairly straightforward. .. note:: @@ -126,4 +104,4 @@ Then install salt using the following command: .. code-block:: bash - sudo pip install salt==2016.11.0rc2 + sudo pip install salt==2017.7.0rc1 From 905be493d4586de841892ee44151083c169b4df9 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 4 Jul 2017 11:09:45 +0300 Subject: [PATCH 031/300] [2017.7] Fix scheduled jobs if `when` parameter is a list --- salt/utils/schedule.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 231f2331c0..888ba3369b 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1129,21 +1129,28 @@ class Schedule(object): log.error('Invalid date string {0}. ' 'Ignoring job {1}.'.format(i, job)) continue - when = int(time.mktime(when__.timetuple())) - if when >= now: - _when.append(when) + _when.append(int(time.mktime(when__.timetuple()))) if data['_splay']: _when.append(data['_splay']) + # Sort the list of "whens" from earlier to later schedules _when.sort() + + for i in _when: + if i < now and len(_when) > 1: + # Remove all missed schedules except the latest one. + # We need it to detect if it was triggered previously. + _when.remove(i) + if _when: - # Grab the first element - # which is the next run time + # Grab the first element, which is the next run time or + # last scheduled time in the past. when = _when[0] if '_run' not in data: - data['_run'] = True + # Prevent run of jobs from the past + data['_run'] = bool(when >= now) if not data['_next_fire_time']: data['_next_fire_time'] = when From 1bb42bb6097596cd1d5920b4cd5ea0a3d26ecb0f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Jul 2017 00:19:24 -0500 Subject: [PATCH 032/300] Fix regression when CLI pillar override is used with salt-call --- salt/state.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 06e63658f1..2a7e4a4507 100644 --- a/salt/state.py +++ b/salt/state.py @@ -660,8 +660,17 @@ class State(object): .format(', '.join(VALID_PILLAR_ENC)) ) self._pillar_enc = pillar_enc - self.opts['pillar'] = initial_pillar if initial_pillar is not None \ - else self._gather_pillar() + if initial_pillar is not None: + self.opts['pillar'] = initial_pillar + if self._pillar_override: + self.opts['pillar'] = salt.utils.dictupdate.merge( + self.opts['pillar'], + self._pillar_override, + self.opts.get('pillar_source_merging_strategy', 'smart'), + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + else: + self.opts['pillar'] = self._gather_pillar() self.state_con = context or {} self.load_modules() self.active = set() From 9a268949e3c5c430247143d7c7bb1da25f92c45d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Jul 2017 00:41:20 -0500 Subject: [PATCH 033/300] Add integration test for 42116 --- .../file/base/issue-42116-cli-pillar-override.sls | 2 ++ tests/integration/shell/call.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/integration/files/file/base/issue-42116-cli-pillar-override.sls diff --git a/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls b/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls new file mode 100644 index 0000000000..d3b2b239ec --- /dev/null +++ b/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls @@ -0,0 +1,2 @@ +ping -c 2 {{ pillar['myhost'] }}: + cmd.run diff --git a/tests/integration/shell/call.py b/tests/integration/shell/call.py index fd73dd92c9..7ed5875ba1 100644 --- a/tests/integration/shell/call.py +++ b/tests/integration/shell/call.py @@ -430,6 +430,21 @@ class CallTest(integration.ShellCase, testprogram.TestProgramCase, integration.S # Restore umask os.umask(current_umask) + @skipIf(sys.platform.startswith('win'), 'This test does not apply on Win') + def test_42116_cli_pillar_override(self): + ret = self.run_call( + 'state.apply issue-42116-cli-pillar-override ' + 'pillar=\'{"myhost": "localhost"}\'' + ) + for line in ret: + line = line.lstrip() + if line == 'Comment: Command "ping -c 2 localhost" run': + # Successful test + break + else: + log.debug('salt-call output:\n\n%s', '\n'.join(ret)) + self.fail('CLI pillar override not found in pillar data') + def tearDown(self): ''' Teardown method to remove installed packages From d14291267f5141715aefe90460959e9ca92abc86 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Jul 2017 02:00:54 -0500 Subject: [PATCH 034/300] Fix pillar.get when saltenv is passed --- salt/modules/pillar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index 5ad8afe27c..e8d6e0a0e1 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -110,7 +110,7 @@ def get(key, default = '' opt_merge_lists = __opts__.get('pillar_merge_lists', False) if \ merge_nested_lists is None else merge_nested_lists - pillar_dict = __pillar__ if saltenv is None else items(saltenv=saltenv) + pillar_dict = __pillar__ if saltenv is None else items(pillarenv=saltenv) if merge: if isinstance(default, dict): From 1435bf177ec4bdbcbba2614d7edde5074868a84e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 3 Jul 2017 14:17:13 -0600 Subject: [PATCH 035/300] require large timediff for ipv6 warning --- salt/grains/core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index ac6f98c043..c7bc2fe49f 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1688,12 +1688,14 @@ def ip_fqdn(): ret[key] = [] else: try: + start_time = datetime.datetime.utcnow() info = socket.getaddrinfo(_fqdn, None, socket_type) ret[key] = list(set(item[4][0] for item in info)) except socket.error: - if __opts__['__role'] == 'master': - log.warning('Unable to find IPv{0} record for "{1}" causing a 10 second timeout when rendering grains. ' - 'Set the dns or /etc/hosts for IPv{0} to clear this.'.format(ipv_num, _fqdn)) + timediff = datetime.datetime.utcnow() - start_time + if timediff.seconds > 5 and __opts__['__role'] == 'master': + log.warning('Unable to find IPv{0} record for "{1}" causing a {2} second timeout when rendering grains. ' + 'Set the dns or /etc/hosts for IPv{0} to clear this.'.format(ipv_num, _fqdn, timediff)) ret[key] = [] return ret From 9c4e132540ff8036a1514bcd4096458abf9b0883 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 3 Jul 2017 16:37:31 -0600 Subject: [PATCH 036/300] Import datetime --- salt/grains/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index c7bc2fe49f..9ed045f45d 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -20,6 +20,7 @@ import re import platform import logging import locale +import datetime import salt.exceptions __proxyenabled__ = ['*'] From bd802432339e648c8854e349f600b8c61fbc43ec Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 3 Jul 2017 14:15:35 -0600 Subject: [PATCH 037/300] Change repo_ng to repo-ng --- doc/topics/windows/windows-package-manager.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index d4ce6b5a07..063c8b44eb 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -140,7 +140,7 @@ packages: - 2015.8.0 and later minions: https://github.com/saltstack/salt-winrepo-ng - Earlier releases: https://github.com/saltstack/salt-winrepo -By default, these repositories are mirrored to ``/srv/salt/win/repo_ng`` +By default, these repositories are mirrored to ``/srv/salt/win/repo-ng`` and ``/srv/salt/win/repo``. This location can be changed in the master config file by setting the From e1694af39c12de643d939f3460ba65ed38068741 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Wed, 5 Jul 2017 17:06:06 -0400 Subject: [PATCH 038/300] Update builds available for rc1 --- doc/topics/releases/releasecandidate.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/topics/releases/releasecandidate.rst b/doc/topics/releases/releasecandidate.rst index e2a628920a..60918d6ac3 100644 --- a/doc/topics/releases/releasecandidate.rst +++ b/doc/topics/releases/releasecandidate.rst @@ -32,6 +32,8 @@ Builds for a few platforms are available as part of the RC at https://repo.salts Available builds: +- Ubuntu16 +- Redhat7 - Windows .. FreeBSD From 4ee24202fc78b0c2a50fc9657520f8a087ba4aa0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 5 Jul 2017 15:49:53 -0600 Subject: [PATCH 039/300] Fix unit tests for test_pip --- tests/unit/modules/test_pip.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/unit/modules/test_pip.py b/tests/unit/modules/test_pip.py index ec4b379928..8a5229b0e6 100644 --- a/tests/unit/modules/test_pip.py +++ b/tests/unit/modules/test_pip.py @@ -10,6 +10,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs +import salt.utils import salt.modules.pip as pip from salt.exceptions import CommandExecutionError @@ -289,17 +290,23 @@ class PipTestCase(TestCase, LoaderModuleMockMixin): mock_path.isdir.return_value = True pkg = 'mock' - venv_path = '/test_env' def join(*args): - return '/'.join(args) + return os.sep.join(args) + mock_path.join = join mock = MagicMock(return_value={'retcode': 0, 'stdout': ''}) with patch.dict(pip.__salt__, {'cmd.run_all': mock}): + if salt.utils.is_windows(): + venv_path = 'c:\\test_env' + bin_path = os.path.join(venv_path, 'Scripts', 'pip.exe').encode('string-escape') + else: + venv_path = '/test_env' + bin_path = os.path.join(venv_path, 'bin', 'pip') pip.install(pkg, bin_env=venv_path) mock.assert_called_once_with( - [os.path.join(venv_path, 'bin', 'pip'), 'install', pkg], - env={'VIRTUAL_ENV': '/test_env'}, + [bin_path, 'install', pkg], + env={'VIRTUAL_ENV': venv_path}, saltenv='base', runas=None, use_vt=False, From 00d9a52802e6d01c6f0826c14c7470d95958e4c4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 5 Jul 2017 17:44:06 -0600 Subject: [PATCH 040/300] Fix problem with handling REG_QWORD in list values --- salt/modules/reg.py | 4 +++- tests/unit/modules/test_reg_win.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/modules/reg.py b/salt/modules/reg.py index 63249dc5fe..644859f141 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -150,7 +150,9 @@ class Registry(object): # pylint: disable=R0903 _winreg.REG_DWORD: 'REG_DWORD', _winreg.REG_EXPAND_SZ: 'REG_EXPAND_SZ', _winreg.REG_MULTI_SZ: 'REG_MULTI_SZ', - _winreg.REG_SZ: 'REG_SZ' + _winreg.REG_SZ: 'REG_SZ', + # REG_QWORD isn't in the winreg library + 11: 'REG_QWORD' } self.opttype_reverse = { _winreg.REG_OPTION_NON_VOLATILE: 'REG_OPTION_NON_VOLATILE', diff --git a/tests/unit/modules/test_reg_win.py b/tests/unit/modules/test_reg_win.py index 960c00e5d0..e83d89b127 100644 --- a/tests/unit/modules/test_reg_win.py +++ b/tests/unit/modules/test_reg_win.py @@ -27,7 +27,7 @@ except ImportError: PY2 = sys.version_info[0] == 2 # The following used to make sure we are not # testing already existing data -# Note strftime retunrns a str, so we need to make it unicode +# Note strftime returns a str, so we need to make it unicode TIMEINT = int(time.time()) if PY2: From fb2cb78a31ce657448e812cc2c7c84695a8ef2d6 Mon Sep 17 00:00:00 2001 From: Steve Katz Date: Wed, 5 Jul 2017 19:56:11 -0400 Subject: [PATCH 041/300] Fix docs for puppet.plugin_sync so code-block renders properly and sync is spelled consistently --- salt/modules/puppet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/puppet.py b/salt/modules/puppet.py index 11067686be..aef4587381 100644 --- a/salt/modules/puppet.py +++ b/salt/modules/puppet.py @@ -337,9 +337,10 @@ def summary(): def plugin_sync(): ''' - Runs a plugin synch between the puppet master and agent + Runs a plugin sync between the puppet master and agent CLI Example: + .. code-block:: bash salt '*' puppet.plugin_sync From b27b1e340a09e8cb3b8b69c1bd5129a92b145632 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 6 Jul 2017 16:43:07 +0300 Subject: [PATCH 042/300] Fix #42115: parse libcloud "rc" version correctly --- salt/cloud/libcloudfuncs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/libcloudfuncs.py b/salt/cloud/libcloudfuncs.py index f627c4cf96..3191251c99 100644 --- a/salt/cloud/libcloudfuncs.py +++ b/salt/cloud/libcloudfuncs.py @@ -25,7 +25,7 @@ try: ) HAS_LIBCLOUD = True LIBCLOUD_VERSION_INFO = tuple([ - int(part) for part in libcloud.__version__.replace('-', '.').split('.')[:3] + int(part) for part in libcloud.__version__.replace('-', '.').replace('rc', '.').split('.')[:3] ]) except ImportError: From 7c0fb248ec025f0043c901badac8370aadfb6f4d Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Thu, 6 Jul 2017 09:46:04 -0400 Subject: [PATCH 043/300] Fix kerberos create_keytab doc --- salt/modules/kerberos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/kerberos.py b/salt/modules/kerberos.py index a5a77a5891..284033daaf 100644 --- a/salt/modules/kerberos.py +++ b/salt/modules/kerberos.py @@ -262,7 +262,7 @@ def create_keytab(name, keytab, enctypes=None): .. code-block:: bash - salt 'kdc.example.com' host/host1.example.com host1.example.com.keytab + salt 'kdc.example.com' kerberos.create_keytab host/host1.example.com host1.example.com.keytab ''' ret = {} From 01addb6053b5defa5ae556439f72e5d3406ecbb1 Mon Sep 17 00:00:00 2001 From: "Michael A. Smith" Date: Wed, 2 Nov 2016 15:41:52 -0400 Subject: [PATCH 044/300] Avoid Early Convert ret['comment'] to String Fixes this exception: An exception occurred in this state: Traceback (most recent call last): File "/var/tmp/.syops_b1274f_salt/py2/salt/state.py", line 1733, in call **cdata['kwargs']) File "/var/tmp/.syops_b1274f_salt/py2/salt/loader.py", line 1652, in wrapper return f(*args, **kwargs) File "/var/tmp/.syops_b1274f_salt/py2/salt/states/gpg.py", line 119, in present ret['comment'].append('Adding {0} to GPG keychain'.format(name)) AttributeError: 'str' object has no attribute 'append' --- salt/states/gpg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/gpg.py b/salt/states/gpg.py index 71b12c3aeb..a5aa4aa4cd 100644 --- a/salt/states/gpg.py +++ b/salt/states/gpg.py @@ -132,7 +132,7 @@ def present(name, else: ret['comment'].append('Invalid trust level {0}'.format(trust)) - ret['comment'] = '\n'.join(ret['comment']) + ret['comment'] = '\n'.join(ret['comment']) return ret @@ -188,5 +188,5 @@ def absent(name, ret['comment'].append('Deleting {0} from GPG keychain'.format(name)) else: ret['comment'].append('{0} not found in GPG keychain'.format(name)) - ret['comment'] = '\n'.join(ret['comment']) + ret['comment'] = '\n'.join(ret['comment']) return ret From 53f7b987e857faf5ae61ae2ae0293fb4999a9380 Mon Sep 17 00:00:00 2001 From: Dan Lloyd Date: Mon, 13 Feb 2017 12:39:09 -0500 Subject: [PATCH 045/300] Pass sig to service.status in after_toggle As on line 350, pass `sig` to `service.status` to ensure that the correct behavior is used. If we don't pass `sig`, upstart/sysv jobs that do not implement `service status` will fail. --- salt/states/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/service.py b/salt/states/service.py index acfc647bc2..0602360864 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -379,7 +379,7 @@ def running(name, enable=None, sig=None, init_delay=None, **kwargs): time.sleep(init_delay) # only force a change state if we have explicitly detected them - after_toggle_status = __salt__['service.status'](name) + after_toggle_status = __salt__['service.status'](name, sig) if 'service.enabled' in __salt__: after_toggle_enable_status = __salt__['service.enabled'](name) else: From 686926daf7f29464f4618e0205ad3fde2c1a4c11 Mon Sep 17 00:00:00 2001 From: Arthur Lutz Date: Tue, 4 Jul 2017 14:00:10 +0200 Subject: [PATCH 046/300] Update aws.rst - add Debian default username --- doc/topics/cloud/aws.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/topics/cloud/aws.rst b/doc/topics/cloud/aws.rst index c2131715f1..b5eb10178e 100644 --- a/doc/topics/cloud/aws.rst +++ b/doc/topics/cloud/aws.rst @@ -78,6 +78,7 @@ parameters are discussed in more detail below. # RHEL -> ec2-user # CentOS -> ec2-user # Ubuntu -> ubuntu + # Debian -> admin # ssh_username: ec2-user From 1bbff572ab003bf13e5e06712cb58c9725d50297 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 6 Jul 2017 11:46:23 -0600 Subject: [PATCH 047/300] Fix some documentation issues found in jinja filters doc topic These doc issues were all found in issue #42151. This commit takes care of all of the issues presented in #42151 except for the last task. --- doc/topics/jinja/index.rst | 38 +++++++++++++++++++++++++++++++------- salt/utils/__init__.py | 18 ++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 5548b29646..75641e4203 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -335,7 +335,7 @@ Returns: .. versionadded:: 2017.7.0 -Wraps a text around quoutes. +This text will be wrapped in quotes. .. jinja_ref:: regex_search @@ -750,19 +750,43 @@ Returns: Check a whitelist and/or blacklist to see if the value matches it. -Example: +This filter can be used with either a whitelist or a blacklist individually, +or a whitelist and a blacklist can be passed simultaneously. + +If whitelist is used alone, value membership is checked against the +whitelist only. If the value is found, the function returns ``True``. +Otherwise, it returns ``False``. + +If blacklist is used alone, value membership is checked against the +blacklist only. If the value is found, the function returns ``False``. +Otherwise, it returns ``True``. + +If both a whitelist and a blacklist are provided, value membership in the +blacklist will be examined first. If the value is not found in the blacklist, +then the whitelist is checked. If the value isn't found in the whitelist, +the function returns ``False``. + +Whitelist Example: .. code-block:: jinja - {{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} - {{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} + {{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} Returns: .. code-block:: python - True + True +Blacklist Example: + +.. code-block:: jinja + + {{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} + +.. code-block:: python + + False .. jinja_ref:: date_format @@ -1186,7 +1210,7 @@ Example: .. code-block:: jinja - {{ ['192.168.0.1', 'foo', 'bar', 'fe80::'] | ipv4 }} + {{ ['192.168.0.1', 'foo', 'bar', 'fe80::'] | ipv6 }} Returns: @@ -1224,7 +1248,7 @@ Returns: .. versionadded:: 2017.7.0 -Return the size of the network. +Return the size of the network. This utility works for both IPv4 and IPv6. Example: diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index b4e3c58abb..ec018f1ad7 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1444,6 +1444,24 @@ def expr_match(line, expr): def check_whitelist_blacklist(value, whitelist=None, blacklist=None): ''' Check a whitelist and/or blacklist to see if the value matches it. + + value + The item to check the whitelist and/or blacklist against. + + whitelist + The list of items that are white-listed. If ``value`` is found + in the whitelist, then the function returns ``True``. Otherwise, + it returns ``False``. + + blacklist + The list of items that are black-listed. If ``value`` is found + in the blacklist, then the function returns ``False``. Otherwise, + it returns ``True``. + + If both a whitelist and a blacklist are provided, value membership + in the blacklist will be examined first. If the value is not found + in the blacklist, then the whitelist is checked. If the value isn't + found in the whitelist, the function returns ``False``. ''' if blacklist is not None: if not hasattr(blacklist, '__iter__'): From 832a3d86ddb1f6d56596ffa18649a0779d5d7779 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 6 Jul 2017 13:11:02 -0600 Subject: [PATCH 048/300] Skip tests that use os.symlink on Windows --- tests/unit/modules/test_timezone.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/modules/test_timezone.py b/tests/unit/modules/test_timezone.py index 7cc6a18d19..0474ca9627 100644 --- a/tests/unit/modules/test_timezone.py +++ b/tests/unit/modules/test_timezone.py @@ -161,6 +161,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_aix', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -175,6 +176,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^ZONE=.*', 'ZONE="UTC"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -189,6 +191,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^TIMEZONE=.*', 'TIMEZONE="UTC"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -207,6 +210,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = _fopen.return_value.__enter__.return_value.write.mock_calls[0] assert args == ('UTC',) + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -225,6 +229,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = _fopen.return_value.__enter__.return_value.write.mock_calls[0] assert args == ('UTC',) + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=True)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -239,6 +244,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._timedatectl', MagicMock(return_value={'stdout': 'rtc in local tz:yes'})): assert timezone.get_hwclock() == 'localtime' + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -254,6 +260,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['tail', '-n', '1', '/etc/adjtime'],) assert kwarg == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -284,6 +291,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['tail', '-n', '1', '/etc/adjtime'],) assert kwarg == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -299,6 +307,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.utils.fopen', mock_open()): assert timezone.get_hwclock() == 'localtime' + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -315,6 +324,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(timezone.__grains__, {'os_family': ['AIX']}): assert timezone.get_hwclock() == hwclock + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -329,6 +339,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert timezone.set_hwclock('forty two') assert timezone.set_hwclock('UTC') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -348,6 +359,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['rtc', '-z', 'GMT'],) assert kwargs == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -364,6 +376,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['timezonectl', 'set-local-rtc', 'false'],) assert kwargs == {'python_shell': False} + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -379,6 +392,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^ZONE=.*', 'ZONE="TEST_TIMEZONE"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -394,6 +408,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^TIMEZONE=.*', 'TIMEZONE="TEST_TIMEZONE"') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @@ -413,6 +428,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[1] assert args == ('/etc/default/rcS', '^UTC=.*', 'UTC=no') + @skipIf(salt.utils.is_windows(), 'os.symlink not available in Windows') @patch('salt.utils.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) From a34970b45b0dfb32788761f66241470a3042b474 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 6 Jul 2017 12:12:50 -0700 Subject: [PATCH 049/300] Back porting the fix for 2017.7 that ensures the order of the names parameter. --- salt/state.py | 12 ++++++++---- .../files/file/base/issue-7649-handle-iorder.sls | 12 ++++++------ tests/integration/states/handle_iorder.py | 7 +++---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/salt/state.py b/salt/state.py index 2a7e4a4507..82c180486b 100644 --- a/salt/state.py +++ b/salt/state.py @@ -547,7 +547,7 @@ class Compiler(object): continue for state, run in six.iteritems(body): funcs = set() - names = set() + names = [] if state.startswith('__'): continue chunk = {'state': state, @@ -564,7 +564,9 @@ class Compiler(object): if isinstance(arg, dict): for key, val in six.iteritems(arg): if key == 'names': - names.update(val) + for _name in val: + if _name not in names: + names.append(_name) continue else: chunk.update(arg) @@ -1287,7 +1289,7 @@ class State(object): continue for state, run in six.iteritems(body): funcs = set() - names = set() + names = [] if state.startswith('__'): continue chunk = {'state': state, @@ -1306,7 +1308,9 @@ class State(object): if isinstance(arg, dict): for key, val in six.iteritems(arg): if key == 'names': - names.update(val) + for _name in val: + if _name not in names: + names.append(_name) elif key == 'state': # Don't pass down a state override continue diff --git a/tests/integration/files/file/base/issue-7649-handle-iorder.sls b/tests/integration/files/file/base/issue-7649-handle-iorder.sls index 4cfcdd11a0..49e6dfe900 100644 --- a/tests/integration/files/file/base/issue-7649-handle-iorder.sls +++ b/tests/integration/files/file/base/issue-7649-handle-iorder.sls @@ -1,9 +1,9 @@ handle-iorder: cmd: - - cwd: /tmp/ruby-1.9.2-p320 - - names: - - ./configure - - make - - make install - - run + - cwd: /tmp/ruby-1.9.2-p320 + - names: + - ./configure + - make + - make install + - run diff --git a/tests/integration/states/handle_iorder.py b/tests/integration/states/handle_iorder.py index 7ef1864cdd..1865f85a12 100644 --- a/tests/integration/states/handle_iorder.py +++ b/tests/integration/states/handle_iorder.py @@ -24,11 +24,10 @@ class HandleOrderTest(integration.ModuleCase): ''' ret = self.run_function('state.show_low_sls', mods='issue-7649-handle-iorder') - sorted_chunks = sorted(ret, key=lambda c: c.get('order')) + sorted_chunks = [chunk['name'] for chunk in sorted(ret, key=lambda c: c.get('order'))] - self.assertEqual(sorted_chunks[0]['name'], './configure') - self.assertEqual(sorted_chunks[1]['name'], 'make') - self.assertEqual(sorted_chunks[2]['name'], 'make install') + expected = ['./configure', 'make', 'make install'] + self.assertEqual(expected, sorted_chunks) if __name__ == '__main__': From 8260a71c07c3bdc22d6af6fcad737ed199cd9706 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 6 Jul 2017 13:19:40 -0600 Subject: [PATCH 050/300] Disable tests that require pwd in Windows --- tests/unit/modules/test_useradd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/modules/test_useradd.py b/tests/unit/modules/test_useradd.py index 2bcbee38d1..7fd457425f 100644 --- a/tests/unit/modules/test_useradd.py +++ b/tests/unit/modules/test_useradd.py @@ -72,6 +72,7 @@ class UserAddTestCase(TestCase, LoaderModuleMockMixin): # 'getent' function tests: 2 + @skipIf(HAS_PWD is False, 'The pwd module is not available') def test_getent(self): ''' Test if user.getent already have a value @@ -359,6 +360,7 @@ class UserAddTestCase(TestCase, LoaderModuleMockMixin): # 'list_users' function tests: 1 + @skipIf(HAS_PWD is False, 'The pwd module is not available') def test_list_users(self): ''' Test if it returns a list of all users From 11862743c2d7fc610f34541e5447cc94b1c1dea5 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 6 Jul 2017 15:03:54 -0600 Subject: [PATCH 051/300] Use long_range function for IPv6Network hosts() function Previous implementation used xrange, which requires that the range fit in a C long data type. Given that an IPv6 subnet is massive, the ``salt.ext.ipadress.IPv6Network.hosts()`` function would stacktrace with an OverflowError due to the use of xrange. However, ``/64`` is a common netmask. This PR uses the ``long_range`` function for the ``IPv6Network.hosts()`` function instead of ``xrange``. While the funciton will no longer stacktrace on large IPv6 subnets, the user should be aware that these large subnets will take a very long time to list. Fixes #42166 --- salt/ext/ipaddress.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/ext/ipaddress.py b/salt/ext/ipaddress.py index d2dfc5ee50..75011f7606 100644 --- a/salt/ext/ipaddress.py +++ b/salt/ext/ipaddress.py @@ -28,9 +28,6 @@ bytes = bytearray # Python 2 does not support exception chaining. # s/ from None$// -# Python 2 ranges need to fit in a C long -# 'fix' hosts() for IPv6Network - # When checking for instances of int, also allow Python 2's long. _builtin_isinstance = isinstance @@ -2259,7 +2256,7 @@ class IPv6Network(_BaseV6, _BaseNetwork): """ network = int(self.network_address) broadcast = int(self.broadcast_address) - for x in range(1, broadcast - network + 1): + for x in long_range(1, broadcast - network + 1): yield self._address_class(network + x) @property From b8bcc0d5990557e993d0fc785a9528278f038604 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 6 Jul 2017 17:36:18 -0600 Subject: [PATCH 052/300] Add note to various network_hosts docs about long_run for IPv6 networks --- doc/topics/jinja/index.rst | 7 ++++++- salt/utils/network.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 5548b29646..7081ce2154 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -1202,7 +1202,12 @@ Returns: .. versionadded:: 2017.7.0 -Return the list of hosts within a networks. +Return the list of hosts within a networks. This utility works for both IPv4 and IPv6. + +.. note:: + + When running this command with a large IPv6 network, the command will + take a long time to gather all of the hosts. Example: diff --git a/salt/utils/network.py b/salt/utils/network.py index aee5be16ab..33c7a14c9f 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -453,6 +453,11 @@ def _network_hosts(ip_addr_entry): def network_hosts(value, options=None, version=None): ''' Return the list of hosts within a network. + + .. note:: + + When running this command with a large IPv6 network, the command will + take a long time to gather all of the hosts. ''' ipaddr_filter_out = _filter_ipaddr(value, options=options, version=version) if not ipaddr_filter_out: From 407b8f4bb385ee398c30d06586c7524f67db625f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FUSIL-DELAHAYE?= Date: Fri, 7 Jul 2017 16:36:42 +0200 Subject: [PATCH 053/300] Fix #42198 If where_args is not set, not using it in the delete request. --- salt/states/sqlite3.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/states/sqlite3.py b/salt/states/sqlite3.py index 7c612cc9d0..abf1953e7d 100644 --- a/salt/states/sqlite3.py +++ b/salt/states/sqlite3.py @@ -147,9 +147,13 @@ def row_absent(name, db, table, where_sql, where_args=None): changes['changes']['old'] = rows[0] else: - cursor = conn.execute("DELETE FROM `" + - table + "` WHERE " + where_sql, - where_args) + if where_args is None: + cursor = conn.execute("DELETE FROM `" + + table + "` WHERE " + where_sql) + else: + cursor = conn.execute("DELETE FROM `" + + table + "` WHERE " + where_sql, + where_args) conn.commit() if cursor.rowcount == 1: changes['result'] = True From 2be4865f48e714b2f1e6743f4d6aaaedd5524bbb Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 7 Jul 2017 09:43:52 -0600 Subject: [PATCH 054/300] [PY3] Fix test that is flaky in Python 3 We can't rely on lists having the same order in Python3 the same way we rely on them in Python2. If we sort them first, and then compare them, this test will be more reliable. --- tests/unit/templates/test_jinja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 471d626540..6817c0c731 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -486,7 +486,7 @@ class TestCustomExtensions(TestCase): env = Environment(extensions=[SerializerExtension]) if six.PY3: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset).strip("'{}").split("', '") - self.assertEqual(rendered, list(unique)) + self.assertEqual(sorted(rendered), sorted(list(unique))) else: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset) self.assertEqual(rendered, u"{0}".format(unique)) From 771ade5d7325ff7bc285968ef6630dd6afe2930a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 7 Jul 2017 11:32:13 -0500 Subject: [PATCH 055/300] Only pass a saltenv in orchestration if one was explicitly passed (2017.7) Also, only pass pillarenv if explicitly passed. _get_opts() in the state module will apply the correct saltenv (if applicable) when no explicit saltenv/pillarenv is passed. --- salt/states/saltmod.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index d9c41f8cb3..ec430d7a60 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -245,14 +245,12 @@ def state(name, if pillar: cmd_kw['kwarg']['pillar'] = pillar - # If pillarenv is directly defined, use it - if pillarenv: + if pillarenv is not None: cmd_kw['kwarg']['pillarenv'] = pillarenv - # Use pillarenv if it's passed from __opts__ (via state.orchestrate for example) - elif __opts__.get('pillarenv'): - cmd_kw['kwarg']['pillarenv'] = __opts__['pillarenv'] - cmd_kw['kwarg']['saltenv'] = saltenv if saltenv is not None else __env__ + if saltenv is not None: + cmd_kw['kwarg']['saltenv'] = saltenv + cmd_kw['kwarg']['queue'] = queue if isinstance(concurrent, bool): From c07e22041aa13d256c5a011e3f5d47792857dd0e Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 7 Jul 2017 12:37:22 -0600 Subject: [PATCH 056/300] Add missing config to example --- salt/states/win_iis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/states/win_iis.py b/salt/states/win_iis.py index 071899886f..0d2661d6c4 100644 --- a/salt/states/win_iis.py +++ b/salt/states/win_iis.py @@ -495,6 +495,7 @@ def container_setting(name, container, settings=None): processModel.maxProcesses: 1 processModel.userName: TestUser processModel.password: TestPassword + processModel.identityType: SpecificUser Example of usage for the ``Sites`` container: From 22a18fa2ed473ae18b15fedd383facce46eb3d27 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 7 Jul 2017 11:32:13 -0500 Subject: [PATCH 057/300] Only pass a saltenv in orchestration if one was explicitly passed (2016.11) Also, only pass pillarenv if explicitly passed. _get_opts() in the state module will apply the correct saltenv (if applicable) when no explicit saltenv/pillarenv is passed. --- salt/states/saltmod.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index 92f46af465..25b0dd1040 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -73,6 +73,7 @@ def state( saltenv=None, test=False, pillar=None, + pillarenv=None, expect_minions=False, fail_minions=None, allow_fail=0, @@ -230,7 +231,12 @@ def state( if pillar: cmd_kw['kwarg']['pillar'] = pillar - cmd_kw['kwarg']['saltenv'] = saltenv if saltenv is not None else __env__ + if pillarenv is not None: + cmd_kw['kwarg']['pillarenv'] = pillarenv + + if saltenv is not None: + cmd_kw['kwarg']['saltenv'] = saltenv + cmd_kw['kwarg']['queue'] = queue if isinstance(concurrent, bool): From 798d29276ed29e3dd209d08952fcaf7633cad87e Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 7 Jul 2017 15:14:06 -0600 Subject: [PATCH 058/300] Add note about "to_bytes" jinja filter issues when using yaml_jinja renderer --- doc/topics/jinja/index.rst | 14 ++++++++++++++ doc/topics/troubleshooting/yaml_idiosyncrasies.rst | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 75641e4203..bcab6c2322 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -849,6 +849,13 @@ Example: {{ 'wall of text' | to_bytes }} +.. note:: + + This option may have adverse effects when using the default renderer, ``yaml_jinja``. + This is due to the fact that YAML requires proper handling in regard to special + characters. Please see the section on :ref:`YAML ASCII support ` + in the :ref:`YAML Idiosyncracies ` documentation for more + information. .. jinja_ref:: json_decode_list @@ -1308,6 +1315,13 @@ Example: {{ '00:11:22:33:44:55' | mac_str_to_bytes }} +.. note:: + + This option may have adverse effects when using the default renderer, ``yaml_jinja``. + This is due to the fact that YAML requires proper handling in regard to special + characters. Please see the section on :ref:`YAML ASCII support ` + in the :ref:`YAML Idiosyncracies ` documentation for more + information. .. jinja_ref:: dns_check diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index e1c6383459..df5eff3d01 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -248,8 +248,10 @@ Alternatively, they can be defined the "old way", or with multiple - require: - user: fred -YAML support only plain ASCII -============================= +.. _yaml_plain_ascii: + +YAML supports only plain ASCII +============================== According to YAML specification, only ASCII characters can be used. From 08e7d8351a2d6a9fde4fad6757954d5e2f7da35d Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Mon, 10 Jul 2017 12:46:32 -0400 Subject: [PATCH 059/300] Abolish references to `dig` in examples. --- salt/modules/dnsutil.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/dnsutil.py b/salt/modules/dnsutil.py index 36b1469a4b..1cd2b6a032 100644 --- a/salt/modules/dnsutil.py +++ b/salt/modules/dnsutil.py @@ -232,7 +232,7 @@ def check_ip(ip_addr): .. code-block:: bash - salt ns1 dig.check_ip 127.0.0.1 + salt ns1 dnsutil.check_ip 127.0.0.1 ''' if _has_dig(): return __salt__['dig.check_ip'](ip_addr) @@ -302,7 +302,7 @@ def NS(domain, resolve=True, nameserver=None): .. code-block:: bash - salt ns1 dig.NS google.com + salt ns1 dnsutil.NS google.com ''' if _has_dig(): @@ -323,7 +323,7 @@ def SPF(domain, record='SPF', nameserver=None): .. code-block:: bash - salt ns1 dig.SPF google.com + salt ns1 dnsutil.SPF google.com ''' if _has_dig(): return __salt__['dig.SPF'](domain, record, nameserver) @@ -346,7 +346,7 @@ def MX(domain, resolve=False, nameserver=None): .. code-block:: bash - salt ns1 dig.MX google.com + salt ns1 dnsutil.MX google.com ''' if _has_dig(): return __salt__['dig.MX'](domain, resolve, nameserver) From 4cb51bd03af6a6a3a4dc0ce07eb109cd07597d28 Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Mon, 10 Jul 2017 13:55:06 -0400 Subject: [PATCH 060/300] Make note of dig partial requirement. --- salt/modules/dnsutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/dnsutil.py b/salt/modules/dnsutil.py index 1cd2b6a032..948cd720ce 100644 --- a/salt/modules/dnsutil.py +++ b/salt/modules/dnsutil.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- ''' -Compendium of generic DNS utilities +Compendium of generic DNS utilities. + +.. note: + Some functions in the `dnsutil` execution module depend on `dig`. ''' from __future__ import absolute_import From 4bf4b14161be83fad83e1c1ca4f8974f2cfae4ae Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Mon, 10 Jul 2017 20:29:35 +0000 Subject: [PATCH 061/300] New option for napalm proxy/minion: provider This option will turn very handy for environemnts when the user requires a library with a different name than napalm-base (eventually on top of the public lib). The sole requirement right now is to preserve the get_network_driver function. However, this can be enhnaced even from this perspective later, if really needed. --- salt/utils/napalm.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index c69a92432e..36914be042 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import traceback import logging +import importlib from functools import wraps log = logging.getLogger(__file__) @@ -264,6 +265,7 @@ def get_device_opts(opts, salt_obj=None): network_device['TIMEOUT'] = device_dict.get('timeout', 60) network_device['OPTIONAL_ARGS'] = device_dict.get('optional_args', {}) network_device['ALWAYS_ALIVE'] = device_dict.get('always_alive', True) + network_device['PROVIDER'] = device_dict.get('provider') network_device['UP'] = False # get driver object form NAPALM if 'config_lock' not in network_device['OPTIONAL_ARGS']: @@ -281,7 +283,24 @@ def get_device(opts, salt_obj=None): ''' log.debug('Setting up NAPALM connection') network_device = get_device_opts(opts, salt_obj=salt_obj) - _driver_ = napalm_base.get_network_driver(network_device.get('DRIVER_NAME')) + provider_lib = napalm_base + if network_device.get('PROVIDER'): + # In case the user requires a different provider library, + # other than napalm-base. + # For example, if napalm-base does not satisfy the requirements + # and needs to be enahanced with more specific features, + # we may need to define a custom library on top of napalm-base + # with the constraint that it still needs to provide the + # `get_network_driver` function. However, even this can be + # extended later, if really needed. + # Configuration example: + # provider: napalm_base_example + try: + provider_lib = importlib.import_module(network_device.get('PROVIDER')) + except ImportError as ierr: + log.error('Unable to import {0}'.format(network_device.get('PROVIDER')), exc_info=True) + log.error('Falling back to napalm-base') + _driver_ = provider_lib.get_network_driver(network_device.get('DRIVER_NAME')) try: network_device['DRIVER'] = _driver_( network_device.get('HOSTNAME', ''), From 1ac69bd7370e05ff278cc995c05133300a7da8de Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Tue, 11 Jul 2017 14:44:51 +0000 Subject: [PATCH 062/300] Document the provider option and rearrange the doc --- salt/proxy/napalm.py | 78 +++++++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index b0cd8d6546..9ae4fd5316 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -23,37 +23,77 @@ Please check Installation_ for complete details. Pillar ------ -The napalm proxy configuration requires four mandatory parameters in order to connect to the network device: +The napalm proxy configuration requires the following parameters in order to connect to the network device: -* driver: specifies the network device operating system. For a complete list of the supported operating systems \ -please refer to the `NAPALM Read the Docs page`_. -* host: hostname -* username: username to be used when connecting to the device -* passwd: the password needed to establish the connection -* optional_args: dictionary with the optional arguments. Check the complete list of supported `optional arguments`_ +driver + Specifies the network device operating system. + For a complete list of the supported operating systems please refer to the + `NAPALM Read the Docs page`_. -.. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems -.. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments +host + The IP Address or FQDN to use when connecting to the device. Alternatively, + the following field names can be used instead: ``hostname``, ``fqdn``, ``ip``. -.. versionadded:: 2017.7.0 +username + The username to be used when connecting to the device. -* always_alive: in certain less dynamic environments, maintaining the remote connection permanently +passwd + The password needed to establish the connection. + + .. note:: + + This field may not be mandatory when working with SSH-based drivers, and + the username has a SSH key properly configured on the device targeted to + be managed. + +optional_args + Dictionary with the optional arguments. + Check the complete list of supported `optional arguments`_. + +always_alive: ``True`` + In certain less dynamic environments, maintaining the remote connection permanently open with the network device is not always beneficial. In that case, the user can select to initialize the connection only when needed, by specifying this field to ``false``. Default: ``true`` (maintains the connection with the remote network device). -Example: + .. versionadded:: 2017.7.0 + +provider: ``napalm_base`` + The module that provides the ``get_network_device`` function. + This option is useful when the user has more specific needs and requires + to extend the NAPALM capabilities using a private library implementation. + The only constraint is that the alternative library needs to have the + ``get_network_device`` function available. + + .. versionadded:: 2017.7.1 + +.. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems +.. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments + +Proxy pillar file example: .. code-block:: yaml proxy: - proxytype: napalm - driver: junos - host: core05.nrt02 - username: my_username - passwd: my_password - optional_args: - port: 12201 + proxytype: napalm + driver: junos + host: core05.nrt02 + username: my_username + passwd: my_password + optional_args: + port: 12201 + +Example using a user-specific library, extending NAPALM's capabilities, e.g. ``custom_napalm_base``: + +.. code-block:: yaml + + proxy: + proxytype: napalm + driver: ios + fqdn: cr1.th2.par.as1234.net + username: salt + password: '' + provider: custom_napalm_base .. seealso:: From 97261bfe69bf018e19082a7616d221d12c908fb2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 10:04:37 -0600 Subject: [PATCH 063/300] Fix win_inet_pton check for malformatted ip addresses --- salt/ext/win_inet_pton.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index ffb47d440a..8c4d7dde3e 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import socket import ctypes import os +import ipaddress class sockaddr(ctypes.Structure): @@ -31,6 +32,24 @@ else: def inet_pton(address_family, ip_string): + # Verify IP Address + # This will catch IP Addresses such as 10.1.2 + if address_family == socket.AF_INET: + try: + ipaddress.ip_address(ip_string.decode()) + except ValueError: + raise socket.error('illegal IP address string passed to inet_pton') + return socket.inet_aton(ip_string) + + # Verify IP Address + # The `WSAStringToAddressA` function handles notations used by Berkeley + # software which includes 3 part IP Addresses such as `10.1.2`. That's why + # the above check is needed to enforce more strict IP Address validation as + # used by the `inet_pton` function in Unix. + # See the following: + # https://stackoverflow.com/a/29286098 + # Docs for the `inet_addr` function on MSDN + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms738563.aspx addr = sockaddr() addr.sa_family = address_family addr_size = ctypes.c_int(ctypes.sizeof(addr)) From e6a9563d472912ebad8eaa658ee2fda1e1401936 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Tue, 11 Jul 2017 10:19:22 -0600 Subject: [PATCH 064/300] simple doc updates --- salt/fileserver/svnfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/fileserver/svnfs.py b/salt/fileserver/svnfs.py index fc17b979b7..5dd1ccc10d 100644 --- a/salt/fileserver/svnfs.py +++ b/salt/fileserver/svnfs.py @@ -2,7 +2,7 @@ ''' Subversion Fileserver Backend -After enabling this backend, branches, and tags in a remote subversion +After enabling this backend, branches and tags in a remote subversion repository are exposed to salt as different environments. To enable this backend, add ``svn`` to the :conf_master:`fileserver_backend` option in the Master config file. @@ -694,7 +694,7 @@ def file_hash(load, fnd): def _file_lists(load, form): ''' - Return a dict containing the file lists for files, dirs, emtydirs and symlinks + Return a dict containing the file lists for files, dirs, emptydirs and symlinks ''' if 'env' in load: salt.utils.warn_until( From 53e25760beb3a504e9451a0826166c41f631a553 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 11 Jul 2017 10:17:51 -0600 Subject: [PATCH 065/300] Only use unassociated ips when unable to allocate When checking the unassociated ips first, in Parallel mode, each server will use the same one unallocated ip. --- salt/cloud/clouds/openstack.py | 55 ++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index 5eaff82c7e..e73af9e0e5 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -135,6 +135,14 @@ Alternatively, one could use the private IP to connect by specifying: ssh_interface: private_ips +.. note:: + + When using floating ips from networks, if the OpenStack driver is unable to + allocate a new ip address for the server, it will check that for + unassociated ip addresses in the floating ip pool. If SaltCloud is running + in parallel mode, it is possible that more than one server will attempt to + use the same ip address. + ''' # Import python libs @@ -866,40 +874,43 @@ def _assign_floating_ips(vm_, conn, kwargs): pool = OpenStack_1_1_FloatingIpPool( net['floating'], conn.connection ) - for idx in pool.list_floating_ips(): - if idx.node_id is None: - floating.append(idx) + try: + floating.append(pool.create_floating_ip()) + except Exception as e: + log.debug('Cannot allocate IP from floating pool \'%s\'. Checking for unassociated ips.', + net['floating']) + for idx in pool.list_floating_ips(): + if idx.node_id is None: + floating.append(idx) + break if not floating: - try: - floating.append(pool.create_floating_ip()) - except Exception as e: - raise SaltCloudSystemExit( - 'Floating pool \'{0}\' does not have any more ' - 'please create some more or use a different ' - 'pool.'.format(net['floating']) - ) + raise SaltCloudSystemExit( + 'There are no more floating IP addresses ' + 'available, please create some more' + ) # otherwise, attempt to obtain list without specifying pool # this is the same as 'nova floating-ip-list' elif ssh_interface(vm_) != 'private_ips': try: # This try/except is here because it appears some - # *cough* Rackspace *cough* # OpenStack providers return a 404 Not Found for the # floating ip pool URL if there are no pools setup pool = OpenStack_1_1_FloatingIpPool( '', conn.connection ) - for idx in pool.list_floating_ips(): - if idx.node_id is None: - floating.append(idx) + try: + floating.append(pool.create_floating_ip()) + except Exception as e: + log.debug('Cannot allocate IP from the default floating pool. Checking for unassociated ips.') + for idx in pool.list_floating_ips(): + if idx.node_id is None: + floating.append(idx) + break if not floating: - try: - floating.append(pool.create_floating_ip()) - except Exception as e: - raise SaltCloudSystemExit( - 'There are no more floating IP addresses ' - 'available, please create some more' - ) + raise SaltCloudSystemExit( + 'There are no more floating IP addresses ' + 'available, please create some more' + ) except Exception as e: if str(e).startswith('404'): pass From acc0345bc87be0e4f619bcc9b78601ef0ac601f0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 12:08:13 -0600 Subject: [PATCH 066/300] Fix unit tests --- salt/modules/win_system.py | 98 +++++++++++++----- tests/unit/modules/test_win_system.py | 143 +++++++++++++------------- 2 files changed, 145 insertions(+), 96 deletions(-) diff --git a/salt/modules/win_system.py b/salt/modules/win_system.py index a92f0923e5..993ab39830 100644 --- a/salt/modules/win_system.py +++ b/salt/modules/win_system.py @@ -37,6 +37,7 @@ except ImportError: import salt.utils import salt.utils.locales import salt.ext.six as six +from salt.exceptions import CommandExecutionError # Set up logging log = logging.getLogger(__name__) @@ -622,7 +623,11 @@ def join_domain(domain, .. versionadded:: 2015.8.2/2015.5.7 Returns: - dict: Dictionary if successful, otherwise False + dict: Dictionary if successful + + Raises: + CommandExecutionError: Raises an error if _join_domain returns anything + other than 0 CLI Example: @@ -655,6 +660,56 @@ def join_domain(domain, account_ou = account_ou.split('\\') account_ou = ''.join(account_ou) + err = _join_domain(domain=domain, username=username, password=password, + account_ou=account_ou, account_exists=account_exists) + + if not err: + ret = {'Domain': domain, + 'Restart': False} + if restart: + ret['Restart'] = reboot() + return ret + + raise CommandExecutionError(win32api.FormatMessage(err).rstrip()) + + +def _join_domain(domain, + username=None, + password=None, + account_ou=None, + account_exists=False): + ''' + Helper function to join the domain. + + Args: + domain (str): The domain to which the computer should be joined, e.g. + ``example.com`` + + username (str): Username of an account which is authorized to join + computers to the specified domain. Need to be either fully qualified + like ``user@domain.tld`` or simply ``user`` + + password (str): Password of the specified user + + account_ou (str): The DN of the OU below which the account for this + computer should be created when joining the domain, e.g. + ``ou=computers,ou=departm_432,dc=my-company,dc=com`` + + account_exists (bool): If set to ``True`` the computer will only join + the domain if the account already exists. If set to ``False`` the + computer account will be created if it does not exist, otherwise it + will use the existing account. Default is False. + + Returns: + int: + + :param domain: + :param username: + :param password: + :param account_ou: + :param account_exists: + :return: + ''' NETSETUP_JOIN_DOMAIN = 0x1 # pylint: disable=invalid-name NETSETUP_ACCOUNT_CREATE = 0x2 # pylint: disable=invalid-name NETSETUP_DOMAIN_JOIN_IF_JOINED = 0x20 # pylint: disable=invalid-name @@ -670,23 +725,13 @@ def join_domain(domain, pythoncom.CoInitialize() conn = wmi.WMI() comp = conn.Win32_ComputerSystem()[0] - err = comp.JoinDomainOrWorkgroup(Name=domain, - Password=password, - UserName=username, - AccountOU=account_ou, - FJoinOptions=join_options) - # you have to do this because JoinDomainOrWorkgroup returns a strangely - # formatted value that looks like (0,) - if not err[0]: - ret = {'Domain': domain, - 'Restart': False} - if restart: - ret['Restart'] = reboot() - return ret - - log.error(win32api.FormatMessage(err[0]).rstrip()) - return False + # Return the results of the command as an error + # JoinDomainOrWorkgroup returns a strangely formatted value that looks like + # (0,) so return the first item + return comp.JoinDomainOrWorkgroup( + Name=domain, Password=password, UserName=username, AccountOU=account_ou, + FJoinOptions=join_options)[0] def unjoin_domain(username=None, @@ -919,7 +964,11 @@ def set_system_date_time(years=None, seconds (int): Seconds digit: 0 - 59 Returns: - bool: True if successful, otherwise False. + bool: True if successful + + Raises: + CommandExecutionError: Raises an error if ``SetLocalTime`` function + fails CLI Example: @@ -972,12 +1021,15 @@ def set_system_date_time(years=None, system_time.wSecond = int(seconds) system_time_ptr = ctypes.pointer(system_time) succeeded = ctypes.windll.kernel32.SetLocalTime(system_time_ptr) - return succeeded is not 0 - except OSError: + if succeeded is not 0: + return True + else: + log.error('Failed to set local time') + raise CommandExecutionError( + win32api.FormatMessage(succeeded).rstrip()) + except OSError as err: log.error('Failed to set local time') - return False - - return True + raise CommandExecutionError(err) def get_system_date(): diff --git a/tests/unit/modules/test_win_system.py b/tests/unit/modules/test_win_system.py index 3506a4ddf6..8b8a05356f 100644 --- a/tests/unit/modules/test_win_system.py +++ b/tests/unit/modules/test_win_system.py @@ -67,30 +67,31 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to reboot the system ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(), True) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_timeout_in_minutes(self): ''' Test to reboot the system with a timeout ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(5, in_seconds=False), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(5, in_seconds=False), True) + shutdown.assert_called_with(timeout=5, in_seconds=False, reboot=True, + only_on_pending_reboot=False) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_timeout_in_seconds(self): ''' Test to reboot the system with a timeout ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(5, in_seconds=True), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '5'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(5, in_seconds=True), True) + shutdown.assert_called_with(timeout=5, in_seconds=True, reboot=True, + only_on_pending_reboot=False) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_wait(self): @@ -98,50 +99,49 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): Test to reboot the system with a timeout and wait for it to finish ''' - mock = MagicMock(return_value='salt') - sleep_mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - with patch('time.sleep', sleep_mock): - self.assertEqual(win_system.reboot(wait_for_reboot=True), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) - sleep_mock.assert_called_once_with(330) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)), \ + patch('salt.modules.win_system.time.sleep', + MagicMock()) as time: + self.assertEqual(win_system.reboot(wait_for_reboot=True), True) + time.assert_called_with(330) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_shutdown(self): ''' Test to shutdown a running system ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.shutdown(), 'salt') + with patch('salt.modules.win_system.win32api.InitiateSystemShutdown', + MagicMock()): + self.assertEqual(win_system.shutdown(), True) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_shutdown_hard(self): ''' Test to shutdown a running system with no timeout or warning ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.shutdown_hard(), 'salt') + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.shutdown_hard(), True) + shutdown.assert_called_with(timeout=0) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_set_computer_name(self): ''' Test to set the Windows computer name ''' - mock = MagicMock(side_effect=[{'Computer Name': {'Current': ""}, - 'ReturnValue = 0;': True}, - {'Computer Name': {'Current': 'salt'}}]) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - mock = MagicMock(return_value='salt') - with patch.object(win_system, 'get_computer_name', mock): - mock = MagicMock(return_value=True) - with patch.object(win_system, - 'get_pending_computer_name', mock): - self.assertDictEqual(win_system.set_computer_name("salt"), + with patch('salt.modules.win_system.windll.kernel32.SetComputerNameExW', + MagicMock(return_value=True)): + with patch.object(win_system, 'get_computer_name', + MagicMock(return_value='salt')): + with patch.object(win_system, 'get_pending_computer_name', + MagicMock(return_value='salt_new')): + self.assertDictEqual(win_system.set_computer_name("salt_new"), {'Computer Name': {'Current': 'salt', - 'Pending': True}}) - + 'Pending': 'salt_new'}}) + # Test set_computer_name failure + with patch('salt.modules.win_system.windll.kernel32.SetComputerNameExW', + MagicMock(return_value=False)): self.assertFalse(win_system.set_computer_name("salt")) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') @@ -149,25 +149,25 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to get a pending computer name. ''' - mock = MagicMock(return_value='salt') - with patch.object(win_system, 'get_computer_name', mock): - mock = MagicMock(side_effect=['salt0', - 'ComputerName REG_SZ (salt)']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): + with patch.object(win_system, 'get_computer_name', + MagicMock(return_value='salt')): + reg_mock = MagicMock(return_value={'vdata': 'salt'}) + with patch.dict(win_system.__salt__, {'reg.read_value': reg_mock}): self.assertFalse(win_system.get_pending_computer_name()) + reg_mock = MagicMock(return_value={'vdata': 'salt_pending'}) + with patch.dict(win_system.__salt__, {'reg.read_value': reg_mock}): self.assertEqual(win_system.get_pending_computer_name(), - '(salt)') + 'salt_pending') @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_get_computer_name(self): ''' Test to get the Windows computer name ''' - mock = MagicMock(side_effect=['Server Name Salt', 'Salt']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.get_computer_name(), 'Salt') - + with patch('salt.modules.win_system.win32api.GetComputerNameEx', + MagicMock(side_effect=['computer name', ''])): + self.assertEqual(win_system.get_computer_name(), 'computer name') self.assertFalse(win_system.get_computer_name()) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') @@ -189,10 +189,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to get the Windows computer description ''' - mock = MagicMock(side_effect=['Server Comment Salt', 'Salt']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.get_computer_desc(), 'Salt') - + with patch('salt.modules.win_system.get_system_info', + MagicMock(side_effect=[{'description': 'salt description'}, + {'description': None}])): + self.assertEqual(win_system.get_computer_desc(), 'salt description') self.assertFalse(win_system.get_computer_desc()) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs w32net and other windows libraries') @@ -200,17 +200,20 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to join a computer to an Active Directory domain ''' - mock = MagicMock(side_effect=[{'ReturnValue = 0;': True}, - {'Salt': True}]) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertDictEqual(win_system.join_domain("saltstack", - "salt", - "salt@123"), - {'Domain': 'saltstack'}) + with patch('salt.modules.win_system._join_domain', + MagicMock(return_value=0)): + with patch('salt.modules.win_system.get_domain_workgroup', + MagicMock(return_value={'Workgroup': 'Workgroup'})): + self.assertDictEqual( + win_system.join_domain( + "saltstack", "salt", "salt@123"), + {'Domain': 'saltstack', 'Restart': False}) - self.assertFalse(win_system.join_domain("saltstack", - "salt", - "salt@123")) + with patch('salt.modules.win_system.get_domain_workgroup', + MagicMock(return_value={'Domain': 'saltstack'})): + self.assertEqual( + win_system.join_domain("saltstack", "salt", "salt@123"), + 'Already joined to saltstack') def test_get_system_time(self): ''' @@ -230,13 +233,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to set system time ''' - mock = MagicMock(side_effect=[False, True]) - with patch.object(win_system, '_validate_time', mock): + with patch('salt.modules.win_system.set_system_date_time', + MagicMock(side_effect=[False, True])): self.assertFalse(win_system.set_system_time("11:31:15 AM")) - - mock = MagicMock(return_value=True) - with patch.dict(win_system.__salt__, {'cmd.retcode': mock}): - self.assertFalse(win_system.set_system_time("11:31:15 AM")) + self.assertTrue(win_system.set_system_time("11:31:15 AM")) def test_get_system_date(self): ''' @@ -250,13 +250,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to set system date ''' - mock = MagicMock(side_effect=[False, True]) - with patch.object(win_system, '_validate_date', mock): + with patch('salt.modules.win_system.set_system_date_time', + MagicMock(side_effect=[False, True])): self.assertFalse(win_system.set_system_date("03-28-13")) - - mock = MagicMock(return_value=True) - with patch.dict(win_system.__salt__, {'cmd.retcode': mock}): - self.assertFalse(win_system.set_system_date("03-28-13")) + self.assertTrue(win_system.set_system_date("03-28-13")) def test_start_time_service(self): ''' From 45be32666a78d82dfc174accf102d608ae0e172f Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 13:08:00 -0600 Subject: [PATCH 067/300] Add error-handling function to shutil.rmtree --- tests/unit/pillar/test_git.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/pillar/test_git.py b/tests/unit/pillar/test_git.py index 7c55175416..f46011d6da 100644 --- a/tests/unit/pillar/test_git.py +++ b/tests/unit/pillar/test_git.py @@ -16,6 +16,7 @@ import tempfile import shutil import subprocess import yaml +import stat # Import Salt Testing libs from tests.integration import AdaptedConfigurationTestCaseMixin @@ -72,9 +73,13 @@ class GitPillarTestCase(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModul git_pillar._update('master', 'file://{0}'.format(self.repo_path)) def tearDown(self): - shutil.rmtree(self.tmpdir) + shutil.rmtree(self.tmpdir, onerror=self._rmtree_error) super(GitPillarTestCase, self).tearDown() + def _rmtree_error(self, func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + def _create_repo(self): 'create source Git repo in temp directory' repo = os.path.join(self.tmpdir, 'repo_pillar') From 55b278c478cb2e1925e6ef417f3b2104112dae8f Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 13:23:11 -0600 Subject: [PATCH 068/300] Mock the reg.read_value function --- tests/unit/states/test_environ.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/states/test_environ.py b/tests/unit/states/test_environ.py index 28c35c525f..45a8e5fb1a 100644 --- a/tests/unit/states/test_environ.py +++ b/tests/unit/states/test_environ.py @@ -96,11 +96,14 @@ class TestEnvironState(TestCase, LoaderModuleMockMixin): ret = envstate.setenv('notimportant', {'foo': 'bar'}) self.assertEqual(ret['changes'], {'foo': 'bar'}) - ret = envstate.setenv('notimportant', {'test': False, 'foo': 'baz'}, false_unsets=True) + with patch.dict(envstate.__salt__, {'reg.read_value': MagicMock()}): + ret = envstate.setenv( + 'notimportant', {'test': False, 'foo': 'baz'}, false_unsets=True) self.assertEqual(ret['changes'], {'test': None, 'foo': 'baz'}) self.assertEqual(envstate.os.environ, {'INITIAL': 'initial', 'foo': 'baz'}) - ret = envstate.setenv('notimportant', {'test': False, 'foo': 'bax'}) + with patch.dict(envstate.__salt__, {'reg.read_value': MagicMock()}): + ret = envstate.setenv('notimportant', {'test': False, 'foo': 'bax'}) self.assertEqual(ret['changes'], {'test': '', 'foo': 'bax'}) self.assertEqual(envstate.os.environ, {'INITIAL': 'initial', 'foo': 'bax', 'test': ''}) From 8c76bbb53d3ce4b3b8eb384f75309a9e793c7ece Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 15:57:32 -0600 Subject: [PATCH 069/300] Some minor doc fixes for dnsutil module so they'll render correctly --- salt/modules/dnsutil.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/modules/dnsutil.py b/salt/modules/dnsutil.py index 948cd720ce..390fbef9b9 100644 --- a/salt/modules/dnsutil.py +++ b/salt/modules/dnsutil.py @@ -2,8 +2,9 @@ ''' Compendium of generic DNS utilities. -.. note: - Some functions in the `dnsutil` execution module depend on `dig`. +.. note:: + + Some functions in the ``dnsutil`` execution module depend on ``dig``. ''' from __future__ import absolute_import @@ -245,7 +246,7 @@ def check_ip(ip_addr): def A(host, nameserver=None): ''' - Return the A record(s) for `host`. + Return the A record(s) for ``host``. Always returns a list. @@ -270,7 +271,7 @@ def A(host, nameserver=None): def AAAA(host, nameserver=None): ''' - Return the AAAA record(s) for `host`. + Return the AAAA record(s) for ``host``. Always returns a list. From c31ded341cf8066a4c70cd2b898e1a7c7ce5d672 Mon Sep 17 00:00:00 2001 From: Thomas Dutrion Date: Sun, 9 Jul 2017 11:34:14 +0200 Subject: [PATCH 070/300] Remove duplicate instruction in Openstack Rackspace config example --- doc/topics/cloud/config.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/topics/cloud/config.rst b/doc/topics/cloud/config.rst index a1739bc701..d7d3ca821e 100644 --- a/doc/topics/cloud/config.rst +++ b/doc/topics/cloud/config.rst @@ -371,7 +371,6 @@ both. compute_name: cloudServersOpenStack protocol: ipv4 compute_region: DFW - protocol: ipv4 user: myuser tenant: 5555555 password: mypass From 30d62f43da5f3b5f824bb763035b5180bcde2201 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 16:24:29 -0600 Subject: [PATCH 071/300] Update minion restart section in FAQ doc for windows Fixes #41116 --- doc/faq.rst | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 8237417816..9911679bb0 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -319,7 +319,27 @@ Restart using states ******************** Now we can apply the workaround to restart the Minion in reliable way. -The following example works on both UNIX-like and Windows operating systems: +The following example works on UNIX-like operating systems: + +.. code-block:: jinja + + {%- if grains['os'] != 'Windows' % + Restart Salt Minion: + cmd.run: + - name: 'salt-call --local service.restart salt-minion' + - bg: True + - onchanges: + - pkg: Upgrade Salt Minion + {%- endif %} + +Note that restarting the salt-minion service on Windows operating systems is +not always necessary when performing an upgrade. The installer stops the +``salt-minion`` service, removes it, deletes the contents of the ``\salt\bin`` +directory, installs the new code, re-creates the ``salt-minion`` service, and +starts it (by default). The restart step **would** be necessary during the +upgrade process, however, if the minion config was edited after the upgrade or +installation. If a minion restart is necessary, the state above can be edited +as follows: .. code-block:: jinja @@ -335,8 +355,8 @@ The following example works on both UNIX-like and Windows operating systems: - pkg: Upgrade Salt Minion However, it requires more advanced tricks to upgrade from legacy version of -Salt (before ``2016.3.0``), where executing commands in the background is not -supported: +Salt (before ``2016.3.0``) on UNIX-like operating systems, where executing +commands in the background is not supported: .. code-block:: jinja From 0c484f89795a5479a80fe683eb5202aa6a170323 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 16:55:43 -0600 Subject: [PATCH 072/300] Fix unit tests on Windows --- salt/modules/win_file.py | 3 - tests/unit/states/test_file.py | 104 +++++++++++++++++++++++++-------- 2 files changed, 81 insertions(+), 26 deletions(-) diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index 12d9319031..461fb5310d 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -1687,9 +1687,6 @@ def check_perms(path, perms = changes[user]['perms'] try: - log.debug('*' * 68) - log.debug(perms) - log.debug('*' * 68) salt.utils.win_dacl.set_permissions( path, user, perms, 'deny', applies_to) ret['changes']['deny_perms'][user] = changes[user] diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index dc58ca2500..2ccce34c29 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -100,7 +100,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): def test_contents_pillar_doesnt_add_more_newlines(self): # make sure the newline - pillar_value = 'i am the pillar value\n' + pillar_value = 'i am the pillar value{0}'.format(os.linesep) self.run_contents_pillar(pillar_value, expected=pillar_value) @@ -167,7 +167,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_empty, - 'file.group_to_gid': mock_empty}): + 'file.group_to_gid': mock_empty, + 'user.info': mock_empty, + 'user.current': mock_user}): comt = ('User {0} does not exist. Group {1} does not exist.'.format(user, group)) ret.update({'comment': comt, 'name': name}) self.assertDictEqual(filestate.symlink(name, target, user=user, @@ -176,7 +178,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, - 'file.is_link': mock_f}): + 'file.is_link': mock_f, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': True}): with patch.object(os.path, 'exists', mock_f): comt = ('Symlink {0} to {1}' @@ -191,7 +195,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, - 'file.is_link': mock_f}): + 'file.is_link': mock_f, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_f): with patch.object(os.path, 'exists', mock_f): @@ -207,7 +213,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_t, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(salt.states.file, '_check_symlink_ownership', mock_t): @@ -224,7 +232,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_f, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(os.path, 'exists', mock_f): @@ -243,7 +253,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_f, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(os.path, 'exists', mock_f): @@ -538,7 +550,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'G12', 'G12', 'G12', 'G12', 'G12']) mock_if = MagicMock(side_effect=[True, False, False, False, False, False, False, False]) - mock_ret = MagicMock(return_value=(ret, None)) + mock_ret = MagicMock(return_value=ret) mock_dict = MagicMock(return_value={}) mock_cp = MagicMock(side_effect=[Exception, True]) mock_ex = MagicMock(side_effect=[Exception, {'changes': {name: name}}, @@ -573,8 +585,14 @@ class TestFileState(TestCase, LoaderModuleMockMixin): self.assertDictEqual(filestate.managed(name, create=False), ret) - comt = ('User salt is not available Group saltstack' - ' is not available') + # Group argument is ignored on Windows systems. Group is set to + # user + if salt.utils.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt, 'result': False}) self.assertDictEqual(filestate.managed(name, user=user, group=group), ret) @@ -689,7 +707,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' Test to ensure that a named directory is present and has the right perms ''' - name = '/etc/grub.conf' + if salt.utils.is_windows(): + name = 'C:\\Windows\\temp\\grub_test' + else: + name = '/etc/grub.conf' user = 'salt' group = 'saltstack' @@ -711,17 +732,28 @@ class TestFileState(TestCase, LoaderModuleMockMixin): mock_t = MagicMock(return_value=True) mock_f = MagicMock(return_value=False) - mock_perms = MagicMock(return_value=(ret, '')) + if salt.utils.is_windows(): + mock_perms = MagicMock(return_value=ret) + else: + mock_perms = MagicMock(return_value=(ret, '')) mock_uid = MagicMock(side_effect=['', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12']) mock_gid = MagicMock(side_effect=['', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12']) + mock_check = MagicMock(return_value=( + None, + 'The directory "{0}" will be changed'.format(name), + {'directory': 'new'})) + mock_error = CommandExecutionError with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.stats': mock_f, 'file.check_perms': mock_perms, - 'file.mkdir': mock_t}): + 'file.mkdir': mock_t}), \ + patch('salt.utils.win_dacl.get_sid', mock_error), \ + patch('os.path.isdir', mock_t), \ + patch('salt.states.file._check_directory_win', mock_check): if salt.utils.is_windows(): comt = ('User salt is not available Group salt' ' is not available') @@ -729,8 +761,8 @@ class TestFileState(TestCase, LoaderModuleMockMixin): comt = ('User salt is not available Group saltstack' ' is not available') ret.update({'comment': comt, 'name': name}) - self.assertDictEqual(filestate.directory(name, user=user, - group=group), ret) + self.assertDictEqual( + filestate.directory(name, user=user, group=group), ret) with patch.object(os.path, 'isabs', mock_f): comt = ('Specified file {0} is not an absolute path' @@ -771,9 +803,19 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.object(os.path, 'isfile', mock_f): with patch.dict(filestate.__opts__, {'test': True}): - comt = ('The following files will be changed:\n{0}:' - ' directory - new\n'.format(name)) - ret.update({'comment': comt, 'result': None, 'pchanges': {'/etc/grub.conf': {'directory': 'new'}}}) + if salt.utils.is_windows(): + comt = 'The directory "{0}" will be changed' \ + ''.format(name) + p_chg = {'directory': 'new'} + ret.update({ + 'comment': comt, + 'result': None, + 'pchanges': p_chg + }) + else: + comt = ('The following files will be changed:\n{0}:' + ' directory - new\n'.format(name)) + ret.update({'comment': comt, 'result': None, 'pchanges': {'/etc/grub.conf': {'directory': 'new'}}}) self.assertDictEqual(filestate.directory(name, user=user, group=group), @@ -845,8 +887,14 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.source_list': mock_lst, 'cp.list_master_dirs': mock_emt, 'cp.list_master': mock_l}): - comt = ('User salt is not available Group saltstack' - ' is not available') + + # Group argument is ignored on Windows systems. Group is set to user + if salt.utils.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt}) self.assertDictEqual(filestate.recurse(name, source, user=user, group=group), ret) @@ -935,7 +983,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' with patch('salt.states.file._load_accumulators', MagicMock(return_value=([], []))): - name = '/etc/hosts' + if salt.utils.is_windows(): + name = 'C:\\Windows\\System32\\drivers\\etc\\hosts' + else: + name = '/etc/hosts' ret = {'name': name, 'result': False, @@ -1312,8 +1363,15 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.get_group': mock_grp, 'file.get_mode': mock_grp, 'file.check_perms': mock_t}): - comt = ('User salt is not available Group ' - 'saltstack is not available') + + # Group argument is ignored on Windows systems. Group is set + # to user + if salt.utils.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt, 'result': False}) self.assertDictEqual(filestate.copy(name, source, user=user, group=group), ret) From a4231c9827acb5a94cf29c9ada21346558cadd3b Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 16:59:42 -0600 Subject: [PATCH 073/300] Fix ret mock for linux --- tests/unit/states/test_file.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index 2ccce34c29..0d190d48e6 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -550,7 +550,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'G12', 'G12', 'G12', 'G12', 'G12']) mock_if = MagicMock(side_effect=[True, False, False, False, False, False, False, False]) - mock_ret = MagicMock(return_value=ret) + if salt.utils.is_windows(): + mock_ret = MagicMock(return_value=ret) + else: + mock_ret = MagicMock(return_value=(ret, None)) mock_dict = MagicMock(return_value={}) mock_cp = MagicMock(side_effect=[Exception, True]) mock_ex = MagicMock(side_effect=[Exception, {'changes': {name: name}}, From 78cdee51d5559b1764ef857b40bcd7c45a26d799 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 17:00:50 -0600 Subject: [PATCH 074/300] Gate boto_elb tests if proper version of moto isn't installed For Python 2 tests, we can use an older version. But when running these tests of Python 3, we need a newer version of moto that supports Python 3. This gates the tests if the expected version of moto is missing. --- tests/unit/modules/test_boto_elb.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/modules/test_boto_elb.py b/tests/unit/modules/test_boto_elb.py index 5d1ee5a9d1..4582155c6f 100644 --- a/tests/unit/modules/test_boto_elb.py +++ b/tests/unit/modules/test_boto_elb.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import logging from copy import deepcopy +import pkg_resources # import Python Third Party Libs # pylint: disable=import-error @@ -45,8 +46,10 @@ except ImportError: # Import Salt Libs import salt.config +import salt.ext.six as six import salt.loader import salt.modules.boto_elb as boto_elb +import salt.utils.versions # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -63,11 +66,32 @@ conn_parameters = {'region': region, 'key': access_key, 'keyid': secret_key, boto_conn_parameters = {'aws_access_key_id': access_key, 'aws_secret_access_key': secret_key} instance_parameters = {'instance_type': 't1.micro'} +required_moto = '0.3.7' +required_moto_py3 = '1.0.1' + + +def _has_required_moto(): + ''' + Returns True or False depending on if ``moto`` is installed and at the correct version, + depending on what version of Python is running these tests. + ''' + if not HAS_MOTO: + return False + else: + moto_version = salt.utils.versions.LooseVersion(pkg_resources.get_distribution('moto').version) + if moto_version < required_moto: + return False + elif six.PY3 and moto_version < required_moto_py3: + return False + + return True @skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(HAS_BOTO is False, 'The boto module must be installed.') @skipIf(HAS_MOTO is False, 'The moto module must be installed.') +@skipIf(_has_required_moto() is False, 'The moto module must be >= to {0} for ' + 'PY2 or {1} for PY3.'.format(required_moto, required_moto_py3)) class BotoElbTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for salt.modules.boto_elb module From 669aaee10d746176e977e4ea16a4c51159cc1768 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 11 Jul 2017 17:09:21 -0600 Subject: [PATCH 075/300] Mock file exists properly --- tests/unit/states/test_file.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index 0d190d48e6..e106423c81 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -710,10 +710,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' Test to ensure that a named directory is present and has the right perms ''' - if salt.utils.is_windows(): - name = 'C:\\Windows\\temp\\grub_test' - else: - name = '/etc/grub.conf' + name = '/etc/grub.conf' user = 'salt' group = 'saltstack' @@ -810,15 +807,15 @@ class TestFileState(TestCase, LoaderModuleMockMixin): comt = 'The directory "{0}" will be changed' \ ''.format(name) p_chg = {'directory': 'new'} - ret.update({ - 'comment': comt, - 'result': None, - 'pchanges': p_chg - }) else: comt = ('The following files will be changed:\n{0}:' ' directory - new\n'.format(name)) - ret.update({'comment': comt, 'result': None, 'pchanges': {'/etc/grub.conf': {'directory': 'new'}}}) + p_chg = {'/etc/grub.conf': {'directory': 'new'}} + ret.update({ + 'comment': comt, + 'result': None, + 'pchanges': p_chg + }) self.assertDictEqual(filestate.directory(name, user=user, group=group), @@ -986,10 +983,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ''' with patch('salt.states.file._load_accumulators', MagicMock(return_value=([], []))): - if salt.utils.is_windows(): - name = 'C:\\Windows\\System32\\drivers\\etc\\hosts' - else: - name = '/etc/hosts' + name = '/etc/hosts' ret = {'name': name, 'result': False, @@ -1008,7 +1002,8 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ret.update({'comment': comt, 'name': name}) self.assertDictEqual(filestate.blockreplace(name), ret) - with patch.object(os.path, 'isabs', mock_t): + with patch.object(os.path, 'isabs', mock_t), \ + patch.object(os.path, 'exists', mock_t): with patch.dict(filestate.__salt__, {'file.blockreplace': mock_t}): with patch.dict(filestate.__opts__, {'test': True}): comt = ('Changes would be made') From 38d9b3d553a424d8163d86db9b0a3e7e32c8e19a Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 11 Jul 2017 17:28:45 -0600 Subject: [PATCH 076/300] Add some clarity to "multiple quotes" section of yaml docs Fixes #41721 --- .../troubleshooting/yaml_idiosyncrasies.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index e1c6383459..bd48691669 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -28,6 +28,7 @@ hit `Enter`. Also, you can convert tabs to 2 spaces by these commands in Vim: Indentation =========== + The suggested syntax for YAML files is to use 2 spaces for indentation, but YAML will follow whatever indentation system that the individual file uses. Indentation of two spaces works very well for SLS files given the @@ -112,8 +113,19 @@ PyYAML will load these values as boolean ``True`` or ``False``. Un-capitalized versions will also be loaded as booleans (``true``, ``false``, ``yes``, ``no``, ``on``, and ``off``). This can be especially problematic when constructing Pillar data. Make sure that your Pillars which need to use the string versions -of these values are enclosed in quotes. Pillars will be parsed twice by salt, -so you'll need to wrap your values in multiple quotes, for example '"false"'. +of these values are enclosed in quotes. Pillars will be parsed twice by salt, +so you'll need to wrap your values in multiple quotes, including double quotation +marks (``" "``) and single quotation marks (``' '``). note that spaces are included +in the quotation type examples for clarity. + +Multiple quoting examples looks like this: + +.. code-block:: yaml + + - '"false"' + - "'True'" + - "'YES'" + - '"No"' The '%' Sign ============ From d1f2a93368f8e06986ce04efe2a6238bb1a8ce51 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Wed, 12 Jul 2017 11:03:34 +0300 Subject: [PATCH 077/300] DOCS: unify hash sum with hash type format --- salt/modules/file.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 9a0f68c7a6..6044296361 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -710,18 +710,21 @@ def check_hash(path, file_hash): hash The hash to check against the file specified in the ``path`` argument. - For versions 2016.11.4 and newer, the hash can be specified without an + + .. versionchanged:: 2016.11.4 + + For this and newer versions the hash can be specified without an accompanying hash type (e.g. ``e138491e9d5b97023cea823fe17bac22``), but for earlier releases it is necessary to also specify the hash type - in the format ``:`` (e.g. - ``md5:e138491e9d5b97023cea823fe17bac22``). + in the format ``=`` (e.g. + ``md5=e138491e9d5b97023cea823fe17bac22``). CLI Example: .. code-block:: bash salt '*' file.check_hash /etc/fstab e138491e9d5b97023cea823fe17bac22 - salt '*' file.check_hash /etc/fstab md5:e138491e9d5b97023cea823fe17bac22 + salt '*' file.check_hash /etc/fstab md5=e138491e9d5b97023cea823fe17bac22 ''' path = os.path.expanduser(path) From f1bc58f6d5a2aec896cf116745452943bec07797 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 6 Jul 2017 11:53:54 +0300 Subject: [PATCH 078/300] Utils: add example of module import --- doc/topics/utils/index.rst | 51 +++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/doc/topics/utils/index.rst b/doc/topics/utils/index.rst index 48728174f5..44380f3541 100644 --- a/doc/topics/utils/index.rst +++ b/doc/topics/utils/index.rst @@ -54,7 +54,7 @@ types like so: salt '*' mymodule.observe_the_awesomeness ''' - print __utils__['foo.bar']() + return __utils__['foo.bar']() Utility modules, like any other kind of Salt extension, support using a :ref:`__virtual__ function ` to conditionally load them, @@ -81,11 +81,56 @@ the ``foo`` utility module with a ``__virtual__`` function. def bar(): return 'baz' +Also you could even write your utility modules in object oriented fashion: + +.. code-block:: python + + # -*- coding: utf-8 -*- + ''' + My utils module + --------------- + + This module contains common functions for use in my other custom types. + ''' + + class Foo(object): + + def __init__(self): + pass + + def bar(self): + return 'baz' + +And import them into other custom modules: + +.. code-block:: python + + # -*- coding: utf-8 -*- + ''' + My awesome execution module + --------------------------- + ''' + + import mymodule + + def observe_the_awesomeness(): + ''' + Prints information from my utility module + + CLI Example: + + .. code-block:: bash + + salt '*' mymodule.observe_the_awesomeness + ''' + foo = mymodule.Foo() + return foo.bar() + These are, of course, contrived examples, but they should serve to show some of the possibilities opened up by writing utility modules. Keep in mind though -that States still have access to all of the execution modules, so it is not +that states still have access to all of the execution modules, so it is not necessary to write a utility module to make a function available to both a -state and an execution module. One good use case for utililty modules is one +state and an execution module. One good use case for utility modules is one where it is necessary to invoke the same function from a custom :ref:`outputter `/returner, as well as an execution module. From 6bb8b8f98cc7e37d08f0f81b4a52cbae1f5409ba Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 7 Jul 2017 18:54:04 +0300 Subject: [PATCH 079/300] Add missing doc for ``utils_dirs`` Minion config option --- doc/ref/configuration/minion.rst | 14 ++++++++++++++ salt/loader.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 6e5dea3ffa..29fbe01dd7 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -1272,6 +1272,20 @@ A list of extra directories to search for Salt renderers render_dirs: - /var/lib/salt/renderers +.. conf_minion:: utils_dirs + +``utils_dirs`` +--------------- + +Default: ``[]`` + +A list of extra directories to search for Salt utilities + +.. code-block:: yaml + + utils_dirs: + - /var/lib/salt/utils + .. conf_minion:: cython_enable ``cython_enable`` diff --git a/salt/loader.py b/salt/loader.py index 60bf78cc56..566597dca4 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -182,7 +182,7 @@ def minion_mods( generated modules in __context__ :param dict utils: Utility functions which should be made available to - Salt modules in __utils__. See `utils_dir` in + Salt modules in __utils__. See `utils_dirs` in salt.config for additional information about configuration. From bd638880e3ceda8c01be33dbf59fa585ebbc2e4a Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 12 Jul 2017 09:36:07 -0600 Subject: [PATCH 080/300] Add mono-spacing to salt-minion reference for consistency --- doc/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/faq.rst b/doc/faq.rst index 9911679bb0..f2a8bace5a 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -332,7 +332,7 @@ The following example works on UNIX-like operating systems: - pkg: Upgrade Salt Minion {%- endif %} -Note that restarting the salt-minion service on Windows operating systems is +Note that restarting the ``salt-minion`` service on Windows operating systems is not always necessary when performing an upgrade. The installer stops the ``salt-minion`` service, removes it, deletes the contents of the ``\salt\bin`` directory, installs the new code, re-creates the ``salt-minion`` service, and From ed89cd0b93ffeee891e91862af47730a575a3c41 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 12 Jul 2017 10:13:26 -0600 Subject: [PATCH 081/300] Use os.sep for path seps --- tests/unit/states/test_winrepo.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/states/test_winrepo.py b/tests/unit/states/test_winrepo.py index a702bbd2de..d5e4321620 100644 --- a/tests/unit/states/test_winrepo.py +++ b/tests/unit/states/test_winrepo.py @@ -67,9 +67,8 @@ class WinrepoTestCase(TestCase, LoaderModuleMockMixin): 'changes': {}, 'result': False, 'comment': ''} - ret.update({'comment': - '{file_roots}/win/repo is ' - 'missing'.format(file_roots=BASE_FILE_ROOTS_DIR)}) + ret.update({'comment': '{0} is missing'.format( + os.sep.join([BASE_FILE_ROOTS_DIR, 'win', 'repo']))}) self.assertDictEqual(winrepo.genrepo('salt'), ret) mock = MagicMock(return_value={'winrepo_dir': 'salt', From f03222384319916b2917be3751ec20a3a1c6f577 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 12 Jul 2017 14:19:35 -0600 Subject: [PATCH 082/300] Handle libcloud objects that throw RepresenterErrors with --out=yaml Fixes #42152 --- salt/cloud/libcloudfuncs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/cloud/libcloudfuncs.py b/salt/cloud/libcloudfuncs.py index 3191251c99..f6465b8f66 100644 --- a/salt/cloud/libcloudfuncs.py +++ b/salt/cloud/libcloudfuncs.py @@ -150,7 +150,7 @@ def avail_locations(conn=None, call=None): ret[img_name] = {} for attr in dir(img): - if attr.startswith('_'): + if attr.startswith('_') or attr == 'driver': continue attr_value = getattr(img, attr) @@ -187,7 +187,7 @@ def avail_images(conn=None, call=None): ret[img_name] = {} for attr in dir(img): - if attr.startswith('_'): + if attr.startswith('_') or attr in ('driver', 'get_uuid'): continue attr_value = getattr(img, attr) if isinstance(attr_value, string_types) and not six.PY3: @@ -222,7 +222,7 @@ def avail_sizes(conn=None, call=None): ret[size_name] = {} for attr in dir(size): - if attr.startswith('_'): + if attr.startswith('_') or attr in ('driver', 'get_uuid'): continue try: From f2250d474acee9ba90e68963109fb7c7847da2ca Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 12 Jul 2017 17:08:23 -0600 Subject: [PATCH 083/300] Add a note about using different styles of quotes. --- doc/topics/troubleshooting/yaml_idiosyncrasies.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index bd48691669..31168afa13 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -115,7 +115,7 @@ versions will also be loaded as booleans (``true``, ``false``, ``yes``, ``no``, Pillar data. Make sure that your Pillars which need to use the string versions of these values are enclosed in quotes. Pillars will be parsed twice by salt, so you'll need to wrap your values in multiple quotes, including double quotation -marks (``" "``) and single quotation marks (``' '``). note that spaces are included +marks (``" "``) and single quotation marks (``' '``). Note that spaces are included in the quotation type examples for clarity. Multiple quoting examples looks like this: @@ -127,6 +127,11 @@ Multiple quoting examples looks like this: - "'YES'" - '"No"' +.. note:: + + When using multiple quotes in this manner, they must be different. Using ``"" ""`` + or ``'' ''`` won't work in this case (spaces are included in examples for clarity). + The '%' Sign ============ From 74689e34626a9c841b9656c51bb55165ec56193c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:12:24 +0200 Subject: [PATCH 084/300] Add ability to use tagged functions in the same set --- salt/states/module.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/states/module.py b/salt/states/module.py index 31a4355464..621c048d16 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -262,6 +262,7 @@ def run(**kwargs): missing = [] tests = [] for func in functions: + func = func.split(':')[0] if func not in __salt__: missing.append(func) elif __opts__['test']: @@ -284,8 +285,9 @@ def run(**kwargs): failures = [] success = [] for func in functions: + _func = func.split(':')[0] try: - func_ret = _call_function(func, returner=kwargs.get('returner'), + func_ret = _call_function(_func, returner=kwargs.get('returner'), func_args=kwargs.get(func)) if not _get_result(func_ret, ret['changes'].get('ret', {})): if isinstance(func_ret, dict): From ea8351362c5aa09f4c622f025e4ed79515a9d544 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:13:19 +0200 Subject: [PATCH 085/300] Bugfix: args gets ignored alongside named parameters --- salt/states/module.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/salt/states/module.py b/salt/states/module.py index 621c048d16..202999e7d8 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -315,22 +315,35 @@ def _call_function(name, returner=None, **kwargs): ''' argspec = salt.utils.args.get_function_argspec(__salt__[name]) func_kw = dict(zip(argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code - argspec.defaults or [])) - func_args = [] - for funcset in kwargs.get('func_args') or {}: - if isinstance(funcset, dict): - func_kw.update(funcset) + argspec.defaults or [])) + arg_type, na_type, kw_type = [], {}, False + for funcset in reversed(kwargs.get('func_args') or []): + if not isinstance(funcset, dict): + kw_type = True + if kw_type: + if isinstance(funcset, dict): + arg_type += funcset.values() + na_type.update(funcset) + else: + arg_type.append(funcset) else: - func_args.append(funcset) + func_kw.update(funcset) + arg_type.reverse() + _exp_prm = len(argspec.args or []) - len(argspec.defaults or []) + _passed_prm = len(arg_type) missing = [] - for arg in argspec.args: - if arg not in func_kw: - missing.append(arg) + if na_type and _exp_prm > _passed_prm: + for arg in argspec.args: + if arg not in func_kw: + missing.append(arg) if missing: raise SaltInvocationError('Missing arguments: {0}'.format(', '.join(missing))) + elif _exp_prm > _passed_prm: + raise SaltInvocationError('Function expects {0} parameters, got only {1}'.format( + _exp_prm, _passed_prm)) - mret = __salt__[name](*func_args, **func_kw) + mret = __salt__[name](*arg_type, **func_kw) if returner is not None: returners = salt.loader.returners(__opts__, __salt__) if returner in returners: From 94c97a8f254e44f1ce40e5919146c5762e419474 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:22:28 +0200 Subject: [PATCH 086/300] Update and correct the error message --- tests/unit/states/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py index 54f3768d37..208829273a 100644 --- a/tests/unit/states/test_module.py +++ b/tests/unit/states/test_module.py @@ -122,7 +122,7 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): ret = module.run(**{CMD: None}) - assert ret['comment'] == "'{0}' failed: Missing arguments: name".format(CMD) + assert ret['comment'] == "'{0}' failed: Function expects 1 parameters, got only 0".format(CMD) def test_run_correct_arg(self): ''' From 8c71257a4b77c5e3ac0a5dbe4f9c1e6c8c60245e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 17:22:46 +0200 Subject: [PATCH 087/300] Call unnamed parameters properly --- tests/unit/states/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py index 208829273a..4588e20ca8 100644 --- a/tests/unit/states/test_module.py +++ b/tests/unit/states/test_module.py @@ -131,7 +131,7 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): ''' with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): - ret = module.run(**{CMD: [{'name': 'Fred'}]}) + ret = module.run(**{CMD: ['Fred']}) assert ret['comment'] == '{0}: Success'.format(CMD) assert ret['result'] From 1391a05d5e9b4d2609d5fc5f818bf718615ecaf0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 18:04:05 +0200 Subject: [PATCH 088/300] Bugfix: syntax error in the example --- doc/topics/releases/2017.7.0.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 52e026e490..7e1d0d3a2d 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -124,13 +124,12 @@ State Module Changes # After run_something: module.run: - mymodule.something: + - mymodule.something: - name: some name - first_arg: one - second_arg: two - do_stuff: True - Since a lot of users are already using :py:func:`module.run ` states, this new behavior must currently be explicitly turned on, to allow users to take their time updating their SLS From 1d7233224b565725b4f87933ff1ce2ae0d25e878 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 18:04:42 +0200 Subject: [PATCH 089/300] Describe function batching --- doc/topics/releases/2017.7.0.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 7e1d0d3a2d..2cce0dbf1a 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -137,6 +137,19 @@ State Module Changes the next feature release of Salt (Oxygen) and the old usage will no longer be supported at that time. + Another feature of the new :py:func:`module.run ` is that + it allows to call many functions in a single batch, such as: + + .. code-block:: yaml + + run_something: + module.run: + - mymodule.function_without_parameters: + - mymodule.another_function: + - myparam + - my_other_param + + In a rare case you have a function that needs to be called several times but To enable the new behavior for :py:func:`module.run `, add the following to the minion config file: From 1e8a56eda5aec6e65b54b034efd6b6c28cd29428 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 12 Jul 2017 18:04:49 +0200 Subject: [PATCH 090/300] Describe function tagging --- doc/topics/releases/2017.7.0.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 2cce0dbf1a..4acd3997b8 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -150,6 +150,23 @@ State Module Changes - my_other_param In a rare case you have a function that needs to be called several times but + with the different parameters, an additional feature of "tagging" is to the + rescue. In order to tag a function, use a colon delimeter. For example + + .. code-block:: yaml + + run_something: + module.run: + - mymodule.same_function:1: + - mymodule.same_function:2: + - myparam + - my_other_param + - mymodule.same_function:3: + - foo: bar + + The example above will run `mymodule.same_function` three times with the + different parameters. + To enable the new behavior for :py:func:`module.run `, add the following to the minion config file: @@ -157,6 +174,7 @@ State Module Changes use_superseded: - module.run + - The default for the ``fingerprint_hash_type`` option used in the ``present`` function in the :mod:`ssh ` state changed from ``md5`` to ``sha256``. From e38d432f906f99447b0beac523952df4199a6072 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 13 Jul 2017 09:19:58 +0200 Subject: [PATCH 091/300] Fix docs --- doc/topics/releases/2017.7.0.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 4acd3997b8..cbda765fd4 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -138,7 +138,7 @@ State Module Changes supported at that time. Another feature of the new :py:func:`module.run ` is that - it allows to call many functions in a single batch, such as: + it allows calling many functions in a single batch, such as: .. code-block:: yaml @@ -149,9 +149,9 @@ State Module Changes - myparam - my_other_param - In a rare case you have a function that needs to be called several times but + In a rare case that you have a function that needs to be called several times but with the different parameters, an additional feature of "tagging" is to the - rescue. In order to tag a function, use a colon delimeter. For example + rescue. In order to tag a function, use a colon delimeter. For example: .. code-block:: yaml From 357dc22f0597b0473b094b4a23b9c5416546250e Mon Sep 17 00:00:00 2001 From: Nicolas Geniteau Date: Thu, 1 Jun 2017 11:39:52 +0200 Subject: [PATCH 092/300] Fix user creation with empty password Emptying the password was only done if the user already existed, not when creating. Signed-off-by: Nicolas Geniteau --- salt/states/user.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/salt/states/user.py b/salt/states/user.py index 57623b2e94..ef02f3db4d 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -126,12 +126,14 @@ def _changes(name, if shell and lusr['shell'] != shell: change['shell'] = shell if 'shadow.info' in __salt__ and 'shadow.default_hash' in __salt__: - if password: + if password and not empty_password: default_hash = __salt__['shadow.default_hash']() if lshad['passwd'] == default_hash \ or lshad['passwd'] != default_hash and enforce_password: if lshad['passwd'] != password: change['passwd'] = password + if empty_password and lshad['passwd'] != '': + change['empty_password'] = True if date and date is not 0 and lshad['lstchg'] != date: change['date'] = date if mindays and mindays is not 0 and lshad['min'] != mindays: @@ -444,9 +446,6 @@ def present(name, if gid_from_name: gid = __salt__['file.group_to_gid'](name) - if empty_password: - __salt__['shadow.del_password'](name) - changes = _changes(name, uid, gid, @@ -497,6 +496,12 @@ def present(name, if key == 'passwd' and not empty_password: __salt__['shadow.set_password'](name, password) continue + if key == 'passwd' and empty_password: + log.warning("No password will be set when empty_password=True") + continue + if key == 'empty_password' and val: + __salt__['shadow.del_password'](name) + continue if key == 'date': __salt__['shadow.set_date'](name, date) continue @@ -662,6 +667,14 @@ def present(name, ' {1}'.format(name, 'XXX-REDACTED-XXX') ret['result'] = False ret['changes']['password'] = 'XXX-REDACTED-XXX' + if empty_password and not password: + __salt__['shadow.del_password'](name) + spost = __salt__['shadow.info'](name) + if spost['passwd'] != '': + ret['comment'] = 'User {0} created but failed to ' \ + 'empty password'.format(name) + ret['result'] = False + ret['changes']['password'] = '' if date: __salt__['shadow.set_date'](name, date) spost = __salt__['shadow.info'](name) From 663874908aee77901378ee744e8079fde608d14c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 12 Jul 2017 10:01:35 -0500 Subject: [PATCH 093/300] pkg.installed: pack name/version into pkgs argument This allows a version of 'latest' to work when just a name and version is passed. --- salt/states/pkg.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 93c7286d9b..260aca1387 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -1248,6 +1248,15 @@ def installed( 'result': True, 'comment': 'No packages to install provided'} + # If just a name (and optionally a version) is passed, just pack them into + # the pkgs argument. + if name and not any((pkgs, sources)): + if version: + pkgs = [{name: version}] + version = None + else: + pkgs = [name] + kwargs['saltenv'] = __env__ refresh = salt.utils.pkg.check_refresh(__opts__, refresh) if not isinstance(pkg_verify, list): @@ -1414,7 +1423,7 @@ def installed( if salt.utils.is_freebsd(): force = True # Downgrades need to be forced. try: - pkg_ret = __salt__['pkg.install'](name, + pkg_ret = __salt__['pkg.install'](name=None, refresh=refresh, version=version, force=force, From 026ccf401a2fc1505392017899f4cc18ce26e07f Mon Sep 17 00:00:00 2001 From: Johannes Scholz Date: Fri, 14 Jul 2017 15:55:23 +0200 Subject: [PATCH 094/300] Force file removal on Windows. Fixes #42295 --- salt/states/file.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index b950acb173..026cd1a913 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -1040,7 +1040,10 @@ def absent(name): ret['comment'] = 'File {0} is set for removal'.format(name) return ret try: - __salt__['file.remove'](name) + if salt.utils.is_windows(): + __salt__['file.remove'](name, force=True) + else: + __salt__['file.remove'](name) ret['comment'] = 'Removed file {0}'.format(name) ret['changes']['removed'] = name return ret From 603f5b7de6fcc97aabc4ec528283a7ea2543033e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 14 Jul 2017 10:21:33 -0500 Subject: [PATCH 095/300] Change "TBD" in versionadded to "2017.7.0" --- salt/modules/boto_elbv2.py | 2 +- salt/states/boto_elbv2.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/boto_elbv2.py b/salt/modules/boto_elbv2.py index 954be55b8d..f9410e45c6 100644 --- a/salt/modules/boto_elbv2.py +++ b/salt/modules/boto_elbv2.py @@ -2,7 +2,7 @@ ''' Connection module for Amazon ALB -.. versionadded:: TBD +.. versionadded:: 2017.7.0 :configuration: This module accepts explicit elb credentials but can also utilize IAM roles assigned to the instance through Instance Profiles. Dynamic diff --git a/salt/states/boto_elbv2.py b/salt/states/boto_elbv2.py index 41566f5916..5146c9d695 100644 --- a/salt/states/boto_elbv2.py +++ b/salt/states/boto_elbv2.py @@ -2,7 +2,7 @@ ''' Manage AWS Application Load Balancer -.. versionadded:: TBD +.. versionadded:: 2017.7.0 Add and remove targets from an ALB target group. @@ -54,6 +54,8 @@ def __virtual__(): def targets_registered(name, targets, region=None, key=None, keyid=None, profile=None): ''' + .. versionadded:: 2017.7.0 + Add targets to an Application Load Balancer target group. This state will not remove targets. name @@ -63,8 +65,6 @@ def targets_registered(name, targets, region=None, key=None, keyid=None, A list of target IDs or a string of a single target that this target group should distribute traffic to. - .. versionadded:: TBD - .. code-block:: yaml add-targets: From c40604694047231691f525b8009a8cc67bf26466 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 14 Jul 2017 11:57:58 -0600 Subject: [PATCH 096/300] Add clarification to salt ssh docs about key auto-generation. Fixes #42267 --- doc/topics/ssh/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/topics/ssh/index.rst b/doc/topics/ssh/index.rst index e55734762f..fa08a96b8c 100644 --- a/doc/topics/ssh/index.rst +++ b/doc/topics/ssh/index.rst @@ -64,7 +64,8 @@ Deploy ssh key for salt-ssh =========================== By default, salt-ssh will generate key pairs for ssh, the default path will be -/etc/salt/pki/master/ssh/salt-ssh.rsa +``/etc/salt/pki/master/ssh/salt-ssh.rsa``. The key generation happens when you run +``salt-ssh`` for the first time. You can use ssh-copy-id, (the OpenSSH key deployment tool) to deploy keys to your servers. From b40f980632ccbac43147bd6071b7658de52f41ef Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 14 Jul 2017 17:26:02 -0600 Subject: [PATCH 097/300] Add more documentation for config options that are missing from master/minion docs --- conf/master | 18 ++ conf/minion | 18 ++ doc/ref/configuration/master.rst | 460 +++++++++++++++++++++++++++---- doc/ref/configuration/minion.rst | 283 ++++++++++++++++++- salt/config/__init__.py | 2 +- 5 files changed, 717 insertions(+), 64 deletions(-) diff --git a/conf/master b/conf/master index 7c09e858b1..1fc76cb89a 100644 --- a/conf/master +++ b/conf/master @@ -302,6 +302,9 @@ # public keys from the minions. Note that this is insecure. #auto_accept: False +# The size of key that should be generated when creating new keys. +#keysize: 2048 + # Time in minutes that an incoming public key with a matching name found in # pki_dir/minion_autosign/keyid is automatically accepted. Expired autosign keys # are removed when the master checks the minion_autosign directory. @@ -916,6 +919,21 @@ #pillar_cache_backend: disk +###### Reactor Settings ##### +########################################### +# Define a salt reactor. See https://docs.saltstack.com/en/latest/topics/reactor/ +#reactor: [] + +#Set the TTL for the cache of the reactor configuration. +#reactor_refresh_interval: 60 + +#Configure the number of workers for the runner/wheel in the reactor. +#reactor_worker_threads: 10 + +#Define the queue size for workers in the reactor. +#reactor_worker_hwm: 10000 + + ##### Syndic settings ##### ########################################## # The Salt syndic is used to pass commands through a master from a higher diff --git a/conf/minion b/conf/minion index 44e5f4e52f..ed2cfde7dc 100644 --- a/conf/minion +++ b/conf/minion @@ -615,6 +615,9 @@ # you do so at your own risk! #open_mode: False +# The size of key that should be generated when creating new keys. +#keysize: 2048 + # Enable permissive access to the salt keys. This allows you to run the # master or minion as root, but have a non-root group be given access to # your pki_dir. To make the access explicit, root must belong to the group @@ -656,6 +659,21 @@ # ssl_version: PROTOCOL_TLSv1_2 +###### Reactor Settings ##### +########################################### +# Define a salt reactor. See https://docs.saltstack.com/en/latest/topics/reactor/ +#reactor: [] + +#Set the TTL for the cache of the reactor configuration. +#reactor_refresh_interval: 60 + +#Configure the number of workers for the runner/wheel in the reactor. +#reactor_worker_threads: 10 + +#Define the queue size for workers in the reactor. +#reactor_worker_hwm: 10000 + + ###### Thread settings ##### ########################################### # Disable multiprocessing support, by default when a minion receives a diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 0ee2f2a4c0..fd310773bf 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -91,64 +91,6 @@ The user to run the Salt processes user: root -.. conf_master:: max_open_files - -``max_open_files`` ------------------- - -Default: ``100000`` - -Each minion connecting to the master uses AT LEAST one file descriptor, the -master subscription connection. If enough minions connect you might start -seeing on the console(and then salt-master crashes): - -.. code-block:: bash - - Too many open files (tcp_listener.cpp:335) - Aborted (core dumped) - -.. code-block:: yaml - - max_open_files: 100000 - -By default this value will be the one of `ulimit -Hn`, i.e., the hard limit for -max open files. - -To set a different value than the default one, uncomment, and configure this -setting. Remember that this value CANNOT be higher than the hard limit. Raising -the hard limit depends on the OS and/or distribution, a good way to find the -limit is to search the internet for something like this: - -.. code-block:: text - - raise max open files hard limit debian - -.. conf_master:: worker_threads - -``worker_threads`` ------------------- - -Default: ``5`` - -The number of threads to start for receiving commands and replies from minions. -If minions are stalling on replies because you have many minions, raise the -worker_threads value. - -Worker threads should not be put below 3 when using the peer system, but can -drop down to 1 worker otherwise. - -.. note:: - When the master daemon starts, it is expected behaviour to see - multiple salt-master processes, even if 'worker_threads' is set to '1'. At - a minimum, a controlling process will start along with a Publisher, an - EventPublisher, and a number of MWorker processes will be started. The - number of MWorker processes is tuneable by the 'worker_threads' - configuration value while the others are not. - -.. code-block:: yaml - - worker_threads: 5 - .. conf_master:: ret_port ``ret_port`` @@ -869,6 +811,74 @@ to socket concurrently. sock_pool_size: 15 +.. conf_master:: ipc_mode + +``ipc_mode`` +------------ + +Default: ``ipc`` + +The ipc strategy. (i.e., sockets versus tcp, etc.) Windows platforms lack +POSIX IPC and must rely on TCP based inter-process communications. ``ipc_mode`` +is set to ``tcp`` by default on Windows. + +.. code-block:: yaml + + ipc_mode: ipc + +.. conf_master:: + +``tcp_master_pub_port`` +----------------------- + +Default: ``4512`` + +The TCP port on which events for the master should be published if ``ipc_mode`` is TCP. + +.. code-block:: yaml + + tcp_master_pub_port: 4512 + +.. conf_master:: tcp_master_pull_port + +``tcp_master_pull_port`` +------------------------ + +Default: ``4513`` + +The TCP port on which events for the master should be pulled if ``ipc_mode`` is TCP. + +.. code-block:: yaml + + tcp_master_pull_port: 4513 + +.. conf_master:: tcp_master_publish_pull + +``tcp_master_publish_pull`` +--------------------------- + +Default: ``4514`` + +The TCP port on which events for the master should be pulled fom and then republished onto +the event bus on the master. + +.. code-block:: yaml + + tcp_master_publish_pull: 4514 + +.. conf_master:: tcp_master_workers + +``tcp_master_workers`` +---------------------- + +Default: ``4515`` + +The TCP port for ``mworkers`` to connect to on the master. + +.. code-block:: yaml + + tcp_master_workers: 4515 + Salt-SSH Configuration ====================== @@ -1103,6 +1113,19 @@ public keys from minions. auto_accept: False +.. conf_master:: keysize + +``keysize`` +----------- + +Default: ``2048`` + +The size of key that should be generated when creating new keys. + +.. code-block:: yaml + + keysize: 2048 + .. conf_master:: autosign_timeout ``autosign_timeout`` @@ -1147,6 +1170,24 @@ minion IDs for which keys will automatically be rejected. Will override both membership in the :conf_master:`autosign_file` and the :conf_master:`auto_accept` setting. +.. conf_master:: permissive_pki_access + +``permissive_pki_access`` +------------------------- + +Default: ``False`` + +Enable permissive access to the salt keys. This allows you to run the +master or minion as root, but have a non-root group be given access to +your pki_dir. To make the access explicit, root must belong to the group +you've given access to. This is potentially quite insecure. If an autosign_file +is specified, enabling permissive_pki_access will allow group access to that +specific file. + +.. code-block:: yaml + + permissive_pki_access: False + .. conf_master:: publisher_acl ``publisher_acl`` @@ -1191,6 +1232,20 @@ This is completely disabled by default. - cmd.* - test.echo +.. conf_master:: sudo_acl + +``sudo_acl`` +------------ + +Default: ``False`` + +Enforce ``publisher_acl`` and ``publisher_acl_blacklist`` when users have sudo +access to the salt command. + +.. code-block:: yaml + + sudo_acl: False + .. conf_master:: external_auth ``external_auth`` @@ -1347,6 +1402,18 @@ Do not disable this unless it is absolutely clear what this does. rotate_aes_key: True +.. conf_master:: publish_session + +``publish_session`` +------------------- + +Default: ``86400`` + +The number of seconds between AES key rotations on the master. + +.. code-block:: yaml + + publish_session: Default: 86400 .. conf_master:: ssl @@ -1378,6 +1445,24 @@ constant names without ssl module prefix: ``CERT_REQUIRED`` or ``PROTOCOL_SSLv23 ``allow_minion_key_revoke`` --------------------------- +Default: ``False`` + +By default, the master deletes its cache of minion data when the key for that +minion is removed. To preserve the cache after key deletion, set +``preserve_minion_cache`` to True. + +WARNING: This may have security implications if compromised minions auth with +a previous deleted minion ID. + +.. code-block:: yaml + + preserve_minion_cache: False + +.. conf_master:: allow_minion_key_revoke + +``allow_minion_key_revoke`` +--------------------------- + Default: ``True`` Controls whether a minion can request its own key revocation. When True @@ -1389,6 +1474,128 @@ the master will drop the request and the minion's key will remain accepted. rotate_aes_key: True + +Master Large Scale Tuning Settings +================================== + +.. conf_master:: max_open_files + +``max_open_files`` +------------------ + +Default: ``100000`` + +Each minion connecting to the master uses AT LEAST one file descriptor, the +master subscription connection. If enough minions connect you might start +seeing on the console(and then salt-master crashes): + +.. code-block:: bash + + Too many open files (tcp_listener.cpp:335) + Aborted (core dumped) + +.. code-block:: yaml + + max_open_files: 100000 + +By default this value will be the one of `ulimit -Hn`, i.e., the hard limit for +max open files. + +To set a different value than the default one, uncomment, and configure this +setting. Remember that this value CANNOT be higher than the hard limit. Raising +the hard limit depends on the OS and/or distribution, a good way to find the +limit is to search the internet for something like this: + +.. code-block:: text + + raise max open files hard limit debian + +.. conf_master:: worker_threads + +``worker_threads`` +------------------ + +Default: ``5`` + +The number of threads to start for receiving commands and replies from minions. +If minions are stalling on replies because you have many minions, raise the +worker_threads value. + +Worker threads should not be put below 3 when using the peer system, but can +drop down to 1 worker otherwise. + +.. note:: + When the master daemon starts, it is expected behaviour to see + multiple salt-master processes, even if 'worker_threads' is set to '1'. At + a minimum, a controlling process will start along with a Publisher, an + EventPublisher, and a number of MWorker processes will be started. The + number of MWorker processes is tuneable by the 'worker_threads' + configuration value while the others are not. + +.. code-block:: yaml + + worker_threads: 5 + +.. conf_master:: pub_hwm + +``pub_hwm`` +----------- + +Default: ``1000`` + +The zeromq high water mark on the publisher interface. + +.. code-block:: yaml + + pub_hwm: 1000 + +.. conf_master:: zmq_backlog + +``zmq_backlog`` +--------------- + +Default: ``1000`` + +The listen queue size of the ZeroMQ backlog. + +.. code-block:: yaml + + zmq_backlog: 1000 + +.. conf_master:: salt_event_pub_hwm +.. conf_master:: event_publisher_pub_hwm + +``salt_event_pub_hwm`` and ``event_publisher_pub_hwm`` +------------------------------------------------------ + +These two ZeroMQ High Water Mark settings, ``salt_event_pub_hwm`` and +``event_publisher_pub_hwm`` are significant for masters with thousands of +minions. When these are insufficiently high it will manifest in random +responses missing in the CLI and even missing from the job cache. Masters +that have fast CPUs and many cores with appropriate ``worker_threads`` +will not need these set as high. + +The ZeroMQ high-water-mark for the ``SaltEvent`` pub socket default is: + +.. code-block:: yaml + + salt_event_pub_hwm: 20000 + +The ZeroMQ high-water-mark for the ``EventPublisher`` pub socket default is: + +.. code-block:: yaml + + event_publisher_pub_hwm: 10000 + +As an example, on single master deployment with 8,000 minions, 2.4GHz CPUs, +24 cores, and 32GiB memory has these settings: + +.. code-block:: yaml + + salt_event_pub_hwm: 128000 + event_publisher_pub_hwm: 64000 + + Master Module Management ======================== @@ -2942,6 +3149,26 @@ configuration. pillar_opts: False +.. conf_master:: pillar_safe_render_error + +``pillar_safe_render_error`` +---------------------------- + +Default: ``True`` + +The pillar_safe_render_error option prevents the master from passing pillar +render errors to the minion. This is set on by default because the error could +contain templating data which would give that minion information it shouldn't +have, like a password! When set ``True`` the error message will only show: + +.. code-block:: shell + + Rendering SLS 'my.sls' failed. Please see master log for details. + +.. code-block:: yaml + + pillar_safe_render_error: True + .. _master-configuration-ext-pillar: .. conf_master:: ext_pillar @@ -3525,6 +3752,63 @@ can be utilized: pillar_cache_backend: disk + +Master Reactor Settings +======================= + +.. conf_master:: reactor + +``reactor`` +----------- + +Default: ``[]`` + +Defines a salt reactor. See the :ref:`Reactor ` documentation for more +information. + +.. code-block:: yaml + + reactor: [] + +.. conf_master:: reactor_refresh_interval + +``reactor_refresh_interval`` +---------------------------- + +Default: ``60`` + +The TTL for the cache of the reactor configuration. + +.. code-block:: yaml + + reactor_refresh_interval: 60 + +.. conf_master:: reactor_worker_threads + +``reactor_worker_threads`` +-------------------------- + +Default: ``10`` + +The number of workers for the runner/wheel in the reactor. + +.. code-block:: yaml + reactor_worker_threads: 10 + +.. conf_master:: reactor_worker_hwm + +``reactor_worker_hwm`` +---------------------- + +Default: ``10000`` + +The queue size for workers in the reactor. + +.. code-block:: yaml + + reactor_worker_hwm: 10000 + + Syndic Server Settings ====================== @@ -3970,6 +4254,64 @@ option then the master will log a warning message. - master.d/* - /etc/roles/webserver + +Keepalive Settings +================== + +.. conf_master:: tcp_keepalive + +``tcp_keepalive`` +----------------- + +Default: ``True`` + +The tcp keepalive interval to set on TCP ports. This setting can be used to tune Salt +connectivity issues in messy network environments with misbehaving firewalls. + +.. code-block:: yaml + + tcp_keepalive: True + +.. conf_master:: tcp_keepalive_cnt + +``tcp_keepalive_cnt`` +--------------------- + +Default: ``-1`` + +Sets the ZeroMQ TCP keepalive count. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_cnt: -1 + +.. conf_master:: tcp_keepalive_idle + +``tcp_keepalive_idle`` +---------------------- + +Default: ``300`` + +Sets ZeroMQ TCP keepalive idle. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_idle: 300 + +.. conf_master:: tcp_keepalive_intvl + +``tcp_keepalive_intvl`` +----------------------- + +Default: ``-1`` + +Sets ZeroMQ TCP keepalive interval. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_intvl': -1 + + .. _winrepo-master-config-opts: Windows Software Repo Settings @@ -4108,7 +4450,7 @@ URL of the repository: .. code-block:: yaml - winrepo_remotes: + winrepo_remotes_ng: - ' https://github.com/saltstack/salt-winrepo-ng.git' Replace ```` with the SHA1 hash of a commit ID. Specifying a commit diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 29fbe01dd7..087f41ef7e 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -750,6 +750,20 @@ seconds each iteration. acceptance_wait_time_max: 0 +.. conf_minion:: rejected_retry + +``rejected_retry`` +------------------ + +Default: ``False`` + +If the master rejects the minion's public key, retry instead of exiting. +Rejected keys will be handled the same as waiting on acceptance. + +.. code-block:: yaml + + rejected_retry: False + .. conf_minion:: random_reauth_delay ``random_reauth_delay`` @@ -1180,7 +1194,7 @@ If certain returners should be disabled, this is the place .. conf_minion:: enable_whitelist_modules ``whitelist_modules`` ----------------------------- +--------------------- Default: ``[]`` (Module whitelisting is disabled. Adding anything to the config option will cause only the listed modules to be enabled. Modules not in the list will @@ -1275,7 +1289,7 @@ A list of extra directories to search for Salt renderers .. conf_minion:: utils_dirs ``utils_dirs`` ---------------- +-------------- Default: ``[]`` @@ -1334,6 +1348,20 @@ below. providers: service: systemd +.. conf_minion:: modules_max_memory + +``modules_max_memory`` +---------------------- + +Default: ``-1`` + +Specify a max size (in bytes) for modules on import. This feature is currently +only supported on *nix operating systems and requires psutil. + +.. code-block:: yaml + + modules_max_memory: -1 + Top File Settings ================= @@ -1465,6 +1493,52 @@ environment lacks one. default_top: dev +.. conf_minion:: startup_states + +``startup_states`` +------------------ + +Default: ``''`` + +States to run when the minion daemon starts. To enable, set ``startup_states`` to: + +- ``highstate``: Execute state.highstate +- ``sls``: Read in the sls_list option and execute the named sls files +- ``top``: Read top_file option and execute based on that file on the Master + +.. code-block:: yaml + + startup_states: '' + +.. conf_minion:: sls_list + +``sls_list`` +------------ + +Default: ``[]`` + +List of states to run when the minion starts up if ``startup_states`` is set to ``sls``. + +.. code-block:: yaml + + sls_list: + - edit.vim + - hyper + +.. conf_minion:: top_file + +``top_file`` +------------ + +Default: ``''`` + +Top file to execute if ``startup_states`` is set to ``top``. + +.. code-block:: yaml + + top_file: '' + + State Management Settings ========================= @@ -1481,7 +1555,7 @@ The default renderer used for local state executions renderer: yaml_jinja -.. conf_master:: test +.. conf_minion:: test ``test`` -------- @@ -1902,6 +1976,35 @@ before the initial key exchange. The master fingerprint can be found by running master_finger: 'ba:30:65:2a:d6:9e:20:4f:d8:b2:f3:a7:d4:65:11:13' +.. conf_minion:: keysize + +``keysize`` +----------- + +Default: ``2048`` + +The size of key that should be generated when creating new keys. + +.. code-block:: yaml + + keysize: 2048 + +.. conf_minion:: permissive_pki_access + +``permissive_pki_access`` +------------------------- + +Default: ``False`` + +Enable permissive access to the salt keys. This allows you to run the +master or minion as root, but have a non-root group be given access to +your pki_dir. To make the access explicit, root must belong to the group +you've given access to. This is potentially quite insecure. + +.. code-block:: yaml + + permissive_pki_access: False + .. conf_minion:: verify_master_pubkey_sign ``verify_master_pubkey_sign`` @@ -2009,7 +2112,7 @@ blocked. If `cmd_whitelist_glob` is NOT SET, then all shell commands are permitt - 'cat /etc/fstab' -.. conf_master:: ssl +.. conf_minion:: ssl ``ssl`` ------- @@ -2035,6 +2138,62 @@ constant names without ssl module prefix: ``CERT_REQUIRED`` or ``PROTOCOL_SSLv23 ssl_version: PROTOCOL_TLSv1_2 +Reactor Settings +================ + +.. conf_minion:: reactor + +``reactor`` +----------- + +Default: ``[]`` + +Defines a salt reactor. See the :ref:`Reactor ` documentation for more +information. + +.. code-block:: yaml + + reactor: [] + +.. conf_minion:: reactor_refresh_interval + +``reactor_refresh_interval`` +---------------------------- + +Default: ``60`` + +The TTL for the cache of the reactor configuration. + +.. code-block:: yaml + + reactor_refresh_interval: 60 + +.. conf_minion:: reactor_worker_threads + +``reactor_worker_threads`` +-------------------------- + +Default: ``10`` + +The number of workers for the runner/wheel in the reactor. + +.. code-block:: yaml + reactor_worker_threads: 10 + +.. conf_minion:: reactor_worker_hwm + +``reactor_worker_hwm`` +---------------------- + +Default: ``10000`` + +The queue size for workers in the reactor. + +.. code-block:: yaml + + reactor_worker_hwm: 10000 + + Thread Settings =============== @@ -2305,6 +2464,62 @@ option then the minion will log a warning message. - /etc/roles/webserver +Keepalive Settings +================== + +.. conf_minion:: tcp_keepalive + +``tcp_keepalive`` +----------------- + +Default: ``True`` + +The tcp keepalive interval to set on TCP ports. This setting can be used to tune Salt +connectivity issues in messy network environments with misbehaving firewalls. + +.. code-block:: yaml + + tcp_keepalive: True + +.. conf_minion:: tcp_keepalive_cnt + +``tcp_keepalive_cnt`` +--------------------- + +Default: ``-1`` + +Sets the ZeroMQ TCP keepalive count. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_cnt: -1 + +.. conf_minion:: tcp_keepalive_idle + +``tcp_keepalive_idle`` +---------------------- + +Default: ``300`` + +Sets ZeroMQ TCP keepalive idle. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_idle: 300 + +.. conf_minion:: tcp_keepalive_intvl + +``tcp_keepalive_intvl`` +----------------------- + +Default: ``-1`` + +Sets ZeroMQ TCP keepalive interval. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_intvl': -1 + Frozen Build Update Settings ============================ @@ -2406,6 +2621,36 @@ out. winrepo_dir: 'D:\winrepo' +.. conf_minion:: winrepo_dir_ng + +``winrepo_dir_ng`` +------------------ + +.. versionadded:: 2015.8.0 + A new :ref:`ng ` repo was added. + +Default: ``/srv/salt/win/repo-ng`` + +Location on the minion where the :conf_minion:`winrepo_remotes_ng` are checked +out for 2015.8.0 and later minions. + +.. code-block:: yaml + + winrepo_dir_ng: /srv/salt/win/repo-ng + +.. conf_minion:: winrepo_source_dir + +``winrepo_source_dir`` +---------------------- + +Default: ``salt://win/repo-ng/`` + +The source location for the winrepo sls files. + +.. code-block:: yaml + + winrepo_source_dir: salt://win/repo-ng/ + .. conf_minion:: winrepo_cachefile .. conf_minion:: win_repo_cachefile @@ -2458,3 +2703,33 @@ URL of the the repository: Replace ```` with the SHA1 hash of a commit ID. Specifying a commit ID is useful in that it allows one to revert back to a previous version in the event that an error is introduced in the latest revision of the repo. + +.. conf_minion:: winrepo_remotes_ng + +``winrepo_remotes_ng`` +---------------------- + +.. versionadded:: 2015.8.0 + A new :ref:`ng ` repo was added. + +Default: ``['https://github.com/saltstack/salt-winrepo-ng.git']`` + +List of git repositories to checkout and include in the winrepo for +2015.8.0 and later minions. + +.. code-block:: yaml + + winrepo_remotes_ng: + - https://github.com/saltstack/salt-winrepo-ng.git + +To specify a specific revision of the repository, prepend a commit ID to the +URL of the repository: + +.. code-block:: yaml + + winrepo_remotes_ng: + - ' https://github.com/saltstack/salt-winrepo-ng.git' + +Replace ```` with the SHA1 hash of a commit ID. Specifying a commit +ID is useful in that it allows one to revert back to a previous version in the +event that an error is introduced in the latest revision of the repo. diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 7c40f82c81..5417790f06 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -331,7 +331,7 @@ VALID_OPTS = { # The TCP port on which minion events should be pulled if ipc_mode is TCP 'tcp_pull_port': int, - # The TCP port on which events for the master should be pulled if ipc_mode is TCP + # The TCP port on which events for the master should be published if ipc_mode is TCP 'tcp_master_pub_port': int, # The TCP port on which events for the master should be pulled if ipc_mode is TCP From c83e6fc696560c674e66eaa2b7ea506e00676b4e Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sat, 15 Jul 2017 23:24:07 +0100 Subject: [PATCH 098/300] Default skip_verify to False This has been partially tackled in https://github.com/saltstack/salt/pull/41528, but exactly the main entry point was missed. --- salt/states/netconfig.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/states/netconfig.py b/salt/states/netconfig.py index 9318e227b4..ed20c1ea6b 100644 --- a/salt/states/netconfig.py +++ b/salt/states/netconfig.py @@ -109,7 +109,7 @@ def managed(name, template_mode='755', saltenv=None, template_engine='jinja', - skip_verify=True, + skip_verify=False, defaults=None, test=False, commit=True, @@ -194,9 +194,11 @@ def managed(name, - :mod:`py` - :mod:`wempy` - skip_verify: True + skip_verify: False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. + + .. versionchanged:: 2017.7.1 test: False Dry run? If set to ``True``, will apply the config, discard and return the changes. Default: ``False`` From c830573a2cb449182b315b33df23a6db28353b02 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sun, 16 Jul 2017 23:09:19 +0100 Subject: [PATCH 099/300] Trailing whitespaces --- salt/states/netconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/netconfig.py b/salt/states/netconfig.py index ed20c1ea6b..1b2ed44d56 100644 --- a/salt/states/netconfig.py +++ b/salt/states/netconfig.py @@ -197,7 +197,7 @@ def managed(name, skip_verify: False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. - + .. versionchanged:: 2017.7.1 test: False From bbba84ce2d3646609a7f3ff84934be1417f970c0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 17 Jul 2017 15:26:54 +0200 Subject: [PATCH 100/300] Bugfix: Jobs scheduled to run at a future time stay pending for Salt minions (bsc#1036125) --- salt/utils/schedule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index a69e5ca418..88118e5b58 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -333,6 +333,7 @@ import logging import errno import random import yaml +import copy # Import Salt libs import salt.config @@ -844,7 +845,7 @@ class Schedule(object): if argspec.keywords: # this function accepts **kwargs, pack in the publish data for key, val in six.iteritems(ret): - kwargs['__pub_{0}'.format(key)] = val + kwargs['__pub_{0}'.format(key)] = copy.deepcopy(val) ret['return'] = self.functions[func](*args, **kwargs) From 774d204d6565924f065fb4b768a7f6c72c20ebe4 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 17 Jul 2017 15:26:54 +0200 Subject: [PATCH 101/300] Bugfix: Jobs scheduled to run at a future time stay pending for Salt minions (bsc#1036125) --- salt/utils/schedule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 231f2331c0..2bb75ffcf1 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -333,6 +333,7 @@ import logging import errno import random import yaml +import copy # Import Salt libs import salt.config @@ -841,7 +842,7 @@ class Schedule(object): if argspec.keywords: # this function accepts **kwargs, pack in the publish data for key, val in six.iteritems(ret): - kwargs['__pub_{0}'.format(key)] = val + kwargs['__pub_{0}'.format(key)] = copy.deepcopy(val) ret['return'] = self.functions[func](*args, **kwargs) From ef1f663fc911bf2ab3a607b92f62652eeea284cc Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 17 Jul 2017 13:44:25 -0600 Subject: [PATCH 102/300] Detect server OS with a desktop release name --- salt/grains/core.py | 28 ++++++++++------------------ salt/version.py | 28 ++++++++++++++++------------ 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 9b4235da10..71ad3d0721 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -984,28 +984,20 @@ def _windows_platform_data(): os_release = platform.release() info = salt.utils.win_osinfo.get_os_version_info() + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function # started reporting the Desktop version instead of the Server version on # Server versions of Windows, so we need to look those up - # Check for Python >=2.7.12 or >=3.5.2 - ver = pythonversion()['pythonversion'] - if ((six.PY2 and - salt.utils.compare_versions(ver, '>=', [2, 7, 12, 'final', 0])) - or - (six.PY3 and - salt.utils.compare_versions(ver, '>=', [3, 5, 2, 'final', 0]))): - # (Product Type 1 is Desktop, Everything else is Server) - if info['ProductType'] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - os_release = server.get(os_release, - 'Grain not found. Update lookup table ' - 'in the `_windows_platform_data` ' - 'function in `grains\\core.py`') + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # (Product Type 1 is Desktop, Everything else is Server) + if info['ProductType'] > 1 and os_release in server: + os_release = server[os_release] service_pack = None if info['ServicePackMajor'] > 0: diff --git a/salt/version.py b/salt/version.py index 6fc4ccbf61..6e5d4e74af 100644 --- a/salt/version.py +++ b/salt/version.py @@ -654,18 +654,22 @@ def system_information(): release = platform.release() if platform.win32_ver()[0]: import win32api # pylint: disable=3rd-party-module-not-gated - if ((sys.version_info.major == 2 and sys.version_info >= (2, 7, 12)) or - (sys.version_info.major == 3 and sys.version_info >= (3, 5, 2))): - if win32api.GetVersionEx(1)[8] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - release = server.get(platform.release(), - 'UNKServer') - _, ver, sp, extra = platform.win32_ver() - version = ' '.join([release, ver, sp, extra]) + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} + # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function + # started reporting the Desktop version instead of the Server version on + # Server versions of Windows, so we need to look those up + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # If this is a Server Platform then `GetVersionEx` will return a number + # greater than 1. + if win32api.GetVersionEx(1)[8] > 1 and release in server: + release = server[release] + _, ver, sp, extra = platform.win32_ver() + version = ' '.join([release, ver, sp, extra]) system = [ ('system', platform.system()), From dc85b5edbe89034778a5d625cced4c5cd9fba6de Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 17 Jul 2017 16:50:54 -0400 Subject: [PATCH 103/300] [2016.3] Update version numbers in doc config for 2017.7.0 release --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 29168d62f6..210bca239e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -213,7 +213,7 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.6' # latest release +latest_release = '2017.7.0' # latest release previous_release = '2016.3.6' # latest release from previous branch previous_release_dir = '2016.3' # path on web server for previous branch next_release = '' # next release From b90b7a7506e6baa9bad7eb21eae72dfe7979e546 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 17 Jul 2017 16:54:46 -0400 Subject: [PATCH 104/300] [2016.11] Update version numbers in doc config for 2017.7.0 release --- doc/conf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index fa2296499c..e891986dc6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -240,9 +240,9 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.6' # latest release -previous_release = '2016.3.6' # latest release from previous branch -previous_release_dir = '2016.3' # path on web server for previous branch +latest_release = '2017.7.0' # latest release +previous_release = '2016.11.6' # latest release from previous branch +previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch @@ -253,8 +253,8 @@ if on_saltstack: copyright = time.strftime("%Y") # < --- START do not merge these settings to other branches START ---> # -build_type = 'latest' # latest, previous, develop, next -release = latest_release # version, latest_release, previous_release +build_type = 'previous' # latest, previous, develop, next +release = previous_release # version, latest_release, previous_release # < --- END do not merge these settings to other branches END ---> # # Set google custom search engine From dc5bb301f7c93814af2122ac86cdde4b0859e8d1 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 17 Jul 2017 16:56:56 -0400 Subject: [PATCH 105/300] [2017.7] Update version numbers in doc config for 2017.7.0 release --- doc/conf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 50357ddcf6..1ecfdfcd99 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -245,9 +245,9 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.6' # latest release -previous_release = '2016.3.6' # latest release from previous branch -previous_release_dir = '2016.3' # path on web server for previous branch +latest_release = '2017.7.0' # latest release +previous_release = '2016.11.6' # latest release from previous branch +previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch @@ -258,8 +258,8 @@ if on_saltstack: copyright = time.strftime("%Y") # < --- START do not merge these settings to other branches START ---> # -build_type = 'develop' # latest, previous, develop, next -release = version # version, latest_release, previous_release +build_type = 'latest' # latest, previous, develop, next +release = latest_release # version, latest_release, previous_release # < --- END do not merge these settings to other branches END ---> # # Set google custom search engine From 8c048403d7327c47e67a9e74961db414ce221e4d Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 17 Jul 2017 14:58:16 -0600 Subject: [PATCH 106/300] Detect Server OS with a desktop release name --- salt/grains/core.py | 27 +++++++++------------------ salt/version.py | 28 ++++++++++++++++------------ 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 9ed045f45d..3f40256299 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -920,28 +920,19 @@ def _windows_platform_data(): os_release = platform.release() info = salt.utils.win_osinfo.get_os_version_info() + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function # started reporting the Desktop version instead of the Server version on # Server versions of Windows, so we need to look those up - # Check for Python >=2.7.12 or >=3.5.2 - ver = pythonversion()['pythonversion'] - if ((six.PY2 and - salt.utils.compare_versions(ver, '>=', [2, 7, 12, 'final', 0])) - or - (six.PY3 and - salt.utils.compare_versions(ver, '>=', [3, 5, 2, 'final', 0]))): - # (Product Type 1 is Desktop, Everything else is Server) - if info['ProductType'] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - os_release = server.get(os_release, - 'Grain not found. Update lookup table ' - 'in the `_windows_platform_data` ' - 'function in `grains\\core.py`') + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + if info['ProductType'] > 1 and os_release in server: + os_release = server[os_release] service_pack = None if info['ServicePackMajor'] > 0: diff --git a/salt/version.py b/salt/version.py index 8549ed9bdf..0c7695568a 100644 --- a/salt/version.py +++ b/salt/version.py @@ -647,18 +647,22 @@ def system_information(): release = platform.release() if platform.win32_ver()[0]: import win32api - if ((sys.version_info.major == 2 and sys.version_info >= (2, 7, 12)) or - (sys.version_info.major == 3 and sys.version_info >= (3, 5, 2))): - if win32api.GetVersionEx(1)[8] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - release = server.get(platform.release(), - 'UNKServer') - _, ver, sp, extra = platform.win32_ver() - version = ' '.join([release, ver, sp, extra]) + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} + # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function + # started reporting the Desktop version instead of the Server version on + # Server versions of Windows, so we need to look those up + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # If this is a Server Platform then `GetVersionEx` will return a number + # greater than 1. + if win32api.GetVersionEx(1)[8] > 1 and release in server: + release = server[release] + _, ver, sp, extra = platform.win32_ver() + version = ' '.join([release, ver, sp, extra]) system = [ ('system', platform.system()), From 526b6ee14d30f34c820f6ece2690bf90756d0a55 Mon Sep 17 00:00:00 2001 From: Corvin Mcpherson Date: Mon, 17 Jul 2017 19:10:15 -0400 Subject: [PATCH 107/300] Multiple documentation fixes The master config 'engines' option takes a list of engines but will accept a single engine instead of the list but adds a warning that it requires a list. The documentation for some of the engine modules pass a single engine instead of a list of engines. This commit fixes the documentation of these engine modules to give correct usage of the 'engines' option. Fixes #42333 Additionally, A few of the documentation pages were missing code-blocks where they were needed to ensure proper formatting. This commit adds the code blocks where they were missing in the engine modules documentation. --- salt/engines/docker_events.py | 4 ++-- salt/engines/hipchat.py | 32 ++++++++++++++++---------------- salt/engines/http_logstash.py | 14 +++++++------- salt/engines/logentries.py | 3 +++ salt/engines/logstash.py | 3 +++ salt/engines/reactor.py | 8 ++++---- salt/engines/redis_sentinel.py | 3 +++ salt/engines/slack.py | 30 +++++++++++++++--------------- 8 files changed, 53 insertions(+), 44 deletions(-) diff --git a/salt/engines/docker_events.py b/salt/engines/docker_events.py index cb6c6dcd2c..9028ad8d82 100644 --- a/salt/engines/docker_events.py +++ b/salt/engines/docker_events.py @@ -50,8 +50,8 @@ def start(docker_url='unix://var/run/docker.sock', .. code-block:: yaml engines: - docker_events: - docker_url: unix://var/run/docker.sock + - docker_events: + docker_url: unix://var/run/docker.sock The config above sets up engines to listen for events from the Docker daemon and publish diff --git a/salt/engines/hipchat.py b/salt/engines/hipchat.py index aede7c9fdc..4493d8b322 100644 --- a/salt/engines/hipchat.py +++ b/salt/engines/hipchat.py @@ -14,22 +14,22 @@ keys make the engine interactive. .. code-block:: yaml engines: - hipchat: - token: 'XXXXXX' - room: 'salt' - control: True - valid_users: - - SomeUser - valid_commands: - - test.ping - - cmd.run - - list_jobs - - list_commands - aliases: - list_jobs: - cmd: jobs.list_jobs - list_commands: - cmd: pillar.get salt:engines:hipchat:valid_commands target=saltmaster + - hipchat: + token: 'XXXXXX' + room: 'salt' + control: True + valid_users: + - SomeUser + valid_commands: + - test.ping + - cmd.run + - list_jobs + - list_commands + aliases: + list_jobs: + cmd: jobs.list_jobs + list_commands: + cmd: pillar.get salt:engines:hipchat:valid_commands target=saltmaster ''' from __future__ import absolute_import diff --git a/salt/engines/http_logstash.py b/salt/engines/http_logstash.py index 3e4b89ad78..78edc0d1b9 100644 --- a/salt/engines/http_logstash.py +++ b/salt/engines/http_logstash.py @@ -12,13 +12,13 @@ them onto a logstash endpoint via HTTP requests. engines: - http_logstash: - url: http://blabla.com/salt-stuff - tags: - - salt/job/*/new - - salt/job/*/ret/* - funs: - - probes.results - - bgp.config + url: http://blabla.com/salt-stuff + tags: + - salt/job/*/new + - salt/job/*/ret/* + funs: + - probes.results + - bgp.config ''' from __future__ import absolute_import diff --git a/salt/engines/logentries.py b/salt/engines/logentries.py index 0fe422edfe..9dd0fc3735 100644 --- a/salt/engines/logentries.py +++ b/salt/engines/logentries.py @@ -24,6 +24,9 @@ master config. :configuration: Example configuration + + .. code-block:: yaml + engines: - logentries: endpoint: data.logentries.com diff --git a/salt/engines/logstash.py b/salt/engines/logstash.py index 7de753b4c8..8965e4dd3d 100644 --- a/salt/engines/logstash.py +++ b/salt/engines/logstash.py @@ -8,6 +8,9 @@ them onto a logstash endpoint. :configuration: Example configuration + + .. code-block:: yaml + engines: - logstash: host: log.my_network.com diff --git a/salt/engines/reactor.py b/salt/engines/reactor.py index b95a8213e3..43829f0a27 100644 --- a/salt/engines/reactor.py +++ b/salt/engines/reactor.py @@ -7,10 +7,10 @@ Example Config in Master or Minion config .. code-block:: yaml engines: - reactor: - refresh_interval: 60 - worker_threads: 10 - worker_hwm: 10000 + - reactor: + refresh_interval: 60 + worker_threads: 10 + worker_hwm: 10000 reactor: - 'salt/cloud/*/destroyed': diff --git a/salt/engines/redis_sentinel.py b/salt/engines/redis_sentinel.py index 8f4e807313..596f610d38 100644 --- a/salt/engines/redis_sentinel.py +++ b/salt/engines/redis_sentinel.py @@ -8,6 +8,9 @@ events based on the channels they are subscribed to. :configuration: Example configuration + + .. code-block:: yaml + engines: - redis_sentinel: hosts: diff --git a/salt/engines/slack.py b/salt/engines/slack.py index d5df1bd574..6b37244daf 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -12,21 +12,21 @@ prefaced with a ``!``. .. code-block:: yaml engines: - slack: - token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' - control: True - valid_users: - - garethgreenaway - valid_commands: - - test.ping - - cmd.run - - list_jobs - - list_commands - aliases: - list_jobs: - cmd: jobs.list_jobs - list_commands: - cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster + - slack: + token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' + control: True + valid_users: + - garethgreenaway + valid_commands: + - test.ping + - cmd.run + - list_jobs + - list_commands + aliases: + list_jobs: + cmd: jobs.list_jobs + list_commands: + cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster :depends: slackclient ''' From 14cf6ce3220394b84b0f0ea4d7d581d70beb471e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 17 Jul 2017 23:36:21 -0500 Subject: [PATCH 108/300] is_windows is a function, not a propery/attribute While not fatal, this could potentially cause problems running this test on Windows. --- tests/integration/modules/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/git.py b/tests/integration/modules/git.py index d404d7101c..548bcd1995 100644 --- a/tests/integration/modules/git.py +++ b/tests/integration/modules/git.py @@ -41,7 +41,7 @@ def _git_version(): git_version = subprocess.Popen( ['git', '--version'], shell=False, - close_fds=False if salt.utils.is_windows else True, + close_fds=False if salt.utils.is_windows() else True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] except OSError: From 915d94219e1915617db5372bc28d610650437353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= Date: Tue, 18 Jul 2017 12:20:34 +0100 Subject: [PATCH 109/300] Allow to check whether a function is available on the AliasesLoader wrapper --- salt/utils/templates.py | 7 +++++++ .../files/file/base/jinja_salt_contains_function.sls | 10 ++++++++++ tests/integration/states/renderers.py | 8 ++++++++ 3 files changed, 25 insertions(+) create mode 100644 tests/integration/files/file/base/jinja_salt_contains_function.sls diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 6370729779..37aa4c22be 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -82,6 +82,13 @@ class AliasedLoader(object): else: return getattr(self.wrapped, name) + def __contains__(self, name): + if name in ALIASES: + salt.utils.warn_until('Nitrogen', ALIAS_WARN) + return ALIASES[name] in self.wrapped + else: + return name in self.wrapped + class AliasedModule(object): ''' diff --git a/tests/integration/files/file/base/jinja_salt_contains_function.sls b/tests/integration/files/file/base/jinja_salt_contains_function.sls new file mode 100644 index 0000000000..24978d1799 --- /dev/null +++ b/tests/integration/files/file/base/jinja_salt_contains_function.sls @@ -0,0 +1,10 @@ +{% set salt_foo_bar_exist = 'foo.bar' in salt %} +{% set salt_test_ping_exist = 'test.ping' in salt %} + +test-ping-exist: + test.succeed_without_changes: + - name: salt_test_ping_exist_{{ salt_test_ping_exist }} + +foo-bar-not-exist: + test.succeed_without_changes: + - name: salt_foo_bar_exist_{{ salt_foo_bar_exist }} diff --git a/tests/integration/states/renderers.py b/tests/integration/states/renderers.py index 61ea0ffb72..1a305ada15 100644 --- a/tests/integration/states/renderers.py +++ b/tests/integration/states/renderers.py @@ -26,6 +26,14 @@ class TestJinjaRenderer(integration.ModuleCase): for state_ret in ret.values(): self.assertTrue(state_ret['result']) + def test_salt_contains_function(self): + ''' + Test if we are able to check if a function exists inside the "salt" + wrapper (AliasLoader) which is available on Jinja templates. + ''' + ret = self.run_function('state.sls', ['jinja_salt_contains_function']) + for state_ret in ret.values(): + self.assertTrue(state_ret['result']) if __name__ == '__main__': from integration import run_tests From 96517d1355eb70efbd2e7d2ac1ef87b267aacbc7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 10:06:55 -0600 Subject: [PATCH 110/300] Add note about patched windows packages --- doc/topics/releases/2017.7.0.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 52e026e490..2d06b20140 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -945,3 +945,13 @@ The ``glusterfs`` state had the following function removed: The ``openvswitch_port`` state had the following change: - The ``type`` option was removed from the ``present`` function. Please use ``tunnel_type`` instead. + +Build Notes +=========== + +Windows Installer Packages +-------------------------- + +Windows Installer packages have been patched with the following PR: 42347_ + +.. _42347: https://github.com/saltstack/salt/pull/42347 From f7c0bb4f46644b19ba18cd7a34e12739cb10a308 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 11:23:16 -0600 Subject: [PATCH 111/300] Remove build and dist directories before install --- pkg/windows/build.bat | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index 9dde8b2e72..cc9a4b34d7 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -110,6 +110,12 @@ if not %errorLevel%==0 ( ) @echo. +:: Remove build and dist directories +@echo %0 :: Remove build and dist directories... +@echo --------------------------------------------------------------------- +rd /s /q "%SrcDir%\build" +rd /s /q "%SrcDir%\dist" + :: Install Current Version of salt @echo %0 :: Install Current Version of salt... @echo --------------------------------------------------------------------- From 0946002713c75caa95a8f5d29c0c72232944dc19 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 11:28:49 -0600 Subject: [PATCH 112/300] Add blank line after delete --- pkg/windows/build.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index cc9a4b34d7..0117718539 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -115,6 +115,7 @@ if not %errorLevel%==0 ( @echo --------------------------------------------------------------------- rd /s /q "%SrcDir%\build" rd /s /q "%SrcDir%\dist" +@echo. :: Install Current Version of salt @echo %0 :: Install Current Version of salt... From a7c910c31e6d88763185789006a49753175e9883 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 18 Jul 2017 11:36:38 -0600 Subject: [PATCH 113/300] Remove build and dist directories before install --- pkg/windows/build.bat | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index 07cf831612..d27d036474 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -42,6 +42,13 @@ if not %errorLevel%==0 ( ) @echo. +:: Remove build and dist directories +@echo %0 :: Remove build and dist directories... +@echo --------------------------------------------------------------------- +rd /s /q "%SrcDir%\build" +rd /s /q "%SrcDir%\dist" +@echo. + :: Install Current Version of salt @echo %0 :: Install Current Version of salt... @echo --------------------------------------------------------------------- From ce1c1b6d282d2c40bb71d124c2594b5a27a7075b Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 18 Jul 2017 18:51:27 -0400 Subject: [PATCH 114/300] Add initial 2017.7.1 Release Notes File --- doc/topics/releases/2017.7.1.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/topics/releases/2017.7.1.rst diff --git a/doc/topics/releases/2017.7.1.rst b/doc/topics/releases/2017.7.1.rst new file mode 100644 index 0000000000..e879d8639a --- /dev/null +++ b/doc/topics/releases/2017.7.1.rst @@ -0,0 +1,5 @@ +============================ +Salt 2017.7.1 Release Notes +============================ + +Version 2017.7.1 is a bugfix release for :ref:`2017.7.0 `. From e721c7eee2b8bdc38a4a5084a5b53fba7faf3870 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 19 Jul 2017 13:21:29 +0300 Subject: [PATCH 115/300] Don't use `key in weakvaluedict` because it could lie. Example: ```python import weakref class O(object): pass d = weakref.WeakValueDictionary() key = 'key' value = O() d[key] = value d[key] # del value key in d # True d.get(key) # None ``` --- salt/crypt.py | 22 ++++++++++++---------- salt/transport/ipc.py | 11 ++++++----- salt/transport/tcp.py | 11 ++++++----- salt/transport/zeromq.py | 21 ++++++--------------- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/salt/crypt.py b/salt/crypt.py index a5411f6ab9..d330594a2a 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -372,17 +372,18 @@ class AsyncAuth(object): loop_instance_map = AsyncAuth.instance_map[io_loop] key = cls.__key(opts) - if key not in loop_instance_map: + auth = loop_instance_map.get(key) + if auth is None: log.debug('Initializing new AsyncAuth for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_auth = object.__new__(cls) - new_auth.__singleton_init__(opts, io_loop=io_loop) - loop_instance_map[key] = new_auth + auth = object.__new__(cls) + auth.__singleton_init__(opts, io_loop=io_loop) + loop_instance_map[key] = auth else: log.debug('Re-using AsyncAuth for {0}'.format(key)) - return loop_instance_map[key] + return auth @classmethod def __key(cls, opts, io_loop=None): @@ -1008,14 +1009,15 @@ class SAuth(AsyncAuth): Only create one instance of SAuth per __key() ''' key = cls.__key(opts) - if key not in SAuth.instances: + auth = SAuth.instances.get(key) + if auth is None: log.debug('Initializing new SAuth for {0}'.format(key)) - new_auth = object.__new__(cls) - new_auth.__singleton_init__(opts) - SAuth.instances[key] = new_auth + auth = object.__new__(cls) + auth.__singleton_init__(opts) + SAuth.instances[key] = auth else: log.debug('Re-using SAuth for {0}'.format(key)) - return SAuth.instances[key] + return auth @classmethod def __key(cls, opts, io_loop=None): diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index c92904196f..2fa7f03e64 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -248,15 +248,16 @@ class IPCClient(object): # FIXME key = str(socket_path) - if key not in loop_instance_map: + client = loop_instance_map.get(key) + if client is None: log.debug('Initializing new IPCClient for path: {0}'.format(key)) - new_client = object.__new__(cls) + client = object.__new__(cls) # FIXME - new_client.__singleton_init__(io_loop=io_loop, socket_path=socket_path) - loop_instance_map[key] = new_client + client.__singleton_init__(io_loop=io_loop, socket_path=socket_path) + loop_instance_map[key] = client else: log.debug('Re-using IPCClient for {0}'.format(key)) - return loop_instance_map[key] + return client def __singleton_init__(self, socket_path, io_loop=None): ''' diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 8aadfef45e..a9001f03a5 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -221,17 +221,18 @@ class AsyncTCPReqChannel(salt.transport.client.ReqChannel): loop_instance_map = cls.instance_map[io_loop] key = cls.__key(opts, **kwargs) - if key not in loop_instance_map: + obj = loop_instance_map.get(key) + if obj is None: log.debug('Initializing new AsyncTCPReqChannel for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj + obj = object.__new__(cls) + obj.__singleton_init__(opts, **kwargs) + loop_instance_map[key] = obj else: log.debug('Re-using AsyncTCPReqChannel for {0}'.format(key)) - return loop_instance_map[key] + return obj @classmethod def __key(cls, opts, **kwargs): diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 5200c07236..94aeb3e21a 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -80,28 +80,19 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): loop_instance_map = cls.instance_map[io_loop] key = cls.__key(opts, **kwargs) - if key not in loop_instance_map: + obj = loop_instance_map.get(key) + if obj is None: log.debug('Initializing new AsyncZeroMQReqChannel for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj + obj = object.__new__(cls) + obj.__singleton_init__(opts, **kwargs) + loop_instance_map[key] = obj log.trace('Inserted key into loop_instance_map id {0} for key {1} and process {2}'.format(id(loop_instance_map), key, os.getpid())) else: log.debug('Re-using AsyncZeroMQReqChannel for {0}'.format(key)) - try: - return loop_instance_map[key] - except KeyError: - # In iterating over the loop_instance_map, we may have triggered - # garbage collection. Therefore, the key is no longer present in - # the map. Re-gen and add to map. - log.debug('Initializing new AsyncZeroMQReqChannel due to GC for {0}'.format(key)) - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj - return loop_instance_map[key] + return obj def __deepcopy__(self, memo): cls = self.__class__ From 664f4b577be632332d428e5d64742ce7ddb4b402 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Wed, 19 Jul 2017 13:24:15 +0200 Subject: [PATCH 116/300] pillar.items pillar_env & pillar_override are never used Fixes a few bugs in pillar_env determination and pillar_enc decryption --- salt/modules/pillar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index d0beb92a06..7293f39244 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -257,8 +257,8 @@ def items(*args, **kwargs): __opts__, __grains__, __opts__['id'], - pillar_override=kwargs.get('pillar'), - pillarenv=kwargs.get('pillarenv') or __opts__['pillarenv']) + pillar_override=pillar_override, + pillarenv=pillarenv) return pillar.compile_pillar() From aa4eed93c82ac4cd8e8206ed5440ab1d257b9d2f Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 19 Jul 2017 11:22:07 -0600 Subject: [PATCH 117/300] Update Python and other reqs --- pkg/osx/build.sh | 2 +- pkg/osx/{build_env.sh => build_env_2.sh} | 24 +++++----- pkg/osx/req.txt | 47 +++++++++---------- pkg/osx/req_ext.txt | 4 +- pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 | 1 - pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 | 1 + .../shasums/libsodium-1.0.12.tar.gz.sha512 | 1 - .../shasums/libsodium-1.0.13.tar.gz.sha512 | 1 + pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 | 1 - pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 | 1 + .../shasums/pkg-config-0.29.2.tar.gz.sha512 | 1 + pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 | 1 - pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 | 1 - pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 | 1 + 14 files changed, 42 insertions(+), 45 deletions(-) rename pkg/osx/{build_env.sh => build_env_2.sh} (95%) delete mode 100644 pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 create mode 100644 pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 delete mode 100644 pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 create mode 100644 pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 delete mode 100644 pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 create mode 100644 pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 create mode 100644 pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 delete mode 100644 pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 delete mode 100644 pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 create mode 100644 pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index e304248328..ab1f51a16b 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -66,7 +66,7 @@ fi # Create the Build Environment ############################################################################ echo -n -e "\033]0;Build: Build Environment\007" -sudo $PKGRESOURCES/build_env.sh +sudo $PKGRESOURCES/build_env_2.sh ############################################################################ # Install Salt diff --git a/pkg/osx/build_env.sh b/pkg/osx/build_env_2.sh similarity index 95% rename from pkg/osx/build_env.sh rename to pkg/osx/build_env_2.sh index f886bd59ad..3e101fd751 100755 --- a/pkg/osx/build_env.sh +++ b/pkg/osx/build_env_2.sh @@ -121,8 +121,8 @@ BUILDDIR=$SCRIPTDIR/build ############################################################################ echo -n -e "\033]0;Build_Env: pkg-config\007" -PKGURL="http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.tar.gz" -PKGDIR="pkg-config-0.29" +PKGURL="http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" +PKGDIR="pkg-config-0.29.2" download $PKGURL @@ -140,8 +140,8 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: libsodium\007" -PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.12.tar.gz" -PKGDIR="libsodium-1.0.12" +PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.13.tar.gz" +PKGDIR="libsodium-1.0.13" download $PKGURL @@ -159,8 +159,8 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: zeromq\007" -PKGURL="http://download.zeromq.org/zeromq-4.1.3.tar.gz" -PKGDIR="zeromq-4.1.3" +PKGURL="http://download.zeromq.org/zeromq-4.1.4.tar.gz" +PKGDIR="zeromq-4.1.4" download $PKGURL @@ -178,13 +178,13 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: OpenSSL\007" -PKGURL="http://openssl.org/source/openssl-1.0.2f.tar.gz" -PKGDIR="openssl-1.0.2f" +PKGURL="http://openssl.org/source/openssl-1.0.2l.tar.gz" +PKGDIR="openssl-1.0.2l" download $PKGURL echo "################################################################################" -echo "Building OpenSSL 1.0.2f" +echo "Building OpenSSL" echo "################################################################################" cd $PKGDIR ./Configure darwin64-x86_64-cc --prefix=/opt/salt --openssldir=/opt/salt/openssl @@ -197,13 +197,13 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: Python\007" -PKGURL="https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tar.xz" -PKGDIR="Python-2.7.12" +PKGURL="https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz" +PKGDIR="Python-2.7.13" download $PKGURL echo "################################################################################" -echo "Building Python 2.7.12" +echo "Building Python" echo "################################################################################" echo "Note there are some test failures" cd $PKGDIR diff --git a/pkg/osx/req.txt b/pkg/osx/req.txt index 3bee76f5b1..2224830a5a 100644 --- a/pkg/osx/req.txt +++ b/pkg/osx/req.txt @@ -1,34 +1,31 @@ -apache-libcloud==0.20.1 +apache-libcloud==2.1.0 backports.ssl_match_hostname==3.5.0.1 -backports_abc==0.4 +backports_abc==0.5 certifi -cffi==1.5.0 -CherryPy==4.0.0 -click==6.2 -enum34==1.1.2 +cffi==1.10.0 +CherryPy==11.0.0 +click==6.7 +enum34==1.1.6 gitdb==0.6.4 -GitPython==1.0.1 -idna==2.0 -ioflo==1.5.0 -ipaddress==1.0.16 -Jinja2==2.9.4 -libnacl==1.4.4 +GitPython==2.1.5 +idna==2.5 +ipaddress==1.0.18 +Jinja2==2.9.6 linode-python==1.1.1 -Mako==1.0.3 -MarkupSafe==0.23 -msgpack-python==0.4.7 -pyasn1==0.1.9 -pycparser==2.14 +Mako==1.0.7 +MarkupSafe==1.0 +msgpack-python==0.4.8 +pyasn1==0.2.3 +pycparser==2.18 pycrypto==2.6.1 -python-dateutil==2.4.2 -python-gnupg==0.3.8 -PyYAML==3.11 -pyzmq==15.2.0 -raet==0.6.5 -requests==2.9.1 +python-dateutil==2.6.1 +python-gnupg==0.4.1 +PyYAML==3.12 +pyzmq==16.0.2 +requests==2.18.1 singledispatch==3.4.0.3 six==1.10.0 smmap==0.9.0 timelib==0.2.4 -tornado==4.3 -vultr==0.1.2 +tornado==4.5.1 +vultr==1.0rc1 diff --git a/pkg/osx/req_ext.txt b/pkg/osx/req_ext.txt index b31fff2b1d..fac429a942 100644 --- a/pkg/osx/req_ext.txt +++ b/pkg/osx/req_ext.txt @@ -1,2 +1,2 @@ -cryptography==1.2.2 -pyOpenSSL==0.15.1 +cryptography==2.0 +pyOpenSSL==17.1.0 diff --git a/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 b/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 deleted file mode 100644 index 513cf3de5a..0000000000 --- a/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 +++ /dev/null @@ -1 +0,0 @@ -6ddbbce47cc49597433d98ca05c2f62f07ed1070807b645602a8e9e9b996adc6fa66fa20a33cd7d23d4e7e925e25071d7301d288149fbe4e8c5f06d5438dda1f ./Python-2.7.12.tar.xz diff --git a/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 b/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 new file mode 100644 index 0000000000..a57f8db9b9 --- /dev/null +++ b/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 @@ -0,0 +1 @@ +f37c9a28ce129d01e63c84d7db627a06402854578f62d17927334ea21ede318e04bbf66e890e3f47c85333e6b19f6e5581fb3f3e27efd24be27017d1b6529c4b ./Python-2.7.13.tar.xz diff --git a/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 b/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 deleted file mode 100644 index 948a5da6b6..0000000000 --- a/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -1e63960da42bcc90945463ae1f5b1355849881dce5bba6d293391f8d6f0932063a5bfd433a071cb184af90ebeab469acc34710587116922144d61f3d7661901b ./libsodium-1.0.12.tar.gz diff --git a/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 b/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 new file mode 100644 index 0000000000..1b5270adbf --- /dev/null +++ b/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 @@ -0,0 +1 @@ +c619b12fdf0b2e59174b6e383a62d5499ebcd720fdbb2c1a41a98a46c285df075202423454b294fefee185432441e943805397d7656f7cd7837de425da623929 ./libsodium-1.0.13.tar.gz diff --git a/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 b/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 deleted file mode 100644 index b107e52b8a..0000000000 --- a/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -50abf6dc94cafd06e7fd20770808bdc675c88daa369e4f752bd584ab17f72a57357c1ca1eca3c83e6745b5a3c9c73c99dce70adaa904d73f6df4c75bc7138351 ./openssl-1.0.2f.tar.gz diff --git a/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 b/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 new file mode 100644 index 0000000000..9dac0d205f --- /dev/null +++ b/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 @@ -0,0 +1 @@ +047d964508ad6025c79caabd8965efd2416dc026a56183d0ef4de7a0a6769ce8e0b4608a3f8393d326f6d03b26a2b067e6e0c750f35b20be190e595e8290c0e3 ./openssl-1.0.2l.tar.gz diff --git a/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 b/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 new file mode 100644 index 0000000000..beb4354b5a --- /dev/null +++ b/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 @@ -0,0 +1 @@ +4861ec6428fead416f5cbbbb0bbad10b9152967e481d4b0ff2eb396a9f297f552984c9bb72f6864a37dcd8fca1d9ccceda3ef18d8f121938dbe4fdf2b870fe75 ./pkg-config-0.29.2.tar.gz diff --git a/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 b/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 deleted file mode 100644 index 8e19abaabb..0000000000 --- a/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -c2857cd67801c0db5d204912453ff6bdc7da3ea61f8b1c6b38983d48dffb958725e7723f909abbc057c7b34a85c27290eec6943808312a75909306076064aa63 ./pkg-config-0.29.tar.gz diff --git a/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 b/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 deleted file mode 100644 index ea85a1581d..0000000000 --- a/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -2c993d18ea44e1cba890e024176af65b85b842ca4f8a22d319be4ace8388ab8828dd706b065f02754025bf271b1d7aa878c3f6655878248f7826452cb2a6134c ./zeromq-4.1.3.tar.gz diff --git a/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 b/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 new file mode 100644 index 0000000000..4f8932f829 --- /dev/null +++ b/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 @@ -0,0 +1 @@ +8a8cf4f52ad78dddfff104bfba0f80bbc12566920906a0fafb9fc340aa92f5577c2923cb2e5346c69835cd2ea1609647a8893c2883cd22c1f0340a720511460c ./zeromq-4.1.4.tar.gz From d9d94fe02fd5610873e21c34b468a910ec2784c8 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 19 Jul 2017 13:29:42 -0600 Subject: [PATCH 118/300] Update old "ref" references to "rev" in git.detached state Fixes #42381 The "rev" kwarg was added to replace "ref" in #38898, however, when switching the state over to "rev", some stacktraces occur due to some remaining "ref" references. --- salt/states/git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 7a710fee10..a269c9b2d9 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -2213,7 +2213,7 @@ def detached(name, # Determine if supplied ref is a hash remote_rev_type = 'ref' - if len(ref) <= 40 \ + if len(rev) <= 40 \ and all(x in string.hexdigits for x in rev): rev = rev.lower() remote_rev_type = 'hash' @@ -2419,7 +2419,7 @@ def detached(name, https_pass=https_pass, ignore_retcode=False) - if 'refs/remotes/'+remote+'/'+ref in all_remote_refs: + if 'refs/remotes/'+remote+'/'+rev in all_remote_refs: checkout_commit_id = all_remote_refs['refs/remotes/' + remote + '/' + rev] elif 'refs/tags/' + rev in all_remote_refs: checkout_commit_id = all_remote_refs['refs/tags/' + rev] From 246a2b3e748e66e081c61d644f62bd26437f03d1 Mon Sep 17 00:00:00 2001 From: Corvin Mcpherson Date: Wed, 19 Jul 2017 19:29:20 -0400 Subject: [PATCH 119/300] Fix documentation misformat in salt.states.file.replace --- salt/states/file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index af3d58d988..731e891b77 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3797,12 +3797,13 @@ def replace(name, A regular expression, to be matched using Python's :py:func:`~re.search`. - ..note:: + .. note:: + If you need to match a literal string that contains regex special characters, you may want to use salt's custom Jinja filter, ``escape_regex``. - ..code-block:: jinja + .. code-block:: jinja {{ 'http://example.com?foo=bar%20baz' | escape_regex }} From 7b8d6cbbd2ff1c2829e83858409072bc32542242 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 19 Jul 2017 17:30:47 -0600 Subject: [PATCH 120/300] Add support for Py3 --- pkg/osx/build.sh | 29 ++++++++--- pkg/osx/{build_env_2.sh => build_env.sh} | 56 ++++++++++++++------ pkg/osx/build_pkg.sh | 59 ++++++++++++++++------ pkg/osx/distribution.xml.dist | 8 +-- pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 | 1 + 5 files changed, 111 insertions(+), 42 deletions(-) rename pkg/osx/{build_env_2.sh => build_env.sh} (86%) create mode 100644 pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index ab1f51a16b..aaf8c78e0d 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -19,14 +19,16 @@ # $1 : : the version of salt to build # (a git tag, not a branch) # (defaults to git-repo state) -# $2 : : the staging area for the package +# $2 : : The version of Python to use in the +# build. Default is 2 +# $3 : : the staging area for the package # defaults to /tmp/salt_pkg # # Example: -# The following will build Salt v2015.8.3 and stage all files -# in /tmp/custom_pkg: +# The following will build Salt v2015.8.3 with Python 2 and +# stage all files in /tmp/custom_pkg: # -# ./build.sh v2015.8.3 /tmp/custom_pkg +# ./build.sh v2015.8.3 2 /tmp/custom_pkg # ############################################################################ echo -n -e "\033]0;Build: Variables\007" @@ -41,9 +43,15 @@ else fi if [ "$2" == "" ]; then + PYVER=2 +else + PYVER=$2 +fi + +if [ "$3" == "" ]; then PKGDIR=/tmp/salt_pkg else - PKGDIR=$2 + PKGDIR=$3 fi ############################################################################ @@ -51,6 +59,11 @@ fi ############################################################################ SRCDIR=`git rev-parse --show-toplevel` PKGRESOURCES=$SRCDIR/pkg/osx +if [ "$PYVER" == "2" ]; then + PYTHON=/opt/salt/bin/python +else + PYTHON=/opt/salt/bin/python3 +fi ############################################################################ # Make sure this is the Salt Repository @@ -66,16 +79,16 @@ fi # Create the Build Environment ############################################################################ echo -n -e "\033]0;Build: Build Environment\007" -sudo $PKGRESOURCES/build_env_2.sh +sudo $PKGRESOURCES/build_env.sh $PYVER ############################################################################ # Install Salt ############################################################################ echo -n -e "\033]0;Build: Install Salt\007" -sudo /opt/salt/bin/python $SRCDIR/setup.py install +sudo $PYTHON $SRCDIR/setup.py install ############################################################################ # Build Package ############################################################################ echo -n -e "\033]0;Build: Package Salt\007" -sudo $PKGRESOURCES/build_pkg.sh $VERSION $PKGDIR +sudo $PKGRESOURCES/build_pkg.sh $VERSION $PYVER $PKGDIR diff --git a/pkg/osx/build_env_2.sh b/pkg/osx/build_env.sh similarity index 86% rename from pkg/osx/build_env_2.sh rename to pkg/osx/build_env.sh index 3e101fd751..3e8ad8dfa5 100755 --- a/pkg/osx/build_env_2.sh +++ b/pkg/osx/build_env.sh @@ -6,18 +6,21 @@ # Authors: CR Oldham, Shane Lee # Date: December 2015 # -# Description: This script sets up a build environment for salt on macOS. +# Description: This script sets up a build environment for Salt on macOS. # # Requirements: # - XCode Command Line Tools (xcode-select --install) # # Usage: -# This script is not passed any parameters +# This script can be passed 1 parameter +# $1 : : the version of Python to use for the +# build environment. Default is 2 # # Example: -# The following will set up a build environment for salt on macOS +# The following will set up a Python 3 build environment for Salt +# on macOS # -# ./dev_env.sh +# ./dev_env.sh 3 # ############################################################################ @@ -31,6 +34,15 @@ quit_on_error() { exit -1 } +############################################################################ +# Check passed parameters, set defaults +############################################################################ +if [ "$1" == "" ]; then + PYVER=2 +else + PYVER=$1 +fi + ############################################################################ # Parameters Required for the script to function properly ############################################################################ @@ -45,6 +57,15 @@ SHADIR=$SCRIPTDIR/shasums PKG_CONFIG_PATH=/opt/salt/lib/pkgconfig CFLAGS="-I/opt/salt/include" LDFLAGS="-L/opt/salt/lib" +if [ "$PYVER" == "2" ]; then + PYDIR=/opt/salt/lib/python2.7 + PYTHON=/opt/salt/bin/python + PIP=/opt/salt/bin/pip +else + PYDIR=/opt/salt/lib/python3.5 + PYTHON=/opt/salt/bin/python3 + PIP=/opt/salt/bin/pip3 +fi ############################################################################ # Determine Which XCode is being used (XCode or XCode Command Line Tools) @@ -197,8 +218,13 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: Python\007" -PKGURL="https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz" -PKGDIR="Python-2.7.13" +if [ "$PYVER" == "2" ]; then + PKGURL="https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz" + PKGDIR="Python-2.7.13" +else + PKGURL="https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tar.xz" + PKGDIR="Python-3.5.3" +fi download $PKGURL @@ -227,23 +253,23 @@ cd $BUILDDIR echo "################################################################################" echo "Installing Salt Dependencies with pip (normal)" echo "################################################################################" -sudo -H /opt/salt/bin/pip install \ - -r $SRCDIR/pkg/osx/req.txt \ - --no-cache-dir +sudo -H $PIP install \ + -r $SRCDIR/pkg/osx/req.txt \ + --no-cache-dir echo "################################################################################" echo "Installing Salt Dependencies with pip (build_ext)" echo "################################################################################" -sudo -H /opt/salt/bin/pip install \ - -r $SRCDIR/pkg/osx/req_ext.txt \ - --global-option=build_ext \ - --global-option="-I/opt/salt/include" \ - --no-cache-dir +sudo -H $PIP install \ + -r $SRCDIR/pkg/osx/req_ext.txt \ + --global-option=build_ext \ + --global-option="-I/opt/salt/include" \ + --no-cache-dir echo "--------------------------------------------------------------------------------" echo "Create Symlink to certifi for openssl" echo "--------------------------------------------------------------------------------" -sudo ln -s /opt/salt/lib/python2.7/site-packages/certifi/cacert.pem /opt/salt/openssl/cert.pem +sudo ln -s $PYDIR/site-packages/certifi/cacert.pem /opt/salt/openssl/cert.pem echo -n -e "\033]0;Build_Env: Finished\007" diff --git a/pkg/osx/build_pkg.sh b/pkg/osx/build_pkg.sh index 86ad9aa450..80f5c4f734 100755 --- a/pkg/osx/build_pkg.sh +++ b/pkg/osx/build_pkg.sh @@ -15,13 +15,16 @@ # This script can be passed 2 parameters # $1 : : the version name to give the package (overrides # version of the git repo) (Defaults to the git repo version) -# $2 : : the staging area for the package defaults to +# $2 : : the version of python that was built (defaults +# to 2) +# $3 : : the staging area for the package defaults to # /tmp/salt_pkg # # Example: -# The following will build Salt and stage all files in /tmp/salt_pkg: +# The following will build Salt version 2017.7.0 with Python 3 and +# stage all files in /tmp/salt_pkg: # -# ./build.sh +# ./build.sh 2017.7.0 3 # ############################################################################ @@ -45,11 +48,18 @@ else VERSION=$1 fi -# Get/Set temp directory +# Get/Set Python Version if [ "$2" == "" ]; then + PYVER=2 +else + PYVER=$2 +fi + +# Get/Set temp directory +if [ "$3" == "" ]; then PKGDIR=/tmp/salt_pkg else - PKGDIR=$2 + PKGDIR=$3 fi CPUARCH=`uname -m` @@ -114,7 +124,11 @@ sudo rm -rdf $PKGDIR/opt/salt/lib/engines sudo rm -rdf $PKGDIR/opt/salt/share/aclocal sudo rm -rdf $PKGDIR/opt/salt/share/doc sudo rm -rdf $PKGDIR/opt/salt/share/man/man1/pkg-config.1 -sudo rm -rdf $PKGDIR/opt/salt/lib/python2.7/test +if [ "$PYVER" == "2" ]; then + sudo rm -rdf $PKGDIR/opt/salt/lib/python2.7/test +else + sudo rm -rdf $PKGDIR/opt/salt/lib/python3.5/test +fi echo -n -e "\033]0;Build_Pkg: Remove compiled python files\007" sudo find $PKGDIR/opt/salt -name '*.pyc' -type f -delete @@ -133,15 +147,30 @@ cp $SRCDIR/conf/master $PKGDIR/etc/salt/master.dist ############################################################################ echo -n -e "\033]0;Build_Pkg: Add Version to .xml\007" +if [ "$PYVER" == "2" ]; then + TITLE="Salt $VERSION" + DESC="Salt $VERSION with Python 2" +else + TITLE="Salt $VERSION (Python 3)" + DESC="Salt $VERSION with Python 3" +fi + cd $PKGRESOURCES cp distribution.xml.dist distribution.xml -SEDSTR="s/@VERSION@/$VERSION/" -echo $SEDSTR -sed -i '' $SEDSTR distribution.xml +SEDSTR="s/@TITLE@/$TITLE/g" +sed -E -i '' "$SEDSTR" distribution.xml -SEDSTR="s/@CPUARCH@/$CPUARCH/" -echo $SEDSTR -sed -i '' $SEDSTR distribution.xml +SEDSTR="s/@DESC@/$DESC/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@VERSION@/$VERSION/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@PYVER@/$PYVER/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@CPUARCH@/$CPUARCH/g" +sed -i '' "$SEDSTR" distribution.xml ############################################################################ # Build the Package @@ -152,10 +181,10 @@ pkgbuild --root=$PKGDIR \ --scripts=pkg-scripts \ --identifier=com.saltstack.salt \ --version=$VERSION \ - --ownership=recommended salt-src-$VERSION-$CPUARCH.pkg + --ownership=recommended salt-src-$VERSION-py$PYVER-$CPUARCH.pkg productbuild --resources=pkg-resources \ --distribution=distribution.xml \ - --package-path=salt-src-$VERSION-$CPUARCH.pkg \ - --version=$VERSION salt-$VERSION-$CPUARCH.pkg + --package-path=salt-src-$VERSION-py$PYVER-$CPUARCH.pkg \ + --version=$VERSION salt-$VERSION-py$PYVER-$CPUARCH.pkg diff --git a/pkg/osx/distribution.xml.dist b/pkg/osx/distribution.xml.dist index 083fef44f6..d31063f5f4 100644 --- a/pkg/osx/distribution.xml.dist +++ b/pkg/osx/distribution.xml.dist @@ -1,6 +1,6 @@ - Salt @VERSION@ + @TITLE@ com.saltstack.salt @@ -25,7 +25,7 @@ salt-src-@VERSION@-@CPUARCH@.pkg + auth="root">salt-src-@VERSION@-py@PYVER@-@CPUARCH@.pkg @@ -34,8 +34,8 @@ diff --git a/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 b/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 new file mode 100644 index 0000000000..d4ab61d5f7 --- /dev/null +++ b/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 @@ -0,0 +1 @@ +bbcc20e315c63dbc8901d7e7bfa29d4dbdad9335720757d8d679730319fd1d9fcfdb55cf62d620c9b052134170f162c28d653a8af60923185b8932524d827864 ./Python-3.5.3.tar.xz From 4ae3911f0143c2e31e4559a0ae263e069d83afe0 Mon Sep 17 00:00:00 2001 From: Richard Clark Date: Wed, 19 Jul 2017 20:30:49 -0700 Subject: [PATCH 121/300] Fix file.managed check_cmd file not found - Issue #42404 --- salt/states/file.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 731e891b77..de86f8c68b 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -2433,12 +2433,16 @@ def managed(name, if sfn and os.path.isfile(sfn): os.remove(sfn) return ret + + if sfn and os.path.isfile(sfn): + os.remove(sfn) + # Since we generated a new tempfile and we are not returning here # lets change the original sfn to the new tempfile or else we will # get file not found - if sfn and os.path.isfile(sfn): - os.remove(sfn) - sfn = tmp_filename + + sfn = tmp_filename + else: ret = {'changes': {}, 'comment': '', From 1c0574d05e68a2f519c6b5f0857fc66894c48711 Mon Sep 17 00:00:00 2001 From: Emmanuel Bouton Date: Thu, 20 Jul 2017 17:01:10 +0200 Subject: [PATCH 122/300] Fix error message when tornado or pycurl is not installed --- salt/utils/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index 0a3c32e3a9..e34280cbfb 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -456,8 +456,8 @@ def query(url, # We want to use curl_http if we have a proxy defined if proxy_host and proxy_port: if HAS_CURL_HTTPCLIENT is False: - ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl, but the ' - 'pycurl library does not seem to be installed') + ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl and tornado, ' + 'but the libraries does not seem to be installed') log.error(ret['error']) return ret From 9d66e273c413527468c16881a5a2ba2496fac855 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 20 Jul 2017 10:37:08 -0600 Subject: [PATCH 123/300] Fix hard coded pip path --- pkg/osx/build_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osx/build_env.sh b/pkg/osx/build_env.sh index 3e8ad8dfa5..5d7f1bac5a 100755 --- a/pkg/osx/build_env.sh +++ b/pkg/osx/build_env.sh @@ -241,7 +241,7 @@ sudo -H $MAKE install ############################################################################ # upgrade pip ############################################################################ -sudo -H /opt/salt/bin/pip install --upgrade pip +sudo -H $PIP install --upgrade pip ############################################################################ # Download and install salt python dependencies From ac0e04af72caf61ca8c172b1f0dfda8663711bac Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 20 Jul 2017 10:59:24 -0600 Subject: [PATCH 124/300] Remove build and dist, sign pkgs --- pkg/osx/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index aaf8c78e0d..c411840a91 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -64,6 +64,7 @@ if [ "$PYVER" == "2" ]; then else PYTHON=/opt/salt/bin/python3 fi +CPUARCH=`uname -m` ############################################################################ # Make sure this is the Salt Repository @@ -85,6 +86,8 @@ sudo $PKGRESOURCES/build_env.sh $PYVER # Install Salt ############################################################################ echo -n -e "\033]0;Build: Install Salt\007" +sudo rm -rm $SRCDIR/build +sudo rm -rm $SRCDIR/dist sudo $PYTHON $SRCDIR/setup.py install ############################################################################ @@ -92,3 +95,8 @@ sudo $PYTHON $SRCDIR/setup.py install ############################################################################ echo -n -e "\033]0;Build: Package Salt\007" sudo $PKGRESOURCES/build_pkg.sh $VERSION $PYVER $PKGDIR + +############################################################################ +# Sign Package +############################################################################ +sudo $PKGRESOURCES/build_sig.sh salt-$VERSION-py$PYVER-$CPUARCH.pkg salt-$VERSION-py$PYVER-$CPUARCH-signed.pkg From 9f4eb80d9072a3fe0e7f3d23528f57b928561f79 Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 20 Jul 2017 14:26:30 -0600 Subject: [PATCH 125/300] Add a test.arg variant that cleans the pub kwargs by default --- salt/modules/test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/modules/test.py b/salt/modules/test.py index 1dc0c28a33..6094b05981 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -311,6 +311,18 @@ def arg_repr(*args, **kwargs): return {"args": repr(args), "kwargs": repr(kwargs)} +def arg_clean(*args, **kwargs): + ''' + Like test.arg but cleans kwargs of the __pub* items + CLI Example: + + .. code-block:: bash + + salt '*' test.arg_clean 1 "two" 3.1 txt="hello" wow='{a: 1, b: "hello"}' + ''' + return dict(args=args, kwargs=salt.utils.clean_kwargs(**kwargs)) + + def fib(num): ''' Return the num-th Fibonacci number, and the time it took to compute in From 622ff5be40df34dadf1f83c4f1dfb77fabdcf25a Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 20 Jul 2017 10:48:04 -0600 Subject: [PATCH 126/300] Add LocalClient.cmd test for arg/kwarg parsing --- tests/integration/client/test_kwarg.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/integration/client/test_kwarg.py b/tests/integration/client/test_kwarg.py index bb5b369de0..a9ec8d27ff 100644 --- a/tests/integration/client/test_kwarg.py +++ b/tests/integration/client/test_kwarg.py @@ -6,7 +6,6 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.case import ModuleCase - class StdTest(ModuleCase): ''' Test standard client calls @@ -94,3 +93,25 @@ class StdTest(ModuleCase): ret = self.client.cmd('minion', 'test.ping', full_return=True) for mid, data in ret.items(): self.assertIn('retcode', data) + + def test_cmd_arg_kwarg_parsing(self): + ret = self.client.cmd('minion', 'test.arg_clean', + arg=[ + 'foo', + 'bar=off', + 'baz={qux: 123}' + ], + kwarg={ + 'quux': 'Quux', + }) + + self.assertEqual(ret['minion'], { + 'args': ['foo'], + 'kwargs': { + 'bar': False, + 'baz': { + 'qux': 123, + }, + 'quux': 'Quux', + }, + }) From 31273c7ec1ee14cdde4f8780280cb0c8019d6b30 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 20 Jul 2017 15:03:14 -0500 Subject: [PATCH 127/300] Modify our custom YAML loader to treat unicode literals as unicode strings --- salt/utils/yamlloader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/utils/yamlloader.py b/salt/utils/yamlloader.py index a51d1f1f25..3d07c005c7 100644 --- a/salt/utils/yamlloader.py +++ b/salt/utils/yamlloader.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import warnings # Import third party libs +import re import yaml from yaml.nodes import MappingNode, SequenceNode from yaml.constructor import ConstructorError @@ -101,6 +102,11 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): # an empty string. Change it to '0'. if node.value == '': node.value = '0' + elif node.tag == 'tag:yaml.org,2002:str': + # If any string comes in as a quoted unicode literal, eval it into + # the proper unicode string type. + if re.match(r'^u([\'"]).+\1$', node.value, flags=re.IGNORECASE): + node.value = eval(node.value, {}, {}) # pylint: disable=W0123 return super(SaltYamlSafeLoader, self).construct_scalar(node) def flatten_mapping(self, node): From 0fd39498c016e1cb3373872bc8fc9385a8ecea99 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Jul 2017 13:48:01 -0700 Subject: [PATCH 128/300] Updating the versions function inside the manage runner to account for when a minion is offline and we are unable to determine it's version. --- salt/runners/manage.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index 249c875c57..ebf6471d9a 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -652,6 +652,7 @@ def versions(): return ret labels = { + -2: 'Minion offline', -1: 'Minion requires update', 0: 'Up to date', 1: 'Minion newer than master', @@ -663,12 +664,19 @@ def versions(): master_version = salt.version.__saltstack_version__ for minion in minions: - minion_version = salt.version.SaltStackVersion.parse(minions[minion]) - ver_diff = cmp(minion_version, master_version) + if not minions[minion]: + minion_version = False + ver_diff = -2 + else: + minion_version = salt.version.SaltStackVersion.parse(minions[minion]) + ver_diff = cmp(minion_version, master_version) if ver_diff not in version_status: version_status[ver_diff] = {} - version_status[ver_diff][minion] = minion_version.string + if minion_version: + version_status[ver_diff][minion] = minion_version.string + else: + version_status[ver_diff][minion] = minion_version # Add version of Master to output version_status[2] = master_version.string From 540977b4b1bc2468c9eac8f216d0b796e3a4f4df Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Thu, 29 Jun 2017 10:10:47 -0500 Subject: [PATCH 129/300] Fix: Reactor emits critical error Under normal usage, the reactor will emit this critical error: ``` [CRITICAL] kwargs must be passed inside the low data within the 'kwarg' key. See usage of salt.utils.args.parse_input() and salt.minion.load_args_and_kwargs() elsewhere in the codebase. ``` It seems like only `salt.utils.reactor.Reactor` uses `salt.state.Compiler`. Due to this, it appears safe to customize `Compiler.compile_high_data` for usage only by the reactor. Since reactor arguments are always named, we ensure that each 'chunk' has an `arg` field that is an empty list and a `kwarg` field that contains all the named arguments for use with the given function call. This conforms to the format expected by `salt.client.mixins.SyncClientMixin._low`. Signed-off-by: Sergey Kizunov --- salt/state.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/state.py b/salt/state.py index 44fb73b97d..d3d7cd3f81 100644 --- a/salt/state.py +++ b/salt/state.py @@ -577,6 +577,8 @@ class Compiler(object): if '__env__' in body: chunk['__env__'] = body['__env__'] chunk['__id__'] = name + chunk['arg'] = [] + chunk['kwarg'] = {} for arg in run: if isinstance(arg, six.string_types): funcs.add(arg) @@ -589,7 +591,7 @@ class Compiler(object): names.append(_name) continue else: - chunk.update(arg) + chunk['kwarg'].update(arg) if names: name_order = 1 for entry in names: From 635810b3e3bc03a9a733ba63203238a983548c9a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Jul 2017 16:50:42 -0700 Subject: [PATCH 130/300] Updating the slack engine in 2016.11 to pass the args and kwrags correctly to LocalClient --- salt/engines/slack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 6b37244daf..1ad9e05ce6 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -202,7 +202,7 @@ def start(token, # Default to trying to run as a client module. else: local = salt.client.LocalClient() - ret = local.cmd('{0}'.format(target), cmd, args, kwargs) + ret = local.cmd('{0}'.format(target), cmd, arg=args, kwarg=kwargs) if ret: return_text = json.dumps(ret, sort_keys=True, indent=1) From f411cfc2a9ce5d65e372bc7c28afec8f09fa15dc Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Jul 2017 16:57:10 -0700 Subject: [PATCH 131/300] Updating the slack engine in 2017.7 to pass the args and kwrags correctly to LocalClient --- salt/engines/slack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 1709364832..efddfda0ea 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -255,7 +255,7 @@ def start(token, # Default to trying to run as a client module. else: local = salt.client.LocalClient() - ret = local.cmd('{0}'.format(target), cmd, args, kwargs, tgt_type='{0}'.format(tgt_type)) + ret = local.cmd('{0}'.format(target), cmd, arg=args, kwarg=kwargs, tgt_type='{0}'.format(tgt_type)) if ret: return_text = json.dumps(ret, sort_keys=True, indent=1) From 3f4a918f733b3875ef31af0c685ad4b779f26998 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 21 Jul 2017 09:45:30 -0400 Subject: [PATCH 132/300] update windows urls to new py2/py3 naming scheme --- doc/conf.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 1ecfdfcd99..6e828afc77 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -320,11 +320,21 @@ rst_prolog = """\ .. _`salt-packagers`: https://groups.google.com/forum/#!forum/salt-packagers .. |windownload| raw:: html -

x86: Salt-Minion-{release}-x86-Setup.exe - | md5

+

Python2 x86: Salt-Minion-{release}-x86-Setup.exe + | md5

+ +

Python2 AMD64: Salt-Minion-{release}-AMD64-Setup.exe + | md5

+

Python3 x86: Salt-Minion-{release}-x86-Setup.exe + | md5

+ +

Python3 AMD64: Salt-Minion-{release}-AMD64-Setup.exe + | md5

-

AMD64: Salt-Minion-{release}-AMD64-Setup.exe - | md5

.. |osxdownload| raw:: html From 98b661406ef90f0a75eef418a0df54af2aab34d5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Jul 2017 09:21:38 -0500 Subject: [PATCH 133/300] Document future renaming of new rand_str jinja filter The name and the documentation for this filter are entirely inaccurate for what it actually does. It will be renamed in the Oxygen release. --- doc/topics/jinja/index.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 5548b29646..9c0dcdfb38 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -876,16 +876,22 @@ Returns: ------------ .. versionadded:: 2017.7.0 +.. versionadded:: Oxygen + Renamed from ``rand_str`` to ``random_hash`` to more accurately describe + what the filter does. -Generate a random string and applies a hash. Default hashing: md5. +Generates a random number between 1 and the number passed to the filter, and +then hashes it. The default hash type is the one specified by the minion's +:conf_minion:`hash_type` config option, but an alternate hash type can be +passed to the filter as an argument. Example: .. code-block:: jinja - {% set passwd_length = 17 %} - {{ passwd_length | rand_str }} - {{ passwd_length | rand_str('sha512') }} + {% set num_range = 99999999 %} + {{ num_range | rand_str }} + {{ num_range | rand_str('sha512') }} Returns: From ea457aa0a564977668791e13e235788e013d55b8 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 21 Jul 2017 09:08:26 -0600 Subject: [PATCH 134/300] Remove ALIASES block from template util These alias warnings were removed in #38218 for 2017.7. --- salt/utils/templates.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 8a2514e796..b4bf049dc1 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -73,11 +73,7 @@ class AliasedLoader(object): return getattr(self.wrapped, name) def __contains__(self, name): - if name in ALIASES: - salt.utils.warn_until('Nitrogen', ALIAS_WARN) - return ALIASES[name] in self.wrapped - else: - return name in self.wrapped + return name in self.wrapped class AliasedModule(object): From c0df0137f5f9c743bbbe512b3904aec0b6264200 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 21 Jul 2017 10:18:20 -0600 Subject: [PATCH 135/300] Bump warning version from Oxygen to Fluorine in roster cache The change was introduced in 2017.7 - we need to provide at least 2 feature releases before removing this warning. Therefore, this PR bumps the version we will warn until from Oxygen to Fluorine. --- salt/roster/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/roster/cache.py b/salt/roster/cache.py index bb03b5f0f3..cf4244ff3a 100644 --- a/salt/roster/cache.py +++ b/salt/roster/cache.py @@ -129,7 +129,7 @@ def targets(tgt, tgt_type='glob', **kwargs): # pylint: disable=W0613 'host': ('ipv6-private', 'ipv6-global', 'ipv4-private', 'ipv4-public') }) if isinstance(roster_order, (tuple, list)): - salt.utils.warn_until('Oxygen', + salt.utils.warn_until('Fluorine', 'Using legacy syntax for roster_order') roster_order = { 'host': roster_order @@ -137,7 +137,7 @@ def targets(tgt, tgt_type='glob', **kwargs): # pylint: disable=W0613 for config_key, order in roster_order.items(): for idx, key in enumerate(order): if key in ('public', 'private', 'local'): - salt.utils.warn_until('Oxygen', + salt.utils.warn_until('Fluorine', 'roster_order {0} will include IPv6 soon. ' 'Set order to ipv4-{0} if needed.'.format(key)) order[idx] = 'ipv4-' + key From 1920dc60793dba34904c686b815ead0b8fbbd94c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Jul 2017 10:00:35 -0700 Subject: [PATCH 136/300] Uncomment the line that removes the temporary identity file. --- salt/modules/git.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index b6592a4607..304276187b 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -273,10 +273,10 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, if not salt.utils.is_windows() and 'GIT_SSH' in env: os.remove(env['GIT_SSH']) - # Cleanup the temporary identify file + # Cleanup the temporary identity file if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identify file {0}'.format(tmp_identity_file)) - #__salt__['file.remove'](tmp_identity_file) + log.debug('Removing identity file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From ff24102d51206a1db6f5ad6b89360761b4077aec Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Jul 2017 10:01:55 -0700 Subject: [PATCH 137/300] Uncomment the line that removes the temporary identity file. --- salt/modules/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 4f1101b755..4b403fe029 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -261,7 +261,7 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # Cleanup the temporary identify file if tmp_identity_file and os.path.exists(tmp_identity_file): log.debug('Removing identify file {0}'.format(tmp_identity_file)) - #__salt__['file.remove'](tmp_identity_file) + __salt__['file.remove'](tmp_identity_file) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From 66093738c8c6d1b5a84d7c8959b2bf74c132d065 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Jul 2017 10:59:22 -0500 Subject: [PATCH 138/300] Add back support for string kwargs --- salt/minion.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 03e0c0cfa3..bc469b63fd 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -306,14 +306,16 @@ def load_args_and_kwargs(func, args, data=None, ignore_invalid=False): else: string_kwarg = salt.utils.args.parse_input([arg], condition=False)[1] # pylint: disable=W0632 if string_kwarg: - log.critical( - 'String kwarg(s) %s passed to ' - 'salt.minion.load_args_and_kwargs(). This is no longer ' - 'supported, so the kwarg(s) will be ignored. Arguments ' - 'passed to salt.minion.load_args_and_kwargs() should be ' - 'passed to salt.utils.args.parse_input() first to load ' - 'and condition them properly.', string_kwarg - ) + if argspec.keywords or next(six.iterkeys(string_kwarg)) in argspec.args: + # Function supports **kwargs or is a positional argument to + # the function. + _kwargs.update(string_kwarg) + else: + # **kwargs not in argspec and parsed argument name not in + # list of positional arguments. This keyword argument is + # invalid. + for key, val in six.iteritems(string_kwarg): + invalid_kwargs.append('{0}={1}'.format(key, val)) else: _args.append(arg) From 99ec634c6be648bdd68633f560b6cda6bd75684a Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 21 Jul 2017 15:19:31 -0600 Subject: [PATCH 139/300] validate ssh_interface for ec2 and default to public_ips --- salt/cloud/clouds/ec2.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index d6f9edcc6e..5223b80457 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -961,10 +961,18 @@ def ssh_interface(vm_): Return the ssh_interface type to connect to. Either 'public_ips' (default) or 'private_ips'. ''' - return config.get_cloud_config_value( + ret = config.get_cloud_config_value( 'ssh_interface', vm_, __opts__, default='public_ips', search_global=False ) + if not ret in ('public_ips', 'private_ips'): + log.warning(( + 'Invalid ssh_interface: {0}. ' + 'Allowed options are ("public_ips", "private_ips"). ' + 'Defaulting to "public_ips".' + ).format(ret)) + ret = 'public_ips' + return ret def get_ssh_gateway_config(vm_): From 0cc0c0967a8ffc5908d275c9800e98ec1ee4213a Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 21 Jul 2017 16:14:32 -0600 Subject: [PATCH 140/300] Lint fixes --- salt/minion.py | 2 +- tests/integration/client/test_kwarg.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index bc469b63fd..3c5046ee93 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -315,7 +315,7 @@ def load_args_and_kwargs(func, args, data=None, ignore_invalid=False): # list of positional arguments. This keyword argument is # invalid. for key, val in six.iteritems(string_kwarg): - invalid_kwargs.append('{0}={1}'.format(key, val)) + invalid_kwargs.append('{0}={1}'.format(key, val)) else: _args.append(arg) diff --git a/tests/integration/client/test_kwarg.py b/tests/integration/client/test_kwarg.py index a9ec8d27ff..ac4d69a0e7 100644 --- a/tests/integration/client/test_kwarg.py +++ b/tests/integration/client/test_kwarg.py @@ -6,6 +6,7 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.case import ModuleCase + class StdTest(ModuleCase): ''' Test standard client calls From 102509029e5fd32f4a86c8a44c22314082c744ef Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 21 Jul 2017 16:36:07 -0600 Subject: [PATCH 141/300] Remove chown mock, fix path seps --- tests/unit/test_crypt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_crypt.py b/tests/unit/test_crypt.py index a26b7583f8..e5713fb641 100644 --- a/tests/unit/test_crypt.py +++ b/tests/unit/test_crypt.py @@ -88,13 +88,13 @@ SIG = ( class CryptTestCase(TestCase): def test_gen_keys(self): - with patch.multiple(os, umask=MagicMock(), chmod=MagicMock(), chown=MagicMock, + with patch.multiple(os, umask=MagicMock(), chmod=MagicMock(), access=MagicMock(return_value=True)): with patch('salt.utils.fopen', mock_open()): - open_priv_wb = call('/keydir/keyname.pem', 'wb+') - open_pub_wb = call('/keydir/keyname.pub', 'wb+') + open_priv_wb = call('/keydir{0}keyname.pem'.format(os.sep), 'wb+') + open_pub_wb = call('/keydir{0}keyname.pub'.format(os.sep), 'wb+') with patch('os.path.isfile', return_value=True): - self.assertEqual(crypt.gen_keys('/keydir', 'keyname', 2048), '/keydir/keyname.pem') + self.assertEqual(crypt.gen_keys('/keydir', 'keyname', 2048), '/keydir{0}keyname.pem'.format(os.sep)) self.assertNotIn(open_priv_wb, salt.utils.fopen.mock_calls) self.assertNotIn(open_pub_wb, salt.utils.fopen.mock_calls) with patch('os.path.isfile', return_value=False): From 0b548c72e1f201467f0caa656721ae6b58dfa809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Fusil-Delahaye?= Date: Sat, 22 Jul 2017 13:01:49 +0200 Subject: [PATCH 142/300] Fix a potential Exception with an explicit error message if sign_remote_certificate is not authorized on master, execution fail with explicite error. --- salt/modules/x509.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/modules/x509.py b/salt/modules/x509.py index 986f137a82..5fc7a97a63 100644 --- a/salt/modules/x509.py +++ b/salt/modules/x509.py @@ -1373,10 +1373,19 @@ def create_certificate( ['listen_in', 'preqrequired', '__prerequired__']: kwargs.pop(ignore, None) - cert_txt = __salt__['publish.publish']( + certs = __salt__['publish.publish']( tgt=ca_server, fun='x509.sign_remote_certificate', - arg=str(kwargs))[ca_server] + arg=str(kwargs)) + + if not any(certs): + raise salt.exceptions.SaltInvocationError( + 'ca_server did not respond' + ' salt master must permit peers to' + ' call the sign_remote_certificate function.') + + cert_txt = certs[ca_server] + if path: return write_pem( text=cert_txt, From 92f18907011b80c72577524a465c910c8943f6f9 Mon Sep 17 00:00:00 2001 From: Aurelien Fusil-Delahaye Date: Mon, 24 Jul 2017 14:12:23 +0200 Subject: [PATCH 143/300] Fix azurerm query to show IPs azurearm.py edited to show IPs instead of interfaces id at queries --- salt/cloud/clouds/azurearm.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py index 8d346c9a0e..6184a89c49 100644 --- a/salt/cloud/clouds/azurearm.py +++ b/salt/cloud/clouds/azurearm.py @@ -407,13 +407,14 @@ def list_nodes_full(conn=None, call=None): # pylint: disable=unused-argument for group in list_resource_groups(): nodes = compconn.virtual_machines.list(group) for node in nodes: + private_ips, public_ips = __get_ips_from_node(group, node) ret[node.name] = object_to_dict(node) ret[node.name]['id'] = node.id ret[node.name]['name'] = node.name ret[node.name]['size'] = node.hardware_profile.vm_size ret[node.name]['state'] = node.provisioning_state - ret[node.name]['private_ips'] = node.network_profile.network_interfaces - ret[node.name]['public_ips'] = node.network_profile.network_interfaces + ret[node.name]['private_ips'] = private_ips + ret[node.name]['public_ips'] = public_ips ret[node.name]['storage_profile']['data_disks'] = [] ret[node.name]['resource_group'] = group for disk in node.storage_profile.data_disks: @@ -433,6 +434,30 @@ def list_nodes_full(conn=None, call=None): # pylint: disable=unused-argument return ret +def __get_ips_from_node(resource_group, node): + ''' + List private and public IPs from a VM interface + ''' + global netconn # pylint: disable=global-statement,invalid-name + if not netconn: + netconn = get_conn(NetworkManagementClient) + + private_ips = [] + public_ips = [] + for node_iface in node.network_profile.network_interfaces: + node_iface_name = node_iface.id.split('/')[-1] + network_interface = netconn.network_interfaces.get(resource_group, node_iface_name) + for ip_configuration in network_interface.ip_configurations: + if ip_configuration.private_ip_address: + private_ips.append(ip_configuration.private_ip_address) + if ip_configuration.public_ip_address and ip_configuration.public_ip_address.id: + public_iface_name = ip_configuration.public_ip_address.id.split('/')[-1] + public_iface = netconn.public_ip_addresses.get(resource_group, public_iface_name) + public_ips.append(public_iface.ip_address) + + return private_ips, public_ips + + def list_resource_groups(conn=None, call=None): # pylint: disable=unused-argument ''' List resource groups associated with the account From 6352f447ce6669c6951d198e2b39b57c16b698f9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 24 Jul 2017 08:50:44 -0500 Subject: [PATCH 144/300] Add PER_REMOTE_ONLY to init_remotes call in git_pillar runner When the mountpoint feature was added, it added a per-remote-only parameter called `mountpoint`. While this is reflected in salt.pillar.git_pillar, it was not in salt.runners.git_pillar. This corrects that oversight, fixing a traceback when the `git_pillar.update` runner is executed and one or more remotes have a `mountpoint` parameter configured. --- salt/runners/git_pillar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/runners/git_pillar.py b/salt/runners/git_pillar.py index 8dfc4412eb..0e8e97beb3 100644 --- a/salt/runners/git_pillar.py +++ b/salt/runners/git_pillar.py @@ -86,7 +86,8 @@ def update(branch=None, repo=None): else: pillar = salt.utils.gitfs.GitPillar(__opts__) pillar.init_remotes(pillar_conf, - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES) + salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + salt.pillar.git_pillar.PER_REMOTE_ONLY) for remote in pillar.remotes: # Skip this remote if it doesn't match the search criteria if branch is not None: From 72924b06b807a4e2442e227015548324a1b83be2 Mon Sep 17 00:00:00 2001 From: clem-compilatio Date: Mon, 24 Jul 2017 16:59:19 +0200 Subject: [PATCH 145/300] Fix _assign_floating_ips in openstack.py Fixes #42417 In salt/cloud/clouds/openstack.py : - _assign_floating_ips() should not raise SaltCloudSystemExit() just yet. - Instead, it should let _query_node_data() a chance to get the IP from node['public_ips']. --- salt/cloud/clouds/openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index e73af9e0e5..775173a348 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -907,9 +907,9 @@ def _assign_floating_ips(vm_, conn, kwargs): floating.append(idx) break if not floating: - raise SaltCloudSystemExit( + log.warning( 'There are no more floating IP addresses ' - 'available, please create some more' + 'available, please create some more if necessary' ) except Exception as e: if str(e).startswith('404'): From c32c1b2803369baaddf3678533f04ee2a47e7235 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 11:31:25 -0600 Subject: [PATCH 146/300] fix pylint --- salt/cloud/clouds/ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index 5223b80457..f47d2d93c3 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -965,7 +965,7 @@ def ssh_interface(vm_): 'ssh_interface', vm_, __opts__, default='public_ips', search_global=False ) - if not ret in ('public_ips', 'private_ips'): + if ret not in ('public_ips', 'private_ips'): log.warning(( 'Invalid ssh_interface: {0}. ' 'Allowed options are ("public_ips", "private_ips"). ' From e3a6717efa7bd77893cec6c0677971a85298cce3 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 24 Jul 2017 12:17:41 -0600 Subject: [PATCH 147/300] Add info about top file to pillar walk-through example to include edit.vim The pillar example in the "Pillar Makes Simple States Grow Easily" section does not meniton that the new file `/srv/pillar/edit/vim.sls` created for the example must be included in the pillar top file in order for the references to work. This PR adds that documentation so that the example works correctly. Fixes #42405 --- doc/topics/tutorials/pillar.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/topics/tutorials/pillar.rst b/doc/topics/tutorials/pillar.rst index e0c97f26bd..3ec2c1ddea 100644 --- a/doc/topics/tutorials/pillar.rst +++ b/doc/topics/tutorials/pillar.rst @@ -75,7 +75,7 @@ The default location for the pillar is in /srv/pillar. .. note:: - The pillar location can be configured via the `pillar_roots` option inside + The pillar location can be configured via the ``pillar_roots`` option inside the master configuration file. It must not be in a subdirectory of the state tree or file_roots. If the pillar is under file_roots, any pillar targeting can be bypassed by minions. @@ -242,7 +242,7 @@ set in the minion's pillar, then the default of ``httpd`` will be used. .. note:: Under the hood, pillar is just a Python dict, so Python dict methods such - as `get` and `items` can be used. + as ``get`` and ``items`` can be used. Pillar Makes Simple States Grow Easily ====================================== @@ -303,6 +303,18 @@ Where the vimrc source location can now be changed via pillar: Ensuring that the right vimrc is sent out to the correct minions. +The pillar top file must include a reference to the new sls pillar file: + +``/srv/pillar/top.sls``: + +.. code-block:: yaml + + base: + '*': + - pkg + - edit.vim + + Setting Pillar Data on the Command Line ======================================= From fa466519c4bdb6baea07e6b9f34b33a1ae4953f1 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 24 Jul 2017 13:22:50 -0600 Subject: [PATCH 148/300] Add a mention of the True/False returns with __virtual__() And their relationship to `__virtualname__`. Fixes #42375 --- doc/ref/modules/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 9f81170fe4..1056ba40f8 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -405,6 +405,10 @@ similar to the following: return __virtualname__ return False +Note that the ``__virtual__()`` function will return either a ``True`` or ``False`` +value. If it returns a ``True`` value, this ``__virtualname__`` module-level attribute +can be set as seen in the above example. This is the name that the module should be +referred to as. Documentation ============= From af3bcc927bd7c728a29a7331d1879734cc56df1c Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 24 Jul 2017 14:34:09 -0600 Subject: [PATCH 149/300] Document changes to Windows Update in 10/2016 --- salt/modules/win_wua.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index a6a5712c38..baf8c8a9ee 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -945,12 +945,26 @@ def set_wu_settings(level=None, Change Windows Update settings. If no parameters are passed, the current value will be returned. + Supported: + - Windows Vista / Server 2008 + - Windows 7 / Server 2008R2 + - Windows 8 / Server 2012 + - Windows 8.1 / Server 2012R2 + + .. note: + Microsoft began using the Unified Update Platform (UUP) starting with + Windows 10 / Server 2016. The Windows Update settings have changed and + the ability to 'Save' Windows Update settings has been removed. Windows + Update settings are read-only. See msdn documentation: + https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx + :param int level: Number from 1 to 4 indicating the update level: 1. Never check for updates 2. Check for updates but let me choose whether to download and install them 3. Download updates but let me choose whether to install them 4. Install updates automatically + :param bool recommended: Boolean value that indicates whether to include optional or recommended updates when a search for updates and installation of updates is @@ -993,8 +1007,28 @@ def set_wu_settings(level=None, salt '*' win_wua.set_wu_settings level=4 recommended=True featured=False """ - ret = {} - ret['Success'] = True + # The AutomaticUpdateSettings.Save() method used in this function does not + # work on Windows 10 / Server 2016. It is called in throughout this function + # like this: + # + # obj_au = win32com.client.Dispatch('Microsoft.Update.AutoUpdate') + # obj_au_settings = obj_au.Settings + # obj_au_settings.Save() + # + # The `Save()` method reports success but doesn't actually change anything. + # Windows Update settings are read-only in Windows 10 / Server 2016. There's + # a little blurb on MSDN that mentions this, but gives no alternative for + # changing these settings in Windows 10 / Server 2016. + # + # https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx + # + # Apparently the Windows Update framework in Windows Vista - Windows 8.1 has + # been changed quite a bit in Windows 10 / Server 2016. It is now called the + # Unified Update Platform (UUP). I haven't found an API or a Powershell + # commandlet for working with the the UUP. Perhaps there will be something + # forthcoming. The `win_lgpo` module might be an option for changing the + # Windows Update settings using local group policy. + ret = {'Success': True} # Initialize the PyCom system pythoncom.CoInitialize() From a02c91adda41c1fca42f62f2359c3415ba1bde7a Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 24 Jul 2017 15:02:15 -0600 Subject: [PATCH 150/300] Namespace `cmp_to_key` in the pkg state for Windows --- salt/states/pkg.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 120565f1f4..f5f85b73a0 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -116,6 +116,7 @@ if salt.utils.is_windows(): from salt.modules.win_pkg import _repo_process_pkg_sls from salt.modules.win_pkg import _get_latest_pkg_version from salt.modules.win_pkg import _reverse_cmp_pkg_versions + from functools import cmp_to_key _get_package_info = _namespaced_function(_get_package_info, globals()) get_repo_data = _namespaced_function(get_repo_data, globals()) _get_repo_details = \ @@ -130,6 +131,7 @@ if salt.utils.is_windows(): _namespaced_function(_get_latest_pkg_version, globals()) _reverse_cmp_pkg_versions = \ _namespaced_function(_reverse_cmp_pkg_versions, globals()) + cmp_to_key = _namespaced_function(cmp_to_key, globals()) # The following imports are used by the namespaced win_pkg funcs # and need to be included in their globals. # pylint: disable=import-error,unused-import From e90ca7a114b11b9a01c383b797dfce58b1ccad9c Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 15:00:15 -0600 Subject: [PATCH 151/300] use salt encoding for joyent on 2017.7 --- salt/cloud/clouds/joyent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/cloud/clouds/joyent.py b/salt/cloud/clouds/joyent.py index 3a2f119f9f..5602e24e8c 100644 --- a/salt/cloud/clouds/joyent.py +++ b/salt/cloud/clouds/joyent.py @@ -1071,10 +1071,10 @@ def query(action=None, timenow = datetime.datetime.utcnow() timestamp = timenow.strftime('%a, %d %b %Y %H:%M:%S %Z').strip() with salt.utils.fopen(ssh_keyfile, 'r') as kh_: - rsa_key = RSA.importKey(kh_) + rsa_key = RSA.importKey(kh_.read()) rsa_ = PKCS1_v1_5.new(rsa_key) hash_ = SHA256.new() - hash_.update(timestamp) + hash_.update(timestamp.encode(__salt_system_encoding__)) signed = base64.b64encode(rsa_.sign(hash_)) keyid = '/{0}/keys/{1}'.format(user.split('/')[0], ssh_keyname) @@ -1085,7 +1085,7 @@ def query(action=None, 'Date': timestamp, 'Authorization': 'Signature keyId="{0}",algorithm="rsa-sha256" {1}'.format( keyid, - signed + signed.decode(__salt_system_encoding__) ), } From b7ebb4d81ae4855d1c87fa85c0a3a5d7c6873f00 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 15:01:57 -0600 Subject: [PATCH 152/300] these drivers do not actually have an issue. Joyent does not use apache-libcloud, and is the only one with this issue. --- doc/topics/releases/2017.7.0.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 2d06b20140..2935ef2a11 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -28,8 +28,6 @@ The following salt-cloud drivers have known issues running with Python 3. These - Joyent -- Any driver that relies on the `apache-libcloud` library such as cloudstack, dimenstiondata, gce, nova, and openstack - - When running under Python 3, users who require Unicode support should ensure that a locale is set on their machines. Users using the `C` locale are advised to switch to a UTF-aware locale to ensure proper functionality with Salt with Python 3. From b242d2d6b5f59b3eb6b022def19dcbba122b7070 Mon Sep 17 00:00:00 2001 From: jmarinaro Date: Mon, 24 Jul 2017 18:15:16 -0600 Subject: [PATCH 153/300] Fixes AttributeError thrown by chocolatey state Fixes #42521 --- salt/states/chocolatey.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/states/chocolatey.py b/salt/states/chocolatey.py index 79c69048ef..d83f9bddd3 100644 --- a/salt/states/chocolatey.py +++ b/salt/states/chocolatey.py @@ -92,7 +92,7 @@ def installed(name, version=None, source=None, force=False, pre_versions=False, # Determine action # Package not installed - if name not in [package.split('|')[0].lower() for package in pre_install.splitlines()]: + if name.lower() not in [package.lower() for package in pre_install.keys()]: if version: ret['changes'] = {name: 'Version {0} will be installed' ''.format(version)} @@ -193,9 +193,13 @@ def uninstalled(name, version=None, uninstall_args=None, override_args=False): pre_uninstall = __salt__['chocolatey.list'](local_only=True) # Determine if package is installed - if name in [package.split('|')[0].lower() for package in pre_uninstall.splitlines()]: - ret['changes'] = {name: '{0} version {1} will be removed' - ''.format(name, pre_uninstall[name][0])} + if name.lower() in [package.lower() for package in pre_uninstall.keys()]: + try: + ret['changes'] = {name: '{0} version {1} will be removed' + ''.format(name, pre_uninstall[name][0])} + except KeyError: + ret['changes'] = {name: '{0} will be removed' + ''.format(name)} else: ret['comment'] = 'The package {0} is not installed'.format(name) return ret From 2fd172e07bca3e6379ddac5095130231ff7c7745 Mon Sep 17 00:00:00 2001 From: Marin Hannache Date: Tue, 25 Jul 2017 16:23:34 +0200 Subject: [PATCH 154/300] Avoid confusing warning when using file.line file.line is internaly using file.managed without any source argument to ensure file creation with the appropriate permissions. file.managed has its 'replace' parameter defaulting to True, resulting in the following warning : Neither 'source' nor 'contents' nor 'contents_pillar' nor 'contents_grains' was defined, yet 'replace' was set to 'True'. As there is no source to replace the file with, 'replace' has been set to 'False' to avoid reading the file unnecessarily. Setting explicitly replace to False solve this issue. Signed-off-by: Marin Hannache --- salt/states/file.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index 64cffa15a2..86d1e42505 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3732,7 +3732,13 @@ def line(name, content=None, match=None, mode=None, location=None, if not name: return _error(ret, 'Must provide name to file.line') - managed(name, create=create, user=user, group=group, mode=file_mode) + managed( + name, + create=create, + user=user, + group=group, + mode=file_mode, + replace=False) check_res, check_msg = _check_file(name) if not check_res: From 118d5134e2e746e56ded813c12eef49573d09fd5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 25 Jul 2017 11:37:06 -0600 Subject: [PATCH 155/300] Remove namespaced function `cmp_to_key` --- salt/states/pkg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index f5f85b73a0..1562ea2fa8 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -131,7 +131,6 @@ if salt.utils.is_windows(): _namespaced_function(_get_latest_pkg_version, globals()) _reverse_cmp_pkg_versions = \ _namespaced_function(_reverse_cmp_pkg_versions, globals()) - cmp_to_key = _namespaced_function(cmp_to_key, globals()) # The following imports are used by the namespaced win_pkg funcs # and need to be included in their globals. # pylint: disable=import-error,unused-import From a040443fa1fee3b6acb09a0ecd6696329073e1ec Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 25 Jul 2017 11:37:59 -0600 Subject: [PATCH 156/300] Move functools import inside pylint escapes --- salt/states/pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 1562ea2fa8..b586ed5dbe 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -105,6 +105,7 @@ if salt.utils.is_windows(): import datetime import errno import time + from functools import cmp_to_key # pylint: disable=import-error # pylint: enable=unused-import from salt.modules.win_pkg import _get_package_info @@ -116,7 +117,6 @@ if salt.utils.is_windows(): from salt.modules.win_pkg import _repo_process_pkg_sls from salt.modules.win_pkg import _get_latest_pkg_version from salt.modules.win_pkg import _reverse_cmp_pkg_versions - from functools import cmp_to_key _get_package_info = _namespaced_function(_get_package_info, globals()) get_repo_data = _namespaced_function(get_repo_data, globals()) _get_repo_details = \ From 685c2cced680ffb5745b3eaa8754ff9eb887fe50 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 25 Jul 2017 14:44:18 -0600 Subject: [PATCH 157/300] Add information about returning a tuple with an error message --- doc/ref/modules/index.rst | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 1056ba40f8..3bd62b3c41 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -405,10 +405,29 @@ similar to the following: return __virtualname__ return False -Note that the ``__virtual__()`` function will return either a ``True`` or ``False`` -value. If it returns a ``True`` value, this ``__virtualname__`` module-level attribute -can be set as seen in the above example. This is the name that the module should be -referred to as. +The ``__virtual__()`` function can return a ``True`` or ``False`` boolean, a tuple, +or a string. If it returns a ``True`` value, this ``__virtualname__`` module-level +attribute can be set as seen in the above example. This is the string that the module +should be referred to as. + +When ``__virtual__()`` returns a tuple, the first item should be a boolean and the +second should be a string. This is typically done when the module should not load. The +first value of the tuple is ``False`` and the second is the error message to display +for why the module did not load. + +For example: + +.. code-block:: python + + def __virtual__(): + ''' + Only load if git exists on the system + ''' + if salt.utils.which('git') is None: + return (False, + 'The git execution module cannot be loaded: git unavailable.') + else: + return True Documentation ============= From c4fabaa192940d84beb53f4f169c4113bcdb1883 Mon Sep 17 00:00:00 2001 From: Richard Phillis Date: Wed, 26 Jul 2017 09:15:59 +1000 Subject: [PATCH 158/300] Remove '-s' (--script) argument to parted within align_check function The -s argument results in no stdout output from parted in this context, so the user must infer success or failure from the exit status. --- salt/modules/parted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/parted.py b/salt/modules/parted.py index 04a7d37d47..f568c3d001 100644 --- a/salt/modules/parted.py +++ b/salt/modules/parted.py @@ -216,7 +216,7 @@ def align_check(device, part_type, partition): 'Invalid partition passed to partition.align_check' ) - cmd = 'parted -m -s {0} align-check {1} {2}'.format( + cmd = 'parted -m {0} align-check {1} {2}'.format( device, part_type, partition ) out = __salt__['cmd.run'](cmd).splitlines() From fb69e710931f3f4d2bdd0ebcec1aeff30d8fd653 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 25 Jul 2017 21:10:58 -0400 Subject: [PATCH 159/300] add changelog to 2017.7.1 release notes --- doc/topics/releases/2017.7.1.rst | 178 +++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/doc/topics/releases/2017.7.1.rst b/doc/topics/releases/2017.7.1.rst index e879d8639a..dcbe2c8f39 100644 --- a/doc/topics/releases/2017.7.1.rst +++ b/doc/topics/releases/2017.7.1.rst @@ -3,3 +3,181 @@ Salt 2017.7.1 Release Notes ============================ Version 2017.7.1 is a bugfix release for :ref:`2017.7.0 `. + +Changes for v2017.7.0..v2017.7.1 +------------------------------ + +Extended changelog courtesy of Todd Stansell (https://github.com/tjstansell/salt-changelogs): + +*Generated at: 2017-07-26T01:09:40Z* + +Statistics: + +- Total Merges: **11** +- Total Issue references: **9** +- Total PR references: **22** + +Changes: + + +- **PR** `#42548`_: (*gtmanfred*) pass in empty kwarg for reactor + @ *2017-07-26T00:41:20Z* + + - **ISSUE** `#460`_: (*whiteinge*) Add a topic and a ref for modules/states/returners/renderers/runners + | refs: `#42548`_ + * 711b742c54 Merge pull request `#42548`_ from gtmanfred/2017.7.1 + * 0257c1dc32 pass in empty kwarg for reactor + + * b948e980d2 update chunk, not kwarg in chunk + +- **PR** `#42522`_: (*gtmanfred*) pacman wildcard is only for repository installs + @ *2017-07-24T20:51:05Z* + + - **ISSUE** `#42519`_: (*xuhcc*) Error when installing package from file under Arch Linux + | refs: `#42522`_ + * 50c1635dcc Merge pull request `#42522`_ from gtmanfred/2017.7.1 + * 7787fb9e1b pacman wildcard is only for repository installs + +- **PR** `#42508`_: (*rallytime*) Back-port `#42474`_ to 2017.7.1 + @ *2017-07-24T20:49:51Z* + + - **PR** `#42474`_: (*whiteinge*) Cmd arg kwarg parsing test + | refs: `#42508`_ + - **PR** `#39646`_: (*terminalmage*) Handle deprecation of passing string args to load_args_and_kwargs + | refs: `#42474`_ + * 05c07ac049 Merge pull request `#42508`_ from rallytime/`bp-42474`_ + * 76fb074433 Add a test.arg variant that cleans the pub kwargs by default + + * 624f63648e Lint fixes + + * d246a5fc61 Add back support for string kwargs + + * 854e098aa0 Add LocalClient.cmd test for arg/kwarg parsing + +- **PR** `#42472`_: (*rallytime*) Back-port `#42435`_ to 2017.7.1 + @ *2017-07-24T15:11:13Z* + + - **ISSUE** `#42427`_: (*grichmond-salt*) Issue Passing Variables created from load_json as Inline Pillar Between States + | refs: `#42435`_ + - **PR** `#42435`_: (*terminalmage*) Modify our custom YAML loader to treat unicode literals as unicode strings + | refs: `#42472`_ + * 95fe2558e4 Merge pull request `#42472`_ from rallytime/`bp-42435`_ + * 5c47af5b98 Modify our custom YAML loader to treat unicode literals as unicode strings + +- **PR** `#42473`_: (*rallytime*) Back-port `#42436`_ to 2017.7.1 + @ *2017-07-24T15:10:29Z* + + - **ISSUE** `#42374`_: (*tyhunt99*) [2017.7.0] salt-run mange.versions throws exception if minion is offline or unresponsive + | refs: `#42436`_ + - **PR** `#42436`_: (*garethgreenaway*) Fixes to versions function in manage runner + | refs: `#42473`_ + * 5b99d45f54 Merge pull request `#42473`_ from rallytime/`bp-42436`_ + * 82ed919803 Updating the versions function inside the manage runner to account for when a minion is offline and we are unable to determine it's version. + +- **PR** `#42471`_: (*rallytime*) Back-port `#42399`_ to 2017.7.1 + @ *2017-07-24T15:09:50Z* + + - **ISSUE** `#42381`_: (*zebooka*) Git.detached broken in 2017.7.0 + | refs: `#42399`_ + - **ISSUE** `#38878`_: (*tomlaredo*) [Naming consistency] git.latest "rev" option VS git.detached "ref" option + | refs: `#38898`_ + - **PR** `#42399`_: (*rallytime*) Update old "ref" references to "rev" in git.detached state + | refs: `#42471`_ + - **PR** `#38898`_: (*terminalmage*) git.detached: rename ref to rev for consistency + | refs: `#42399`_ + * 3d1a2d3f9f Merge pull request `#42471`_ from rallytime/`bp-42399`_ + * b9a4669e5a Update old "ref" references to "rev" in git.detached state + +- **PR** `#42470`_: (*rallytime*) Back-port `#42031`_ to 2017.7.1 + @ *2017-07-24T15:09:30Z* + + - **ISSUE** `#42400`_: (*Enquier*) Conflict in execution of passing pillar data to orch/reactor event executions 2017.7.0 + | refs: `#42031`_ + - **PR** `#42031`_: (*skizunov*) Fix: Reactor emits critical error + | refs: `#42470`_ + * 09766bccbc Merge pull request `#42470`_ from rallytime/`bp-42031`_ + * 0a0c6287a4 Fix: Reactor emits critical error + +- **PR** `#42469`_: (*rallytime*) Back-port `#42027`_ to 2017.7.1 + @ *2017-07-21T22:41:02Z* + + - **ISSUE** `#41949`_: (*jrporcaro*) Event returner doesn't work with Windows Master + | refs: `#42027`_ + - **PR** `#42027`_: (*gtmanfred*) import salt.minion for EventReturn for Windows + | refs: `#42469`_ + * d7b172a15b Merge pull request `#42469`_ from rallytime/`bp-42027`_ + * ed612b4ee7 import salt.minion for EventReturn for Windows + +- **PR** `#42466`_: (*rallytime*) Back-port `#42452`_ to 2017.7.1 + @ *2017-07-21T19:41:24Z* + + - **PR** `#42452`_: (*Ch3LL*) update windows urls to new py2/py3 naming scheme + | refs: `#42466`_ + * 8777b1a825 Merge pull request `#42466`_ from rallytime/`bp-42452`_ + * c10196f68c update windows urls to new py2/py3 naming scheme + +- **PR** `#42439`_: (*rallytime*) Back-port `#42409`_ to 2017.7.1 + @ *2017-07-21T17:38:10Z* + + - **PR** `#42409`_: (*twangboy*) Add Scripts to build Py3 on Mac + | refs: `#42439`_ + * fceaaf41d0 Merge pull request `#42439`_ from rallytime/`bp-42409`_ + * 8176964b41 Remove build and dist, sign pkgs + + * 2c14d92a07 Fix hard coded pip path + + * 82fdd7c2e1 Add support for Py3 + + * 2478447246 Update Python and other reqs + +- **PR** `#42441`_: (*rallytime*) Back-port `#42433`_ to 2017.7.1 + @ *2017-07-21T17:37:01Z* + + - **ISSUE** `#42403`_: (*astronouth7303*) [2017.7] Pillar empty when state is applied from orchestrate + | refs: `#42433`_ + - **PR** `#42433`_: (*terminalmage*) Only force saltenv/pillarenv to be a string when not None + | refs: `#42441`_ + * 660400560b Merge pull request `#42441`_ from rallytime/`bp-42433`_ + * 17f347123a Only force saltenv/pillarenv to be a string when not None + + +.. _`#38878`: https://github.com/saltstack/salt/issues/38878 +.. _`#38898`: https://github.com/saltstack/salt/pull/38898 +.. _`#39646`: https://github.com/saltstack/salt/pull/39646 +.. _`#41949`: https://github.com/saltstack/salt/issues/41949 +.. _`#42027`: https://github.com/saltstack/salt/pull/42027 +.. _`#42031`: https://github.com/saltstack/salt/pull/42031 +.. _`#42374`: https://github.com/saltstack/salt/issues/42374 +.. _`#42381`: https://github.com/saltstack/salt/issues/42381 +.. _`#42399`: https://github.com/saltstack/salt/pull/42399 +.. _`#42400`: https://github.com/saltstack/salt/issues/42400 +.. _`#42403`: https://github.com/saltstack/salt/issues/42403 +.. _`#42409`: https://github.com/saltstack/salt/pull/42409 +.. _`#42427`: https://github.com/saltstack/salt/issues/42427 +.. _`#42433`: https://github.com/saltstack/salt/pull/42433 +.. _`#42435`: https://github.com/saltstack/salt/pull/42435 +.. _`#42436`: https://github.com/saltstack/salt/pull/42436 +.. _`#42439`: https://github.com/saltstack/salt/pull/42439 +.. _`#42441`: https://github.com/saltstack/salt/pull/42441 +.. _`#42452`: https://github.com/saltstack/salt/pull/42452 +.. _`#42466`: https://github.com/saltstack/salt/pull/42466 +.. _`#42469`: https://github.com/saltstack/salt/pull/42469 +.. _`#42470`: https://github.com/saltstack/salt/pull/42470 +.. _`#42471`: https://github.com/saltstack/salt/pull/42471 +.. _`#42472`: https://github.com/saltstack/salt/pull/42472 +.. _`#42473`: https://github.com/saltstack/salt/pull/42473 +.. _`#42474`: https://github.com/saltstack/salt/pull/42474 +.. _`#42508`: https://github.com/saltstack/salt/pull/42508 +.. _`#42519`: https://github.com/saltstack/salt/issues/42519 +.. _`#42522`: https://github.com/saltstack/salt/pull/42522 +.. _`#42548`: https://github.com/saltstack/salt/pull/42548 +.. _`#460`: https://github.com/saltstack/salt/issues/460 +.. _`bp-42027`: https://github.com/saltstack/salt/pull/42027 +.. _`bp-42031`: https://github.com/saltstack/salt/pull/42031 +.. _`bp-42399`: https://github.com/saltstack/salt/pull/42399 +.. _`bp-42409`: https://github.com/saltstack/salt/pull/42409 +.. _`bp-42433`: https://github.com/saltstack/salt/pull/42433 +.. _`bp-42435`: https://github.com/saltstack/salt/pull/42435 +.. _`bp-42436`: https://github.com/saltstack/salt/pull/42436 +.. _`bp-42452`: https://github.com/saltstack/salt/pull/42452 +.. _`bp-42474`: https://github.com/saltstack/salt/pull/42474 From ee3bc6eb10dcf217395ad07ca3e49c4c25eb665f Mon Sep 17 00:00:00 2001 From: "J. Beard" Date: Sun, 23 Jul 2017 21:02:13 -0700 Subject: [PATCH 160/300] Fixing output so --force-color and --no-color override master and minion config color value --- salt/output/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/output/__init__.py b/salt/output/__init__.py index d9ce631b4e..8a1732c512 100644 --- a/salt/output/__init__.py +++ b/salt/output/__init__.py @@ -168,6 +168,13 @@ def get_printout(out, opts=None, **kwargs): opts['color'] = False else: opts['color'] = True + else: + if opts.get('force_color', False): + opts['color'] = True + elif opts.get('no_color', False) or salt.utils.is_windows(): + opts['color'] = False + else: + pass outputters = salt.loader.outputters(opts) if out not in outputters: From afa7a13ce334874a51cd382dd985f7dbd9d0775b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 21 Jul 2017 08:31:08 -0600 Subject: [PATCH 161/300] use logic from file.directory for makedirs --- salt/states/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/archive.py b/salt/states/archive.py index e36f178f2e..f053d3c207 100644 --- a/salt/states/archive.py +++ b/salt/states/archive.py @@ -1221,7 +1221,7 @@ def extracted(name, return ret if not os.path.isdir(name): - __salt__['file.makedirs'](name, user=user) + __states__['file.directory'](name, user=user, makedirs=True) created_destdir = True log.debug('Extracting {0} to {1}'.format(cached_source, name)) From 0f0b7e3e0ac3ee0aa9c7a28d2c16bc0c3b9e681f Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Tue, 25 Jul 2017 20:56:28 -0500 Subject: [PATCH 162/300] Fix disable_ config option Previously, the logic always looked for `disable_s` in the config options. The issue with this is that the following tag names already end with 's': `beacons`, `engines`, `grains`, `log_handlers`, `serializers`, `states`, and `utils`. So previously, if you wanted to disable a beacon, the config option to set is `disable_beaconss` (with 'ss' at the end). Fix this so that we only append an 's' if the tag name does not already end with an 's'. Signed-off-by: Sergey Kizunov --- salt/loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/loader.py b/salt/loader.py index 000c0fef60..1c03e475be 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -1094,7 +1094,8 @@ class LazyLoader(salt.utils.lazy.LazyDict): virtual_funcs = [] self.virtual_funcs = virtual_funcs - self.disabled = set(self.opts.get('disable_{0}s'.format(self.tag), [])) + self.disabled = set(self.opts.get('disable_{0}{1}'.format( + self.tag, '' if self.tag[-1] == 's' else 's'), [])) self.refresh_file_mapping() From 4e2fb03a95e7cf8249c7a4a49d415859406551f4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 12:21:14 -0600 Subject: [PATCH 163/300] Add pythonpath to batch files and service --- pkg/windows/buildenv/salt-call.bat | 3 +++ pkg/windows/buildenv/salt-cp.bat | 3 +++ pkg/windows/buildenv/salt-key.bat | 3 +++ pkg/windows/buildenv/salt-master.bat | 3 +++ pkg/windows/buildenv/salt-minion-debug.bat | 3 +++ pkg/windows/buildenv/salt-minion.bat | 3 +++ pkg/windows/buildenv/salt-run.bat | 3 +++ pkg/windows/buildenv/salt.bat | 3 +++ pkg/windows/installer/Salt-Minion-Setup.nsi | 2 +- 9 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/windows/buildenv/salt-call.bat b/pkg/windows/buildenv/salt-call.bat index 095f51e4c1..3949253030 100644 --- a/pkg/windows/buildenv/salt-call.bat +++ b/pkg/windows/buildenv/salt-call.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-call +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-cp.bat b/pkg/windows/buildenv/salt-cp.bat index 29274320b1..4c5eae9be3 100644 --- a/pkg/windows/buildenv/salt-cp.bat +++ b/pkg/windows/buildenv/salt-cp.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-cp +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-key.bat b/pkg/windows/buildenv/salt-key.bat index 471e3626b2..08b05f04c5 100644 --- a/pkg/windows/buildenv/salt-key.bat +++ b/pkg/windows/buildenv/salt-key.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-key +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-master.bat b/pkg/windows/buildenv/salt-master.bat index 9a124ffd46..0f66885d91 100644 --- a/pkg/windows/buildenv/salt-master.bat +++ b/pkg/windows/buildenv/salt-master.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-master +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat index ad0ebafee0..bb58e89132 100644 --- a/pkg/windows/buildenv/salt-minion-debug.bat +++ b/pkg/windows/buildenv/salt-minion-debug.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Stop the Salt Minion service net stop salt-minion diff --git a/pkg/windows/buildenv/salt-minion.bat b/pkg/windows/buildenv/salt-minion.bat index 0a1aafa0c0..41c434aa83 100644 --- a/pkg/windows/buildenv/salt-minion.bat +++ b/pkg/windows/buildenv/salt-minion.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt-run.bat b/pkg/windows/buildenv/salt-run.bat index 5d62775d5e..c931d87ec5 100644 --- a/pkg/windows/buildenv/salt-run.bat +++ b/pkg/windows/buildenv/salt-run.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-run +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/buildenv/salt.bat b/pkg/windows/buildenv/salt.bat index 050387da90..6a7302372a 100644 --- a/pkg/windows/buildenv/salt.bat +++ b/pkg/windows/buildenv/salt.bat @@ -8,6 +8,9 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt +:: Set PYTHONPATH +Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages + :: Launch Script "%Python%" "%Script%" %* diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 070621a52a..8d0c6fd1b6 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -335,7 +335,7 @@ Section -Post ; Register the Salt-Minion Service nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" - nsExec::Exec "nssm.exe set salt-minion AppEnvironmentExtra PYTHONHOME=" + nsExec::Exec "nssm.exe set salt-minion AppEnvironmentExtra PYTHONHOME= PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" From 9af1eb274112d03370c393f35045cec74c7f738f Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 13:42:12 -0600 Subject: [PATCH 164/300] Ignore any PYTHON* environment vars already on the system --- pkg/windows/buildenv/salt-call.bat | 6 +----- pkg/windows/buildenv/salt-cp.bat | 6 +----- pkg/windows/buildenv/salt-key.bat | 6 +----- pkg/windows/buildenv/salt-master.bat | 6 +----- pkg/windows/buildenv/salt-minion-debug.bat | 6 +----- pkg/windows/buildenv/salt-minion.bat | 6 +----- pkg/windows/buildenv/salt-run.bat | 6 +----- pkg/windows/buildenv/salt.bat | 6 +----- pkg/windows/installer/Salt-Minion-Setup.nsi | 3 +-- 9 files changed, 9 insertions(+), 42 deletions(-) diff --git a/pkg/windows/buildenv/salt-call.bat b/pkg/windows/buildenv/salt-call.bat index 3949253030..6fc4086a9d 100644 --- a/pkg/windows/buildenv/salt-call.bat +++ b/pkg/windows/buildenv/salt-call.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-call -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-cp.bat b/pkg/windows/buildenv/salt-cp.bat index 4c5eae9be3..6c63dce338 100644 --- a/pkg/windows/buildenv/salt-cp.bat +++ b/pkg/windows/buildenv/salt-cp.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-cp -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-key.bat b/pkg/windows/buildenv/salt-key.bat index 08b05f04c5..c78b41580b 100644 --- a/pkg/windows/buildenv/salt-key.bat +++ b/pkg/windows/buildenv/salt-key.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-key -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-master.bat b/pkg/windows/buildenv/salt-master.bat index 0f66885d91..c1b9575a56 100644 --- a/pkg/windows/buildenv/salt-master.bat +++ b/pkg/windows/buildenv/salt-master.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-master -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat index bb58e89132..e78887ea63 100644 --- a/pkg/windows/buildenv/salt-minion-debug.bat +++ b/pkg/windows/buildenv/salt-minion-debug.bat @@ -8,12 +8,8 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Stop the Salt Minion service net stop salt-minion :: Launch Script -"%Python%" "%Script%" -l debug - +"%Python%" -E "%Script%" -l debug diff --git a/pkg/windows/buildenv/salt-minion.bat b/pkg/windows/buildenv/salt-minion.bat index 41c434aa83..8ca7ba4d8d 100644 --- a/pkg/windows/buildenv/salt-minion.bat +++ b/pkg/windows/buildenv/salt-minion.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt-run.bat b/pkg/windows/buildenv/salt-run.bat index c931d87ec5..ca4741c6a9 100644 --- a/pkg/windows/buildenv/salt-run.bat +++ b/pkg/windows/buildenv/salt-run.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-run -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/buildenv/salt.bat b/pkg/windows/buildenv/salt.bat index 6a7302372a..5eab5b5618 100644 --- a/pkg/windows/buildenv/salt.bat +++ b/pkg/windows/buildenv/salt.bat @@ -8,9 +8,5 @@ Set SaltDir=%SaltDir:~0,-1% Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt -:: Set PYTHONPATH -Set PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages - :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E "%Script%" %* diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 8d0c6fd1b6..24adb6eb08 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -334,8 +334,7 @@ Section -Post WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "Path" "$INSTDIR\bin\" ; Register the Salt-Minion Service - nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" - nsExec::Exec "nssm.exe set salt-minion AppEnvironmentExtra PYTHONHOME= PYTHONPATH=C:\salt\bin;C:\salt\bin\Lib\site-packages" + nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" From 0293429e24eea2f98ebb54c82a161789f25f5384 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 20 Jul 2017 13:55:06 -0500 Subject: [PATCH 165/300] Only force saltenv/pillarenv to be a string when not None This fixes a regression introduced to make numeric saltenv/pillarenv work properly. --- salt/modules/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index fc64d8d97d..5d6bf5f009 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -263,14 +263,14 @@ def _get_opts(**kwargs): if 'saltenv' in kwargs: saltenv = kwargs['saltenv'] - if not isinstance(saltenv, six.string_types): + if saltenv is not None and not isinstance(saltenv, six.string_types): opts['environment'] = str(kwargs['saltenv']) else: opts['environment'] = kwargs['saltenv'] if 'pillarenv' in kwargs: pillarenv = kwargs['pillarenv'] - if not isinstance(pillarenv, six.string_types): + if pillarenv is not None and not isinstance(pillarenv, six.string_types): opts['pillarenv'] = str(kwargs['pillarenv']) else: opts['pillarenv'] = kwargs['pillarenv'] From d55a44dd1a12933d25c2741f5b2b9f5fc6f6d485 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 14:21:41 -0600 Subject: [PATCH 166/300] Avoid loading user site packages --- pkg/windows/buildenv/salt-call.bat | 2 +- pkg/windows/buildenv/salt-cp.bat | 2 +- pkg/windows/buildenv/salt-key.bat | 2 +- pkg/windows/buildenv/salt-master.bat | 2 +- pkg/windows/buildenv/salt-minion-debug.bat | 2 +- pkg/windows/buildenv/salt-minion.bat | 2 +- pkg/windows/buildenv/salt-run.bat | 2 +- pkg/windows/buildenv/salt.bat | 2 +- pkg/windows/installer/Salt-Minion-Setup.nsi | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/windows/buildenv/salt-call.bat b/pkg/windows/buildenv/salt-call.bat index 6fc4086a9d..55f7cfac3b 100644 --- a/pkg/windows/buildenv/salt-call.bat +++ b/pkg/windows/buildenv/salt-call.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-call :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-cp.bat b/pkg/windows/buildenv/salt-cp.bat index 6c63dce338..61fd1ce444 100644 --- a/pkg/windows/buildenv/salt-cp.bat +++ b/pkg/windows/buildenv/salt-cp.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-cp :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-key.bat b/pkg/windows/buildenv/salt-key.bat index c78b41580b..4928b696d5 100644 --- a/pkg/windows/buildenv/salt-key.bat +++ b/pkg/windows/buildenv/salt-key.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-key :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-master.bat b/pkg/windows/buildenv/salt-master.bat index c1b9575a56..134875c072 100644 --- a/pkg/windows/buildenv/salt-master.bat +++ b/pkg/windows/buildenv/salt-master.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-master :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat index e78887ea63..2e6b5c5bcf 100644 --- a/pkg/windows/buildenv/salt-minion-debug.bat +++ b/pkg/windows/buildenv/salt-minion-debug.bat @@ -12,4 +12,4 @@ Set Script=%SaltDir%\bin\Scripts\salt-minion net stop salt-minion :: Launch Script -"%Python%" -E "%Script%" -l debug +"%Python%" -E -s "%Script%" -l debug diff --git a/pkg/windows/buildenv/salt-minion.bat b/pkg/windows/buildenv/salt-minion.bat index 8ca7ba4d8d..0eb041219c 100644 --- a/pkg/windows/buildenv/salt-minion.bat +++ b/pkg/windows/buildenv/salt-minion.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-run.bat b/pkg/windows/buildenv/salt-run.bat index ca4741c6a9..a8766fc9b0 100644 --- a/pkg/windows/buildenv/salt-run.bat +++ b/pkg/windows/buildenv/salt-run.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-run :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt.bat b/pkg/windows/buildenv/salt.bat index 5eab5b5618..6732ca2968 100644 --- a/pkg/windows/buildenv/salt.bat +++ b/pkg/windows/buildenv/salt.bat @@ -9,4 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt :: Launch Script -"%Python%" -E "%Script%" %* +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 24adb6eb08..ab890529d5 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -334,7 +334,7 @@ Section -Post WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "Path" "$INSTDIR\bin\" ; Register the Salt-Minion Service - nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" + nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E -s $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" From 2868061ee4d15e17327ce66a5b2f14b61f6bc379 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 25 Jul 2017 14:05:58 -0600 Subject: [PATCH 167/300] update chunk, not kwarg in chunk --- salt/state.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/state.py b/salt/state.py index d3d7cd3f81..a3b9563011 100644 --- a/salt/state.py +++ b/salt/state.py @@ -571,14 +571,14 @@ class Compiler(object): if state.startswith('__'): continue chunk = {'state': state, - 'name': name} + 'name': name, + 'arg': [], + 'kwarg': {}} if '__sls__' in body: chunk['__sls__'] = body['__sls__'] if '__env__' in body: chunk['__env__'] = body['__env__'] chunk['__id__'] = name - chunk['arg'] = [] - chunk['kwarg'] = {} for arg in run: if isinstance(arg, six.string_types): funcs.add(arg) @@ -591,7 +591,7 @@ class Compiler(object): names.append(_name) continue else: - chunk['kwarg'].update(arg) + chunk.update(arg) if names: name_order = 1 for entry in names: From 63bb0fb2c461908879663b998c8a06d31443e23d Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 25 Jul 2017 15:34:44 -0600 Subject: [PATCH 168/300] pass in empty kwarg for reactor --- salt/state.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/state.py b/salt/state.py index a3b9563011..44fb73b97d 100644 --- a/salt/state.py +++ b/salt/state.py @@ -571,9 +571,7 @@ class Compiler(object): if state.startswith('__'): continue chunk = {'state': state, - 'name': name, - 'arg': [], - 'kwarg': {}} + 'name': name} if '__sls__' in body: chunk['__sls__'] = body['__sls__'] if '__env__' in body: From 69d5973651ca2230fb409ce8e8c5732b83c62f45 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 26 Jul 2017 16:36:25 -0600 Subject: [PATCH 169/300] Compile scripts with -E -s params for python --- pkg/osx/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index c411840a91..7850d48cd8 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -86,9 +86,9 @@ sudo $PKGRESOURCES/build_env.sh $PYVER # Install Salt ############################################################################ echo -n -e "\033]0;Build: Install Salt\007" -sudo rm -rm $SRCDIR/build -sudo rm -rm $SRCDIR/dist -sudo $PYTHON $SRCDIR/setup.py install +sudo rm -rf $SRCDIR/build +sudo rm -rf $SRCDIR/dist +sudo $PYTHON $SRCDIR/setup.py build -e "$PYTHON -E -s" install ############################################################################ # Build Package From a96f7c09e0ee8589ed36c01c9e293ae2d885f131 Mon Sep 17 00:00:00 2001 From: Guillaume DUBROEUCQ Date: Thu, 27 Jul 2017 14:34:56 +0200 Subject: [PATCH 170/300] yumpkg.py: add option to the command "check-update" --- salt/modules/yumpkg.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 448d347bb6..98b618c734 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -896,7 +896,13 @@ def refresh_db(**kwargs): branch_arg = _get_branch_option(**kwargs) clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache'] - update_cmd = [_yum(), '--quiet', 'check-update'] + + if __grains__.get('os_family') == 'RedHat' and __grains__.get('osmajorrelease') == '7': + # This feature is disabled because it is not used by Salt. Furthermore, this feature lasts a lot with using large repo like EPEL + update_cmd = [_yum(), '--quiet', '--setopt=autocheck_running_kernel=false', 'check-update'] + else: + update_cmd = [_yum(), '--quiet', 'check-update'] + for args in (repo_arg, exclude_arg, branch_arg): if args: clean_cmd.extend(args) From d2ef4483e4370749a6757f2a40bd9f24fedc9ab1 Mon Sep 17 00:00:00 2001 From: Guillaume DUBROEUCQ Date: Thu, 27 Jul 2017 15:03:34 +0200 Subject: [PATCH 171/300] yumpkg.py: clean --- salt/modules/yumpkg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 98b618c734..1947d038ce 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -896,12 +896,12 @@ def refresh_db(**kwargs): branch_arg = _get_branch_option(**kwargs) clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache'] + update_cmd = [_yum(), '--quiet', 'check-update'] if __grains__.get('os_family') == 'RedHat' and __grains__.get('osmajorrelease') == '7': - # This feature is disabled because it is not used by Salt. Furthermore, this feature lasts a lot with using large repo like EPEL - update_cmd = [_yum(), '--quiet', '--setopt=autocheck_running_kernel=false', 'check-update'] - else: - update_cmd = [_yum(), '--quiet', 'check-update'] + # This feature is disable because it is not used by Salt and lasts a lot with using large repo like EPEL + update_cmd.append('--setopt=autocheck_running_kernel=false') + for args in (repo_arg, exclude_arg, branch_arg): if args: From 9c0b5cc1d6fbac5f01a6df0058562551bfbd50fb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 27 Jul 2017 10:59:07 -0600 Subject: [PATCH 172/300] Remove extra newline --- salt/modules/yumpkg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 1947d038ce..9ccd66d105 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -902,7 +902,6 @@ def refresh_db(**kwargs): # This feature is disable because it is not used by Salt and lasts a lot with using large repo like EPEL update_cmd.append('--setopt=autocheck_running_kernel=false') - for args in (repo_arg, exclude_arg, branch_arg): if args: clean_cmd.extend(args) From 0373791f2a9d885fb94b9eec62099566d4367be1 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 27 Jul 2017 11:45:29 -0600 Subject: [PATCH 173/300] Correct capatlization --- salt/modules/win_wua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index baf8c8a9ee..782459365a 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -955,7 +955,7 @@ def set_wu_settings(level=None, Microsoft began using the Unified Update Platform (UUP) starting with Windows 10 / Server 2016. The Windows Update settings have changed and the ability to 'Save' Windows Update settings has been removed. Windows - Update settings are read-only. See msdn documentation: + Update settings are read-only. See MSDN documentation: https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx :param int level: From 559d432930b77a0f5389332b6d8b0857852427da Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 27 Jul 2017 12:00:38 -0600 Subject: [PATCH 174/300] fix tests --- tests/unit/states/test_archive.py | 51 ++++++++++++++++--------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/unit/states/test_archive.py b/tests/unit/states/test_archive.py index 2823711fe7..30d256773d 100644 --- a/tests/unit/states/test_archive.py +++ b/tests/unit/states/test_archive.py @@ -102,16 +102,17 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': mock_run, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): - ret = archive.extracted(tmp_dir, - source, - options=test_opts, - enforce_toplevel=False) - ret_opts.append(source) - mock_run.assert_called_with(ret_opts, - cwd=tmp_dir + os.sep, - python_shell=False) + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): + ret = archive.extracted(tmp_dir, + source, + options=test_opts, + enforce_toplevel=False) + ret_opts.append(source) + mock_run.assert_called_with(ret_opts, + cwd=tmp_dir + os.sep, + python_shell=False) def test_tar_gnutar(self): ''' @@ -142,13 +143,14 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': run_all, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stdout') + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + ret = archive.extracted('/tmp/out', + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stdout') def test_tar_bsdtar(self): ''' @@ -179,10 +181,11 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': run_all, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stderr') + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + ret = archive.extracted('/tmp/out', + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stderr') From 928a4808dd85d0fcf5e5cf60d91db4a20802bacd Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 27 Jul 2017 11:25:23 -0700 Subject: [PATCH 175/300] Updating the superseded and deprecated decorators to work when specified as pillar values. --- salt/utils/decorators/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 8c3daeacd3..86e605933e 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -542,8 +542,14 @@ class _WithDeprecated(_DeprecationDecorator): f_name=function.__name__)) opts = self._globals.get('__opts__', '{}') - use_deprecated = full_name in opts.get(self.CFG_USE_DEPRECATED, list()) - use_superseded = full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) + pillar = self._globals.get('__pillar__', '{}') + + use_deprecated = full_name in opts.get(self.CFG_USE_DEPRECATED, list()) or \ + full_name in pillar.get(self.CFG_USE_DEPRECATED, list()) + + use_superseded = full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) or \ + full_name in pillar.get(self.CFG_USE_SUPERSEDED, list()) + if use_deprecated and use_superseded: raise SaltConfigurationError("Function '{0}' is mentioned both in deprecated " "and superseded sections. Please remove any of that.".format(full_name)) @@ -567,6 +573,8 @@ class _WithDeprecated(_DeprecationDecorator): return func_path in self._globals.get('__opts__').get( self.CFG_USE_DEPRECATED, list()) or (self._policy == self.OPT_IN and not (func_path in self._globals.get('__opts__', {}).get( + self.CFG_USE_SUPERSEDED, list())) + and not (func_path in self._globals.get('__pillar__', {}).get( self.CFG_USE_SUPERSEDED, list()))), func_path def __call__(self, function): From 74bae139399da116a6fb0e2dbe3457cd82caa9f7 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 27 Jul 2017 13:19:48 -0700 Subject: [PATCH 176/300] Small update to something I missed in the first commit. Updating tests to also test for pillar values. --- salt/utils/decorators/__init__.py | 1 + tests/unit/utils/test_decorators.py | 80 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 86e605933e..9d459169bb 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -571,6 +571,7 @@ class _WithDeprecated(_DeprecationDecorator): f_name=self._orig_f_name) return func_path in self._globals.get('__opts__').get( + self.CFG_USE_DEPRECATED, list()) or func_path in self._globals.get('__pillar__').get( self.CFG_USE_DEPRECATED, list()) or (self._policy == self.OPT_IN and not (func_path in self._globals.get('__opts__', {}).get( self.CFG_USE_SUPERSEDED, list())) diff --git a/tests/unit/utils/test_decorators.py b/tests/unit/utils/test_decorators.py index a729ec024c..35ea6da502 100644 --- a/tests/unit/utils/test_decorators.py +++ b/tests/unit/utils/test_decorators.py @@ -59,6 +59,7 @@ class DecoratorsTest(TestCase): self.globs = { '__virtualname__': 'test', '__opts__': {}, + '__pillar__': {}, 'old_function': self.old_function, 'new_function': self.new_function, '_new_function': self._new_function, @@ -149,6 +150,23 @@ class DecoratorsTest(TestCase): ['The function "test.new_function" is using its deprecated ' 'version and will expire in version "Beryllium".']) + def test_with_deprecated_notfound_in_pillar(self): + ''' + Test with_deprecated should raise an exception, if a same name + function with the "_" prefix not implemented. + + :return: + ''' + del self.globs['_new_function'] + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + with self.assertRaises(CommandExecutionError): + depr(self.new_function)() + self.assertEqual(self.messages, + ['The function "test.new_function" is using its deprecated ' + 'version and will expire in version "Beryllium".']) + def test_with_deprecated_found(self): ''' Test with_deprecated should not raise an exception, if a same name @@ -166,6 +184,23 @@ class DecoratorsTest(TestCase): 'and will expire in version "Beryllium".'] self.assertEqual(self.messages, log_msg) + def test_with_deprecated_found_in_pillar(self): + ''' + Test with_deprecated should not raise an exception, if a same name + function with the "_" prefix is implemented, but should use + an old version instead, if "use_deprecated" is requested. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['_new_function'] = self.old_function + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + self.assertEqual(depr(self.new_function)(), self.old_function()) + log_msg = ['The function "test.new_function" is using its deprecated version ' + 'and will expire in version "Beryllium".'] + self.assertEqual(self.messages, log_msg) + def test_with_deprecated_found_eol(self): ''' Test with_deprecated should raise an exception, if a same name @@ -185,6 +220,25 @@ class DecoratorsTest(TestCase): 'is configured as its deprecated version. The lifetime of the function ' '"new_function" expired. Please use its successor "new_function" instead.']) + def test_with_deprecated_found_eol_in_pillar(self): + ''' + Test with_deprecated should raise an exception, if a same name + function with the "_" prefix is implemented, "use_deprecated" is requested + and EOL is reached. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['_new_function'] = self.old_function + depr = decorators.with_deprecated(self.globs, "Helium") + depr._curr_version = self._mk_version("Beryllium")[1] + with self.assertRaises(CommandExecutionError): + depr(self.new_function)() + self.assertEqual(self.messages, + ['Although function "new_function" is called, an alias "new_function" ' + 'is configured as its deprecated version. The lifetime of the function ' + '"new_function" expired. Please use its successor "new_function" instead.']) + def test_with_deprecated_no_conf(self): ''' Test with_deprecated should not raise an exception, if a same name @@ -260,6 +314,19 @@ class DecoratorsTest(TestCase): assert depr(self.new_function)() == self.new_function() assert not self.messages + def test_with_deprecated_opt_in_use_superseded_in_pillar(self): + ''' + Test with_deprecated using opt-in policy, + where newer function is used as per configuration. + + :return: + ''' + self.globs['__pillar__']['use_superseded'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium", policy=decorators._DeprecationDecorator.OPT_IN) + depr._curr_version = self._mk_version("Helium")[1] + assert depr(self.new_function)() == self.new_function() + assert not self.messages + def test_with_deprecated_opt_in_use_superseded_and_deprecated(self): ''' Test with_deprecated misconfiguration. @@ -272,3 +339,16 @@ class DecoratorsTest(TestCase): depr._curr_version = self._mk_version("Helium")[1] with self.assertRaises(SaltConfigurationError): assert depr(self.new_function)() == self.new_function() + + def test_with_deprecated_opt_in_use_superseded_and_deprecated_in_pillar(self): + ''' + Test with_deprecated misconfiguration. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['__pillar__']['use_superseded'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + with self.assertRaises(SaltConfigurationError): + assert depr(self.new_function)() == self.new_function() From cc4e45656d7c99f7d1271a864b936c76e5349e9b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 24 Jul 2017 11:54:42 -0600 Subject: [PATCH 177/300] Allow not interpreting backslashes in the repl Without this, if append or prepend is used with file.replace, the backslashes will not be interpreted when appending the file, but the next time the state is run, the backslashes could be interpreted and removed from the line. --- salt/modules/file.py | 20 +++++++++++++++++--- salt/states/file.py | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 9a0f68c7a6..33e7708abe 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1884,6 +1884,7 @@ def replace(path, show_changes=True, ignore_if_missing=False, preserve_inode=True, + backslash_literal=False, ): ''' .. versionadded:: 0.17.0 @@ -1984,6 +1985,14 @@ def replace(path, filename. Hard links will then share an inode with the backup, instead (if using ``backup`` to create a backup copy). + backslash_literal : False + .. versionadded:: 2016.11.7 + + Interpret backslashes as literal backslashes for the repl and not + escape characters. This will help when using append/prepend so that + the backslashes are not interpreted for the repl on the second run of + the state. + If an equal sign (``=``) appears in an argument to a Salt command it is interpreted as a keyword argument in the format ``key=val``. That processing can be bypassed in order to pass an equal sign through to the @@ -2080,7 +2089,10 @@ def replace(path, if re.search(cpattern, r_data): return True # `with` block handles file closure else: - result, nrepl = re.subn(cpattern, repl, r_data, count) + result, nrepl = re.subn(cpattern, + repl.replace('\\', '\\\\') if backslash_literal else repl, + r_data, + count) # found anything? (even if no change) if nrepl > 0: @@ -2138,8 +2150,10 @@ def replace(path, r_data = mmap.mmap(r_file.fileno(), 0, access=mmap.ACCESS_READ) - result, nrepl = re.subn(cpattern, repl, - r_data, count) + result, nrepl = re.subn(cpattern, + repl.replace('\\', '\\\\') if backslash_literal else repl, + r_data, + count) try: w_file.write(salt.utils.to_str(result)) except (OSError, IOError) as exc: diff --git a/salt/states/file.py b/salt/states/file.py index b950acb173..d404f24eb8 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -3261,7 +3261,8 @@ def replace(name, not_found_content=None, backup='.bak', show_changes=True, - ignore_if_missing=False): + ignore_if_missing=False, + backslash_literal=False): r''' Maintain an edit in a file. @@ -3351,6 +3352,14 @@ def replace(name, state will display an error raised by the execution module. If set to ``True``, the state will simply report no changes. + backslash_literal : False + .. versionadded:: 2016.11.7 + + Interpret backslashes as literal backslashes for the repl and not + escape characters. This will help when using append/prepend so that + the backslashes are not interpreted for the repl on the second run of + the state. + For complex regex patterns, it can be useful to avoid the need for complex quoting and escape sequences by making use of YAML's multiline string syntax. @@ -3399,7 +3408,8 @@ def replace(name, backup=backup, dry_run=__opts__['test'], show_changes=show_changes, - ignore_if_missing=ignore_if_missing) + ignore_if_missing=ignore_if_missing, + backslash_literal=backslash_literal) if changes: ret['pchanges']['diff'] = changes From b9c91eba60fbd7e8fef4cc086d54b171c33cb8e3 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 27 Jul 2017 15:46:58 -0600 Subject: [PATCH 178/300] Add runas_passwd as a global for states --- doc/ref/states/requisites.rst | 23 ++++++++++++++++++++++- salt/modules/cmdmod.py | 3 +++ salt/state.py | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index 6b00b3e856..4fbc0ad972 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -519,7 +519,8 @@ runas .. versionadded:: 2017.7.0 -The ``runas`` global option is used to set the user which will be used to run the command in the ``cmd.run`` module. +The ``runas`` global option is used to set the user which will be used to run +the command in the ``cmd.run`` module. .. code-block:: yaml @@ -532,6 +533,26 @@ The ``runas`` global option is used to set the user which will be used to run th In the above state, the pip command run by ``cmd.run`` will be run by the daniel user. +runas_passwd +~~~~~~~~~~~~ + +.. versionadded:: 2017.7.2 + +The ``runas_passwd`` global option is used to set the password used by the runas +global option. This is required by ``cmd.run`` on Windows when ``runas`` is +specified. It will be set when ``password`` is defined in the state. + +.. code-block:: yaml + + run_script: + cmd.run: + - name: Powershell -NonInteractive -ExecutionPolicy Bypass -File C:\\Temp\\script.ps1 + - runas: frank + - password: supersecret + +In the above state, the Powershell script run by ``cmd.run`` will be run by the +frank user with the password ``supersecret``. + .. _requisites-require-in: .. _requisites-watch-in: .. _requisites-onchanges-in: diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index b576174a6d..ea75865280 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -294,6 +294,9 @@ def _run(cmd, if runas is None and '__context__' in globals(): runas = __context__.get('runas') + if password is None and '__context__' in globals(): + password = __context__.get('runas_passwd') + # Set the default working directory to the home directory of the user # salt-minion is running as. Defaults to home directory of user under which # the minion is running. diff --git a/salt/state.py b/salt/state.py index 44fb73b97d..af50b2c073 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1751,6 +1751,7 @@ class State(object): ret = {'result': False, 'name': low['name'], 'changes': {}} self.state_con['runas'] = low.get('runas', None) + self.state_con['runas_passwd'] = low.get('runas_passwd', None) if not low.get('__prereq__'): log.info( From 41f0f75a0687af29c5954f90dbe59d5eec14362d Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 27 Jul 2017 18:01:26 -0600 Subject: [PATCH 179/300] Add new var to list, change to runas_password --- salt/modules/cmdmod.py | 2 +- salt/state.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index ea75865280..9f3364c594 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -295,7 +295,7 @@ def _run(cmd, runas = __context__.get('runas') if password is None and '__context__' in globals(): - password = __context__.get('runas_passwd') + password = __context__.get('runas_password') # Set the default working directory to the home directory of the user # salt-minion is running as. Defaults to home directory of user under which diff --git a/salt/state.py b/salt/state.py index af50b2c073..2ce6a3adae 100644 --- a/salt/state.py +++ b/salt/state.py @@ -97,6 +97,7 @@ STATE_RUNTIME_KEYWORDS = frozenset([ 'reload_grains', 'reload_pillar', 'runas', + 'runas_password', 'fire_event', 'saltenv', 'use', @@ -1751,7 +1752,7 @@ class State(object): ret = {'result': False, 'name': low['name'], 'changes': {}} self.state_con['runas'] = low.get('runas', None) - self.state_con['runas_passwd'] = low.get('runas_passwd', None) + self.state_con['runas_password'] = low.get('runas_password', None) if not low.get('__prereq__'): log.info( From d42f781c645bac43e09c6b309b66d70cbfe784b0 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 28 Jul 2017 09:38:55 +0100 Subject: [PATCH 180/300] Add trace logging of docker.networks result Added during debugging of failing test to determine what docker.networks is expected to return, keeping as it seems useful. --- salt/states/docker_network.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/states/docker_network.py b/salt/states/docker_network.py index a10162ba9d..28eaa05f67 100644 --- a/salt/states/docker_network.py +++ b/salt/states/docker_network.py @@ -90,6 +90,9 @@ def present(name, driver=None, containers=None): # map containers to container's Ids. containers = [__salt__['docker.inspect_container'](c)['Id'] for c in containers] networks = __salt__['docker.networks'](names=[name]) + log.trace( + 'docker_network.present: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. @@ -159,6 +162,9 @@ def absent(name, driver=None): 'comment': ''} networks = __salt__['docker.networks'](names=[name]) + log.trace( + 'docker_network.absent: current networks: {0}'.format(networks) + ) # networks will contain all Docker networks which partially match 'name'. # We need to loop through to find the matching network, if there is one. From d31f2913bdd5c76f64686fe81c576677939b65e1 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 28 Jul 2017 09:57:45 +0100 Subject: [PATCH 181/300] Fix broken unit test test_network_absent This started failing following commit 515c612, which relied on the 'Name' key being present in the return value of docker.networks - as the mock didn't have this set the test started failing. --- tests/unit/states/test_docker_network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/test_docker_network.py b/tests/unit/states/test_docker_network.py index 7b0037a7d4..aca1e9061e 100644 --- a/tests/unit/states/test_docker_network.py +++ b/tests/unit/states/test_docker_network.py @@ -69,10 +69,14 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): ''' docker_remove_network = Mock(return_value='removed') docker_disconnect_container_from_network = Mock(return_value='disconnected') + docker_networks = Mock(return_value=[{ + 'Name': 'network_foo', + 'Containers': {'container': {}} + }]) __salt__ = { 'docker.remove_network': docker_remove_network, 'docker.disconnect_container_from_network': docker_disconnect_container_from_network, - 'docker.networks': Mock(return_value=[{'Containers': {'container': {}}}]), + 'docker.networks': docker_networks, } with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): From c7d364ec5620a907e105d953b5542a6c057e6e06 Mon Sep 17 00:00:00 2001 From: Andrew Bulford Date: Fri, 28 Jul 2017 10:41:44 +0100 Subject: [PATCH 182/300] Add regression tests for #41982 These test the scenarios where another network with a similar name already exists, verifying that absent doesn't attempt to remove a network which isn't specified, and present still attempts to create the specified network despite a similarly named network already being present. --- tests/unit/states/test_docker_network.py | 40 +++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/test_docker_network.py b/tests/unit/states/test_docker_network.py index aca1e9061e..367441b937 100644 --- a/tests/unit/states/test_docker_network.py +++ b/tests/unit/states/test_docker_network.py @@ -43,10 +43,18 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): docker_create_network = Mock(return_value='created') docker_connect_container_to_network = Mock(return_value='connected') docker_inspect_container = Mock(return_value={'Id': 'abcd'}) + # Get docker.networks to return a network with a name which is a superset of the name of + # the network which is to be created, despite this network existing we should still expect + # that the new network will be created. + # Regression test for #41982. + docker_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) __salt__ = {'docker.create_network': docker_create_network, 'docker.inspect_container': docker_inspect_container, 'docker.connect_container_to_network': docker_connect_container_to_network, - 'docker.networks': Mock(return_value=[]), + 'docker.networks': docker_networks, } with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): @@ -89,3 +97,33 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): 'changes': {'disconnected': 'disconnected', 'removed': 'removed'}, 'result': True}) + + def test_absent_with_matching_network(self): + ''' + Test docker_network.absent when the specified network does not exist, + but another network with a name which is a superset of the specified + name does exist. In this case we expect there to be no attempt to remove + any network. + Regression test for #41982. + ''' + docker_remove_network = Mock(return_value='removed') + docker_disconnect_container_from_network = Mock(return_value='disconnected') + docker_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) + __salt__ = { + 'docker.remove_network': docker_remove_network, + 'docker.disconnect_container_from_network': docker_disconnect_container_from_network, + 'docker.networks': docker_networks, + } + with patch.dict(docker_state.__dict__, + {'__salt__': __salt__}): + ret = docker_state.absent('network_foo') + docker_disconnect_container_from_network.assert_not_called() + docker_remove_network.assert_not_called() + self.assertEqual(ret, {'name': 'network_foo', + 'comment': 'Network \'network_foo\' already absent', + 'changes': {}, + 'result': True}) + From ab21bd9b5b4b260b0533c0f5249a4969e6143962 Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Fri, 28 Jul 2017 06:00:16 -0600 Subject: [PATCH 183/300] Sync cloud modules when saltutil.sync_all is run Fixes #12587 --- salt/runners/saltutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index a6099e18b3..339a8b44ef 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -38,6 +38,7 @@ def sync_all(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' log.debug('Syncing all') ret = {} + ret['clouds'] = sync_clouds(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['modules'] = sync_modules(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['states'] = sync_states(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['grains'] = sync_grains(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) From f83960c02a510ea79ef153e29a41d72ade8e3591 Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Fri, 28 Jul 2017 09:13:59 -0600 Subject: [PATCH 184/300] Lint: Remove extra line at end of file. --- tests/unit/states/test_docker_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/states/test_docker_network.py b/tests/unit/states/test_docker_network.py index 367441b937..552170bdfb 100644 --- a/tests/unit/states/test_docker_network.py +++ b/tests/unit/states/test_docker_network.py @@ -126,4 +126,3 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): 'comment': 'Network \'network_foo\' already absent', 'changes': {}, 'result': True}) - From 2e132daa7370ee4c67a55768bb731016546fefcf Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 28 Jul 2017 08:33:21 -0700 Subject: [PATCH 185/300] Slight update to formatting --- salt/utils/decorators/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 9d459169bb..855b93a5dc 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -544,11 +544,11 @@ class _WithDeprecated(_DeprecationDecorator): opts = self._globals.get('__opts__', '{}') pillar = self._globals.get('__pillar__', '{}') - use_deprecated = full_name in opts.get(self.CFG_USE_DEPRECATED, list()) or \ - full_name in pillar.get(self.CFG_USE_DEPRECATED, list()) + use_deprecated = (full_name in opts.get(self.CFG_USE_DEPRECATED, list()) or + full_name in pillar.get(self.CFG_USE_DEPRECATED, list())) - use_superseded = full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) or \ - full_name in pillar.get(self.CFG_USE_SUPERSEDED, list()) + use_superseded = (full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) or + full_name in pillar.get(self.CFG_USE_SUPERSEDED, list())) if use_deprecated and use_superseded: raise SaltConfigurationError("Function '{0}' is mentioned both in deprecated " From ee4ea6b860574ec43d8ea53f8a721363e6cbe0a4 Mon Sep 17 00:00:00 2001 From: m03 Date: Sun, 11 Jun 2017 17:22:35 -0700 Subject: [PATCH 186/300] Fix #34245 ini.options_present reporting changes --- salt/states/ini_manage.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/states/ini_manage.py b/salt/states/ini_manage.py index 10cff758ca..8afef101d8 100644 --- a/salt/states/ini_manage.py +++ b/salt/states/ini_manage.py @@ -86,8 +86,9 @@ def options_present(name, sections=None, separator='=', strict=False): changes[section_name].update({key_to_remove: ''}) changes[section_name].update({key_to_remove: {'before': orig_value, 'after': None}}) - changes[section_name].update( - __salt__['ini.set_option'](name, {section_name: section_body}, separator)[section_name]) + options_updated = __salt__['ini.set_option'](name, {section_name: section_body}, separator) + if options_updated: + changes[section_name].update(options_updated[section_name]) else: changes = __salt__['ini.set_option'](name, sections, separator) except IOError as err: @@ -99,8 +100,12 @@ def options_present(name, sections=None, separator='=', strict=False): ret['comment'] = 'Errors encountered. {0}'.format(changes['error']) ret['changes'] = {} else: - ret['comment'] = 'Changes take effect' - ret['changes'] = changes + if all(value == {} for value in changes.values()): + ret['changes'] = {} + ret['comment'] = 'No changes take effect' + else: + ret['changes'] = changes + ret['comment'] = 'Changes take effect' return ret From 22c6a7c7ff5a82df7db094aba300a103e56a4515 Mon Sep 17 00:00:00 2001 From: m03 Date: Sun, 11 Jun 2017 18:01:55 -0700 Subject: [PATCH 187/300] Improve output precision --- salt/states/ini_manage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/states/ini_manage.py b/salt/states/ini_manage.py index 8afef101d8..e22202e0f1 100644 --- a/salt/states/ini_manage.py +++ b/salt/states/ini_manage.py @@ -89,6 +89,8 @@ def options_present(name, sections=None, separator='=', strict=False): options_updated = __salt__['ini.set_option'](name, {section_name: section_body}, separator) if options_updated: changes[section_name].update(options_updated[section_name]) + if not changes[section_name]: + del changes[section_name] else: changes = __salt__['ini.set_option'](name, sections, separator) except IOError as err: @@ -100,12 +102,12 @@ def options_present(name, sections=None, separator='=', strict=False): ret['comment'] = 'Errors encountered. {0}'.format(changes['error']) ret['changes'] = {} else: - if all(value == {} for value in changes.values()): - ret['changes'] = {} - ret['comment'] = 'No changes take effect' - else: + if changes: ret['changes'] = changes ret['comment'] = 'Changes take effect' + else: + ret['changes'] = {} + ret['comment'] = 'No changes take effect' return ret From 61aba35718f1157c1addd80676066c529251c51c Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 10:12:44 -0600 Subject: [PATCH 188/300] Deprecate password, make runas_password a named arg --- salt/states/cmd.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 8f55c85832..e032a6caba 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -654,6 +654,7 @@ def run(name, creates=None, cwd=None, runas=None, + runas_password=None, shell=None, env=None, stateful=False, @@ -687,6 +688,9 @@ def run(name, runas The user name to run the command as + runas_password + The password of the user specified in runas. Required for Windows + shell The shell to use for execution, defaults to the shell grain @@ -829,9 +833,19 @@ def run(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') + if 'password' in kwargs: + runas_password = kwargs.pop('password') + salt.utils.warn_until( + 'Neon', + 'The legacy password argument is deprecated. Use runas_password' + 'instead. This warning will be removed in Salt Neon') + if runas is not None: + __context__['runas'] = runas + if runas_password is not None: + __context__['runas_password'] = runas_password + cmd_kwargs = copy.deepcopy(kwargs) cmd_kwargs.update({'cwd': cwd, - 'runas': runas, 'use_vt': use_vt, 'shell': shell or __grains__['shell'], 'env': env, @@ -891,6 +905,7 @@ def script(name, creates=None, cwd=None, runas=None, + runas_password=None, shell=None, env=None, stateful=False, @@ -933,6 +948,9 @@ def script(name, runas The name of the user to run the command as + runas_password + The password of the user specified in runas. Required for Windows + shell The shell to use for execution. The default is set in grains['shell'] @@ -1065,9 +1083,19 @@ def script(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') + if 'password' in kwargs: + runas_password = kwargs.pop('password') + salt.utils.warn_until( + 'Neon', + 'The legacy password argument is deprecated. Use runas_password' + 'instead. This warning will be removed in Salt Neon') + if runas is not None: + __context__['runas'] = runas + if runas_password is not None: + __context__['runas_password'] = runas_password + cmd_kwargs = copy.deepcopy(kwargs) - cmd_kwargs.update({'runas': runas, - 'shell': shell or __grains__['shell'], + cmd_kwargs.update({'shell': shell or __grains__['shell'], 'env': env, 'onlyif': onlyif, 'unless': unless, @@ -1082,7 +1110,6 @@ def script(name, run_check_cmd_kwargs = { 'cwd': cwd, - 'runas': runas, 'shell': shell or __grains__['shell'] } From 4ea264e3dbc04f5781f72258c88d892344024b81 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 10:16:38 -0600 Subject: [PATCH 189/300] Change to runas_password in docs --- doc/ref/states/requisites.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index 4fbc0ad972..c77b7e44ca 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -538,9 +538,9 @@ runas_passwd .. versionadded:: 2017.7.2 -The ``runas_passwd`` global option is used to set the password used by the runas -global option. This is required by ``cmd.run`` on Windows when ``runas`` is -specified. It will be set when ``password`` is defined in the state. +The ``runas_password`` global option is used to set the password used by the +runas global option. This is required by ``cmd.run`` on Windows when ``runas`` +is specified. It will be set when ``runas_password`` is defined in the state. .. code-block:: yaml @@ -548,7 +548,7 @@ specified. It will be set when ``password`` is defined in the state. cmd.run: - name: Powershell -NonInteractive -ExecutionPolicy Bypass -File C:\\Temp\\script.ps1 - runas: frank - - password: supersecret + - runas_password: supersecret In the above state, the Powershell script run by ``cmd.run`` will be run by the frank user with the password ``supersecret``. From cd5eb9390386f7898d6a368f09269046fcc6ea0b Mon Sep 17 00:00:00 2001 From: Andreas Thienemann Date: Thu, 27 Jul 2017 16:40:35 +0200 Subject: [PATCH 190/300] Fix ssh-salt calls with scan roster for uncached clients When using salt-ssh with a scanning roster uncached minions lead to a traceback indicating an error creating the Single() object. Fix this error by ensuring that the target dictionary always contains a host key and the appropriate value. Fixes #42588 --- salt/client/ssh/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 22c770ea16..a25e8838b5 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -467,6 +467,8 @@ class SSH(object): for default in self.defaults: if default not in self.targets[host]: self.targets[host][default] = self.defaults[default] + if 'host' not in self.targets[host]: + self.targets[host]['host'] = host args = ( que, self.opts, From 181a1beecc30d451c46b96b060f9ec1a66db3adc Mon Sep 17 00:00:00 2001 From: Slawomir Bojarski Date: Wed, 26 Jul 2017 16:16:59 -0400 Subject: [PATCH 191/300] Fixed error reporting in "boto_cfn.present" function. Fixes #41433 --- salt/states/boto_cfn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index fcc7375464..87723f7b7f 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -141,10 +141,14 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi stack_policy_body = _get_template(stack_policy_body, name) stack_policy_during_update_body = _get_template(stack_policy_during_update_body, name) + for i in [template_body, stack_policy_body, stack_policy_during_update_body]: + if isinstance(i, dict): + return i + _valid = _validate(template_body, template_url, region, key, keyid, profile) log.debug('Validate is : {0}.'.format(_valid)) if _valid is not True: - code, message = _get_error(_valid) + code, message = _valid ret['result'] = False ret['comment'] = 'Template could not be validated.\n{0} \n{1}'.format(code, message) return ret From 5c945f10c244011a57dd9d97c41ced41677ea7d0 Mon Sep 17 00:00:00 2001 From: Slawomir Bojarski Date: Fri, 28 Jul 2017 12:57:10 -0400 Subject: [PATCH 192/300] Fix debug message in "boto_cfn._validate" function. --- salt/states/boto_cfn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index 87723f7b7f..c996b9006b 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -254,7 +254,7 @@ def _get_template(template, name): def _validate(template_body=None, template_url=None, region=None, key=None, keyid=None, profile=None): # Validates template. returns true if template syntax is correct. validate = __salt__['boto_cfn.validate_template'](template_body, template_url, region, key, keyid, profile) - log.debug('Validate is result is {0}.'.format(str(validate))) + log.debug('Validate result is {0}.'.format(str(validate))) if isinstance(validate, str): code, message = _get_error(validate) log.debug('Validate error is {0} and message is {1}.'.format(code, message)) From 6c71ab6f803320931b38ac07d642bc0a4adea02e Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:03:21 -0600 Subject: [PATCH 193/300] Remove runas and runas_password after state run --- salt/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/state.py b/salt/state.py index 2ce6a3adae..78cc8d128c 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1866,6 +1866,9 @@ class State(object): sys.modules[self.states[cdata['full']].__module__].__opts__[ 'test'] = test + self.state_con.pop('runas') + self.state_con.pop('runas_password') + # If format_call got any warnings, let's show them to the user if 'warnings' in cdata: ret.setdefault('warnings', []).extend(cdata['warnings']) From 18d6ce4d55ca60c6323aca172531bb520ff4642a Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:15:37 -0600 Subject: [PATCH 194/300] Add global vars to cmd.call --- salt/states/cmd.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index e032a6caba..2289dded97 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -839,6 +839,7 @@ def run(name, 'Neon', 'The legacy password argument is deprecated. Use runas_password' 'instead. This warning will be removed in Salt Neon') + if runas is not None: __context__['runas'] = runas if runas_password is not None: @@ -1089,6 +1090,7 @@ def script(name, 'Neon', 'The legacy password argument is deprecated. Use runas_password' 'instead. This warning will be removed in Salt Neon') + if runas is not None: __context__['runas'] = runas if runas_password is not None: @@ -1212,8 +1214,29 @@ def call(name, 'result': False, 'comment': ''} + if 'user' in kwargs or 'group' in kwargs: + salt.utils.warn_until( + 'Oxygen', + 'The legacy user/group arguments are deprecated. ' + 'Replace them with runas. ' + 'These arguments will be removed in Salt Oxygen.' + ) + if 'user' in kwargs and kwargs['user'] is not None and runas is None: + runas = kwargs.pop('user') + + if 'password' in kwargs: + runas_password = kwargs.pop('password') + salt.utils.warn_until( + 'Neon', + 'The legacy password argument is deprecated. Use runas_password' + 'instead. This warning will be removed in Salt Neon') + + if runas is not None: + __context__['runas'] = runas + if runas_password is not None: + __context__['runas_password'] = runas_password + cmd_kwargs = {'cwd': kwargs.get('cwd'), - 'runas': kwargs.get('user'), 'shell': kwargs.get('shell') or __grains__['shell'], 'env': kwargs.get('env'), 'use_vt': use_vt, From 464ec347139917c0b9c8233021012d7e55245f6f Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:20:48 -0600 Subject: [PATCH 195/300] Fix another instance of runas_passwd --- doc/ref/states/requisites.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index c77b7e44ca..3e3d29213c 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -533,8 +533,8 @@ the command in the ``cmd.run`` module. In the above state, the pip command run by ``cmd.run`` will be run by the daniel user. -runas_passwd -~~~~~~~~~~~~ +runas_password +~~~~~~~~~~~~~~ .. versionadded:: 2017.7.2 From 0c9e40012b473f51570516496b7b70e270c3d86f Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 11:39:02 -0600 Subject: [PATCH 196/300] Remove deprecation, add logic to state.py --- salt/state.py | 6 ++++- salt/states/cmd.py | 60 ++++------------------------------------------ 2 files changed, 10 insertions(+), 56 deletions(-) diff --git a/salt/state.py b/salt/state.py index 78cc8d128c..8d1ce6dae0 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1752,7 +1752,11 @@ class State(object): ret = {'result': False, 'name': low['name'], 'changes': {}} self.state_con['runas'] = low.get('runas', None) - self.state_con['runas_password'] = low.get('runas_password', None) + + if low['state'] == 'cmd' and 'password' in low: + self.state_con['runas_password'] = low['password'] + else: + self.state_con['runas_password'] = low.get('runas_password', None) if not low.get('__prereq__'): log.info( diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 2289dded97..8f55c85832 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -654,7 +654,6 @@ def run(name, creates=None, cwd=None, runas=None, - runas_password=None, shell=None, env=None, stateful=False, @@ -688,9 +687,6 @@ def run(name, runas The user name to run the command as - runas_password - The password of the user specified in runas. Required for Windows - shell The shell to use for execution, defaults to the shell grain @@ -833,20 +829,9 @@ def run(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') - if 'password' in kwargs: - runas_password = kwargs.pop('password') - salt.utils.warn_until( - 'Neon', - 'The legacy password argument is deprecated. Use runas_password' - 'instead. This warning will be removed in Salt Neon') - - if runas is not None: - __context__['runas'] = runas - if runas_password is not None: - __context__['runas_password'] = runas_password - cmd_kwargs = copy.deepcopy(kwargs) cmd_kwargs.update({'cwd': cwd, + 'runas': runas, 'use_vt': use_vt, 'shell': shell or __grains__['shell'], 'env': env, @@ -906,7 +891,6 @@ def script(name, creates=None, cwd=None, runas=None, - runas_password=None, shell=None, env=None, stateful=False, @@ -949,9 +933,6 @@ def script(name, runas The name of the user to run the command as - runas_password - The password of the user specified in runas. Required for Windows - shell The shell to use for execution. The default is set in grains['shell'] @@ -1084,20 +1065,9 @@ def script(name, if 'user' in kwargs and kwargs['user'] is not None and runas is None: runas = kwargs.pop('user') - if 'password' in kwargs: - runas_password = kwargs.pop('password') - salt.utils.warn_until( - 'Neon', - 'The legacy password argument is deprecated. Use runas_password' - 'instead. This warning will be removed in Salt Neon') - - if runas is not None: - __context__['runas'] = runas - if runas_password is not None: - __context__['runas_password'] = runas_password - cmd_kwargs = copy.deepcopy(kwargs) - cmd_kwargs.update({'shell': shell or __grains__['shell'], + cmd_kwargs.update({'runas': runas, + 'shell': shell or __grains__['shell'], 'env': env, 'onlyif': onlyif, 'unless': unless, @@ -1112,6 +1082,7 @@ def script(name, run_check_cmd_kwargs = { 'cwd': cwd, + 'runas': runas, 'shell': shell or __grains__['shell'] } @@ -1214,29 +1185,8 @@ def call(name, 'result': False, 'comment': ''} - if 'user' in kwargs or 'group' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'The legacy user/group arguments are deprecated. ' - 'Replace them with runas. ' - 'These arguments will be removed in Salt Oxygen.' - ) - if 'user' in kwargs and kwargs['user'] is not None and runas is None: - runas = kwargs.pop('user') - - if 'password' in kwargs: - runas_password = kwargs.pop('password') - salt.utils.warn_until( - 'Neon', - 'The legacy password argument is deprecated. Use runas_password' - 'instead. This warning will be removed in Salt Neon') - - if runas is not None: - __context__['runas'] = runas - if runas_password is not None: - __context__['runas_password'] = runas_password - cmd_kwargs = {'cwd': kwargs.get('cwd'), + 'runas': kwargs.get('user'), 'shell': kwargs.get('shell') or __grains__['shell'], 'env': kwargs.get('env'), 'use_vt': use_vt, From fcf45889ddf655256fd79d66ff0837c5b49955b8 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 28 Jul 2017 15:35:30 -0500 Subject: [PATCH 197/300] Fix unicode constructor in custom YAML loader 2e7f743 added this in the wrong place, it should not have been added within the if block since we will _always_ want this constructor available. --- salt/utils/yamlloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/utils/yamlloader.py b/salt/utils/yamlloader.py index a51d1f1f25..7beb94b767 100644 --- a/salt/utils/yamlloader.py +++ b/salt/utils/yamlloader.py @@ -45,9 +45,9 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): self.add_constructor( u'tag:yaml.org,2002:omap', type(self).construct_yaml_map) - self.add_constructor( - u'tag:yaml.org,2002:python/unicode', - type(self).construct_unicode) + self.add_constructor( + u'tag:yaml.org,2002:python/unicode', + type(self).construct_unicode) self.dictclass = dictclass def construct_yaml_map(self, node): From b17495c9c86c15cbd30dd0df5709baa5f7b83b5e Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 14:46:01 -0600 Subject: [PATCH 198/300] Fix problem with list when install=True --- salt/modules/win_wua.py | 193 ++++++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 77 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 796da27ee4..74843868a6 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -63,27 +63,31 @@ def available(software=True, Args: - software (bool): Include software updates in the results (default is - True) + software (bool): + Include software updates in the results (default is True) - drivers (bool): Include driver updates in the results (default is False) + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip updates that are already installed. Default - is False. + skip_installed (bool): + Skip updates that are already installed. Default is False. - skip_hidden (bool): Skip updates that have been hidden. Default is True. + skip_hidden (bool): + Skip updates that have been hidden. Default is True. - skip_mandatory (bool): Skip mandatory updates. Default is False. + skip_mandatory (bool): + Skip mandatory updates. Default is False. - skip_reboot (bool): Skip updates that require a reboot. Default is - False. + skip_reboot (bool): + Skip updates that require a reboot. Default is False. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -101,8 +105,9 @@ def available(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -152,19 +157,19 @@ def available(software=True, salt '*' win_wua.available # List all updates with categories of Critical Updates and Drivers - salt '*' win_wua.available categories=['Critical Updates','Drivers'] + salt '*' win_wua.available categories=["Critical Updates","Drivers"] # List all Critical Security Updates - salt '*' win_wua.available categories=['Security Updates'] severities=['Critical'] + salt '*' win_wua.available categories=["Security Updates"] severities=["Critical"] # List all updates with a severity of Critical - salt '*' win_wua.available severities=['Critical'] + salt '*' win_wua.available severities=["Critical"] # A summary of all available updates salt '*' win_wua.available summary=True # A summary of all Feature Packs and Windows 8.1 Updates - salt '*' win_wua.available categories=['Feature Packs','Windows 8.1'] summary=True + salt '*' win_wua.available categories=["Feature Packs","Windows 8.1"] summary=True ''' # Create a Windows Update Agent instance @@ -261,18 +266,21 @@ def get(name, download=False, install=False): Returns details for all updates that match the search criteria Args: - name (str): The name of the update you're searching for. This can be the - GUID, a KB number, or any part of the name of the update. GUIDs and - KBs are preferred. Run ``list`` to get the GUID for the update - you're looking for. + name (str): + The name of the update you're searching for. This can be the GUID, a + KB number, or any part of the name of the update. GUIDs and KBs are + preferred. Run ``list`` to get the GUID for the update you're + looking for. - download (bool): Download the update returned by this function. Run this - function first to see if the update exists, then set ``download=True`` - to download the update. + download (bool): + Download the update returned by this function. Run this function + first to see if the update exists, then set ``download=True`` to + download the update. - install (bool): Install the update returned by this function. Run this - function first to see if the update exists, then set ``install=True`` to - install the update. + install (bool): + Install the update returned by this function. Run this function + first to see if the update exists, then set ``install=True`` to + install the update. Returns: dict: Returns a dict containing a list of updates that match the name if @@ -486,30 +494,34 @@ def list(software=True, install is True the same list will be downloaded and/or installed. Args: - software (bool): Include software updates in the results (default is - True) + software (bool): + Include software updates in the results (default is True) - drivers (bool): Include driver updates in the results (default is False) + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip installed updates in the results (default is - False) + skip_installed (bool): + Skip installed updates in the results (default is False) - download (bool): (Overrides reporting functionality) Download the list - of updates returned by this function. Run this function first with - ``download=False`` to see what will be downloaded, then set - ``download=True`` to download the updates. + download (bool): + (Overrides reporting functionality) Download the list of updates + returned by this function. Run this function first with + ``download=False`` to see what will be downloaded, then set + ``download=True`` to download the updates. - install (bool): (Overrides reporting functionality) Install the list of - updates returned by this function. Run this function first with - ``install=False`` to see what will be installed, then set - ``install=True`` to install the updates. + install (bool): + (Overrides reporting functionality) Install the list of updates + returned by this function. Run this function first with + ``install=False`` to see what will be installed, then set + ``install=True`` to install the updates. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -527,8 +539,9 @@ def list(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -575,22 +588,22 @@ def list(software=True, .. code-block:: bash # Normal Usage (list all software updates) - salt '*' win_wua.list_updates + salt '*' win_wua.list # List all updates with categories of Critical Updates and Drivers - salt '*' win_wua.list_updates categories=['Critical Updates','Drivers'] + salt '*' win_wua.list categories=['Critical Updates','Drivers'] # List all Critical Security Updates - salt '*' win_wua.list_updates categories=['Security Updates'] severities=['Critical'] + salt '*' win_wua.list categories=['Security Updates'] severities=['Critical'] # List all updates with a severity of Critical - salt '*' win_wua.list_updates severities=['Critical'] + salt '*' win_wua.list severities=['Critical'] # A summary of all available updates - salt '*' win_wua.list_updates summary=True + salt '*' win_wua.list summary=True # A summary of all Feature Packs and Windows 8.1 Updates - salt '*' win_wua.list_updates categories=['Feature Packs','Windows 8.1'] summary=True + salt '*' win_wua.list categories=['Feature Packs','Windows 8.1'] summary=True ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -604,11 +617,11 @@ def list(software=True, # Download if download or install: - ret['Download'] = wua.download(updates.updates) + ret['Download'] = wua.download(updates) # Install if install: - ret['Install'] = wua.install(updates.updates) + ret['Install'] = wua.install(updates) if not ret: return updates.summary() if summary else updates.list() @@ -690,9 +703,14 @@ def download(names): Args: - names (str, list): A single update or a list of updates to download. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to download. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. + + .. note:: + An error will be raised if there are more results than there are items + in the names parameter Returns: @@ -703,7 +721,7 @@ def download(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.download guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] + salt '*' win_wua.download names=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -714,6 +732,13 @@ def download(names): if updates.count() == 0: raise CommandExecutionError('No updates found') + # Make sure it's a list so count comparison is correct + if isinstance(names, six.string_types): + names = [names] + + if isinstance(names, six.integer_types): + names = [str(names)] + if updates.count() > len(names): raise CommandExecutionError('Multiple updates found, names need to be ' 'more specific') @@ -795,9 +820,14 @@ def install(names): Args: - names (str, list): A single update or a list of updates to install. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to install. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. + + .. note:: + An error will be raised if there are more results than there are items + in the names parameter Returns: @@ -808,7 +838,7 @@ def install(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.install_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB12323211'] + salt '*' win_wua.install KB12323211 ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -819,6 +849,13 @@ def install(names): if updates.count() == 0: raise CommandExecutionError('No updates found') + # Make sure it's a list so count comparison is correct + if isinstance(names, six.string_types): + names = [names] + + if isinstance(names, six.integer_types): + names = [str(names)] + if updates.count() > len(names): raise CommandExecutionError('Multiple updates found, names need to be ' 'more specific') @@ -834,9 +871,10 @@ def uninstall(names): Args: - names (str, list): A single update or a list of updates to uninstall. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to uninstall. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. Returns: @@ -1110,30 +1148,31 @@ def get_wu_settings(): Boolean value that indicates whether to display notifications for featured updates. Group Policy Required (Read-only): - Boolean value that indicates whether Group Policy requires the Automatic - Updates service. + Boolean value that indicates whether Group Policy requires the + Automatic Updates service. Microsoft Update: Boolean value that indicates whether to turn on Microsoft Update for other Microsoft Products Needs Reboot: - Boolean value that indicates whether the machine is in a reboot pending - state. + Boolean value that indicates whether the machine is in a reboot + pending state. Non Admins Elevated: - Boolean value that indicates whether non-administrators can perform some - update-related actions without administrator approval. + Boolean value that indicates whether non-administrators can perform + some update-related actions without administrator approval. Notification Level: Number 1 to 4 indicating the update level: 1. Never check for updates - 2. Check for updates but let me choose whether to download and install them + 2. Check for updates but let me choose whether to download and + install them 3. Download updates but let me choose whether to install them 4. Install updates automatically Read Only (Read-only): Boolean value that indicates whether the Automatic Update settings are read-only. Recommended Updates: - Boolean value that indicates whether to include optional or recommended - updates when a search for updates and installation of updates is - performed. + Boolean value that indicates whether to include optional or + recommended updates when a search for updates and installation of + updates is performed. Scheduled Day: Days of the week on which Automatic Updates installs or uninstalls updates. From 19f34bda55f18aeb72ce2accb0340887d7689356 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 15:06:52 -0600 Subject: [PATCH 199/300] Fix docs, formatting --- salt/modules/win_wua.py | 156 +++++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 66 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 74843868a6..9ef58881cc 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -188,23 +188,29 @@ def list_update(name, download=False, install=False): ''' .. deprecated:: 2017.7.0 Use :func:`get` instead + Returns details for all updates that match the search criteria Args: - name (str): The name of the update you're searching for. This can be the - GUID, a KB number, or any part of the name of the update. GUIDs and - KBs are preferred. Run ``list_updates`` to get the GUID for the update - you're looking for. - download (bool): Download the update returned by this function. Run this - function first to see if the update exists, then set ``download=True`` - to download the update. + name (str): + The name of the update you're searching for. This can be the GUID, a + KB number, or any part of the name of the update. GUIDs and KBs are + preferred. Run ``list_updates`` to get the GUID for the update + you're looking for. - install (bool): Install the update returned by this function. Run this - function first to see if the update exists, then set ``install=True`` to - install the update. + download (bool): + Download the update returned by this function. Run this function + first to see if the update exists, then set ``download=True`` to + download the update. + + install (bool): + Install the update returned by this function. Run this function + first to see if the update exists, then set ``install=True`` to + install the update. Returns: + dict: Returns a dict containing a list of updates that match the name if download and install are both set to False. Should usually be a single update, but can return multiple if a partial name is given. @@ -263,9 +269,10 @@ def get(name, download=False, install=False): ''' .. versionadded:: 2017.7.0 - Returns details for all updates that match the search criteria + Returns details for the named update Args: + name (str): The name of the update you're searching for. This can be the GUID, a KB number, or any part of the name of the update. GUIDs and KBs are @@ -283,6 +290,7 @@ def get(name, download=False, install=False): install the update. Returns: + dict: Returns a dict containing a list of updates that match the name if download and install are both set to False. Should usually be a single update, but can return multiple if a partial name is given. @@ -365,30 +373,35 @@ def list_updates(software=True, install is True the same list will be downloaded and/or installed. Args: - software (bool): Include software updates in the results (default is - True) - drivers (bool): Include driver updates in the results (default is False) + software (bool): + Include software updates in the results (default is True) + + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip installed updates in the results (default is - False) + skip_installed (bool): + Skip installed updates in the results (default is False) - download (bool): (Overrides reporting functionality) Download the list - of updates returned by this function. Run this function first with - ``download=False`` to see what will be downloaded, then set - ``download=True`` to download the updates. + download (bool): + (Overrides reporting functionality) Download the list of updates + returned by this function. Run this function first with + ``download=False`` to see what will be downloaded, then set + ``download=True`` to download the updates. - install (bool): (Overrides reporting functionality) Install the list of - updates returned by this function. Run this function first with - ``install=False`` to see what will be installed, then set - ``install=True`` to install the updates. + install (bool): + (Overrides reporting functionality) Install the list of updates + returned by this function. Run this function first with + ``install=False`` to see what will be installed, then set + ``install=True`` to install the updates. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -406,8 +419,9 @@ def list_updates(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -494,6 +508,7 @@ def list(software=True, install is True the same list will be downloaded and/or installed. Args: + software (bool): Include software updates in the results (default is True) @@ -638,13 +653,16 @@ def download_update(name): Args: - name (str): The name of the update to download. This can be a GUID, a KB - number, or any part of the name. To ensure a single item is matched the - GUID is preferred. + name (str): + The name of the update to download. This can be a GUID, a KB number, + or any part of the name. To ensure a single item is matched the GUID + is preferred. - .. note:: If more than one result is returned an error will be raised. + .. note:: + If more than one result is returned an error will be raised. Returns: + dict: A dictionary containing the results of the download CLI Examples: @@ -654,7 +672,6 @@ def download_update(name): salt '*' win_wua.download_update 12345678-abcd-1234-abcd-1234567890ab salt '*' win_wua.download_update KB12312321 - ''' salt.utils.warn_until( 'Fluorine', @@ -673,8 +690,9 @@ def download_updates(names): Args: - names (list): A list of updates to download. This can be any combination - of GUIDs, KB numbers, or names. GUIDs or KBs are preferred. + names (list): + A list of updates to download. This can be any combination of GUIDs, + KB numbers, or names. GUIDs or KBs are preferred. Returns: @@ -685,7 +703,7 @@ def download_updates(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.download guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] + salt '*' win_wua.download_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] ''' salt.utils.warn_until( 'Fluorine', @@ -759,10 +777,12 @@ def install_update(name): number, or any part of the name. To ensure a single item is matched the GUID is preferred. - .. note:: If no results or more than one result is returned an error - will be raised. + .. note:: + If no results or more than one result is returned an error will be + raised. Returns: + dict: A dictionary containing the results of the install CLI Examples: @@ -926,34 +946,37 @@ def set_wu_settings(level=None, Update settings are read-only. See MSDN documentation: https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx - :param int level: - Number from 1 to 4 indicating the update level: + Args: + + level (int): + Number from 1 to 4 indicating the update level: + 1. Never check for updates 2. Check for updates but let me choose whether to download and install them 3. Download updates but let me choose whether to install them 4. Install updates automatically - :param bool recommended: - Boolean value that indicates whether to include optional or recommended - updates when a search for updates and installation of updates is - performed. + recommended (bool): + Boolean value that indicates whether to include optional or + recommended updates when a search for updates and installation of + updates is performed. - :param bool featured: - Boolean value that indicates whether to display notifications for - featured updates. + featured (bool): + Boolean value that indicates whether to display notifications for + featured updates. - :param bool elevated: - Boolean value that indicates whether non-administrators can perform some - update-related actions without administrator approval. + elevated (bool): + Boolean value that indicates whether non-administrators can perform + some update-related actions without administrator approval. - :param bool msupdate: - Boolean value that indicates whether to turn on Microsoft Update for - other Microsoft products + msupdate (bool): + Boolean value that indicates whether to turn on Microsoft Update for + other Microsoft products + + day (str): + Days of the week on which Automatic Updates installs or uninstalls + updates. Accepted values: - :param str day: - Days of the week on which Automatic Updates installs or uninstalls - updates. - Accepted values: - Everyday - Monday - Tuesday @@ -962,18 +985,20 @@ def set_wu_settings(level=None, - Friday - Saturday - :param str time: - Time at which Automatic Updates installs or uninstalls updates. Must be - in the ##:## 24hr format, eg. 3:00 PM would be 15:00 + time (str): + Time at which Automatic Updates installs or uninstalls updates. Must + be in the ##:## 24hr format, eg. 3:00 PM would be 15:00. Must be in + 1 hour increments. - :return: Returns a dictionary containing the results. + Returns: + + dict: Returns a dictionary containing the results. CLI Examples: .. code-block:: bash salt '*' win_wua.set_wu_settings level=4 recommended=True featured=False - ''' # The AutomaticUpdateSettings.Save() method used in this function does not # work on Windows 10 / Server 2016. It is called in throughout this function @@ -1255,13 +1280,12 @@ def get_needs_reboot(): Returns: - bool: True if the system requires a reboot, False if not + bool: True if the system requires a reboot, otherwise False CLI Examples: .. code-block:: bash salt '*' win_wua.get_needs_reboot - ''' return salt.utils.win_update.needs_reboot() From 1340c15ce7c0b518e8aed6bb9d89e924e192930d Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Jul 2017 15:38:14 -0600 Subject: [PATCH 200/300] Add general usage instructions --- salt/modules/win_wua.py | 57 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 9ef58881cc..237fb74924 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -2,6 +2,49 @@ ''' Module for managing Windows Updates using the Windows Update Agent. +List updates on the system using the following functions: + +- :ref:`available` +- :ref:`list` + +This is an easy way to find additional information about updates available to +to the system, such as the GUID, KB number, or description. + +Once you have the GUID or a KB number for the update you can get information +about the update, download, install, or uninstall it using these functions: + +- :ref:`get` +- :ref:`download` +- :ref:`install` +- :ref:`uninstall` + +The get function expects a name in the form of a GUID, KB, or Title and should +return information about a single update. The other functions accept either a +single item or a list of items for downloading/installing/uninstalling a +specific list of items. + +The :ref:`list` and :ref:`get` functions are utility functions. In addition to +returning information about updates they can also download and install updates +by setting ``download=True`` or ``install=True``. So, with :ref:`list` for +example, you could run the function with the filters you want to see what is +available. Then just add ``install=True`` to install everything on that list. + +If you want to download, install, or uninstall specific updates, use +:ref:`download`, :ref:`install`, or :ref:`uninstall`. To update your system +with the latest updates use :ref:`list` and set ``install=True`` + +You can also adjust the Windows Update settings using the :ref:`set_wu_settings` +function. This function is only supported on the following operating systems: + +- Windows Vista / Server 2008 +- Windows 7 / Server 2008R2 +- Windows 8 / Server 2012 +- Windows 8.1 / Server 2012R2 + +As of Windows 10 and Windows Server 2016, the ability to modify the Windows +Update settings has been restricted. The settings can be modified in the Local +Group Policy using the ``lgpo`` module. + .. versionadded:: 2015.8.0 :depends: @@ -54,12 +97,12 @@ def available(software=True, skip_mandatory=False, skip_reboot=False, categories=None, - severities=None, - ): + severities=None,): ''' .. versionadded:: 2017.7.0 - List updates that match the passed criteria. + List updates that match the passed criteria. This allows for more filter + options than :func:`list`. Good for finding a specific GUID or KB. Args: @@ -176,9 +219,11 @@ def available(software=True, wua = salt.utils.win_update.WindowsUpdateAgent() # Look for available - updates = wua.available(skip_hidden, skip_installed, skip_mandatory, - skip_reboot, software, drivers, categories, - severities) + updates = wua.available( + skip_hidden=skip_hidden, skip_installed=skip_installed, + skip_mandatory=skip_mandatory, skip_reboot=skip_reboot, + software=software, drivers=drivers, categories=categories, + severities=severities) # Return results as Summary or Details return updates.summary() if summary else updates.list() From 1e13383b95eca138b690433c4ad12b08e123b3a3 Mon Sep 17 00:00:00 2001 From: "xiaofei.sun" Date: Sat, 29 Jul 2017 11:40:02 +0800 Subject: [PATCH 201/300] tornado api --- salt/netapi/rest_tornado/saltnado.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 6622b14635..d70fb6e753 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -521,8 +521,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin try: # Use cgi.parse_header to correctly separate parameters from value - header = cgi.parse_header(self.request.headers['Content-Type']) - value, parameters = header + value, parameters = cgi.parse_header(self.request.headers['Content-Type']) return ct_in_map[value](data) except KeyError: self.send_error(406) @@ -536,7 +535,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin if not self.request.body: return data = self.deserialize(self.request.body) - self.raw_data = copy(data) + self.request_payload = copy(data) if 'arg' in data and not isinstance(data['arg'], list): data['arg'] = [data['arg']] @@ -694,15 +693,13 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 }} ''' try: - request_payload = self.deserialize(self.request.body) - - if not isinstance(request_payload, dict): + if not isinstance(self.request_payload, dict): self.send_error(400) return - creds = {'username': request_payload['username'], - 'password': request_payload['password'], - 'eauth': request_payload['eauth'], + creds = {'username': self.request_payload['username'], + 'password': self.request_payload['password'], + 'eauth': self.request_payload['eauth'], } # if any of the args are missing, its a bad request except KeyError: @@ -1633,7 +1630,7 @@ class WebhookSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 listen=False) ret = self.event.fire_event({ - 'post': self.raw_data, + 'post': self.request_payload, 'get': dict(self.request.query_arguments), # In Tornado >= v4.0.3, the headers come # back as an HTTPHeaders instance, which From 49023deb9438a67c46c92776b09896ce1feb24ad Mon Sep 17 00:00:00 2001 From: Morgan Willcock Date: Mon, 31 Jul 2017 21:17:35 +0100 Subject: [PATCH 202/300] Disable ZFS grain on NetBSD --- salt/grains/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index 71ad3d0721..824ec0fccd 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2351,6 +2351,10 @@ def _zpool_data(grains): if salt.utils.is_windows() or 'proxyminion' in __opts__: return {} + # quickly return if NetBSD (ZFS still under development) + if salt.utils.is_netbsd(): + return {} + # quickly return if no zpool and zfs command if not salt.utils.which('zpool'): return {} From 6bd91c8b03ef14bf1de1e66bcfd23b557a011701 Mon Sep 17 00:00:00 2001 From: Seth House Date: Mon, 31 Jul 2017 14:15:52 -0600 Subject: [PATCH 203/300] Reenable cpstats for rest_cherrypy --- salt/netapi/rest_cherrypy/app.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 48f1330018..5c55433bb3 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -468,11 +468,19 @@ import tarfile from multiprocessing import Process, Pipe # Import third-party libs -# pylint: disable=import-error +# pylint: disable=import-error, 3rd-party-module-not-gated import cherrypy +try: + from cherrypy.lib import cpstats +except ImportError: + cpstats = None + logger.warn('Import of cherrypy.cpstats failed. ' + 'Possible upstream bug: ' + 'https://github.com/cherrypy/cherrypy/issues/1444') + import yaml import salt.ext.six as six -# pylint: enable=import-error +# pylint: enable=import-error, 3rd-party-module-not-gated # Import Salt libs @@ -2446,13 +2454,6 @@ class Stats(object): :status 406: |406| ''' if hasattr(logging, 'statistics'): - # Late import - try: - from cherrypy.lib import cpstats - except ImportError: - logger.error('Import of cherrypy.cpstats failed. Possible ' - 'upstream bug here: https://github.com/cherrypy/cherrypy/issues/1444') - return {} return cpstats.extrapolate_statistics(logging.statistics) return {} @@ -2559,13 +2560,14 @@ class API(object): 'tools.trailing_slash.on': True, 'tools.gzip.on': True, - 'tools.cpstats.on': self.apiopts.get('collect_stats', False), - 'tools.html_override.on': True, 'tools.cors_tool.on': True, }, } + if cpstats and self.apiopts.get('collect_stats', False): + conf['/']['tools.cpstats.on'] = True + if 'favicon' in self.apiopts: conf['/favicon.ico'] = { 'tools.staticfile.on': True, From d73c4b55b7ca288c25b8e21ddfb879ede646cc9b Mon Sep 17 00:00:00 2001 From: Neile Havens Date: Mon, 31 Jul 2017 17:10:06 -0500 Subject: [PATCH 204/300] back-port #42612 to 2017.7 --- salt/states/selinux.py | 12 +++++++++--- tests/unit/states/test_selinux.py | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/states/selinux.py b/salt/states/selinux.py index 868dbc16e3..8187ea8338 100644 --- a/salt/states/selinux.py +++ b/salt/states/selinux.py @@ -171,8 +171,14 @@ def boolean(name, value, persist=False): name, rvalue) return ret - if __salt__['selinux.setsebool'](name, rvalue, persist): + ret['result'] = __salt__['selinux.setsebool'](name, rvalue, persist) + if ret['result']: ret['comment'] = 'Boolean {0} has been set to {1}'.format(name, rvalue) + ret['changes'].update({'State': {'old': bools[name]['State'], + 'new': rvalue}}) + if persist and not default: + ret['changes'].update({'Default': {'old': bools[name]['Default'], + 'new': rvalue}}) return ret ret['comment'] = 'Failed to set the boolean {0} to {1}'.format(name, rvalue) return ret @@ -262,7 +268,7 @@ def module_install(name): name Path to file with module to install - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' ret = {'name': name, 'result': True, @@ -283,7 +289,7 @@ def module_remove(name): name The name of the module to remove - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' ret = {'name': name, 'result': True, diff --git a/tests/unit/states/test_selinux.py b/tests/unit/states/test_selinux.py index 36474b95c4..f6ce3bfb8f 100644 --- a/tests/unit/states/test_selinux.py +++ b/tests/unit/states/test_selinux.py @@ -118,9 +118,11 @@ class SelinuxTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(selinux.__opts__, {'test': False}): comt = ('Boolean samba_create_home_dirs has been set to on') ret.update({'comment': comt, 'result': True}) + ret.update({'changes': {'State': {'old': 'off', 'new': 'on'}}}) self.assertDictEqual(selinux.boolean(name, value), ret) comt = ('Failed to set the boolean ' 'samba_create_home_dirs to on') - ret.update({'comment': comt, 'result': True}) + ret.update({'comment': comt, 'result': False}) + ret.update({'changes': {}}) self.assertDictEqual(selinux.boolean(name, value), ret) From 24413084e2255c6319e1213fdbdacc21a4ccc8cb Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 1 Aug 2017 12:40:31 -0700 Subject: [PATCH 205/300] Updating the call to shlex_split to pass the posix=False argument so that quotes are preserved. --- salt/modules/augeas_cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/augeas_cfg.py b/salt/modules/augeas_cfg.py index 6b1f1e7b1b..401a11eb1b 100644 --- a/salt/modules/augeas_cfg.py +++ b/salt/modules/augeas_cfg.py @@ -199,7 +199,7 @@ def execute(context=None, lens=None, commands=(), load_path=None): method = METHOD_MAP[cmd] nargs = arg_map[method] - parts = salt.utils.shlex_split(arg) + parts = salt.utils.shlex_split(arg, posix=False) if len(parts) not in nargs: err = '{0} takes {1} args: {2}'.format(method, nargs, parts) From deb6316d6782807a692756347a2ad3889b9d71a8 Mon Sep 17 00:00:00 2001 From: Seth House Date: Tue, 1 Aug 2017 14:17:13 -0600 Subject: [PATCH 206/300] Fix lint errors --- salt/netapi/rest_cherrypy/app.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 5c55433bb3..6eb389242e 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -467,8 +467,10 @@ import signal import tarfile from multiprocessing import Process, Pipe +logger = logging.getLogger(__name__) + # Import third-party libs -# pylint: disable=import-error, 3rd-party-module-not-gated +# pylint: disable=import-error import cherrypy try: from cherrypy.lib import cpstats @@ -480,8 +482,7 @@ except ImportError: import yaml import salt.ext.six as six -# pylint: enable=import-error, 3rd-party-module-not-gated - +# pylint: enable=import-error # Import Salt libs import salt @@ -491,8 +492,6 @@ import salt.utils.event # Import salt-api libs import salt.netapi -logger = logging.getLogger(__name__) - # Imports related to websocket try: from .tools import websockets From a2565ba8e570f3e969ed5660e3a798e6fd852610 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 2 Aug 2017 09:39:18 +0000 Subject: [PATCH 207/300] Add new napalm option: multiprocessing --- salt/utils/napalm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index 36914be042..0ca387cc68 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -243,6 +243,11 @@ def get_device_opts(opts, salt_obj=None): network_device = {} # by default, look in the proxy config details device_dict = opts.get('proxy', {}) or opts.get('napalm', {}) + if opts.get('proxy') or opts.get('napalm'): + opts['multiprocessing'] = device_dict.get('multiprocessing', False) + # Most NAPALM drivers are SSH-based, so multiprocessing should default to False. + # But the user can be allows to have a different value for the multiprocessing, which will + # override the opts. if salt_obj and not device_dict: # get the connection details from the opts device_dict = salt_obj['config.merge']('napalm') From 1598571f52583213ccc6d5d0ef8b403483cdc72f Mon Sep 17 00:00:00 2001 From: Frankie Hui Date: Wed, 2 Aug 2017 17:47:31 +0800 Subject: [PATCH 208/300] Add eos.rst in the installation guide --- doc/topics/installation/eos.rst | 155 ++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 doc/topics/installation/eos.rst diff --git a/doc/topics/installation/eos.rst b/doc/topics/installation/eos.rst new file mode 100644 index 0000000000..f7537c334b --- /dev/null +++ b/doc/topics/installation/eos.rst @@ -0,0 +1,155 @@ + +========================================= +Arista EOS Salt minion installation guide +========================================= + +The Salt minion for Arista EOS is distributed as a SWIX extension and can be installed directly on the switch. The EOS network operating system is based on old Fedora distributions and the installation of the ``salt-minion`` requires backports. This SWIX extension contains the necessary backports, together with the Salt basecode. + +.. note:: + + This SWIX extension has been tested on Arista DCS-7280SE-68-R, running EOS 4.17.5M and vEOS 4.18.3F. + +Important Notes +=============== + +This package is in beta, make sure to test it carefully before running it in production. + +If confirmed working correctly, please report and add a note on this page with the platform model and EOS version. + +If you want to uninstall this package, please refer to the uninstalling_ section. + +Installation from the Official SaltStack Repository +=================================================== + +Download the swix package and save it to flash. + +.. code-block:: bash + + veos#copy https://salt-eos.netops.life/salt-eos-latest.swix flash: + veos#copy https://salt-eos.netops.life/startup.sh flash: + +Install the Extension +===================== + +Copy the Salt package to extension + +.. code-block:: bash + + veos#copy flash:salt-eos-latest.swix extension: + +Install the SWIX + +.. code-block:: bash + + veos#extension salt-eos-latest.swix force + +Verify the installation + +.. code-block:: bash + + veos#show extensions | include salt-eos + salt-eos-2017-07-19.swix 1.0.11/1.fc25 A, F 27 + +Change the Salt master IP address or FQDN, by edit the variable (SALT_MASTER) + +.. code-block:: bash + + veos#bash vi /mnt/flash/startup.sh + +Make sure you enable the eAPI with unix-socket + +.. code-block:: bash + + veos(config)#management api http-commands + protocol unix-socket + no shutdown + +Post-installation tasks +======================= + +Generate Keys and host record and start Salt minion + +.. code-block:: bash + + veos#bash + #sudo /mnt/flash/startup.sh + +``salt-minion`` should be running + +Copy the installed extensions to boot-extensions + +.. code-block:: bash + + veos#copy installed-extensions boot-extensions + +Apply event-handler to let EOS start salt-minion during boot-up + +.. code-block:: bash + + veos(config)#event-handler boot-up-script + trigger on-boot + action bash sudo /mnt/flash/startup.sh + +For more specific installation details of the ``salt-minion``, please refer to :ref:`Configuring Salt`. + +.. _uninstalling: + +Uninstalling +============ + +If you decide to uninstall this package, the following steps are recommended for safety: + +1. Remove the extension from boot-extensions + +.. code-block:: bash + + veos#bash rm /mnt/flash/boot-extensions + +2. Remove the extension from extensions folder + +.. code-block:: bash + + veos#bash rm /mnt/flash/.extensions/salt-eos-latest.swix + +2. Remove boot-up script + +.. code-block:: bash + + veos(config)#no event-handler boot-up-script + +Additional Information +====================== + +This SWIX extension contains the following RPM packages: + +.. code-block:: + + libsodium-1.0.11-1.fc25.i686.rpm + libstdc++-6.2.1-2.fc25.i686.rpm + openpgm-5.2.122-6.fc24.i686.rpm + python-Jinja2-2.8-0.i686.rpm + python-PyYAML-3.12-0.i686.rpm + python-babel-0.9.6-5.fc18.noarch.rpm + python-backports-1.0-3.fc18.i686.rpm + python-backports-ssl_match_hostname-3.4.0.2-1.fc18.noarch.rpm + python-backports_abc-0.5-0.i686.rpm + python-certifi-2016.9.26-0.i686.rpm + python-chardet-2.0.1-5.fc18.noarch.rpm + python-crypto-1.4.1-1.noarch.rpm + python-crypto-2.6.1-1.fc18.i686.rpm + python-futures-3.1.1-1.noarch.rpm + python-jtextfsm-0.3.1-0.noarch.rpm + python-kitchen-1.1.1-2.fc18.noarch.rpm + python-markupsafe-0.18-1.fc18.i686.rpm + python-msgpack-python-0.4.8-0.i686.rpm + python-napalm-base-0.24.3-1.noarch.rpm + python-napalm-eos-0.6.0-1.noarch.rpm + python-netaddr-0.7.18-0.noarch.rpm + python-pyeapi-0.7.0-0.noarch.rpm + python-salt-2017.7.0_1414_g2fb986f-1.noarch.rpm + python-singledispatch-3.4.0.3-0.i686.rpm + python-six-1.10.0-0.i686.rpm + python-tornado-4.4.2-0.i686.rpm + python-urllib3-1.5-7.fc18.noarch.rpm + python2-zmq-15.3.0-2.fc25.i686.rpm + zeromq-4.1.4-5.fc25.i686.rpm From 37bca1b90203f8faf952ed4af17f4047f8dc27dc Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 2 Aug 2017 10:23:53 +0000 Subject: [PATCH 209/300] Add multiprocessing option for NAPALM proxy Overrides the :conf_minion:`multiprocessing` option, per proxy minion. The ``multiprocessing`` option must be turned off for SSH-based proxies. However, some NAPALM drivers (e.g. Arista, NX-OS) are not SSH-based. As multiple proxy minions may share the same configuration file, this option permits the configuration of the ``multiprocessing`` option more specifically, for some proxy minions. Additionally, this helps to default the option to False, in case the user does not configure it explicitly in the proxy config file. --- salt/proxy/napalm.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index 9ae4fd5316..3a9b6a0465 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -67,6 +67,17 @@ provider: ``napalm_base`` .. versionadded:: 2017.7.1 +multiprocessing: ``False`` + Overrides the :conf_minion:`multiprocessing` option, per proxy minion. + The ``multiprocessing`` option must be turned off for SSH-based proxies. + However, some NAPALM drivers (e.g. Arista, NX-OS) are not SSH-based. + As multiple proxy minions may share the same configuration file, + this option permits the configuration of the ``multiprocessing`` option + more specifically, for some proxy minions. + + .. versionadded:: 2017.7.1 + + .. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems .. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments From 9c4566db0c7c69631e30a436645d5bef7766988c Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 2 Aug 2017 14:51:58 +0000 Subject: [PATCH 210/300] multiprocessing option tagged for 2017.7.2 --- salt/proxy/napalm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index 3a9b6a0465..1acee33148 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -75,7 +75,7 @@ multiprocessing: ``False`` this option permits the configuration of the ``multiprocessing`` option more specifically, for some proxy minions. - .. versionadded:: 2017.7.1 + .. versionadded:: 2017.7.2 .. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems From 157fb28851dc4db152975a2efbe11c11d9aa72e3 Mon Sep 17 00:00:00 2001 From: Andy Hibbert Date: Wed, 2 Aug 2017 15:59:08 +0100 Subject: [PATCH 211/300] boto_efs_fix_tags: Fix #42688 invalid type for parameter tags Change-Id: Ibab5417074b1ac98ee025c421d04674070745edf --- salt/modules/boto_efs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/boto_efs.py b/salt/modules/boto_efs.py index 0bb20c31d2..1ef85d9233 100644 --- a/salt/modules/boto_efs.py +++ b/salt/modules/boto_efs.py @@ -158,7 +158,7 @@ def create_file_system(name, import os import base64 creation_token = base64.b64encode(os.urandom(46), ['-', '_']) - tags = {"Key": "Name", "Value": name} + tags = [{"Key": "Name", "Value": name}] client = _get_conn(key=key, keyid=keyid, profile=profile, region=region) From 287b57b5c5b7dca4365543951221454f30eb41e9 Mon Sep 17 00:00:00 2001 From: NLR Date: Wed, 2 Aug 2017 17:50:02 +0200 Subject: [PATCH 212/300] Fix RabbitMQ tags not properly set. --- salt/modules/rabbitmq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index 4cfdeb4d3d..44cf514c9f 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -472,11 +472,11 @@ def set_user_tags(name, tags, runas=None): if runas is None: runas = salt.utils.get_user() - if tags and isinstance(tags, (list, tuple)): - tags = ' '.join(tags) + if not isinstance(tags, (list, tuple)): + tags = [tags] res = __salt__['cmd.run']( - ['rabbitmqctl', 'set_user_tags', name, tags], + ['rabbitmqctl', 'set_user_tags', name] + tags, runas=runas, python_shell=False) msg = "Tag(s) set" From 78fccdc7e2480d59b4625471925f5e792a4e36a8 Mon Sep 17 00:00:00 2001 From: NLR Date: Wed, 2 Aug 2017 17:57:41 +0200 Subject: [PATCH 213/300] Cast to list in case tags is a tuple. --- salt/modules/rabbitmq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index 44cf514c9f..3ae6f71212 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -476,7 +476,7 @@ def set_user_tags(name, tags, runas=None): tags = [tags] res = __salt__['cmd.run']( - ['rabbitmqctl', 'set_user_tags', name] + tags, + ['rabbitmqctl', 'set_user_tags', name] + list(tags), runas=runas, python_shell=False) msg = "Tag(s) set" From 1a0457af51769b4e3f6132dd8bba36eec4546587 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 2 Aug 2017 10:30:29 -0600 Subject: [PATCH 214/300] allow adding extra remotes to a repository --- salt/states/git.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 274662e83b..caf75399de 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1154,13 +1154,22 @@ def latest(name, password=password, https_user=https_user, https_pass=https_pass) - comments.append( - 'Remote \'{0}\' changed from {1} to {2}'.format( - remote, - salt.utils.url.redact_http_basic_auth(fetch_url), - redacted_fetch_url + if fetch_url is None: + comments.append( + 'Remote \'{0}\' set to {1}'.format( + remote, + redacted_fetch_url + ) + ) + ret['changes']['new'] = name + ' => ' + remote + else: + comments.append( + 'Remote \'{0}\' changed from {1} to {2}'.format( + remote, + salt.utils.url.redact_http_basic_auth(fetch_url), + redacted_fetch_url + ) ) - ) if remote_rev is not None: if __opts__['test']: From 03b675a6185fe824808499695444234d5b729b2a Mon Sep 17 00:00:00 2001 From: Seth House Date: Wed, 2 Aug 2017 12:51:59 -0600 Subject: [PATCH 215/300] Add import to work around likely multiprocessing scoping bug Fixes #42649. --- salt/client/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 1073223ae1..2e4aead071 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -544,6 +544,7 @@ class LocalClient(object): {'stewart': {...}} ''' if 'expr_form' in kwargs: + import salt salt.utils.warn_until( 'Fluorine', 'The target type should be passed using the \'tgt_type\' ' From a260e913b5e4ee9fcfb7bd1420ddc53ad7064db2 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Wed, 2 Aug 2017 13:44:32 -0600 Subject: [PATCH 216/300] Do not change the arguments of the function when memoizing --- salt/utils/decorators/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index 855b93a5dc..f7066b48bf 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -251,9 +251,8 @@ def memoize(func): str_args.append(str(arg)) else: str_args.append(arg) - args = str_args - args_ = ','.join(list(args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)]) + args_ = ','.join(list(str_args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)]) if args_ not in cache: cache[args_] = func(*args, **kwargs) return cache[args_] From 8604312a7b453ce056a4a0af193c25a2c341dbff Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 2 Aug 2017 17:12:02 -0600 Subject: [PATCH 217/300] Remove master conf in minion install --- pkg/windows/build_pkg.bat | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/windows/build_pkg.bat b/pkg/windows/build_pkg.bat index b6b52f3bda..0d30f047ac 100644 --- a/pkg/windows/build_pkg.bat +++ b/pkg/windows/build_pkg.bat @@ -108,9 +108,9 @@ xcopy /E /Q "%PyDir%" "%BinDir%\" @echo Copying configs to buildenv\conf... @echo ---------------------------------------------------------------------- @echo xcopy /E /Q "%SrcDir%\conf\master" "%CnfDir%\" -xcopy /Q "%SrcDir%\conf\master" "%CnfDir%\" +xcopy /Q /Y "%SrcDir%\conf\master" "%CnfDir%\" @echo xcopy /E /Q "%SrcDir%\conf\minion" "%CnfDir%\" -xcopy /Q "%SrcDir%\conf\minion" "%CnfDir%\" +xcopy /Q /Y "%SrcDir%\conf\minion" "%CnfDir%\" @echo. @echo Copying VCRedist to Prerequisites @@ -582,6 +582,10 @@ If Exist "%BinDir%\Scripts\salt-run*"^ If Exist "%BldDir%\salt-run.bat"^ del /Q "%BldDir%\salt-run.bat" 1>nul +:: Remove the master config file +if Exist "%CnfDir%\master"^ + del /Q "%CnfDir%\master" 1>nul + :: Make the Salt Minion Installer makensis.exe /DSaltVersion=%Version% /DPythonVersion=%Python% "%InsDir%\Salt-Minion-Setup.nsi" @echo. From 834d6c605e221e349ec3454419ee3cd5c9a96b7b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 2 Aug 2017 23:11:52 -0600 Subject: [PATCH 218/300] Set fact gathering style to 'old' for test_junos Without this, we stacktrace because it does not appear that setting 'gather_facts' to False prevents the library from assuming the presence of facts. I believe this to be an upstream bug with jnpr. Because they have listed this as being a deprecated option in the future this may re-break in the future. --- tests/unit/modules/test_junos.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/modules/test_junos.py b/tests/unit/modules/test_junos.py index b40793e71e..34176d9d71 100644 --- a/tests/unit/modules/test_junos.py +++ b/tests/unit/modules/test_junos.py @@ -52,6 +52,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): host='1.1.1.1', user='test', password='test123', + fact_style='old', gather_facts=False) self.dev.open() self.dev.timeout = 30 From f58256802ab1a2c1f00b4c47be3c55bb79fd6a6e Mon Sep 17 00:00:00 2001 From: Andy Hibbert Date: Thu, 3 Aug 2017 12:01:47 +0100 Subject: [PATCH 219/300] allow_no_ip_sg: Allow user to not supply ipaddress or securitygroups when running boto_efs.create_mount_target Change-Id: I558de2e80e71399a442913f33fc69b5d84490361 --- salt/modules/boto_efs.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/salt/modules/boto_efs.py b/salt/modules/boto_efs.py index 0bb20c31d2..8e20a04cec 100644 --- a/salt/modules/boto_efs.py +++ b/salt/modules/boto_efs.py @@ -223,10 +223,23 @@ def create_mount_target(filesystemid, client = _get_conn(key=key, keyid=keyid, profile=profile, region=region) - return client.create_mount_point(FileSystemId=filesystemid, - SubnetId=subnetid, - IpAddress=ipaddress, - SecurityGroups=securitygroups) + if ipaddress is None and securitygroups is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid) + + if ipaddress is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + SecurityGroups=securitygroups) + if securitygroups is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + IpAddress=ipaddress) + + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + IpAddress=ipaddress, + SecurityGroups=securitygroups) def create_tags(filesystemid, From 4bbfc751ae9e8c786229dacb3a7ebda1b303ed16 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 1 Aug 2017 15:26:17 -0600 Subject: [PATCH 220/300] render kubernetes docs --- doc/ref/modules/all/index.rst | 1 + doc/ref/modules/all/salt.modules.kubernetes.rst | 6 ++++++ doc/ref/states/all/index.rst | 1 + doc/ref/states/all/salt.states.kubernetes.rst | 6 ++++++ 4 files changed, 14 insertions(+) create mode 100644 doc/ref/modules/all/salt.modules.kubernetes.rst create mode 100644 doc/ref/states/all/salt.states.kubernetes.rst diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index e0b0198791..f325f8f375 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -195,6 +195,7 @@ execution modules keyboard keystone kmod + kubernetes launchctl layman ldap3 diff --git a/doc/ref/modules/all/salt.modules.kubernetes.rst b/doc/ref/modules/all/salt.modules.kubernetes.rst new file mode 100644 index 0000000000..a0f715d617 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.kubernetes.rst @@ -0,0 +1,6 @@ +======================= +salt.modules.kubernetes +======================= + +.. automodule:: salt.modules.kubernetes + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 03de35190e..61e38a2b52 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -135,6 +135,7 @@ state modules keyboard keystone kmod + kubernetes layman ldap libcloud_dns diff --git a/doc/ref/states/all/salt.states.kubernetes.rst b/doc/ref/states/all/salt.states.kubernetes.rst new file mode 100644 index 0000000000..a0f715d617 --- /dev/null +++ b/doc/ref/states/all/salt.states.kubernetes.rst @@ -0,0 +1,6 @@ +======================= +salt.modules.kubernetes +======================= + +.. automodule:: salt.modules.kubernetes + :members: From bca17902f5d968ed1710820efdfcabc2e422495b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 2 Aug 2017 08:55:52 -0600 Subject: [PATCH 221/300] add version added info --- doc/topics/releases/2017.7.0.rst | 2 ++ salt/modules/kubernetes.py | 1 + salt/states/kubernetes.py | 2 ++ 3 files changed, 5 insertions(+) diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index f170176d01..a0e257b347 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -706,6 +706,7 @@ Execution modules - :mod:`salt.modules.grafana4 ` - :mod:`salt.modules.heat ` - :mod:`salt.modules.icinga2 ` +- :mod:`salt.modules.kubernetes ` - :mod:`salt.modules.logmod ` - :mod:`salt.modules.mattermost ` - :mod:`salt.modules.namecheap_dns ` @@ -784,6 +785,7 @@ States - :mod:`salt.states.icinga2 ` - :mod:`salt.states.influxdb_continuous_query ` - :mod:`salt.states.influxdb_retention_policy ` +- :mod:`salt.states.kubernetes ` - :mod:`salt.states.logadm ` - :mod:`salt.states.logrotate ` - :mod:`salt.states.msteams ` diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index b68b83057b..850534bf56 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -16,6 +16,7 @@ or `api_password` parameters when calling a function: .. code-block:: bash salt '*' kubernetes.nodes api_url=http://k8s-api-server:port api_user=myuser api_password=pass +.. versionadded: 2017.7.0 ''' # Import Python Futures diff --git a/salt/states/kubernetes.py b/salt/states/kubernetes.py index 64cd451532..bc62da1ab7 100644 --- a/salt/states/kubernetes.py +++ b/salt/states/kubernetes.py @@ -73,6 +73,8 @@ The kubernetes module is used to manage different kubernetes resources. key1: value1 key2: value2 key3: value3 + +.. versionadded: 2017.7.0 ''' from __future__ import absolute_import From 1958d1863424b95f8673d2d3eec8d54dffc37247 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:25:27 -0600 Subject: [PATCH 222/300] python2- prefix for fedora 26 packages --- salt/modules/yumpkg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 8f68747c75..4b10f34c0b 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,7 +181,10 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - vl_plugin = 'python-dnf-plugins-extras-versionlock' + if int(__grains__.get('osmajorrelease')) < 26: + vl_plugin = 'python-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python2-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From f179b97b52987d124fe522a43ff570f235bdf10b Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:56:30 -0600 Subject: [PATCH 223/300] add py3 for versionlock --- salt/modules/yumpkg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 4b10f34c0b..064a6aaa45 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,7 +181,9 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if int(__grains__.get('osmajorrelease')) < 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) < 26: vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'python2-dnf-plugins-extras-versionlock' From 178cc1bd8183b8285c4b6806a4fe093451cbc89a Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 14:15:52 -0600 Subject: [PATCH 224/300] make sure names are correct --- salt/modules/yumpkg.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 064a6aaa45..1495033153 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,12 +181,16 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if six.PY3: - vl_plugin = 'python3-dnf-plugins-extras-versionlock' - elif int(__grains__.get('osmajorrelease')) < 26: - vl_plugin = 'python-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) >= 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugin-versionlock' + else: + vl_plugin = 'python2-dnf-plugin-versionlock' else: - vl_plugin = 'python2-dnf-plugins-extras-versionlock' + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From 8784899942aa6ed48ccd7c99b226df2535a985e5 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 1 Aug 2017 15:23:28 -0600 Subject: [PATCH 225/300] fix syntax --- salt/modules/yumpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 1495033153..c105bbb95b 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -181,7 +181,7 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - elif int(__grains__.get('osmajorrelease')) >= 26: + if int(__grains__.get('osmajorrelease')) >= 26: if six.PY3: vl_plugin = 'python3-dnf-plugin-versionlock' else: From 683561a711005ec875b9f4175a0aa0d42aa94899 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 2 Aug 2017 14:30:18 -0600 Subject: [PATCH 226/300] use subtraction instead of or If we use or, we will get minions that are in the ret list, but not connected to the master in cases where the syndic is used. --- salt/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 1073223ae1..c27dbf7328 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -738,7 +738,7 @@ class LocalClient(object): ret[mid] = (data if full_return else data.get('ret', {})) - for failed in list(set(pub_data['minions']) ^ set(ret)): + for failed in list(set(pub_data['minions']) - set(ret)): ret[failed] = False return ret finally: From 58b997c67f5e60dfc7c1079aa5393e88588c7f2c Mon Sep 17 00:00:00 2001 From: Kristjan Koppel Date: Fri, 4 Aug 2017 15:35:41 +0300 Subject: [PATCH 227/300] Added a helper function that removes container names from container HostConfig:Links values to enable compare_container() to make the correct decision about differences in links. Fixes #42741 --- salt/modules/dockermod.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 0f87a9b17f..1127f9a6ca 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -558,6 +558,19 @@ def _prep_pull(): ''' __context__['docker._pull_status'] = [x[:12] for x in images(all=True)] +def _scrub_links(links, name): + ''' + Remove container name from HostConfig:Links values to enable comparing + container configurations correctly. + ''' + if isinstance(links, list): + ret = [] + for l in links: + ret.append(l.replace('/{0}/'.format(name), '/', 1)) + else: + ret = links + + return ret def _size_fmt(num): ''' @@ -884,6 +897,9 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict][item] val2 = result2[conf_dict].get(item) + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) if val1 != val2: ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} # Check for optionally-present items that were in the second container @@ -895,6 +911,9 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict].get(item) val2 = result2[conf_dict][item] + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) if val1 != val2: ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} return ret From f3dcfca4e0f4937b066767321bdcb30e80aa515d Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Fri, 4 Aug 2017 09:09:15 -0600 Subject: [PATCH 228/300] Fix infinite loops on failed Windows deployments --- salt/utils/cloud.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index e8a98ee9c3..25817b26d7 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -806,21 +806,21 @@ def wait_for_winexesvc(host, port, username, password, timeout=900): log.debug('winexe connected...') return True log.debug('Return code was {0}'.format(ret_code)) - time.sleep(1) except socket.error as exc: log.debug('Caught exception in wait_for_winexesvc: {0}'.format(exc)) - time.sleep(1) - if time.time() - start > timeout: - log.error('winexe connection timed out: {0}'.format(timeout)) - return False - log.debug( - 'Retrying winexe connection to host {0} on port {1} ' - '(try {2})'.format( - host, - port, - try_count - ) + + if time.time() - start > timeout: + log.error('winexe connection timed out: {0}'.format(timeout)) + return False + log.debug( + 'Retrying winexe connection to host {0} on port {1} ' + '(try {2})'.format( + host, + port, + try_count ) + ) + time.sleep(1) def wait_for_winrm(host, port, username, password, timeout=900): @@ -846,19 +846,19 @@ def wait_for_winrm(host, port, username, password, timeout=900): log.debug('WinRM session connected...') return s log.debug('Return code was {0}'.format(r.status_code)) - time.sleep(1) except WinRMTransportError as exc: log.debug('Caught exception in wait_for_winrm: {0}'.format(exc)) - if time.time() - start > timeout: - log.error('WinRM connection timed out: {0}'.format(timeout)) - return None - log.debug( - 'Retrying WinRM connection to host {0} on port {1} ' - '(try {2})'.format( - host, port, trycount - ) + + if time.time() - start > timeout: + log.error('WinRM connection timed out: {0}'.format(timeout)) + return None + log.debug( + 'Retrying WinRM connection to host {0} on port {1} ' + '(try {2})'.format( + host, port, trycount ) - time.sleep(1) + ) + time.sleep(1) def validate_windows_cred(host, From de6d3cc0cff9dedd4373e0fec5e8f634ed2f3957 Mon Sep 17 00:00:00 2001 From: garethgreenaway Date: Fri, 4 Aug 2017 08:59:33 -0700 Subject: [PATCH 229/300] Update dockermod.py --- salt/modules/dockermod.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 1127f9a6ca..dda1d6e94a 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -558,6 +558,7 @@ def _prep_pull(): ''' __context__['docker._pull_status'] = [x[:12] for x in images(all=True)] + def _scrub_links(links, name): ''' Remove container name from HostConfig:Links values to enable comparing @@ -572,6 +573,7 @@ def _scrub_links(links, name): return ret + def _size_fmt(num): ''' Format bytes as human-readable file sizes From de60b77c82bfab349c4d60a561c7326f89555ff9 Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 4 Aug 2017 12:36:06 -0600 Subject: [PATCH 230/300] Workaround Orchestrate problem that highstate outputter mutates data --- salt/client/mixins.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 634133c352..78d0bb4bb6 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -408,8 +408,6 @@ class SyncClientMixin(object): ) data['success'] = False - namespaced_event.fire_event(data, 'ret') - if self.store_job: try: salt.utils.job.store_job( @@ -427,6 +425,9 @@ class SyncClientMixin(object): log.error('Could not store job cache info. ' 'Job details for this run may be unavailable.') + # Outputters _can_ mutate data so write to the job cache first! + namespaced_event.fire_event(data, 'ret') + # if we fired an event, make sure to delete the event object. # This will ensure that we call destroy, which will do the 0MQ linger log.info('Runner completed: {0}'.format(data['jid'])) From 4a9f6ba44f1ab7a6e4bc2382b0dcfb3055ed0c53 Mon Sep 17 00:00:00 2001 From: Seth House Date: Wed, 2 Aug 2017 14:19:20 -0600 Subject: [PATCH 231/300] Add token_expire_user_override link to auth runner docstring --- salt/runners/auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/runners/auth.py b/salt/runners/auth.py index 8aec8c3389..30c359558d 100644 --- a/salt/runners/auth.py +++ b/salt/runners/auth.py @@ -20,6 +20,10 @@ def mk_token(**load): ''' Create an eauth token using provided credentials + Non-root users may specify an expiration date (if allowed via the + :conf_master:`token_expire_user_override` setting) by passing an additional + ``token_expire`` param. + CLI Example: .. code-block:: shell From 710bdf611586992829906e704995df9087d0e04d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 4 Aug 2017 10:19:27 -0500 Subject: [PATCH 232/300] docker.compare_container: treat null oom_kill_disable as False Resolves #42705. --- salt/modules/dockermod.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index dda1d6e94a..09056cb927 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -899,11 +899,15 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict][item] val2 = result2[conf_dict].get(item) - if item == 'Links': - val1 = _scrub_links(val1, first) - val2 = _scrub_links(val2, second) - if val1 != val2: - ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + if item in ('OomKillDisable',): + if bool(val1) != bool(val2): + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + else: + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) + if val1 != val2: + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} # Check for optionally-present items that were in the second container # and not the first. for item in result2[conf_dict]: @@ -913,11 +917,15 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict].get(item) val2 = result2[conf_dict][item] - if item == 'Links': - val1 = _scrub_links(val1, first) - val2 = _scrub_links(val2, second) - if val1 != val2: - ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + if item in ('OomKillDisable',): + if bool(val1) != bool(val2): + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + else: + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) + if val1 != val2: + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} return ret From 665de2d1f9e7af0b24bc635a39d98a2b2efba4b5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 7 Aug 2017 09:23:10 -0500 Subject: [PATCH 233/300] Fix domainname parameter input translation Resolves #42538 --- salt/modules/dockermod.py | 6 +++--- salt/states/docker_container.py | 16 +++------------- salt/utils/docker/translate.py | 2 +- tests/unit/utils/test_docker.py | 2 +- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 09056cb927..4d4ca55bac 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -1836,7 +1836,7 @@ def create(image, generate one for you (it will be included in the return data). skip_translate - This function translates Salt CLI input into the format which + This function translates Salt CLI or SLS input into the format which docker-py_ expects. However, in the event that Salt's translation logic fails (due to potential changes in the Docker Remote API, or to bugs in the translation code), this argument can be used to exert granular @@ -2104,9 +2104,9 @@ def create(image, - ``dns_search="[foo1.domain.tld, foo2.domain.tld]"`` domainname - Set custom DNS search domains + The domain name to use for the container - Example: ``domainname=domain.tld,domain2.tld`` + Example: ``domainname=domain.tld`` entrypoint Entrypoint for the container. Either a string (e.g. ``"mycmd --arg1 diff --git a/salt/states/docker_container.py b/salt/states/docker_container.py index 1bfc8cf489..1f5996c3d5 100644 --- a/salt/states/docker_container.py +++ b/salt/states/docker_container.py @@ -145,7 +145,7 @@ def running(name, .. _docker-container-running-skip-translate: skip_translate - This function translates Salt CLI input into the format which + This function translates Salt CLI or SLS input into the format which docker-py_ expects. However, in the event that Salt's translation logic fails (due to potential changes in the Docker Remote API, or to bugs in the translation code), this argument can be used to exert granular @@ -677,24 +677,14 @@ def running(name, - foo2.domain.tld domainname - Set custom DNS search domains. Can be expressed as a comma-separated - list or a YAML list. The below two examples are equivalent: + The domain name to use for the container .. code-block:: yaml foo: docker_container.running: - image: bar/baz:latest - - dommainname: domain.tld,domain2.tld - - .. code-block:: yaml - - foo: - docker_container.running: - - image: bar/baz:latest - - dommainname: - - domain.tld - - domain2.tld + - dommainname: domain.tld entrypoint Entrypoint for the container diff --git a/salt/utils/docker/translate.py b/salt/utils/docker/translate.py index b6d8717316..372596a759 100644 --- a/salt/utils/docker/translate.py +++ b/salt/utils/docker/translate.py @@ -387,7 +387,7 @@ def dns(val, **kwargs): def domainname(val, **kwargs): # pylint: disable=unused-argument - return _translate_stringlist(val) + return _translate_str(val) def entrypoint(val, **kwargs): # pylint: disable=unused-argument diff --git a/tests/unit/utils/test_docker.py b/tests/unit/utils/test_docker.py index 5b0587f18c..5ee6edc236 100644 --- a/tests/unit/utils/test_docker.py +++ b/tests/unit/utils/test_docker.py @@ -820,7 +820,7 @@ class TranslateInputTestCase(TestCase): expected ) - @assert_stringlist + @assert_string def test_domainname(self): ''' Should be a list of strings or converted to one From 7ef691e8da519f0e44783b7f73863048e369ecd2 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 7 Aug 2017 11:43:11 -0600 Subject: [PATCH 234/300] make sure to use the correct out_file When installing more than one package, the out_file is not correct when doing the install. --- salt/spm/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/spm/__init__.py b/salt/spm/__init__.py index b2f367e5ac..2bd2646b61 100644 --- a/salt/spm/__init__.py +++ b/salt/spm/__init__.py @@ -359,6 +359,7 @@ class SPMClient(object): # First we download everything, then we install for package in dl_list: + out_file = dl_list[package]['dest_file'] # Kick off the install self._install_indv_pkg(package, out_file) return From 560510428514c1fab10c35a70a90741480d7f817 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 7 Aug 2017 15:05:46 -0400 Subject: [PATCH 235/300] Add a cmp compatibility function utility The ``cmp`` function has been removed in Python 3. This PR adds the functionality as a salt utility, so the function can be used across all Python versions. Fixes #42697 --- salt/runners/manage.py | 3 ++- salt/utils/compat.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index ebf6471d9a..fbf8ea3f23 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -22,6 +22,7 @@ from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: dis # Import salt libs import salt.key import salt.utils +import salt.utils.compat import salt.utils.minions import salt.client import salt.client.ssh @@ -669,7 +670,7 @@ def versions(): ver_diff = -2 else: minion_version = salt.version.SaltStackVersion.parse(minions[minion]) - ver_diff = cmp(minion_version, master_version) + ver_diff = salt.utils.compat.cmp(minion_version, master_version) if ver_diff not in version_status: version_status[ver_diff] = {} diff --git a/salt/utils/compat.py b/salt/utils/compat.py index b97820db92..09c5cc2828 100644 --- a/salt/utils/compat.py +++ b/salt/utils/compat.py @@ -46,3 +46,15 @@ def deepcopy_bound(name): finally: copy._deepcopy_dispatch = pre_dispatch return ret + + +def cmp(x, y): + ''' + Compatibility helper function to replace the ``cmp`` function from Python 2. The + ``cmp`` function is no longer available in Python 3. + + cmp(x, y) -> integer + + Return negative if xy. + ''' + return (x > y) - (x < y) From d707f9486306ef65245b7eb2970c477bc014c863 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 7 Aug 2017 15:38:39 -0400 Subject: [PATCH 236/300] Update all other calls to "cmp" function Some should just use the new utility function, while others can just be compared more directly. --- salt/states/bigip.py | 4 ++-- salt/states/boto_cfn.py | 5 ++++- salt/states/cisconso.py | 8 +++++++- salt/states/ini_manage.py | 2 +- salt/states/zabbix_usergroup.py | 2 +- salt/utils/cloud.py | 3 ++- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/salt/states/bigip.py b/salt/states/bigip.py index 8a575b0d1d..44462534dc 100644 --- a/salt/states/bigip.py +++ b/salt/states/bigip.py @@ -85,7 +85,7 @@ def _check_for_changes(entity_type, ret, existing, modified): if 'generation' in existing['content'].keys(): del existing['content']['generation'] - if cmp(modified['content'], existing['content']) == 0: + if modified['content'] == existing['content']: ret['comment'] = '{entity_type} is currently enforced to the desired state. No changes made.'.format(entity_type=entity_type) else: ret['comment'] = '{entity_type} was enforced to the desired state. Note: Only parameters specified ' \ @@ -94,7 +94,7 @@ def _check_for_changes(entity_type, ret, existing, modified): ret['changes']['new'] = modified['content'] else: - if cmp(modified, existing) == 0: + if modified == existing: ret['comment'] = '{entity_type} is currently enforced to the desired state. No changes made.'.format(entity_type=entity_type) else: ret['comment'] = '{entity_type} was enforced to the desired state. Note: Only parameters specified ' \ diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index c996b9006b..97f2cf6260 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -43,6 +43,9 @@ from __future__ import absolute_import import logging import json +# Import Salt libs +import salt.utils.compat + # Import 3rd party libs try: from salt._compat import ElementTree as ET @@ -158,7 +161,7 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi template = template['GetTemplateResponse']['GetTemplateResult']['TemplateBody'].encode('ascii', 'ignore') template = json.loads(template) _template_body = json.loads(template_body) - compare = cmp(template, _template_body) + compare = salt.utils.compat.cmp(template, _template_body) if compare != 0: log.debug('Templates are not the same. Compare value is {0}'.format(compare)) # At this point we should be able to run update safely since we already validated the template diff --git a/salt/states/cisconso.py b/salt/states/cisconso.py index eb26cfaf33..7622a3102d 100644 --- a/salt/states/cisconso.py +++ b/salt/states/cisconso.py @@ -8,6 +8,12 @@ For documentation on setting up the cisconso proxy minion look in the documentat for :mod:`salt.proxy.cisconso `. ''' +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +import salt.utils.compat + def __virtual__(): return 'cisconso.set_data_value' in __salt__ @@ -53,7 +59,7 @@ def value_present(name, datastore, path, config): existing = __salt__['cisconso.get_data'](datastore, path) - if cmp(existing, config): + if salt.utils.compat.cmp(existing, config): ret['result'] = True ret['comment'] = 'Config is already set' diff --git a/salt/states/ini_manage.py b/salt/states/ini_manage.py index e22202e0f1..7ea26e05b7 100644 --- a/salt/states/ini_manage.py +++ b/salt/states/ini_manage.py @@ -206,7 +206,7 @@ def sections_present(name, sections=None, separator='='): ret['result'] = False ret['comment'] = "{0}".format(err) return ret - if cmp(dict(sections[section]), cur_section) == 0: + if dict(sections[section]) == cur_section: ret['comment'] += 'Section unchanged {0}.\n'.format(section) continue elif cur_section: diff --git a/salt/states/zabbix_usergroup.py b/salt/states/zabbix_usergroup.py index a5d68b1546..292a2aafc9 100644 --- a/salt/states/zabbix_usergroup.py +++ b/salt/states/zabbix_usergroup.py @@ -84,7 +84,7 @@ def present(name, **kwargs): for right in kwargs['rights']: for key in right: right[key] = str(right[key]) - if cmp(sorted(kwargs['rights']), sorted(usergroup['rights'])) != 0: + if sorted(kwargs['rights']) != sorted(usergroup['rights']): update_rights = True else: update_rights = True diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index cd6d59e9b5..b67f42cedb 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -58,6 +58,7 @@ import salt.config import salt.loader import salt.template import salt.utils +import salt.utils.compat import salt.utils.event from salt.utils import vt from salt.utils.nb_popen import NonBlockingPopen @@ -3041,7 +3042,7 @@ def diff_node_cache(prov_dir, node, new_data, opts): # Perform a simple diff between the old and the new data, and if it differs, # return both dicts. # TODO: Return an actual diff - diff = cmp(new_data, cache_data) + diff = salt.utils.compat.cmp(new_data, cache_data) if diff != 0: fire_event( 'event', From 998834fbacf319c2369750f02559d7d741dd4d1d Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 7 Aug 2017 14:38:10 -0400 Subject: [PATCH 237/300] Sort lists before compairing them in python 3 unit test --- tests/unit/templates/test_jinja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 6817c0c731..7a96608825 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -497,7 +497,7 @@ class TestCustomExtensions(TestCase): env = Environment(extensions=[SerializerExtension]) if six.PY3: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset).strip("'{}").split("', '") - self.assertEqual(rendered, list(unique)) + self.assertEqual(sorted(rendered), sorted(list(unique))) else: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset) self.assertEqual(rendered, u"{0}".format(unique)) From d397c90e92dfc61a36b216cb2049e28fabdf85f7 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 7 Aug 2017 14:37:10 -0600 Subject: [PATCH 238/300] only read file if it is not a string --- salt/utils/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index 87da823099..eaec8df33a 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -779,7 +779,8 @@ def _render(template, render, renderer, template_dict, opts): blacklist = opts.get('renderer_blacklist') whitelist = opts.get('renderer_whitelist') ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict) - ret = ret.read() + if salt.utils.stringio.is_readable(ret): + ret = ret.read() if str(ret).startswith('#!') and not str(ret).startswith('#!/'): ret = str(ret).split('\n', 1)[1] return ret From 90a2fb66a25fe9e39d52f63d9ba5c6df4436513d Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 7 Aug 2017 16:45:42 -0400 Subject: [PATCH 239/300] Fix typo for template_dict in http docs --- doc/topics/tutorials/http.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/topics/tutorials/http.rst b/doc/topics/tutorials/http.rst index 1eaee62071..e6b20c62a2 100644 --- a/doc/topics/tutorials/http.rst +++ b/doc/topics/tutorials/http.rst @@ -110,7 +110,7 @@ To pass through a file that contains jinja + yaml templating (the default): method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) To pass through a file that contains mako templating: @@ -123,7 +123,7 @@ To pass through a file that contains mako templating: data_file='/srv/salt/somefile.mako', data_render=True, data_renderer='mako', - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) Because this function uses Salt's own rendering system, any Salt renderer can @@ -140,7 +140,7 @@ However, this can be changed to ``master`` if necessary. method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'}, + template_dict={'key1': 'value1', 'key2': 'value2'}, opts=__opts__ ) @@ -149,7 +149,7 @@ However, this can be changed to ``master`` if necessary. method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'}, + template_dict={'key1': 'value1', 'key2': 'value2'}, node='master' ) @@ -170,11 +170,11 @@ a Python dict. header_file='/srv/salt/headers.jinja', header_render=True, header_renderer='jinja', - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) Because much of the data that would be templated between headers and data may be -the same, the ``template_data`` is the same for both. Correcting possible +the same, the ``template_dict`` is the same for both. Correcting possible variable name collisions is up to the user. Authentication From c7ea631558d570e1afdc8611bf428f3cb92c5f5c Mon Sep 17 00:00:00 2001 From: Seth House Date: Mon, 7 Aug 2017 14:11:49 -0600 Subject: [PATCH 240/300] Add more docs on the token_expire param --- salt/runners/auth.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/salt/runners/auth.py b/salt/runners/auth.py index 30c359558d..7e8e64855a 100644 --- a/salt/runners/auth.py +++ b/salt/runners/auth.py @@ -17,20 +17,28 @@ import salt.netapi def mk_token(**load): - ''' + r''' Create an eauth token using provided credentials - Non-root users may specify an expiration date (if allowed via the - :conf_master:`token_expire_user_override` setting) by passing an additional - ``token_expire`` param. + Non-root users may specify an expiration date -- if allowed via the + :conf_master:`token_expire_user_override` setting -- by passing an + additional ``token_expire`` param. This overrides the + :conf_master:`token_expire` setting of the same name in the Master config + and is how long a token should live in seconds. CLI Example: .. code-block:: shell salt-run auth.mk_token username=saltdev password=saltdev eauth=auto - salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \\ + + # Create a token valid for three years. + salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \ token_expire=94670856 + + # Calculate the number of seconds using expr. + salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \ + token_expire=$(expr \( 365 \* 24 \* 60 \* 60 \) \* 3) ''' # This will hang if the master daemon is not running. netapi = salt.netapi.NetapiClient(__opts__) From 928b523797edf69c75fecd4c95a5466917211fde Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Fri, 4 Aug 2017 17:01:02 -0600 Subject: [PATCH 241/300] Remove waits and retries from Saltify deployment The waits and retries are there to give a new VM a chance to be created and booted. Saltify works against existing VMs, so there is no need to wait for services to become available. --- salt/utils/cloud.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index 25817b26d7..069533558d 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -311,6 +311,11 @@ def bootstrap(vm_, opts): } } + if vm_.get('driver', 'none:none').split(':')[1] == 'saltify': + saltify_driver = True + else: + saltify_driver = False + key_filename = salt.config.get_cloud_config_value( 'key_filename', vm_, opts, search_global=False, default=salt.config.get_cloud_config_value( @@ -475,6 +480,9 @@ def bootstrap(vm_, opts): 'make_minion', vm_, opts, default=True ) + if saltify_driver: + deploy_kwargs['wait_for_passwd_maxtries'] = 0 # No need to wait/retry with Saltify + win_installer = salt.config.get_cloud_config_value( 'win_installer', vm_, opts ) @@ -499,6 +507,8 @@ def bootstrap(vm_, opts): deploy_kwargs['winrm_port'] = salt.config.get_cloud_config_value( 'winrm_port', vm_, opts, default=5986 ) + if saltify_driver: + deploy_kwargs['port_timeout'] = 1 # No need to wait/retry with Saltify # Store what was used to the deploy the VM event_kwargs = copy.deepcopy(deploy_kwargs) From 0acffc6df54347c3128847da67fe2854c1b040fc Mon Sep 17 00:00:00 2001 From: lomeroe Date: Thu, 3 Aug 2017 12:07:49 -0500 Subject: [PATCH 242/300] fix #42600 in develop attempt to write data to regpol file even if data_to_write is empty (i.e. no policies configured) --- salt/modules/win_lgpo.py | 143 +++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 6bf5d437d8..347ca742f9 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -3982,78 +3982,77 @@ def _write_regpol_data(data_to_write, gpt_extension_guid: admx registry extension guid for the class ''' try: - if data_to_write: - reg_pol_header = u'\u5250\u6765\x01\x00' - if not os.path.exists(policy_file_path): - ret = __salt__['file.makedirs'](policy_file_path) - with salt.utils.fopen(policy_file_path, 'wb') as pol_file: - if not data_to_write.startswith(reg_pol_header): - pol_file.write(reg_pol_header.encode('utf-16-le')) - pol_file.write(data_to_write.encode('utf-16-le')) - try: - gpt_ini_data = '' - if os.path.exists(gpt_ini_path): - with salt.utils.fopen(gpt_ini_path, 'rb') as gpt_file: - gpt_ini_data = gpt_file.read() - if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): - gpt_ini_data = '[General]\r\n' + gpt_ini_data - if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): - # ensure the line contains the ADM guid - gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] - if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), - gpt_ext_str): - gpt_ext_str = gpt_ext_str.split('=') - gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] - gpt_ext_str = '='.join(gpt_ext_str) - gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - gpt_extension, gpt_extension_guid, - gpt_ini_data[general_location.end():]) - # https://technet.microsoft.com/en-us/library/cc978247.aspx - if _regexSearchRegPolData(r'Version=', gpt_ini_data): - version_loc = re.search(r'^Version=.*\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - version_str = gpt_ini_data[version_loc.start():version_loc.end()] - version_str = version_str.split('=') - version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (version_nums[0], version_nums[1] + 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (version_nums[0] + 1, version_nums[1]) - version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[0:version_loc.start()], - 'Version', version_num, - gpt_ini_data[version_loc.end():]) - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (0, 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (1, 0) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - 'Version', - int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), - gpt_ini_data[general_location.end():]) - if gpt_ini_data: - with salt.utils.fopen(gpt_ini_path, 'wb') as gpt_file: - gpt_file.write(gpt_ini_data) - except Exception as e: - msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( - gpt_ini_path, e) - raise CommandExecutionError(msg) + reg_pol_header = u'\u5250\u6765\x01\x00' + if not os.path.exists(policy_file_path): + ret = __salt__['file.makedirs'](policy_file_path) + with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: + if not data_to_write.startswith(reg_pol_header): + pol_file.write(reg_pol_header.encode('utf-16-le')) + pol_file.write(data_to_write.encode('utf-16-le')) + try: + gpt_ini_data = '' + if os.path.exists(gpt_ini_path): + with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + gpt_ini_data = gpt_file.read() + if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): + gpt_ini_data = '[General]\r\n' + gpt_ini_data + if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): + # ensure the line contains the ADM guid + gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] + if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), + gpt_ext_str): + gpt_ext_str = gpt_ext_str.split('=') + gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] + gpt_ext_str = '='.join(gpt_ext_str) + gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + gpt_extension, gpt_extension_guid, + gpt_ini_data[general_location.end():]) + # https://technet.microsoft.com/en-us/library/cc978247.aspx + if _regexSearchRegPolData(r'Version=', gpt_ini_data): + version_loc = re.search(r'^Version=.*\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + version_str = gpt_ini_data[version_loc.start():version_loc.end()] + version_str = version_str.split('=') + version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (version_nums[0], version_nums[1] + 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (version_nums[0] + 1, version_nums[1]) + version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[0:version_loc.start()], + 'Version', version_num, + gpt_ini_data[version_loc.end():]) + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (0, 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (1, 0) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + 'Version', + int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), + gpt_ini_data[general_location.end():]) + if gpt_ini_data: + with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: + gpt_file.write(gpt_ini_data) + except Exception as e: + msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( + gpt_ini_path, e) + raise CommandExecutionError(msg) except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format(policy_file_path, e) raise CommandExecutionError(msg) From 695f8c1ae41cbcc6675930a379d277fb81edbeac Mon Sep 17 00:00:00 2001 From: lomeroe Date: Thu, 3 Aug 2017 12:07:49 -0500 Subject: [PATCH 243/300] fix #42600 in develop attempt to write data to regpol file even if data_to_write is empty (i.e. no policies configured) --- salt/modules/win_lgpo.py | 143 +++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index d941521dda..21f855e3c2 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -3989,78 +3989,77 @@ def _write_regpol_data(data_to_write, gpt_extension_guid: admx registry extension guid for the class ''' try: - if data_to_write: - reg_pol_header = u'\u5250\u6765\x01\x00' - if not os.path.exists(policy_file_path): - ret = __salt__['file.makedirs'](policy_file_path) - with open(policy_file_path, 'wb') as pol_file: - if not data_to_write.startswith(reg_pol_header): - pol_file.write(reg_pol_header.encode('utf-16-le')) - pol_file.write(data_to_write.encode('utf-16-le')) - try: - gpt_ini_data = '' - if os.path.exists(gpt_ini_path): - with open(gpt_ini_path, 'rb') as gpt_file: - gpt_ini_data = gpt_file.read() - if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): - gpt_ini_data = '[General]\r\n' + gpt_ini_data - if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): - # ensure the line contains the ADM guid - gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] - if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), - gpt_ext_str): - gpt_ext_str = gpt_ext_str.split('=') - gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] - gpt_ext_str = '='.join(gpt_ext_str) - gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - gpt_extension, gpt_extension_guid, - gpt_ini_data[general_location.end():]) - # https://technet.microsoft.com/en-us/library/cc978247.aspx - if _regexSearchRegPolData(r'Version=', gpt_ini_data): - version_loc = re.search(r'^Version=.*\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - version_str = gpt_ini_data[version_loc.start():version_loc.end()] - version_str = version_str.split('=') - version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (version_nums[0], version_nums[1] + 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (version_nums[0] + 1, version_nums[1]) - version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[0:version_loc.start()], - 'Version', version_num, - gpt_ini_data[version_loc.end():]) - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (0, 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (1, 0) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - 'Version', - int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), - gpt_ini_data[general_location.end():]) - if gpt_ini_data: - with open(gpt_ini_path, 'wb') as gpt_file: - gpt_file.write(gpt_ini_data) - except Exception as e: - msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( - gpt_ini_path, e) - raise CommandExecutionError(msg) + reg_pol_header = u'\u5250\u6765\x01\x00' + if not os.path.exists(policy_file_path): + ret = __salt__['file.makedirs'](policy_file_path) + with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: + if not data_to_write.startswith(reg_pol_header): + pol_file.write(reg_pol_header.encode('utf-16-le')) + pol_file.write(data_to_write.encode('utf-16-le')) + try: + gpt_ini_data = '' + if os.path.exists(gpt_ini_path): + with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + gpt_ini_data = gpt_file.read() + if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): + gpt_ini_data = '[General]\r\n' + gpt_ini_data + if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): + # ensure the line contains the ADM guid + gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] + if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), + gpt_ext_str): + gpt_ext_str = gpt_ext_str.split('=') + gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] + gpt_ext_str = '='.join(gpt_ext_str) + gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + gpt_extension, gpt_extension_guid, + gpt_ini_data[general_location.end():]) + # https://technet.microsoft.com/en-us/library/cc978247.aspx + if _regexSearchRegPolData(r'Version=', gpt_ini_data): + version_loc = re.search(r'^Version=.*\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + version_str = gpt_ini_data[version_loc.start():version_loc.end()] + version_str = version_str.split('=') + version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (version_nums[0], version_nums[1] + 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (version_nums[0] + 1, version_nums[1]) + version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[0:version_loc.start()], + 'Version', version_num, + gpt_ini_data[version_loc.end():]) + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (0, 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (1, 0) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + 'Version', + int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), + gpt_ini_data[general_location.end():]) + if gpt_ini_data: + with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: + gpt_file.write(gpt_ini_data) + except Exception as e: + msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( + gpt_ini_path, e) + raise CommandExecutionError(msg) except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format(policy_file_path, e) raise CommandExecutionError(msg) From fa5822009f00760802c44d5589b1a1dbac9f3494 Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Tue, 8 Aug 2017 11:18:20 -0500 Subject: [PATCH 244/300] Fix exception when master_type=disable The following exception occasionally occurs when `master_type=disable`: ``` File "/usr/lib/python2.7/site-packages/salt/minion.py", line 1989, in handle_event self._fire_master(data['data'], data['tag'], data['events'], data['pretag']) File "/usr/lib/python2.7/site-packages/salt/minion.py", line 1261, in _fire_master 'tok': self.tok} AttributeError: 'Minion' object has no attribute 'tok' ``` This occurs because it tries to fire a master event when the minion is not connected to the master, in this case due to an action from a beacon. Signed-off-by: Sergey Kizunov --- salt/minion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 3c5046ee93..2c7f4cb212 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1985,8 +1985,9 @@ class Minion(MinionBase): elif tag.startswith('_minion_mine'): self._mine_send(tag, data) elif tag.startswith('fire_master'): - log.debug('Forwarding master event tag={tag}'.format(tag=data['tag'])) - self._fire_master(data['data'], data['tag'], data['events'], data['pretag']) + if self.connected: + log.debug('Forwarding master event tag={tag}'.format(tag=data['tag'])) + self._fire_master(data['data'], data['tag'], data['events'], data['pretag']) elif tag.startswith(master_event(type='disconnected')) or tag.startswith(master_event(type='failback')): # if the master disconnect event is for a different master, raise an exception if tag.startswith(master_event(type='disconnected')) and data['master'] != self.opts['master']: From dbd29e4aaaef6d3ad631959f93392752dff49588 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 7 Aug 2017 14:37:10 -0600 Subject: [PATCH 245/300] only read file if it is not a string --- salt/utils/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index e34280cbfb..51d43dcfdd 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -763,7 +763,8 @@ def _render(template, render, renderer, template_dict, opts): blacklist = opts.get('renderer_blacklist') whitelist = opts.get('renderer_whitelist') ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict) - ret = ret.read() + if salt.utils.stringio.is_readable(ret): + ret = ret.read() if str(ret).startswith('#!') and not str(ret).startswith('#!/'): ret = str(ret).split('\n', 1)[1] return ret From 5a91c1f2d1311d8d16d82f6c101a8c5829b8c088 Mon Sep 17 00:00:00 2001 From: remijouannet Date: Wed, 26 Jul 2017 01:27:38 +0200 Subject: [PATCH 246/300] update consul module following this documentation https://www.consul.io/api/acl.html --- salt/modules/consul.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/modules/consul.py b/salt/modules/consul.py index 4ed96efea9..2629971a58 100644 --- a/salt/modules/consul.py +++ b/salt/modules/consul.py @@ -1972,6 +1972,8 @@ def acl_create(consul_url=None, **kwargs): Create a new ACL token. :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to create + leave it blank to let consul server generate one :param name: Meaningful indicator of the ACL's purpose. :param type: Type is either client or management. A management token is comparable to a root user and has the @@ -2002,6 +2004,9 @@ def acl_create(consul_url=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') + if 'id' in kwargs: + data['ID'] = kwargs['id'] + if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2120,7 +2125,7 @@ def acl_delete(consul_url=None, **kwargs): ret['res'] = False return ret - function = 'acl/delete/{0}'.format(kwargs['id']) + function = 'acl/destroy/{0}'.format(kwargs['id']) res = _query(consul_url=consul_url, data=data, method='PUT', From 8c8640d6b8fa39cb5c0bda1b922788d0c449af15 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 8 Aug 2017 14:35:53 -0400 Subject: [PATCH 247/300] Update doc references in glusterfs.volume_present The "created" option has been deprecated in favor of volume_present and the docs need to match. Fixes #42683 --- salt/states/glusterfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/glusterfs.py b/salt/states/glusterfs.py index 5ee7120c5a..9116243916 100644 --- a/salt/states/glusterfs.py +++ b/salt/states/glusterfs.py @@ -121,13 +121,13 @@ def volume_present(name, bricks, stripe=False, replica=False, device_vg=False, .. code-block:: yaml myvolume: - glusterfs.created: + glusterfs.volume_present: - bricks: - host1:/srv/gluster/drive1 - host2:/srv/gluster/drive2 Replicated Volume: - glusterfs.created: + glusterfs.volume_present: - name: volume2 - bricks: - host1:/srv/gluster/drive2 From 152eb88d9f2ff719a99e5d9f3a0bad94b55dae32 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 8 Aug 2017 14:42:34 -0400 Subject: [PATCH 248/300] Update modules --> states in kubernetes doc module The kubernetes state docs are not rendering/building due to a typo in the doc module. Fixes #42639 --- doc/ref/states/all/salt.states.kubernetes.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ref/states/all/salt.states.kubernetes.rst b/doc/ref/states/all/salt.states.kubernetes.rst index a0f715d617..55bd416492 100644 --- a/doc/ref/states/all/salt.states.kubernetes.rst +++ b/doc/ref/states/all/salt.states.kubernetes.rst @@ -1,6 +1,6 @@ -======================= -salt.modules.kubernetes -======================= +====================== +salt.states.kubernetes +====================== -.. automodule:: salt.modules.kubernetes +.. automodule:: salt.states.kubernetes :members: From 78d826dd147d721e5a041c8fe8f5999758b0b2b6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 13:55:35 -0500 Subject: [PATCH 249/300] Fix regression in yum/dnf version specification Resolves #42774. --- salt/states/pkg.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index b586ed5dbe..97a9d254a0 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -186,15 +186,12 @@ def _fulfills_version_spec(versions, oper, desired_version, if isinstance(versions, dict) and 'version' in versions: versions = versions['version'] for ver in versions: - if oper == '==': - if fnmatch.fnmatch(ver, desired_version): - return True - - elif salt.utils.compare_versions(ver1=ver, - oper=oper, - ver2=desired_version, - cmp_func=cmp_func, - ignore_epoch=ignore_epoch): + if (oper == '==' and fnmatch.fnmatch(ver, desired_version)) \ + or salt.utils.compare_versions(ver1=ver, + oper=oper, + ver2=desired_version, + cmp_func=cmp_func, + ignore_epoch=ignore_epoch): return True return False From c69f17dd183a9301058c5f4ac6e35387b420506b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 14:15:29 -0500 Subject: [PATCH 250/300] Add integration test for #42774 --- tests/integration/states/test_pkg.py | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/integration/states/test_pkg.py b/tests/integration/states/test_pkg.py index c2f112e026..814e3578c8 100644 --- a/tests/integration/states/test_pkg.py +++ b/tests/integration/states/test_pkg.py @@ -673,6 +673,44 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) + @requires_system_grains + def test_pkg_014_installed_missing_release(self, grains=None): # pylint: disable=unused-argument + ''' + Tests that a version number missing the release portion still resolves + as correctly installed. For example, version 2.0.2 instead of 2.0.2-1.el7 + ''' + os_family = grains.get('os_family', '') + + if os_family.lower() != 'redhat': + self.skipTest('Test only runs on RedHat OS family') + + pkg_targets = _PKG_TARGETS.get(os_family, []) + + # Make sure that we have targets that match the os_family. If this + # fails then the _PKG_TARGETS dict above needs to have an entry added, + # with two packages that are not installed before these tests are run + self.assertTrue(pkg_targets) + + target = pkg_targets[0] + version = self.run_function('pkg.version', [target]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + + ret = self.run_state( + 'pkg.installed', + name=target, + version=salt.utils.str_version_to_evr(version)[1], + refresh=False, + ) + self.assertSaltTrueReturn(ret) + + # Clean up + ret = self.run_state('pkg.removed', name=target) + self.assertSaltTrueReturn(ret) + @requires_salt_modules('pkg.group_install') @requires_system_grains def test_group_installed_handle_missing_package_group(self, grains=None): # pylint: disable=unused-argument From dc20e4651b4911be3fc6be715f629826c2e80360 Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Tue, 8 Aug 2017 14:39:34 -0600 Subject: [PATCH 251/300] Ignore error values when listing Windows SNMP community strings --- salt/modules/win_snmp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/modules/win_snmp.py b/salt/modules/win_snmp.py index 15653a7510..6fd3bd515f 100644 --- a/salt/modules/win_snmp.py +++ b/salt/modules/win_snmp.py @@ -303,6 +303,11 @@ def get_community_names(): # Windows SNMP service GUI. if isinstance(current_values, list): for current_value in current_values: + + # Ignore error values + if not isinstance(current_value, dict): + continue + permissions = str() for permission_name in _PERMISSION_TYPES: if current_value['vdata'] == _PERMISSION_TYPES[permission_name]: From 00f93142e4ed39fc0b09a0cdef4339ac227e12aa Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 9 Aug 2017 12:24:06 -0500 Subject: [PATCH 252/300] Fix misspelling of "versions" --- salt/modules/boto_iam.py | 2 +- salt/modules/status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/boto_iam.py b/salt/modules/boto_iam.py index 2bc34168b0..e955504ade 100644 --- a/salt/modules/boto_iam.py +++ b/salt/modules/boto_iam.py @@ -1892,7 +1892,7 @@ def list_policy_versions(policy_name, return ret.get('list_policy_versions_response', {}).get('list_policy_versions_result', {}).get('versions') except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to list {0} policy vesions.' + msg = 'Failed to list {0} policy versions.' log.error(msg.format(policy_name)) return [] diff --git a/salt/modules/status.py b/salt/modules/status.py index 26871084f5..90c10b68b5 100644 --- a/salt/modules/status.py +++ b/salt/modules/status.py @@ -214,7 +214,7 @@ def uptime(): raise CommandExecutionError("File {ut_path} was not found.".format(ut_path=ut_path)) seconds = int(float(salt.utils.fopen(ut_path).read().split()[0])) elif salt.utils.is_sunos(): - # note: some flavors/vesions report the host uptime inside a zone + # note: some flavors/versions report the host uptime inside a zone # https://support.oracle.com/epmos/faces/BugDisplay?id=15611584 res = __salt__['cmd.run_all']('kstat -p unix:0:system_misc:boot_time') if res['retcode'] > 0: From 81fefa6e67cf598c9ca5b6b13cbdddbb2f2245d5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 11:59:32 -0600 Subject: [PATCH 253/300] Add ability to pass version in pkgs list --- salt/modules/win_pkg.py | 122 +++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 38 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index b0d4052496..9872f9891d 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -861,56 +861,93 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): r''' Install the passed package(s) on the system using winrepo - :param name: - The name of a single package, or a comma-separated list of packages to - install. (no spaces after the commas) - :type name: str, list, or None + Args: - :param bool refresh: Boolean value representing whether or not to refresh - the winrepo db + name (str): + The name of a single package, or a comma-separated list of packages + to install. (no spaces after the commas) - :param pkgs: A list of packages to install from a software repository. - All packages listed under ``pkgs`` will be installed via a single - command. + refresh (bool): + Boolean value representing whether or not to refresh the winrepo db - :type pkgs: list or None + pkgs (list): + A list of packages to install from a software repository. All + packages listed under ``pkgs`` will be installed via a single + command. - *Keyword Arguments (kwargs)* + You can specify a version by passing the item as a dict: - :param str version: - The specific version to install. If omitted, the latest version will be - installed. If passed with multiple install, the version will apply to - all packages. Recommended for single installation only. + CLI Example: - :param str cache_file: - A single file to copy down for use with the installer. Copied to the - same location as the installer. Use this over ``cache_dir`` if there - are many files in the directory and you only need a specific file and - don't want to cache additional files that may reside in the installer - directory. Only applies to files on ``salt://`` + .. code-block:: bash - :param bool cache_dir: - True will copy the contents of the installer directory. This is useful - for installations that are not a single file. Only applies to - directories on ``salt://`` + # will install the latest version of foo and bar + salt '*' pkg.install pkgs='["foo", "bar"]' - :param str saltenv: Salt environment. Default 'base' + # will install the latest version of foo and version 1.2.3 of bar + salt '*' pkg.install pkgs='["foo", {"bar": "1.2.3"}]' - :param bool report_reboot_exit_codes: - If the installer exits with a recognized exit code indicating that - a reboot is required, the module function + Kwargs: + + version (str): + The specific version to install. If omitted, the latest version will + be installed. Recommend for use when installing a single package. + + If passed with a list of packages in the ``pkgs`` parameter, the + version will be ignored. + + CLI Example: + + .. code-block:: bash + + # Version is ignored + salt '*' pkg.install pkgs="['foo', 'bar']" version=1.2.3 + + If passed with a comma seperated list in the ``name`` parameter, the + version will apply to all packages in the list. + + CLI Example: + + .. code-block:: bash + + # Version 1.2.3 will apply to packages foo and bar + salt '*' pkg.install foo,bar version=1.2.3 + + cache_file (str): + A single file to copy down for use with the installer. Copied to the + same location as the installer. Use this over ``cache_dir`` if there + are many files in the directory and you only need a specific file + and don't want to cache additional files that may reside in the + installer directory. Only applies to files on ``salt://`` + + cache_dir (bool): + True will copy the contents of the installer directory. This is + useful for installations that are not a single file. Only applies to + directories on ``salt://`` + + extra_install_flags (str): + Additional install flags that will be appended to the + ``install_flags`` defined in the software definition file. Only + applies when single package is passed. + + saltenv (str): + Salt environment. Default 'base' + + report_reboot_exit_codes (bool): + If the installer exits with a recognized exit code indicating that + a reboot is required, the module function *win_system.set_reboot_required_witnessed* - will be called, preserving the knowledge of this event - for the remainder of the current boot session. For the time being, - 3010 is the only recognized exit code. The value of this param - defaults to True. + will be called, preserving the knowledge of this event + for the remainder of the current boot session. For the time being, + 3010 is the only recognized exit code. The value of this param + defaults to True. - .. versionadded:: 2016.11.0 + .. versionadded:: 2016.11.0 - :return: Return a dict containing the new package names and versions:: - :rtype: dict + Returns: + dict: Return a dict containing the new package names and versions: If the package is installed by ``pkg.install``: @@ -989,13 +1026,22 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # "sources" argument pkg_params = __salt__['pkg_resource.parse_targets'](name, pkgs, **kwargs)[0] + if len(pkg_params) > 1: + if kwargs.get('extra_install_flags') is not None: + log.warning('\'extra_install_flags\' argument will be ignored for ' + 'multiple package targets') + + # Windows expects an Options dictionary containing 'version' + for pkg in pkg_params: + pkg_params[pkg] = {'version': pkg_params[pkg]} + if pkg_params is None or len(pkg_params) == 0: log.error('No package definition found') return {} if not pkgs and len(pkg_params) == 1: - # Only use the 'version' param if 'name' was not specified as a - # comma-separated list + # Only use the 'version' param if a single item was passed to the 'name' + # parameter pkg_params = { name: { 'version': kwargs.get('version'), From 83b9b230cd83a044891c95df8fbc1337eb60b8f2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 12:05:42 -0600 Subject: [PATCH 254/300] Add winrepo to docs about supporting versions in pkgs --- salt/states/pkg.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 260aca1387..fe2a67a057 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -1073,8 +1073,11 @@ def installed( ``NOTE:`` For :mod:`apt `, :mod:`ebuild `, - :mod:`pacman `, :mod:`yumpkg `, - and :mod:`zypper `, version numbers can be specified + :mod:`pacman `, + :mod:`winrepo `, + :mod:`yumpkg `, and + :mod:`zypper `, + version numbers can be specified in the ``pkgs`` argument. For example: .. code-block:: yaml From 7de687aa5747a0cac28b58f057fe2e467cd15bc8 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 17:49:20 -0600 Subject: [PATCH 255/300] Document requirements for win_pki --- salt/modules/win_pki.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_pki.py b/salt/modules/win_pki.py index 5b046b364a..25228cc567 100644 --- a/salt/modules/win_pki.py +++ b/salt/modules/win_pki.py @@ -1,9 +1,18 @@ # -*- coding: utf-8 -*- ''' -Microsoft certificate management via the Pki PowerShell module. +Microsoft certificate management via the PKIClient PowerShell module. +https://technet.microsoft.com/en-us/itpro/powershell/windows/pkiclient/pkiclient + +The PKI Client PowerShell module is only available on Windows 8+ and Windows +Server 2012+. +https://technet.microsoft.com/en-us/library/hh848636(v=wps.620).aspx :platform: Windows +:depends: + - PowerShell 4 + - PKIClient Module (Windows 8+ / Windows Server 2012+) + .. versionadded:: 2016.11.0 ''' # Import python libs @@ -29,11 +38,17 @@ __virtualname__ = 'win_pki' def __virtual__(): ''' - Only works on Windows systems with the PKI PowerShell module installed. + Requires Windows + Requires Windows 8+ / Windows Server 2012+ + Requires PowerShell + Requires PKIClient PowerShell module installed. ''' if not salt.utils.is_windows(): return False, 'Only available on Windows Systems' + if salt.utils.version_cmp(__grains__['osversion'], '6.2.9200') == -1: + return False, 'Only available on Windows 8+ / Windows Server 2012 +' + if not __salt__['cmd.shell_info']('powershell')['installed']: return False, 'Powershell not available' From f0a1d06b46e87fa44d8784c07348df42cc8bee00 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 9 Aug 2017 17:53:22 -0600 Subject: [PATCH 256/300] Standardize PKI Client --- salt/modules/win_pki.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_pki.py b/salt/modules/win_pki.py index 25228cc567..ca17b6d626 100644 --- a/salt/modules/win_pki.py +++ b/salt/modules/win_pki.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Microsoft certificate management via the PKIClient PowerShell module. +Microsoft certificate management via the PKI Client PowerShell module. https://technet.microsoft.com/en-us/itpro/powershell/windows/pkiclient/pkiclient The PKI Client PowerShell module is only available on Windows 8+ and Windows @@ -11,7 +11,7 @@ https://technet.microsoft.com/en-us/library/hh848636(v=wps.620).aspx :depends: - PowerShell 4 - - PKIClient Module (Windows 8+ / Windows Server 2012+) + - PKI Client Module (Windows 8+ / Windows Server 2012+) .. versionadded:: 2016.11.0 ''' @@ -41,7 +41,7 @@ def __virtual__(): Requires Windows Requires Windows 8+ / Windows Server 2012+ Requires PowerShell - Requires PKIClient PowerShell module installed. + Requires PKI Client PowerShell module installed. ''' if not salt.utils.is_windows(): return False, 'Only available on Windows Systems' From 497241fbcb1226b00f021f7c9082db161e25a701 Mon Sep 17 00:00:00 2001 From: Mapel88 Date: Thu, 10 Aug 2017 10:39:43 +0300 Subject: [PATCH 257/300] Fix bug #42818 in win_iis module Exception in function "create_cert_binding". function fails with the following exception: 2017-08-09 00:23:32,096 [salt.state ][ERROR ][2948] An exception occurred in this state: Traceback (most recent call last): File "c:\salt\bin\lib\site-packages\salt\state.py", line 1837, in call **cdata['kwargs']) File "c:\salt\bin\lib\site-packages\salt\loader.py", line 1794, in wrapper return f(*args, **kwargs) File "c:\salt\var\cache\salt\minion\extmods\states\win_iisV2.py", line 326, in create_cert_binding ipaddress, port, sslflags) File "c:\salt\var\cache\salt\minion\extmods\modules\win_iisV2.py", line 861, in create_cert_binding if binding_info not in new_cert_bindings(site): TypeError: 'dict' object is not callable **This is the problematic code: new_cert_bindings = list_cert_bindings(site) if binding_info not in new_cert_bindings(site): Just need to remove (site) from second line as follows and it's fixed: new_cert_bindings = list_cert_bindings(site) if binding_info not in new_cert_bindings:** --- salt/modules/win_iis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index d0fe647b03..9309873ac4 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -856,7 +856,7 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, new_cert_bindings = list_cert_bindings(site) - if binding_info not in new_cert_bindings(site): + if binding_info not in new_cert_bindings: log.error('Binding not present: {0}'.format(binding_info)) return False From d8f7d7a7c0a4f2a95860bddfb8c048da00d16de5 Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Thu, 10 Aug 2017 16:35:45 +0200 Subject: [PATCH 258/300] API changes for Kubernetes version 2.0.0 Switching to the new Kubernetes client lib API introduced with version 2.0.0. --- salt/modules/kubernetes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 850534bf56..10e10acef1 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -2,7 +2,7 @@ ''' Module for handling kubernetes calls. -:optdepends: - kubernetes Python client +:optdepends: - kubernetes Python client >= version 2.0.0 :configuration: The k8s API settings are provided either in a pillar, in the minion's config file, or in master's config file:: @@ -762,7 +762,7 @@ def create_deployment( ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.V1beta1Deployment, + obj_class=kubernetes.client.AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1013,7 +1013,7 @@ def replace_deployment(name, ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.V1beta1Deployment, + obj_class=kubernetes.client.AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1276,9 +1276,9 @@ def __dict_to_object_meta(name, namespace, metadata): def __dict_to_deployment_spec(spec): ''' - Converts a dictionary into kubernetes V1beta1DeploymentSpec instance. + Converts a dictionary into kubernetes AppsV1beta1DeploymentSpec instance. ''' - spec_obj = kubernetes.client.V1beta1DeploymentSpec() + spec_obj = kubernetes.client.AppsV1beta1DeploymentSpec() for key, value in iteritems(spec): if hasattr(spec_obj, key): setattr(spec_obj, key, value) From ff66b7aaf09c2684f11c2f3fcb400aa001bfca72 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 10 Aug 2017 17:55:26 +0300 Subject: [PATCH 259/300] Execute fire_master asynchronously in the main minion thread. In another case it will block minion execution if master is not responding. This is actual for MultiMaster configuration because blocks minion to respond to the active master requests if another one is down. --- salt/minion.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 3c5046ee93..b1d08ab6ff 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1251,7 +1251,7 @@ class Minion(MinionBase): ret = yield channel.send(load, timeout=timeout) raise tornado.gen.Return(ret) - def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True): + def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True, timeout_handler=None): ''' Fire an event on the master, or drop message if unable to send. ''' @@ -1270,9 +1270,10 @@ class Minion(MinionBase): else: return - def timeout_handler(*_): - log.info('fire_master failed: master could not be contacted. Request timed out.') - return True + if timeout_handler is None: + def timeout_handler(*_): + log.info('fire_master failed: master could not be contacted. Request timed out.') + return True if sync: try: @@ -2205,13 +2206,15 @@ class Minion(MinionBase): if ping_interval > 0 and self.connected: def ping_master(): try: - if not self._fire_master('ping', 'minion_ping'): + def ping_timeout_handler(*_): if not self.opts.get('auth_safemode', True): log.error('** Master Ping failed. Attempting to restart minion**') delay = self.opts.get('random_reauth_delay', 5) log.info('delaying random_reauth_delay {0}s'.format(delay)) # regular sys.exit raises an exception -- which isn't sufficient in a thread os._exit(salt.defaults.exitcodes.SALT_KEEPALIVE) + + self._fire_master('ping', 'minion_ping', sync=False, timeout_handler=ping_timeout_handler) except Exception: log.warning('Attempt to ping master failed.', exc_on_loglevel=logging.DEBUG) self.periodic_callbacks['ping'] = tornado.ioloop.PeriodicCallback(ping_master, ping_interval * 1000, io_loop=self.io_loop) @@ -2226,7 +2229,7 @@ class Minion(MinionBase): except Exception: log.critical('The beacon errored: ', exc_info=True) if beacons and self.connected: - self._fire_master(events=beacons) + self._fire_master(events=beacons, sync=False) self.periodic_callbacks['beacons'] = tornado.ioloop.PeriodicCallback(handle_beacons, loop_interval * 1000, io_loop=self.io_loop) From 7f5412c19e5727bee98a16728f1a82178e2661cb Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 10 Aug 2017 18:54:46 +0300 Subject: [PATCH 260/300] Make lint happier. --- salt/minion.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index b1d08ab6ff..208ea128d3 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1270,11 +1270,6 @@ class Minion(MinionBase): else: return - if timeout_handler is None: - def timeout_handler(*_): - log.info('fire_master failed: master could not be contacted. Request timed out.') - return True - if sync: try: self._send_req_sync(load, timeout) @@ -1285,6 +1280,12 @@ class Minion(MinionBase): log.info('fire_master failed: {0}'.format(traceback.format_exc())) return False else: + if timeout_handler is None: + def handle_timeout(*_): + log.info('fire_master failed: master could not be contacted. Request timed out.') + return True + timeout_handler = handle_timeout + with tornado.stack_context.ExceptionStackContext(timeout_handler): self._send_req_async(load, timeout, callback=lambda f: None) # pylint: disable=unexpected-keyword-arg return True From 21934f61bbba1243d0b4120939d948f0338deb78 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:25:27 -0600 Subject: [PATCH 261/300] python2- prefix for fedora 26 packages --- salt/modules/yumpkg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 9ccd66d105..890a78fb04 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,7 +180,10 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - vl_plugin = 'python-dnf-plugins-extras-versionlock' + if int(__grains__.get('osmajorrelease')) < 26: + vl_plugin = 'python-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python2-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From f83b553d6e84ec87676354e280d62cb4d2b90859 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 13:56:30 -0600 Subject: [PATCH 262/300] add py3 for versionlock --- salt/modules/yumpkg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 890a78fb04..f4b80b10c2 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,7 +180,9 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if int(__grains__.get('osmajorrelease')) < 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) < 26: vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'python2-dnf-plugins-extras-versionlock' From 6ecdbcec1d14ecdbf494359243cc8948df45cd99 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 31 Jul 2017 14:15:52 -0600 Subject: [PATCH 263/300] make sure names are correct --- salt/modules/yumpkg.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index f4b80b10c2..371cea8863 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,12 +180,16 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - if six.PY3: - vl_plugin = 'python3-dnf-plugins-extras-versionlock' - elif int(__grains__.get('osmajorrelease')) < 26: - vl_plugin = 'python-dnf-plugins-extras-versionlock' + elif int(__grains__.get('osmajorrelease')) >= 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugin-versionlock' + else: + vl_plugin = 'python2-dnf-plugin-versionlock' else: - vl_plugin = 'python2-dnf-plugins-extras-versionlock' + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ From a3da86eea8b4b3ef00edbbd8f5a8e1ab7cd88e28 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 1 Aug 2017 15:23:28 -0600 Subject: [PATCH 264/300] fix syntax --- salt/modules/yumpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 371cea8863..b0a183d70a 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -180,7 +180,7 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - elif int(__grains__.get('osmajorrelease')) >= 26: + if int(__grains__.get('osmajorrelease')) >= 26: if six.PY3: vl_plugin = 'python3-dnf-plugin-versionlock' else: From b458b89fb84313e787a8ce7578ab67115b569c81 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 10 Aug 2017 13:49:45 -0600 Subject: [PATCH 265/300] skip cache_clean test if npm version is >= 5.0.0 --- tests/integration/states/npm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/states/npm.py b/tests/integration/states/npm.py index 7035ee3047..44f43a9a07 100644 --- a/tests/integration/states/npm.py +++ b/tests/integration/states/npm.py @@ -6,6 +6,7 @@ ''' # Import Python libs from __future__ import absolute_import +from distutils.version import LooseVersion # Import Salt Testing libs from salttesting import skipIf @@ -15,6 +16,9 @@ ensure_in_syspath('../../') # Import salt libs import integration import salt.utils +import salt.modules.cmdmod as cmd + +MAX_NPM_VERSION = '5.0.0' @skipIf(salt.utils.which('npm') is None, 'npm not installed') @@ -50,6 +54,7 @@ class NpmStateTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ret = self.run_state('npm.installed', name=None, pkgs=['pm2', 'grunt']) self.assertSaltTrueReturn(ret) + @skipIf(LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION), 'Skip with npm >= 5.0.0 until #41770 is fixed') @destructiveTest def test_npm_cache_clean(self): ''' From 744bf954ff1d3cb95a9a53f0dbd9afa50da9cb01 Mon Sep 17 00:00:00 2001 From: Robert James Hernandez Date: Thu, 10 Aug 2017 13:19:08 -0700 Subject: [PATCH 266/300] Adding missing output flags to salt cli --- doc/ref/cli/_includes/output-options.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/ref/cli/_includes/output-options.rst b/doc/ref/cli/_includes/output-options.rst index 0128a1cfb0..c679269e76 100644 --- a/doc/ref/cli/_includes/output-options.rst +++ b/doc/ref/cli/_includes/output-options.rst @@ -33,6 +33,10 @@ Output Options Write the output to the specified file. +.. option:: --out-file-append, --output-file-append + + Append the output to the specified file. + .. option:: --no-color Disable all colored output @@ -46,3 +50,14 @@ Output Options ``green`` denotes success, ``red`` denotes failure, ``blue`` denotes changes and success and ``yellow`` denotes a expected future change in configuration. + +.. option:: --state-output=STATE_OUTPUT, --state_output=STATE_OUTPUT + + Override the configured state_output value for minion + output. One of 'full', 'terse', 'mixed', 'changes' or + 'filter'. Default: 'none'. + +.. option:: --state-verbose=STATE_VERBOSE, --state_verbose=STATE_VERBOSE + + Override the configured state_verbose value for minion + output. Set to True or False. Default: none. From 3055e17ed5396c38ac2372de53f03e42e58d5558 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 16:40:11 -0400 Subject: [PATCH 267/300] Replace @mock_ec2 calls with @mock_ec2_deprecated calls moto versions >= 1.0.0 have changed the way the mocked connections through boto are handled with the @mock_ec2 decorator. They use the boto3 connection method. However, since we are still using boto in many places, we need to use the new @mock_ec2_deprecated decorator instead to handle the boto connection functions for the unit tests. Versions of moto < 1.0.0 are not Python 3 compatible, so salt-jenkins should be installing newer versions of moto for those tests. Unfortunately, we cannot install an older version of moto for Python2 that use the original @mock_ec2 call and also import the @mock_ec2_deprecated function for newer versions of moto simultaneously as the @mock_ec2_deprecated function doesn't exist in older versions of moto. --- tests/unit/modules/test_boto_elb.py | 26 +-- tests/unit/modules/test_boto_secgroup.py | 38 ++-- tests/unit/modules/test_boto_vpc.py | 252 +++++++++++------------ tests/unit/states/test_boto_vpc.py | 38 ++-- 4 files changed, 177 insertions(+), 177 deletions(-) diff --git a/tests/unit/modules/test_boto_elb.py b/tests/unit/modules/test_boto_elb.py index 4582155c6f..15a995ca4a 100644 --- a/tests/unit/modules/test_boto_elb.py +++ b/tests/unit/modules/test_boto_elb.py @@ -16,17 +16,17 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2, mock_elb + from moto import mock_ec2_deprecated, mock_elb HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -34,10 +34,10 @@ except ImportError: def mock_elb(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -114,7 +114,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): # __virtual__ must be caller in order for _get_conn to be injected boto_elb.__virtual__() - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_register_instances_valid_id_result_true(self): ''' @@ -133,7 +133,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(True, register_result) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_register_instances_valid_id_string(self): ''' @@ -156,7 +156,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): log.debug(load_balancer_refreshed.instances) self.assertEqual([reservations.instances[0].id], registered_instance_ids) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_result_true(self): ''' @@ -177,7 +177,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(True, deregister_result) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_string(self): ''' @@ -203,7 +203,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): load_balancer_refreshed.instances] self.assertEqual(actual_instances, expected_instances) - @mock_ec2 + @mock_ec2_deprecated @mock_elb def test_deregister_instances_valid_id_list(self): ''' diff --git a/tests/unit/modules/test_boto_secgroup.py b/tests/unit/modules/test_boto_secgroup.py index 7a1d9c0197..51ced62319 100644 --- a/tests/unit/modules/test_boto_secgroup.py +++ b/tests/unit/modules/test_boto_secgroup.py @@ -29,17 +29,17 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_secgroup unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_secgroup unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -117,7 +117,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): {'to_port': 80, 'from_port': 80, 'ip_protocol': u'tcp', 'cidr_ip': u'0.0.0.0/0'}] self.assertEqual(boto_secgroup._split_rules(rules), split_rules) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_classic(self): ''' Test of creation of an EC2-Classic security group. The test ensures @@ -137,7 +137,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_created_group[0].vpc_id] self.assertEqual(expected_create_result, secgroup_create_result) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_vpc(self): ''' test of creation of an EC2-VPC security group. The test ensures that a @@ -155,7 +155,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_create_result = [secgroup_created_group[0].name, secgroup_created_group[0].description, secgroup_created_group[0].vpc_id] self.assertEqual(expected_create_result, secgroup_create_result) - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_classic(self): ''' tests that given a name of a group in EC2-Classic that the correct @@ -177,7 +177,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): @skipIf(True, 'test skipped because moto does not yet support group' ' filters https://github.com/spulec/moto/issues/154') - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_vpc(self): ''' tests that given a name of a group in EC2-VPC that the correct @@ -197,7 +197,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(group_vpc.id, retrieved_group_id) - @mock_ec2 + @mock_ec2_deprecated def test_get_config_single_rule_group_name(self): ''' tests return of 'config' when given group name. get_config returns an OrderedDict. @@ -221,7 +221,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_get_config_result = boto_secgroup.get_config(group_id=group.id, **conn_parameters) self.assertEqual(expected_get_config_result, secgroup_get_config_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_classic(self): ''' tests 'true' existence of a group in EC2-Classic when given name @@ -234,11 +234,11 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(name=group_name, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_classic(self): pass - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_vpc(self): ''' tests 'true' existence of a group in EC2-VPC when given name and vpc_id @@ -250,7 +250,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(name=group_name, vpc_id=vpc_id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_vpc(self): ''' tests 'false' existence of a group in vpc when given name and vpc_id @@ -259,7 +259,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_name, vpc_id=vpc_id, **conn_parameters) self.assertFalse(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_group_id(self): ''' tests 'true' existence of a group when given group_id @@ -271,7 +271,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_id=group.id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_group_id(self): ''' tests 'false' existence of a group when given group_id @@ -280,7 +280,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_id=group_id, **conn_parameters) self.assertFalse(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_ec2_classic(self): ''' test deletion of a group in EC2-Classic. Test does the following: @@ -306,11 +306,11 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): actual_groups = [group.id for group in conn.get_all_security_groups()] self.assertEqual(expected_groups, actual_groups) - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_name_ec2_vpc(self): pass - @mock_ec2 + @mock_ec2_deprecated def test__get_conn_true(self): ''' tests ensures that _get_conn returns an boto.ec2.connection.EC2Connection object. diff --git a/tests/unit/modules/test_boto_vpc.py b/tests/unit/modules/test_boto_vpc.py index 2f53edf5e5..e5b337c2d1 100644 --- a/tests/unit/modules/test_boto_vpc.py +++ b/tests/unit/modules/test_boto_vpc.py @@ -40,17 +40,17 @@ except ImportError: try: import moto - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -63,7 +63,7 @@ except ImportError: # which was added in boto 2.8.0 # https://github.com/boto/boto/commit/33ac26b416fbb48a60602542b4ce15dcc7029f12 required_boto_version = '2.8.0' -required_moto_version = '0.3.7' +required_moto_version = '1.0.0' region = 'us-east-1' access_key = 'GKTADJGHEIQSXMKKRBJ08H' @@ -272,7 +272,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.modules.boto_vpc module ''' - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via id when the vpc already exists @@ -283,7 +283,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -295,7 +295,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via name when vpc exists @@ -306,7 +306,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -318,7 +318,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via tag when vpc exists @@ -329,7 +329,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -341,7 +341,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via cidr when vpc exists @@ -352,7 +352,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -364,7 +364,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_vpc_exists_but_providing_no_filters_the_vpc_exists_method_raises_a_salt_invocation_error(self): ''' @@ -375,7 +375,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): 'cidr or tags.'): boto_vpc.exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_name(self): ''' Tests getting vpc id when filtering by name @@ -386,7 +386,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_name(self): ''' Tests getting vpc id when filtering by invalid name @@ -397,7 +397,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_cidr(self): ''' Tests getting vpc id when filtering by cidr @@ -408,7 +408,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_cidr(self): ''' Tests getting vpc id when filtering by invalid cidr @@ -419,7 +419,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_tags(self): ''' Tests getting vpc id when filtering by tags @@ -430,7 +430,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_tags(self): ''' Tests getting vpc id when filtering by invalid tags @@ -441,7 +441,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_get_vpc_id_method_when_not_providing_filters_raises_a_salt_invocation_error(self): ''' @@ -450,7 +450,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegex(SaltInvocationError, 'At least one of the following must be provided: vpc_id, vpc_name, cidr or tags.'): boto_vpc.get_id(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_more_than_one_vpc_is_matched_raises_a_salt_command_execution_error(self): ''' Tests getting vpc id but providing no filters @@ -461,7 +461,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegex(CommandExecutionError, 'Found more than one VPC matching the criteria.'): boto_vpc.get_id(cidr=u'10.0.0.0/24', **conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -470,7 +470,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_a_vpc_name_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -479,7 +479,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_tags_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -488,7 +488,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_vpc_fails_the_create_vpc_method_returns_false(self): ''' @@ -499,7 +499,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_creation_result['created']) self.assertTrue('error' in vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_vpc_the_delete_vpc_method_returns_true(self): ''' Tests deleting an existing vpc @@ -510,7 +510,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_vpc_the_delete_vpc_method_returns_false(self): ''' Tests deleting a non-existent vpc @@ -519,7 +519,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(delete_vpc_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_true(self): ''' Tests describing parameters via vpc id if vpc exist @@ -546,7 +546,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_vpc, {'vpc': vpc_properties}) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_false(self): ''' Tests describing parameters via vpc id if vpc does not exist @@ -557,7 +557,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(describe_vpc['vpc']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_describing_vpc_by_id_on_connection_error_it_returns_error(self): ''' @@ -570,7 +570,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): describe_result = boto_vpc.describe(vpc_id=vpc.id, **conn_parameters) self.assertTrue('error' in describe_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_but_providing_no_vpc_id_the_describe_method_raises_a_salt_invocation_error(self): ''' Tests describing vpc without vpc id @@ -588,7 +588,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version, _get_boto_version())) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_single_subnet(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -601,7 +601,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_same_vpc(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -614,7 +614,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_different_vpc(self): ''' tests that given multiple subnet ids in different VPCs that False is @@ -628,7 +628,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(set(subnet_association['vpc_ids']), set([vpc_a.id, vpc_b.id])) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully @@ -640,7 +640,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) self.assertTrue('id' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_a_name_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a name @@ -651,7 +651,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_tags_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a tag @@ -663,7 +663,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_subnet_fails_the_create_subnet_method_returns_error(self): ''' @@ -675,7 +675,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_creation_result = boto_vpc.create_subnet(vpc.id, '10.0.0.0/24', **conn_parameters) self.assertTrue('error' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_subnet_the_delete_subnet_method_returns_true(self): ''' Tests deleting an existing subnet @@ -687,7 +687,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_deletion_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_subnet_the_delete_vpc_method_returns_false(self): ''' Tests deleting a subnet that doesn't exist @@ -695,7 +695,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): delete_subnet_result = boto_vpc.delete_subnet(subnet_id='1234', **conn_parameters) self.assertTrue('error' in delete_subnet_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' Tests checking if a subnet exists when it does exist @@ -707,7 +707,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' Tests checking if a subnet exists which doesn't exist @@ -716,7 +716,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by name @@ -728,7 +728,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by name when it doesn't exist @@ -740,7 +740,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by tag @@ -752,7 +752,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by tag when subnet doesn't exist @@ -764,7 +764,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_subnet_exists_but_providing_no_filters_the_subnet_exists_method_raises_a_salt_invocation_error(self): ''' @@ -776,7 +776,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): boto_vpc.subnet_exists(**conn_parameters) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by id. @@ -791,7 +791,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -805,7 +805,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by name. @@ -820,7 +820,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -834,7 +834,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -852,7 +852,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): set(['id', 'cidr_block', 'availability_zone', 'tags'])) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_name_for_existing_subnets_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -869,7 +869,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnets'][0].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_create_subnet_passes_availability_zone(self): ''' Tests that the availability_zone kwarg is passed on to _create_resource @@ -890,7 +890,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway successfully (with no vpc id or name) @@ -901,7 +901,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_non_existent_vpc_the_create_internet_gateway_method_returns_an_error(self): ''' Tests that creating an internet gateway for a non-existent VPC fails. @@ -913,7 +913,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): vpc_name='non-existent-vpc') self.assertTrue('error' in igw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_name_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -928,7 +928,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_id_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -951,7 +951,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway successfully (with subnet_id specified) @@ -965,7 +965,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(ngw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_non_existent_subnet_the_create_nat_gateway_method_returns_an_error(self): ''' Tests that creating an nat gateway for a non-existent subnet fails. @@ -977,7 +977,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_name='non-existent-subnet') self.assertTrue('error' in ngw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_subnet_name_specified_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway with subnet name specified. @@ -1000,7 +1000,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_customer_gateway_the_create_customer_gateway_method_returns_true(self): ''' @@ -1010,7 +1010,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_creation_result = boto_vpc.create_customer_gateway('ipsec.1', '10.1.1.1', None) self.assertTrue(gw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' @@ -1021,7 +1021,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_exists_result = boto_vpc.customer_gateway_exists(customer_gateway_id=gw_creation_result['id']) self.assertTrue(gw_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' @@ -1039,7 +1039,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version, _get_boto_version())) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_succeeds_the_create_dhcp_options_method_returns_true(self): ''' Tests creating dhcp options successfully @@ -1048,7 +1048,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_dhcp_options_and_specifying_a_name_succeeds_the_create_dhcp_options_method_returns_true( self): @@ -1060,7 +1060,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_and_specifying_tags_succeeds_the_create_dhcp_options_method_returns_true( self): ''' @@ -1071,7 +1071,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_dhcp_options_fails_the_create_dhcp_options_method_returns_error(self): ''' @@ -1082,7 +1082,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = dhcp_options_creation_result = boto_vpc.create_dhcp_options(**dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_true( self): ''' @@ -1096,7 +1096,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_association_result['associated']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_error( self): ''' @@ -1108,7 +1108,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_a_non_existent_vpc_the_associate_dhcp_options_method_returns_false( self): ''' @@ -1121,7 +1121,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_an_existing_vpc_succeeds_the_associate_new_dhcp_options_method_returns_true( self): ''' @@ -1133,7 +1133,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_creating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception( self): @@ -1147,7 +1147,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_associating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception(self): ''' @@ -1160,7 +1160,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_a_non_existent_vpc_the_dhcp_options_the_associate_new_dhcp_options_method_returns_false( self): ''' @@ -1170,7 +1170,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.create_dhcp_options(vpc_name='fake', **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_exists_the_dhcp_options_exists_method_returns_true(self): ''' Tests existence of dhcp options successfully @@ -1181,7 +1181,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_do_not_exist_the_dhcp_options_exists_method_returns_false(self): ''' Tests existence of dhcp options failure @@ -1189,7 +1189,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.dhcp_options_exists('fake', **conn_parameters) self.assertFalse(r['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_dhcp_options_exists_but_providing_no_filters_the_dhcp_options_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1206,7 +1206,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_the_create_network_acl_method_returns_true(self): ''' Tests creation of network acl with existing vpc @@ -1217,7 +1217,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_a_name_the_create_network_acl_method_returns_true( self): ''' @@ -1229,7 +1229,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_tags_the_create_network_acl_method_returns_true( self): ''' @@ -1241,7 +1241,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_a_non_existent_vpc_the_create_network_acl_method_returns_an_error(self): ''' Tests creation of network acl with a non-existent vpc @@ -1250,7 +1250,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_network_acl_fails_the_create_network_acl_method_returns_false(self): ''' @@ -1264,7 +1264,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_network_acl_the_delete_network_acl_method_returns_true(self): ''' Tests deletion of existing network acl successfully @@ -1276,7 +1276,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_network_acl_the_delete_network_acl_method_returns_an_error(self): ''' Tests deleting a non-existent network acl @@ -1285,7 +1285,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_network_acl_exists_the_network_acl_exists_method_returns_true(self): ''' Tests existence of network acl @@ -1297,7 +1297,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_network_acl_does_not_exist_the_network_acl_exists_method_returns_false(self): ''' Tests checking network acl does not exist @@ -1306,7 +1306,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_deletion_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_network_acl_exists_but_providing_no_filters_the_network_acl_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1318,7 +1318,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_successfully_the_create_network_acl_entry_method_returns_true(self): ''' @@ -1333,7 +1333,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_for_a_non_existent_network_acl_the_create_network_acl_entry_method_returns_false( self): @@ -1345,7 +1345,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_successfully_the_replace_network_acl_entry_method_returns_true( self): @@ -1362,7 +1362,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_for_a_non_existent_network_acl_the_replace_network_acl_entry_method_returns_false( self): @@ -1373,7 +1373,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_an_existing_network_acl_entry_the_delete_network_acl_entry_method_returns_true(self): ''' @@ -1388,7 +1388,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_network_acl_entry_the_delete_network_acl_entry_method_returns_false( self): @@ -1400,7 +1400,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_true( self): @@ -1416,7 +1416,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_an_error( self): ''' @@ -1430,7 +1430,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_a_non_existent_subnet_the_associate_network_acl_method_returns_false( self): @@ -1445,7 +1445,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1460,7 +1460,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_a_name_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1476,7 +1476,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_tags_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1493,7 +1493,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_non_existent_subnet_the_associate_new_network_acl_to_subnet_method_returns_false( self): @@ -1507,7 +1507,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_network_acl_to_a_non_existent_vpc_the_associate_new_network_acl_to_subnet_method_returns_an_error( self): ''' @@ -1520,7 +1520,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_succeeds_the_disassociate_network_acl_method_should_return_true(self): ''' @@ -1533,7 +1533,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_vpc_the_disassociate_network_acl_method_should_return_false( self): @@ -1547,7 +1547,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_subnet_the_disassociate_network_acl_method_should_return_false( self): @@ -1568,7 +1568,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_succeeds_the_create_route_table_method_returns_true(self): ''' @@ -1580,7 +1580,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_on_a_non_existent_vpc_the_create_route_table_method_returns_false(self): ''' @@ -1590,7 +1590,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_table_succeeds_the_delete_route_table_method_returns_true(self): ''' @@ -1603,7 +1603,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_route_table_the_delete_route_table_method_returns_false(self): ''' @@ -1613,7 +1613,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_exists_the_route_table_exists_method_returns_true(self): ''' @@ -1626,7 +1626,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_does_not_exist_the_route_table_exists_method_returns_false(self): ''' @@ -1636,7 +1636,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_route_table_exists_but_providing_no_filters_the_route_table_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1648,7 +1648,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_succeeds_the_associate_route_table_method_should_return_the_association_id( self): @@ -1663,7 +1663,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_route_table_the_associate_route_table_method_should_return_false( self): @@ -1677,7 +1677,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_subnet_the_associate_route_table_method_should_return_false( self): @@ -1691,7 +1691,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_a_route_table_succeeds_the_disassociate_route_table_method_should_return_true( self): @@ -1708,7 +1708,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_succeeds_the_create_route_method_should_return_true(self): ''' @@ -1721,7 +1721,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_with_a_non_existent_route_table_the_create_route_method_should_return_false( self): @@ -1732,7 +1732,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_succeeds_the_delete_route_method_should_return_true(self): ''' @@ -1745,7 +1745,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_with_a_non_existent_route_table_the_delete_route_method_should_return_false( self): @@ -1756,7 +1756,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_succeeds_the_replace_route_method_should_return_true(self): ''' @@ -1769,7 +1769,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_replacing_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_with_a_non_existent_route_table_the_replace_route_method_should_return_false( self): @@ -1790,7 +1790,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_request_vpc_peering_connection(self): ''' Run with 2 vpc ids and returns a message @@ -1803,7 +1803,7 @@ class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): peer_vpc_id=other_vpc.id, **conn_parameters)) - @mock_ec2 + @mock_ec2_deprecated def test_raises_error_if_both_vpc_name_and_vpc_id_are_specified(self): ''' Must specify only one diff --git a/tests/unit/states/test_boto_vpc.py b/tests/unit/states/test_boto_vpc.py index e632b1ec2e..34b74bd068 100644 --- a/tests/unit/states/test_boto_vpc.py +++ b/tests/unit/states/test_boto_vpc.py @@ -34,18 +34,18 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -131,7 +131,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.states.boto_vpc state.module ''' - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_does_not_exist(self): ''' Tests present on a VPC that does not exist. @@ -142,14 +142,14 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes']['new']['vpc']['state'], 'available') - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_exists(self): vpc = self._create_vpc(name='test') vpc_present_result = self.salt_states['boto_vpc.present']('test', cidr_block) self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): with patch('moto.ec2.models.VPCBackend.create_vpc', side_effect=BotoServerError(400, 'Mocked error')): @@ -157,7 +157,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_present_result['result']) self.assertTrue('Mocked error' in vpc_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_does_not_exist(self): ''' Tests absent on a VPC that does not exist. @@ -167,7 +167,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_exists(self): vpc = self._create_vpc(name='test') with patch.dict(salt.utils.boto.__salt__, self.funcs): @@ -175,7 +175,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes']['new']['vpc'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -195,7 +195,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): _create = getattr(self, '_create_' + self.resource_type) _create(vpc_id=vpc_id, name=name, **self.extra_kwargs) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_does_not_exist(self): ''' Tests present on a resource that does not exist. @@ -210,7 +210,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = self.funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertTrue(exists) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_exists(self): vpc = self._create_vpc(name='test') resource = self._create_resource(vpc_id=vpc.id, name='test') @@ -220,7 +220,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_present_result['result']) self.assertEqual(resource_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): vpc = self._create_vpc(name='test') @@ -231,7 +231,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertFalse(resource_present_result['result']) self.assertTrue('Mocked error' in resource_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_does_not_exist(self): ''' Tests absent on a resource that does not exist. @@ -241,7 +241,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_absent_result['result']) self.assertEqual(resource_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_exists(self): vpc = self._create_vpc(name='test') self._create_resource(vpc_id=vpc.id, name='test') @@ -253,7 +253,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = self.funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertFalse(exists) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -301,7 +301,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas backend_create = 'RouteTableBackend.create_route_table' backend_delete = 'RouteTableBackend.delete_route_table' - @mock_ec2 + @mock_ec2_deprecated def test_present_with_subnets(self): vpc = self._create_vpc(name='test') subnet1 = self._create_subnet(vpc_id=vpc.id, name='test1') @@ -326,7 +326,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas new_subnets = changes['new']['subnets_associations'] self.assertEqual(new_subnets[0]['subnet_id'], subnet2.id) - @mock_ec2 + @mock_ec2_deprecated def test_present_with_routes(self): vpc = self._create_vpc(name='test') igw = self._create_internet_gateway(name='test', vpc_id=vpc.id) From c1f673eca4b55b8c70f81596eba8c8290acf63e2 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 10 Aug 2017 14:43:33 -0600 Subject: [PATCH 268/300] use older name if _create_unverified_context is unvailable --- .../unit/utils/vmware_test/test_connection.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/unit/utils/vmware_test/test_connection.py b/tests/unit/utils/vmware_test/test_connection.py index 812cb1231c..da5e5ae63f 100644 --- a/tests/unit/utils/vmware_test/test_connection.py +++ b/tests/unit/utils/vmware_test/test_connection.py @@ -40,6 +40,11 @@ if sys.version_info[:3] > (2, 7, 8): else: SSL_VALIDATION = False +if hasattr(ssl, '_create_unverified_context'): + ssl_context = 'ssl._create_unverified_context' +else: + ssl_context = 'ssl._create_stdlib_context' + # Get Logging Started log = logging.getLogger(__name__) @@ -337,14 +342,14 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_successful_connection(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' mock_sc = MagicMock(side_effect=[exc, None]) mock_ssl = MagicMock() with patch('salt.utils.vmware.SmartConnect', mock_sc): - with patch('ssl._create_unverified_context', + with patch(ssl_context, mock_ssl): salt.utils.vmware._get_service_instance( @@ -378,7 +383,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_successful_connection(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') @@ -387,9 +392,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): mock_ssl_context = MagicMock() with patch('salt.utils.vmware.SmartConnect', mock_sc): - with patch('ssl._create_unverified_context', - mock_ssl_unverif): - + with patch(ssl_context, mock_ssl_unverif): with patch('ssl.SSLContext', mock_ssl_context): salt.utils.vmware._get_service_instance( @@ -473,7 +476,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_unsuccsessful_connection_default_error(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('Exception') @@ -498,7 +501,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_unsuccsessful_connection_vim_fault(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = vim.fault.VimFault() @@ -523,7 +526,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_unsuccessful_connection_detault_error(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') @@ -548,7 +551,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_unsuccessful_connection_vim_fault(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') From 7c1d493fdd7c1f15e44c99ac20ba8af876c56ea7 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 17:15:45 -0400 Subject: [PATCH 269/300] @mock_elb needs to be changed to @mock_elb_deprecated as well --- tests/unit/modules/test_boto_elb.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unit/modules/test_boto_elb.py b/tests/unit/modules/test_boto_elb.py index 15a995ca4a..b577f586a7 100644 --- a/tests/unit/modules/test_boto_elb.py +++ b/tests/unit/modules/test_boto_elb.py @@ -16,7 +16,7 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2_deprecated, mock_elb + from moto import mock_ec2_deprecated, mock_elb_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False @@ -25,19 +25,19 @@ except ImportError: ''' if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + Allows boto_elb unit tests to use the @mock_ec2_deprecated decorator without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass return stub_function - def mock_elb(self): + def mock_elb_deprecated(self): ''' - if the mock_ec2_deprecated function is not available due to import failure + if the mock_elb_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator - without a "NameError: name 'mock_ec2_deprecated' is not defined" error. + Allows boto_elb unit tests to use the @mock_elb_deprecated decorator + without a "NameError: name 'mock_elb_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -115,7 +115,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): boto_elb.__virtual__() @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_register_instances_valid_id_result_true(self): ''' tests that given a valid instance id and valid ELB that @@ -134,7 +134,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(True, register_result) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_register_instances_valid_id_string(self): ''' tests that given a string containing a instance id and valid ELB that @@ -157,7 +157,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual([reservations.instances[0].id], registered_instance_ids) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_result_true(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -178,7 +178,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(True, deregister_result) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_string(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -204,7 +204,7 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(actual_instances, expected_instances) @mock_ec2_deprecated - @mock_elb + @mock_elb_deprecated def test_deregister_instances_valid_id_list(self): ''' tests that given an valid ids in the form of a list that the boto_elb From 7f46603e9cb895318f4d609e6a4be0ae6b37f18d Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 10 Aug 2017 17:31:14 -0400 Subject: [PATCH 270/300] Update account id value in boto_secgroup module unit test This value was updated in moto 1.0.0 with the following commit: https://github.com/spulec/moto/commit/5f3fbff627029c0c260546a674a3b372cacdae82 --- tests/unit/modules/test_boto_secgroup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_boto_secgroup.py b/tests/unit/modules/test_boto_secgroup.py index 51ced62319..80afac0436 100644 --- a/tests/unit/modules/test_boto_secgroup.py +++ b/tests/unit/modules/test_boto_secgroup.py @@ -213,7 +213,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): group = conn.create_security_group(name=group_name, description=group_name) group.authorize(ip_protocol=ip_protocol, from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) # setup the expected get_config result - expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'111122223333'), + expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id),('owner_id', u'123456789012'), ('description', group.description), ('tags', {}), ('rules', [{'to_port': to_port, 'from_port': from_port, 'ip_protocol': ip_protocol, 'cidr_ip': cidr_ip}]), From 35e05c95153c9212022d7eca41701fbbffa53083 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 10 Aug 2017 16:47:47 -0500 Subject: [PATCH 271/300] Add note about git CLI requirement for GitPython to GitFS tutorial --- doc/topics/tutorials/gitfs.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/topics/tutorials/gitfs.rst b/doc/topics/tutorials/gitfs.rst index 160589199f..43bf9b4071 100644 --- a/doc/topics/tutorials/gitfs.rst +++ b/doc/topics/tutorials/gitfs.rst @@ -166,13 +166,15 @@ Ubuntu 14.04 LTS and Debian Wheezy (7.x) also have a compatible version packaged # apt-get install python-git -If your master is running an older version (such as Ubuntu 12.04 LTS or Debian -Squeeze), then you will need to install GitPython using either pip_ or -easy_install (it is recommended to use pip). Version 0.3.2.RC1 is now marked as -the stable release in PyPI, so it should be a simple matter of running ``pip -install GitPython`` (or ``easy_install GitPython``) as root. +GitPython_ requires the ``git`` CLI utility to work. If installed from a system +package, then git should already be installed, but if installed via pip_ then +it may still be necessary to install git separately. For MacOS users, +GitPython_ comes bundled in with the Salt installer, but git must still be +installed for it to work properly. Git can be installed in several ways, +including by installing XCode_. -.. _`pip`: http://www.pip-installer.org/ +.. _pip: http://www.pip-installer.org/ +.. _XCode: https://developer.apple.com/xcode/ .. warning:: From 5f85a03636755ab6c7dfbb63060ff0a267e5ef1d Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Thu, 10 Aug 2017 16:35:45 -0500 Subject: [PATCH 272/300] hash_and_stat_file should return a 2-tuple Callers of `hash_and_stat_file` expect a 2-tuple and an exception will be raised if only a single value is returned. Signed-off-by: Sergey Kizunov --- salt/fileclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index a3faa10cc1..5f02723ef9 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -1257,7 +1257,7 @@ class RemoteClient(Client): if not os.path.isfile(path): msg = 'specified file {0} is not present to generate hash: {1}' log.warning(msg.format(path, err)) - return {} + return {}, None else: ret = {} hash_type = self.opts.get('hash_type', 'md5') From 4b1f55da9c3219f2687c37dedc748b9b9845fbdc Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 10 Aug 2017 16:59:11 -0600 Subject: [PATCH 273/300] Make syndic_log_file respect root_dir setting --- salt/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 5417790f06..9f69900f65 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2136,7 +2136,7 @@ def syndic_config(master_config_path, 'pki_dir', 'cachedir', 'pidfile', 'sock_dir', 'extension_modules', 'autosign_file', 'autoreject_file', 'token_dir' ] - for config_key in ('log_file', 'key_logfile'): + for config_key in ('log_file', 'key_logfile', 'syndic_log_file'): # If this is not a URI and instead a local path if urlparse(opts.get(config_key, '')).scheme == '': prepend_root_dirs.append(config_key) From 28053a84a63457713cabb55436e4c39a558dcd32 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 10 Aug 2017 17:39:58 -0600 Subject: [PATCH 274/300] Change GitPython version to 2.1.1 Version 2.1.4+ requires git to be installed, usually in the form of xcode command line tools. 2.1.3 has a memory leak problem, 2.1.2 is not available. So that leaves 2.1.1. --- pkg/osx/req.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osx/req.txt b/pkg/osx/req.txt index 2224830a5a..338454f456 100644 --- a/pkg/osx/req.txt +++ b/pkg/osx/req.txt @@ -7,7 +7,7 @@ CherryPy==11.0.0 click==6.7 enum34==1.1.6 gitdb==0.6.4 -GitPython==2.1.5 +GitPython==2.1.1 idna==2.5 ipaddress==1.0.18 Jinja2==2.9.6 From 71995505bc3aaf71bb205c7ea08f0abf9bdf2daf Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Fri, 11 Aug 2017 08:57:03 +0200 Subject: [PATCH 275/300] Not depending on specific K8s version anymore Kubernetes imports are no longer version specific. Seems like the API change was just a rename, so now we are importing either `V1beta1Deployment` or `AppsV1beta1Deployment`. Usage stays the same. --- salt/modules/kubernetes.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 10e10acef1..6ecfdd1b13 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -2,7 +2,7 @@ ''' Module for handling kubernetes calls. -:optdepends: - kubernetes Python client >= version 2.0.0 +:optdepends: - kubernetes Python client up to version 2.0.0 :configuration: The k8s API settings are provided either in a pillar, in the minion's config file, or in master's config file:: @@ -40,6 +40,14 @@ try: except ImportError: HAS_LIBS = False +try: + # There is an API change in Kubernetes >= 2.0.0. + from kubernetes.client import V1beta1Deployment as AppsV1beta1Deployment + from kubernetes.client import V1beta1DeploymentSpec as AppsV1beta1DeploymentSpec +except ImportError: + from kubernetes.client import AppsV1beta1Deployment + from kubernetes.client import AppsV1beta1DeploymentSpec + log = logging.getLogger(__name__) @@ -762,7 +770,7 @@ def create_deployment( ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.AppsV1beta1Deployment, + obj_class=AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1013,7 +1021,7 @@ def replace_deployment(name, ''' body = __create_object_body( kind='Deployment', - obj_class=kubernetes.client.AppsV1beta1Deployment, + obj_class=AppsV1beta1Deployment, spec_creator=__dict_to_deployment_spec, name=name, namespace=namespace, @@ -1278,7 +1286,7 @@ def __dict_to_deployment_spec(spec): ''' Converts a dictionary into kubernetes AppsV1beta1DeploymentSpec instance. ''' - spec_obj = kubernetes.client.AppsV1beta1DeploymentSpec() + spec_obj = AppsV1beta1DeploymentSpec() for key, value in iteritems(spec): if hasattr(spec_obj, key): setattr(spec_obj, key, value) From 81674aa88a0254d5d510e9b803ee9975bc844a2a Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Fri, 11 Aug 2017 11:20:42 +0200 Subject: [PATCH 276/300] Version info in :optdepends: not needed anymore Since this should work with 1.x.x and 2.x.x we don't have to provide a ersion info. --- salt/modules/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 6ecfdd1b13..3a54e34fd3 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -2,7 +2,7 @@ ''' Module for handling kubernetes calls. -:optdepends: - kubernetes Python client up to version 2.0.0 +:optdepends: - kubernetes Python client :configuration: The k8s API settings are provided either in a pillar, in the minion's config file, or in master's config file:: From f1de196740d00803594a51f1bfac9ec6a91e73c5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 11 Aug 2017 08:32:19 -0500 Subject: [PATCH 277/300] Add virtual func for cron state module This prevents a traceback when you attempt to use a cron state without cron being installed. --- salt/states/cron.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/states/cron.py b/salt/states/cron.py index ca8f69afb6..381bb173ab 100644 --- a/salt/states/cron.py +++ b/salt/states/cron.py @@ -150,6 +150,13 @@ from salt.modules.cron import ( ) +def __virtual__(): + if 'cron.list_tab' in __salt__: + return True + else: + return (False, 'cron module could not be loaded') + + def _check_cron(user, cmd, minute=None, From 05ecc6ac8dd43d01991c5143b64acb6231ce0b14 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 11 Aug 2017 09:59:23 -0600 Subject: [PATCH 278/300] fix vmware for python 3.4.2 in salt.utils.vmware --- salt/utils/vmware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/vmware.py b/salt/utils/vmware.py index 19292f2128..4b8f5a091a 100644 --- a/salt/utils/vmware.py +++ b/salt/utils/vmware.py @@ -238,7 +238,7 @@ def _get_service_instance(host, username, password, protocol, pwd=password, protocol=protocol, port=port, - sslContext=ssl._create_unverified_context(), + sslContext=getattr(ssl, '_create_unverified_context', getattr(ssl, '_create_stdlib_context'))(), b64token=token, mechanism=mechanism) else: From 80fd733c995f77cec90a875d78576a5b76a57e63 Mon Sep 17 00:00:00 2001 From: Giandom Date: Fri, 11 Aug 2017 18:37:52 +0200 Subject: [PATCH 279/300] Update slack.py --- salt/engines/slack.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 1709364832..024889ceb6 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -62,6 +62,7 @@ import logging import time import re import yaml +import ast try: import slackclient @@ -246,6 +247,10 @@ def start(token, tgt_type = kwargs['tgt_type'] del kwargs['tgt_type'] + # Check for pillar string representation of dict and convert it to dict + if 'pillar' in kwargs: + kwargs.update(pillar=ast.literal_eval(kwargs['pillar'])) + ret = {} if cmd in runner_functions: @@ -255,7 +260,7 @@ def start(token, # Default to trying to run as a client module. else: local = salt.client.LocalClient() - ret = local.cmd('{0}'.format(target), cmd, args, kwargs, tgt_type='{0}'.format(tgt_type)) + ret = local.cmd('{0}'.format(target), cmd, arg=args, kwarg=kwargs, tgt_type='{0}'.format(tgt_type)) if ret: return_text = json.dumps(ret, sort_keys=True, indent=1) From 43643227c62b9f5fc6b94dfef457ae4257e90d4a Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 12:57:39 -0400 Subject: [PATCH 280/300] Skip 2 failing tests in Python 3 due to upstream bugs --- tests/unit/states/test_boto_vpc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/states/test_boto_vpc.py b/tests/unit/states/test_boto_vpc.py index 34b74bd068..00336d20d5 100644 --- a/tests/unit/states/test_boto_vpc.py +++ b/tests/unit/states/test_boto_vpc.py @@ -14,6 +14,7 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch import salt.config import salt.loader import salt.utils.boto +from salt.ext import six from salt.utils.versions import LooseVersion import salt.states.boto_vpc as boto_vpc @@ -291,6 +292,9 @@ class BotoVpcInternetGatewayTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTe @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(six.PY3, 'Disabled for Python 3 due to upstream bugs: ' + 'https://github.com/spulec/moto/issues/548 and ' + 'https://github.com/gabrielfalcao/HTTPretty/issues/325') @skipIf(HAS_BOTO is False, 'The boto module must be installed.') @skipIf(HAS_MOTO is False, 'The moto module must be installed.') @skipIf(_has_required_boto() is False, 'The boto module must be greater than' From da3402a53dbb77e1d1ed55bfe8c0ec890ed5330d Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 11 Aug 2017 10:58:00 -0600 Subject: [PATCH 281/300] make sure cmd is not run when npm isn't installed apparently the skipIf on the functions still get run, even if the function is going to be skipped based on a skipIf on the class. --- tests/integration/states/npm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/states/npm.py b/tests/integration/states/npm.py index 44f43a9a07..140eaa92ca 100644 --- a/tests/integration/states/npm.py +++ b/tests/integration/states/npm.py @@ -54,7 +54,8 @@ class NpmStateTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ret = self.run_state('npm.installed', name=None, pkgs=['pm2', 'grunt']) self.assertSaltTrueReturn(ret) - @skipIf(LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION), 'Skip with npm >= 5.0.0 until #41770 is fixed') + @skipIf(salt.utils.which('npm') and LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION), + 'Skip with npm >= 5.0.0 until #41770 is fixed') @destructiveTest def test_npm_cache_clean(self): ''' From 462d65308236e0fd81d5dc938d432031dc904966 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 11 Aug 2017 12:20:50 -0500 Subject: [PATCH 282/300] Move weird tearDown test to an actual tearDown Also catch KeyError when user doesn't exist --- tests/integration/shell/test_auth.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/integration/shell/test_auth.py b/tests/integration/shell/test_auth.py index 3c4c7000a2..94793af51d 100644 --- a/tests/integration/shell/test_auth.py +++ b/tests/integration/shell/test_auth.py @@ -54,7 +54,6 @@ class AuthTest(ShellCase): group = 'saltops' def setUp(self): - # This is a little wasteful but shouldn't be a problem for user in (self.userA, self.userB): try: pwd.getpwnam(user) @@ -68,6 +67,21 @@ class AuthTest(ShellCase): self.run_call('group.add {0}'.format(self.group)) self.run_call('user.chgroups {0} {1} True'.format(self.userB, self.group)) + def tearDown(self): + for user in (self.userA, self.userB): + try: + pwd.getpwnam(user) + except KeyError: + pass + else: + self.run_call('user.delete {0}'.format(user)) + try: + grp.getgrnam(self.group) + except KeyError: + pass + else: + self.run_call('group.delete {0}'.format(self.group)) + def test_pam_auth_valid_user(self): ''' test that pam auth mechanism works with a valid user @@ -121,10 +135,3 @@ class AuthTest(ShellCase): self.assertTrue( 'minion:' in resp ) - - def test_zzzz_tearDown(self): - for user in (self.userA, self.userB): - if pwd.getpwnam(user): - self.run_call('user.delete {0}'.format(user)) - if grp.getgrnam(self.group): - self.run_call('group.delete {0}'.format(self.group)) From 6a7bf998484afd437a71f48346e4915d8488b92d Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 14:19:15 -0400 Subject: [PATCH 283/300] Lint fix: add missing space --- tests/unit/modules/test_boto_secgroup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_boto_secgroup.py b/tests/unit/modules/test_boto_secgroup.py index 80afac0436..ef90958dec 100644 --- a/tests/unit/modules/test_boto_secgroup.py +++ b/tests/unit/modules/test_boto_secgroup.py @@ -213,7 +213,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): group = conn.create_security_group(name=group_name, description=group_name) group.authorize(ip_protocol=ip_protocol, from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) # setup the expected get_config result - expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id),('owner_id', u'123456789012'), + expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'123456789012'), ('description', group.description), ('tags', {}), ('rules', [{'to_port': to_port, 'from_port': from_port, 'ip_protocol': ip_protocol, 'cidr_ip': cidr_ip}]), From 885bee2a7d9326071416d1abc4248e526253f026 Mon Sep 17 00:00:00 2001 From: Carson Anderson Date: Fri, 11 Aug 2017 10:39:11 -0600 Subject: [PATCH 284/300] Stub out required functions for redis cache Without these functions the masters are not able to use the redis cache completely Signed-off-by: Carson Anderson --- salt/cache/redis_cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/cache/redis_cache.py b/salt/cache/redis_cache.py index 6678787efd..434e5f4d8c 100644 --- a/salt/cache/redis_cache.py +++ b/salt/cache/redis_cache.py @@ -115,7 +115,7 @@ from salt.exceptions import SaltCacheError __virtualname__ = 'redis' __func_alias__ = { - 'list_': 'list' + 'ls': 'list' } log = logging.getLogger(__file__) @@ -145,6 +145,9 @@ def __virtual__(): # helper functions -- will not be exported # ----------------------------------------------------------------------------- +def init_kwargs(kwargs): + return {} + def _get_redis_cache_opts(): ''' @@ -415,7 +418,7 @@ def flush(bank, key=None): return True -def list_(bank): +def ls(bank): ''' Lists entries stored in the specified bank. ''' From 71e7581a2d8d2b14dfaf32018053a0b9d065f202 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 14:31:17 -0400 Subject: [PATCH 285/300] Remove extraneous "deprecated" notation This deprecated notation was supposed to be on the "useradd_all" function, but somehow was moved to the "useradd" function. Fixes #42870 --- salt/modules/htpasswd.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/modules/htpasswd.py b/salt/modules/htpasswd.py index 02f48457c2..cbb09e99ea 100644 --- a/salt/modules/htpasswd.py +++ b/salt/modules/htpasswd.py @@ -35,8 +35,6 @@ def useradd(pwfile, user, password, opts='', runas=None): Add a user to htpasswd file using the htpasswd command. If the htpasswd file does not exist, it will be created. - .. deprecated:: 2016.3.0 - pwfile Path to htpasswd file From fb7117f2acdb54560048d319235b5f3d35ae989b Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 11 Aug 2017 16:31:24 -0400 Subject: [PATCH 286/300] Use salt.utils.versions.LooseVersion instead of distutils --- tests/integration/states/test_npm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/states/test_npm.py b/tests/integration/states/test_npm.py index e5f298930e..42bc94e4b0 100644 --- a/tests/integration/states/test_npm.py +++ b/tests/integration/states/test_npm.py @@ -6,7 +6,6 @@ ''' # Import Python libs from __future__ import absolute_import -from distutils.version import LooseVersion # Import Salt Testing libs from tests.support.case import ModuleCase @@ -17,6 +16,7 @@ from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs import salt.utils import salt.modules.cmdmod as cmd +from salt.utils.versions import LooseVersion MAX_NPM_VERSION = '5.0.0' From f903e7bc394242703d990cd3a7450838d9c0629d Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sat, 12 Aug 2017 23:11:25 +0100 Subject: [PATCH 287/300] Minor eos doc correction --- doc/topics/installation/eos.rst | 61 ++++++++++++++++----------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/doc/topics/installation/eos.rst b/doc/topics/installation/eos.rst index f7537c334b..c8c7787be7 100644 --- a/doc/topics/installation/eos.rst +++ b/doc/topics/installation/eos.rst @@ -1,4 +1,3 @@ - ========================================= Arista EOS Salt minion installation guide ========================================= @@ -122,34 +121,34 @@ Additional Information This SWIX extension contains the following RPM packages: -.. code-block:: +.. code-block:: text - libsodium-1.0.11-1.fc25.i686.rpm - libstdc++-6.2.1-2.fc25.i686.rpm - openpgm-5.2.122-6.fc24.i686.rpm - python-Jinja2-2.8-0.i686.rpm - python-PyYAML-3.12-0.i686.rpm - python-babel-0.9.6-5.fc18.noarch.rpm - python-backports-1.0-3.fc18.i686.rpm - python-backports-ssl_match_hostname-3.4.0.2-1.fc18.noarch.rpm - python-backports_abc-0.5-0.i686.rpm - python-certifi-2016.9.26-0.i686.rpm - python-chardet-2.0.1-5.fc18.noarch.rpm - python-crypto-1.4.1-1.noarch.rpm - python-crypto-2.6.1-1.fc18.i686.rpm - python-futures-3.1.1-1.noarch.rpm - python-jtextfsm-0.3.1-0.noarch.rpm - python-kitchen-1.1.1-2.fc18.noarch.rpm - python-markupsafe-0.18-1.fc18.i686.rpm - python-msgpack-python-0.4.8-0.i686.rpm - python-napalm-base-0.24.3-1.noarch.rpm - python-napalm-eos-0.6.0-1.noarch.rpm - python-netaddr-0.7.18-0.noarch.rpm - python-pyeapi-0.7.0-0.noarch.rpm - python-salt-2017.7.0_1414_g2fb986f-1.noarch.rpm - python-singledispatch-3.4.0.3-0.i686.rpm - python-six-1.10.0-0.i686.rpm - python-tornado-4.4.2-0.i686.rpm - python-urllib3-1.5-7.fc18.noarch.rpm - python2-zmq-15.3.0-2.fc25.i686.rpm - zeromq-4.1.4-5.fc25.i686.rpm + libsodium-1.0.11-1.fc25.i686.rpm + libstdc++-6.2.1-2.fc25.i686.rpm + openpgm-5.2.122-6.fc24.i686.rpm + python-Jinja2-2.8-0.i686.rpm + python-PyYAML-3.12-0.i686.rpm + python-babel-0.9.6-5.fc18.noarch.rpm + python-backports-1.0-3.fc18.i686.rpm + python-backports-ssl_match_hostname-3.4.0.2-1.fc18.noarch.rpm + python-backports_abc-0.5-0.i686.rpm + python-certifi-2016.9.26-0.i686.rpm + python-chardet-2.0.1-5.fc18.noarch.rpm + python-crypto-1.4.1-1.noarch.rpm + python-crypto-2.6.1-1.fc18.i686.rpm + python-futures-3.1.1-1.noarch.rpm + python-jtextfsm-0.3.1-0.noarch.rpm + python-kitchen-1.1.1-2.fc18.noarch.rpm + python-markupsafe-0.18-1.fc18.i686.rpm + python-msgpack-python-0.4.8-0.i686.rpm + python-napalm-base-0.24.3-1.noarch.rpm + python-napalm-eos-0.6.0-1.noarch.rpm + python-netaddr-0.7.18-0.noarch.rpm + python-pyeapi-0.7.0-0.noarch.rpm + python-salt-2017.7.0_1414_g2fb986f-1.noarch.rpm + python-singledispatch-3.4.0.3-0.i686.rpm + python-six-1.10.0-0.i686.rpm + python-tornado-4.4.2-0.i686.rpm + python-urllib3-1.5-7.fc18.noarch.rpm + python2-zmq-15.3.0-2.fc25.i686.rpm + zeromq-4.1.4-5.fc25.i686.rpm From 93be79a1352ea048c3987d3631d560c904d60b45 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Sat, 12 Aug 2017 23:15:11 +0100 Subject: [PATCH 288/300] Index eos under the installation instructions list --- doc/topics/installation/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index 0ee4b38d7a..2f96830c4e 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -46,6 +46,7 @@ These guides go into detail how to install Salt on a given platform. arch debian + eos fedora freebsd gentoo From 99046b441f890bf36f0cff72c4a591da8df7553e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 14 Aug 2017 10:26:21 -0600 Subject: [PATCH 289/300] cloud driver isn't a provider This will not have a ':' in it at all. --- salt/utils/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index f4ee061958..4677fde631 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -312,7 +312,7 @@ def bootstrap(vm_, opts): } } - if vm_.get('driver', 'none:none').split(':')[1] == 'saltify': + if vm_.get('driver') == 'saltify': saltify_driver = True else: saltify_driver = False From a3becf8342c46173b8522190c0afb3082b3004dd Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 14 Aug 2017 11:19:09 -0600 Subject: [PATCH 290/300] Change service shutdown timeouts Windows default for stopping a service is 30 seconds. This changes the default timeouts for the salt-minion service to more closely resemble how it is handled in Windows. This gives Python a chance to cleanly exit before being forcibly closed. --- pkg/windows/installer/Salt-Minion-Setup.nsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index b3566583e3..dbb795e89b 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -383,6 +383,8 @@ Section -Post nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion Start SERVICE_AUTO_START" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" + nsExec::Exec "nssm.exe set salt-minion AppStopMethodConsole 24000" + nsExec::Exec "nssm.exe set salt-minion AppStopMethodWindow 2000" RMDir /R "$INSTDIR\var\cache\salt" ; removing cache from old version From ffb23fbe47f5b16bd6b61c589f1e08bda1f3de49 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 14 Aug 2017 12:26:27 -0600 Subject: [PATCH 291/300] Remove the line that wipes out the cache --- pkg/windows/installer/Salt-Minion-Setup.nsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index dbb795e89b..46fb821fb8 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -386,8 +386,6 @@ Section -Post nsExec::Exec "nssm.exe set salt-minion AppStopMethodConsole 24000" nsExec::Exec "nssm.exe set salt-minion AppStopMethodWindow 2000" - RMDir /R "$INSTDIR\var\cache\salt" ; removing cache from old version - Call updateMinionConfig Push "C:\salt" From 5e930b8cbda8fddec48770b4d4c60050e5a7eecb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 14 Aug 2017 12:23:27 -0700 Subject: [PATCH 292/300] If we catch the pid file in a transistory state, return None --- salt/utils/process.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/utils/process.py b/salt/utils/process.py index aefe8731ee..fc47877d92 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -130,8 +130,10 @@ def get_pidfile(pidfile): ''' with salt.utils.fopen(pidfile) as pdf: pid = pdf.read() - - return int(pid) + if pid: + return int(pid) + else: + return def clean_proc(proc, wait_for_kill=10): From 8165f46165d2aeeebbeedc0cb89de816601ddc48 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 14 Aug 2017 16:25:13 -0500 Subject: [PATCH 293/300] Add debug logging to troubleshoot test failures This adds logging to troubleshoot https://github.com/saltstack/salt-jenkins/issues/477 --- tests/integration/shell/test_auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/shell/test_auth.py b/tests/integration/shell/test_auth.py index 94793af51d..fc80c7ac05 100644 --- a/tests/integration/shell/test_auth.py +++ b/tests/integration/shell/test_auth.py @@ -6,6 +6,7 @@ # Import python libs from __future__ import absolute_import +import logging import pwd import grp import random @@ -21,6 +22,8 @@ from salt.utils.pycrypto import gen_hash # Import 3rd-party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin +log = logging.getLogger(__name__) + def gen_password(): ''' @@ -99,6 +102,7 @@ class AuthTest(ShellCase): cmd = ('-a pam "*" test.ping ' '--username {0} --password {1}'.format(self.userA, password)) resp = self.run_salt(cmd) + log.debug('resp = %s', resp) self.assertTrue( 'minion:' in resp ) From a0118bcecec895b8247245b44611bc003d71ea43 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 16:33:42 -0500 Subject: [PATCH 294/300] Remove bytestrings and use textwrap.dedent for readability PyYAML works with and without bytestrings on PY3, and Python 3 can read all of the test data as regular strings, so bytestrings are unnecessary here. This also adds use of textwrap.dedent to make the YAML easier to read, so it can be indented away from the far left side of the line. --- tests/unit/utils/test_yamlloader.py | 69 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/tests/unit/utils/test_yamlloader.py b/tests/unit/utils/test_yamlloader.py index 69a2724f5d..2f90f29ff9 100644 --- a/tests/unit/utils/test_yamlloader.py +++ b/tests/unit/utils/test_yamlloader.py @@ -5,6 +5,7 @@ # Import python libs from __future__ import absolute_import +import textwrap # Import Salt Libs from yaml.constructor import ConstructorError @@ -36,12 +37,11 @@ class YamlLoaderTestCase(TestCase): ''' Test parsing an ordinary path ''' - self.assertEqual( - self._render_yaml(b''' -p1: - - alpha - - beta'''), + self._render_yaml(textwrap.dedent('''\ + p1: + - alpha + - beta''')), {'p1': ['alpha', 'beta']} ) @@ -49,38 +49,37 @@ p1: ''' Test YAML anchors ''' - # Simple merge test self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v2: beta'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v2: beta''')), {'p1': {'v1': 'alpha'}, 'p2': {'v1': 'alpha', 'v2': 'beta'}} ) # Test that keys/nodes are overwritten self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v1: new_alpha'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v1: new_alpha''')), {'p1': {'v1': 'alpha'}, 'p2': {'v1': 'new_alpha'}} ) # Test merging of lists self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: &v1 - - t1 - - t2 -p2: - v2: *v1'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: &v1 + - t1 + - t2 + p2: + v2: *v1''')), {"p2": {"v2": ["t1", "t2"]}, "p1": {"v1": ["t1", "t2"]}} ) @@ -89,15 +88,15 @@ p2: Test that duplicates still throw an error ''' with self.assertRaises(ConstructorError): - self._render_yaml(b''' -p1: alpha -p1: beta''') + self._render_yaml(textwrap.dedent('''\ + p1: alpha + p1: beta''')) with self.assertRaises(ConstructorError): - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v2: beta - v2: betabeta''') + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v2: beta + v2: betabeta''')) From 9d8486a89484b772d728441d734b3f4be6db525d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 8 Aug 2017 16:48:43 -0500 Subject: [PATCH 295/300] Add test for custom YAML loader with unicode literal strings --- tests/unit/utils/test_yamlloader.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/utils/test_yamlloader.py b/tests/unit/utils/test_yamlloader.py index 2f90f29ff9..11228ff7e0 100644 --- a/tests/unit/utils/test_yamlloader.py +++ b/tests/unit/utils/test_yamlloader.py @@ -100,3 +100,15 @@ class YamlLoaderTestCase(TestCase): <<: *p1 v2: beta v2: betabeta''')) + + def test_yaml_with_unicode_literals(self): + ''' + Test proper loading of unicode literals + ''' + self.assertEqual( + self._render_yaml(textwrap.dedent('''\ + foo: + a: Д + b: {'a': u'\\u0414'}''')), + {'foo': {'a': u'\u0414', 'b': {'a': u'\u0414'}}} + ) From 73f9135340843325e7ed3eb3dd0bdd18935cfbbc Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Tue, 15 Aug 2017 16:08:58 +0000 Subject: [PATCH 296/300] extension_modules should default to /proxy/extmods Additionally, append it to the append_minionid_config_dirs list, so each proxy caches its extension modules separately. --- salt/config/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 4feaff3293..0f06f9ccca 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1633,7 +1633,8 @@ DEFAULT_PROXY_MINION_OPTS = { 'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'proxy'), 'add_proxymodule_to_opts': False, 'proxy_merge_grains_in_module': True, - 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include'], + 'extension_modules': os.path.join(salt.syspaths.CACHE_DIR, 'proxy', 'extmods'), + 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include', 'extension_modules'], 'default_include': 'proxy.d/*.conf', # By default, proxies will preserve the connection. From 1d8f827c5840efe4680fe4c33598000e032812b8 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:37:50 -0400 Subject: [PATCH 297/300] Add Security Notice to 2017.7.1 Release Notes --- doc/topics/releases/2017.7.1.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/topics/releases/2017.7.1.rst b/doc/topics/releases/2017.7.1.rst index 69f7a9aa79..7cc616c94b 100644 --- a/doc/topics/releases/2017.7.1.rst +++ b/doc/topics/releases/2017.7.1.rst @@ -4,6 +4,13 @@ Salt 2017.7.1 Release Notes Version 2017.7.1 is a bugfix release for :ref:`2017.7.0 `. +Security Fix +============ + +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master + +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com + Changes for v2017.7.0..v2017.7.1 -------------------------------- From cbecf658230fedd8bb8135e24d3b9c522cbee341 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 15 Aug 2017 12:55:02 -0400 Subject: [PATCH 298/300] [2017.7] Bump latest and previous versions --- doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 6e828afc77..de8db8ea90 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -245,8 +245,8 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2017.7.0' # latest release -previous_release = '2016.11.6' # latest release from previous branch +latest_release = '2017.7.1' # latest release +previous_release = '2016.11.7' # latest release from previous branch previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch From e9febe4893280b50cb56686ac4253b980567f245 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 15 Aug 2017 16:34:03 -0600 Subject: [PATCH 299/300] Fix unit.test_fileclient Use os.sep instead of hard-coded, unix-style paths --- tests/unit/test_fileclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_fileclient.py b/tests/unit/test_fileclient.py index 077630693a..b6bd2207e0 100644 --- a/tests/unit/test_fileclient.py +++ b/tests/unit/test_fileclient.py @@ -6,6 +6,7 @@ # Import Python libs from __future__ import absolute_import import errno +import os # Import Salt Testing libs from tests.support.mock import patch, Mock @@ -38,7 +39,7 @@ class FileclientTestCase(TestCase): for exists in range(2): with patch('os.makedirs', self._fake_makedir()): with Client(self.opts)._cache_loc('testfile') as c_ref_itr: - assert c_ref_itr == '/__test__/files/base/testfile' + assert c_ref_itr == os.sep + os.sep.join(['__test__', 'files', 'base', 'testfile']) def test_cache_raises_exception_on_non_eexist_ioerror(self): ''' From 121cd4ef818dbb55e6bce95ae2c159a0b42f299d Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 15 Aug 2017 16:53:28 -0600 Subject: [PATCH 300/300] Fix `salt.utils.recursive_copy` for Windows Use os.sep instead of hard-coded `/` --- salt/utils/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/files.py b/salt/utils/files.py index 12dd6987c6..d4893608a2 100644 --- a/salt/utils/files.py +++ b/salt/utils/files.py @@ -57,7 +57,7 @@ def recursive_copy(source, dest): (identical to cp -r on a unix machine) ''' for root, _, files in os.walk(source): - path_from_source = root.replace(source, '').lstrip('/') + path_from_source = root.replace(source, '').lstrip(os.sep) target_directory = os.path.join(dest, path_from_source) if not os.path.exists(target_directory): os.makedirs(target_directory)