From 2e6252c273f5e195c473a7dcb0edfd05da82ece5 Mon Sep 17 00:00:00 2001 From: "U-KEIRIN\\lwebb" Date: Tue, 15 Jan 2019 12:19:37 +1100 Subject: [PATCH 01/67] Support rubygems 3 cli param changes --- salt/modules/gem.py | 58 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/salt/modules/gem.py b/salt/modules/gem.py index 9af3c09389..7db6baf98f 100644 --- a/salt/modules/gem.py +++ b/salt/modules/gem.py @@ -91,8 +91,12 @@ def install(gems, # pylint: disable=C0103 Doesn't play nice with multiple gems at once :param rdoc: boolean : False Generate RDoc documentation for the gem(s). + For rubygems > 3 this is interpreted as the --no-document arg and the + ri option will then be ignored :param ri: boolean : False Generate RI documentation for the gem(s). + For rubygems > 3 this is interpreted as the --no-document arg and the + rdoc option will then be ignored :param pre_releases: boolean : False Include pre-releases in the available versions :param proxy: string : None @@ -119,12 +123,18 @@ def install(gems, # pylint: disable=C0103 options = [] if version: options.extend(['--version', version]) - if not rdoc: - options.append('--no-rdoc') - if not ri: - options.append('--no-ri') - if pre_releases: - options.append('--pre') + if _has_rubygems_3(ruby=ruby, runas=runas, gem_bin=gem_bin): + if not rdoc or not ri: + options.append('--no-document') + if pre_releases: + options.append('--prerelease') + else: + if not rdoc: + options.append('--no-rdoc') + if not ri: + options.append('--no-ri') + if pre_releases: + options.append('--pre') if proxy: options.extend(['-p', proxy]) if source: @@ -223,6 +233,42 @@ def update_system(version='', ruby=None, runas=None, gem_bin=None): gem_bin=gem_bin, runas=runas) +def version(ruby=None, runas=None, gem_bin=None): + ''' + Print out the version of gem + + :param gem_bin: string : None + Full path to ``gem`` binary to use. + :param ruby: string : None + If RVM or rbenv are installed, the ruby version and gemset to use. + Ignored if ``gem_bin`` is specified. + :param runas: string : None + The user to run gem as. + + CLI Example: + + .. code-block:: bash + + salt '*' gem.version + ''' + cmd = ['--version'] + stdout = _gem(cmd, + ruby, + gem_bin=gem_bin, + runas=runas) + ret = {} + for line in salt.utils.itertools.split(stdout, '\n'): + match = re.match(r'[.0-9]+', line) + if match: + ret = line + break + return ret + +def _has_rubygems_3(ruby=None, runas=None, gem_bin=None): + match = re.match(r'^3\..*', version(ruby=ruby, runas=runas, gem_bin=gem_bin)) + if match: + return True + return False def list_(prefix='', ruby=None, runas=None, gem_bin=None): ''' From 22eda16d64280bd655c4e670c075da379cc140a1 Mon Sep 17 00:00:00 2001 From: "U-KEIRIN\\lwebb" Date: Tue, 15 Jan 2019 15:27:57 +1100 Subject: [PATCH 02/67] Fix lint violations --- salt/modules/gem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/modules/gem.py b/salt/modules/gem.py index 7db6baf98f..5a31e60bff 100644 --- a/salt/modules/gem.py +++ b/salt/modules/gem.py @@ -233,6 +233,7 @@ def update_system(version='', ruby=None, runas=None, gem_bin=None): gem_bin=gem_bin, runas=runas) + def version(ruby=None, runas=None, gem_bin=None): ''' Print out the version of gem @@ -264,12 +265,14 @@ def version(ruby=None, runas=None, gem_bin=None): break return ret + def _has_rubygems_3(ruby=None, runas=None, gem_bin=None): match = re.match(r'^3\..*', version(ruby=ruby, runas=runas, gem_bin=gem_bin)) if match: return True return False + def list_(prefix='', ruby=None, runas=None, gem_bin=None): ''' List locally installed gems. From 241707efe813b1e75c32a40cc751d32e648b11a4 Mon Sep 17 00:00:00 2001 From: "menglong.zhou" Date: Wed, 16 Jan 2019 15:21:40 +0800 Subject: [PATCH 03/67] ss commmand replace to netstat command --- salt/utils/network.py | 50 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/salt/utils/network.py b/salt/utils/network.py index 9641739e76..1aa6d74f9d 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1394,8 +1394,11 @@ def _remotes_on(port, which_end): Return a set of ip addrs active tcp connections ''' port = int(port) - ret = set() + ret = _netlink_tool_remote_on(port, which_end) + if ret is not None: + return ret + ret = set() proc_available = False for statf in ['/proc/net/tcp', '/proc/net/tcp6']: if os.path.isfile(statf): @@ -1446,6 +1449,51 @@ def _parse_tcp_line(line): return ret +def _netlink_tool_remote_on(port, which_end): + ''' + Returns set of ipv4 host addresses of remote established connections + on local or remote tcp port. + + Parses output of shell 'ss' to get connections + + [root@salt-master ~]# ss -ant + State Recv-Q Send-Q Local Address:Port Peer Address:Port + LISTEN 0 511 *:80 *:* + LISTEN 0 128 *:22 *:* + ESTAB 0 0 127.0.0.1:56726 127.0.0.1:4505 + ''' + remotes = set() + valid = False + try: + data = subprocess.check_output(['ss', '-ant']) # pylint: disable=minimum-python-version + except subprocess.CalledProcessError: + log.error('Failed ss') + raise + except OSError: # not command "No such file or directory" + return None + + lines = salt.utils.stringutils.to_str(data).split('\n') + for line in lines: + if 'Address:Port' in line: # ss tools may not be valid + valid = True + continue + elif 'ESTAB' not in line: + continue + chunks = line.split() + local_host, local_port = chunks[3].split(':') + remote_host, remote_port = chunks[4].split(':') + + if which_end == 'remote_port' and int(remote_port) != port: + continue + if which_end == 'local_port' and int(local_port) != port: + continue + remotes.add(remote_host) + + if valid is False: + remotes = None + return remotes + + def _sunos_remotes_on(port, which_end): ''' SunOS specific helper function. From 76770f33265dff4790ec22c0eb88c6bfa705e4aa Mon Sep 17 00:00:00 2001 From: "menglong.zhou" Date: Wed, 16 Jan 2019 15:24:09 +0800 Subject: [PATCH 04/67] ss commmand replace to netstat command --- salt/utils/network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/utils/network.py b/salt/utils/network.py index 1aa6d74f9d..6247d85f29 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1394,6 +1394,7 @@ def _remotes_on(port, which_end): Return a set of ip addrs active tcp connections ''' port = int(port) + ret = _netlink_tool_remote_on(port, which_end) if ret is not None: return ret From 5de5e643da6661c9ee985b7395798435e9702149 Mon Sep 17 00:00:00 2001 From: "U-KEIRIN\\lwebb" Date: Thu, 17 Jan 2019 15:50:38 +1100 Subject: [PATCH 05/67] Update tests --- tests/unit/modules/test_gem.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/test_gem.py b/tests/unit/modules/test_gem.py index f2a2adc1f2..b3a7e85903 100644 --- a/tests/unit/modules/test_gem.py +++ b/tests/unit/modules/test_gem.py @@ -73,12 +73,29 @@ class TestGemModule(TestCase, LoaderModuleMockMixin): runas=None ) - def test_install_pre(self): + + def test_install_pre_rubygems_3(self): mock = MagicMock(return_value={'retcode': 0, 'stdout': ''}) with patch.dict(gem.__salt__, {'rvm.is_installed': MagicMock(return_value=False), 'rbenv.is_installed': MagicMock(return_value=False), - 'cmd.run_all': mock}): + 'cmd.run_all': mock}),\ + patch.object(gem, '_has_rubygems_3', MagicMock(return_value=True)): + gem.install('rails', pre_releases=True) + mock.assert_called_once_with( + ['gem', 'install', 'rails', '--no-document', '--prerelease'], + runas=None, + python_shell=False + ) + + + def test_install_pre_rubygems_3(self): + mock = MagicMock(return_value={'retcode': 0, 'stdout': ''}) + with patch.dict(gem.__salt__, + {'rvm.is_installed': MagicMock(return_value=False), + 'rbenv.is_installed': MagicMock(return_value=False), + 'cmd.run_all': mock}),\ + patch.object(gem, '_has_rubygems_3', MagicMock(return_value=False)): gem.install('rails', pre_releases=True) mock.assert_called_once_with( ['gem', 'install', 'rails', '--no-rdoc', '--no-ri', '--pre'], From d879d18dcd1a5f53a84330c5ec7302666501649f Mon Sep 17 00:00:00 2001 From: "menglong.zhou" Date: Thu, 17 Jan 2019 19:09:00 +0800 Subject: [PATCH 06/67] fix broken network.py --- salt/utils/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/network.py b/salt/utils/network.py index 6247d85f29..6f127f3917 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1481,8 +1481,8 @@ def _netlink_tool_remote_on(port, which_end): elif 'ESTAB' not in line: continue chunks = line.split() - local_host, local_port = chunks[3].split(':') - remote_host, remote_port = chunks[4].split(':') + local_host, local_port = chunks[3].split(':', 1) + remote_host, remote_port = chunks[4].split(':', 1) if which_end == 'remote_port' and int(remote_port) != port: continue From 1945c30feb36ff8e7f002f3c9b4e4433a84a7701 Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Thu, 17 Jan 2019 15:28:59 -0500 Subject: [PATCH 07/67] Document the client-defined dunders --- doc/topics/development/modules/developing.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/topics/development/modules/developing.rst b/doc/topics/development/modules/developing.rst index a01137d744..df167c2cba 100644 --- a/doc/topics/development/modules/developing.rst +++ b/doc/topics/development/modules/developing.rst @@ -235,3 +235,17 @@ Defined in: State __sdb__ ------- Defined in: SDB + + +Additional Globals +================== + +Defined for: Runners, Execution Modules, Wheels + +* ``__jid__``: The job ID +* ``__user__``: The user +* ``__tag__``: The jid tag +* ``__jid_event__``: A :py:class:`salt.utils.event.NamespacedEvent`. + +:py:class:`NamespacedEvent ` defines a single +method :py:meth:`fire_event `, that takes data and tag. The :ref:`Runner docs ` has examples. From 80a38262c478e80710f93a551bb87e96cb309c62 Mon Sep 17 00:00:00 2001 From: Jamie Bliss Date: Thu, 17 Jan 2019 15:30:37 -0500 Subject: [PATCH 08/67] Update TODO --- salt/client/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 3717e8bccb..6d29697da8 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -324,7 +324,7 @@ class SyncClientMixin(object): print_func=print_func ) - # TODO: document these, and test that they exist + # TODO: test that they exist # TODO: Other things to inject?? func_globals = {'__jid__': jid, '__user__': data['user'], From 13d3288b884711ee6edda8488a2c9fba5e84c459 Mon Sep 17 00:00:00 2001 From: Lee Webb Date: Fri, 18 Jan 2019 09:17:47 +1100 Subject: [PATCH 09/67] Fix lint issues --- tests/unit/modules/test_gem.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/modules/test_gem.py b/tests/unit/modules/test_gem.py index b3a7e85903..4012cc1c40 100644 --- a/tests/unit/modules/test_gem.py +++ b/tests/unit/modules/test_gem.py @@ -73,14 +73,14 @@ class TestGemModule(TestCase, LoaderModuleMockMixin): runas=None ) - def test_install_pre_rubygems_3(self): mock = MagicMock(return_value={'retcode': 0, 'stdout': ''}) with patch.dict(gem.__salt__, {'rvm.is_installed': MagicMock(return_value=False), 'rbenv.is_installed': MagicMock(return_value=False), 'cmd.run_all': mock}),\ - patch.object(gem, '_has_rubygems_3', MagicMock(return_value=True)): + patch.object( + gem, '_has_rubygems_3', MagicMock(return_value=True)): gem.install('rails', pre_releases=True) mock.assert_called_once_with( ['gem', 'install', 'rails', '--no-document', '--prerelease'], @@ -88,14 +88,14 @@ class TestGemModule(TestCase, LoaderModuleMockMixin): python_shell=False ) - - def test_install_pre_rubygems_3(self): + def test_install_pre(self): mock = MagicMock(return_value={'retcode': 0, 'stdout': ''}) with patch.dict(gem.__salt__, {'rvm.is_installed': MagicMock(return_value=False), 'rbenv.is_installed': MagicMock(return_value=False), 'cmd.run_all': mock}),\ - patch.object(gem, '_has_rubygems_3', MagicMock(return_value=False)): + patch.object( + gem, '_has_rubygems_3', MagicMock(return_value=False)): gem.install('rails', pre_releases=True) mock.assert_called_once_with( ['gem', 'install', 'rails', '--no-rdoc', '--no-ri', '--pre'], From aa108d5dd14526cd67d32c48c6adf1ba507f4c5f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 21 Jan 2019 18:18:58 +0000 Subject: [PATCH 10/67] Run tests from tox, wether runtests or pytest --- Gemfile | 7 +- tests/tox-helper.py | 44 +++++++ tests/unit/test_test_module_names.py | 5 +- tox.ini | 188 ++++++++++++++++++++++++++- 4 files changed, 235 insertions(+), 9 deletions(-) create mode 100644 tests/tox-helper.py diff --git a/Gemfile b/Gemfile index 360fb4bfca..b014c8fa1b 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,8 @@ source 'https://rubygems.org' -# Point this back at the test-kitchen package after 1.23.3 is relased -gem 'test-kitchen', :git => 'https://github.com/dwoz/test-kitchen.git', :branch => 'winrm_opts' -gem 'kitchen-salt', '~>0.2' +gem 'test-kitchen', '~>1.23.3' +gem 'kitchen-salt', '~>0.4.1' gem 'kitchen-sync' gem 'git' @@ -14,7 +13,7 @@ end group :windows do gem 'winrm', '~>2.0' - gem 'winrm-fs', '~>1.3.1' + gem 'winrm-fs', '~>1.3.1' end group :ec2 do diff --git a/tests/tox-helper.py b/tests/tox-helper.py new file mode 100644 index 0000000000..98e60fe0f3 --- /dev/null +++ b/tests/tox-helper.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This script exists so that path handling when running tox works for both Linux and Windows + +# Import Python Libs +from __future__ import absolute_import, unicode_literals +import os +import shutil +import argparse +import tempfile + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--rootdir', + default=os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + ) + subparsers = parser.add_subparsers(help='sub-command help', dest='subparser') + + subparsers.add_parser('create-dirs') + subparsers.add_parser('move-artifacts') + + options = parser.parse_args() + if options.subparser == 'create-dirs': + for dirname in ('logs', 'coverage', 'xml-unittests-output'): + path = os.path.join(options.rootdir, 'artifacts', dirname) + if not os.path.exists(path): + os.makedirs(path) + + if options.subparser == 'move-artifacts': + tmp_artifacts_dir = os.path.join(tempfile.gettempdir(), 'artifacts') + if not os.path.exists(tmp_artifacts_dir): + os.makedirs(tmp_artifacts_dir) + + for dirname in ('logs', 'coverage', 'xml-unittests-output'): + src = os.path.join(options.rootdir, 'artifacts', dirname) + dst = os.path.join(tmp_artifacts_dir, dirname) + shutil.copytree(src, dst) + + +if __name__ == '__main__': + main() diff --git a/tests/unit/test_test_module_names.py b/tests/unit/test_test_module_names.py index 485f737685..c601333ac4 100644 --- a/tests/unit/test_test_module_names.py +++ b/tests/unit/test_test_module_names.py @@ -38,6 +38,7 @@ EXCLUDED_FILES = [ os.path.join('tests', 'modparser.py'), os.path.join('tests', 'committer_parser.py'), os.path.join('tests', 'zypp_plugin.py'), + os.path.join('tests', 'tox-helper.py'), os.path.join('tests', 'unit', 'transport', 'mixins.py'), os.path.join('tests', 'integration', 'utils', 'testprogram.py'), ] @@ -57,7 +58,7 @@ class BadTestModuleNamesTestCase(TestCase): excluded_dirs = tuple(EXCLUDED_DIRS) tests_dir = os.path.join(CODE_DIR, 'tests') bad_names = [] - for root, dirs, files in os.walk(tests_dir): + for root, _, files in os.walk(tests_dir): reldir = os.path.relpath(root, CODE_DIR) if reldir.startswith(excluded_dirs) or reldir.endswith('__pycache__'): continue @@ -73,7 +74,7 @@ class BadTestModuleNamesTestCase(TestCase): error_msg = '\n\nPlease rename the following files:\n' for path in bad_names: directory, filename = path.rsplit(os.sep, 1) - filename, ext = os.path.splitext(filename) + filename, _ = os.path.splitext(filename) error_msg += ' {} -> {}/test_{}.py\n'.format(path, directory, filename.split('_test')[0]) error_msg += '\nIf you believe one of the entries above should be ignored, please add it to either\n' diff --git a/tox.ini b/tox.ini index ea50d3d284..d8beec1275 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,194 @@ [tox] -envlist = py27,py34,py35,py36,pylint-salt,pylint-tests +envlist = + py{27,34,35,36}, + py{27,34,35,36}-coverage, + py{27,34,35,36}-pytest, + py{27,34,35,36}-runtests, + py{27,34,35,36}-pytest-coverage, + py{27,34,35,36}-runtests-coverage, + pylint-salt, + pylint-tests skip_missing_interpreters = True skipsdist = True [testenv] deps = -Ur{toxinidir}/requirements/tests.txt -commands = pytest --rootdir {toxinidir} {posargs} +changedir = {toxinidir} +commands_pre = {envpython} tests/tox-helper.py create-dirs passenv = LANG HOME sitepackages = True +commands = {[testenv:runtests]commands} + +[testenv:runtests] +deps = + {[testenv]deps} + unittest-xml-reporting +commands = {envpython} {toxinidir}/tests/runtests.py --tests-logfile={toxinidir}/artifacts/logs/runtests.log {posargs} + +[testenv:pytest] +commands = pytest --rootdir {toxinidir} --log-file={toxinidir}/artifacts/logs/runtests.log {posargs} + +[testenv:runtests-coverage] +commands_pre = + - coverage erase +commands = + coverage run -m tests.runtests {posargs} +commands_post = + - coverage combine + - coverage xml -o {toxinidir}/artifacts/coverage/coverage.xml + +[testenv:pytest-coverage] +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands = coverage run -m py.test --rootdir {toxinidir} {posargs} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py2-pytest] +commands = {[testenv:pytest]commands} + +[testenv:py2-runtests] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests]commands} + +[testenv:py2-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py2-runtests-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py2-pytest-coverage] +commands = {[testenv:pytest-coverage]commands} +commands_pre = {[testenv:pytest-coverage]commands_pre} +commands_post = {[testenv:pytest-coverage]commands_post} + +[testenv:py27-pytest] +commands = {[testenv:pytest]commands} + +[testenv:py27-runtests] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests]commands} + +[testenv:py27-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py27-runtests-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py27-pytest-coverage] +commands = {[testenv:pytest-coverage]commands} +commands_pre = {[testenv:pytest-coverage]commands_pre} +commands_post = {[testenv:pytest-coverage]commands_post} + +[testenv:py3-pytest] +commands = {[testenv:pytest]commands} + +[testenv:py3-runtests] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests]commands} + +[testenv:py3-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py3-runtests-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py3-pytest-coverage] +commands = {[testenv:pytest-coverage]commands} +commands_pre = {[testenv:pytest-coverage]commands_pre} +commands_post = {[testenv:pytest-coverage]commands_post} + + +[testenv:py34-pytest] +commands = {[testenv:pytest]commands} + +[testenv:py34-runtests] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests]commands} + +[testenv:py34-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py34-runtests-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py34-pytest-coverage] +commands = {[testenv:pytest-coverage]commands} +commands_pre = {[testenv:pytest-coverage]commands_pre} +commands_post = {[testenv:pytest-coverage]commands_post} + + +[testenv:py35-pytest] +commands = {[testenv:pytest]commands} + +[testenv:py35-runtests] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests]commands} + +[testenv:py35-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py35-runtests-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py35-pytest-coverage] +commands = {[testenv:pytest-coverage]commands} +commands_pre = {[testenv:pytest-coverage]commands_pre} +commands_post = {[testenv:pytest-coverage]commands_post} + + +[testenv:py36-pytest] +commands = {[testenv:pytest]commands} + +[testenv:py36-runtests] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests]commands} + +[testenv:py36-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py36-runtests-coverage] +deps = {[testenv:runtests]deps} +commands = {[testenv:runtests-coverage]commands} +commands_pre = {[testenv:runtests-coverage]commands_pre} +commands_post = {[testenv:runtests-coverage]commands_post} + +[testenv:py36-pytest-coverage] +commands = {[testenv:pytest-coverage]commands} +commands_pre = {[testenv:pytest-coverage]commands_pre} +commands_post = {[testenv:pytest-coverage]commands_post} + [testenv:pylint-salt] basepython = python2.7 @@ -17,6 +198,7 @@ commands = pylint --rcfile=.testing.pylintrc --disable=I,W1307,C0411,C0413,W8410,str-format-in-logging {posargs:setup.py salt/} sitepackages = False + [testenv:pylint-tests] basepython = python2.7 deps = -r{toxinidir}/requirements/dev.txt @@ -26,6 +208,6 @@ commands = sitepackages = False [pytest] -addopts = --log-file /tmp/salt-runtests.log --no-print-logs --ssh-tests -ra -sv +addopts = --no-print-logs --ssh-tests -ra -sv testpaths = tests norecursedirs = tests/kitchen From 5eec144873c2aa19b1af16424bfc641353af8caf Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 10 Dec 2018 19:02:13 +0000 Subject: [PATCH 11/67] Add `.coveragerc` --- .coveragerc | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..7771c44f33 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,34 @@ +[run] +branch = True +cover_pylib = False +source = + salt +parallel = True +concurrency = multiprocessing +omit = + tests/*.py + setup.py + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + + +ignore_errors = True + +[paths] +source = + salt From 8d2c4d25df9446a38214246c9d0c94a2e0456c3c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 22 Jan 2019 11:00:01 +0000 Subject: [PATCH 12/67] Remove duplicate dependency --- requirements/tests.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/tests.txt b/requirements/tests.txt index a3c3382518..ba3b0e3242 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -28,7 +28,6 @@ PyMySQL; sys.platform != 'win32' and sys.platform != 'darwin' jsonschema strict_rfc3339 rfc3987 -jinja2 pyOpenSSL ioflo dnspython From 97ced3a5ba555006abe9ae2fce135b661233f454 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 22 Jan 2019 13:22:46 +0000 Subject: [PATCH 13/67] Add `pytest-salt-from-filenames` to pytest requirements --- requirements/pytest.txt | 1 + tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/pytest.txt b/requirements/pytest.txt index c27d5cb2ea..7b166d46ee 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -5,3 +5,4 @@ pytest-salt == 2018.12.8 pytest-timeout >= 1.3.3 pytest-tempdir >= 2018.8.11 pytest-helpers-namespace >= 2017.11.11 +pytest-salt-from-filenames >= 2019.1.22 diff --git a/tests/conftest.py b/tests/conftest.py index dba1b42903..57bbab3d5c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,7 +54,7 @@ import salt.log.setup from salt.utils.odict import OrderedDict # Define the pytest plugins we rely on -pytest_plugins = ['tempdir', 'helpers_namespace'] # pylint: disable=invalid-name +pytest_plugins = ['tempdir', 'helpers_namespace', 'salt-from-filenames'] # pylint: disable=invalid-name # Define where not to collect tests from collect_ignore = ['setup.py'] From 9a31be7271084eda1eccf17241a0cb9dac43e7d8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 22 Jan 2019 16:15:11 +0000 Subject: [PATCH 14/67] Don't call `.keys()` on dictionaries --- salt/modules/pillar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index 834d525948..d27e3fabed 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -10,7 +10,6 @@ import collections # Import third party libs import copy import os -import copy import logging import yaml import salt.ext.six as six @@ -154,7 +153,7 @@ def get(key, 'skipped.', default, ret, type(ret).__name__ ) elif isinstance(default, list): - ret = salt.utils.traverse_dict_and_list( + ret = salt.utils.traverse_dict_and_list( # pylint: disable=redefined-variable-type pillar_dict, key, [], @@ -273,6 +272,7 @@ def items(*args, **kwargs): return pillar.compile_pillar() + # Allow pillar.data to also be used to return pillar data data = salt.utils.alias_function(items, 'data') @@ -340,7 +340,7 @@ def ls(*args): salt '*' pillar.ls ''' - return list(items(*args).keys()) + return list(items(*args)) def item(*args, **kwargs): @@ -538,7 +538,7 @@ def keys(key, delimiter=DEFAULT_TARGET_DELIM): if not isinstance(ret, dict): raise ValueError("Pillar value in key {0} is not a dict".format(key)) - return ret.keys() + return list(ret) def file_exists(path, saltenv=None): From 0281fc5b50236cb7259db911cf285d7987f153cd Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 22 Jan 2019 16:15:45 +0000 Subject: [PATCH 15/67] Don't fail the test because of order --- tests/integration/modules/test_supervisord.py | 2 +- tests/unit/modules/test_cassandra.py | 7 ++++-- tests/unit/modules/test_inspect_fsdb.py | 23 +++---------------- tests/unit/modules/test_pillar.py | 5 ++-- tests/unit/output/test_json_out.py | 2 +- tests/unit/states/test_proxy.py | 4 +--- 6 files changed, 14 insertions(+), 29 deletions(-) diff --git a/tests/integration/modules/test_supervisord.py b/tests/integration/modules/test_supervisord.py index 0329baef5b..185235bd51 100644 --- a/tests/integration/modules/test_supervisord.py +++ b/tests/integration/modules/test_supervisord.py @@ -222,7 +222,7 @@ class SupervisordModuleTest(ModuleCase): ret = self.run_function( 'supervisord.status', [], conf_file=self.supervisor_conf, bin_env=self.venv_dir) - self.assertEqual(list(ret.keys()), ['sleep_service', 'sleep_service2']) + self.assertEqual(sorted(ret), ['sleep_service', 'sleep_service2']) def test_status_one(self): ''' diff --git a/tests/unit/modules/test_cassandra.py b/tests/unit/modules/test_cassandra.py index 2c028b7766..fef004b9df 100644 --- a/tests/unit/modules/test_cassandra.py +++ b/tests/unit/modules/test_cassandra.py @@ -118,9 +118,12 @@ class CassandraTestCase(TestCase, LoaderModuleMockMixin): self.assertCountEqual(cassandra.column_families(), {'A': ['a', 'b'], 'B': ['c', 'd']}) else: - self.assertEqual(cassandra.column_families('A'), + self.assertEqual(sorted(cassandra.column_families('A')), ['a', 'b']) - self.assertEqual(cassandra.column_families(), + column_families = cassandra.column_families() + for key in ('A', 'B'): + column_families[key] = sorted(column_families[key]) + self.assertEqual(column_families, {'A': ['a', 'b'], 'B': ['c', 'd']}) def test_column_family_definition(self): diff --git a/tests/unit/modules/test_inspect_fsdb.py b/tests/unit/modules/test_inspect_fsdb.py index 3f0317baef..92afedca2a 100644 --- a/tests/unit/modules/test_inspect_fsdb.py +++ b/tests/unit/modules/test_inspect_fsdb.py @@ -135,26 +135,9 @@ class InspectorFSDBTestCase(TestCase): csvdb.open() csvdb.create_table_from_object(FoobarEntity()) - if six.PY2: - assert writable.data[0].strip() == "foo:int,bar:str,spam:float" - else: - # Order in PY3 is not the same for every run - writable_data = writable.data[0].strip() - assert_order_options = ['bar:str,foo:int,spam:float', - 'bar:str,spam:float,foo:int', - 'foo:int,spam:float,bar:str', - 'foo:int,bar:str,spam:float', - 'spam:float,foo:int,bar:str', - 'spam:float,bar:str,foo:int'] - while assert_order_options: - assert_option = assert_order_options.pop() - try: - assert writable_data == assert_option - break - except AssertionError: - if not assert_order_options: - raise - continue + sorted_writable_data = sorted(writable.data[0].strip().split(',')) + sorted_expected_data = sorted("foo:int,bar:str,spam:float".split(',')) + self.assertEqual(sorted_writable_data, sorted_expected_data) def test_list_databases(self): ''' diff --git a/tests/unit/modules/test_pillar.py b/tests/unit/modules/test_pillar.py index 85225f05b7..7631f5d1e3 100644 --- a/tests/unit/modules/test_pillar.py +++ b/tests/unit/modules/test_pillar.py @@ -54,10 +54,11 @@ class PillarModuleTestCase(TestCase, LoaderModuleMockMixin): @skipIf(NO_MOCK, NO_MOCK_REASON) def test_ls(self): with patch('salt.modules.pillar.items', MagicMock(return_value=pillar_value_1)): + ls = sorted(pillarmod.ls()) if six.PY3: - self.assertCountEqual(pillarmod.ls(), ['a', 'b']) + self.assertCountEqual(ls, ['a', 'b']) else: - self.assertEqual(pillarmod.ls(), ['a', 'b']) + self.assertEqual(ls, ['a', 'b']) @skipIf(NO_MOCK, NO_MOCK_REASON) def test_pillar_get_default_merge(self): diff --git a/tests/unit/output/test_json_out.py b/tests/unit/output/test_json_out.py index 13c9659607..18a95796dc 100644 --- a/tests/unit/output/test_json_out.py +++ b/tests/unit/output/test_json_out.py @@ -66,6 +66,6 @@ class JsonTestCase(TestCase, LoaderModuleMockMixin): '"error": "Unable to serialize output to json"}') ret = json_out.output(data) if six.PY2: - self.assertEqual(expect, ret) + self.assertEqual(sorted(expect), sorted(ret)) else: self.assertEqual(json.loads(ret), data) diff --git a/tests/unit/states/test_proxy.py b/tests/unit/states/test_proxy.py index 30990f614e..f72a454e57 100644 --- a/tests/unit/states/test_proxy.py +++ b/tests/unit/states/test_proxy.py @@ -70,9 +70,7 @@ class ProxyTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(proxy.__salt__, patches): out = proxy.managed('192.168.0.1', '3128', user='frank', password='passw0rd', bypass_domains=['salt.com', 'test.com']) - if six.PY3: - # Sorting is different in Py3 - out['changes']['new'][-1]['bypass_domains'] = sorted(out['changes']['new'][-1]['bypass_domains']) + out['changes']['new'][-1]['bypass_domains'] = sorted(out['changes']['new'][-1]['bypass_domains']) calls = [ call('192.168.0.1', '3128', 'frank', 'passw0rd', 'Ethernet'), From 15ab061b8cf810c6ae14579926ed955fac014bf7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 22 Jan 2019 16:24:44 +0000 Subject: [PATCH 16/67] Show objects on assertion failure --- tests/unit/serializers/test_serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/serializers/test_serializers.py b/tests/unit/serializers/test_serializers.py index 093d887544..71fd4d6020 100644 --- a/tests/unit/serializers/test_serializers.py +++ b/tests/unit/serializers/test_serializers.py @@ -143,7 +143,7 @@ class TestSerializers(TestCase): # BLAAM! yml_src is not valid ! final_obj = OrderedDict(yaml.deserialize(yml_src)) - assert obj != final_obj + assert obj != final_obj, 'Objects matched! {} == {}'.format(obj, final_obj) @skipIf(not yamlex.available, SKIP_MESSAGE % 'sls') def test_sls_aggregate(self): From f67a529b7a9658e4d9f9124630b3299572c23854 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 24 Jan 2019 20:33:08 +0000 Subject: [PATCH 17/67] Remove unused imports --- tests/unit/modules/test_inspect_fsdb.py | 1 - tests/unit/states/test_proxy.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/tests/unit/modules/test_inspect_fsdb.py b/tests/unit/modules/test_inspect_fsdb.py index 92afedca2a..3be383e101 100644 --- a/tests/unit/modules/test_inspect_fsdb.py +++ b/tests/unit/modules/test_inspect_fsdb.py @@ -30,7 +30,6 @@ from salt.modules.inspectlib.fsdb import CsvDB from salt.modules.inspectlib.entities import CsvDBEntity from salt.utils.odict import OrderedDict -import salt.ext.six as six from salt.ext.six.moves import StringIO diff --git a/tests/unit/states/test_proxy.py b/tests/unit/states/test_proxy.py index f72a454e57..780588b6b7 100644 --- a/tests/unit/states/test_proxy.py +++ b/tests/unit/states/test_proxy.py @@ -14,9 +14,6 @@ from tests.support.mock import ( call ) -# Import 3rd-party libs -import salt.ext.six as six - # Import Salt Libs import salt.states.proxy as proxy From 3840af86dfe14b7e4c324861d6f189a0e8729c0e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 25 Jan 2019 12:58:52 +0000 Subject: [PATCH 18/67] "Tell" coverage to track subprocesses. --- tests/support/coverage/sitecustomize.py | 11 +++++++++++ tox.ini | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/support/coverage/sitecustomize.py diff --git a/tests/support/coverage/sitecustomize.py b/tests/support/coverage/sitecustomize.py new file mode 100644 index 0000000000..76f7babf5f --- /dev/null +++ b/tests/support/coverage/sitecustomize.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +''' +Python will always try to import sitecustomize. +We use that fact to try and support code coverage for sub-processes +''' +from __future__ import absolute_import +try: + import coverage + coverage.process_startup() +except ImportError: + pass diff --git a/tox.ini b/tox.ini index d8beec1275..b4ee8d0b87 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,12 @@ commands = {envpython} {toxinidir}/tests/runtests.py --tests-logfile={toxinidir} commands = pytest --rootdir {toxinidir} --log-file={toxinidir}/artifacts/logs/runtests.log {posargs} [testenv:runtests-coverage] +# Add tests/support/coverage to PYTHONPATH in order to get code coverage from subprocesses. +# Additional, set the COVERAGE_PROCESS_START environment variable so that the coverage library +# knows it's supposed to track subprocesses. +setenv = + PYTHONPATH={toxinidir}/tests/support/coverage + COVERAGE_PROCESS_START={toxinidir}/.coveragerc commands_pre = - coverage erase commands = @@ -38,6 +44,7 @@ commands_post = - coverage xml -o {toxinidir}/artifacts/coverage/coverage.xml [testenv:pytest-coverage] +setenv = {[testenv:runtests-coverage]setenv} commands_pre = {[testenv:runtests-coverage]commands_pre} commands = coverage run -m py.test --rootdir {toxinidir} {posargs} commands_post = {[testenv:runtests-coverage]commands_post} @@ -51,17 +58,20 @@ commands = {[testenv:runtests]commands} [testenv:py2-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py2-runtests-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py2-pytest-coverage] +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:pytest-coverage]commands} commands_pre = {[testenv:pytest-coverage]commands_pre} commands_post = {[testenv:pytest-coverage]commands_post} @@ -75,17 +85,20 @@ commands = {[testenv:runtests]commands} [testenv:py27-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py27-runtests-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py27-pytest-coverage] +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:pytest-coverage]commands} commands_pre = {[testenv:pytest-coverage]commands_pre} commands_post = {[testenv:pytest-coverage]commands_post} @@ -99,17 +112,20 @@ commands = {[testenv:runtests]commands} [testenv:py3-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py3-runtests-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py3-pytest-coverage] +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:pytest-coverage]commands} commands_pre = {[testenv:pytest-coverage]commands_pre} commands_post = {[testenv:pytest-coverage]commands_post} @@ -124,17 +140,20 @@ commands = {[testenv:runtests]commands} [testenv:py34-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py34-runtests-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py34-pytest-coverage] +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:pytest-coverage]commands} commands_pre = {[testenv:pytest-coverage]commands_pre} commands_post = {[testenv:pytest-coverage]commands_post} @@ -149,17 +168,20 @@ commands = {[testenv:runtests]commands} [testenv:py35-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py35-runtests-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py35-pytest-coverage] +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:pytest-coverage]commands} commands_pre = {[testenv:pytest-coverage]commands_pre} commands_post = {[testenv:pytest-coverage]commands_post} @@ -174,17 +196,20 @@ commands = {[testenv:runtests]commands} [testenv:py36-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py36-runtests-coverage] deps = {[testenv:runtests]deps} +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:runtests-coverage]commands} commands_pre = {[testenv:runtests-coverage]commands_pre} commands_post = {[testenv:runtests-coverage]commands_post} [testenv:py36-pytest-coverage] +setenv = {[testenv:runtests-coverage]setenv} commands = {[testenv:pytest-coverage]commands} commands_pre = {[testenv:pytest-coverage]commands_pre} commands_post = {[testenv:pytest-coverage]commands_post} From 004d9b828fb397e1edca086745d69deaf68520c3 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 28 Jan 2019 17:15:37 +0000 Subject: [PATCH 19/67] Cloud provider and profile configs are merged --- salt/cloud/__init__.py | 31 ++++++++++++---- tests/unit/cloud/__init__.py | 69 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 745e4e87ee..4e7b0ebc77 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -1338,6 +1338,24 @@ class Cloud(object): output['ret'] = action_out return output + @staticmethod + def vm_config(name, main, provider, profile, overrides): + ''' + Create vm config. + + :param str name: The name of the vm + :param dict main: The main cloud config + :param dict provider: The provider config + :param dict profile: The profile config + :param dict overrides: The vm's config overrides + ''' + vm = main.copy() + vm = salt.utils.dictupdate.update(vm, provider) + vm = salt.utils.dictupdate.update(vm, profile) + vm.update(overrides) + vm['name'] = name + return vm + def extras(self, extra_): ''' Extra actions @@ -1424,12 +1442,13 @@ class Cloud(object): ret[name] = {'Error': msg} continue - vm_ = main_cloud_config.copy() - vm_.update(provider_details) - vm_.update(profile_details) - vm_.update(vm_overrides) - - vm_['name'] = name + vm_ = self.vm_config( + name, + main_cloud_config, + provider_details, + profile_details, + vm_overrides, + ) if self.opts['parallel']: process = multiprocessing.Process( target=self.create, diff --git a/tests/unit/cloud/__init__.py b/tests/unit/cloud/__init__.py index 40a96afc6f..358bc5ba14 100644 --- a/tests/unit/cloud/__init__.py +++ b/tests/unit/cloud/__init__.py @@ -1 +1,70 @@ # -*- coding: utf-8 -*- +''' + tests.unit.cloud + ~~~~~~~~~~~~~~~~ +''' +from __future__ import absolute_import, print_function, unicode_literals +from tests.support.unit import TestCase + +import salt.cloud + +class CloudTest(TestCase): + + def test_vm_config_merger(self): + ''' + Validate the vm's config is generated correctly. + + https://github.com/saltstack/salt/issues/49226 + ''' + main = { + 'minion': {'master': '172.31.39.213'}, + 'log_file': 'var/log/salt/cloud.log', + 'pool_size': 10 + } + provider = { + 'private_key': 'dwoz.pem', + 'grains': {'foo1': 'bar', 'foo2': 'bang'}, + 'availability_zone': 'us-west-2b', + 'driver': 'ec2', + 'ssh_interface': 'private_ips', + 'ssh_username': 'admin', + 'location': 'us-west-2' + } + profile = { + 'profile': 'default', + 'grains': {'meh2': 'bar', 'meh1': 'foo'}, + 'provider': 'ec2-default:ec2', + 'ssh_username': 'admin', + 'image': 'ami-0a1fbca0e5b419fd1', + 'size': 't2.micro' + } + vm = salt.cloud.Cloud.vm_config( + 'test_vm', + main, + provider, + profile, + {} + ) + self.assertEqual({ + 'minion': {'master': '172.31.39.213'}, + 'log_file': 'var/log/salt/cloud.log', + 'pool_size': 10, + 'private_key': 'dwoz.pem', + 'grains': { + 'foo1': 'bar', + 'foo2': 'bang', + 'meh2': 'bar', + 'meh1': 'foo', + }, + 'availability_zone': 'us-west-2b', + 'driver': 'ec2', + 'ssh_interface': 'private_ips', + 'ssh_username': 'admin', + 'location': 'us-west-2', + 'profile': 'default', + 'provider': 'ec2-default:ec2', + 'ssh_username': 'admin', + 'image': 'ami-0a1fbca0e5b419fd1', + 'size': 't2.micro', + 'name': 'test_vm', + }, vm) From a2bbf4dea85d0e777eaea6f5f77c550bec1f058f Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 28 Jan 2019 10:30:35 -0700 Subject: [PATCH 20/67] Fix linter issues --- tests/unit/cloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cloud/__init__.py b/tests/unit/cloud/__init__.py index 358bc5ba14..cae15ad6c5 100644 --- a/tests/unit/cloud/__init__.py +++ b/tests/unit/cloud/__init__.py @@ -8,6 +8,7 @@ from tests.support.unit import TestCase import salt.cloud + class CloudTest(TestCase): def test_vm_config_merger(self): @@ -63,7 +64,6 @@ class CloudTest(TestCase): 'location': 'us-west-2', 'profile': 'default', 'provider': 'ec2-default:ec2', - 'ssh_username': 'admin', 'image': 'ami-0a1fbca0e5b419fd1', 'size': 't2.micro', 'name': 'test_vm', From 0de189d2723822c7f7736cf2fa44b3b1f7adaf79 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 26 Nov 2018 11:10:35 -0700 Subject: [PATCH 21/67] Handle pipenv like version restrictions We need to handle python_version filters in requirement files in order to install cleanly using pip --- setup.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c67e9ac774..a67e93f326 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,8 @@ import os import sys import glob import time +import operator +import platform try: from urllib2 import urlopen except ImportError: @@ -136,6 +138,74 @@ exec(compile(open(SALT_VERSION).read(), SALT_VERSION, 'exec')) # ----- Helper Functions --------------------------------------------------------------------------------------------> + +def _parse_op(op): + ''' + >>> _parse_op('>') + 'gt' + >>> _parse_op('>=') + 'ge' + >>> _parse_op('=>') + 'ge' + >>> _parse_op('=> ') + 'ge' + >>> _parse_op('<') + 'lt' + >>> _parse_op('<=') + 'le' + >>> _parse_op('==') + 'eq' + >>> _parse_op(' <= ') + 'le' + ''' + op = op.strip() + if '>' in op: + if '=' in op: + return 'ge' + else: + return 'gt' + elif '<' in op: + if '=' in op: + return 'le' + else: + return 'lt' + elif '!' in op: + return 'ne' + else: + return 'eq' + + +def _parse_ver(ver): + ''' + >>> _parse_ver("'3.4' # pyzmq 17.1.0 stopped building wheels for python3.4") + '3.4' + >>> _parse_ver('"3.4"') + '3.4' + >>> _parse_ver('"2.6.17"') + '2.6.17' + ''' + if '#' in ver: + ver, _ = ver.split('#', 1) + ver = ver.strip() + return ver.strip('\'').strip('"') + + +def _check_ver(pyver, op, wanted): + ''' + >>> _check_ver('2.7.15', 'gt', '2.7') + True + >>> _check_ver('2.7.15', 'gt', '2.7.15') + False + >>> _check_ver('2.7.15', 'ge', '2.7.15') + True + >>> _check_ver('2.7.15', 'eq', '2.7.15') + True + ''' + pyver = distutils.version.LooseVersion(pyver) + wanted = distutils.version.LooseVersion(wanted) + return getattr(operator, '__{}__'.format(op))(pyver, wanted) + + def _parse_requirements_file(requirements_file): parsed_requirements = [] with open(requirements_file) as rfh: @@ -150,7 +220,16 @@ def _parse_requirements_file(requirements_file): # Python 3 already has futures, installing it will only break # the current python installation whenever futures is imported continue - parsed_requirements.append(line) + try: + pkg, pyverspec = line.rsplit(';', 1) + except ValueError: + pkg, pyverspec = line, '' + pyverspec = pyverspec.strip() + if pyverspec: + _, op, ver = pyverspec.split(' ', 2) + if not _check_ver(platform.python_version(), _parse_op(op), _parse_ver(ver)): + continue + parsed_requirements.append(pkg) return parsed_requirements # <---- Helper Functions --------------------------------------------------------------------------------------------- @@ -362,7 +441,6 @@ class DownloadWindowsDlls(Command): if getattr(self.distribution, 'salt_download_windows_dlls', None) is None: print('This command is not meant to be called on it\'s own') exit(1) - import platform import pip # pip has moved many things to `_internal` starting with pip 10 if LooseVersion(pip.__version__) < LooseVersion('10.0'): @@ -932,7 +1010,6 @@ class SaltDistribution(distutils.dist.Distribution): if IS_WINDOWS_PLATFORM: install_requires = _parse_requirements_file(SALT_WINDOWS_REQS) - return install_requires @property From 43ac36e5ff918d07b2a449d3acbcac2115558122 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 29 Jan 2019 14:24:27 -0700 Subject: [PATCH 22/67] Use gems.github.com no longer exists, use gemcutter.org --- tests/integration/modules/test_gem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/test_gem.py b/tests/integration/modules/test_gem.py index f0a6d048aa..26b21f21a5 100644 --- a/tests/integration/modules/test_gem.py +++ b/tests/integration/modules/test_gem.py @@ -110,7 +110,7 @@ class GemModuleTest(ModuleCase): gem.sources_add gem.sources_remove ''' - source = 'http://gems.github.com' + source = 'http://gemcutter.org/' self.run_function('gem.sources_add', [source]) sources_list = self.run_function('gem.sources_list') From bac652f439dfca8c9c6a46f07aaf00b0638f3cea Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 29 Jan 2019 17:26:04 -0700 Subject: [PATCH 23/67] Add testing path to run_script commands --- tests/support/case.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/support/case.py b/tests/support/case.py index 0e159eef40..b1cc8ec9c4 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -243,9 +243,15 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): script_path = self.get_script_path(script) if not os.path.isfile(script_path): return False + popen_kwargs = popen_kwargs or {} if salt.utils.is_windows(): cmd = 'python ' + if 'cwd' not in popen_kwargs: + popen_kwargs['cwd'] = os.getcwd() + if 'env' not in popen_kwargs: + popen_kwargs['env'] = os.environ.copy() + popen_kwargs['env'][b'PYTHONPATH'] = os.getcwd().encode() else: cmd = 'PYTHONPATH=' python_path = os.environ.get('PYTHONPATH', None) @@ -262,7 +268,6 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): tmp_file = tempfile.SpooledTemporaryFile() - popen_kwargs = popen_kwargs or {} popen_kwargs = dict({ 'shell': True, 'stdout': tmp_file, From 7bb4ceb620cd8f511f6500c7539a29a063e40c50 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 30 Jan 2019 03:13:21 -0700 Subject: [PATCH 24/67] Fix python 3 path for shell test cases --- tests/support/case.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/support/case.py b/tests/support/case.py index 0e159eef40..5f2b013e00 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -246,6 +246,14 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): if salt.utils.is_windows(): cmd = 'python ' + if 'cwd' not in popen_kwargs: + popen_kwargs['cwd'] = os.getcwd() + if 'env' not in popen_kwargs: + popen_kwargs['env'] = os.environ.copy() + if sys.version_info[0] < 3: + popen_kwargs['env'][b'PYTHONPATH'] = os.getcwd().encode() + else: + popen_kwargs['env']['PYTHONPATH'] = os.getcwd() else: cmd = 'PYTHONPATH=' python_path = os.environ.get('PYTHONPATH', None) From e1e06a0f1b1c04c26efd02c63ecab7c471ba9b43 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 30 Jan 2019 03:13:21 -0700 Subject: [PATCH 25/67] Fix python 3 path for shell test cases --- tests/support/case.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/support/case.py b/tests/support/case.py index b1cc8ec9c4..0e2b468446 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -251,7 +251,10 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): popen_kwargs['cwd'] = os.getcwd() if 'env' not in popen_kwargs: popen_kwargs['env'] = os.environ.copy() - popen_kwargs['env'][b'PYTHONPATH'] = os.getcwd().encode() + if sys.version_info[0] < 3: + popen_kwargs['env'][b'PYTHONPATH'] = os.getcwd().encode() + else: + popen_kwargs['env']['PYTHONPATH'] = os.getcwd() else: cmd = 'PYTHONPATH=' python_path = os.environ.get('PYTHONPATH', None) From 1c62e2750e0d4bdebd54f3d4a213f2c5c6e01723 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 30 Jan 2019 11:46:50 +0000 Subject: [PATCH 26/67] Switch required PyTest dependency --- requirements/pytest.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/pytest.txt b/requirements/pytest.txt index 7b166d46ee..1159d11129 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -5,4 +5,4 @@ pytest-salt == 2018.12.8 pytest-timeout >= 1.3.3 pytest-tempdir >= 2018.8.11 pytest-helpers-namespace >= 2017.11.11 -pytest-salt-from-filenames >= 2019.1.22 +pytest-salt-runtests-bridge >= 2019.1.30 From 3a21afa73d771b5b321bd5fe76b9cf6dc8991872 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 30 Jan 2019 11:47:15 +0000 Subject: [PATCH 27/67] Coverage on windows chokes with the tox env var `COVERAGE_PROCESS_START` --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index b4ee8d0b87..6d72c1831f 100644 --- a/tox.ini +++ b/tox.ini @@ -30,11 +30,8 @@ commands = pytest --rootdir {toxinidir} --log-file={toxinidir}/artifacts/logs/ru [testenv:runtests-coverage] # Add tests/support/coverage to PYTHONPATH in order to get code coverage from subprocesses. -# Additional, set the COVERAGE_PROCESS_START environment variable so that the coverage library -# knows it's supposed to track subprocesses. setenv = PYTHONPATH={toxinidir}/tests/support/coverage - COVERAGE_PROCESS_START={toxinidir}/.coveragerc commands_pre = - coverage erase commands = From f3c9cd4166a23467347f74d6ff00b24fcf3c9259 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 30 Jan 2019 11:20:56 -0700 Subject: [PATCH 28/67] Add missing raise statement --- salt/modules/win_dsc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_dsc.py b/salt/modules/win_dsc.py index 9bd06fbb67..6d0b8e2b54 100644 --- a/salt/modules/win_dsc.py +++ b/salt/modules/win_dsc.py @@ -738,7 +738,7 @@ def set_lcm_config(config_mode=None, cmd += ' RefreshFrequencyMins = {0};'.format(refresh_freq) if reboot_if_needed is not None: if not isinstance(reboot_if_needed, bool): - SaltInvocationError('reboot_if_needed must be a boolean value') + raise SaltInvocationError('reboot_if_needed must be a boolean value') if reboot_if_needed: reboot_if_needed = '$true' else: From a8f42a0d423547cf11c3869b64b53fe445e18f29 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 29 Jan 2019 10:32:12 -0800 Subject: [PATCH 29/67] Various documentation fixes. --- doc/topics/cloud/azure.rst | 2 +- doc/topics/index.rst | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/topics/cloud/azure.rst b/doc/topics/cloud/azure.rst index 7599605aa4..3066c39d6e 100644 --- a/doc/topics/cloud/azure.rst +++ b/doc/topics/cloud/azure.rst @@ -532,7 +532,7 @@ services. For more information on service certificates, see the following link: * `Manage Certificates`__ -.. __: https://msdn.microsoft.com/en-us/library/azure/gg981929.aspx +.. __: https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-certs-create The following functions are available. diff --git a/doc/topics/index.rst b/doc/topics/index.rst index f641334537..2e825e5867 100644 --- a/doc/topics/index.rst +++ b/doc/topics/index.rst @@ -163,10 +163,10 @@ A few examples of salt states from the community: * https://github.com/bclermont/states * https://github.com/pcrews/salt-data -Follow on ohloh +Follow on Open Hub =============== -https://www.ohloh.net/p/salt +https://www.openhub.net/p/salt Other community links ===================== @@ -178,6 +178,7 @@ Other community links - `Facebook `_ - `Twitter `_ - `Wikipedia page `_ +- `Stack Overflow `_ Hack the Source =============== From 3582f6e209db2dc3a31e05d3c90c496090495a32 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 31 Jan 2019 10:30:21 -0800 Subject: [PATCH 30/67] Fixing the underline. --- doc/topics/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/index.rst b/doc/topics/index.rst index 2e825e5867..47656ebb4f 100644 --- a/doc/topics/index.rst +++ b/doc/topics/index.rst @@ -164,7 +164,7 @@ A few examples of salt states from the community: * https://github.com/pcrews/salt-data Follow on Open Hub -=============== +================== https://www.openhub.net/p/salt From a3ee70ca3a9b6a3f77e11fb2f8f0e3e484f94eb5 Mon Sep 17 00:00:00 2001 From: Alexey Aksenov Date: Fri, 1 Feb 2019 16:06:21 +0300 Subject: [PATCH 31/67] fix #50556 state.orchestrate_show_sls --- salt/runners/state.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/runners/state.py b/salt/runners/state.py index 573bc666f4..77b059944f 100644 --- a/salt/runners/state.py +++ b/salt/runners/state.py @@ -209,7 +209,7 @@ def orchestrate_high(data, test=None, queue=False, pillar=None, **kwargs): def orchestrate_show_sls(mods, saltenv='base', test=None, - exclude=None, + queue=False, pillar=None, pillarenv=None, pillar_enc=None): @@ -234,12 +234,12 @@ def orchestrate_show_sls(mods, minion = salt.minion.MasterMinion(__opts__) running = minion.functions['state.show_sls']( mods, - saltenv, test, - exclude, + queue, pillar=pillar, pillarenv=pillarenv, - pillar_enc=pillar_enc) + pillar_enc=pillar_enc, + saltenv=saltenv) ret = {minion.opts['id']: running} return ret From 2aaa9f9c950a2c832be11973e547ad023b336d25 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Dec 2018 10:35:09 -0600 Subject: [PATCH 32/67] pip.installed: Fix traceback when _find_key doesn't return a match --- salt/states/pip_state.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 4ebffeeb7d..493d95d661 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -92,7 +92,7 @@ def _find_key(prefix, pip_list): except StopIteration: return None else: - return match + return match.lower() def _fulfills_version_spec(version, version_spec): @@ -913,11 +913,10 @@ def installed(name, ) else: pkg_name = _find_key(prefix, pipsearch) - if pkg_name.lower() in already_installed_packages: - continue - ver = pipsearch[pkg_name] - ret['changes']['{0}=={1}'.format(pkg_name, - ver)] = 'Installed' + if pkg_name is not None \ + and pkg_name not in already_installed_packages: + ver = pipsearch[pkg_name] + ret['changes']['{0}=={1}'.format(pkg_name, ver)] = 'Installed' # Case for packages that are an URL else: ret['changes']['{0}==???'.format(state_name)] = 'Installed' From 5ca9f82ab5c60713254f5e3212b8883860979a93 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 30 Jan 2019 15:13:16 -0600 Subject: [PATCH 33/67] Add a CaseInsensitiveDict implementation --- salt/output/nested.py | 11 +++-- salt/utils/data.py | 99 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 salt/utils/data.py diff --git a/salt/output/nested.py b/salt/output/nested.py index 21983c73b1..2a71028c43 100644 --- a/salt/output/nested.py +++ b/salt/output/nested.py @@ -34,6 +34,11 @@ from salt.ext.six import string_types from salt.utils import get_colors import salt.utils.locales +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping + class NestDisplay(object): ''' @@ -109,7 +114,7 @@ class NestDisplay(object): first_line = False elif isinstance(ret, (list, tuple)): for ind in ret: - if isinstance(ind, (list, tuple, dict)): + if isinstance(ind, (list, tuple, Mapping)): out.append( self.ustring( indent, @@ -117,11 +122,11 @@ class NestDisplay(object): '|_' ) ) - prefix = '' if isinstance(ind, dict) else '- ' + prefix = '' if isinstance(ind, Mapping) else '- ' self.display(ind, indent + 2, prefix, out) else: self.display(ind, indent, '- ', out) - elif isinstance(ret, dict): + elif isinstance(ret, Mapping): if indent: out.append( self.ustring( diff --git a/salt/utils/data.py b/salt/utils/data.py new file mode 100644 index 0000000000..a781befca8 --- /dev/null +++ b/salt/utils/data.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +''' +Functions for manipulating, inspecting, or otherwise working with data types +and data structures. +''' + +from __future__ import absolute_import, print_function, unicode_literals + +try: + from collections.abc import Mapping, MutableMapping, Sequence +except ImportError: + from collections import Mapping, MutableMapping, Sequence + +# Import Salt libs +from salt.utils.odict import OrderedDict + +# Import 3rd-party libs +from salt.ext import six + + +class CaseInsensitiveDict(MutableMapping): + ''' + Inspired by requests' case-insensitive dict implementation, but works with + non-string keys as well. + ''' + def __init__(self, init=None): + ''' + Force internal dict to be ordered to ensure a consistent iteration + order, irrespective of case. + ''' + self._data = OrderedDict() + self.update(init or {}) + + def __len__(self): + return len(self._data) + + def __setitem__(self, key, value): + # Store the case-sensitive key so it is available for dict iteration + self._data[to_lowercase(key)] = (key, value) + + def __delitem__(self, key): + del self._data[to_lowercase(key)] + + def __getitem__(self, key): + return self._data[to_lowercase(key)][1] + + def __iter__(self): + return (item[0] for item in six.itervalues(self._data)) + + def __eq__(self, rval): + if not isinstance(rval, Mapping): + # Comparing to non-mapping type (e.g. int) is always False + return False + return dict(self.items_lower()) == dict(CaseInsensitiveDict(rval).items_lower()) + + def __repr__(self): + return repr(dict(six.iteritems(self))) + + def items_lower(self): + ''' + Returns a generator iterating over keys and values, with the keys all + being lowercase. + ''' + return ((key, val[1]) for key, val in six.iteritems(self._data)) + + def copy(self): + ''' + Returns a copy of the object + ''' + return CaseInsensitiveDict(six.iteritems(self._data)) + + +def __change_case(data, attr, preserve_dict_class=False): + try: + return getattr(data, attr)() + except AttributeError: + pass + + data_type = data.__class__ + + if isinstance(data, Mapping): + return (data_type if preserve_dict_class else dict)( + (__change_case(key, attr, preserve_dict_class), + __change_case(val, attr, preserve_dict_class)) + for key, val in six.iteritems(data) + ) + elif isinstance(data, Sequence): + return data_type( + __change_case(item, attr, preserve_dict_class) for item in data) + else: + return data + + +def to_lowercase(data, preserve_dict_class=False): + return __change_case(data, 'lower', preserve_dict_class) + + +def to_uppercase(data, preserve_dict_class=False): + return __change_case(data, 'upper', preserve_dict_class) From 0e760b59ba2c5d7110e152b0e5c423758c171752 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 30 Jan 2019 15:15:40 -0600 Subject: [PATCH 34/67] pip states: Use case-insensitive dictionaries for pip.list return --- salt/states/pip_state.py | 57 +++++++++++++++------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 493d95d661..86044b9eb4 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -27,6 +27,7 @@ import pkg_resources # Import salt libs import salt.utils +import salt.utils.data from salt.version import SaltStackVersion as _SaltStackVersion from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -81,20 +82,6 @@ def __virtual__(): return False -def _find_key(prefix, pip_list): - ''' - Does a case-insensitive match in the pip_list for the desired package. - ''' - try: - match = next( - iter(x for x in pip_list if x.lower() == prefix.lower()) - ) - except StopIteration: - return None - else: - return match.lower() - - def _fulfills_version_spec(version, version_spec): ''' Check version number against version specification info and return a @@ -210,23 +197,20 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, ret = {'result': False, 'comment': None} # If we are not passed a pip list, get one: - if not pip_list: - pip_list = __salt__['pip.list'](prefix, bin_env=bin_env, - user=user, cwd=cwd, - env_vars=env_vars, **kwargs) - - # Check if the requested package is already installed. - prefix_realname = _find_key(prefix, pip_list) + pip_list = salt.utils.data.CaseInsensitiveDict( + pip_list or __salt__['pip.list'](prefix, bin_env=bin_env, + user=user, cwd=cwd, + env_vars=env_vars, **kwargs) + ) # If the package was already installed, check # the ignore_installed and force_reinstall flags - if ignore_installed is False and prefix_realname is not None: + if ignore_installed is False and prefix in pip_list: if force_reinstall is False and not upgrade: # Check desired version (if any) against currently-installed if ( any(version_spec) and - _fulfills_version_spec(pip_list[prefix_realname], - version_spec) + _fulfills_version_spec(pip_list[prefix], version_spec) ) or (not any(version_spec)): ret['result'] = True ret['comment'] = ('Python package {0} was already ' @@ -246,7 +230,7 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, if 'rc' in spec[1]: include_rc = True available_versions = __salt__['pip.list_all_versions']( - prefix_realname, bin_env=bin_env, include_alpha=include_alpha, + prefix, bin_env=bin_env, include_alpha=include_alpha, include_beta=include_beta, include_rc=include_rc, user=user, cwd=cwd) desired_version = '' @@ -262,9 +246,9 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, ret['comment'] = ('Python package {0} was already ' 'installed and\nthe available upgrade ' 'doesn\'t fulfills the version ' - 'requirements'.format(prefix_realname)) + 'requirements'.format(prefix)) return ret - if _pep440_version_cmp(pip_list[prefix_realname], desired_version) == 0: + if _pep440_version_cmp(pip_list[prefix], desired_version) == 0: ret['result'] = True ret['comment'] = ('Python package {0} was already ' 'installed'.format(state_pkg_name)) @@ -898,10 +882,12 @@ def installed(name, # Case for packages that are not an URL if prefix: - pipsearch = __salt__['pip.list'](prefix, bin_env, - user=user, cwd=cwd, - env_vars=env_vars, - **kwargs) + pipsearch = salt.utils.data.CaseInsensitiveDict( + __salt__['pip.list'](prefix, bin_env, + user=user, cwd=cwd, + env_vars=env_vars, + **kwargs) + ) # If we didn't find the package in the system after # installing it report it @@ -912,11 +898,10 @@ def installed(name, '\'pip.freeze\'.'.format(pkg) ) else: - pkg_name = _find_key(prefix, pipsearch) - if pkg_name is not None \ - and pkg_name not in already_installed_packages: - ver = pipsearch[pkg_name] - ret['changes']['{0}=={1}'.format(pkg_name, ver)] = 'Installed' + if prefix in pipsearch \ + and prefix.lower() not in already_installed_packages: + ver = pipsearch[prefix] + ret['changes']['{0}=={1}'.format(prefix, ver)] = 'Installed' # Case for packages that are an URL else: ret['changes']['{0}==???'.format(state_name)] = 'Installed' From d06526c7a28ceac21dfeb61ce41bc3e5cf9a9118 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 30 Jan 2019 15:35:37 -0600 Subject: [PATCH 35/67] Allow for kwargs to be used in object initialization --- salt/utils/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/data.py b/salt/utils/data.py index a781befca8..d91b384ac4 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -23,13 +23,13 @@ class CaseInsensitiveDict(MutableMapping): Inspired by requests' case-insensitive dict implementation, but works with non-string keys as well. ''' - def __init__(self, init=None): + def __init__(self, init=None, **kwargs): ''' Force internal dict to be ordered to ensure a consistent iteration order, irrespective of case. ''' self._data = OrderedDict() - self.update(init or {}) + self.update(init or {}, **kwargs) def __len__(self): return len(self._data) From 550e9abfe733412f86eb87459c6f089a48a4f342 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 4 Feb 2019 09:57:09 -0600 Subject: [PATCH 36/67] Don't log shell warning if output_loglevel is quiet This will prevent grains generation from logging a bunch of these sort of warnings. --- salt/modules/cmdmod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 69014c6e94..a3b2e406e4 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -280,7 +280,7 @@ def _run(cmd, ''' if 'pillar' in kwargs and not pillar_override: pillar_override = kwargs['pillar'] - if _is_valid_shell(shell) is False: + if output_loglevel != 'quiet' and _is_valid_shell(shell) is False: log.warning( 'Attempt to run a shell command with what may be an invalid shell! ' 'Check to ensure that the shell <%s> is valid for this user.', From 432428d833bcaf385c46fee4043e18c41bc7ea5a Mon Sep 17 00:00:00 2001 From: lomeroe Date: Tue, 5 Feb 2019 12:02:00 -0600 Subject: [PATCH 37/67] Don't assume that each item in "pkgs" is a dict (as packed by pkg.installed state). Fixes #50874 Also, corrects an issue where a trailing space was added to package names when pkgs was passed as a dict (i.e. pkg.installed state) --- salt/modules/solarisips.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/salt/modules/solarisips.py b/salt/modules/solarisips.py index 30016ecf93..96840b530d 100644 --- a/salt/modules/solarisips.py +++ b/salt/modules/solarisips.py @@ -46,6 +46,7 @@ import salt.utils.data import salt.utils.functools import salt.utils.path import salt.utils.pkg +from salt.ext.six import string_types from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -332,7 +333,6 @@ def latest_version(name, **kwargs): return ret return '' - # available_version is being deprecated available_version = salt.utils.functools.alias_function(latest_version, 'available_version') @@ -474,12 +474,16 @@ def install(name=None, refresh=False, pkgs=None, version=None, test=False, **kwa pkg2inst = '' if pkgs: # multiple packages specified + pkg2inst = [] for pkg in pkgs: - if list(pkg.items())[0][1]: # version specified - pkg2inst += '{0}@{1} '.format(list(pkg.items())[0][0], - list(pkg.items())[0][1]) + if getattr(pkg, 'items', False): + if list(pkg.items())[0][1]: # version specified + pkg2inst.append('{0}@{1}'.format(list(pkg.items())[0][0], + list(pkg.items())[0][1])) + else: + pkg2inst.append(list(pkg.items())[0][0]) else: - pkg2inst += '{0} '.format(list(pkg.items())[0][0]) + pkg2inst.append("{0}".format(pkg)) log.debug('Installing these packages instead of %s: %s', name, pkg2inst) @@ -499,7 +503,10 @@ def install(name=None, refresh=False, pkgs=None, version=None, test=False, **kwa # Install or upgrade the package # If package is already installed - cmd.append(pkg2inst) + if isinstance(pkg2inst, string_types): + cmd.append(pkg2inst) + elif isinstance(pkg2inst, list): + cmd = cmd + pkg2inst out = __salt__['cmd.run_all'](cmd, output_loglevel='trace') From eaa229dde597d3bff74e8218147335a3c4d5aeb0 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Wed, 6 Feb 2019 11:44:32 -0600 Subject: [PATCH 38/67] add unit test --- tests/unit/modules/test_solarisips.py | 177 ++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 tests/unit/modules/test_solarisips.py diff --git a/tests/unit/modules/test_solarisips.py b/tests/unit/modules/test_solarisips.py new file mode 100644 index 0000000000..8dbfc87e27 --- /dev/null +++ b/tests/unit/modules/test_solarisips.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- + +# Import Python Libs +from __future__ import absolute_import +import os + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + Mock, + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + +# Import Salt libs +import salt.modules.solarisips as solarisips +import salt.modules.pkg_resource as pkg_resource +import salt.utils.data + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class IpsTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.solarisips + ''' + def setup_loader_modules(self): + self.opts = opts = salt.config.DEFAULT_MINION_OPTS + utils = salt.loader.utils( + opts, + whitelist=['pkg', 'path', 'platform']) + return { + pkg_resource: { + '__grains__': { + 'osarch': 'sparcv9', + 'os_family': 'Solaris', + 'osmajorrelease': 11, + 'kernelrelease': 5.11, + }, + }, + solarisips: { + '__opts__': opts, + '__utils__': utils, + } + } + + def test_install_single_package(self): + ''' + Test installing a single package + ''' + pkg_list_pre = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + } + pkg_list_post = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + 'pkg://solaris/text/less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + } + install_cmd = { + 'pid': 1234, + 'retcode': 0, + 'stderr': '', + 'stdout': '', + } + mock_install_cmd = MagicMock(return_value=install_cmd) + list_pkgs_responses = [pkg_list_pre, pkg_list_post] + with patch.object(solarisips, 'is_installed', return_value=False), \ + patch.object(solarisips, 'list_pkgs', side_effect=list_pkgs_responses), \ + patch.dict(solarisips.__salt__, {'cmd.run_all': mock_install_cmd}): + result = solarisips.install(name='less', refresh=False) + self.assertEqual(result, salt.utils.data.compare_dicts(pkg_list_pre, pkg_list_post)) + + def test_install_list_pkgs(self): + ''' + Test installing a list of packages + ''' + pkg_list_pre = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + } + pkg_list_post = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + 'pkg://solaris/text/less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + 'pkg://solaris/system/library/security/libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z', + } + install_cmd = { + 'pid': 1234, + 'retcode': 0, + 'stderr': '', + 'stdout': '', + } + mock_install_cmd = MagicMock(return_value=install_cmd) + list_pkgs_responses = [pkg_list_pre, pkg_list_post] + with patch.object(solarisips, 'is_installed', return_value=False), \ + patch.object(solarisips, 'list_pkgs', side_effect=list_pkgs_responses), \ + patch.dict(solarisips.__salt__, {'cmd.run_all': mock_install_cmd}): + result = solarisips.install(pkgs=['less', 'libsasl'], refresh=False) + self.assertEqual(result, salt.utils.data.compare_dicts(pkg_list_pre, pkg_list_post)) + + def test_install_dict_pkgs_no_version(self): + ''' + Test installing a list of packages + ''' + pkg_list_pre = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + } + pkg_list_post = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + 'pkg://solaris/text/less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + 'pkg://solaris/system/library/security/libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z', + } + install_cmd = { + 'pid': 1234, + 'retcode': 0, + 'stderr': '', + 'stdout': '', + } + mock_install_cmd = MagicMock(return_value=install_cmd) + list_pkgs_responses = [pkg_list_pre, pkg_list_post] + with patch.object(solarisips, 'is_installed', return_value=False), \ + patch.object(solarisips, 'list_pkgs', side_effect=list_pkgs_responses), \ + patch.dict(solarisips.__salt__, {'cmd.run_all': mock_install_cmd}): + result = solarisips.install(pkgs=[{'less': ''}, {'libsasl': ''}], refresh=False) + self.assertEqual(result, salt.utils.data.compare_dicts(pkg_list_pre, pkg_list_post)) + + def test_install_dict_pkgs_with_version(self): + ''' + Test installing a list of packages + ''' + pkg_list_pre = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + } + pkg_list_post = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + 'pkg://solaris/text/less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + 'pkg://solaris/system/library/security/libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z', + } + install_cmd = { + 'pid': 1234, + 'retcode': 0, + 'stderr': '', + 'stdout': '', + } + mock_install_cmd = MagicMock(return_value=install_cmd) + list_pkgs_responses = [pkg_list_pre, pkg_list_post] + with patch.object(solarisips, 'is_installed', return_value=False), \ + patch.object(solarisips, 'list_pkgs', side_effect=list_pkgs_responses), \ + patch.dict(solarisips.__salt__, {'cmd.run_all': mock_install_cmd}): + result = solarisips.install(pkgs=[ + {'less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z'}, + {'libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z'}], refresh=False) + self.assertEqual(result, salt.utils.data.compare_dicts(pkg_list_pre, pkg_list_post)) + + def test_install_already_installed_single_pkg(self): + ''' + Test installing a package that is already installed + ''' + result = None + expected_result = 'Package already installed.' + with patch.object(solarisips, 'is_installed', return_value=True): + result = solarisips.install(name='less') + self.assertEqual(result, expected_result) \ No newline at end of file From cdaccf46798f1d1d7c4c14adc74e68d9ecc70e7a Mon Sep 17 00:00:00 2001 From: lomeroe Date: Wed, 6 Feb 2019 12:30:23 -0600 Subject: [PATCH 39/67] add tests to validate command being sent to cmd.run_all --- tests/unit/modules/test_solarisips.py | 123 +++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/tests/unit/modules/test_solarisips.py b/tests/unit/modules/test_solarisips.py index 8dbfc87e27..4a7345ec29 100644 --- a/tests/unit/modules/test_solarisips.py +++ b/tests/unit/modules/test_solarisips.py @@ -174,4 +174,125 @@ class IpsTestCase(TestCase, LoaderModuleMockMixin): expected_result = 'Package already installed.' with patch.object(solarisips, 'is_installed', return_value=True): result = solarisips.install(name='less') - self.assertEqual(result, expected_result) \ No newline at end of file + self.assertEqual(result, expected_result) + + def test_install_dict_pkgs_with_version_validate_cmd(self): + ''' + Test installing a list of packages + ''' + def check_param(arg, **kwargs): + self.assertEqual(arg, [ + 'pkg', + 'install', + '-v', + '--accept', + 'less@458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + 'libsasl@0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z' + ]) + return { + 'pid': 1234, + 'retcode': 0, + 'stderr': '', + 'stdout': '', + } + + pkg_list_pre = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + } + pkg_list_post = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + 'pkg://solaris/text/less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + 'pkg://solaris/system/library/security/libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z', + } + mock_install_cmd = MagicMock(side_effect=check_param) + list_pkgs_responses = [pkg_list_pre, pkg_list_post] + with patch.object(solarisips, 'is_installed', return_value=False), \ + patch.object(solarisips, 'list_pkgs', side_effect=list_pkgs_responses): + with patch.dict(solarisips.__salt__, {'cmd.run_all': mock_install_cmd}): + result = solarisips.install(pkgs=[ + {'less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z'}, + {'libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z'}], refresh=False) + + def test_install_dict_pkgs_no_version_validate_cmd(self): + ''' + Test installing a list of packages + ''' + def check_param(arg, **kwargs): + self.assertEqual(arg, [ + 'pkg', + 'install', + '-v', + '--accept', + 'less', + 'libsasl' + ]) + return { + 'pid': 1234, + 'retcode': 0, + 'stderr': '', + 'stdout': '', + } + + pkg_list_pre = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + } + pkg_list_post = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + 'pkg://solaris/text/less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + 'pkg://solaris/system/library/security/libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z', + } + mock_install_cmd = MagicMock(side_effect=check_param) + list_pkgs_responses = [pkg_list_pre, pkg_list_post] + with patch.object(solarisips, 'is_installed', return_value=False), \ + patch.object(solarisips, 'list_pkgs', side_effect=list_pkgs_responses): + with patch.dict(solarisips.__salt__, {'cmd.run_all': mock_install_cmd}): + result = solarisips.install(pkgs=[ + {'less': ''}, + {'libsasl': ''}], refresh=False) + + def test_install_list_pkgs_validate_cmd(self): + ''' + Test installing a list of packages + ''' + def check_param(arg, **kwargs): + self.assertEqual(arg, [ + 'pkg', + 'install', + '-v', + '--accept', + 'less', + 'libsasl' + ]) + return { + 'pid': 1234, + 'retcode': 0, + 'stderr': '', + 'stdout': '', + } + + pkg_list_pre = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + } + pkg_list_post = { + 'pkg://solaris/compress/bzip2': '1.0.6,5.11-0.175.3.10.0.4.0:20160630T215500Z', + 'pkg://solaris/compress/gzip': '1.5,5.11-0.175.3.0.0.30.0:20150821T161446Z', + 'pkg://solaris/compress/p7zip': '16.2.3,5.11-0.175.3.34.0.2.0:20180614T204908Z', + 'pkg://solaris/text/less': '458,5.11-0.175.3.0.0.30.0:20150821T172730Z', + 'pkg://solaris/system/library/security/libsasl': '0.5.11,5.11-0.175.3.32.0.1.0:20180406T191209Z', + } + mock_install_cmd = MagicMock(side_effect=check_param) + list_pkgs_responses = [pkg_list_pre, pkg_list_post] + with patch.object(solarisips, 'is_installed', return_value=False), \ + patch.object(solarisips, 'list_pkgs', side_effect=list_pkgs_responses): + with patch.dict(solarisips.__salt__, {'cmd.run_all': mock_install_cmd}): + result = solarisips.install(pkgs=['less', 'libsasl'], refresh=False) From b7bfe1766b740580b052ec49907020962d0c71ea Mon Sep 17 00:00:00 2001 From: lomeroe Date: Thu, 7 Feb 2019 08:48:06 -0600 Subject: [PATCH 40/67] lint fixes --- salt/modules/solarisips.py | 1 + tests/unit/modules/test_solarisips.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/solarisips.py b/salt/modules/solarisips.py index 96840b530d..ca939525d6 100644 --- a/salt/modules/solarisips.py +++ b/salt/modules/solarisips.py @@ -333,6 +333,7 @@ def latest_version(name, **kwargs): return ret return '' + # available_version is being deprecated available_version = salt.utils.functools.alias_function(latest_version, 'available_version') diff --git a/tests/unit/modules/test_solarisips.py b/tests/unit/modules/test_solarisips.py index 4a7345ec29..29dadc1e85 100644 --- a/tests/unit/modules/test_solarisips.py +++ b/tests/unit/modules/test_solarisips.py @@ -2,13 +2,11 @@ # Import Python Libs from __future__ import absolute_import -import os # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import ( - Mock, MagicMock, patch, NO_MOCK, @@ -20,6 +18,7 @@ import salt.modules.solarisips as solarisips import salt.modules.pkg_resource as pkg_resource import salt.utils.data + @skipIf(NO_MOCK, NO_MOCK_REASON) class IpsTestCase(TestCase, LoaderModuleMockMixin): ''' From f9d682851cb2a697d2df04ea0df3b7ce8536619e Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 11 Feb 2019 09:32:40 -0800 Subject: [PATCH 41/67] Fixing missed merge. --- salt/utils/data.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/salt/utils/data.py b/salt/utils/data.py index 88f676892b..613dadb8f4 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -6,40 +6,31 @@ and data structures. from __future__ import absolute_import, print_function, unicode_literals -<<<<<<< HEAD # Import Python libs import copy import fnmatch import logging import re -======= ->>>>>>> 2017.7 try: from collections.abc import Mapping, MutableMapping, Sequence except ImportError: from collections import Mapping, MutableMapping, Sequence # Import Salt libs -<<<<<<< HEAD import salt.utils.dictupdate import salt.utils.stringutils import salt.utils.yaml from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import SaltException from salt.utils.decorators.jinja import jinja_filter -======= ->>>>>>> 2017.7 from salt.utils.odict import OrderedDict # Import 3rd-party libs from salt.ext import six -<<<<<<< HEAD from salt.ext.six.moves import range # pylint: disable=redefined-builtin log = logging.getLogger(__name__) -======= ->>>>>>> 2017.7 class CaseInsensitiveDict(MutableMapping): @@ -121,7 +112,6 @@ def to_lowercase(data, preserve_dict_class=False): def to_uppercase(data, preserve_dict_class=False): return __change_case(data, 'upper', preserve_dict_class) -<<<<<<< HEAD @jinja_filter('compare_dicts') @@ -978,5 +968,3 @@ def stringify(data): item = six.text_type(item) ret.append(item) return ret -======= ->>>>>>> 2017.7 From cbac390a7dcbe5f8114d163184df5cf15e8b8aca Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 11 Feb 2019 14:47:19 -0700 Subject: [PATCH 42/67] Use the code directory instead of cwd for python path --- tests/support/case.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/support/case.py b/tests/support/case.py index 96be004492..700393bebf 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -276,9 +276,9 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): if 'env' not in popen_kwargs: popen_kwargs['env'] = os.environ.copy() if sys.version_info[0] < 3: - popen_kwargs['env'][b'PYTHONPATH'] = os.getcwd().encode() + popen_kwargs['env'][b'PYTHONPATH'] = CODE_DIR.encode() else: - popen_kwargs['env']['PYTHONPATH'] = os.getcwd() + popen_kwargs['env']['PYTHONPATH'] = CODE_DIR else: cmd = 'PYTHONPATH=' python_path = os.environ.get('PYTHONPATH', None) From e1dcbb541e5ef2d5e1a7a24f7aebbdbb562f1e55 Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Thu, 7 Feb 2019 11:47:28 +0100 Subject: [PATCH 43/67] Silence linux_distribution deprecation warning The salt master log is flooded with deprecation warnings: ``` Feb 07 10:45:34 debian salt-master[1657]: [WARNING ] /usr/lib/python3/dist-packages/salt/grains/core.py:1759: DeprecationWarning: dist() and linux_distribution() functions are deprecated in Python 3.5 Feb 07 10:45:34 debian salt-master[1657]: linux_distribution(supported_dists=_supported_dists)] Feb 07 10:45:34 debian salt-master[1657]: [WARNING ] /usr/lib/python3/dist-packages/salt/grains/core.py:1759: DeprecationWarning: dist() and linux_distribution() functions are deprecated in Python 3.5 ``` Since the import statement already falls back to use distro.linux_distribution, silence the deprecation warning. Bug-Debian: https://bugs.debian.org/921630 Signed-off-by: Benjamin Drung --- salt/grains/core.py | 8 +++++++- salt/version.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index acda07ddd7..9ca8341660 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -22,6 +22,7 @@ import locale import uuid from errno import EACCES, EPERM import datetime +import warnings __proxyenabled__ = ['*'] __FQDN__ = None @@ -34,7 +35,12 @@ _supported_dists += ('arch', 'mageia', 'meego', 'vmware', 'bluewhite64', # linux_distribution deprecated in py3.7 try: - from platform import linux_distribution + from platform import linux_distribution as _deprecated_linux_distribution + + def linux_distribution(**kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return _deprecated_linux_distribution(**kwargs) except ImportError: from distro import linux_distribution diff --git a/salt/version.py b/salt/version.py index 715a9eb43e..af08d6495b 100644 --- a/salt/version.py +++ b/salt/version.py @@ -8,10 +8,16 @@ from __future__ import absolute_import, print_function, unicode_literals import re import sys import platform +import warnings # linux_distribution deprecated in py3.7 try: - from platform import linux_distribution + from platform import linux_distribution as _deprecated_linux_distribution + + def linux_distribution(**kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return _deprecated_linux_distribution(**kwargs) except ImportError: from distro import linux_distribution From 7678c28be7c5f43c132101edeb5d8343613e2b92 Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Fri, 8 Feb 2019 10:51:05 +0100 Subject: [PATCH 44/67] Do not load zyppnotify file on module import The call imp.load_source() could fail (i.e. when the specified zyppnotify does not exist). To prevent an import failure in that case, move the loading of the zyppnotify file into the test case. Signed-off-by: Benjamin Drung --- tests/unit/test_zypp_plugins.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_zypp_plugins.py b/tests/unit/test_zypp_plugins.py index e4d37a661b..a18f048489 100644 --- a/tests/unit/test_zypp_plugins.py +++ b/tests/unit/test_zypp_plugins.py @@ -25,8 +25,10 @@ if sys.version_info >= (3,): else: BUILTINS_OPEN = '__builtin__.open' -zyppnotify = imp.load_source('zyppnotify', os.path.sep.join(os.path.dirname(__file__).split( - os.path.sep)[:-2] + ['scripts', 'suse', 'zypper', 'plugins', 'commit', 'zyppnotify'])) +ZYPPNOTIFY_FILE = os.path.sep.join( + os.path.dirname(__file__).split(os.path.sep)[:-2] + + ['scripts', 'suse', 'zypper', 'plugins', 'commit', 'zyppnotify'] +) @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -40,6 +42,7 @@ class ZyppPluginsTestCase(TestCase): Returns: ''' + zyppnotify = imp.load_source('zyppnotify', ZYPPNOTIFY_FILE) drift = zyppnotify.DriftDetector() drift._get_mtime = MagicMock(return_value=123) drift._get_checksum = MagicMock(return_value='deadbeef') From 997ce2650c6eb372a15e0d1ab1bec48cc5e37777 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 12 Feb 2019 08:56:48 -0600 Subject: [PATCH 45/67] Add documentation for the metadata_server_grains config option --- doc/ref/configuration/minion.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 06bfbeea41..326f5ebc84 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -803,6 +803,22 @@ A value of 10 minutes is a reasonable default. grains_refresh_every: 0 +.. conf_minion:: metadata_server_grains + +``metadata_server_grains`` +-------------------------- + +.. versionadded:: 2017.7.0 + +Default: ``False`` + +Set this option to enable gathering of cloud metadata from +``http://169.254.169.254/latest`` for use in grains. + +.. code-block:: yaml + + metadata_server_grains: True + .. conf_minion:: fibre_channel_grains ``fibre_channel_grains`` From 9c02104feac2feb1c32c84f168efb29a551aaf1b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 12 Feb 2019 09:01:19 -0600 Subject: [PATCH 46/67] Add link to metadata grain module docs --- doc/ref/configuration/minion.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 326f5ebc84..ade63dec1e 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -813,7 +813,8 @@ A value of 10 minutes is a reasonable default. Default: ``False`` Set this option to enable gathering of cloud metadata from -``http://169.254.169.254/latest`` for use in grains. +``http://169.254.169.254/latest`` for use in grains (see :py:mod:`here +` for more information). .. code-block:: yaml From 4ee94409a18b27a53a520ed69f5ba79d66964293 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 12 Feb 2019 09:12:40 -0600 Subject: [PATCH 47/67] Improve ec2 pillar documentation This also incorporates some docs proposed by @mr337 in #50860. --- salt/pillar/ec2_pillar.py | 51 +++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/salt/pillar/ec2_pillar.py b/salt/pillar/ec2_pillar.py index ac3781cf36..f129fb3290 100644 --- a/salt/pillar/ec2_pillar.py +++ b/salt/pillar/ec2_pillar.py @@ -2,26 +2,36 @@ ''' Retrieve EC2 instance data for minions for ec2_tags and ec2_tags_list -The minion id must be the AWS instance-id or value in 'tag_match_key'. -For example set 'tag_match_key' to 'Name', to have the minion-id matched against the -tag 'Name'. The tag contents must be unique. The value of tag_match_value can -be 'uqdn' or 'asis'. if 'uqdn' strips any domain before comparison. +The minion id must be the AWS instance-id or value in ``tag_match_key``. For +example set ``tag_match_key`` to ``Name`` to have the minion-id matched against +the tag 'Name'. The tag contents must be unique. The value of +``tag_match_value`` can be 'uqdn' or 'asis'. if 'uqdn', then the domain will be +stripped before comparison. -The option use_grain can be set to True. This allows the use of an -instance-id grain instead of the minion-id. Since this is a potential -security risk, the configuration can be further expanded to include -a list of minions that are trusted to only allow the alternate id -of the instances to specific hosts. There is no glob matching at -this time. +Additionally, the ``use_grain`` option can be set to ``True``. This allows the +use of an instance-id grain instead of the minion-id. Since this is a potential +security risk, the configuration can be further expanded to include a list of +minions that are trusted to only allow the alternate id of the instances to +specific hosts. There is no glob matching at this time. -The optional 'tag_list_key' indicates which keys should be added to -'ec2_tags_list' and be split by tag_list_sep (default `;`). If a tag key is -included in 'tag_list_key' it is removed from ec2_tags. If a tag does not -exist it is still included as an empty list. +.. note:: + If you are using ``use_grain: True`` in the configuration for this external + pillar module, the minion must have :conf_minion:`metadata_server_grains` + enabled in the minion config file (see also :py:mod:`here + `). + + It is important to also note that enabling the ``use_grain`` option allows + the minion to manipulate the pillar data returned, as described above. + +The optional ``tag_list_key`` indicates which keys should be added to +``ec2_tags_list`` and be split by ``tag_list_sep`` (by default ``;``). If a tag +key is included in ``tag_list_key`` it is removed from ec2_tags. If a tag does +not exist it is still included as an empty list. - Note: restart the salt-master for changes to take effect. - +..note:: + As with any master configuration change, restart the salt-master daemon for + changes to take effect. .. code-block:: yaml @@ -38,11 +48,10 @@ exist it is still included as an empty list. - trusted-minion-2 - trusted-minion-3 -This is a very simple pillar that simply retrieves the instance data -from AWS. Currently the only portion implemented are EC2 tags, which -returns a list of key/value pairs for all of the EC2 tags assigned to -the instance. - +This is a very simple pillar configuration that simply retrieves the instance +data from AWS. Currently the only portion implemented are EC2 tags, which +returns a list of key/value pairs for all of the EC2 tags assigned to the +instance. ''' # Import python libs From ed194c7f3c5ad0a349eb53b2c1b52502ed376ebc Mon Sep 17 00:00:00 2001 From: Dafydd Jones Date: Tue, 5 Feb 2019 15:34:39 +0000 Subject: [PATCH 48/67] grains: assign os_family Debian to Debian derivative TurnKey Linux --- salt/grains/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index acda07ddd7..76766d7d16 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1389,7 +1389,9 @@ _OS_FAMILY_MAP = { 'KDE neon': 'Debian', 'Void': 'Void', 'IDMS': 'Debian', - 'AIX': 'AIX' + 'Funtoo': 'Gentoo', + 'AIX': 'AIX', + 'TurnKey': 'Debian', } # Matches any possible format: From 3eee0383c6e1d9e2d06c753af26bba5c3dea4084 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 12 Feb 2019 09:34:03 -0800 Subject: [PATCH 49/67] Fixing a couple issues that did not merge properly. --- salt/modules/pillar.py | 2 +- salt/output/nested.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index 3896e326f7..6327f7ab5e 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -156,7 +156,7 @@ def get(key, 'skipped.', default, ret, type(ret).__name__ ) elif isinstance(default, list): - ret = salt.utils.traverse_dict_and_list( # pylint: disable=redefined-variable-type + ret = salt.utils.data.traverse_dict_and_list( # pylint: disable=redefined-variable-type pillar_dict, key, [], diff --git a/salt/output/nested.py b/salt/output/nested.py index 0f1af83329..d32c8e7696 100644 --- a/salt/output/nested.py +++ b/salt/output/nested.py @@ -39,11 +39,6 @@ try: except ImportError: from collections import Mapping -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping - class NestDisplay(object): ''' From d6720d2d144e95a1916acc5916f2852507ee20bc Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 12 Feb 2019 18:01:52 +0000 Subject: [PATCH 50/67] Wait for minions to be pingable before starting tests --- tests/integration/__init__.py | 99 +++++++---------------------------- 1 file changed, 20 insertions(+), 79 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 86b3a8d5dc..926503335f 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -183,7 +183,7 @@ class TestDaemon(object): ''' Set up the master and minion daemons, and run related cases ''' - MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 120 + MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 300 def __init__(self, parser): self.parser = parser @@ -219,6 +219,8 @@ class TestDaemon(object): if getattr(self.parser.options, 'ssh', False): self.prep_ssh() + self.wait_for_minions(time.time(), self.MINIONS_CONNECT_TIMEOUT) + if self.parser.options.sysinfo: try: print_header( @@ -1185,84 +1187,6 @@ class TestDaemon(object): k for (k, v) in six.iteritems(running) if v and v[0]['jid'] == jid ] - def wait_for_minion_connections(self, targets, timeout): - salt.utils.process.appendproctitle('WaitForMinionConnections') - sys.stdout.write( - ' {LIGHT_BLUE}*{ENDC} Waiting at most {0} for minions({1}) to ' - 'connect back\n'.format( - (timeout > 60 and - timedelta(seconds=timeout) or - '{0} secs'.format(timeout)), - ', '.join(targets), - **self.colors - ) - ) - sys.stdout.flush() - expected_connections = set(targets) - now = datetime.now() - expire = now + timedelta(seconds=timeout) - while now <= expire: - sys.stdout.write( - '\r{0}\r'.format( - ' ' * getattr(self.parser.options, 'output_columns', PNUM) - ) - ) - sys.stdout.write( - ' * {LIGHT_YELLOW}[Quit in {0}]{ENDC} Waiting for {1}'.format( - '{0}'.format(expire - now).rsplit('.', 1)[0], - ', '.join(expected_connections), - **self.colors - ) - ) - sys.stdout.flush() - - try: - responses = self.client.cmd( - list(expected_connections), 'test.ping', tgt_type='list', - ) - # we'll get this exception if the master process hasn't finished starting yet - except SaltClientError: - time.sleep(0.1) - now = datetime.now() - continue - for target in responses: - if target not in expected_connections: - # Someone(minion) else "listening"? - continue - expected_connections.remove(target) - sys.stdout.write( - '\r{0}\r'.format( - ' ' * getattr(self.parser.options, 'output_columns', - PNUM) - ) - ) - sys.stdout.write( - ' {LIGHT_GREEN}*{ENDC} {0} connected.\n'.format( - target, **self.colors - ) - ) - sys.stdout.flush() - - if not expected_connections: - return - - time.sleep(1) - now = datetime.now() - else: # pylint: disable=W0120 - print( - '\n {LIGHT_RED}*{ENDC} WARNING: Minions failed to connect ' - 'back. Tests requiring them WILL fail'.format(**self.colors) - ) - try: - print_header( - '=', sep='=', inline=True, - width=getattr(self.parser.options, 'output_columns', PNUM) - - ) - except TypeError: - print_header('=', sep='=', inline=True) - raise SystemExit() - def sync_minion_modules_(self, modules_kind, targets, timeout=None): if not timeout: timeout = 120 @@ -1339,3 +1263,20 @@ class TestDaemon(object): def sync_minion_grains(self, targets, timeout=None): salt.utils.process.appendproctitle('SyncMinionGrains') self.sync_minion_modules_('grains', targets, timeout=timeout) + + def wait_for_minions(self, start, timeout, sleep=5): + ''' + Ensure all minions and masters (including sub-masters) are connected. + ''' + while True: + try: + ret = self.client.run_job('*', 'test.ping') + except salt.exceptions.SaltClientError: + ret = None + if ret and 'minions' not in ret: + continue + if ret and sorted(ret['minions']) == ['minion', 'sub_minion']: + break + if time.time() - start >= timeout: + raise RuntimeError("Ping Minions Failed") + time.sleep(sleep) From d794edb5158a4a2be1042d3c35c2c1cb852b3cd0 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 12 Feb 2019 11:13:42 -0700 Subject: [PATCH 51/67] fix linter --- tests/integration/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 926503335f..845eaa0faa 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1275,7 +1275,7 @@ class TestDaemon(object): ret = None if ret and 'minions' not in ret: continue - if ret and sorted(ret['minions']) == ['minion', 'sub_minion']: + if ret and sorted(ret['minions']) == ['minion', 'sub_minion']: break if time.time() - start >= timeout: raise RuntimeError("Ping Minions Failed") From 7376cb9151a12adcdcbfe8d8d1c7cbf817863c4b Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 12 Feb 2019 14:38:20 -0700 Subject: [PATCH 52/67] Handle explicit lists properly --- salt/modules/win_lgpo.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 3c469dc833..28a9fea37c 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -5605,7 +5605,7 @@ def _getDataFromRegPolData(search_string, policy_data, return_value_name=False): ) ].split(encoded_semicolon) if len(pol_entry) >= 2: - valueName = pol_entry[1] + valueName = pol_entry[1].decode('utf-16-le').rstrip(chr(0)) if len(pol_entry) >= 5: value = pol_entry[4] if vtype == 'REG_DWORD' or vtype == 'REG_QWORD': @@ -5923,18 +5923,18 @@ def _processValueItem(element, reg_key, reg_valuename, policy, parent_element, ']'.encode('utf-16-le')]) if 'expandable' in element.attrib: this_vtype = 'REG_EXPAND_SZ' - if 'explicitValue' in element.attrib and element.attrib['explicitValue'].lower() == 'true': + if element.attrib.get('explicitValue', 'false').lower() == 'true': if this_element_value is not None: - element_valuenames = this_element_value.keys() - element_values = this_element_value.values() - if 'valuePrefix' in element.attrib: + element_valuenames = [str(k) for k in this_element_value.keys()] + element_values = [str(v) for v in this_element_value.values()] + elif 'valuePrefix' in element.attrib: # if the valuePrefix attribute exists, the valuenames are # most prefixes attributes are empty in the admx files, so the valuenames # end up being just numbers if element.attrib['valuePrefix'] != '': if this_element_value is not None: - element_valuenames = ['{0}{1}'.format(element.attrib['valuePrefix'], - k) for k in element_valuenames] + element_valuenames = ['{0}{1}'.format( + element.attrib['valuePrefix'], k) for k in element_valuenames] else: # if there is no valuePrefix attribute, the valuename is the value if element_values is not None: From 5342305725b91142fc569281fe66b14249b95f15 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 7 Feb 2019 13:29:18 +0000 Subject: [PATCH 53/67] Allow NOT to install pip dependencies --- pkg/windows/build_env_2.ps1 | 65 +++++++++++++++++++++---------- pkg/windows/build_env_3.ps1 | 78 +++++++++++++++++++++++++------------ pkg/windows/req.txt | 3 +- pkg/windows/req_win.txt | 2 + 4 files changed, 101 insertions(+), 47 deletions(-) create mode 100644 pkg/windows/req_win.txt diff --git a/pkg/windows/build_env_2.ps1 b/pkg/windows/build_env_2.ps1 index 914017da73..13e405410e 100644 --- a/pkg/windows/build_env_2.ps1 +++ b/pkg/windows/build_env_2.ps1 @@ -22,22 +22,10 @@ # Load parameters param( - [switch]$Silent + [switch]$Silent, + [switch]$NoPipDependencies ) -Write-Output "=================================================================" -Write-Output "" -Write-Output " Development Environment Installation" -Write-Output "" -Write-Output " - Installs All Salt Dependencies" -Write-Output " - Detects 32/64 bit Architectures" -Write-Output "" -Write-Output " To run silently add -Silent" -Write-Output " eg: dev_env.ps1 -Silent" -Write-Output "" -Write-Output "=================================================================" -Write-Output "" - #============================================================================== # Get the Directory of actual script #============================================================================== @@ -49,6 +37,22 @@ $script_path = $script_path.DirectoryName #============================================================================== $script_name = $MyInvocation.MyCommand.Name +Write-Output "=================================================================" +Write-Output "" +Write-Output " Development Environment Installation" +Write-Output "" +Write-Output " - Installs All Salt Dependencies" +Write-Output " - Detects 32/64 bit Architectures" +Write-Output "" +Write-Output " To run silently add -Silent" +Write-Output " eg: ${script_name} -Silent" +Write-Output "" +Write-Output " To run skip installing pip dependencies add -NoPipDependencies" +Write-Output " eg: ${script_name} -NoPipDependencies" +Write-Output "" +Write-Output "=================================================================" +Write-Output "" + #============================================================================== # Import Modules #============================================================================== @@ -211,25 +215,46 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) { } #============================================================================== -# Install pypi resources using pip +# Install windows specific pypi resources using pip # caching depends on environment variable SALT_REQ_LOCAL_CACHE #============================================================================== Write-Output " ----------------------------------------------------------------" -Write-Output " - $script_name :: Installing pypi resources using pip . . ." +Write-Output " - $script_name :: Installing windows specific pypi resources using pip . . ." Write-Output " ----------------------------------------------------------------" if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_win.txt" "pip install" } else { if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) { # folder empty - Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download" + Write-Output " pip download from req_win.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip download" } Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE" Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip install" } +#============================================================================== +# Install pypi resources using pip +# caching depends on environment variable SALT_REQ_LOCAL_CACHE +#============================================================================== +If ($NoPipDependencies -eq $false) { + Write-Output " ----------------------------------------------------------------" + Write-Output " - $script_name :: Installing pypi resources using pip . . ." + Write-Output " ----------------------------------------------------------------" + if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install" + } else { + if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) { + # folder empty + Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download" + } + Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE" + Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install" + } +} #============================================================================== # Cleaning Up PyWin32 #============================================================================== diff --git a/pkg/windows/build_env_3.ps1 b/pkg/windows/build_env_3.ps1 index 01fc428b8f..20b05b6cc1 100644 --- a/pkg/windows/build_env_3.ps1 +++ b/pkg/windows/build_env_3.ps1 @@ -22,22 +22,10 @@ # Load parameters param( - [switch]$Silent + [switch]$Silent, + [switch]$NoPipDependencies ) -Write-Output "=================================================================" -Write-Output "" -Write-Output " Development Environment Installation" -Write-Output "" -Write-Output " - Installs All Salt Dependencies" -Write-Output " - Detects 32/64 bit Architectures" -Write-Output "" -Write-Output " To run silently add -Silent" -Write-Output " eg: dev_env.ps1 -Silent" -Write-Output "" -Write-Output "=================================================================" -Write-Output "" - #============================================================================== # Get the Directory of actual script #============================================================================== @@ -49,6 +37,22 @@ $script_path = $script_path.DirectoryName #============================================================================== $script_name = $MyInvocation.MyCommand.Name +Write-Output "=================================================================" +Write-Output "" +Write-Output " Development Environment Installation" +Write-Output "" +Write-Output " - Installs All Salt Dependencies" +Write-Output " - Detects 32/64 bit Architectures" +Write-Output "" +Write-Output " To run silently add -Silent" +Write-Output " eg: ${script_name} -Silent" +Write-Output "" +Write-Output " To run skip installing pip dependencies add -NoPipDependencies" +Write-Output " eg: ${script_name} -NoPipDependencies" +Write-Output "" +Write-Output "=================================================================" +Write-Output "" + #============================================================================== # Import Modules #============================================================================== @@ -211,23 +215,45 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) { } #============================================================================== -# Install pypi resources using pip +# Install windows specific pypi resources using pip # caching depends on environment variable SALT_REQ_LOCAL_CACHE #============================================================================== Write-Output " ----------------------------------------------------------------" -Write-Output " - $script_name :: Installing pypi resources using pip . . ." +Write-Output " - $script_name :: Installing windows specific pypi resources using pip . . ." Write-Output " ----------------------------------------------------------------" if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_win.txt" "pip install" } else { if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) { # folder empty - Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download" + Write-Output " pip download from req_win.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip download" } Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE" Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip install" +} + +#============================================================================== +# Install pypi resources using pip +# caching depends on environment variable SALT_REQ_LOCAL_CACHE +#============================================================================== +If ($NoPipDependencies -eq $false) { + Write-Output " ----------------------------------------------------------------" + Write-Output " - $script_name :: Installing pypi resources using pip . . ." + Write-Output " ----------------------------------------------------------------" + if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install" + } else { + if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) { + # folder empty + Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download" + } + Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE" + Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install" + } } #============================================================================== @@ -262,11 +288,13 @@ Remove-Item "$($ini['Settings']['Scripts3Dir'])\pywin32_*" -Force -Recurse #============================================================================== # Fix PyCrypto #============================================================================== -Write-Output " ----------------------------------------------------------------" -Write-Output " - $script_name :: Fixing PyCrypto . . ." -Write-Output " ----------------------------------------------------------------" -$nt_file = "$($ini['Settings']['Python3Dir'])\Lib\site-packages\Crypto\Random\OSRNG\nt.py" -(Get-Content $nt_file) | Foreach-Object {$_ -replace '^import winrandom$', 'from Crypto.Random.OSRNG import winrandom'} | Set-Content $nt_file +If ($NoPipDependencies -eq $false) { + Write-Output " ----------------------------------------------------------------" + Write-Output " - $script_name :: Fixing PyCrypto . . ." + Write-Output " ----------------------------------------------------------------" + $nt_file = "$($ini['Settings']['Python3Dir'])\Lib\site-packages\Crypto\Random\OSRNG\nt.py" + (Get-Content $nt_file) | Foreach-Object {$_ -replace '^import winrandom$', 'from Crypto.Random.OSRNG import winrandom'} | Set-Content $nt_file +} #============================================================================== # Copy DLLs to Python Directory diff --git a/pkg/windows/req.txt b/pkg/windows/req.txt index 998164da01..24eef93852 100644 --- a/pkg/windows/req.txt +++ b/pkg/windows/req.txt @@ -1,3 +1,4 @@ +-r req_win.txt backports-abc==0.5 backports.ssl-match-hostname==3.5.0.1 certifi @@ -28,7 +29,6 @@ pyOpenSSL==17.5.0 python-dateutil==2.6.1 python-gnupg==0.4.1 pythonnet==2.3.0 -pywin32==223 PyYAML==3.12 pyzmq==16.0.3 requests==2.21.0 @@ -37,4 +37,3 @@ smmap==0.9.0 timelib==0.2.4 tornado==4.5.1 wheel==0.30.0a0 -WMI==1.4.9 diff --git a/pkg/windows/req_win.txt b/pkg/windows/req_win.txt new file mode 100644 index 0000000000..3b4bca7f4c --- /dev/null +++ b/pkg/windows/req_win.txt @@ -0,0 +1,2 @@ +pywin32==223 +WMI==1.4.9 From d263410e46c02b059142b0bd28d209c11cdc9d59 Mon Sep 17 00:00:00 2001 From: Jochen Breuer Date: Wed, 13 Feb 2019 16:50:34 +0100 Subject: [PATCH 54/67] Prevents crash when there is no job entry Seems like in rare race conditions it might happen that there is no job entry. This prevents Salt from crashing at that point. --- salt/returners/local_cache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 9eeb044c8d..7fedb7167b 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -77,6 +77,9 @@ def _walk_through(job_dir): except Exception: log.exception('Failed to deserialize %s', load_path) continue + if not job: + log.error('Deserialization of job succeded but there is no data in %s', load_path) + continue jid = job['jid'] yield jid, job, t_path, final From 1c6d4fbc7435a026fb558efc1b26906080fb691d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 12 Feb 2019 11:47:08 -0800 Subject: [PATCH 55/67] When looping through the schedule to remove hidden attributes, we should only do that if the item in question is a dictionary. It could be the attribute that determines if the entire scheudle is enabled or disable. --- salt/utils/schedule.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 310b400598..8d2947649f 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -160,9 +160,10 @@ class Schedule(object): if remove_hidden: _schedule = copy.deepcopy(schedule) for job in _schedule: - for item in _schedule[job]: - if item.startswith('_'): - del schedule[job][item] + if isinstance(_schedule[job], dict): + for item in _schedule[job]: + if item.startswith('_'): + del schedule[job][item] return schedule def _check_max_running(self, func, data, opts): From 086066b75f457441a5ef857375ffa8a4189140bc Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 12 Feb 2019 12:29:53 -0800 Subject: [PATCH 56/67] Adding a test to ensure _get_schedule which is used by the save functionality works when there is an enabled attribute. --- tests/integration/scheduler/test_helpers.py | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tests/integration/scheduler/test_helpers.py diff --git a/tests/integration/scheduler/test_helpers.py b/tests/integration/scheduler/test_helpers.py new file mode 100644 index 0000000000..137d23cb7c --- /dev/null +++ b/tests/integration/scheduler/test_helpers.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +# Import Python libs +from __future__ import absolute_import +import copy +import datetime +import logging +import os +import random +import time + +import dateutil.parser as dateutil_parser +import datetime + +# Import Salt Testing libs +from tests.support.case import ModuleCase +from tests.support.mixins import SaltReturnAssertsMixin + +# Import Salt Testing Libs +from tests.support.mock import MagicMock, patch +from tests.support.unit import skipIf +import tests.integration as integration + +# Import Salt libs +import salt.utils.schedule +import salt.utils.platform + +from salt.modules.test import ping as ping + +try: + import croniter # pylint: disable=W0611 + HAS_CRONITER = True +except ImportError: + HAS_CRONITER = False + +log = logging.getLogger(__name__) +ROOT_DIR = os.path.join(integration.TMP, 'schedule-unit-tests') +SOCK_DIR = os.path.join(ROOT_DIR, 'test-socks') + +DEFAULT_CONFIG = salt.config.minion_config(None) +DEFAULT_CONFIG['conf_dir'] = ROOT_DIR +DEFAULT_CONFIG['root_dir'] = ROOT_DIR +DEFAULT_CONFIG['sock_dir'] = SOCK_DIR +DEFAULT_CONFIG['pki_dir'] = os.path.join(ROOT_DIR, 'pki') +DEFAULT_CONFIG['cachedir'] = os.path.join(ROOT_DIR, 'cache') + + +class SchedulerHelpersTest(ModuleCase, SaltReturnAssertsMixin): + ''' + Test scheduler helper functions + ''' + def setUp(self): + with patch('salt.utils.schedule.clean_proc_dir', MagicMock(return_value=None)): + functions = {'test.ping': ping} + self.schedule = salt.utils.schedule.Schedule(copy.deepcopy(DEFAULT_CONFIG), functions, returners={}) + self.schedule.opts['loop_interval'] = 1 + + def tearDown(self): + self.schedule.reset() + + def test_get_schedule(self): + ''' + verify that the _get_schedule function works + when remove_hidden is True and schedule data + contains enabled key + ''' + job_name = 'test_get_schedule' + job = { + 'schedule': { + 'enabled': True, + job_name: { + 'function': 'test.ping', + 'seconds': 60 + } + } + } + # Add the job to the scheduler + self.schedule.opts.update(job) + + ret = self.schedule._get_schedule(remove_hidden=True) + self.assertEqual(job['schedule'], ret) From 1552fba0bca304d7de5d63754b4b5a54aa8e7cc5 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 12 Feb 2019 12:34:36 -0800 Subject: [PATCH 57/67] lint cleanup --- tests/integration/scheduler/test_helpers.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/integration/scheduler/test_helpers.py b/tests/integration/scheduler/test_helpers.py index 137d23cb7c..91a8e06f1e 100644 --- a/tests/integration/scheduler/test_helpers.py +++ b/tests/integration/scheduler/test_helpers.py @@ -3,14 +3,8 @@ # Import Python libs from __future__ import absolute_import import copy -import datetime import logging import os -import random -import time - -import dateutil.parser as dateutil_parser -import datetime # Import Salt Testing libs from tests.support.case import ModuleCase @@ -18,7 +12,6 @@ from tests.support.mixins import SaltReturnAssertsMixin # Import Salt Testing Libs from tests.support.mock import MagicMock, patch -from tests.support.unit import skipIf import tests.integration as integration # Import Salt libs @@ -27,12 +20,6 @@ import salt.utils.platform from salt.modules.test import ping as ping -try: - import croniter # pylint: disable=W0611 - HAS_CRONITER = True -except ImportError: - HAS_CRONITER = False - log = logging.getLogger(__name__) ROOT_DIR = os.path.join(integration.TMP, 'schedule-unit-tests') SOCK_DIR = os.path.join(ROOT_DIR, 'test-socks') From 07601333367ed8de9777242adab22f1ceea28d0a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 13 Feb 2019 09:27:10 -0800 Subject: [PATCH 58/67] adding integration.scheduler.test_helpers to various files to by pass the test_module_name_source_match check. --- tests/filename_map.yml | 1 + tests/unit/test_module_names.py | 1 + tests/whitelist.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/filename_map.yml b/tests/filename_map.yml index cc5808e94d..085c1c1103 100644 --- a/tests/filename_map.yml +++ b/tests/filename_map.yml @@ -220,6 +220,7 @@ salt/utils/schedule.py: - integration.scheduler.test_postpone - integration.scheduler.test_skip - integration.scheduler.test_maxrunning + - integration.scheduler.test_helpers salt/utils/vt.py: - integration.cli.test_custom_module diff --git a/tests/unit/test_module_names.py b/tests/unit/test_module_names.py index feae262eff..04f7d47f67 100644 --- a/tests/unit/test_module_names.py +++ b/tests/unit/test_module_names.py @@ -148,6 +148,7 @@ class BadTestModuleNamesTestCase(TestCase): 'integration.scheduler.test_postpone', 'integration.scheduler.test_skip', 'integration.scheduler.test_maxrunning', + 'integration.scheduler.test_helpers', 'integration.shell.test_spm', 'integration.shell.test_cp', 'integration.shell.test_syndic', diff --git a/tests/whitelist.txt b/tests/whitelist.txt index d33a719ce5..6bf0ac8625 100644 --- a/tests/whitelist.txt +++ b/tests/whitelist.txt @@ -72,6 +72,7 @@ integration.runners.test_salt integration.scheduler.test_eval integration.scheduler.test_postpone integration.scheduler.test_skip +integration.scheduler.test_helpers integration.sdb.test_env integration.shell.test_arguments integration.shell.test_auth From b2c9c3b7589e829caf8c6fbb94581aef8d10e302 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 13 Feb 2019 11:21:19 -0700 Subject: [PATCH 59/67] Use makedirs instead of mkdir --- salt/modules/win_lgpo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 3c469dc833..216b7b569c 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -5083,7 +5083,7 @@ def _findOptionValueAdvAudit(option): field_names = _get_audit_defaults('fieldnames') # If the file doesn't exist anywhere, create it with default # fieldnames - __salt__['file.mkdir'](os.path.dirname(f_audit)) + __salt__['file.makedirs'](f_audit) __salt__['file.write'](f_audit, ','.join(field_names)) audit_settings = {} @@ -5187,7 +5187,7 @@ def _set_audit_file_data(option, value): # Copy the temporary csv file over the existing audit.csv in both # locations if a value was written __salt__['file.copy'](f_temp.name, f_audit, remove_existing=True) - __salt__['file.mkdir'](os.path.dirname(f_audit_gpo)) + __salt__['file.makedirs'](f_audit_gpo) __salt__['file.copy'](f_temp.name, f_audit_gpo, remove_existing=True) finally: f_temp.close() From e3d3c5a8e2c1fce6ae52cba736985a70203057bc Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 13 Feb 2019 12:22:36 -0700 Subject: [PATCH 60/67] Add tests for issue 51309 --- tests/unit/utils/test_filebuffer.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/utils/test_filebuffer.py b/tests/unit/utils/test_filebuffer.py index c3147ff1af..3608f1a8cd 100644 --- a/tests/unit/utils/test_filebuffer.py +++ b/tests/unit/utils/test_filebuffer.py @@ -9,11 +9,14 @@ # Import Python libs from __future__ import absolute_import, unicode_literals, print_function +import os # Import Salt Testing libs from tests.support.unit import TestCase +from tests.support.helpers import generate_random_name # Import salt libs +import salt.modules.cmdmod as cmdmod from salt.utils.filebuffer import BufferedReader, InvalidFileMode @@ -30,3 +33,26 @@ class TestFileBuffer(TestCase): with self.assertRaises(InvalidFileMode): BufferedReader('/tmp/foo', mode='wb') + + def test_issue_51309(self): + ''' + https://github.com/saltstack/salt/issues/51309 + ''' + temp_name = os.path.join(os.environ.get('TEMP'), + generate_random_name(prefix='salt-test-')) + cmd = 'tzutil /l > {0}'.format(temp_name) + cmdmod.run(cmd=cmd, python_shell=True) + + def find_value(text): + stripped_text = text.strip() + try: + with BufferedReader(temp_name) as breader: + for chunk in breader: + if stripped_text in chunk: + return True + return False + except (IOError, OSError): + return False + + self.assertTrue(find_value('(UTC) Coordinated Universal Time')) + os.remove(temp_name) From 30c18324a461850132f597ba776cfe387e12f4fe Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 13 Feb 2019 12:40:19 -0700 Subject: [PATCH 61/67] Use tempfile to get temp dir --- tests/unit/utils/test_filebuffer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/test_filebuffer.py b/tests/unit/utils/test_filebuffer.py index 3608f1a8cd..3bfc7c6051 100644 --- a/tests/unit/utils/test_filebuffer.py +++ b/tests/unit/utils/test_filebuffer.py @@ -10,6 +10,7 @@ # Import Python libs from __future__ import absolute_import, unicode_literals, print_function import os +import tempfile # Import Salt Testing libs from tests.support.unit import TestCase @@ -38,7 +39,7 @@ class TestFileBuffer(TestCase): ''' https://github.com/saltstack/salt/issues/51309 ''' - temp_name = os.path.join(os.environ.get('TEMP'), + temp_name = os.path.join(tempfile.gettempdir(), generate_random_name(prefix='salt-test-')) cmd = 'tzutil /l > {0}'.format(temp_name) cmdmod.run(cmd=cmd, python_shell=True) From 5ed8eb648e68023bc95f5e4733d81b70e87b5eb9 Mon Sep 17 00:00:00 2001 From: lomeroe Date: Wed, 13 Feb 2019 15:24:45 -0600 Subject: [PATCH 62/67] Properly create the value for a "True" boolean element item Closes #51634 --- salt/modules/win_lgpo.py | 2 +- tests/integration/modules/test_win_lgpo.py | 57 ++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 3c469dc833..6da87d010a 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -5857,7 +5857,7 @@ def _processValueItem(element, reg_key, reg_valuename, policy, parent_element, check_deleted = True if not check_deleted: this_vtype = 'REG_DWORD' - this_element_value = chr(1).encode('utf-16-le') + this_element_value = struct.pack('I', 1) standard_element_expected_string = False elif etree.QName(element).localname == 'decimal': # https://msdn.microsoft.com/en-us/library/dn605987(v=vs.85).aspx diff --git a/tests/integration/modules/test_win_lgpo.py b/tests/integration/modules/test_win_lgpo.py index b87fb7c6f9..24e477cdad 100644 --- a/tests/integration/modules/test_win_lgpo.py +++ b/tests/integration/modules/test_win_lgpo.py @@ -260,6 +260,63 @@ class WinLgpoTest(ModuleCase): ''' Test setting/unsetting/changing WindowsUpdate policy ''' + the_policy = { + 'Configure automatic updating': '4 - Auto download and schedule the install', + 'Install during automatic maintenance': False, + 'Scheduled install day': '7 - Every Saturday', + 'Scheduled install time': '17:00', + 'Install updates for other Microsoft products': True + } + the_policy_check = [ + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*NoAutoUpdate[\s]*DWORD:0', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*AUOptions[\s]*DWORD:4', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*AutomaticMaintenanceEnabled[\s]*DELETE', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*ScheduledInstallDay[\s]*DWORD:7', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*ScheduledInstallTime[\s]*DWORD:17', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*AllowMUUpdateService[\s]*DWORD:1\s*' + ] + + # Configure Automatic Updates has different options in 2016 than in 2012 + # and has only one boolean item, so we'll test it "False" in this block + # and then "True" in next block + if self.osrelease in ['2012Server', '2012ServerR2']: + the_policy = { + 'Configure automatic updating': '4 - Auto download and schedule the install', + 'Install during automatic maintenance': False, + 'Schedule install day': '7 - Every Saturday', + 'Schedule install time': '17:00', + } + the_policy_check = [ + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*NoAutoUpdate[\s]*DWORD:0', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*AUOptions[\s]*DWORD:4', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*AutomaticMaintenanceEnabled[\s]*DELETE', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*ScheduledInstallDay[\s]*DWORD:7', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*ScheduledInstallTime[\s]*DWORD:17', + ] + # test as False + self._testComputerAdmxPolicy(r'Windows Components\Windows Update\Configure Automatic Updates', + the_policy, + the_policy_check) + # configure as True for "enable Automatic Updates" test below + the_policy = { + 'Configure automatic updating': '4 - Auto download and schedule the install', + 'Install during automatic maintenance': True, + 'Schedule install day': '7 - Every Saturday', + 'Schedule install time': '17:00', + } + the_policy_check = [ + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*NoAutoUpdate[\s]*DWORD:0', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*AUOptions[\s]*DWORD:4', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*AutomaticMaintenanceEnabled[\s]*DWORD:1\s*', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*ScheduledInstallDay[\s]*DWORD:7', + r'Computer[\s]*Software\\Policies\\Microsoft\\Windows\\WindowsUpdate\\AU[\s]*ScheduledInstallTime[\s]*DWORD:17', + ] + + # enable Automatic Updates + self._testComputerAdmxPolicy(r'Windows Components\Windows Update\Configure Automatic Updates', + the_policy, + the_policy_check) + # disable Configure Automatic Updates self._testComputerAdmxPolicy(r'Windows Components\Windows Update\Configure Automatic Updates', 'Disabled', From 9e3a01b9302eb10854aa0505ebdbeb9e1e16604e Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 13 Feb 2019 15:32:16 -0700 Subject: [PATCH 63/67] Use existing file instead of tzutil --- tests/unit/utils/test_filebuffer.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/unit/utils/test_filebuffer.py b/tests/unit/utils/test_filebuffer.py index 3bfc7c6051..c335ff05da 100644 --- a/tests/unit/utils/test_filebuffer.py +++ b/tests/unit/utils/test_filebuffer.py @@ -10,14 +10,12 @@ # Import Python libs from __future__ import absolute_import, unicode_literals, print_function import os -import tempfile # Import Salt Testing libs from tests.support.unit import TestCase -from tests.support.helpers import generate_random_name +from tests.support.paths import BASE_FILES # Import salt libs -import salt.modules.cmdmod as cmdmod from salt.utils.filebuffer import BufferedReader, InvalidFileMode @@ -39,15 +37,12 @@ class TestFileBuffer(TestCase): ''' https://github.com/saltstack/salt/issues/51309 ''' - temp_name = os.path.join(tempfile.gettempdir(), - generate_random_name(prefix='salt-test-')) - cmd = 'tzutil /l > {0}'.format(temp_name) - cmdmod.run(cmd=cmd, python_shell=True) + file_name = os.path.join(BASE_FILES, 'grail', 'scene33') def find_value(text): stripped_text = text.strip() try: - with BufferedReader(temp_name) as breader: + with BufferedReader(file_name) as breader: for chunk in breader: if stripped_text in chunk: return True @@ -55,5 +50,4 @@ class TestFileBuffer(TestCase): except (IOError, OSError): return False - self.assertTrue(find_value('(UTC) Coordinated Universal Time')) - os.remove(temp_name) + self.assertTrue(find_value('We have the Holy Hand Grenade')) From 11681412afec264c67ec01cdb7ce42d3d6d175a1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 13 Feb 2019 16:59:07 -0700 Subject: [PATCH 64/67] Add test for explicit lists --- tests/integration/modules/test_win_lgpo.py | 55 +++++++++++++++++----- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/tests/integration/modules/test_win_lgpo.py b/tests/integration/modules/test_win_lgpo.py index b87fb7c6f9..a36066da95 100644 --- a/tests/integration/modules/test_win_lgpo.py +++ b/tests/integration/modules/test_win_lgpo.py @@ -123,25 +123,25 @@ class WinLgpoTest(ModuleCase): ret = self.run_function('lgpo.set_computer_policy', (policy_name, policy_config)) log.debug('lgpo set_computer_policy ret == %s', ret) + cmd = ['lgpo.exe', + '/parse', + '/m', + r'c:\Windows\System32\GroupPolicy\Machine\Registry.pol'] if assert_true: self.assertTrue(ret) - lgpo_output = self.run_function( - 'cmd.run', - (), - cmd='lgpo.exe /parse /m c:\\Windows\\System32\\GroupPolicy\\Machine\\Registry.pol') + lgpo_output = self.run_function('cmd.run', (), cmd=' '.join(cmd)) # validate that the lgpo output doesn't say the format is invalid self.assertIsNone( - re.search( - r'Invalid file format\.', - lgpo_output, - re.IGNORECASE), 'Failed validating Registry.pol file format') + re.search(r'Invalid file format\.', lgpo_output, re.IGNORECASE), + msg='Failed validating Registry.pol file format') # validate that the regexes we expect are in the output for expected_regex in expected_regexes: - match = re.search( - expected_regex, - lgpo_output, - re.IGNORECASE) - self.assertIsNotNone(match, 'Failed validating policy "{0}" configuration, regex "{1}" not found in lgpo output'.format(policy_name, expected_regex)) + match = re.search(expected_regex, lgpo_output, re.IGNORECASE) + self.assertIsNotNone( + match, + msg='Failed validating policy "{0}" configuration, regex ' + '"{1}" not found in lgpo output:\n{2}' + ''.format(policy_name, expected_regex, lgpo_output)) else: # expecting it to fail self.assertNotEqual(ret, True) @@ -255,6 +255,35 @@ class WinLgpoTest(ModuleCase): 'Not Configured', [r'; Source file: c:\\windows\\system32\\grouppolicy\\machine\\registry.pol[\s]*; PARSING COMPLETED.']) + @destructiveTest + def test_set_computer_policy_Pol_HardenedPaths(self): + # Disable Pol_HardenedPaths + log.debug('Attempting to disable Pol_HardenedPaths') + self._testComputerAdmxPolicy( + 'Pol_HardenedPaths', + 'Disabled', + [r'Computer[\s]*Software\\policies\\Microsoft\\Windows\\NetworkProvider\\HardenedPaths[\s]*\*[\s]*DELETEALLVALUES']) + # Configure Pol_HardenedPaths + log.debug('Attempting to configure Pol_HardenedPaths') + self._testComputerAdmxPolicy( + 'Pol_HardenedPaths', + { + 'Hardened UNC Paths': { + r'\\*\NETLOGON': 'RequireMutualAuthentication=1, RequireIntegrity=1', + r'\\*\SYSVOL': 'RequireMutualAuthentication=1, RequireIntegrity=1' + } + }, + [ + r'Computer[\s]*Software\\policies\\Microsoft\\Windows\\NetworkProvider\\HardenedPaths[\s]*\\\\\*\\NETLOGON[\s]*SZ:RequireMutualAuthentication=1, RequireIntegrity=1[\s]*', + r'Computer[\s]*Software\\policies\\Microsoft\\Windows\\NetworkProvider\\HardenedPaths[\s]*\\\\\*\\SYSVOL[\s]*SZ:RequireMutualAuthentication=1, RequireIntegrity=1[\s]*', + ]) + # Not Configure Pol_HardenedPaths + log.debug('Attempting to set Pol_HardenedPaths to Not Configured') + self._testComputerAdmxPolicy( + 'Pol_HardenedPaths', + 'Not Configured', + [r'; Source file: c:\\windows\\system32\\grouppolicy\\machine\\registry.pol[\s]*; PARSING COMPLETED.']) + @destructiveTest def test_set_computer_policy_WindowsUpdate(self): ''' From d8d2b863a5c6bdd446728562ee787a5ffe0cffae Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 13 Feb 2019 17:31:13 -0700 Subject: [PATCH 65/67] backport #50887 --- salt/states/file.py | 8 +++++--- tests/integration/states/test_file.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 2feb98f48b..475c3579d0 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -781,10 +781,12 @@ def _check_directory_win(name, if not os.path.isdir(name): changes = {name: {'directory': 'new'}} else: - # Check owner + # Check owner by SID if win_owner is not None: - owner = salt.utils.win_dacl.get_owner(name) - if not owner.lower() == win_owner.lower(): + current_owner = salt.utils.win_dacl.get_owner(name) + current_owner_sid = salt.utils.win_functions.get_sid_from_name(current_owner) + expected_owner_sid = salt.utils.win_functions.get_sid_from_name(win_owner) + if not current_owner_sid == expected_owner_sid: changes['owner'] = win_owner # Check perms diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index f4fca4c887..34df82cf9d 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -889,6 +889,31 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): self.assertFalse(os.path.exists(straydir)) self.assertTrue(os.path.isdir(name)) + def test_directory_is_idempotent(self): + ''' + Ensure the file.directory state produces no changes when rerun. + ''' + name = os.path.join(TMP, 'a_dir_twice') + + if IS_WINDOWS: + username = os.environ.get('USERNAME', 'Administrators') + domain = os.environ.get('USERDOMAIN', '') + fullname = '{0}\\{1}'.format(domain, username) + + ret = self.run_state('file.directory', name=name, win_owner=fullname) + else: + ret = self.run_state('file.directory', name=name) + + self.assertSaltTrueReturn(ret) + + if IS_WINDOWS: + ret = self.run_state('file.directory', name=name, win_owner=username) + else: + ret = self.run_state('file.directory', name=name) + + self.assertSaltTrueReturn(ret) + self.assertSaltStateChangesEqual(ret, {}) + @with_tempdir() def test_directory_clean_exclude(self, base_dir): ''' From 1087d10bea8bb5cecbbc9471809fd4674b1420ef Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 14 Feb 2019 01:18:45 -0700 Subject: [PATCH 66/67] increase batch test timeout on windows --- tests/integration/cli/test_batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cli/test_batch.py b/tests/integration/cli/test_batch.py index 05f3228d17..7f8dc28836 100644 --- a/tests/integration/cli/test_batch.py +++ b/tests/integration/cli/test_batch.py @@ -17,7 +17,7 @@ class BatchTest(ShellCase): Integration tests for the salt.cli.batch module ''' if salt.utils.platform.is_windows(): - run_timeout = 90 + run_timeout = 180 else: run_timeout = 30 From 918030e48d439ec74b98a94929c423afbada1a9b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 14 Feb 2019 01:46:53 -0700 Subject: [PATCH 67/67] Increase minion wait for slow boxes --- tests/integration/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 845eaa0faa..eb246534ce 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -183,7 +183,7 @@ class TestDaemon(object): ''' Set up the master and minion daemons, and run related cases ''' - MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 300 + MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 500 def __init__(self, parser): self.parser = parser