From f3c9cd4166a23467347f74d6ff00b24fcf3c9259 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 30 Jan 2019 11:20:56 -0700 Subject: [PATCH 1/7] Add missing raise statement --- salt/modules/win_dsc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_dsc.py b/salt/modules/win_dsc.py index 9bd06fbb67..6d0b8e2b54 100644 --- a/salt/modules/win_dsc.py +++ b/salt/modules/win_dsc.py @@ -738,7 +738,7 @@ def set_lcm_config(config_mode=None, cmd += ' RefreshFrequencyMins = {0};'.format(refresh_freq) if reboot_if_needed is not None: if not isinstance(reboot_if_needed, bool): - SaltInvocationError('reboot_if_needed must be a boolean value') + raise SaltInvocationError('reboot_if_needed must be a boolean value') if reboot_if_needed: reboot_if_needed = '$true' else: From 2aaa9f9c950a2c832be11973e547ad023b336d25 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Dec 2018 10:35:09 -0600 Subject: [PATCH 2/7] pip.installed: Fix traceback when _find_key doesn't return a match --- salt/states/pip_state.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 4ebffeeb7d..493d95d661 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -92,7 +92,7 @@ def _find_key(prefix, pip_list): except StopIteration: return None else: - return match + return match.lower() def _fulfills_version_spec(version, version_spec): @@ -913,11 +913,10 @@ def installed(name, ) else: pkg_name = _find_key(prefix, pipsearch) - if pkg_name.lower() in already_installed_packages: - continue - ver = pipsearch[pkg_name] - ret['changes']['{0}=={1}'.format(pkg_name, - ver)] = 'Installed' + if pkg_name is not None \ + and pkg_name not in already_installed_packages: + ver = pipsearch[pkg_name] + ret['changes']['{0}=={1}'.format(pkg_name, ver)] = 'Installed' # Case for packages that are an URL else: ret['changes']['{0}==???'.format(state_name)] = 'Installed' From 5ca9f82ab5c60713254f5e3212b8883860979a93 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 30 Jan 2019 15:13:16 -0600 Subject: [PATCH 3/7] Add a CaseInsensitiveDict implementation --- salt/output/nested.py | 11 +++-- salt/utils/data.py | 99 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 salt/utils/data.py diff --git a/salt/output/nested.py b/salt/output/nested.py index 21983c73b1..2a71028c43 100644 --- a/salt/output/nested.py +++ b/salt/output/nested.py @@ -34,6 +34,11 @@ from salt.ext.six import string_types from salt.utils import get_colors import salt.utils.locales +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping + class NestDisplay(object): ''' @@ -109,7 +114,7 @@ class NestDisplay(object): first_line = False elif isinstance(ret, (list, tuple)): for ind in ret: - if isinstance(ind, (list, tuple, dict)): + if isinstance(ind, (list, tuple, Mapping)): out.append( self.ustring( indent, @@ -117,11 +122,11 @@ class NestDisplay(object): '|_' ) ) - prefix = '' if isinstance(ind, dict) else '- ' + prefix = '' if isinstance(ind, Mapping) else '- ' self.display(ind, indent + 2, prefix, out) else: self.display(ind, indent, '- ', out) - elif isinstance(ret, dict): + elif isinstance(ret, Mapping): if indent: out.append( self.ustring( diff --git a/salt/utils/data.py b/salt/utils/data.py new file mode 100644 index 0000000000..a781befca8 --- /dev/null +++ b/salt/utils/data.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +''' +Functions for manipulating, inspecting, or otherwise working with data types +and data structures. +''' + +from __future__ import absolute_import, print_function, unicode_literals + +try: + from collections.abc import Mapping, MutableMapping, Sequence +except ImportError: + from collections import Mapping, MutableMapping, Sequence + +# Import Salt libs +from salt.utils.odict import OrderedDict + +# Import 3rd-party libs +from salt.ext import six + + +class CaseInsensitiveDict(MutableMapping): + ''' + Inspired by requests' case-insensitive dict implementation, but works with + non-string keys as well. + ''' + def __init__(self, init=None): + ''' + Force internal dict to be ordered to ensure a consistent iteration + order, irrespective of case. + ''' + self._data = OrderedDict() + self.update(init or {}) + + def __len__(self): + return len(self._data) + + def __setitem__(self, key, value): + # Store the case-sensitive key so it is available for dict iteration + self._data[to_lowercase(key)] = (key, value) + + def __delitem__(self, key): + del self._data[to_lowercase(key)] + + def __getitem__(self, key): + return self._data[to_lowercase(key)][1] + + def __iter__(self): + return (item[0] for item in six.itervalues(self._data)) + + def __eq__(self, rval): + if not isinstance(rval, Mapping): + # Comparing to non-mapping type (e.g. int) is always False + return False + return dict(self.items_lower()) == dict(CaseInsensitiveDict(rval).items_lower()) + + def __repr__(self): + return repr(dict(six.iteritems(self))) + + def items_lower(self): + ''' + Returns a generator iterating over keys and values, with the keys all + being lowercase. + ''' + return ((key, val[1]) for key, val in six.iteritems(self._data)) + + def copy(self): + ''' + Returns a copy of the object + ''' + return CaseInsensitiveDict(six.iteritems(self._data)) + + +def __change_case(data, attr, preserve_dict_class=False): + try: + return getattr(data, attr)() + except AttributeError: + pass + + data_type = data.__class__ + + if isinstance(data, Mapping): + return (data_type if preserve_dict_class else dict)( + (__change_case(key, attr, preserve_dict_class), + __change_case(val, attr, preserve_dict_class)) + for key, val in six.iteritems(data) + ) + elif isinstance(data, Sequence): + return data_type( + __change_case(item, attr, preserve_dict_class) for item in data) + else: + return data + + +def to_lowercase(data, preserve_dict_class=False): + return __change_case(data, 'lower', preserve_dict_class) + + +def to_uppercase(data, preserve_dict_class=False): + return __change_case(data, 'upper', preserve_dict_class) From 0e760b59ba2c5d7110e152b0e5c423758c171752 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 30 Jan 2019 15:15:40 -0600 Subject: [PATCH 4/7] pip states: Use case-insensitive dictionaries for pip.list return --- salt/states/pip_state.py | 57 +++++++++++++++------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 493d95d661..86044b9eb4 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -27,6 +27,7 @@ import pkg_resources # Import salt libs import salt.utils +import salt.utils.data from salt.version import SaltStackVersion as _SaltStackVersion from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -81,20 +82,6 @@ def __virtual__(): return False -def _find_key(prefix, pip_list): - ''' - Does a case-insensitive match in the pip_list for the desired package. - ''' - try: - match = next( - iter(x for x in pip_list if x.lower() == prefix.lower()) - ) - except StopIteration: - return None - else: - return match.lower() - - def _fulfills_version_spec(version, version_spec): ''' Check version number against version specification info and return a @@ -210,23 +197,20 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, ret = {'result': False, 'comment': None} # If we are not passed a pip list, get one: - if not pip_list: - pip_list = __salt__['pip.list'](prefix, bin_env=bin_env, - user=user, cwd=cwd, - env_vars=env_vars, **kwargs) - - # Check if the requested package is already installed. - prefix_realname = _find_key(prefix, pip_list) + pip_list = salt.utils.data.CaseInsensitiveDict( + pip_list or __salt__['pip.list'](prefix, bin_env=bin_env, + user=user, cwd=cwd, + env_vars=env_vars, **kwargs) + ) # If the package was already installed, check # the ignore_installed and force_reinstall flags - if ignore_installed is False and prefix_realname is not None: + if ignore_installed is False and prefix in pip_list: if force_reinstall is False and not upgrade: # Check desired version (if any) against currently-installed if ( any(version_spec) and - _fulfills_version_spec(pip_list[prefix_realname], - version_spec) + _fulfills_version_spec(pip_list[prefix], version_spec) ) or (not any(version_spec)): ret['result'] = True ret['comment'] = ('Python package {0} was already ' @@ -246,7 +230,7 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, if 'rc' in spec[1]: include_rc = True available_versions = __salt__['pip.list_all_versions']( - prefix_realname, bin_env=bin_env, include_alpha=include_alpha, + prefix, bin_env=bin_env, include_alpha=include_alpha, include_beta=include_beta, include_rc=include_rc, user=user, cwd=cwd) desired_version = '' @@ -262,9 +246,9 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, ret['comment'] = ('Python package {0} was already ' 'installed and\nthe available upgrade ' 'doesn\'t fulfills the version ' - 'requirements'.format(prefix_realname)) + 'requirements'.format(prefix)) return ret - if _pep440_version_cmp(pip_list[prefix_realname], desired_version) == 0: + if _pep440_version_cmp(pip_list[prefix], desired_version) == 0: ret['result'] = True ret['comment'] = ('Python package {0} was already ' 'installed'.format(state_pkg_name)) @@ -898,10 +882,12 @@ def installed(name, # Case for packages that are not an URL if prefix: - pipsearch = __salt__['pip.list'](prefix, bin_env, - user=user, cwd=cwd, - env_vars=env_vars, - **kwargs) + pipsearch = salt.utils.data.CaseInsensitiveDict( + __salt__['pip.list'](prefix, bin_env, + user=user, cwd=cwd, + env_vars=env_vars, + **kwargs) + ) # If we didn't find the package in the system after # installing it report it @@ -912,11 +898,10 @@ def installed(name, '\'pip.freeze\'.'.format(pkg) ) else: - pkg_name = _find_key(prefix, pipsearch) - if pkg_name is not None \ - and pkg_name not in already_installed_packages: - ver = pipsearch[pkg_name] - ret['changes']['{0}=={1}'.format(pkg_name, ver)] = 'Installed' + if prefix in pipsearch \ + and prefix.lower() not in already_installed_packages: + ver = pipsearch[prefix] + ret['changes']['{0}=={1}'.format(prefix, ver)] = 'Installed' # Case for packages that are an URL else: ret['changes']['{0}==???'.format(state_name)] = 'Installed' From d06526c7a28ceac21dfeb61ce41bc3e5cf9a9118 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 30 Jan 2019 15:35:37 -0600 Subject: [PATCH 5/7] Allow for kwargs to be used in object initialization --- salt/utils/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/data.py b/salt/utils/data.py index a781befca8..d91b384ac4 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -23,13 +23,13 @@ class CaseInsensitiveDict(MutableMapping): Inspired by requests' case-insensitive dict implementation, but works with non-string keys as well. ''' - def __init__(self, init=None): + def __init__(self, init=None, **kwargs): ''' Force internal dict to be ordered to ensure a consistent iteration order, irrespective of case. ''' self._data = OrderedDict() - self.update(init or {}) + self.update(init or {}, **kwargs) def __len__(self): return len(self._data) From e5972f5a03aa92c45081cd73ad9667cffcefdd40 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 11 Feb 2019 14:47:19 -0700 Subject: [PATCH 6/7] Use the code directory instead of cwd for python path --- tests/support/case.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/support/case.py b/tests/support/case.py index 0e2b468446..faf9bbad38 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -252,9 +252,9 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): if 'env' not in popen_kwargs: popen_kwargs['env'] = os.environ.copy() if sys.version_info[0] < 3: - popen_kwargs['env'][b'PYTHONPATH'] = os.getcwd().encode() + popen_kwargs['env'][b'PYTHONPATH'] = CODE_DIR.encode() else: - popen_kwargs['env']['PYTHONPATH'] = os.getcwd() + popen_kwargs['env']['PYTHONPATH'] = CODE_DIR else: cmd = 'PYTHONPATH=' python_path = os.environ.get('PYTHONPATH', None) From 8697ce703e308cf1b0cf44493c4951be287b72c0 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 11 Feb 2019 17:25:22 -0700 Subject: [PATCH 7/7] Disable pylint checks, only for 2017.7 --- tests/support/case.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/support/case.py b/tests/support/case.py index faf9bbad38..3afb17036f 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -141,14 +141,14 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): ) return self.run_script('salt-ssh', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr, raw=True) - def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None): + def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None): # pylint: disable=W8606 ''' Execute salt-run ''' arg_str = '-c {0}{async_flag} -t {timeout} {1}'.format(config_dir or self.get_config_dir(), arg_str, timeout=timeout, - async_flag=' --async' if async else '') + async_flag=' --async' if async else '') # pylint: disable=W8606 return self.run_script('salt-run', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr) def run_run_plus(self, fun, *arg, **kwargs): @@ -498,14 +498,14 @@ class ShellCase(ShellTestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixi timeout=timeout, raw=True) - def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None): + def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None): # pylint: disable=W8606 ''' Execute salt-run ''' arg_str = '-c {0}{async_flag} -t {timeout} {1}'.format(config_dir or self.get_config_dir(), arg_str, timeout=timeout, - async_flag=' --async' if async else '') + async_flag=' --async' if async else '') # pylint: disable=W8606 return self.run_script('salt-run', arg_str, with_retcode=with_retcode,