Merge pull request #42854 from rallytime/merge-develop

[develop] Merge forward from 2017.7 to develop

doc/ref/states/all/salt.states.kubernetes.rst

@@ -1,6 +1,6 @@
-=======================
+======================
-salt.modules.kubernetes
+salt.states.kubernetes
-=======================
+======================
 
-.. automodule:: salt.modules.kubernetes
+.. automodule:: salt.states.kubernetes
     :members:

salt/modules/augeas_cfg.py

@@ -200,7 +200,7 @@ def execute(context=None, lens=None, commands=(), load_path=None):
             method = METHOD_MAP[cmd]
             nargs = arg_map[method]
 
-            parts = salt.utils.args.shlex_split(arg)
+            parts = salt.utils.args.shlex_split(arg, posix=False)
 
             if len(parts) not in nargs:
                 err = '{0} takes {1} args: {2}'.format(method, nargs, parts)

salt/modules/boto_efs.py

@@ -223,10 +223,23 @@ def create_mount_target(filesystemid,
 
     client = _get_conn(key=key, keyid=keyid, profile=profile, region=region)
 
-    return client.create_mount_point(FileSystemId=filesystemid,
+    if ipaddress is None and securitygroups is None:
-                                     SubnetId=subnetid,
+        return client.create_mount_target(FileSystemId=filesystemid,
-                                     IpAddress=ipaddress,
+                                          SubnetId=subnetid)
-                                     SecurityGroups=securitygroups)
+
+    if ipaddress is None:
+        return client.create_mount_target(FileSystemId=filesystemid,
+                                          SubnetId=subnetid,
+                                          SecurityGroups=securitygroups)
+    if securitygroups is None:
+        return client.create_mount_target(FileSystemId=filesystemid,
+                                          SubnetId=subnetid,
+                                          IpAddress=ipaddress)
+
+    return client.create_mount_target(FileSystemId=filesystemid,
+                                      SubnetId=subnetid,
+                                      IpAddress=ipaddress,
+                                      SecurityGroups=securitygroups)
 
 
 def create_tags(filesystemid,

salt/modules/dockermod.py

@@ -1839,7 +1839,7 @@ def create(image,
         generate one for you (it will be included in the return data).
 
     skip_translate
-        This function translates Salt CLI input into the format which
+        This function translates Salt CLI or SLS input into the format which
         docker-py_ expects. However, in the event that Salt's translation logic
         fails (due to potential changes in the Docker Remote API, or to bugs in
         the translation code), this argument can be used to exert granular
@@ -2107,9 +2107,9 @@ def create(image,
         - ``dns_search="[foo1.domain.tld, foo2.domain.tld]"``
 
     domainname
-        Set custom DNS search domains
+        The domain name to use for the container
 
-        Example: ``domainname=domain.tld,domain2.tld``
+        Example: ``domainname=domain.tld``
 
     entrypoint
         Entrypoint for the container. Either a string (e.g. ``"mycmd --arg1

salt/modules/pillar.py

@@ -257,8 +257,8 @@ def items(*args, **kwargs):
         __opts__,
         __grains__,
         __opts__['id'],
-        pillar_override=kwargs.get('pillar'),
+        pillar_override=pillar_override,
-        pillarenv=kwargs.get('pillarenv') or __opts__['pillarenv'])
+        pillarenv=pillarenv)
 
     return pillar.compile_pillar()
 

salt/modules/rabbitmq.py

@@ -611,11 +611,11 @@ def set_user_tags(name, tags, runas=None):
     if runas is None and not salt.utils.platform.is_windows():
         runas = salt.utils.get_user()
 
-    if tags and isinstance(tags, (list, tuple)):
+    if not isinstance(tags, (list, tuple)):
-        tags = ' '.join(tags)
+        tags = [tags]
 
     res = __salt__['cmd.run_all'](
-        [RABBITMQCTL, 'set_user_tags', name, tags],
+        [RABBITMQCTL, 'set_user_tags', name] + list(tags),
         runas=runas,
         python_shell=False)
     msg = "Tag(s) set"

salt/modules/win_iis.py

@@ -856,7 +856,7 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443,
 
     new_cert_bindings = list_cert_bindings(site)
 
-    if binding_info not in new_cert_bindings(site):
+    if binding_info not in new_cert_bindings:
         log.error('Binding not present: {0}'.format(binding_info))
         return False
 

salt/netapi/rest_cherrypy/app.py

@@ -580,13 +580,21 @@ import signal
 import tarfile
 from multiprocessing import Process, Pipe
 
-# Import third-party libs
+logger = logging.getLogger(__name__)
-# pylint: disable=import-error
-import cherrypy  # pylint: disable=3rd-party-module-not-gated
-import yaml
-from salt.ext import six
-# pylint: enable=import-error
 
+# Import third-party libs
+# pylint: disable=import-error, 3rd-party-module-not-gated
+import cherrypy
+try:
+    from cherrypy.lib import cpstats
+except ImportError:
+    cpstats = None
+    logger.warn('Import of cherrypy.cpstats failed. '
+        'Possible upstream bug: '
+        'https://github.com/cherrypy/cherrypy/issues/1444')
+
+import yaml
+# pylint: enable=import-error, 3rd-party-module-not-gated
 
 # Import Salt libs
 import salt
@@ -594,12 +602,11 @@ import salt.auth
 import salt.utils
 import salt.utils.event
 import salt.utils.stringutils
+from salt.ext import six
 
 # Import salt-api libs
 import salt.netapi
 
-logger = logging.getLogger(__name__)
-
 # Imports related to websocket
 try:
     from .tools import websockets
@@ -2717,13 +2724,6 @@ class Stats(object):
             :status 406: |406|
         '''
         if hasattr(logging, 'statistics'):
-            # Late import
-            try:
-                from cherrypy.lib import cpstats
-            except ImportError:
-                logger.error('Import of cherrypy.cpstats failed. Possible '
-                        'upstream bug here: https://github.com/cherrypy/cherrypy/issues/1444')
-                return {}
             return cpstats.extrapolate_statistics(logging.statistics)
 
         return {}
@@ -2843,13 +2843,14 @@ class API(object):
                 'tools.trailing_slash.on': True,
                 'tools.gzip.on': True,
 
-                'tools.cpstats.on': self.apiopts.get('collect_stats', False),
-
                 'tools.html_override.on': True,
                 'tools.cors_tool.on': True,
             },
         }
 
+        if cpstats and self.apiopts.get('collect_stats', False):
+            conf['/']['tools.cpstats.on'] = True
+
         if 'favicon' in self.apiopts:
             conf['/favicon.ico'] = {
                 'tools.staticfile.on': True,

salt/netapi/rest_tornado/saltnado.py

@@ -523,8 +523,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn):  # pylin
 
         try:
             # Use cgi.parse_header to correctly separate parameters from value
-            header = cgi.parse_header(self.request.headers['Content-Type'])
+            value, parameters = cgi.parse_header(self.request.headers['Content-Type'])
-            value, parameters = header
             return ct_in_map[value](tornado.escape.native_str(data))
         except KeyError:
             self.send_error(406)
@@ -538,7 +537,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn):  # pylin
         if not self.request.body:
             return
         data = self.deserialize(self.request.body)
-        self.raw_data = copy(data)
+        self.request_payload = copy(data)
 
         if data and 'arg' in data and not isinstance(data['arg'], list):
             data['arg'] = [data['arg']]
@@ -696,15 +695,13 @@ class SaltAuthHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
             }}
         '''
         try:
-            request_payload = self.deserialize(self.request.body)
+            if not isinstance(self.request_payload, dict):
-
-            if not isinstance(request_payload, dict):
                 self.send_error(400)
                 return
 
-            creds = {'username': request_payload['username'],
+            creds = {'username': self.request_payload['username'],
-                     'password': request_payload['password'],
+                     'password': self.request_payload['password'],
-                     'eauth': request_payload['eauth'],
+                     'eauth': self.request_payload['eauth'],
                      }
         # if any of the args are missing, its a bad request
         except KeyError:
@@ -1641,7 +1638,7 @@ class WebhookSaltAPIHandler(SaltAPIHandler):  # pylint: disable=W0223
                 value = value[0]
             arguments[argname] = value
         ret = self.event.fire_event({
-            'post': self.raw_data,
+            'post': self.request_payload,
             'get': arguments,
             # In Tornado >= v4.0.3, the headers come
             # back as an HTTPHeaders instance, which

salt/runners/manage.py

@@ -22,6 +22,7 @@ from salt.ext.six.moves.urllib.request import urlopen as _urlopen  # pylint: dis
 # Import salt libs
 import salt.key
 import salt.utils
+import salt.utils.compat
 import salt.utils.files
 import salt.utils.minions
 import salt.client
@@ -670,7 +671,7 @@ def versions():
             ver_diff = -2
         else:
             minion_version = salt.version.SaltStackVersion.parse(minions[minion])
-            ver_diff = cmp(minion_version, master_version)
+            ver_diff = salt.utils.compat.cmp(minion_version, master_version)
 
         if ver_diff not in version_status:
             version_status[ver_diff] = {}

salt/states/bigip.py

@@ -85,7 +85,7 @@ def _check_for_changes(entity_type, ret, existing, modified):
         if 'generation' in existing['content'].keys():
             del existing['content']['generation']
 
-        if cmp(modified['content'], existing['content']) == 0:
+        if modified['content'] == existing['content']:
             ret['comment'] = '{entity_type} is currently enforced to the desired state.  No changes made.'.format(entity_type=entity_type)
         else:
             ret['comment'] = '{entity_type} was enforced to the desired state.  Note: Only parameters specified ' \
@@ -94,7 +94,7 @@ def _check_for_changes(entity_type, ret, existing, modified):
             ret['changes']['new'] = modified['content']
 
     else:
-        if cmp(modified, existing) == 0:
+        if modified == existing:
             ret['comment'] = '{entity_type} is currently enforced to the desired state.  No changes made.'.format(entity_type=entity_type)
         else:
             ret['comment'] = '{entity_type} was enforced to the desired state.  Note: Only parameters specified ' \

salt/states/boto_cfn.py

@@ -37,14 +37,16 @@ Connection module for Amazon Cloud Formation
             - name: mystack
 '''
 
-from __future__ import absolute_import
-
 # Import Python libs
+from __future__ import absolute_import
 import logging
 import json
 
-# Import 3rd-party libs
+# Import Salt libs
+import salt.utils.compat
 from salt.ext import six
+
+# Import 3rd-party libs
 try:
     from salt._compat import ElementTree as ET
     HAS_ELEMENT_TREE = True
@@ -142,10 +144,14 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi
     stack_policy_body = _get_template(stack_policy_body, name)
     stack_policy_during_update_body = _get_template(stack_policy_during_update_body, name)
 
+    for i in [template_body, stack_policy_body, stack_policy_during_update_body]:
+        if isinstance(i, dict):
+            return i
+
     _valid = _validate(template_body, template_url, region, key, keyid, profile)
     log.debug('Validate is : {0}.'.format(_valid))
     if _valid is not True:
-        code, message = _get_error(_valid)
+        code, message = _valid
         ret['result'] = False
         ret['comment'] = 'Template could not be validated.\n{0} \n{1}'.format(code, message)
         return ret
@@ -155,7 +161,7 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi
         template = template['GetTemplateResponse']['GetTemplateResult']['TemplateBody'].encode('ascii', 'ignore')
         template = json.loads(template)
         _template_body = json.loads(template_body)
-        compare = cmp(template, _template_body)
+        compare = salt.utils.compat.cmp(template, _template_body)
         if compare != 0:
             log.debug('Templates are not the same. Compare value is {0}'.format(compare))
             # At this point we should be able to run update safely since we already validated the template
@@ -251,7 +257,7 @@ def _get_template(template, name):
 def _validate(template_body=None, template_url=None, region=None, key=None, keyid=None, profile=None):
     # Validates template. returns true if template syntax is correct.
     validate = __salt__['boto_cfn.validate_template'](template_body, template_url, region, key, keyid, profile)
-    log.debug('Validate is result is {0}.'.format(str(validate)))
+    log.debug('Validate result is {0}.'.format(str(validate)))
     if isinstance(validate, six.string_types):
         code, message = _get_error(validate)
         log.debug('Validate error is {0} and message is {1}.'.format(code, message))

salt/states/cisconso.py

@@ -8,6 +8,12 @@ For documentation on setting up the cisconso proxy minion look in the documentat
 for :mod:`salt.proxy.cisconso <salt.proxy.cisconso>`.
 '''
 
+# Import Python libs
+from __future__ import absolute_import
+
+# Import Salt libs
+import salt.utils.compat
+
 
 def __virtual__():
     return 'cisconso.set_data_value' in __salt__
@@ -53,7 +59,7 @@ def value_present(name, datastore, path, config):
 
     existing = __salt__['cisconso.get_data'](datastore, path)
 
-    if cmp(existing, config):
+    if salt.utils.compat.cmp(existing, config):
         ret['result'] = True
         ret['comment'] = 'Config is already set'
 

salt/states/docker_container.py

@@ -146,7 +146,7 @@ def running(name,
     .. _docker-container-running-skip-translate:
 
     skip_translate
-        This function translates Salt CLI input into the format which
+        This function translates Salt CLI or SLS input into the format which
         docker-py_ expects. However, in the event that Salt's translation logic
         fails (due to potential changes in the Docker Remote API, or to bugs in
         the translation code), this argument can be used to exert granular
@@ -678,24 +678,14 @@ def running(name,
                   - foo2.domain.tld
 
     domainname
-        Set custom DNS search domains. Can be expressed as a comma-separated
+        The domain name to use for the container
-        list or a YAML list. The below two examples are equivalent:
 
         .. code-block:: yaml
 
             foo:
               docker_container.running:
                 - image: bar/baz:latest
-                - dommainname: domain.tld,domain2.tld
+                - dommainname: domain.tld
-
-        .. code-block:: yaml
-
-            foo:
-              docker_container.running:
-                - image: bar/baz:latest
-                - dommainname:
-                  - domain.tld
-                  - domain2.tld
 
     entrypoint
         Entrypoint for the container

salt/states/ini_manage.py

@@ -206,7 +206,7 @@ def sections_present(name, sections=None, separator='='):
                 ret['result'] = False
                 ret['comment'] = "{0}".format(err)
                 return ret
-            if cmp(dict(sections[section]), cur_section) == 0:
+            if dict(sections[section]) == cur_section:
                 ret['comment'] += 'Section unchanged {0}.\n'.format(section)
                 continue
             elif cur_section:

salt/states/pkg.py

@@ -178,15 +178,12 @@ def _fulfills_version_spec(versions, oper, desired_version,
         if isinstance(versions, dict) and 'version' in versions:
             versions = versions['version']
     for ver in versions:
-        if oper == '==':
+        if (oper == '==' and fnmatch.fnmatch(ver, desired_version)) \
-            if fnmatch.fnmatch(ver, desired_version):
+                or salt.utils.compare_versions(ver1=ver,
-                return True
+                                               oper=oper,
-
+                                               ver2=desired_version,
-        elif salt.utils.compare_versions(ver1=ver,
+                                               cmp_func=cmp_func,
-                                         oper=oper,
+                                               ignore_epoch=ignore_epoch):
-                                         ver2=desired_version,
-                                         cmp_func=cmp_func,
-                                         ignore_epoch=ignore_epoch):
             return True
     return False
 

salt/states/zabbix_usergroup.py

@@ -84,7 +84,7 @@ def present(name, **kwargs):
                     for right in kwargs['rights']:
                         for key in right:
                             right[key] = str(right[key])
-                    if cmp(sorted(kwargs['rights']), sorted(usergroup['rights'])) != 0:
+                    if sorted(kwargs['rights']) != sorted(usergroup['rights']):
                         update_rights = True
                 else:
                     update_rights = True

salt/utils/cloud.py

@@ -47,6 +47,7 @@ import salt.config
 import salt.loader
 import salt.template
 import salt.utils  # Can be removed when pem_finger is moved
+import salt.utils.compat
 import salt.utils.event
 import salt.utils.files
 import salt.utils.platform
@@ -3055,7 +3056,7 @@ def diff_node_cache(prov_dir, node, new_data, opts):
     # Perform a simple diff between the old and the new data, and if it differs,
     # return both dicts.
     # TODO: Return an actual diff
-    diff = cmp(new_data, cache_data)
+    diff = salt.utils.compat.cmp(new_data, cache_data)
     if diff != 0:
         fire_event(
             'event',

salt/utils/compat.py

@@ -46,3 +46,15 @@ def deepcopy_bound(name):
     finally:
         copy._deepcopy_dispatch = pre_dispatch
     return ret
+
+
+def cmp(x, y):
+    '''
+    Compatibility helper function to replace the ``cmp`` function from Python 2. The
+    ``cmp`` function is no longer available in Python 3.
+
+    cmp(x, y) -> integer
+
+    Return negative if x<y, zero if x==y, positive if x>y.
+    '''
+    return (x > y) - (x < y)

salt/utils/decorators/__init__.py

@@ -188,9 +188,8 @@ def memoize(func):
                 str_args.append(str(arg))
             else:
                 str_args.append(arg)
-        args = str_args
 
-        args_ = ','.join(list(args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)])
+        args_ = ','.join(list(str_args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)])
         if args_ not in cache:
             cache[args_] = func(*args, **kwargs)
         return cache[args_]

salt/utils/docker/translate.py

@@ -387,7 +387,7 @@ def dns(val, **kwargs):
 
 
 def domainname(val, **kwargs):  # pylint: disable=unused-argument
-    return _translate_stringlist(val)
+    return _translate_str(val)
 
 
 def entrypoint(val, **kwargs):  # pylint: disable=unused-argument

salt/utils/http.py

@@ -777,7 +777,8 @@ def _render(template, render, renderer, template_dict, opts):
         blacklist = opts.get('renderer_blacklist')
         whitelist = opts.get('renderer_whitelist')
         ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict)
-        ret = ret.read()
+        if salt.utils.stringio.is_readable(ret):
+            ret = ret.read()
         if str(ret).startswith('#!') and not str(ret).startswith('#!/'):
             ret = str(ret).split('\n', 1)[1]
         return ret

salt/utils/reactor.py

@@ -274,6 +274,8 @@ class ReactWrap(object):
         try:
             f_call = salt.utils.format_call(l_fun, low)
             kwargs = f_call.get('kwargs', {})
+            if 'arg' not in kwargs:
+                kwargs['arg'] = []
             if 'kwarg' not in kwargs:
                 kwargs['kwarg'] = {}
 

tests/integration/states/test_pkg.py

@@ -726,6 +726,43 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin):
             ret = self.run_state('pkg.removed', name=target)
             self.assertSaltTrueReturn(ret)
 
+    def test_pkg_014_installed_missing_release(self, grains=None):  # pylint: disable=unused-argument
+        '''
+        Tests that a version number missing the release portion still resolves
+        as correctly installed. For example, version 2.0.2 instead of 2.0.2-1.el7
+        '''
+        os_family = grains.get('os_family', '')
+
+        if os_family.lower() != 'redhat':
+            self.skipTest('Test only runs on RedHat OS family')
+
+        pkg_targets = _PKG_TARGETS.get(os_family, [])
+
+        # Make sure that we have targets that match the os_family. If this
+        # fails then the _PKG_TARGETS dict above needs to have an entry added,
+        # with two packages that are not installed before these tests are run
+        self.assertTrue(pkg_targets)
+
+        target = pkg_targets[0]
+        version = self.run_function('pkg.version', [target])
+
+        # If this assert fails, we need to find new targets, this test needs to
+        # be able to test successful installation of packages, so this package
+        # needs to not be installed before we run the states below
+        self.assertFalse(version)
+
+        ret = self.run_state(
+            'pkg.installed',
+            name=target,
+            version=salt.utils.str_version_to_evr(version)[1],
+            refresh=False,
+        )
+        self.assertSaltTrueReturn(ret)
+
+        # Clean up
+        ret = self.run_state('pkg.removed', name=target)
+        self.assertSaltTrueReturn(ret)
+
     @requires_salt_modules('pkg.group_install')
     @requires_system_grains
     def test_group_installed_handle_missing_package_group(self, grains=None):  # pylint: disable=unused-argument

tests/unit/utils/test_docker.py

@@ -821,7 +821,7 @@ class TranslateInputTestCase(TestCase):
             expected
         )
 
-    @assert_stringlist
+    @assert_string
     def test_domainname(self):
         '''
         Should be a list of strings or converted to one