Merge branch '2016.11' into 'nitrogen'

Conflicts:
  - salt/modules/yumpkg.py
  - salt/spm/__init__.py
  - tests/unit/modules/test_zypper.py
  - tests/unit/utils/test_network.py

doc/conf.py

@@ -244,7 +244,7 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ
 project = 'Salt'
 
 version = salt.version.__version__
-latest_release = '2016.11.4'  # latest release
+latest_release = '2016.11.5'  # latest release
 previous_release = '2016.3.6'  # latest release from previous branch
 previous_release_dir = '2016.3'  # path on web server for previous branch
 next_release = ''  # next release

doc/topics/releases/2016.11.5.rst

@@ -19,6 +19,25 @@ Statistics:
 
 Changes:
 
+Patched Packages
+----------------
+Due to the critical nature of issue `#41230`_ we have decided to patch the 2016.11.5 packages with PR `#41244`_. This issue affects all calls to a salt-minion if there is an ipv6 nameserver set on the minion's host. The patched packages on repo.saltstack.com will divert from the v2016.11.5 tag and pypi packages due to the additional PR applied to the packages.
+
+- **PR** `#41244`_: (*cachedout*) Fix ipv6 nameserver grains
+  @ *2017-05-15T17:55:39Z*
+
+  - **ISSUE** `#41230`_: (*RealKelsar*) 2016.11.5 IPv6 nameserver in resolv.conf leads to minion exception
+    | refs: `#41244`_ `#41244`_
+  - **ISSUE** `#40912`_: (*razed11*) IPV6 Warning when ipv6 set to False
+    | refs: `#40934`_
+  - **PR** `#40934`_: (*gtmanfred*) Only display IPvX warning if role is master
+    | refs: `#41244`_ `#41244`_
+  * 53d5b3e Merge pull request `#41244`_ from cachedout/fix_ipv6_nameserver_grains
+  * f745db1 Lint
+
+  * 6e1ab69 Partial revert of `#40934`_
+
+  * 88f49f9 Revert "Only display IPvX warning if role is master"
 
 - **PR** `#41173`_: (*twangboy*) Add silent action to MsgBox for Path Actions
   @ *2017-05-10T17:22:18Z*
@@ -780,6 +799,8 @@ Changes:
 .. _`#40930`: https://github.com/saltstack/salt/pull/40930
 .. _`#40933`: https://github.com/saltstack/salt/pull/40933
 .. _`#40934`: https://github.com/saltstack/salt/pull/40934
+.. _`#41230`: https://github.com/saltstack/salt/issues/41230
+.. _`#41244`: https://github.com/saltstack/salt/pull/41244
 .. _`#40935`: https://github.com/saltstack/salt/pull/40935
 .. _`#40936`: https://github.com/saltstack/salt/pull/40936
 .. _`#40939`: https://github.com/saltstack/salt/pull/40939

doc/topics/releases/2016.11.6.rst

@@ -0,0 +1,6 @@
+============================
+Salt 2016.11.6 Release Notes
+============================
+
+Version 2016.11.6 is a bugfix release for :ref:`2016.11.0 <release-2016-11-0>`.
+

doc/topics/spm/master.rst

@@ -26,6 +26,16 @@ the name of the repository, and the link to the repository:
     my_repo:
       url: https://spm.example.com/
 
+For HTTP/HTTPS Basic authorization you can define credentials:
+
+.. code-block:: yaml
+    my_repo:
+      url: https://spm.example.com/
+      username: user
+      password: pass
+
+Beware of unauthorized access to this file, please set at least 0640 permissions for this configuration file:
+
 The URL can use ``http``, ``https``, ``ftp``, or ``file``.
 
 .. code-block:: yaml

salt/cache/consul.py

@@ -61,7 +61,7 @@ api = None
 # Define the module's virtual name
 __virtualname__ = 'consul'
 
-__func_alias__ = {'ls': 'list'}
+__func_alias__ = {'list': 'ls'}
 
 
 def __virtual__():

salt/modules/win_lgpo.py

@@ -3091,9 +3091,14 @@ def _getDataFromRegPolData(search_string, policy_data, return_value_name=False):
                 if len(pol_entry) >= 5:
                     value = pol_entry[4]
                     if vtype == 'REG_DWORD' or vtype == 'REG_QWORD':
-                        value = value.replace(chr(0), '')
                         if value:
-                            value = ord(value)
+                            vlist = list(ord(v) for v in value)
+                            if vtype == 'REG_DWORD':
+                                for v in struct.unpack('I', struct.pack('2H', *vlist)):
+                                    value = v
+                            elif vtype == 'REG_QWORD':
+                                for v in struct.unpack('I', struct.pack('4H', *vlist)):
+                                    value = v
                         else:
                             value = 0
                     elif vtype == 'REG_MULTI_SZ':
@@ -4024,8 +4029,7 @@ def _write_regpol_data(data_to_write,
                         version_nums = (version_nums[0], version_nums[1] + 1)
                     elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower():
                         version_nums = (version_nums[0] + 1, version_nums[1])
-                    version_num = int("{0}{1}".format(str(version_nums[0]).zfill(4),
+                    version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0]
-                                                      str(version_nums[1]).zfill(4)), 16)
                     gpt_ini_data = "{0}{1}={2}\r\n{3}".format(
                             gpt_ini_data[0:version_loc.start()],
                             'Version', version_num,

salt/modules/win_network.py

@@ -232,7 +232,7 @@ def get_route(ip):
     '''
     Return routing information for given destination ip
 
-    .. versionadded:: 2016.11.6
+    .. versionadded:: 2016.11.5
 
     CLI Example::
 

salt/modules/yumpkg.py

@@ -41,6 +41,7 @@ except ImportError:
 # Import salt libs
 import salt.utils
 import salt.utils.pkg
+import salt.ext.six as six
 import salt.utils.itertools
 import salt.utils.systemd
 import salt.utils.decorators as decorators
@@ -197,10 +198,10 @@ def _get_repo_options(**kwargs):
     in the yum command, based on the kwargs.
     '''
     # Get repo options from the kwargs
-    fromrepo = kwargs.get('fromrepo', '')
+    fromrepo = kwargs.pop('fromrepo', '')
-    repo = kwargs.get('repo', '')
+    repo = kwargs.pop('repo', '')
-    disablerepo = kwargs.get('disablerepo', '')
+    disablerepo = kwargs.pop('disablerepo', '')
-    enablerepo = kwargs.get('enablerepo', '')
+    enablerepo = kwargs.pop('enablerepo', '')
 
     # Support old 'repo' argument
     if repo and not fromrepo:
@@ -233,7 +234,7 @@ def _get_excludes_option(**kwargs):
     Returns a list of '--disableexcludes' option to be used in the yum command,
     based on the kwargs.
     '''
-    disable_excludes = kwargs.get('disableexcludes', '')
+    disable_excludes = kwargs.pop('disableexcludes', '')
     ret = []
     if disable_excludes:
         log.info('Disabling excludes for \'%s\'', disable_excludes)
@@ -246,7 +247,7 @@ def _get_branch_option(**kwargs):
     Returns a list of '--branch' option to be used in the yum command,
     based on the kwargs. This feature requires 'branch' plugin for YUM.
     '''
-    branch = kwargs.get('branch', '')
+    branch = kwargs.pop('branch', '')
     ret = []
     if branch:
         log.info('Adding branch \'%s\'', branch)
@@ -254,6 +255,20 @@ def _get_branch_option(**kwargs):
     return ret
 
 
+def _get_extra_options(**kwargs):
+    '''
+    Returns list of extra options for yum
+    '''
+    ret = []
+    kwargs = salt.utils.clean_kwargs(**kwargs)
+    for key, value in six.iteritems(kwargs):
+        if isinstance(key, six.string_types):
+            ret.append('--{0}=\'{1}\''.format(key, value))
+        elif value is True:
+            ret.append('--{0}'.format(key))
+    return ret
+
+
 def _get_yum_config():
     '''
     Returns a dict representing the yum config options and values.
@@ -1619,10 +1634,21 @@ def upgrade(name=None,
             salt -G role:nsd pkg.upgrade gpfs.gplbin-2.6.32-279.31.1.el6.x86_64 normalize=False
 
         .. versionadded:: 2016.3.0
+
+    .. note::
+
+        To add extra arguments to the ``yum upgrade`` command, pass them as key
+        word arguments.  For arguments without assignments, pass ``True``
+
+    .. code-block:: bash
+
+        salt '*' pkg.upgrade security=True exclude='kernel*'
+
     '''
     repo_arg = _get_repo_options(**kwargs)
     exclude_arg = _get_excludes_option(**kwargs)
     branch_arg = _get_branch_option(**kwargs)
+    extra_args = _get_extra_options(**kwargs)
 
     if salt.utils.is_true(refresh):
         refresh_db(**kwargs)
@@ -1651,7 +1677,7 @@ def upgrade(name=None,
             and __salt__['config.get']('systemd.scope', True):
         cmd.extend(['systemd-run', '--scope'])
     cmd.extend([_yum(), '--quiet', '-y'])
-    for args in (repo_arg, exclude_arg, branch_arg):
+    for args in (repo_arg, exclude_arg, branch_arg, extra_args):
         if args:
             cmd.extend(args)
     if skip_verify:

salt/modules/zypper.py

@@ -1069,6 +1069,12 @@ def install(name=None,
 
     __context__.pop('pkg.list_pkgs', None)
     new = list_pkgs()
+
+    # Handle packages which report multiple new versions
+    # (affects only kernel packages at this point)
+    for pkg in new:
+        new[pkg] = new[pkg].split(',')[-1]
+
     ret = salt.utils.compare_dicts(old, new)
 
     if errors:
@@ -1176,6 +1182,11 @@ def upgrade(refresh=True,
     __zypper__(systemd_scope=_systemd_scope()).noraise.call(*cmd_update)
     __context__.pop('pkg.list_pkgs', None)
     new = list_pkgs()
+
+    # Handle packages which report multiple new versions
+    # (affects only kernel packages at this point)
+    for pkg in new:
+        new[pkg] = new[pkg].split(',')[-1]
     ret = salt.utils.compare_dicts(old, new)
 
     if __zypper__.exit_code not in __zypper__.SUCCESS_EXIT_CODES:

salt/spm/__init__.py

@@ -354,9 +354,8 @@ class SPMClient(object):
                 dl_url = dl_url.replace('file://', '')
                 shutil.copyfile(dl_url, out_file)
             else:
-                response = http.query(dl_url, text=True)
                 with salt.utils.fopen(out_file, 'w') as outf:
-                    outf.write(response.get('text'))
+                    outf.write(self._query_http(dl_path, repo_info['info']))
 
         # First we download everything, then we install
         for package in dl_list:
@@ -622,6 +621,45 @@ class SPMClient(object):
                         continue
                     callback(repo, repo_data[repo])
 
+    def _query_http(self, dl_path, repo_info):
+        '''
+        Download files via http
+        '''
+        query = None
+        response = None
+
+        try:
+            if 'username' in repo_info:
+                try:
+                    if 'password' in repo_info:
+                        query = http.query(
+                            dl_path, text=True,
+                            username=repo_info['username'],
+                            password=repo_info['password']
+                        )
+                    else:
+                        raise SPMException('Auth defined, but password is not set for username: \'{0}\''
+                                           .format(repo_info['username']))
+                except SPMException as exc:
+                    self.ui.error(str(exc))
+            else:
+                query = http.query(dl_path, text=True)
+        except SPMException as exc:
+            self.ui.error(str(exc))
+
+        try:
+            if query:
+                if 'SPM-METADATA' in dl_path:
+                    response = yaml.safe_load(query.get('text', '{}'))
+                else:
+                    response = query.get('text')
+            else:
+                raise SPMException('Response is empty, please check for Errors above.')
+        except SPMException as exc:
+            self.ui.error(str(exc))
+
+        return response
+
     def _download_repo_metadata(self, args):
         '''
         Connect to all repos and download metadata
@@ -635,8 +673,7 @@ class SPMClient(object):
                 with salt.utils.fopen(dl_path, 'r') as rpm:
                     metadata = yaml.safe_load(rpm)
             else:
-                response = http.query(dl_path, text=True)
+                metadata = self._query_http(dl_path, repo_info)
-                metadata = yaml.safe_load(response.get('text', '{}'))
 
             cache.store('.', repo, metadata)
 

salt/states/cron.py

@@ -160,7 +160,8 @@ def _check_cron(user,
                 dayweek=None,
                 comment=None,
                 commented=None,
-                identifier=None):
+                identifier=None,
+                special=None):
     '''
     Return the changes
     '''
@@ -181,6 +182,7 @@ def _check_cron(user,
     if cmd is not None:
         cmd = str(cmd)
     lst = __salt__['cron.list_tab'](user)
+    if special is None:
         for cron in lst['crons']:
             if _cron_matched(cron, cmd, identifier):
                 if any([_needs_change(x, y) for x, y in
@@ -191,6 +193,10 @@ def _check_cron(user,
                          (cron['commented'], commented))]):
                     return 'update'
                 return 'present'
+    else:
+        for cron in lst['special']:
+            if special == cron['spec'] and cmd == cron['cmd']:
+                return 'present'
     return 'absent'
 
 
@@ -311,7 +317,8 @@ def present(name,
                              dayweek=dayweek,
                              comment=comment,
                              commented=commented,
-                             identifier=identifier)
+                             identifier=identifier,
+                             special=special)
         ret['result'] = None
         if status == 'absent':
             ret['comment'] = 'Cron {0} is set to be added'.format(name)

salt/states/pkg.py

@@ -1177,7 +1177,7 @@ def installed(
         package, the held package(s) will be skipped and the state will fail.
         By default, this parameter is set to ``False``.
 
-        This option is currently supported only for YUM/DNF.
+        Currently works with YUM/DNF & APT based systems.
 
         .. versionadded:: 2016.11.0
 
@@ -2360,7 +2360,7 @@ def purged(name,
         return ret
 
 
-def uptodate(name, refresh=False, **kwargs):
+def uptodate(name, refresh=False, pkgs=None, **kwargs):
     '''
     .. versionadded:: 2014.7.0
 
@@ -2373,6 +2373,9 @@ def uptodate(name, refresh=False, **kwargs):
     refresh
         refresh the package database before checking for new upgrades
 
+    pkgs
+        list of packages to upgrade
+
     :param str cache_valid_time:
         This parameter sets the value in seconds after which cache marked as invalid,
         and cache update is necessary. This overwrite ``refresh`` parameter
@@ -2408,6 +2411,8 @@ def uptodate(name, refresh=False, **kwargs):
     if isinstance(refresh, bool):
         try:
             packages = __salt__['pkg.list_upgrades'](refresh=refresh, **kwargs)
+            if isinstance(pkgs, list):
+                packages = [pkg for pkg in packages if pkg in pkgs]
         except Exception as exc:
             ret['comment'] = str(exc)
             return ret
@@ -2425,7 +2430,7 @@ def uptodate(name, refresh=False, **kwargs):
         return ret
 
     try:
-        ret['changes'] = __salt__['pkg.upgrade'](refresh=refresh, **kwargs)
+        ret['changes'] = __salt__['pkg.upgrade'](refresh=refresh, pkgs=pkgs, **kwargs)
     except CommandExecutionError as exc:
         if exc.info:
             # Get information for state return from the exception.

salt/states/saltmod.py

@@ -694,6 +694,16 @@ def runner(name, **kwargs):
             'Unable to fire args event due to missing __orchestration_jid__'
         )
         jid = None
+
+    if __opts__.get('test', False):
+        ret = {
+            'name': name,
+            'result': True,
+            'changes': {},
+            'comment': "Runner function '{0}' would be executed.".format(name)
+        }
+        return ret
+
     out = __salt__['saltutil.runner'](name,
                                       __orchestration_jid__=jid,
                                       __env__=__env__,
@@ -751,6 +761,12 @@ def wheel(name, **kwargs):
             'Unable to fire args event due to missing __orchestration_jid__'
         )
         jid = None
+
+    if __opts__.get('test', False):
+        ret['result'] = True,
+        ret['comment'] = "Wheel function '{0}' would be executed.".format(name)
+        return ret
+
     out = __salt__['saltutil.wheel'](name,
                                      __orchestration_jid__=jid,
                                      __env__=__env__,

salt/utils/minions.py

@@ -251,12 +251,9 @@ class CkMinions(object):
         If not 'greedy' return the only minions have cache data and matched by the condition.
         '''
         cache_enabled = self.opts.get('minion_data_cache', False)
-        cdir = os.path.join(self.opts['cachedir'], 'minions')
 
         def list_cached_minions():
-            if not os.path.isdir(cdir):
+            return self.cache.list('minions')
-                return []
-            return os.listdir(cdir)
 
         if greedy:
             minions = []

salt/utils/network.py

@@ -98,8 +98,8 @@ def _generate_minion_id():
         Needs to work on Python 2.6, because of collections.OrderedDict only since 2.7 version.
         Override 'filter()' for custom filtering.
         '''
-        localhost_matchers = ['localhost.*', 'ip6-.*', '127.*', r'0\.0\.0\.0',
+        localhost_matchers = [r'localhost.*', r'ip6-.*', r'127[.]\d', r'0\.0\.0\.0',
-                              '::1.*', 'ipv6-.*', 'fe00::.*', 'fe02::.*', '1.0.0.*.ip6.arpa']
+                              r'::1.*', r'ipv6-.*', r'fe00::.*', r'fe02::.*', r'1.0.0.*.ip6.arpa']
 
         def append(self, p_object):
             if p_object and p_object not in self and not self.filter(p_object):

tests/unit/modules/test_zypper.py

@@ -376,6 +376,12 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin):
                     self.assertDictEqual(ret, {"vim": {"old": "1.1", "new": "1.2"}})
                     zypper_mock.assert_any_call('update', '--auto-agree-with-licenses')
 
+                with patch('salt.modules.zypper.list_pkgs',
+                           MagicMock(side_effect=[{"vim": "1.1"}, {"vim": "1.1,1.2"}])):
+                    ret = zypper.upgrade()
+                    self.assertDictEqual(ret, {"vim": {"old": "1.1", "new": "1.2"}})
+                    zypper_mock.assert_any_call('update', '--auto-agree-with-licenses')
+
                 with patch('salt.modules.zypper.list_pkgs', MagicMock(side_effect=[{"vim": "1.1"}, {"vim": "1.2"}])):
                     ret = zypper.upgrade(dist_upgrade=True)
                     self.assertDictEqual(ret, {"vim": {"old": "1.1", "new": "1.2"}})
@@ -396,6 +402,22 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin):
                     self.assertDictEqual(ret, {"vim": {"old": "1.1", "new": "1.2"}})
                     zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--from', "Dummy", '--from', 'Dummy2', '--no-allow-vendor-change')
 
+    def test_upgrade_kernel(self):
+        '''
+        Test kernel package upgrade success.
+
+        :return:
+        '''
+        with patch.dict(zypper.__grains__, {'osrelease_info': [12, 1]}), \
+             patch('salt.modules.zypper.refresh_db', MagicMock(return_value=True)), \
+             patch('salt.modules.zypper._systemd_scope', MagicMock(return_value=False)):
+            with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=(['kernel-default'], None))}):
+                with patch('salt.modules.zypper.__zypper__.noraise.call', MagicMock()):
+                    with patch('salt.modules.zypper.list_pkgs', MagicMock(side_effect=[
+                        {"kernel-default": "3.12.49-11.1"}, {"kernel-default": "3.12.49-11.1,3.12.51-60.20.2"}])):
+                        ret = zypper.install('kernel-default', '--auto-agree-with-licenses')
+                        self.assertDictEqual(ret, {"kernel-default": {"old": "3.12.49-11.1", "new": "3.12.51-60.20.2"}})
+
     def test_upgrade_failure(self):
         '''
         Test system upgrade failure.

tests/unit/utils/test_network.py

@@ -264,6 +264,38 @@ class NetworkTestCase(TestCase):
             self.assertEqual(network._generate_minion_id(),
                              ['hostname.domainname.blank', 'nodename', 'hostname', '1.2.3.4', '5.6.7.8'])
 
+    def test_generate_minion_id_127_name(self):
+        '''
+        Test if minion IDs can be named 127.foo
+
+        :return:
+        '''
+        with patch('platform.node', MagicMock(return_value='127')), \
+                patch('socket.gethostname', MagicMock(return_value='127')), \
+                patch('socket.getfqdn', MagicMock(return_value='127.domainname.blank')), \
+                patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'attrname', ('127.0.1.1', 0))])), \
+                patch('salt.utils.fopen', MagicMock(return_value=False)), \
+                patch('os.path.exists', MagicMock(return_value=False)), \
+                patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '5.6.7.8'])):
+            self.assertEqual(network._generate_minion_id(),
+                             ['127.domainname.blank', '127', '1.2.3.4', '5.6.7.8'])
+
+    def test_generate_minion_id_127_name_startswith(self):
+        '''
+        Test if minion IDs can be named starting from "127"
+
+        :return:
+        '''
+        with patch('platform.node', MagicMock(return_value='127890')), \
+                patch('socket.gethostname', MagicMock(return_value='127890')), \
+                patch('socket.getfqdn', MagicMock(return_value='127890.domainname.blank')), \
+                patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'attrname', ('127.0.1.1', 0))])), \
+                patch('salt.utils.fopen', MagicMock(return_value=False)), \
+                patch('os.path.exists', MagicMock(return_value=False)), \
+                patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '5.6.7.8'])):
+            self.assertEqual(network._generate_minion_id(),
+                             ['127890.domainname.blank', '127890', '1.2.3.4', '5.6.7.8'])
+
     def test_generate_minion_id_duplicate(self):
         '''
         Test if IP addresses in the minion IDs are distinct in the pool