Merge branch '2016.3' into 'develop'

Conflicts:
  - salt/client/mixins.py
  - salt/grains/core.py
  - salt/modules/saltutil.py
  - salt/runner.py
  - salt/states/saltmod.py
  - salt/wheel/__init__.py
  - tests/unit/states/saltmod_test.py

salt/cli/salt.py

@@ -252,8 +252,8 @@ class SaltCMD(parsers.SaltCMDOptionParser):
             print_cli('---------------------------')
             print_cli('Errors')
             print_cli('---------------------------')
-            for minion in errors:
-                print_cli(self._format_error(minion))
+            for error in errors:
+                print_cli(self._format_error(error))
 
     def _print_returns_summary(self, ret):
         '''
@@ -376,13 +376,16 @@ class SaltCMD(parsers.SaltCMDOptionParser):
         if isinstance(ret, str):
             self.exit(2, '{0}\n'.format(ret))
         for host in ret:
-            if ret[host] == 'Minion did not return. [Not connected]':
+            if isinstance(ret[host], string_types) and ret[host].startswith("Minion did not return"):
                 continue
             for fun in ret[host]:
-                if fun not in docs:
-                    if ret[host][fun]:
-                        docs[fun] = ret[host][fun]
-        for fun in sorted(docs):
-            salt.output.display_output(fun + ':', 'nested', self.config)
-            print_cli(docs[fun])
-            print_cli('')
+                if fun not in docs and ret[host][fun]:
+                    docs[fun] = ret[host][fun]
+        if self.options.output:
+            for fun in sorted(docs):
+                salt.output.display_output({fun: docs[fun]}, 'nested', self.config)
+        else:
+            for fun in sorted(docs):
+                print_cli('{0}:'.format(fun))
+                print_cli(docs[fun])
+                print_cli('')

salt/client/mixins.py

@@ -167,7 +167,7 @@ class SyncClientMixin(object):
 
         return ret['data']['return']
 
-    def cmd(self, fun, arg=None, pub_data=None, kwarg=None, full_return=False):
+    def cmd(self, fun, arg=None, pub_data=None, kwarg=None, print_event=True, full_return=False):
         '''
         Execute a function
 
@@ -227,7 +227,7 @@ class SyncClientMixin(object):
         low = {'fun': fun,
                'arg': args,
                'kwarg': kwargs}
-        return self.low(fun, low, full_return=full_return)
+        return self.low(fun, low, print_event=print_event, full_return=full_return)
 
     @property
     def mminion(self):
@@ -235,7 +235,7 @@ class SyncClientMixin(object):
             self._mminion = salt.minion.MasterMinion(self.opts, states=False, rend=False)
         return self._mminion
 
-    def low(self, fun, low, full_return=False):
+    def low(self, fun, low, print_event=True, full_return=False):
         '''
         Check for deprecated usage and allow until Salt Oxygen.
         '''
@@ -288,12 +288,18 @@ class SyncClientMixin(object):
                 opts=self.opts,
                 listen=False)
 
+        if print_event:
+            print_func = self.print_async_event \
+                if hasattr(self, 'print_async_event') \
+                else None
+        else:
+            # Suppress printing of return event (this keeps us from printing
+            # runner/wheel output during orchestration).
+            print_func = None
         namespaced_event = salt.utils.event.NamespacedEvent(
             event,
             tag,
-            print_func=self.print_async_event
-                       if hasattr(self, 'print_async_event')
-                       else None
+            print_func=print_func
         )
         # TODO: document these, and test that they exist
         # TODO: Other things to inject??

salt/grains/core.py

@@ -1401,8 +1401,7 @@ def os_data():
                 grains.pop('lsb_distrib_release', None)
             grains['osrelease'] = \
                 grains.get('lsb_distrib_release', osrelease).strip()
-        grains['oscodename'] = grains.get('lsb_distrib_codename',
-                                          oscodename).strip()
+        grains['oscodename'] = grains.get('lsb_distrib_codename', '').strip() or oscodename
         if 'Red Hat' in grains['oscodename']:
             grains['oscodename'] = oscodename
         distroname = _REPLACE_LINUX_RE.sub('', grains['osfullname']).strip()
@@ -1519,9 +1518,12 @@ def os_data():
                 continue
             osrelease_info[idx] = int(value)
         grains['osrelease_info'] = tuple(osrelease_info)
-        grains['osmajorrelease'] = grains['osrelease_info'][0]
-        os_name = 'os' if grains.get('os') in ('FreeBSD', 'OpenBSD', 'NetBSD', 'Mac', 'Raspbian') else 'osfullname'
-        grains['osfinger'] = '{0}-{1}'.format(grains[os_name], grains['osrelease_info'][0])
+        grains['osmajorrelease'] = str(grains['osrelease_info'][0])  # This will be an integer in the two releases
+        salt.utils.warn_until('Nitrogen', 'The "osmajorrelease" will be a type of an integer.')
+        os_name = grains['os' if grains.get('os') in (
+            'FreeBSD', 'OpenBSD', 'NetBSD', 'Mac', 'Raspbian') else 'osfullname']
+        grains['osfinger'] = '{0}-{1}'.format(
+            os_name, grains['osrelease'] if os_name in ('Ubuntu',) else grains['osrelease_info'][0])
 
     return grains
 

salt/minion.py

@@ -517,6 +517,8 @@ class MinionBase(object):
             conn = False
             # shuffle the masters and then loop through them
             opts['local_masters'] = copy.copy(opts['master'])
+            if opts['random_master']:
+                shuffle(opts['local_masters'])
             last_exc = None
 
             while True:
@@ -570,6 +572,8 @@ class MinionBase(object):
 
         # single master sign in
         else:
+            if opts['random_master']:
+                log.warning('random_master is True but there is only one master specified. Ignoring.')
             while True:
                 attempts += 1
                 if tries > 0:

salt/modules/mac_shadow.py

@@ -12,7 +12,13 @@ of a configuration profile.
 
 from __future__ import absolute_import
 from datetime import datetime
-import pwd
+
+
+try:
+    import pwd
+    HAS_PWD = True
+except ImportError:
+    HAS_PWD = False
 
 # Import salt libs
 import salt.utils
@@ -30,7 +36,10 @@ def __virtual__():
     if not salt.utils.is_darwin():
         return False, 'Not Darwin'
 
-    return __virtualname__
+    if HAS_PWD:
+        return __virtualname__
+    else:
+        return (False, 'The pwd module failed to load.')
 
 
 def _get_account_policy(name):

salt/modules/rpmbuild.py

@@ -425,7 +425,7 @@ def make_repo(repodir, keyid=None, env=None, use_passphrase=False, gnupghome='/e
         for file in os.listdir(repodir):
             if file.endswith('.rpm'):
                 abs_file = os.path.join(repodir, file)
-                number_retries = 5
+                number_retries = 20
                 times_looped = 0
                 error_msg = 'Failed to sign file {0}'.format(abs_file)
                 cmd = 'rpm {0} --addsign {1}'.format(define_gpg_name, abs_file)

salt/modules/saltutil.py

@@ -1170,7 +1170,7 @@ def runner(name, **kwargs):
             prefix='run'
         )
 
-    return rclient.cmd(name, kwarg=kwargs, full_return=True)
+    return rclient.cmd(name, kwarg=kwargs, print_event=False, full_return=True)
 
 
 def wheel(name, *args, **kwargs):
@@ -1249,6 +1249,7 @@ def wheel(name, *args, **kwargs):
                                arg=args,
                                pub_data=pub_data,
                                kwarg=valid_kwargs,
+                               print_event=False,
                                full_return=True)
     except SaltInvocationError:
         raise CommandExecutionError(

salt/output/highstate.py

@@ -216,7 +216,8 @@ def _format_host(host, data):
                               .format(ret.get('duration', 0)))
 
             tcolor = colors['GREEN']
-            schanged, ctext = _format_changes(ret['changes'])
+            orchestration = ret.get('__orchestration__', False)
+            schanged, ctext = _format_changes(ret['changes'], orchestration)
             nchanges += 1 if schanged else 0
 
             # Skip this state if it was successful & diff output was requested
@@ -469,15 +470,37 @@ def _format_host(host, data):
     return u'\n'.join(hstrs), nchanges > 0
 
 
-def _format_changes(changes):
+def _nested_changes(changes):
     '''
-    Format the changes dict based on what the data is
+    Print the changes data using the nested outputter
     '''
     global __opts__  # pylint: disable=W0601
 
+    opts = __opts__.copy()
+    # Pass the __opts__ dict. The loader will splat this modules __opts__ dict
+    # anyway so have to restore it after the other outputter is done
+    if __opts__['color']:
+        __opts__['color'] = u'CYAN'
+    __opts__['nested_indent'] = 14
+    ret = u'\n'
+    ret += salt.output.out_format(
+            changes,
+            'nested',
+            __opts__)
+    __opts__ = opts
+    return ret
+
+
+def _format_changes(changes, orchestration=False):
+    '''
+    Format the changes dict based on what the data is
+    '''
     if not changes:
         return False, u''
 
+    if orchestration:
+        return True, _nested_changes(changes)
+
     if not isinstance(changes, dict):
         return True, u'Invalid Changes data: {0}'.format(changes)
 
@@ -491,18 +514,7 @@ def _format_changes(changes):
             changed = changed or c
     else:
         changed = True
-        opts = __opts__.copy()
-        # Pass the __opts__ dict. The loader will splat this modules __opts__ dict
-        # anyway so have to restore it after the other outputter is done
-        if __opts__['color']:
-            __opts__['color'] = u'CYAN'
-        __opts__['nested_indent'] = 14
-        ctext = u'\n'
-        ctext += salt.output.out_format(
-                changes,
-                'nested',
-                __opts__)
-        __opts__ = opts
+        ctext = _nested_changes(changes)
     return changed, ctext
 
 

salt/pillar/mysql.py

@@ -11,17 +11,6 @@ This module is a concrete implementation of the sql_base ext_pillar for MySQL.
 :depends: python-mysqldb
 :platform: all
 
-Legacy compatibility
-=====================================
-
-This module has an extra addition for backward compatibility.
-
-If there's a keyword arg of mysql_query, that'll go first before other args.
-This legacy compatibility translates to depth 1.
-
-We do this so that it's backward compatible with older configs.
-This is deprecated and slated to be removed in Carbon.
-
 Configuring the mysql ext_pillar
 =====================================
 

salt/runner.py

@@ -132,7 +132,7 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object):
         reformatted_low = self._reformat_low(low)
         return mixins.SyncClientMixin.cmd_sync(self, reformatted_low, timeout)
 
-    def cmd(self, fun, arg=None, pub_data=None, kwarg=None, full_return=False):
+    def cmd(self, fun, arg=None, pub_data=None, kwarg=None, print_event=True, full_return=False):
         '''
         Execute a function
         '''
@@ -140,6 +140,7 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object):
                                              arg,
                                              pub_data,
                                              kwarg,
+                                             print_event,
                                              full_return)
 
 

salt/runners/jobs.py

@@ -507,14 +507,19 @@ def exit_success(jid, ext_source=None):
     '''
     ret = dict()
 
-    data = lookup_jid(
+    data = list_job(
         jid,
         ext_source=ext_source
     )
 
-    for minion in data:
-        if "retcode" in data[minion]:
-            ret[minion] = True if not data[minion]['retcode'] else False
+    minions = data['Minions']
+    result = data['Result']
+
+    for minion in minions:
+        if minion in result and 'return' in result[minion]:
+            ret[minion] = True if result[minion]['return'] else False
+        else:
+            ret[minion] = False
 
     return ret
 

salt/states/saltmod.py

@@ -648,6 +648,7 @@ def runner(name, **kwargs):
     ret['result'] = True
     ret['comment'] = "Runner function '{0}' executed.".format(name)
 
+    ret['__orchestration__'] = True
     if 'jid' in out:
         ret['__jid__'] = out['jid']
 
@@ -692,6 +693,7 @@ def wheel(name, **kwargs):
     ret['result'] = True
     ret['comment'] = "Wheel function '{0}' executed.".format(name)
 
+    ret['__orchestration__'] = True
     if 'jid' in out:
         ret['__jid__'] = out['jid']
 

salt/wheel/__init__.py

@@ -118,7 +118,7 @@ class WheelClient(salt.client.mixins.SyncClientMixin,
         fun = low.pop('fun')
         return self.async(fun, low)
 
-    def cmd(self, fun, arg=None, pub_data=None, kwarg=None, full_return=False):
+    def cmd(self, fun, arg=None, pub_data=None, kwarg=None, print_event=True, full_return=False):
         '''
         Execute a function
 
@@ -131,6 +131,7 @@ class WheelClient(salt.client.mixins.SyncClientMixin,
                                             arg,
                                             pub_data,
                                             kwarg,
+                                            print_event,
                                             full_return)
 
 

tests/unit/grains/core_test.py

@@ -388,6 +388,77 @@ PATCHLEVEL = 3
         }
         self._run_suse_os_grains_tests(_os_release_map)
 
+    @skipIf(not salt.utils.is_linux(), 'System is not Linux')
+    def test_suse_os_grains_ubuntu(self):
+        '''
+        Test if OS grains are parsed correctly in Ubuntu Xenial Xerus
+        '''
+        _os_release_map = {
+            'os_release_file': {
+                'NAME': 'Ubuntu',
+                'VERSION': '16.04.1 LTS (Xenial Xerus)',
+                'VERSION_ID': '16.04',
+                'PRETTY_NAME': '',
+                'ID': 'ubuntu',
+            },
+            'oscodename': 'xenial',
+            'osfullname': 'Ubuntu',
+            'osrelease': '16.04',
+            'osrelease_info': [16, 4],
+            'osmajorrelease': '16',
+            'osfinger': 'Ubuntu-16.04',
+        }
+        self._run_ubuntu_os_grains_tests(_os_release_map)
+
+    def _run_ubuntu_os_grains_tests(self, os_release_map):
+        path_isfile_mock = MagicMock(side_effect=lambda x: x in ['/etc/os-release'])
+        empty_mock = MagicMock(return_value={})
+        osarch_mock = MagicMock(return_value="amd64")
+        os_release_mock = MagicMock(return_value=os_release_map.get('os_release_file'))
+
+        orig_import = __import__
+
+        def _import_mock(name, *args):
+            if name == 'lsb_release':
+                raise ImportError('No module named lsb_release')
+            return orig_import(name, *args)
+
+        # Skip the first if statement
+        with patch.object(salt.utils, 'is_proxy',
+                          MagicMock(return_value=False)):
+            # Skip the selinux/systemd stuff (not pertinent)
+            with patch.object(core, '_linux_bin_exists',
+                              MagicMock(return_value=False)):
+                # Skip the init grain compilation (not pertinent)
+                with patch.object(os.path, 'exists', path_isfile_mock):
+                    # Ensure that lsb_release fails to import
+                    with patch('__builtin__.__import__',
+                               side_effect=_import_mock):
+                        # Skip all the /etc/*-release stuff (not pertinent)
+                        with patch.object(os.path, 'isfile', path_isfile_mock):
+                            with patch.object(core, '_parse_os_release', os_release_mock):
+                                # Mock platform.linux_distribution to give us the
+                                # OS name that we want.
+                                distro_mock = MagicMock(return_value=('Ubuntu', '16.04', 'xenial'))
+                                with patch("salt.utils.fopen", mock_open()) as suse_release_file:
+                                    suse_release_file.return_value.__iter__.return_value = os_release_map.get(
+                                        'suse_release_file', '').splitlines()
+                                    with patch.object(platform, 'linux_distribution', distro_mock):
+                                        with patch.object(core, '_linux_gpu_data', empty_mock):
+                                            with patch.object(core, '_linux_cpudata', empty_mock):
+                                                with patch.object(core, '_virtual', empty_mock):
+                                                    # Mock the osarch
+                                                    with patch.dict(core.__salt__, {'cmd.run': osarch_mock}):
+                                                        os_grains = core.os_data()
+
+        self.assertEqual(os_grains.get('os'), 'Ubuntu')
+        self.assertEqual(os_grains.get('os_family'), 'Debian')
+        self.assertEqual(os_grains.get('osfullname'), os_release_map['osfullname'])
+        self.assertEqual(os_grains.get('oscodename'), os_release_map['oscodename'])
+        self.assertEqual(os_grains.get('osrelease'), os_release_map['osrelease'])
+        self.assertListEqual(list(os_grains.get('osrelease_info')), os_release_map['osrelease_info'])
+        self.assertEqual(os_grains.get('osmajorrelease'), os_release_map['osmajorrelease'])
+
 
 if __name__ == '__main__':
     from integration import run_tests

tests/unit/states/saltmod_test.py

@@ -171,7 +171,8 @@ class SaltmodTestCase(TestCase):
         name = 'state'
 
         ret = {'changes': True, 'name': 'state', 'result': True,
-               'comment': "Runner function 'state' executed."}
+               'comment': 'Runner function \'state\' executed.',
+               '__orchestration__': True}
         runner_mock = MagicMock(return_value={'return': True})
 
         with patch.dict(saltmod.__salt__, {'saltutil.runner': runner_mock}):
@@ -186,7 +187,8 @@ class SaltmodTestCase(TestCase):
         name = 'state'
 
         ret = {'changes': True, 'name': 'state', 'result': True,
-               'comment': "Wheel function 'state' executed."}
+               'comment': 'Wheel function \'state\' executed.',
+               '__orchestration__': True}
         wheel_mock = MagicMock(return_value={'return': True})
 
         with patch.dict(saltmod.__salt__, {'saltutil.wheel': wheel_mock}):