Merge pull request #35276 from rallytime/merge-2016.3

[2016.3] Merge forward from 2015.8 to 2016.3

doc/topics/ssh/roster.rst

@@ -44,6 +44,10 @@ The information which can be stored in a roster ``target`` is the following:
         # Optional parameters
         port:        # The target system's ssh port number
         sudo:        # Boolean to run command via sudo
+        sudo_user:   # Str: Set this to execute Salt as a sudo user other than root.
+                     # This user must be in the same system group as the remote user
+                     # that is used to login and is specified above. Alternatively,
+                     # the user must be a super-user.
         tty:         # Boolean: Set this option to True if sudo is also set to
                      # True and requiretty is also set on the target system
         priv:        # File path to ssh private key, defaults to salt-ssh.rsa

doc/topics/tutorials/writing_tests.rst

@@ -418,9 +418,9 @@ against the ``return`` statement in the ``if`` clause.
 There are more examples of writing unit tests of varying complexities available
 in the following docs:
 
-* `Simple Unit Test Example<simple-unit-example>`
+* :ref:`Simple Unit Test Example<simple-unit-example>`
-* `Complete Unit Test Example<complete-unit-example>`
+* :ref:`Complete Unit Test Example<complete-unit-example>`
-* `Complex Unit Test Example<complex-unit-example>`
+* :ref:`Complex Unit Test Example<complex-unit-example>`
 
 .. note::
 
@@ -435,7 +435,7 @@ Automated Test Runs
 ===================
 
 SaltStack maintains a Jenkins server which can be viewed at
-http://jenkins.saltstack.com. The tests executed from this Jenkins server
+https://jenkins.saltstack.com. The tests executed from this Jenkins server
 create fresh virtual machines for each test run, then execute the destructive
 tests on the new, clean virtual machine. This allows for the execution of tests
 across supported platforms.

salt/client/ssh/__init__.py

@@ -134,6 +134,9 @@ SUDO=""
 if [ -n "{{SUDO}}" ]
     then SUDO="sudo "
 fi
+if [ "$SUDO" ]
+then SUDO="sudo -u {{SUDO_USER}}"
+fi
 EX_PYTHON_INVALID={EX_THIN_PYTHON_INVALID}
 PYTHON_CMDS="python3 python27 python2.7 python26 python2.6 python2 python"
 for py_cmd in $PYTHON_CMDS
@@ -271,6 +274,10 @@ class SSH(object):
                 'ssh_sudo',
                 salt.config.DEFAULT_MASTER_OPTS['ssh_sudo']
             ),
+            'sudo_user': self.opts.get(
+                'ssh_sudo_user',
+                salt.config.DEFAULT_MASTER_OPTS['ssh_sudo_user']
+            ),
             'identities_only': self.opts.get(
                 'ssh_identities_only',
                 salt.config.DEFAULT_MASTER_OPTS['ssh_identities_only']
@@ -656,6 +663,7 @@ class Single(object):
             mine=False,
             minion_opts=None,
             identities_only=False,
+            sudo_user=None,
             **kwargs):
         # Get mine setting and mine_functions if defined in kwargs (from roster)
         self.mine = mine
@@ -710,7 +718,8 @@ class Single(object):
                 'sudo': sudo,
                 'tty': tty,
                 'mods': self.mods,
-                'identities_only': identities_only}
+                'identities_only': identities_only,
+                'sudo_user': sudo_user}
         # Pre apply changeable defaults
         self.minion_opts = {
                     'grains_cache': True,
@@ -950,6 +959,7 @@ class Single(object):
         Prepare the command string
         '''
         sudo = 'sudo' if self.target['sudo'] else ''
+        sudo_user = self.target['sudo_user']
         if '_caller_cachedir' in self.opts:
             cachedir = self.opts['_caller_cachedir']
         else:
@@ -994,6 +1004,7 @@ ARGS = {10}\n'''.format(self.minion_config,
         cmd = SSH_SH_SHIM.format(
             DEBUG=debug,
             SUDO=sudo,
+            SUDO_USER=sudo_user,
             SSH_PY_CODE=py_code_enc,
             HOST_PY_MAJOR=sys.version_info[0],
         )

salt/client/ssh/shell.py

@@ -62,7 +62,8 @@ class Shell(object):
             sudo=False,
             tty=False,
             mods=None,
-            identities_only=False):
+            identities_only=False,
+            sudo_user=None):
         self.opts = opts
         self.host = host
         self.user = user

salt/client/ssh/ssh_py_shim.py

@@ -65,9 +65,14 @@ def need_deployment():
     # If SUDOing then also give the super user group write permissions
     sudo_gid = os.environ.get('SUDO_GID')
     if sudo_gid:
-        os.chown(OPTIONS.saltdir, -1, int(sudo_gid))
+        try:
-        stt = os.stat(OPTIONS.saltdir)
+            os.chown(OPTIONS.saltdir, -1, int(sudo_gid))
-        os.chmod(OPTIONS.saltdir, stt.st_mode | stat.S_IWGRP | stat.S_IRGRP | stat.S_IXGRP)
+            stt = os.stat(OPTIONS.saltdir)
+            os.chmod(OPTIONS.saltdir, stt.st_mode | stat.S_IWGRP | stat.S_IRGRP | stat.S_IXGRP)
+        except OSError:
+            sys.stdout.write('\n\nUnable to set permissions on thin directory.\nIf sudo_user is set '
+                    'and is not root, be certain the user is in the same group\nas the login user')
+            sys.exit(1)
 
     # Delimiter emitted on stdout *only* to indicate shim message to master.
     sys.stdout.write("{0}\ndeploy\n".format(OPTIONS.delimiter))

salt/config/__init__.py

@@ -772,6 +772,7 @@ VALID_OPTS = {
     'ssh_passwd': str,
     'ssh_port': str,
     'ssh_sudo': bool,
+    'ssh_sudo_user': str,
     'ssh_timeout': float,
     'ssh_user': str,
     'ssh_scan_ports': str,
@@ -1298,6 +1299,7 @@ DEFAULT_MASTER_OPTS = {
     'ssh_passwd': '',
     'ssh_port': '22',
     'ssh_sudo': False,
+    'ssh_sudo_user': '',
     'ssh_timeout': 60,
     'ssh_user': 'root',
     'ssh_scan_ports': '22',

salt/grains/core.py

@@ -1562,62 +1562,30 @@ def append_domain():
     return grain
 
 
-def ip4():
+def ip_fqdn():
     '''
-    Return a list of ipv4 addrs
+    Return ip address and FQDN grains
     '''
-
     if salt.utils.is_proxy():
         return {}
 
-    return {'ipv4': salt.utils.network.ip_addrs(include_loopback=True)}
+    ret = {}
+    ret['ipv4'] = salt.utils.network.ip_addrs(include_loopback=True)
+    ret['ipv6'] = salt.utils.network.ip_addrs6(include_loopback=True)
 
+    _fqdn = hostname()['fqdn']
+    for socket_type, ipv_num in ((socket.AF_INET, '4'), (socket.AF_INET6, '6')):
+        key = 'fqdn_ip' + ipv_num
+        if not ret['ipv' + ipv_num]:
+            ret[key] = []
+        else:
+            try:
+                info = socket.getaddrinfo(_fqdn, None, socket_type)
+                ret[key] = list(set(item[4][0] for item in info))
+            except socket.error:
+                ret[key] = []
 
-def fqdn_ip4():
+    return ret
-    '''
-    Return a list of ipv4 addrs of fqdn
-    '''
-
-    if salt.utils.is_proxy():
-        return {}
-
-    addrs = []
-    try:
-        hostname_grains = hostname()
-        info = socket.getaddrinfo(hostname_grains['fqdn'], None, socket.AF_INET)
-        addrs = list(set(item[4][0] for item in info))
-    except socket.error:
-        pass
-    return {'fqdn_ip4': addrs}
-
-
-def ip6():
-    '''
-    Return a list of ipv6 addrs
-    '''
-
-    if salt.utils.is_proxy():
-        return {}
-
-    return {'ipv6': salt.utils.network.ip_addrs6(include_loopback=True)}
-
-
-def fqdn_ip6():
-    '''
-    Return a list of ipv6 addrs of fqdn
-    '''
-
-    if salt.utils.is_proxy():
-        return {}
-
-    addrs = []
-    try:
-        hostname_grains = hostname()
-        info = socket.getaddrinfo(hostname_grains['fqdn'], None, socket.AF_INET6)
-        addrs = list(set(item[4][0] for item in info))
-    except socket.error:
-        pass
-    return {'fqdn_ip6': addrs}
 
 
 def ip_interfaces():

salt/modules/rpm.py

@@ -655,13 +655,21 @@ def version_cmp(ver1, ver2, ignore_epoch=False):
                     'more accurate version comparisons'
                 )
         else:
-            cmp_result = cmp_func(salt.utils.str_version_to_evr(ver1),
+            # If one EVR is missing a release but not the other and they
-                                  salt.utils.str_version_to_evr(ver2))
+            # otherwise would be equal, ignore the release. This can happen if
+            # e.g. you are checking if a package version 3.2 is satisfied by
+            # 3.2-1.
+            (ver1_e, ver1_v, ver1_r) = salt.utils.str_version_to_evr(ver1)
+            (ver2_e, ver2_v, ver2_r) = salt.utils.str_version_to_evr(ver2)
+            if not ver1_r or not ver2_r:
+                ver1_r = ver2_r = ''
+
+            cmp_result = cmp_func((ver1_e, ver1_v, ver1_r),
+                                  (ver2_e, ver2_v, ver2_r))
             if cmp_result not in (-1, 0, 1):
                 raise CommandExecutionError(
                     'Comparison result \'{0}\' is invalid'.format(cmp_result)
                 )
-
             return cmp_result
 
     except Exception as exc:

salt/output/highstate.py

@@ -443,7 +443,7 @@ def _format_host(host, data):
                                                line_max_len - 7)
         hstrs.append(colorfmt.format(colors['CYAN'], totals, colors))
 
-        if __opts__.get('state_output_profile', False):
+        if __opts__.get('state_output_profile', True):
             sum_duration = sum(rdurations)
             duration_unit = 'ms'
             # convert to seconds if duration is 1000ms or more
@@ -516,7 +516,7 @@ def _format_terse(tcolor, comps, ret, colors, tabular):
                 c=colors, w='\n'.join(ret['warnings'])
             )
         fmt_string += u'{0}'
-        if __opts__.get('state_output_profile', False):
+        if __opts__.get('state_output_profile', True):
             fmt_string += u'{6[start_time]!s} [{6[duration]!s} ms] '
         fmt_string += u'{2:>10}.{3:<10} {4:7}   Name: {1}{5}'
     elif isinstance(tabular, str):
@@ -528,7 +528,7 @@ def _format_terse(tcolor, comps, ret, colors, tabular):
                 c=colors, w='\n'.join(ret['warnings'])
             )
         fmt_string += u' {0} Name: {1} - Function: {2}.{3} - Result: {4}'
-        if __opts__.get('state_output_profile', False):
+        if __opts__.get('state_output_profile', True):
             fmt_string += u' Started: - {6[start_time]!s} Duration: {6[duration]!s} ms'
         fmt_string += u'{5}'
 

salt/states/git.py

@@ -839,16 +839,11 @@ def latest(name,
                         elif remote_rev_type == 'sha1':
                             has_remote_rev = True
 
-            # If has_remote_rev is False, then either the remote rev could not
+            # If fast_forward is not boolean, then we don't know if this will
-            # be found with git ls-remote (in which case we won't know more
+            # be a fast forward or not, because a fetch is requirde.
-            # until fetching) or we're going to be checking out a new branch
-            # and don't have to worry about fast-forwarding. So, we will set
-            # fast_forward to None (to signify uncertainty) unless there are
-            # local changes, in which case we will set it to False.
             fast_forward = None if not local_changes else False
 
             if has_remote_rev:
-                # Remote rev already present
                 if (not revs_match and not update_head) \
                         and (branch is None or branch == local_branch):
                     ret['comment'] = remote_loc.capitalize() \
@@ -861,25 +856,26 @@ def latest(name,
                     )
                     return ret
 
-            # No need to check if this is a fast_forward if we already know
+                # No need to check if this is a fast_forward if we already know
-            # that it won't be (due to local changes).
+                # that it won't be (due to local changes).
-            if fast_forward is not False:
+                if fast_forward is not False:
-                if base_rev is None:
+                    if base_rev is None:
-                    # If we're here, the remote_rev exists in the local
+                        # If we're here, the remote_rev exists in the local
-                    # checkout but there is still no HEAD locally. A possible
+                        # checkout but there is still no HEAD locally. A
-                    # reason for this is that an empty repository existed there
+                        # possible reason for this is that an empty repository
-                    # and a remote was added and fetched, but the repository
+                        # existed there and a remote was added and fetched, but
-                    # was not fast-forwarded. Regardless, going from no HEAD to
+                        # the repository was not fast-forwarded. Regardless,
-                    # a locally-present rev is considered a fast-forward
+                        # going from no HEAD to a locally-present rev is
-                    # update, unless there are local changes.
+                        # considered a fast-forward update, unless there are
-                    fast_forward = not bool(local_changes)
+                        # local changes.
-                else:
+                        fast_forward = not bool(local_changes)
-                    fast_forward = __salt__['git.merge_base'](
+                    else:
-                        target,
+                        fast_forward = __salt__['git.merge_base'](
-                        refs=[base_rev, remote_rev],
+                            target,
-                        is_ancestor=True,
+                            refs=[base_rev, remote_rev],
-                        user=user,
+                            is_ancestor=True,
-                        ignore_retcode=True)
+                            user=user,
+                            ignore_retcode=True)
 
             if fast_forward is False:
                 if not force_reset:

salt/states/pkg.py

@@ -706,6 +706,9 @@ def installed(
                   - dos2unix
                   - salt-minion: 2015.8.5-1.el6
 
+        If the version given is the string ``latest``, the latest available
+        package version will be installed à la ``pkg.latest``.
+
     :param bool refresh:
         This parameter controls whether or not the packge repo database is
         updated prior to installing the requested package(s).

salt/utils/gitfs.py

@@ -857,6 +857,7 @@ class GitPython(GitProvider):
         while True:
             depth += 1
             if depth > SYMLINK_RECURSE_DEPTH:
+                blob = None
                 break
             try:
                 file_blob = tree / path
@@ -878,6 +879,7 @@ class GitPython(GitProvider):
                     break
             except KeyError:
                 # File not found or repo_path points to a directory
+                blob = None
                 break
         return blob, blob.hexsha if blob is not None else blob
 
@@ -1400,6 +1402,7 @@ class Pygit2(GitProvider):
         while True:
             depth += 1
             if depth > SYMLINK_RECURSE_DEPTH:
+                blob = None
                 break
             try:
                 if stat.S_ISLNK(tree[path].filemode):
@@ -1414,7 +1417,9 @@ class Pygit2(GitProvider):
                 else:
                     oid = tree[path].oid
                     blob = self.repo[oid]
+                    break
             except KeyError:
+                blob = None
                 break
         return blob, blob.hex if blob is not None else blob
 
@@ -1750,6 +1755,7 @@ class Dulwich(GitProvider):  # pylint: disable=abstract-method
         while True:
             depth += 1
             if depth > SYMLINK_RECURSE_DEPTH:
+                blob = None
                 break
             prefix_dirs, _, filename = path.rpartition(os.path.sep)
             tree = self.walk_tree(tree, prefix_dirs)
@@ -1771,6 +1777,7 @@ class Dulwich(GitProvider):  # pylint: disable=abstract-method
                     blob = self.repo.get_object(oid)
                     break
             except KeyError:
+                blob = None
                 break
         return blob, blob.sha().hexdigest() if blob is not None else blob
 

tests/integration/states/git.py

@@ -197,6 +197,62 @@ class GitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
         finally:
             shutil.rmtree(name, ignore_errors=True)
 
+    def test_latest_fast_forward(self):
+        '''
+        Test running git.latest state a second time after changes have been
+        made to the remote repo.
+        '''
+        def _head(cwd):
+            return self.run_function('git.rev_parse', [cwd, 'HEAD'])
+
+        repo_url = 'https://{0}/saltstack/salt-test-repo.git'.format(self.__domain)
+        mirror_dir = os.path.join(integration.TMP, 'salt_repo_mirror')
+        mirror_url = 'file://' + mirror_dir
+        admin_dir = os.path.join(integration.TMP, 'salt_repo_admin')
+        clone_dir = os.path.join(integration.TMP, 'salt_repo')
+
+        try:
+            # Mirror the repo
+            self.run_function('git.clone',
+                              [mirror_dir, repo_url, None, '--mirror'])
+
+            # Make sure the directory for the mirror now exists
+            self.assertTrue(os.path.exists(mirror_dir))
+
+            # Clone the mirror twice, once to the admin location and once to
+            # the clone_dir
+            ret = self.run_state('git.latest', name=mirror_url, target=admin_dir)
+            self.assertSaltTrueReturn(ret)
+            ret = self.run_state('git.latest', name=mirror_url, target=clone_dir)
+            self.assertSaltTrueReturn(ret)
+
+            # Make a change to the repo by editing the file in the admin copy
+            # of the repo and committing.
+            head_pre = _head(admin_dir)
+            with open(os.path.join(admin_dir, 'LICENSE'), 'a') as fp_:
+                fp_.write('Hello world!')
+            self.run_function('git.commit', [admin_dir, 'Added a line', '-a'])
+            # Make sure HEAD is pointing to a new SHA so we know we properly
+            # committed our change.
+            head_post = _head(admin_dir)
+            self.assertNotEqual(head_pre, head_post)
+
+            # Push the change to the mirror
+            # NOTE: the test will fail if the salt-test-repo's default branch
+            # is changed.
+            self.run_function('git.push', [admin_dir, 'origin', 'develop'])
+
+            # Re-run the git.latest state on the clone_dir
+            ret = self.run_state('git.latest', name=mirror_url, target=clone_dir)
+            self.assertSaltTrueReturn(ret)
+
+            # Make sure that the clone_dir now has the correct SHA
+            self.assertEqual(head_post, _head(clone_dir))
+
+        finally:
+            for path in (mirror_dir, admin_dir, clone_dir):
+                shutil.rmtree(path, ignore_errors=True)
+
     def test_present(self):
         '''
         git.present