Merge pull request #36443 from rallytime/merge-carbon

[carbon] Merge forward from 2016.3 to carbon

doc/topics/beacons/index.rst

@@ -109,7 +109,7 @@ a change is detected.
     ``salt myminion pkg.install python-inotify``.
 
 First, on the Salt minion, add the following beacon configuration to
-``/ect/salt/minion``:
+``/etc/salt/minion``:
 
 .. code-block:: yaml
 

requirements/windows.txt

@@ -0,0 +1,10 @@
+backports-abc
+backports.ssl-match-hostname
+certifi
+psutil
+python-dateutil
+pypiwin32
+pyzmq
+six
+timelib
+WMI

salt/client/__init__.py

@@ -134,7 +134,7 @@ class LocalClient(object):
     def __init__(self,
                  c_path=os.path.join(syspaths.CONFIG_DIR, 'master'),
                  mopts=None, skip_perm_errors=False,
-                 io_loop=None):
+                 io_loop=None, keep_loop=False):
         '''
         :param IOLoop io_loop: io_loop used for events.
                                Pass in an io_loop if you want asynchronous
@@ -163,7 +163,8 @@ class LocalClient(object):
                 self.opts['transport'],
                 opts=self.opts,
                 listen=False,
-                io_loop=io_loop)
+                io_loop=io_loop,
+                keep_loop=keep_loop)
         self.utils = salt.loader.utils(self.opts)
         self.functions = salt.loader.minion_mods(self.opts, utils=self.utils)
         self.returners = salt.loader.returners(self.opts, self.functions)

salt/fileclient.py

@@ -709,12 +709,17 @@ class Client(object):
         elif not os.path.isabs(cachedir):
             cachedir = os.path.join(self.opts['cachedir'], cachedir)
 
+        if url_data.query is not None:
+            file_name = '-'.join([url_data.path, url_data.query])
+        else:
+            file_name = url_data.path
+
         return salt.utils.path_join(
             cachedir,
             'extrn_files',
             saltenv,
             netloc,
-            url_data.path
+            file_name
         )
 
 

salt/grains/core.py

@@ -1005,6 +1005,7 @@ _OS_NAME_MAP = {
     'enterprise': 'OEL',
     'oracleserv': 'OEL',
     'cloudserve': 'CloudLinux',
+    'cloudlinux': 'CloudLinux',
     'pidora': 'Fedora',
     'scientific': 'ScientificLinux',
     'synology': 'Synology',
@@ -1289,7 +1290,12 @@ def os_data():
                             grains[
                                 'lsb_{0}'.format(match.groups()[0].lower())
                             ] = match.groups()[1].rstrip()
-            if 'lsb_distrib_id' not in grains:
+            if grains.get('lsb_distrib_description', '').lower().startswith('antergos'):
+                # Antergos incorrectly configures their /etc/lsb-release,
+                # setting the DISTRIB_ID to "Arch". This causes the "os" grain
+                # to be incorrectly set to "Arch".
+                grains['osfullname'] = 'Antergos Linux'
+            elif 'lsb_distrib_id' not in grains:
                 if os.path.isfile('/etc/os-release') or os.path.isfile('/usr/lib/os-release'):
                     os_release = _parse_os_release()
                     if 'NAME' in os_release:

salt/modules/aptpkg.py

@@ -1048,14 +1048,20 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs):
         force_conf = '--force-confnew'
     else:
         force_conf = '--force-confold'
-
     cmd = []
     if salt.utils.systemd.has_scope(__context__) \
             and __salt__['config.get']('systemd.scope', True):
         cmd.extend(['systemd-run', '--scope'])
+
     cmd.extend(['apt-get', '-q', '-y',
                 '-o', 'DPkg::Options::={0}'.format(force_conf),
                 '-o', 'DPkg::Options::=--force-confdef'])
+
+    if kwargs.get('force_yes', False):
+        cmd.append('--force-yes')
+    if kwargs.get('skip_verify', False):
+        cmd.append('--allow-unauthenticated')
+
     cmd.append('dist-upgrade' if dist_upgrade else 'upgrade')
 
     call = __salt__['cmd.run_all'](cmd,

salt/modules/boto_ec2.py

@@ -958,7 +958,7 @@ def create_key(key_name, save_path, region=None, key=None, keyid=None,
 
     .. code-block:: bash
 
-        salt myminion boto_ec2.create mykey /root/
+        salt myminion boto_ec2.create_key mykey /root/
     '''
     conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
 

salt/modules/boto_iam.py

@@ -1299,6 +1299,7 @@ def get_account_id(region=None, key=None, keyid=None, profile=None):
             # The get_user call returns an user ARN:
             #    arn:aws:iam::027050522557:user/salt-test
             arn = ret['get_user_response']['get_user_result']['user']['arn']
+            account_id = arn.split(':')[4]
         except boto.exception.BotoServerError:
             # If call failed, then let's try to get the ARN from the metadata
             timeout = boto.config.getfloat(
@@ -1307,15 +1308,15 @@ def get_account_id(region=None, key=None, keyid=None, profile=None):
             attempts = boto.config.getint(
                 'Boto', 'metadata_service_num_attempts', 1
             )
-            metadata = boto.utils.get_instance_metadata(
+            identity = boto.utils.get_instance_identity(
                 timeout=timeout, num_retries=attempts
             )
             try:
-                arn = metadata['iam']['info']['InstanceProfileArn']
+                account_id = identity['document']['accountId']
             except KeyError:
-                log.error('Failed to get user or metadata ARN information in'
+                log.error('Failed to get account id from instance_identity in'
                           ' boto_iam.get_account_id.')
-        __context__[cache_key] = arn.split(':')[4]
+        __context__[cache_key] = account_id
     return __context__[cache_key]
 
 

salt/modules/boto_sns.py

@@ -227,6 +227,9 @@ def get_arn(name, region=None, key=None, keyid=None, profile=None):
 def _get_region(region=None, profile=None):
     if profile and 'region' in profile:
         return profile['region']
+    if not region and __salt__['config.option'](profile):
+        _profile = __salt__['config.option'](profile)
+        region = _profile.get('region', None)
     if not region and __salt__['config.option']('sns.region'):
         region = __salt__['config.option']('sns.region')
     if not region:

salt/modules/pacman.py

@@ -36,7 +36,7 @@ def __virtual__():
     '''
     Set the virtual pkg module if the os is Arch
     '''
-    if __grains__['os'] in ('Arch', 'Arch ARM', 'Antergos', 'ManjaroLinux'):
+    if __grains__['os_family'] == 'Arch':
         return __virtualname__
     return (False, 'The pacman module could not be loaded: unsupported OS family.')
 
@@ -155,10 +155,17 @@ def list_upgrades(refresh=False, root=None, **kwargs):  # pylint: disable=W0613
         out = call['stdout']
 
     for line in salt.utils.itertools.split(out, '\n'):
-        comps = line.split(' ')
-        if len(comps) != 2:
+        try:
+            pkgname, pkgver = line.split()
+        except ValueError:
             continue
-        upgrades[comps[0]] = comps[1]
+        if pkgname.lower() == 'downloading' and '.db' in pkgver.lower():
+            # Antergos (and possibly other Arch derivatives) add lines when pkg
+            # metadata is being downloaded. Because these lines, when split,
+            # contain two columns (i.e. 'downloading community.db...'), we will
+            # skip this line to keep it from being interpreted as an upgrade.
+            continue
+        upgrades[pkgname] = pkgver
     return upgrades
 
 

salt/modules/win_service.py

@@ -406,7 +406,13 @@ def stop(name):
 
         salt '*' service.stop <service name>
     '''
-    if not status(name):
+    # net stop issues a stop command and waits briefly (~30s), but will give
+    # up if the service takes too long to stop with a misleading
+    # "service could not be stopped" message and RC 0.
+
+    cmd = ['net', 'stop', '/y', name]
+    res = __salt__['cmd.run'](cmd, python_shell=False)
+    if 'service was stopped' in res:
         return True
 
     try:

salt/netapi/rest_cherrypy/app.py

@@ -900,7 +900,7 @@ def lowdata_fmt():
     # if the data was sent as urlencoded, we need to make it a list.
     # this is a very forgiving implementation as different clients set different
     # headers for form encoded data (including charset or something similar)
-    if data and not isinstance(data, list):
+    if data and isinstance(data, collections.Mapping):
         # Make the 'arg' param a list if not already
         if 'arg' in data and not isinstance(data['arg'], list):
             data['arg'] = [data['arg']]

salt/state.py

@@ -1818,6 +1818,8 @@ class State(object):
         Check if the low data chunk should send a failhard signal
         '''
         tag = _gen_tag(low)
+        if self.opts['test']:
+            return False
         if (low.get('failhard', False) or self.opts['failhard']
                 and tag in running):
             if running[tag]['result'] is None:

salt/states/git.py

@@ -1142,6 +1142,20 @@ def latest(name,
                 else:
                     branch_opts = None
 
+                if branch_opts is not None and local_branch is None:
+                    return _fail(
+                        ret,
+                        'Cannot set/unset upstream tracking branch, local '
+                        'HEAD refers to nonexistent branch. This may have '
+                        'been caused by cloning a remote repository for which '
+                        'the default branch was renamed or deleted. If you '
+                        'are unable to fix the remote repository, you can '
+                        'work around this by setting the \'branch\' argument '
+                        '(which will ensure that the named branch is created '
+                        'if it does not already exist).',
+                        comments
+                    )
+
                 if not has_remote_rev:
                     try:
                         fetch_changes = __salt__['git.fetch'](
@@ -1561,6 +1575,21 @@ def latest(name,
                     local_rev, local_branch = \
                         _get_local_rev_and_branch(target, user, password)
 
+                    if local_branch is None \
+                            and remote_rev is not None \
+                            and 'HEAD' not in all_remote_refs:
+                        return _fail(
+                            ret,
+                            'Remote HEAD refers to a ref that does not exist. '
+                            'This can happen when the default branch on the '
+                            'remote repository is renamed or deleted. If you '
+                            'are unable to fix the remote repository, you can '
+                            'work around this by setting the \'branch\' argument '
+                            '(which will ensure that the named branch is created '
+                            'if it does not already exist).',
+                            comments
+                        )
+
                     if not _revs_equal(local_rev, remote_rev, remote_rev_type):
                         __salt__['git.reset'](
                             target,

salt/utils/event.py

@@ -111,7 +111,7 @@ TAGS = {
 
 def get_event(
         node, sock_dir=None, transport='zeromq',
-        opts=None, listen=True, io_loop=None):
+        opts=None, listen=True, io_loop=None, keep_loop=False):
     '''
     Return an event object suitable for the named transport
 
@@ -124,8 +124,8 @@ def get_event(
     # TODO: AIO core is separate from transport
     if transport in ('zeromq', 'tcp'):
         if node == 'master':
-            return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop)
-        return SaltEvent(node, sock_dir, opts, listen=listen, io_loop=io_loop)
+            return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop, keep_loop=keep_loop)
+        return SaltEvent(node, sock_dir, opts, listen=listen, io_loop=io_loop, keep_loop=keep_loop)
     elif transport == 'raet':
         import salt.utils.raetevent
         return salt.utils.raetevent.RAETEvent(node,
@@ -197,14 +197,19 @@ class SaltEvent(object):
     '''
     def __init__(
             self, node, sock_dir=None,
-            opts=None, listen=True, io_loop=None):
+            opts=None, listen=True, io_loop=None, keep_loop=False):
         '''
         :param IOLoop io_loop: Pass in an io_loop if you want asynchronous
                                operation for obtaining events. Eg use of
                                set_event_handler() API. Otherwise, operation
                                will be synchronous.
+        :param Bool keep_loop: Pass a boolean to determine if we want to keep
+                               the io loop or destroy it when the event handle
+                               is destroyed. This is useful when using event
+                               loops from within third party async code
         '''
         self.serial = salt.payload.Serial({'serial': 'msgpack'})
+        self.keep_loop = keep_loop
         if io_loop is not None:
             self.io_loop = io_loop
             self._run_io_loop_sync = False
@@ -727,7 +732,7 @@ class SaltEvent(object):
             self.subscriber.close()
         if self.pusher is not None:
             self.pusher.close()
-        if self._run_io_loop_sync:
+        if self._run_io_loop_sync and not self.keep_loop:
             self.io_loop.close()
 
     def fire_ret_load(self, load):
@@ -790,9 +795,20 @@ class MasterEvent(SaltEvent):
     RAET compatible
     Create a master event management object
     '''
-    def __init__(self, sock_dir, opts=None, listen=True, io_loop=None):
+    def __init__(
+            self,
+            sock_dir,
+            opts=None,
+            listen=True,
+            io_loop=None,
+            keep_loop=False):
         super(MasterEvent, self).__init__(
-            'master', sock_dir, opts, listen=listen, io_loop=io_loop)
+            'master',
+            sock_dir,
+            opts,
+            listen=listen,
+            io_loop=io_loop,
+            keep_loop=keep_loop)
 
 
 class LocalClientEvent(MasterEvent):

setup.py

@@ -119,6 +119,7 @@ SALT_VERSION = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', 'version.py'
 SALT_VERSION_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_version.py')
 SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_syspaths.py')
 SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt')
+SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'windows.txt')
 SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt')
 SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt')
 
@@ -384,11 +385,11 @@ class InstallPyCryptoWindowsWheel(Command):
         call_arguments = ['pip', 'install', 'wheel']
         if platform_bits == '64bit':
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl'
+                'https://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl'
             )
         else:
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl'
+                'https://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl'
             )
         with indent_log():
             call_subprocess(call_arguments)
@@ -415,11 +416,11 @@ class InstallCompiledPyYaml(Command):
         call_arguments = ['easy_install', '-Z']
         if platform_bits == '64bit':
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe'
+                'https://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe'
             )
         else:
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win-amd64-py2.7.exe'
+                'https://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win32-py2.7.exe'
             )
         with indent_log():
             call_subprocess(call_arguments)
@@ -442,7 +443,7 @@ class DownloadWindowsDlls(Command):
         import platform
         from pip.utils.logging import indent_log
         platform_bits, _ = platform.architecture()
-        url = 'http://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll'
+        url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll'
         dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll')
         with indent_log():
             for fname in ('libeay32', 'ssleay32', 'libsodium', 'msvcr120'):
@@ -1023,8 +1024,7 @@ class SaltDistribution(distutils.dist.Distribution):
         install_requires = _parse_requirements_file(SALT_REQS)
 
         if IS_WINDOWS_PLATFORM:
-            install_requires.append('WMI')
-            install_requires.append('pypiwin32 >= 219')
+            install_requires += _parse_requirements_file(SALT_WINDOWS_REQS)
 
         if self.salt_transport == 'zeromq':
             install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS)

tests/integration/states/git.py

@@ -8,6 +8,7 @@ from __future__ import absolute_import
 import os
 import shutil
 import socket
+import string
 import subprocess
 import tempfile
 
@@ -328,6 +329,136 @@ class GitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
         self.assertSaltTrueReturn(ret)
 
 
+@skip_if_binaries_missing('git')
+class LocalRepoGitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
+    '''
+    Tests which do no require connectivity to github.com
+    '''
+    def test_renamed_default_branch(self):
+        '''
+        Test the case where the remote branch has been removed
+        https://github.com/saltstack/salt/issues/36242
+        '''
+        cwd = os.getcwd()
+        repo = tempfile.mkdtemp(dir=integration.TMP)
+        admin = tempfile.mkdtemp(dir=integration.TMP)
+        name = tempfile.mkdtemp(dir=integration.TMP)
+        for dirname in (repo, admin, name):
+            self.addCleanup(shutil.rmtree, dirname, ignore_errors=True)
+        self.addCleanup(os.chdir, cwd)
+
+        with salt.utils.fopen(os.devnull, 'w') as devnull:
+            # Create bare repo
+            subprocess.check_call(['git', 'init', '--bare', repo],
+                                  stdout=devnull, stderr=devnull)
+            # Clone bare repo
+            subprocess.check_call(['git', 'clone', repo, admin],
+                                  stdout=devnull, stderr=devnull)
+
+            # Create, add, commit, and push file
+            os.chdir(admin)
+            with salt.utils.fopen('foo', 'w'):
+                pass
+            subprocess.check_call(['git', 'add', '.'],
+                                  stdout=devnull, stderr=devnull)
+            subprocess.check_call(['git', 'commit', '-m', 'init'],
+                                  stdout=devnull, stderr=devnull)
+            subprocess.check_call(['git', 'push', 'origin', 'master'],
+                                  stdout=devnull, stderr=devnull)
+
+        # Change back to the original cwd
+        os.chdir(cwd)
+
+        # Rename remote 'master' branch to 'develop'
+        os.rename(
+            os.path.join(repo, 'refs', 'heads', 'master'),
+            os.path.join(repo, 'refs', 'heads', 'develop')
+        )
+
+        # Run git.latest state. This should successfuly clone and fail with a
+        # specific error in the comment field.
+        ret = self.run_state(
+            'git.latest',
+            name=repo,
+            target=name,
+            rev='develop',
+        )
+        self.assertSaltFalseReturn(ret)
+        self.assertEqual(
+            ret[next(iter(ret))]['comment'],
+            'Remote HEAD refers to a ref that does not exist. '
+            'This can happen when the default branch on the '
+            'remote repository is renamed or deleted. If you '
+            'are unable to fix the remote repository, you can '
+            'work around this by setting the \'branch\' argument '
+            '(which will ensure that the named branch is created '
+            'if it does not already exist).\n\n'
+            'Changes already made: {0} cloned to {1}'
+            .format(repo, name)
+        )
+        self.assertEqual(
+            ret[next(iter(ret))]['changes'],
+            {'new': '{0} => {1}'.format(repo, name)}
+        )
+
+        # Run git.latest state again. This should fail again, with a different
+        # error in the comment field, and should not change anything.
+        ret = self.run_state(
+            'git.latest',
+            name=repo,
+            target=name,
+            rev='develop',
+        )
+        self.assertSaltFalseReturn(ret)
+        self.assertEqual(
+            ret[next(iter(ret))]['comment'],
+            'Cannot set/unset upstream tracking branch, local '
+            'HEAD refers to nonexistent branch. This may have '
+            'been caused by cloning a remote repository for which '
+            'the default branch was renamed or deleted. If you '
+            'are unable to fix the remote repository, you can '
+            'work around this by setting the \'branch\' argument '
+            '(which will ensure that the named branch is created '
+            'if it does not already exist).'
+        )
+        self.assertEqual(ret[next(iter(ret))]['changes'], {})
+
+        # Run git.latest state again with a branch manually set. This should
+        # checkout a new branch and the state should pass.
+        ret = self.run_state(
+            'git.latest',
+            name=repo,
+            target=name,
+            rev='develop',
+            branch='develop',
+        )
+        # State should succeed
+        self.assertSaltTrueReturn(ret)
+        self.assertSaltCommentRegexpMatches(
+            ret,
+            'New branch \'develop\' was checked out, with origin/develop '
+            r'\([0-9a-f]{7}\) as a starting point'
+        )
+        # Only the revision should be in the changes dict.
+        self.assertEqual(
+            list(ret[next(iter(ret))]['changes'].keys()),
+            ['revision']
+        )
+        # Since the remote repo was incorrectly set up, the local head should
+        # not exist (therefore the old revision should be None).
+        self.assertEqual(
+            ret[next(iter(ret))]['changes']['revision']['old'],
+            None
+        )
+        # Make sure the new revision is a SHA (40 chars, all hex)
+        self.assertTrue(
+            len(ret[next(iter(ret))]['changes']['revision']['new']) == 40)
+        self.assertTrue(
+            all([x in string.hexdigits for x in
+                 ret[next(iter(ret))]['changes']['revision']['new']])
+        )
+
+
 if __name__ == '__main__':
     from integration import run_tests
     run_tests(GitTest)