Merge branch 'carbon' into 'develop'

No conflicts.

doc/faq.rst

@@ -14,10 +14,10 @@ open and proprietary projects.
 
 To expand on this a little:
 
-There is much argument over the actual definition of "open core".  From our standpoint, Salt is open source because 
+There is much argument over the actual definition of "open core".  From our standpoint, Salt is open source because
 
 1. It is a standalone product that anyone is free to use.
-2. It is developed in the open with contributions accepted from the community for the good of the project. 
+2. It is developed in the open with contributions accepted from the community for the good of the project.
 3. There are no features of Salt itself that are restricted to separate proprietary products distributed by SaltStack, Inc.
 4. Because of our Apache 2.0 license, Salt can be used as the foundation for a project or even a proprietary tool.
 5. Our APIs are open and documented (any lack of documentation is an oversight as opposed to an intentional decision by SaltStack the company) and available for use by anyone.

requirements/windows.txt

@@ -0,0 +1,10 @@
+backports-abc
+backports.ssl-match-hostname
+certifi
+psutil
+python-dateutil
+pypiwin32
+pyzmq
+six
+timelib
+WMI

salt/client/__init__.py

@@ -134,7 +134,7 @@ class LocalClient(object):
     def __init__(self,
                  c_path=os.path.join(syspaths.CONFIG_DIR, 'master'),
                  mopts=None, skip_perm_errors=False,
-                 io_loop=None):
+                 io_loop=None, keep_loop=False):
         '''
         :param IOLoop io_loop: io_loop used for events.
                                Pass in an io_loop if you want asynchronous
@@ -163,7 +163,8 @@ class LocalClient(object):
                 self.opts['transport'],
                 opts=self.opts,
                 listen=False,
-                io_loop=io_loop)
+                io_loop=io_loop,
+                keep_loop=keep_loop)
         self.utils = salt.loader.utils(self.opts)
         self.functions = salt.loader.minion_mods(self.opts, utils=self.utils)
         self.returners = salt.loader.returners(self.opts, self.functions)

salt/fileclient.py

@@ -709,12 +709,17 @@ class Client(object):
         elif not os.path.isabs(cachedir):
             cachedir = os.path.join(self.opts['cachedir'], cachedir)
 
+        if url_data.query is not None:
+            file_name = '-'.join([url_data.path, url_data.query])
+        else:
+            file_name = url_data.path
+
         return salt.utils.path_join(
             cachedir,
             'extrn_files',
             saltenv,
             netloc,
-            url_data.path
+            file_name
         )
 
 

salt/grains/core.py

@@ -1005,6 +1005,7 @@ _OS_NAME_MAP = {
     'enterprise': 'OEL',
     'oracleserv': 'OEL',
     'cloudserve': 'CloudLinux',
+    'cloudlinux': 'CloudLinux',
     'pidora': 'Fedora',
     'scientific': 'ScientificLinux',
     'synology': 'Synology',
@@ -1289,7 +1290,12 @@ def os_data():
                             grains[
                                 'lsb_{0}'.format(match.groups()[0].lower())
                             ] = match.groups()[1].rstrip()
-            if 'lsb_distrib_id' not in grains:
+            if grains.get('lsb_distrib_description', '').lower().startswith('antergos'):
+                # Antergos incorrectly configures their /etc/lsb-release,
+                # setting the DISTRIB_ID to "Arch". This causes the "os" grain
+                # to be incorrectly set to "Arch".
+                grains['osfullname'] = 'Antergos Linux'
+            elif 'lsb_distrib_id' not in grains:
                 if os.path.isfile('/etc/os-release') or os.path.isfile('/usr/lib/os-release'):
                     os_release = _parse_os_release()
                     if 'NAME' in os_release:

salt/modules/aptpkg.py

@@ -1048,14 +1048,20 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs):
         force_conf = '--force-confnew'
     else:
         force_conf = '--force-confold'
-
     cmd = []
     if salt.utils.systemd.has_scope(__context__) \
             and __salt__['config.get']('systemd.scope', True):
         cmd.extend(['systemd-run', '--scope'])
+
     cmd.extend(['apt-get', '-q', '-y',
                 '-o', 'DPkg::Options::={0}'.format(force_conf),
                 '-o', 'DPkg::Options::=--force-confdef'])
+
+    if kwargs.get('force_yes', False):
+        cmd.append('--force-yes')
+    if kwargs.get('skip_verify', False):
+        cmd.append('--allow-unauthenticated')
+
     cmd.append('dist-upgrade' if dist_upgrade else 'upgrade')
 
     call = __salt__['cmd.run_all'](cmd,

salt/modules/boto_iam.py

@@ -1299,6 +1299,7 @@ def get_account_id(region=None, key=None, keyid=None, profile=None):
             # The get_user call returns an user ARN:
             #    arn:aws:iam::027050522557:user/salt-test
             arn = ret['get_user_response']['get_user_result']['user']['arn']
+            account_id = arn.split(':')[4]
         except boto.exception.BotoServerError:
             # If call failed, then let's try to get the ARN from the metadata
             timeout = boto.config.getfloat(
@@ -1307,15 +1308,15 @@ def get_account_id(region=None, key=None, keyid=None, profile=None):
             attempts = boto.config.getint(
                 'Boto', 'metadata_service_num_attempts', 1
             )
-            metadata = boto.utils.get_instance_metadata(
+            identity = boto.utils.get_instance_identity(
                 timeout=timeout, num_retries=attempts
             )
             try:
-                arn = metadata['iam']['info']['InstanceProfileArn']
+                account_id = identity['document']['accountId']
             except KeyError:
-                log.error('Failed to get user or metadata ARN information in'
+                log.error('Failed to get account id from instance_identity in'
                           ' boto_iam.get_account_id.')
-        __context__[cache_key] = arn.split(':')[4]
+        __context__[cache_key] = account_id
     return __context__[cache_key]
 
 

salt/modules/boto_sns.py

@@ -227,6 +227,9 @@ def get_arn(name, region=None, key=None, keyid=None, profile=None):
 def _get_region(region=None, profile=None):
     if profile and 'region' in profile:
         return profile['region']
+    if not region and __salt__['config.option'](profile):
+        _profile = __salt__['config.option'](profile)
+        region = _profile.get('region', None)
     if not region and __salt__['config.option']('sns.region'):
         region = __salt__['config.option']('sns.region')
     if not region:

salt/modules/cmdmod.py

@@ -20,6 +20,7 @@ import time
 import traceback
 import fnmatch
 import base64
+import re
 
 # Import salt libs
 import salt.utils
@@ -2643,6 +2644,151 @@ def shells():
     return ret
 
 
+def shell_info(shell):
+    '''
+    Provides information about a shell or languages often use #!
+    The values returned are dependant on the shell or languages all return the
+    ``installed``, ``path``, ``version``, ``version_major`` and
+    ``version_major_minor`` e.g. 5.0 or 6-3.
+    The shell must be within the exeuctable search path.
+
+    :param str shell: Name of the shell.
+    Support are bash, cmd, perl, php, powershell, python, ruby and zsh
+    :return: Properies of the shell specifically its and other information if
+    available.
+    :rtype: dict
+
+    .. code-block:: cfg
+        {'version': '<version string>',
+         'version_major': '<version string',
+         'version_minor': '<version string>',
+         'path': '<full path to binary>',
+         'installed': <True, False or None>,
+         '<attribute>': '<attribute value>'}
+
+    ``installed`` is always returned, if None or False also returns error and
+     may also return stdout for diagnostics.
+
+    .. versionadded:: 2016.9.0
+
+    CLI Example::
+    .. code-block:: bash
+        salt '*' cmd.shell bash
+        salt '*' cmd.shell powershell
+
+    :codeauthor: Damon Atkins <https://github.com/damon-atkins>
+    '''
+    regex_shells = {
+        'bash': [r'version ([-\w.]+)', 'bash', '--version'],
+        'bash-test-error': [r'versioZ ([-\w.]+)', 'bash', '--version'],  # used to test a error result
+        'bash-test-env': [r'(HOME=.*)', 'bash', '-c', 'declare'],  # used to test a error result
+        'zsh': [r'^zsh ([\d.]+)', 'zsh', '--version'],
+        'tcsh': [r'^tcsh ([\d.]+)', 'tcsh', '--version'],
+        'cmd': [r'Version ([\d.]+)', 'cmd.exe', '/C', 'ver'],
+        'powershell': [r'PSVersion\s+([\d.]+)', 'powershell', '-NonInteractive', '$PSVersionTable'],
+        'perl': [r'^([\d.]+)', 'perl', '-e', 'printf "%vd\n", $^V;'],
+        'python': [r'^Python ([\d.]+)', 'python', '-V'],
+        'ruby': [r'^ruby ([\d.]+)', 'ruby', '-v'],
+        'php': [r'^PHP ([\d.]+)', 'php', '-v']
+    }
+    # Ensure ret['installed'] always as a value of True, False or None (not sure)
+    ret = {}
+    ret['installed'] = None
+    if salt.utils.is_windows() and shell == 'powershell':
+        pw_keys = __salt__['reg.list_keys']('HKEY_LOCAL_MACHINE', 'Software\\Microsoft\\PowerShell')
+        pw_keys.sort(key=int)
+        if len(pw_keys) == 0:
+            return {
+                'error': 'Unable to locate \'powershell\' Reason: Cannot be found in registry.',
+                'installed': False,
+                'version': None
+            }
+        for reg_ver in pw_keys:
+            install_data = __salt__['reg.read_value']('HKEY_LOCAL_MACHINE', 'Software\\Microsoft\\PowerShell\\{0}'.format(reg_ver), 'Install')
+            if 'vtype' in install_data and install_data['vtype'] == 'REG_DWORD' and install_data['vdata'] == 1:
+                details = __salt__['reg.list_values']('HKEY_LOCAL_MACHINE', 'Software\\Microsoft\\PowerShell\\{0}\\PowerShellEngine'.format(reg_ver))
+                ret = {}  # reset data, want the newest version details only as powershell is backwards compatible
+                ret['installed'] = True
+                ret['path'] = which('powershell.exe')
+                for attribute in details:
+                    if attribute['vname'].lower() == '(default)':
+                        continue
+                    elif attribute['vname'].lower() == 'powershellversion':
+                        ret['psversion'] = attribute['vdata']
+                        ret['version'] = attribute['vdata']
+                    elif attribute['vname'].lower() == 'runtimeversion':
+                        ret['crlversion'] = attribute['vdata']
+                        if ret['crlversion'][0] == 'v' or ret['crlversion'][0] == 'V':
+                            ret['crlversion'] = ret['crlversion'][1::]
+                    elif attribute['vname'].lower() == 'pscompatibleversion':
+                        # reg attribute does not end in s, the powershell attibute does
+                        ret['pscompatibleversions'] = attribute['vdata']
+                    else:
+                        # keys are lower case as python is case sensitive the registry is not
+                        ret[attribute['vname'].lower()] = attribute['vdata']
+    else:
+        if shell not in regex_shells:
+            return {
+                'error': 'Salt does not know how to get the version number for {0}'.format(shell),
+                'installed': None
+            }
+        shell_data = regex_shells[shell]
+        pattern = shell_data.pop(0)
+        # We need to make sure HOME set, so shells work correctly
+        # salt-call will general have home set, the salt-minion service may not
+        # We need to assume ports of unix shells to windows will look after themselves
+        # in setting HOME as they do it in many different ways
+        newenv = os.environ
+        if ('HOME' not in newenv) and (not salt.utils.is_windows()):
+            newenv['HOME'] = os.path.expanduser('~')
+            log.debug('HOME environment set to {0}'.format(newenv['HOME']))
+        try:
+            proc = salt.utils.timed_subprocess.TimedProc(
+                shell_data,
+                stdin=None,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                timeout=10,
+                env=newenv
+                )
+        except (OSError, IOError) as exc:
+            return {
+                'error': 'Unable to run command \'{0}\' Reason: {1}'.format(' '.join(shell_data), exc),
+                'installed': False,
+                'version': None
+            }
+        try:
+            proc.run()
+        except TimedProcTimeoutError as exc:
+            return {
+                'error': 'Unable to run command \'{0}\' Reason: Timed out.'.format(' '.join(shell_data)),
+                'installed': False,
+                'version': None
+            }
+
+        ret['installed'] = True
+        ret['path'] = which(shell_data[0])
+        pattern_result = re.search(pattern, proc.stdout, flags=re.IGNORECASE)
+        # only set version if we find it, so code later on can deal with it
+        if pattern_result:
+            ret['version'] = pattern_result.group(1)
+
+    if 'version' not in ret:
+        ret['error'] = 'The version regex pattern for shell {0}, could not find the version string'.format(shell)
+        ret['version'] = None
+        ret['stdout'] = proc.stdout  # include stdout so they can see the issue
+        log.error(ret['error'])
+    else:
+        major_result = re.match('(\\d+)', ret['version'])
+        if major_result:
+            ret['version_major'] = major_result.group(1)
+            major_minor_result = re.match('(\\d+[-.]\\d+)', ret['version'])
+            if major_minor_result:
+                ret['version_major_minor'] = major_minor_result.group(1)
+
+    return ret
+
+
 def powershell(cmd,
         cwd=None,
         stdin=None,

salt/modules/pacman.py

@@ -36,7 +36,7 @@ def __virtual__():
     '''
     Set the virtual pkg module if the os is Arch
     '''
-    if __grains__['os'] in ('Arch', 'Arch ARM', 'Antergos', 'ManjaroLinux'):
+    if __grains__['os_family'] == 'Arch':
         return __virtualname__
     return (False, 'The pacman module could not be loaded: unsupported OS family.')
 
@@ -155,10 +155,17 @@ def list_upgrades(refresh=False, root=None, **kwargs):  # pylint: disable=W0613
         out = call['stdout']
 
     for line in salt.utils.itertools.split(out, '\n'):
-        comps = line.split(' ')
-        if len(comps) != 2:
+        try:
+            pkgname, pkgver = line.split()
+        except ValueError:
             continue
-        upgrades[comps[0]] = comps[1]
+        if pkgname.lower() == 'downloading' and '.db' in pkgver.lower():
+            # Antergos (and possibly other Arch derivatives) add lines when pkg
+            # metadata is being downloaded. Because these lines, when split,
+            # contain two columns (i.e. 'downloading community.db...'), we will
+            # skip this line to keep it from being interpreted as an upgrade.
+            continue
+        upgrades[pkgname] = pkgver
     return upgrades
 
 

salt/modules/win_dsc.py

@@ -7,13 +7,14 @@ Module for managing PowerShell modules
 
 Support for PowerShell
 '''
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import python libs
 import copy
 import logging
 import json
 import os
+import distutils.version
 
 # Import salt libs
 import salt.utils
@@ -33,25 +34,39 @@ def __virtual__():
     if not salt.utils.is_windows():
         return (False, 'Module DSC: Module only works on Windows systems ')
 
-    if psversion() < 5:
-        return (False, 'Module DSC: Module only works with PowerShell 5 or later.')
+    powershell_info = __salt__['cmd.shell_info']('powershell')
+    if (
+           powershell_info['installed'] and
+           ('version_major' in powershell_info) and
+           (distutils.version.LooseVersion(powershell_info['version_major']) < distutils.version.LooseVersion('5'))
+        ):
+        return (False, 'Module DSC: Module only works with PowerShell 5 or newer.')
 
     return __virtualname__
 
 
-def _pshell(cmd, cwd=None):
+def _pshell(cmd, cwd=None, json_depth=2):
     '''
     Execute the desired powershell command and ensure that it returns data
     in json format and load that into python
     '''
     if 'convertto-json' not in cmd.lower():
-        cmd = ' '.join([cmd, '| ConvertTo-Json'])
+        cmd = '{0} | ConvertTo-Json -Depth {1}'.format(cmd, json_depth)
     log.debug('DSC: {0}'.format(cmd))
-    ret = __salt__['cmd.shell'](cmd, shell='powershell', cwd=cwd)
+    results = __salt__['cmd.run_all'](cmd, shell='powershell', cwd=cwd, python_shell=True)
+
+    if 'pid' in results:
+        del results['pid']
+
+    if 'retcode' not in results or results['retcode'] != 0:
+        # run_all logs an error to log.error, fail hard back to the user
+        raise CommandExecutionError('Issue executing powershell {0}'.format(cmd), info=results)
+
     try:
-        ret = json.loads(ret, strict=False)
+        ret = json.loads(results['stdout'], strict=False)
     except ValueError:
-        log.debug('Json not returned')
+        raise CommandExecutionError('No JSON results from powershell', info=results)
+
     return ret
 
 
@@ -92,15 +107,27 @@ def psversion():
     '''
     Returns the Powershell version
 
+    This has been deprecated and has been replaced by ``cmd.shell_info`` Note
+    the minimum version return is 5 as ``dsc`` is not available for version
+    less than 5.  This function will be removed in 'Nitrogen' release.
+
     CLI Example:
 
     .. code-block:: bash
 
         salt 'win01' dsc.psversion
     '''
-    cmd = '$PSVersionTable.PSVersion.Major'
-    ret = _pshell(cmd)
-    return ret
+    salt.utils.warn_until('Nitrogen',
+        'The \'psversion\' has been deprecated and has been '
+        'replaced by \'cmd.shell_info\'.'
+    )
+    powershell_info = __salt__['cmd.shell_info']('powershell')
+    if powershell_info['installed'] and 'version_major' in powershell_info:
+        try:
+            return int(powershell_info['version_major'])
+        except ValueError:
+            pass
+    return 0
 
 
 def avail_modules(desc=False):

salt/modules/win_psget.py

@@ -8,15 +8,17 @@ Module for managing PowerShell through PowerShellGet (PSGet)
 
 Support for PowerShell
 '''
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import python libs
 import copy
 import logging
 import json
+import distutils.version
 
 # Import salt libs
 import salt.utils
+from salt.exceptions import CommandExecutionError
 
 # Set up logging
 log = logging.getLogger(__name__)
@@ -32,25 +34,39 @@ def __virtual__():
     if not salt.utils.is_windows():
         return (False, 'Module PSGet: Module only works on Windows systems ')
 
-    if psversion() < 5:
-        return (False, 'Module PSGet: Module only works with PowerShell 5 or later.')
+    powershell_info = __salt__['cmd.shell_info']('powershell')
+    if (
+           powershell_info['installed'] and
+           ('version_major' in powershell_info) and
+           (distutils.version.LooseVersion(powershell_info['version_major']) < distutils.version.LooseVersion('5'))
+       ):
+        return (False, 'Module PSGet: Module only works with PowerShell 5 or newer.')
 
     return __virtualname__
 
 
-def _pshell(cmd, cwd=None):
+def _pshell(cmd, cwd=None, json_depth=2):
     '''
     Execute the desired powershell command and ensure that it returns data
     in json format and load that into python
     '''
     if 'convertto-json' not in cmd.lower():
-        cmd = ' '.join([cmd, '| ConvertTo-Json'])
-    log.debug('PSGET: {0}'.format(cmd))
-    ret = __salt__['cmd.shell'](cmd, shell='powershell', cwd=cwd)
+        cmd = '{0} | ConvertTo-Json -Depth {1}'.format(cmd, json_depth)
+    log.debug('DSC: {0}'.format(cmd))
+    results = __salt__['cmd.run_all'](cmd, shell='powershell', cwd=cwd, python_shell=True)
+
+    if 'pid' in results:
+        del results['pid']
+
+    if 'retcode' not in results or results['retcode'] != 0:
+        # run_all logs an error to log.error, fail hard back to the user
+        raise CommandExecutionError('Issue executing powershell {0}'.format(cmd), info=results)
+
     try:
-        ret = json.loads(ret, strict=False)
+        ret = json.loads(results['stdout'], strict=False)
     except ValueError:
-        log.debug('Json not returned')
+        raise CommandExecutionError('No JSON results from powershell', info=results)
+
     return ret
 
 
@@ -58,15 +74,27 @@ def psversion():
     '''
     Returns the Powershell version
 
+    This has been deprecated and has been replaced by ``cmd.shell_info`` Note
+    the minimum version return is 5 as ``dsc`` is not available for version
+    less than 5.  This function will be removed in 'Nitrogen' release.
+
     CLI Example:
 
     .. code-block:: bash
 
         salt 'win01' dsc.psversion
     '''
-    cmd = '$PSVersionTable.PSVersion.Major'
-    ret = _pshell(cmd)
-    return ret
+    salt.utils.warn_until('Nitrogen',
+        'The \'psversion\' has been deprecated and has been '
+        'replaced by \'cmd.shell_info\'.'
+    )
+    powershell_info = __salt__['cmd.shell_info']('powershell')
+    if powershell_info['installed'] and 'version_major' in powershell_info:
+        try:
+            return int(powershell_info['version_major'])
+        except ValueError:
+            pass
+    return 0
 
 
 def bootstrap():

salt/modules/win_service.py

@@ -406,7 +406,13 @@ def stop(name):
 
         salt '*' service.stop <service name>
     '''
-    if not status(name):
+    # net stop issues a stop command and waits briefly (~30s), but will give
+    # up if the service takes too long to stop with a misleading
+    # "service could not be stopped" message and RC 0.
+
+    cmd = ['net', 'stop', '/y', name]
+    res = __salt__['cmd.run'](cmd, python_shell=False)
+    if 'service was stopped' in res:
         return True
 
     try:

salt/netapi/rest_cherrypy/app.py

@@ -900,7 +900,7 @@ def lowdata_fmt():
     # if the data was sent as urlencoded, we need to make it a list.
     # this is a very forgiving implementation as different clients set different
     # headers for form encoded data (including charset or something similar)
-    if data and not isinstance(data, list):
+    if data and isinstance(data, collections.Mapping):
         # Make the 'arg' param a list if not already
         if 'arg' in data and not isinstance(data['arg'], list):
             data['arg'] = [data['arg']]

salt/state.py

@@ -1832,6 +1832,8 @@ class State(object):
         Check if the low data chunk should send a failhard signal
         '''
         tag = _gen_tag(low)
+        if self.opts['test']:
+            return False
         if (low.get('failhard', False) or self.opts['failhard']
                 and tag in running):
             if running[tag]['result'] is None:

salt/states/git.py

@@ -1158,6 +1158,20 @@ def latest(name,
                 else:
                     branch_opts = None
 
+                if branch_opts is not None and local_branch is None:
+                    return _fail(
+                        ret,
+                        'Cannot set/unset upstream tracking branch, local '
+                        'HEAD refers to nonexistent branch. This may have '
+                        'been caused by cloning a remote repository for which '
+                        'the default branch was renamed or deleted. If you '
+                        'are unable to fix the remote repository, you can '
+                        'work around this by setting the \'branch\' argument '
+                        '(which will ensure that the named branch is created '
+                        'if it does not already exist).',
+                        comments
+                    )
+
                 if not has_remote_rev:
                     try:
                         fetch_changes = __salt__['git.fetch'](
@@ -1577,6 +1591,21 @@ def latest(name,
                     local_rev, local_branch = \
                         _get_local_rev_and_branch(target, user, password)
 
+                    if local_branch is None \
+                            and remote_rev is not None \
+                            and 'HEAD' not in all_remote_refs:
+                        return _fail(
+                            ret,
+                            'Remote HEAD refers to a ref that does not exist. '
+                            'This can happen when the default branch on the '
+                            'remote repository is renamed or deleted. If you '
+                            'are unable to fix the remote repository, you can '
+                            'work around this by setting the \'branch\' argument '
+                            '(which will ensure that the named branch is created '
+                            'if it does not already exist).',
+                            comments
+                        )
+
                     if not _revs_equal(local_rev, remote_rev, remote_rev_type):
                         __salt__['git.reset'](
                             target,

salt/utils/event.py

@@ -111,7 +111,7 @@ TAGS = {
 
 def get_event(
         node, sock_dir=None, transport='zeromq',
-        opts=None, listen=True, io_loop=None):
+        opts=None, listen=True, io_loop=None, keep_loop=False):
     '''
     Return an event object suitable for the named transport
 
@@ -124,8 +124,8 @@ def get_event(
     # TODO: AIO core is separate from transport
     if transport in ('zeromq', 'tcp'):
         if node == 'master':
-            return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop)
-        return SaltEvent(node, sock_dir, opts, listen=listen, io_loop=io_loop)
+            return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop, keep_loop=keep_loop)
+        return SaltEvent(node, sock_dir, opts, listen=listen, io_loop=io_loop, keep_loop=keep_loop)
     elif transport == 'raet':
         import salt.utils.raetevent
         return salt.utils.raetevent.RAETEvent(node,
@@ -197,14 +197,19 @@ class SaltEvent(object):
     '''
     def __init__(
             self, node, sock_dir=None,
-            opts=None, listen=True, io_loop=None):
+            opts=None, listen=True, io_loop=None, keep_loop=False):
         '''
         :param IOLoop io_loop: Pass in an io_loop if you want asynchronous
                                operation for obtaining events. Eg use of
                                set_event_handler() API. Otherwise, operation
                                will be synchronous.
+        :param Bool keep_loop: Pass a boolean to determine if we want to keep
+                               the io loop or destroy it when the event handle
+                               is destroyed. This is useful when using event
+                               loops from within third party async code
         '''
         self.serial = salt.payload.Serial({'serial': 'msgpack'})
+        self.keep_loop = keep_loop
         if io_loop is not None:
             self.io_loop = io_loop
             self._run_io_loop_sync = False
@@ -727,7 +732,7 @@ class SaltEvent(object):
             self.subscriber.close()
         if self.pusher is not None:
             self.pusher.close()
-        if self._run_io_loop_sync:
+        if self._run_io_loop_sync and not self.keep_loop:
             self.io_loop.close()
 
     def fire_ret_load(self, load):
@@ -790,9 +795,20 @@ class MasterEvent(SaltEvent):
     RAET compatible
     Create a master event management object
     '''
-    def __init__(self, sock_dir, opts=None, listen=True, io_loop=None):
+    def __init__(
+            self,
+            sock_dir,
+            opts=None,
+            listen=True,
+            io_loop=None,
+            keep_loop=False):
         super(MasterEvent, self).__init__(
-            'master', sock_dir, opts, listen=listen, io_loop=io_loop)
+            'master',
+            sock_dir,
+            opts,
+            listen=listen,
+            io_loop=io_loop,
+            keep_loop=keep_loop)
 
 
 class LocalClientEvent(MasterEvent):

salt/utils/schedule.py

@@ -449,7 +449,7 @@ class Schedule(object):
             with salt.utils.fopen(schedule_conf, 'wb+') as fp_:
                 fp_.write(
                     salt.utils.to_bytes(
-                        yaml.dump({'schedule': self.opts['schedule']})
+                        yaml.dump({'schedule': self.option('schedule')})
                     )
                 )
         except (IOError, OSError):
@@ -462,9 +462,9 @@ class Schedule(object):
         '''
         if where is None or where != 'pillar':
             # ensure job exists, then delete it
-            if name in self.opts['schedule']:
-                del self.opts['schedule'][name]
-            schedule = self.opts['schedule']
+            schedule = self.option('schedule')
+            if name in schedule:
+                del schedule[name]
         else:
             # If job is in pillar, delete it there too
             if 'schedule' in self.opts['pillar']:
@@ -491,10 +491,10 @@ class Schedule(object):
         '''
         if where is None or where != 'pillar':
             # ensure job exists, then delete it
-            for job in list(self.opts['schedule'].keys()):
+            schedule = self.option('schedule')
+            for job in list(schedule.keys()):
                 if job.startswith(name):
-                    del self.opts['schedule'][job]
-            schedule = self.opts['schedule']
+                    del schedule[job]
         else:
             # If job is in pillar, delete it there too
             if 'schedule' in self.opts['pillar']:
@@ -539,17 +539,18 @@ class Schedule(object):
 
         new_job = next(six.iterkeys(data))
 
-        if new_job in self.opts['schedule']:
+        schedule = self.option('schedule')
+        if new_job in schedule:
             log.info('Updating job settings for scheduled '
                      'job: {0}'.format(new_job))
         else:
             log.info('Added new job {0} to scheduler'.format(new_job))
 
-        self.opts['schedule'].update(data)
+        schedule.update(data)
 
         # Fire the complete event back along with updated list of schedule
         evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False)
-        evt.fire_event({'complete': True, 'schedule': self.opts['schedule']},
+        evt.fire_event({'complete': True, 'schedule': schedule},
                        tag='/salt/minion/minion_schedule_add_complete')
 
         if persist:
@@ -563,8 +564,8 @@ class Schedule(object):
             self.opts['pillar']['schedule'][name]['enabled'] = True
             schedule = self.opts['pillar']['schedule']
         else:
-            self.opts['schedule'][name]['enabled'] = True
-            schedule = self.opts['schedule']
+            schedule = self.option('schedule')
+            schedule[name]['enabled'] = True
 
         # Fire the complete event back along with updated list of schedule
         evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False)
@@ -584,8 +585,8 @@ class Schedule(object):
             self.opts['pillar']['schedule'][name]['enabled'] = False
             schedule = self.opts['pillar']['schedule']
         else:
-            self.opts['schedule'][name]['enabled'] = False
-            schedule = self.opts['schedule']
+            schedule = self.option('schedule')
+            schedule[name]['enabled'] = False
 
         # Fire the complete event back along with updated list of schedule
         evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False)
@@ -606,9 +607,9 @@ class Schedule(object):
                 self.delete_job(name, persist, where=where)
             self.opts['pillar']['schedule'][name] = schedule
         else:
-            if name in self.opts['schedule']:
+            if name in self.option('schedule'):
                 self.delete_job(name, persist, where=where)
-            self.opts['schedule'][name] = schedule
+            self.option('schedule')[name] = schedule
 
         if persist:
             self.persist()
@@ -617,7 +618,7 @@ class Schedule(object):
         '''
         Run a schedule job now
         '''
-        schedule = self.opts['schedule']
+        schedule = self.option('schedule')
         if 'schedule' in self.opts['pillar']:
             schedule.update(self.opts['pillar']['schedule'])
         data = schedule[name]
@@ -664,22 +665,24 @@ class Schedule(object):
         '''
         Enable the scheduler.
         '''
-        self.opts['schedule']['enabled'] = True
+        schedule = self.option('schedule')
+        schedule['enabled'] = True
 
         # Fire the complete event back along with updated list of schedule
         evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False)
-        evt.fire_event({'complete': True, 'schedule': self.opts['schedule']},
+        evt.fire_event({'complete': True, 'schedule': schedule},
                        tag='/salt/minion/minion_schedule_enabled_complete')
 
     def disable_schedule(self):
         '''
         Disable the scheduler.
         '''
-        self.opts['schedule']['enabled'] = False
+        schedule = self.option('schedule')
+        schedule['enabled'] = False
 
         # Fire the complete event back along with updated list of schedule
         evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False)
-        evt.fire_event({'complete': True, 'schedule': self.opts['schedule']},
+        evt.fire_event({'complete': True, 'schedule': schedule},
                        tag='/salt/minion/minion_schedule_disabled_complete')
 
     def reload(self, schedule):
@@ -702,9 +705,9 @@ class Schedule(object):
             if 'schedule' in self.opts['pillar']:
                 schedule.update(self.opts['pillar']['schedule'])
         elif where == 'opts':
-            schedule.update(self.opts['schedule'])
+            schedule.update(self.option('schedule'))
         else:
-            schedule.update(self.opts['schedule'])
+            schedule.update(self.option('schedule'))
             if 'schedule' in self.opts['pillar']:
                 schedule.update(self.opts['pillar']['schedule'])
 

setup.py

@@ -119,6 +119,7 @@ SALT_VERSION = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', 'version.py'
 SALT_VERSION_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_version.py')
 SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_syspaths.py')
 SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt')
+SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'windows.txt')
 SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt')
 SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt')
 
@@ -384,11 +385,11 @@ class InstallPyCryptoWindowsWheel(Command):
         call_arguments = ['pip', 'install', 'wheel']
         if platform_bits == '64bit':
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl'
+                'https://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl'
             )
         else:
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl'
+                'https://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl'
             )
         with indent_log():
             call_subprocess(call_arguments)
@@ -415,11 +416,11 @@ class InstallCompiledPyYaml(Command):
         call_arguments = ['easy_install', '-Z']
         if platform_bits == '64bit':
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe'
+                'https://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe'
             )
         else:
             call_arguments.append(
-                'http://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win-amd64-py2.7.exe'
+                'https://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win32-py2.7.exe'
             )
         with indent_log():
             call_subprocess(call_arguments)
@@ -442,7 +443,7 @@ class DownloadWindowsDlls(Command):
         import platform
         from pip.utils.logging import indent_log
         platform_bits, _ = platform.architecture()
-        url = 'http://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll'
+        url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll'
         dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll')
         with indent_log():
             for fname in ('libeay32', 'ssleay32', 'libsodium', 'msvcr120'):
@@ -1023,8 +1024,7 @@ class SaltDistribution(distutils.dist.Distribution):
         install_requires = _parse_requirements_file(SALT_REQS)
 
         if IS_WINDOWS_PLATFORM:
-            install_requires.append('WMI')
-            install_requires.append('pypiwin32 >= 219')
+            install_requires += _parse_requirements_file(SALT_WINDOWS_REQS)
 
         if self.salt_transport == 'zeromq':
             install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS)

tests/integration/states/git.py

@@ -8,6 +8,7 @@ from __future__ import absolute_import
 import os
 import shutil
 import socket
+import string
 import subprocess
 import tempfile
 
@@ -328,6 +329,136 @@ class GitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
         self.assertSaltTrueReturn(ret)
 
 
+@skip_if_binaries_missing('git')
+class LocalRepoGitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
+    '''
+    Tests which do no require connectivity to github.com
+    '''
+    def test_renamed_default_branch(self):
+        '''
+        Test the case where the remote branch has been removed
+        https://github.com/saltstack/salt/issues/36242
+        '''
+        cwd = os.getcwd()
+        repo = tempfile.mkdtemp(dir=integration.TMP)
+        admin = tempfile.mkdtemp(dir=integration.TMP)
+        name = tempfile.mkdtemp(dir=integration.TMP)
+        for dirname in (repo, admin, name):
+            self.addCleanup(shutil.rmtree, dirname, ignore_errors=True)
+        self.addCleanup(os.chdir, cwd)
+
+        with salt.utils.fopen(os.devnull, 'w') as devnull:
+            # Create bare repo
+            subprocess.check_call(['git', 'init', '--bare', repo],
+                                  stdout=devnull, stderr=devnull)
+            # Clone bare repo
+            subprocess.check_call(['git', 'clone', repo, admin],
+                                  stdout=devnull, stderr=devnull)
+
+            # Create, add, commit, and push file
+            os.chdir(admin)
+            with salt.utils.fopen('foo', 'w'):
+                pass
+            subprocess.check_call(['git', 'add', '.'],
+                                  stdout=devnull, stderr=devnull)
+            subprocess.check_call(['git', 'commit', '-m', 'init'],
+                                  stdout=devnull, stderr=devnull)
+            subprocess.check_call(['git', 'push', 'origin', 'master'],
+                                  stdout=devnull, stderr=devnull)
+
+        # Change back to the original cwd
+        os.chdir(cwd)
+
+        # Rename remote 'master' branch to 'develop'
+        os.rename(
+            os.path.join(repo, 'refs', 'heads', 'master'),
+            os.path.join(repo, 'refs', 'heads', 'develop')
+        )
+
+        # Run git.latest state. This should successfuly clone and fail with a
+        # specific error in the comment field.
+        ret = self.run_state(
+            'git.latest',
+            name=repo,
+            target=name,
+            rev='develop',
+        )
+        self.assertSaltFalseReturn(ret)
+        self.assertEqual(
+            ret[next(iter(ret))]['comment'],
+            'Remote HEAD refers to a ref that does not exist. '
+            'This can happen when the default branch on the '
+            'remote repository is renamed or deleted. If you '
+            'are unable to fix the remote repository, you can '
+            'work around this by setting the \'branch\' argument '
+            '(which will ensure that the named branch is created '
+            'if it does not already exist).\n\n'
+            'Changes already made: {0} cloned to {1}'
+            .format(repo, name)
+        )
+        self.assertEqual(
+            ret[next(iter(ret))]['changes'],
+            {'new': '{0} => {1}'.format(repo, name)}
+        )
+
+        # Run git.latest state again. This should fail again, with a different
+        # error in the comment field, and should not change anything.
+        ret = self.run_state(
+            'git.latest',
+            name=repo,
+            target=name,
+            rev='develop',
+        )
+        self.assertSaltFalseReturn(ret)
+        self.assertEqual(
+            ret[next(iter(ret))]['comment'],
+            'Cannot set/unset upstream tracking branch, local '
+            'HEAD refers to nonexistent branch. This may have '
+            'been caused by cloning a remote repository for which '
+            'the default branch was renamed or deleted. If you '
+            'are unable to fix the remote repository, you can '
+            'work around this by setting the \'branch\' argument '
+            '(which will ensure that the named branch is created '
+            'if it does not already exist).'
+        )
+        self.assertEqual(ret[next(iter(ret))]['changes'], {})
+
+        # Run git.latest state again with a branch manually set. This should
+        # checkout a new branch and the state should pass.
+        ret = self.run_state(
+            'git.latest',
+            name=repo,
+            target=name,
+            rev='develop',
+            branch='develop',
+        )
+        # State should succeed
+        self.assertSaltTrueReturn(ret)
+        self.assertSaltCommentRegexpMatches(
+            ret,
+            'New branch \'develop\' was checked out, with origin/develop '
+            r'\([0-9a-f]{7}\) as a starting point'
+        )
+        # Only the revision should be in the changes dict.
+        self.assertEqual(
+            list(ret[next(iter(ret))]['changes'].keys()),
+            ['revision']
+        )
+        # Since the remote repo was incorrectly set up, the local head should
+        # not exist (therefore the old revision should be None).
+        self.assertEqual(
+            ret[next(iter(ret))]['changes']['revision']['old'],
+            None
+        )
+        # Make sure the new revision is a SHA (40 chars, all hex)
+        self.assertTrue(
+            len(ret[next(iter(ret))]['changes']['revision']['new']) == 40)
+        self.assertTrue(
+            all([x in string.hexdigits for x in
+                 ret[next(iter(ret))]['changes']['revision']['new']])
+        )
+
+
 if __name__ == '__main__':
     from integration import run_tests
     run_tests(GitTest)