Merge branch 'develop' into beacon_state_fix

doc/topics/releases/oxygen.rst

@@ -51,6 +51,19 @@ New NaCl Renderer
 
 A new renderer has been added for encrypted data.
 
+New support for Cisco UCS Chassis
+---------------------------------
+
+The salt proxy minion now allows for control of Cisco USC chassis. See
+the `cimc` modules for details.
+
+New salt-ssh roster
+-------------------
+
+A new roster has been added that allows users to pull in a list of hosts
+for salt-ssh targeting from a ~/.ssh configuration. For full details,
+please see the `sshconfig` roster.
+
 New GitFS Features
 ------------------
 

salt/cloud/clouds/linode.py

@@ -44,9 +44,6 @@ from salt.exceptions import (
     SaltCloudSystemExit
 )
 
-# Import Salt-Cloud Libs
-import salt.utils.cloud
-
 # Get logging started
 log = logging.getLogger(__name__)
 
@@ -1193,7 +1190,7 @@ def list_nodes_select(call=None):
     '''
     Return a list of the VMs that are on the provider, with select fields.
     '''
-    return salt.utils.cloud.list_nodes_select(
+    return __utils__['cloud.list_nodes_select'](
         list_nodes_full(), __opts__['query.selection'], call,
     )
 
@@ -1503,7 +1500,7 @@ def _query(action=None,
     if LASTCALL >= now:
         time.sleep(ratelimit_sleep)
 
-    result = salt.utils.http.query(
+    result = __utils__['http.query'](
         url,
         method,
         params=args,

salt/config/schemas/esxcluster.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Alexandru Bleotu (alexandru.bleotu@morganstanley.com)`
+
+
+    salt.config.schemas.esxcluster
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    ESX Cluster configuration schemas
+'''
+
+# Import Python libs
+from __future__ import absolute_import
+
+# Import Salt libs
+from salt.utils.schema import (Schema,
+                               ArrayItem,
+                               IntegerItem,
+                               StringItem)
+
+
+class EsxclusterProxySchema(Schema):
+    '''
+    Schema of the esxcluster proxy input
+    '''
+
+    title = 'Esxcluster Proxy Schema'
+    description = 'Esxcluster proxy schema'
+    additional_properties = False
+    proxytype = StringItem(required=True,
+                           enum=['esxcluster'])
+    vcenter = StringItem(required=True, pattern=r'[^\s]+')
+    datacenter = StringItem(required=True)
+    cluster = StringItem(required=True)
+    mechanism = StringItem(required=True, enum=['userpass', 'sspi'])
+    username = StringItem()
+    passwords = ArrayItem(min_items=1,
+                          items=StringItem(),
+                          unique_items=True)
+    # TODO Should be changed when anyOf is supported for schemas
+    domain = StringItem()
+    principal = StringItem()
+    protocol = StringItem()
+    port = IntegerItem(minimum=1)

salt/modules/aptpkg.py

@@ -99,7 +99,7 @@ def __virtual__():
     '''
     Confirm this module is on a Debian based system
     '''
-    if __grains__.get('os_family') in ('Kali', 'Debian', 'neon'):
+    if __grains__.get('os_family') in ('Kali', 'Debian', 'neon', 'Deepin'):
         return __virtualname__
     elif __grains__.get('os_family', False) == 'Cumulus':
         return __virtualname__

salt/modules/esxcluster.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+'''
+Module used to access the esxcluster proxy connection methods
+'''
+from __future__ import absolute_import
+
+# Import python libs
+import logging
+import salt.utils.platform
+
+
+log = logging.getLogger(__name__)
+
+__proxyenabled__ = ['esxcluster']
+# Define the module's virtual name
+__virtualname__ = 'esxcluster'
+
+
+def __virtual__():
+    '''
+    Only work on proxy
+    '''
+    if salt.utils.platform.is_proxy():
+        return __virtualname__
+    return (False, 'Must be run on a proxy minion')
+
+
+def get_details():
+    return __proxy__['esxcluster.get_details']()

salt/modules/vsphere.py

@@ -195,7 +195,7 @@ else:
 log = logging.getLogger(__name__)
 
 __virtualname__ = 'vsphere'
-__proxyenabled__ = ['esxi', 'esxdatacenter']
+__proxyenabled__ = ['esxi', 'esxcluster', 'esxdatacenter']
 
 
 def __virtual__():
@@ -227,6 +227,8 @@ def _get_proxy_connection_details():
     proxytype = get_proxy_type()
     if proxytype == 'esxi':
         details = __salt__['esxi.get_details']()
+    elif proxytype == 'esxcluster':
+        details = __salt__['esxcluster.get_details']()
     elif proxytype == 'esxdatacenter':
         details = __salt__['esxdatacenter.get_details']()
     else:
@@ -267,7 +269,7 @@ def gets_service_instance_via_proxy(fn):
     proxy details and passes the connection (vim.ServiceInstance) to
     the decorated function.
 
-    Supported proxies: esxi, esxdatacenter.
+    Supported proxies: esxi, esxcluster, esxdatacenter.
 
     Notes:
         1. The decorated function must have a ``service_instance`` parameter
@@ -354,7 +356,7 @@ def gets_service_instance_via_proxy(fn):
 
 
 @depends(HAS_PYVMOMI)
-@supports_proxies('esxi', 'esxdatacenter')
+@supports_proxies('esxi', 'esxcluster', 'esxdatacenter')
 def get_service_instance_via_proxy(service_instance=None):
     '''
     Returns a service instance to the proxied endpoint (vCenter/ESXi host).
@@ -374,7 +376,7 @@ def get_service_instance_via_proxy(service_instance=None):
 
 
 @depends(HAS_PYVMOMI)
-@supports_proxies('esxi', 'esxdatacenter')
+@supports_proxies('esxi', 'esxcluster', 'esxdatacenter')
 def disconnect(service_instance):
     '''
     Disconnects from a vCenter or ESXi host
@@ -1909,7 +1911,7 @@ def get_vsan_eligible_disks(host, username, password, protocol=None, port=None,
 
 
 @depends(HAS_PYVMOMI)
-@supports_proxies('esxi', 'esxdatacenter')
+@supports_proxies('esxi', 'esxcluster', 'esxdatacenter')
 @gets_service_instance_via_proxy
 def test_vcenter_connection(service_instance=None):
     '''
@@ -3598,7 +3600,7 @@ def vsan_enable(host, username, password, protocol=None, port=None, host_names=N
 
 
 @depends(HAS_PYVMOMI)
-@supports_proxies('esxdatacenter')
+@supports_proxies('esxdatacenter', 'esxcluster')
 @gets_service_instance_via_proxy
 def list_datacenters_via_proxy(datacenter_names=None, service_instance=None):
     '''
@@ -4294,3 +4296,14 @@ def _get_esxdatacenter_proxy_details():
     return det.get('vcenter'), det.get('username'), det.get('password'), \
             det.get('protocol'), det.get('port'), det.get('mechanism'), \
             det.get('principal'), det.get('domain'), det.get('datacenter')
+
+
+def _get_esxcluster_proxy_details():
+    '''
+    Returns the running esxcluster's proxy details
+    '''
+    det = __salt__['esxcluster.get_details']()
+    return det.get('vcenter'), det.get('username'), det.get('password'), \
+            det.get('protocol'), det.get('port'), det.get('mechanism'), \
+            det.get('principal'), det.get('domain'), det.get('datacenter'), \
+            det.get('cluster')

salt/proxy/esxcluster.py

@@ -0,0 +1,310 @@
+# -*- coding: utf-8 -*-
+'''
+Proxy Minion interface module for managing VMWare ESXi clusters.
+
+Dependencies
+============
+
+- pyVmomi
+- jsonschema
+
+Configuration
+=============
+To use this integration proxy module, please configure the following:
+
+Pillar
+------
+
+Proxy minions get their configuration from Salt's Pillar. This can now happen
+from the proxy's configuration file.
+
+Example pillars:
+
+``userpass`` mechanism:
+
+.. code-block:: yaml
+
+    proxy:
+      proxytype: esxcluster
+      cluster: <cluster name>
+      datacenter: <datacenter name>
+      vcenter: <ip or dns name of parent vcenter>
+      mechanism: userpass
+      username: <vCenter username>
+      passwords: (required if userpass is used)
+        - first_password
+        - second_password
+        - third_password
+
+``sspi`` mechanism:
+
+.. code-block:: yaml
+
+    proxy:
+      proxytype: esxcluster
+      cluster: <cluster name>
+      datacenter: <datacenter name>
+      vcenter: <ip or dns name of parent vcenter>
+      mechanism: sspi
+      domain: <user domain>
+      principal: <host kerberos principal>
+
+proxytype
+^^^^^^^^^
+To use this Proxy Module, set this to ``esxdatacenter``.
+
+cluster
+^^^^^^^
+Name of the managed cluster. Required.
+
+datacenter
+^^^^^^^^^^
+Name of the datacenter the managed cluster is in. Required.
+
+vcenter
+^^^^^^^
+The location of the VMware vCenter server (host of ip) where the datacenter
+should be managed. Required.
+
+mechanism
+^^^^^^^^
+The mechanism used to connect to the vCenter server. Supported values are
+``userpass`` and ``sspi``. Required.
+
+Note:
+    Connections are attempted using all (``username``, ``password``)
+    combinations on proxy startup.
+
+username
+^^^^^^^^
+The username used to login to the host, such as ``root``. Required if mechanism
+is ``userpass``.
+
+passwords
+^^^^^^^^^
+A list of passwords to be used to try and login to the vCenter server. At least
+one password in this list is required if mechanism is ``userpass``.  When the
+proxy comes up, it will try the passwords listed in order.
+
+domain
+^^^^^^
+User domain. Required if mechanism is ``sspi``.
+
+principal
+^^^^^^^^
+Kerberos principal. Rquired if mechanism is ``sspi``.
+
+protocol
+^^^^^^^^
+If the ESXi host is not using the default protocol, set this value to an
+alternate protocol. Default is ``https``.
+
+port
+^^^^
+If the ESXi host is not using the default port, set this value to an
+alternate port. Default is ``443``.
+
+Salt Proxy
+----------
+
+After your pillar is in place, you can test the proxy. The proxy can run on
+any machine that has network connectivity to your Salt Master and to the
+vCenter server in the pillar. SaltStack recommends that the machine running the
+salt-proxy process also run a regular minion, though it is not strictly
+necessary.
+
+To start a proxy minion one needs to establish its identity <id>:
+
+.. code-block:: bash
+
+    salt-proxy --proxyid <proxy_id>
+
+On the machine that will run the proxy, make sure there is a configuration file
+present. By default this is ``/etc/salt/proxy``. If in a different location, the
+``<configuration_folder>`` has to be specified when running the proxy:
+file with at least the following in it:
+
+.. code-block:: bash
+
+    salt-proxy --proxyid <proxy_id> -c <configuration_folder>
+
+Commands
+--------
+
+Once the proxy is running it will connect back to the specified master and
+individual commands can be runs against it:
+
+.. code-block:: bash
+
+    # Master - minion communication
+    salt <cluster_name> test.ping
+
+    # Test vcenter connection
+    salt <cluster_name> vsphere.test_vcenter_connection
+
+States
+------
+
+Associated states are documented in
+:mod:`salt.states.esxcluster </ref/states/all/salt.states.esxcluster>`.
+Look there to find an example structure for Pillar as well as an example
+``.sls`` file for configuring an ESX cluster from scratch.
+'''
+
+
+# Import Python Libs
+from __future__ import absolute_import
+import logging
+import os
+
+# Import Salt Libs
+import salt.exceptions
+from salt.config.schemas.esxcluster import EsxclusterProxySchema
+from salt.utils.dictupdate import merge
+
+# This must be present or the Salt loader won't load this module.
+__proxyenabled__ = ['esxcluster']
+
+# External libraries
+try:
+    import jsonschema
+    HAS_JSONSCHEMA = True
+except ImportError:
+    HAS_JSONSCHEMA = False
+
+# Variables are scoped to this module so we can have persistent data
+# across calls to fns in here.
+GRAINS_CACHE = {}
+DETAILS = {}
+
+
+# Set up logging
+log = logging.getLogger(__name__)
+# Define the module's virtual name
+__virtualname__ = 'esxcluster'
+
+
+def __virtual__():
+    '''
+    Only load if the vsphere execution module is available.
+    '''
+    if HAS_JSONSCHEMA:
+        return __virtualname__
+
+    return False, 'The esxcluster proxy module did not load.'
+
+
+def init(opts):
+    '''
+    This function gets called when the proxy starts up. For
+    login
+    the protocol and port are cached.
+    '''
+    log.debug('Initting esxcluster proxy module in process '
+              '{}'.format(os.getpid()))
+    log.debug('Validating esxcluster proxy input')
+    schema = EsxclusterProxySchema.serialize()
+    log.trace('schema = {}'.format(schema))
+    proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {}))
+    log.trace('proxy_conf = {0}'.format(proxy_conf))
+    try:
+        jsonschema.validate(proxy_conf, schema)
+    except jsonschema.exceptions.ValidationError as exc:
+        raise salt.exceptions.InvalidConfigError(exc)
+
+    # Save mandatory fields in cache
+    for key in ('vcenter', 'datacenter', 'cluster', 'mechanism'):
+        DETAILS[key] = proxy_conf[key]
+
+    # Additional validation
+    if DETAILS['mechanism'] == 'userpass':
+        if 'username' not in proxy_conf:
+            raise salt.exceptions.InvalidConfigError(
+                'Mechanism is set to \'userpass\', but no '
+                '\'username\' key found in proxy config.')
+        if 'passwords' not in proxy_conf:
+            raise salt.exceptions.InvalidConfigError(
+                'Mechanism is set to \'userpass\', but no '
+                '\'passwords\' key found in proxy config.')
+        for key in ('username', 'passwords'):
+            DETAILS[key] = proxy_conf[key]
+    else:
+        if 'domain' not in proxy_conf:
+            raise salt.exceptions.InvalidConfigError(
+                'Mechanism is set to \'sspi\', but no '
+                '\'domain\' key found in proxy config.')
+        if 'principal' not in proxy_conf:
+            raise salt.exceptions.InvalidConfigError(
+                'Mechanism is set to \'sspi\', but no '
+                '\'principal\' key found in proxy config.')
+        for key in ('domain', 'principal'):
+            DETAILS[key] = proxy_conf[key]
+
+    # Save optional
+    DETAILS['protocol'] = proxy_conf.get('protocol')
+    DETAILS['port'] = proxy_conf.get('port')
+
+    # Test connection
+    if DETAILS['mechanism'] == 'userpass':
+        # Get the correct login details
+        log.debug('Retrieving credentials and testing vCenter connection for '
+                  'mehchanism \'userpass\'')
+        try:
+            username, password = find_credentials()
+            DETAILS['password'] = password
+        except salt.exceptions.SaltSystemExit as err:
+            log.critical('Error: {0}'.format(err))
+            return False
+    return True
+
+
+def ping():
+    '''
+    Returns True.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt esx-cluster test.ping
+    '''
+    return True
+
+
+def shutdown():
+    '''
+    Shutdown the connection to the proxy device. For this proxy,
+    shutdown is a no-op.
+    '''
+    log.debug('esxcluster proxy shutdown() called...')
+
+
+def find_credentials():
+    '''
+    Cycle through all the possible credentials and return the first one that
+    works.
+    '''
+
+    # if the username and password were already found don't fo though the
+    # connection process again
+    if 'username' in DETAILS and 'password' in DETAILS:
+        return DETAILS['username'], DETAILS['password']
+
+    passwords = DETAILS['passwords']
+    for password in passwords:
+        DETAILS['password'] = password
+        if not __salt__['vsphere.test_vcenter_connection']():
+            # We are unable to authenticate
+            continue
+        # If we have data returned from above, we've successfully authenticated.
+        return DETAILS['username'], password
+    # We've reached the end of the list without successfully authenticating.
+    raise salt.exceptions.VMwareConnectionError('Cannot complete login due to '
+                                                'incorrect credentials.')
+
+
+def get_details():
+    '''
+    Function that returns the cached details
+    '''
+    return DETAILS

salt/states/acme.py

@@ -116,9 +116,14 @@ def cert(name,
     if res['result'] is None:
         ret['changes'] = {}
     else:
+        if not __salt__['acme.has'](name):
+            new = None
+        else:
+            new = __salt__['acme.info'](name)
+
         ret['changes'] = {
             'old': old,
-            'new': __salt__['acme.info'](name)
+            'new': new
         }
 
     return ret

salt/utils/configparser.py

@@ -0,0 +1,269 @@
+# -*- coding: utf-8 -*-
+# Import Python libs
+from __future__ import absolute_import
+import re
+
+# Import Salt libs
+import salt.utils.stringutils
+
+# Import 3rd-party libs
+from salt.ext import six
+from salt.ext.six.moves.configparser import *  # pylint: disable=no-name-in-module,wildcard-import
+
+try:
+    from collections import OrderedDict as _default_dict
+except ImportError:
+    # fallback for setup.py which hasn't yet built _collections
+    _default_dict = dict
+
+
+# pylint: disable=string-substitution-usage-error
+class GitConfigParser(RawConfigParser, object):  # pylint: disable=undefined-variable
+    '''
+    Custom ConfigParser which reads and writes git config files.
+
+    READ A GIT CONFIG FILE INTO THE PARSER OBJECT
+
+    >>> import salt.utils.configparser
+    >>> conf = salt.utils.configparser.GitConfigParser()
+    >>> conf.read('/home/user/.git/config')
+
+    MAKE SOME CHANGES
+
+    >>> # Change user.email
+    >>> conf.set('user', 'email', 'myaddress@mydomain.tld')
+    >>> # Add another refspec to the "origin" remote's "fetch" multivar
+    >>> conf.set_multivar('remote "origin"', 'fetch', '+refs/tags/*:refs/tags/*')
+
+    WRITE THE CONFIG TO A FILEHANDLE
+
+    >>> import salt.utils.files
+    >>> with salt.utils.files.fopen('/home/user/.git/config', 'w') as fh:
+    ...     conf.write(fh)
+    >>>
+    '''
+    DEFAULTSECT = u'DEFAULT'
+    SPACEINDENT = u' ' * 8
+
+    def __init__(self, defaults=None, dict_type=_default_dict,
+                 allow_no_value=True):
+        '''
+        Changes default value for allow_no_value from False to True
+        '''
+        super(GitConfigParser, self).__init__(
+            defaults, dict_type, allow_no_value)
+
+    def _read(self, fp, fpname):
+        '''
+        Makes the following changes from the RawConfigParser:
+
+        1. Strip leading tabs from non-section-header lines.
+        2. Treat 8 spaces at the beginning of a line as a tab.
+        3. Treat lines beginning with a tab as options.
+        4. Drops support for continuation lines.
+        5. Multiple values for a given option are stored as a list.
+        6. Keys and values are decoded to the system encoding.
+        '''
+        cursect = None                        # None, or a dictionary
+        optname = None
+        lineno = 0
+        e = None                              # None, or an exception
+        while True:
+            line = fp.readline()
+            if six.PY2:
+                line = line.decode(__salt_system_encoding__)
+            if not line:
+                break
+            lineno = lineno + 1
+            # comment or blank line?
+            if line.strip() == u'' or line[0] in u'#;':
+                continue
+            if line.split(None, 1)[0].lower() == u'rem' and line[0] in u'rR':
+                # no leading whitespace
+                continue
+            # Replace space indentation with a tab. Allows parser to work
+            # properly in cases where someone has edited the git config by hand
+            # and indented using spaces instead of tabs.
+            if line.startswith(self.SPACEINDENT):
+                line = u'\t' + line[len(self.SPACEINDENT):]
+            # is it a section header?
+            mo = self.SECTCRE.match(line)
+            if mo:
+                sectname = mo.group(u'header')
+                if sectname in self._sections:
+                    cursect = self._sections[sectname]
+                elif sectname == self.DEFAULTSECT:
+                    cursect = self._defaults
+                else:
+                    cursect = self._dict()
+                    self._sections[sectname] = cursect
+                # So sections can't start with a continuation line
+                optname = None
+            # no section header in the file?
+            elif cursect is None:
+                raise MissingSectionHeaderError(  # pylint: disable=undefined-variable
+                    salt.utils.stringutils.to_str(fpname),
+                    lineno,
+                    salt.utils.stringutils.to_str(line))
+            # an option line?
+            else:
+                mo = self._optcre.match(line.lstrip())
+                if mo:
+                    optname, vi, optval = mo.group(u'option', u'vi', u'value')
+                    optname = self.optionxform(optname.rstrip())
+                    if optval is None:
+                        optval = u''
+                    if optval:
+                        if vi in (u'=', u':') and u';' in optval:
+                            # ';' is a comment delimiter only if it follows
+                            # a spacing character
+                            pos = optval.find(u';')
+                            if pos != -1 and optval[pos-1].isspace():
+                                optval = optval[:pos]
+                        optval = optval.strip()
+                        # Empty strings should be considered as blank strings
+                        if optval in (u'""', u"''"):
+                            optval = u''
+                    self._add_option(cursect, optname, optval)
+                else:
+                    # a non-fatal parsing error occurred.  set up the
+                    # exception but keep going. the exception will be
+                    # raised at the end of the file and will contain a
+                    # list of all bogus lines
+                    if not e:
+                        e = ParsingError(fpname)  # pylint: disable=undefined-variable
+                    e.append(lineno, repr(line))
+        # if any parsing errors occurred, raise an exception
+        if e:
+            raise e  # pylint: disable=raising-bad-type
+
+    def _string_check(self, value, allow_list=False):
+        '''
+        Based on the string-checking code from the SafeConfigParser's set()
+        function, this enforces string values for config options.
+        '''
+        if self._optcre is self.OPTCRE or value:
+            is_list = isinstance(value, list)
+            if is_list and not allow_list:
+                raise TypeError('option value cannot be a list unless allow_list is True')  # future lint: disable=non-unicode-string
+            elif not is_list:
+                value = [value]
+            if not all(isinstance(x, six.string_types) for x in value):
+                raise TypeError('option values must be strings')  # future lint: disable=non-unicode-string
+
+    def get(self, section, option, as_list=False):
+        '''
+        Adds an optional "as_list" argument to ensure a list is returned. This
+        is helpful when iterating over an option which may or may not be a
+        multivar.
+        '''
+        ret = super(GitConfigParser, self).get(section, option)
+        if as_list and not isinstance(ret, list):
+            ret = [ret]
+        return ret
+
+    def set(self, section, option, value=u''):
+        '''
+        This is overridden from the RawConfigParser merely to change the
+        default value for the 'value' argument.
+        '''
+        self._string_check(value)
+        super(GitConfigParser, self).set(section, option, value)
+
+    def _add_option(self, sectdict, key, value):
+        if isinstance(value, list):
+            sectdict[key] = value
+        elif isinstance(value, six.string_types):
+            try:
+                sectdict[key].append(value)
+            except KeyError:
+                # Key not present, set it
+                sectdict[key] = value
+            except AttributeError:
+                # Key is present but the value is not a list. Make it into a list
+                # and then append to it.
+                sectdict[key] = [sectdict[key]]
+                sectdict[key].append(value)
+        else:
+            raise TypeError('Expected str or list for option value, got %s' % type(value).__name__)  # future lint: disable=non-unicode-string
+
+    def set_multivar(self, section, option, value=u''):
+        '''
+        This function is unique to the GitConfigParser. It will add another
+        value for the option if it already exists, converting the option's
+        value to a list if applicable.
+
+        If "value" is a list, then any existing values for the specified
+        section and option will be replaced with the list being passed.
+        '''
+        self._string_check(value, allow_list=True)
+        if not section or section == self.DEFAULTSECT:
+            sectdict = self._defaults
+        else:
+            try:
+                sectdict = self._sections[section]
+            except KeyError:
+                raise NoSectionError(  # pylint: disable=undefined-variable
+                    salt.utils.stringutils.to_str(section))
+        key = self.optionxform(option)
+        self._add_option(sectdict, key, value)
+
+    def remove_option_regexp(self, section, option, expr):
+        '''
+        Remove an option with a value matching the expression. Works on single
+        values and multivars.
+        '''
+        if not section or section == self.DEFAULTSECT:
+            sectdict = self._defaults
+        else:
+            try:
+                sectdict = self._sections[section]
+            except KeyError:
+                raise NoSectionError(  # pylint: disable=undefined-variable
+                    salt.utils.stringutils.to_str(section))
+        option = self.optionxform(option)
+        if option not in sectdict:
+            return False
+        regexp = re.compile(expr)
+        if isinstance(sectdict[option], list):
+            new_list = [x for x in sectdict[option] if not regexp.search(x)]
+            # Revert back to a list if we removed all but one item
+            if len(new_list) == 1:
+                new_list = new_list[0]
+            existed = new_list != sectdict[option]
+            if existed:
+                del sectdict[option]
+                sectdict[option] = new_list
+            del new_list
+        else:
+            existed = bool(regexp.search(sectdict[option]))
+            if existed:
+                del sectdict[option]
+        return existed
+
+    def write(self, fp_):
+        '''
+        Makes the following changes from the RawConfigParser:
+
+        1. Prepends options with a tab character.
+        2. Does not write a blank line between sections.
+        3. When an option's value is a list, a line for each option is written.
+           This allows us to support multivars like a remote's "fetch" option.
+        4. Drops support for continuation lines.
+        '''
+        convert = salt.utils.stringutils.to_bytes \
+            if u'b' in fp_.mode \
+            else salt.utils.stringutils.to_str
+        if self._defaults:
+            fp_.write(convert(u'[%s]\n' % self.DEFAULTSECT))
+            for (key, value) in six.iteritems(self._defaults):
+                value = salt.utils.stringutils.to_unicode(value).replace(u'\n', u'\n\t')
+                fp_.write(convert(u'%s = %s\n' % (key, value)))
+        for section in self._sections:
+            fp_.write(convert(u'[%s]\n' % section))
+            for (key, value) in six.iteritems(self._sections[section]):
+                if (value is not None) or (self._optcre == self.OPTCRE):
+                    if not isinstance(value, list):
+                        value = [value]
+                    for item in value:
+                        fp_.write(convert(u'\t%s\n' % u' = '.join((key, item)).rstrip()))

salt/utils/gitfs.py

@@ -21,6 +21,7 @@ from datetime import datetime
 
 # Import salt libs
 import salt.utils
+import salt.utils.configparser
 import salt.utils.files
 import salt.utils.itertools
 import salt.utils.path
@@ -29,13 +30,12 @@ import salt.utils.stringutils
 import salt.utils.url
 import salt.utils.versions
 import salt.fileserver
-from salt.config import DEFAULT_MASTER_OPTS as __DEFAULT_MASTER_OPTS
+from salt.config import DEFAULT_MASTER_OPTS as _DEFAULT_MASTER_OPTS
 from salt.utils.odict import OrderedDict
 from salt.utils.process import os_is_running as pid_exists
 from salt.exceptions import (
     FileserverConfigError,
     GitLockError,
-    GitRemoteError,
     get_error_message
 )
 from salt.utils.event import tagify
@@ -44,7 +44,7 @@ from salt.utils.versions import LooseVersion as _LooseVersion
 # Import third party libs
 from salt.ext import six
 
-VALID_REF_TYPES = __DEFAULT_MASTER_OPTS['gitfs_ref_types']
+VALID_REF_TYPES = _DEFAULT_MASTER_OPTS['gitfs_ref_types']
 
 # Optional per-remote params that can only be used on a per-remote basis, and
 # thus do not have defaults in salt/config.py.
@@ -327,6 +327,28 @@ class GitProvider(object):
                 setattr(self, '_' + key, self.conf[key])
             self.add_conf_overlay(key)
 
+        if not hasattr(self, 'refspecs'):
+            # This was not specified as a per-remote overrideable parameter
+            # when instantiating an instance of a GitBase subclass. Make sure
+            # that we set this attribute so we at least have a sane default and
+            # are able to fetch.
+            key = '{0}_refspecs'.format(self.role)
+            try:
+                default_refspecs = _DEFAULT_MASTER_OPTS[key]
+            except KeyError:
+                log.critical(
+                    'The \'%s\' option has no default value in '
+                    'salt/config/__init__.py.', key
+                )
+                failhard(self.role)
+
+            setattr(self, 'refspecs', default_refspecs)
+            log.debug(
+                'The \'refspecs\' option was not explicitly defined as a '
+                'configurable parameter. Falling back to %s for %s remote '
+                '\'%s\'.', default_refspecs, self.role, self.id
+            )
+
         for item in ('env_whitelist', 'env_blacklist'):
             val = getattr(self, item, None)
             if val:
@@ -493,12 +515,6 @@ class GitProvider(object):
                 return strip_sep(getattr(self, '_' + name))
         setattr(cls, name, _getconf)
 
-    def add_refspecs(self, *refspecs):
-        '''
-        This function must be overridden in a sub-class
-        '''
-        raise NotImplementedError()
-
     def check_root(self):
         '''
         Check if the relative root path exists in the checked-out copy of the
@@ -591,55 +607,74 @@ class GitProvider(object):
             success.append(msg)
         return success, failed
 
-    def configure_refspecs(self):
+    def enforce_git_config(self):
         '''
-        Ensure that the configured refspecs are set
+        For the config options which need to be maintained in the git config,
+        ensure that the git config file is configured as desired.
         '''
-        try:
+        git_config = os.path.join(self.gitdir, 'config')
-            refspecs = set(self.get_refspecs())
+        conf = salt.utils.configparser.GitConfigParser()
-        except (git.exc.GitCommandError, GitRemoteError) as exc:
+        if not conf.read(git_config):
-            log.error(
+            log.error('Failed to read from git config file %s', git_config)
-                'Failed to get refspecs for %s remote \'%s\': %s',
-                self.role,
-                self.id,
-                exc
-            )
-            return
-
-        desired_refspecs = set(self.refspecs)
-        to_delete = refspecs - desired_refspecs if refspecs else set()
-        if to_delete:
-            # There is no native unset support in Pygit2, and GitPython just
-            # wraps the CLI anyway. So we'll just use the git CLI to
-            # --unset-all the config value. Then, we will add back all
-            # configured refspecs. This is more foolproof than trying to remove
-            # specific refspecs, as removing specific ones necessitates
-            # formulating a regex to match, and the fact that slashes and
-            # asterisks are in refspecs complicates this.
-            cmd_str = 'git config --unset-all remote.origin.fetch'
-            cmd = subprocess.Popen(
-                shlex.split(cmd_str),
-                close_fds=not salt.utils.platform.is_windows(),
-                cwd=os.path.dirname(self.gitdir),
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT)
-            output = cmd.communicate()[0]
-            if cmd.returncode != 0:
-                log.error(
-                    'Failed to unset git config value for %s remote \'%s\'. '
-                    'Output from \'%s\' follows:\n%s',
-                    self.role, self.id, cmd_str, output
-                )
-                return
-            # Since we had to remove all refspecs, we now need to add all
-            # desired refspecs to achieve the desired configuration.
-            to_add = desired_refspecs
         else:
-            # We didn't need to delete any refspecs, so we'll only need to add
+            # We are currently enforcing the following git config items:
-            # the desired refspecs that aren't currently configured.
+            # 1. refspecs used in fetch
-            to_add = desired_refspecs - refspecs
+            # 2. http.sslVerify
+            conf_changed = False
 
-        self.add_refspecs(*to_add)
+            # 1. refspecs
+            try:
+                refspecs = sorted(
+                    conf.get('remote "origin"', 'fetch', as_list=True))
+            except salt.utils.configparser.NoSectionError:
+                # First time we've init'ed this repo, we need to add the
+                # section for the remote to the git config
+                conf.add_section('remote "origin"')
+                conf.set('remote "origin"', 'url', self.url)
+                conf_changed = True
+                refspecs = []
+            desired_refspecs = sorted(self.refspecs)
+            log.debug(
+                'Current refspecs for %s remote \'%s\': %s (desired: %s)',
+                self.role, self.id, refspecs, desired_refspecs
+            )
+            if refspecs != desired_refspecs:
+                conf.set_multivar('remote "origin"', 'fetch', self.refspecs)
+                log.debug(
+                    'Refspecs for %s remote \'%s\' set to %s',
+                    self.role, self.id, desired_refspecs
+                )
+                conf_changed = True
+
+            # 2. http.sslVerify
+            try:
+                ssl_verify = conf.get('http', 'sslVerify')
+            except salt.utils.configparser.NoSectionError:
+                conf.add_section('http')
+                ssl_verify = None
+            except salt.utils.configparser.NoOptionError:
+                ssl_verify = None
+            desired_ssl_verify = six.text_type(self.ssl_verify).lower()
+            log.debug(
+                'Current http.sslVerify for %s remote \'%s\': %s (desired: %s)',
+                self.role, self.id, ssl_verify, desired_ssl_verify
+            )
+            if ssl_verify != desired_ssl_verify:
+                conf.set('http', 'sslVerify', desired_ssl_verify)
+                log.debug(
+                    'http.sslVerify for %s remote \'%s\' set to %s',
+                    self.role, self.id, desired_ssl_verify
+                )
+                conf_changed = True
+
+            # Write changes, if necessary
+            if conf_changed:
+                with salt.utils.files.fopen(git_config, 'w') as fp_:
+                    conf.write(fp_)
+                    log.debug(
+                        'Config updates for %s remote \'%s\' written to %s',
+                        self.role, self.id, git_config
+                    )
 
     def fetch(self):
         '''
@@ -853,12 +888,6 @@ class GitProvider(object):
                 else target
         return self.branch
 
-    def get_refspecs(self):
-        '''
-        This function must be overridden in a sub-class
-        '''
-        raise NotImplementedError()
-
     def get_tree(self, tgt_env):
         '''
         Return a tree object for the specified environment
@@ -935,23 +964,6 @@ class GitPython(GitProvider):
             override_params, cache_root, role
         )
 
-    def add_refspecs(self, *refspecs):
-        '''
-        Add the specified refspecs to the "origin" remote
-        '''
-        for refspec in refspecs:
-            try:
-                self.repo.git.config('--add', 'remote.origin.fetch', refspec)
-                log.debug(
-                    'Added refspec \'%s\' to %s remote \'%s\'',
-                    refspec, self.role, self.id
-                )
-            except git.exc.GitCommandError as exc:
-                log.error(
-                    'Failed to add refspec \'%s\' to %s remote \'%s\': %s',
-                    refspec, self.role, self.id, exc
-                )
-
     def checkout(self):
         '''
         Checkout the configured branch/tag. We catch an "Exception" class here
@@ -1039,29 +1051,7 @@ class GitPython(GitProvider):
                 return new
 
         self.gitdir = salt.utils.path.join(self.repo.working_dir, '.git')
-
+        self.enforce_git_config()
-        if not self.repo.remotes:
-            try:
-                self.repo.create_remote('origin', self.url)
-            except os.error:
-                # This exception occurs when two processes are trying to write
-                # to the git config at once, go ahead and pass over it since
-                # this is the only write. This should place a lock down.
-                pass
-            else:
-                new = True
-
-            try:
-                ssl_verify = self.repo.git.config('--get', 'http.sslVerify')
-            except git.exc.GitCommandError:
-                ssl_verify = ''
-            desired_ssl_verify = str(self.ssl_verify).lower()
-            if ssl_verify != desired_ssl_verify:
-                self.repo.git.config('http.sslVerify', desired_ssl_verify)
-
-            # Ensure that refspecs for the "origin" remote are set up as configured
-            if hasattr(self, 'refspecs'):
-                self.configure_refspecs()
 
         return new
 
@@ -1213,13 +1203,6 @@ class GitPython(GitProvider):
             return blob, blob.hexsha, blob.mode
         return None, None, None
 
-    def get_refspecs(self):
-        '''
-        Return the configured refspecs
-        '''
-        refspecs = self.repo.git.config('--get-all', 'remote.origin.fetch')
-        return [x.strip() for x in refspecs.splitlines()]
-
     def get_tree_from_branch(self, ref):
         '''
         Return a git.Tree object matching a head ref fetched into
@@ -1272,27 +1255,6 @@ class Pygit2(GitProvider):
             override_params, cache_root, role
         )
 
-    def add_refspecs(self, *refspecs):
-        '''
-        Add the specified refspecs to the "origin" remote
-        '''
-        for refspec in refspecs:
-            try:
-                self.repo.config.set_multivar(
-                    'remote.origin.fetch',
-                    'FOO',
-                    refspec
-                )
-                log.debug(
-                    'Added refspec \'%s\' to %s remote \'%s\'',
-                    refspec, self.role, self.id
-                )
-            except Exception as exc:
-                log.error(
-                    'Failed to add refspec \'%s\' to %s remote \'%s\': %s',
-                    refspec, self.role, self.id, exc
-                )
-
     def checkout(self):
         '''
         Checkout the configured branch/tag
@@ -1519,30 +1481,7 @@ class Pygit2(GitProvider):
                 return new
 
         self.gitdir = salt.utils.path.join(self.repo.workdir, '.git')
-
+        self.enforce_git_config()
-        if not self.repo.remotes:
-            try:
-                self.repo.create_remote('origin', self.url)
-            except os.error:
-                # This exception occurs when two processes are trying to write
-                # to the git config at once, go ahead and pass over it since
-                # this is the only write. This should place a lock down.
-                pass
-            else:
-                new = True
-
-            try:
-                ssl_verify = self.repo.config.get_bool('http.sslVerify')
-            except KeyError:
-                ssl_verify = None
-            if ssl_verify != self.ssl_verify:
-                self.repo.config.set_multivar('http.sslVerify',
-                                            '',
-                                            str(self.ssl_verify).lower())
-
-            # Ensure that refspecs for the "origin" remote are set up as configured
-            if hasattr(self, 'refspecs'):
-                self.configure_refspecs()
 
         return new
 
@@ -1760,14 +1699,6 @@ class Pygit2(GitProvider):
             return blob, blob.hex, mode
         return None, None, None
 
-    def get_refspecs(self):
-        '''
-        Return the configured refspecs
-        '''
-        if not [x for x in self.repo.config if x.startswith('remote.origin.')]:
-            raise GitRemoteError('\'origin\' remote not not present')
-        return list(self.repo.config.get_multivar('remote.origin.fetch'))
-
     def get_tree_from_branch(self, ref):
         '''
         Return a pygit2.Tree object matching a head ref fetched into

tests/unit/modules/test_esxcluster.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Alexandru Bleotu <alexandru.bleotu@morganstanley.com>`
+
+    Tests for functions in salt.modules.esxcluster
+'''
+
+# Import Python Libs
+from __future__ import absolute_import
+
+# Import Salt Libs
+import salt.modules.esxcluster as esxcluster
+
+# Import Salt Testing Libs
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.unit import TestCase, skipIf
+from tests.support.mock import (
+    MagicMock,
+    patch,
+    NO_MOCK,
+    NO_MOCK_REASON
+)
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class GetDetailsTestCase(TestCase, LoaderModuleMockMixin):
+    '''Tests for salt.modules.esxcluster.get_details'''
+    def setup_loader_modules(self):
+        return {esxcluster: {'__virtual__':
+                             MagicMock(return_value='esxcluster'),
+                             '__proxy__': {}}}
+
+    def test_get_details(self):
+        mock_get_details = MagicMock()
+        with patch.dict(esxcluster.__proxy__,
+                        {'esxcluster.get_details': mock_get_details}):
+            esxcluster.get_details()
+        mock_get_details.assert_called_once_with()

tests/unit/modules/test_vsphere.py

@@ -620,6 +620,7 @@ class _GetProxyConnectionDetailsTestCase(TestCase, LoaderModuleMockMixin):
                                      'principal': 'fake_principal',
                                      'domain': 'fake_domain'}
         self.esxdatacenter_details = {'vcenter': 'fake_vcenter',
+                                      'datacenter': 'fake_dc',
                                       'username': 'fake_username',
                                       'password': 'fake_password',
                                       'protocol': 'fake_protocol',
@@ -627,9 +628,20 @@ class _GetProxyConnectionDetailsTestCase(TestCase, LoaderModuleMockMixin):
                                       'mechanism': 'fake_mechanism',
                                       'principal': 'fake_principal',
                                       'domain': 'fake_domain'}
+        self.esxcluster_details = {'vcenter': 'fake_vcenter',
+                                   'datacenter': 'fake_dc',
+                                   'cluster': 'fake_cluster',
+                                   'username': 'fake_username',
+                                   'password': 'fake_password',
+                                   'protocol': 'fake_protocol',
+                                   'port': 'fake_port',
+                                   'mechanism': 'fake_mechanism',
+                                   'principal': 'fake_principal',
+                                   'domain': 'fake_domain'}
 
     def tearDown(self):
-        for attrname in ('esxi_host_details', 'esxi_vcenter_details'):
+        for attrname in ('esxi_host_details', 'esxi_vcenter_details',
+                         'esxdatacenter_details', 'esxcluster_details'):
             try:
                 delattr(self, attrname)
             except AttributeError:
@@ -651,8 +663,22 @@ class _GetProxyConnectionDetailsTestCase(TestCase, LoaderModuleMockMixin):
                    MagicMock(return_value='esxdatacenter')):
             with patch.dict(vsphere.__salt__,
                             {'esxdatacenter.get_details': MagicMock(
-                                return_value=self.esxdatacenter_details)}):
+                             return_value=self.esxdatacenter_details)}):
                 ret = vsphere._get_proxy_connection_details()
+        self.assertEqual(('fake_vcenter', 'fake_username', 'fake_password',
+                          'fake_protocol', 'fake_port', 'fake_mechanism',
+                          'fake_principal', 'fake_domain'), ret)
+
+    def test_esxcluster_proxy_details(self):
+        with patch('salt.modules.vsphere.get_proxy_type',
+                   MagicMock(return_value='esxcluster')):
+            with patch.dict(vsphere.__salt__,
+                            {'esxcluster.get_details': MagicMock(
+                             return_value=self.esxcluster_details)}):
+                ret = vsphere._get_proxy_connection_details()
+        self.assertEqual(('fake_vcenter', 'fake_username', 'fake_password',
+                          'fake_protocol', 'fake_port', 'fake_mechanism',
+                          'fake_principal', 'fake_domain'), ret)
 
     def test_esxi_proxy_vcenter_details(self):
         with patch('salt.modules.vsphere.get_proxy_type',
@@ -862,8 +888,8 @@ class GetServiceInstanceViaProxyTestCase(TestCase, LoaderModuleMockMixin):
             }
         }
 
-    def test_supported_proxes(self):
+    def test_supported_proxies(self):
-        supported_proxies = ['esxi', 'esxdatacenter']
+        supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter']
         for proxy_type in supported_proxies:
             with patch('salt.modules.vsphere.get_proxy_type',
                        MagicMock(return_value=proxy_type)):
@@ -905,8 +931,8 @@ class DisconnectTestCase(TestCase, LoaderModuleMockMixin):
             }
         }
 
-    def test_supported_proxes(self):
+    def test_supported_proxies(self):
-        supported_proxies = ['esxi', 'esxdatacenter']
+        supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter']
         for proxy_type in supported_proxies:
             with patch('salt.modules.vsphere.get_proxy_type',
                        MagicMock(return_value=proxy_type)):
@@ -946,8 +972,8 @@ class TestVcenterConnectionTestCase(TestCase, LoaderModuleMockMixin):
             }
         }
 
-    def test_supported_proxes(self):
+    def test_supported_proxies(self):
-        supported_proxies = ['esxi', 'esxdatacenter']
+        supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter']
         for proxy_type in supported_proxies:
             with patch('salt.modules.vsphere.get_proxy_type',
                        MagicMock(return_value=proxy_type)):
@@ -1022,7 +1048,7 @@ class ListDatacentersViaProxyTestCase(TestCase, LoaderModuleMockMixin):
         }
 
     def test_supported_proxies(self):
-        supported_proxies = ['esxdatacenter']
+        supported_proxies = ['esxcluster', 'esxdatacenter']
         for proxy_type in supported_proxies:
             with patch('salt.modules.vsphere.get_proxy_type',
                        MagicMock(return_value=proxy_type)):
@@ -1099,7 +1125,7 @@ class CreateDatacenterTestCase(TestCase, LoaderModuleMockMixin):
             }
         }
 
-    def test_supported_proxes(self):
+    def test_supported_proxies(self):
         supported_proxies = ['esxdatacenter']
         for proxy_type in supported_proxies:
             with patch('salt.modules.vsphere.get_proxy_type',

tests/unit/proxy/test_esxcluster.py

@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Alexandru Bleotu <alexandru.bleotu@morganstanley.com>`
+
+    Tests for esxcluster proxy
+'''
+
+# Import Python Libs
+from __future__ import absolute_import
+
+# Import external libs
+try:
+    import jsonschema
+    HAS_JSONSCHEMA = True
+except ImportError:
+    HAS_JSONSCHEMA = False
+
+# Import Salt Libs
+import salt.proxy.esxcluster as esxcluster
+import salt.exceptions
+from salt.config.schemas.esxcluster import EsxclusterProxySchema
+
+# Import Salt Testing Libs
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.unit import TestCase, skipIf
+from tests.support.mock import (
+    MagicMock,
+    patch,
+    NO_MOCK,
+    NO_MOCK_REASON
+)
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+@skipIf(not HAS_JSONSCHEMA, 'jsonschema is required')
+class InitTestCase(TestCase, LoaderModuleMockMixin):
+    '''Tests for salt.proxy.esxcluster.init'''
+    def setup_loader_modules(self):
+        return {esxcluster: {'__virtual__':
+                             MagicMock(return_value='esxcluster'),
+                             'DETAILS': {}, '__pillar__': {}}}
+
+    def setUp(self):
+        self.opts_userpass = {'proxy': {'proxytype': 'esxcluster',
+                                        'vcenter': 'fake_vcenter',
+                                        'datacenter': 'fake_dc',
+                                        'cluster': 'fake_cluster',
+                                        'mechanism': 'userpass',
+                                        'username': 'fake_username',
+                                        'passwords': ['fake_password'],
+                                        'protocol': 'fake_protocol',
+                                        'port': 100}}
+        self.opts_sspi = {'proxy': {'proxytype': 'esxcluster',
+                                    'vcenter': 'fake_vcenter',
+                                    'datacenter': 'fake_dc',
+                                    'cluster': 'fake_cluster',
+                                    'mechanism': 'sspi',
+                                    'domain': 'fake_domain',
+                                    'principal': 'fake_principal',
+                                    'protocol': 'fake_protocol',
+                                    'port': 100}}
+        patches = (('salt.proxy.esxcluster.merge',
+                    MagicMock(return_value=self.opts_sspi['proxy'])),)
+        for mod, mock in patches:
+            patcher = patch(mod, mock)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+
+    def test_merge(self):
+        mock_pillar_proxy = MagicMock()
+        mock_opts_proxy = MagicMock()
+        mock_merge = MagicMock(return_value=self.opts_sspi['proxy'])
+        with patch.dict(esxcluster.__pillar__,
+                        {'proxy': mock_pillar_proxy}):
+            with patch('salt.proxy.esxcluster.merge', mock_merge):
+                esxcluster.init(opts={'proxy': mock_opts_proxy})
+        mock_merge.assert_called_once_with(mock_opts_proxy, mock_pillar_proxy)
+
+    def test_esxcluster_schema(self):
+        mock_json_validate = MagicMock()
+        serialized_schema = EsxclusterProxySchema().serialize()
+        with patch('salt.proxy.esxcluster.jsonschema.validate',
+                   mock_json_validate):
+            esxcluster.init(self.opts_sspi)
+        mock_json_validate.assert_called_once_with(
+            self.opts_sspi['proxy'], serialized_schema)
+
+    def test_invalid_proxy_input_error(self):
+        with patch('salt.proxy.esxcluster.jsonschema.validate',
+                   MagicMock(side_effect=jsonschema.exceptions.ValidationError(
+                       'Validation Error'))):
+            with self.assertRaises(salt.exceptions.InvalidConfigError) as \
+                    excinfo:
+                esxcluster.init(self.opts_userpass)
+        self.assertEqual(excinfo.exception.strerror.message,
+                         'Validation Error')
+
+    def test_no_username(self):
+        opts = self.opts_userpass.copy()
+        del opts['proxy']['username']
+        with patch('salt.proxy.esxcluster.merge',
+                   MagicMock(return_value=opts['proxy'])):
+            with self.assertRaises(salt.exceptions.InvalidConfigError) as \
+                    excinfo:
+                esxcluster.init(opts)
+        self.assertEqual(excinfo.exception.strerror,
+                         'Mechanism is set to \'userpass\', but no '
+                         '\'username\' key found in proxy config.')
+
+    def test_no_passwords(self):
+        opts = self.opts_userpass.copy()
+        del opts['proxy']['passwords']
+        with patch('salt.proxy.esxcluster.merge',
+                   MagicMock(return_value=opts['proxy'])):
+            with self.assertRaises(salt.exceptions.InvalidConfigError) as \
+                    excinfo:
+                esxcluster.init(opts)
+        self.assertEqual(excinfo.exception.strerror,
+                         'Mechanism is set to \'userpass\', but no '
+                         '\'passwords\' key found in proxy config.')
+
+    def test_no_domain(self):
+        opts = self.opts_sspi.copy()
+        del opts['proxy']['domain']
+        with patch('salt.proxy.esxcluster.merge',
+                   MagicMock(return_value=opts['proxy'])):
+            with self.assertRaises(salt.exceptions.InvalidConfigError) as \
+                    excinfo:
+                esxcluster.init(opts)
+        self.assertEqual(excinfo.exception.strerror,
+                         'Mechanism is set to \'sspi\', but no '
+                         '\'domain\' key found in proxy config.')
+
+    def test_no_principal(self):
+        opts = self.opts_sspi.copy()
+        del opts['proxy']['principal']
+        with patch('salt.proxy.esxcluster.merge',
+                   MagicMock(return_value=opts['proxy'])):
+            with self.assertRaises(salt.exceptions.InvalidConfigError) as \
+                    excinfo:
+                esxcluster.init(opts)
+        self.assertEqual(excinfo.exception.strerror,
+                         'Mechanism is set to \'sspi\', but no '
+                         '\'principal\' key found in proxy config.')
+
+    def test_find_credentials(self):
+        mock_find_credentials = MagicMock(return_value=('fake_username',
+                                                        'fake_password'))
+        with patch('salt.proxy.esxcluster.merge',
+                   MagicMock(return_value=self.opts_userpass['proxy'])):
+            with patch('salt.proxy.esxcluster.find_credentials',
+                       mock_find_credentials):
+                esxcluster.init(self.opts_userpass)
+        mock_find_credentials.assert_called_once_with()
+
+    def test_details_userpass(self):
+        mock_find_credentials = MagicMock(return_value=('fake_username',
+                                                        'fake_password'))
+        with patch('salt.proxy.esxcluster.merge',
+                   MagicMock(return_value=self.opts_userpass['proxy'])):
+            with patch('salt.proxy.esxcluster.find_credentials',
+                       mock_find_credentials):
+                esxcluster.init(self.opts_userpass)
+        self.assertDictEqual(esxcluster.DETAILS,
+                             {'vcenter': 'fake_vcenter',
+                              'datacenter': 'fake_dc',
+                              'cluster': 'fake_cluster',
+                              'mechanism': 'userpass',
+                              'username': 'fake_username',
+                              'password': 'fake_password',
+                              'passwords': ['fake_password'],
+                              'protocol': 'fake_protocol',
+                              'port': 100})
+
+    def test_details_sspi(self):
+        esxcluster.init(self.opts_sspi)
+        self.assertDictEqual(esxcluster.DETAILS,
+                             {'vcenter': 'fake_vcenter',
+                              'datacenter': 'fake_dc',
+                              'cluster': 'fake_cluster',
+                              'mechanism': 'sspi',
+                              'domain': 'fake_domain',
+                              'principal': 'fake_principal',
+                              'protocol': 'fake_protocol',
+                              'port': 100})

tests/unit/utils/test_configparser.py

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*-
+'''
+tests.unit.utils.test_configparser
+==================================
+
+Test the funcs in the custom parsers in salt.utils.configparser
+'''
+# Import Python Libs
+from __future__ import absolute_import
+import copy
+import errno
+import logging
+import os
+
+log = logging.getLogger(__name__)
+
+# Import Salt Testing Libs
+from tests.support.unit import TestCase
+from tests.support.paths import TMP
+
+# Import salt libs
+import salt.utils.files
+import salt.utils.stringutils
+import salt.utils.configparser
+
+# The user.name param here is intentionally indented with spaces instead of a
+# tab to test that we properly load a file with mixed indentation.
+ORIG_CONFIG = u'''[user]
+        name = Артём Анисимов
+\temail = foo@bar.com
+[remote "origin"]
+\turl = https://github.com/terminalmage/salt.git
+\tfetch = +refs/heads/*:refs/remotes/origin/*
+\tpushurl = git@github.com:terminalmage/salt.git
+[color "diff"]
+\told = 196
+\tnew = 39
+[core]
+\tpager = less -R
+\trepositoryformatversion = 0
+\tfilemode = true
+\tbare = false
+\tlogallrefupdates = true
+[alias]
+\tmodified = ! git status --porcelain | awk 'match($1, "M"){print $2}'
+\tgraph = log --all --decorate --oneline --graph
+\thist = log --pretty=format:\\"%h %ad | %s%d [%an]\\" --graph --date=short
+[http]
+\tsslverify = false'''.split(u'\n')  # future lint: disable=non-unicode-string
+
+
+class TestGitConfigParser(TestCase):
+    '''
+    Tests for salt.utils.configparser.GitConfigParser
+    '''
+    maxDiff = None
+    orig_config = os.path.join(TMP, u'test_gitconfig.orig')
+    new_config = os.path.join(TMP, u'test_gitconfig.new')
+    remote = u'remote "origin"'
+
+    def tearDown(self):
+        del self.conf
+        try:
+            os.remove(self.new_config)
+        except OSError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+    def setUp(self):
+        if not os.path.exists(self.orig_config):
+            with salt.utils.files.fopen(self.orig_config, u'wb') as fp_:
+                fp_.write(
+                    salt.utils.stringutils.to_bytes(
+                        u'\n'.join(ORIG_CONFIG)
+                    )
+                )
+        self.conf = salt.utils.configparser.GitConfigParser()
+        self.conf.read(self.orig_config)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            os.remove(cls.orig_config)
+        except OSError as exc:
+            if exc.errno != errno.ENOENT:
+                raise
+
+    @staticmethod
+    def fix_indent(lines):
+        '''
+        Fixes the space-indented 'user' line, because when we write the config
+        object to a file space indentation will be replaced by tab indentation.
+        '''
+        ret = copy.copy(lines)
+        for i, _ in enumerate(ret):
+            if ret[i].startswith(salt.utils.configparser.GitConfigParser.SPACEINDENT):
+                ret[i] = ret[i].replace(salt.utils.configparser.GitConfigParser.SPACEINDENT, u'\t')
+        return ret
+
+    @staticmethod
+    def get_lines(path):
+        with salt.utils.files.fopen(path, u'r') as fp_:
+            return salt.utils.stringutils.to_unicode(fp_.read()).splitlines()
+
+    def _test_write(self, mode):
+        with salt.utils.files.fopen(self.new_config, mode) as fp_:
+            self.conf.write(fp_)
+        self.assertEqual(
+            self.get_lines(self.new_config),
+            self.fix_indent(ORIG_CONFIG)
+        )
+
+    def test_get(self):
+        '''
+        Test getting an option's value
+        '''
+        # Numeric values should be loaded as strings
+        self.assertEqual(self.conf.get(u'color "diff"', u'old'), u'196')
+        # Complex strings should be loaded with their literal quotes and
+        # slashes intact
+        self.assertEqual(
+            self.conf.get(u'alias', u'modified'),
+            u"""! git status --porcelain | awk 'match($1, "M"){print $2}'"""
+        )
+        # future lint: disable=non-unicode-string
+        self.assertEqual(
+            self.conf.get(u'alias', u'hist'),
+            salt.utils.stringutils.to_unicode(
+                r"""log --pretty=format:\"%h %ad | %s%d [%an]\" --graph --date=short"""
+            )
+        )
+        # future lint: enable=non-unicode-string
+
+    def test_read_space_indent(self):
+        '''
+        Test that user.name was successfully loaded despite being indented
+        using spaces instead of a tab. Additionally, this tests that the value
+        was loaded as a unicode type on PY2.
+        '''
+        self.assertEqual(self.conf.get(u'user', u'name'), u'Артём Анисимов')
+
+    def test_set_new_option(self):
+        '''
+        Test setting a new option in an existing section
+        '''
+        self.conf.set(u'http', u'useragent', u'myawesomeagent')
+        self.assertEqual(self.conf.get(u'http', u'useragent'), u'myawesomeagent')
+
+    def test_add_section(self):
+        '''
+        Test adding a section and adding an item to that section
+        '''
+        self.conf.add_section(u'foo')
+        self.conf.set(u'foo', u'bar', u'baz')
+        self.assertEqual(self.conf.get(u'foo', u'bar'), u'baz')
+
+    def test_replace_option(self):
+        '''
+        Test replacing an existing option
+        '''
+        # We're also testing the normalization of key names, here. Setting
+        # "sslVerify" should actually set an "sslverify" option.
+        self.conf.set(u'http', u'sslVerify', u'true')
+        self.assertEqual(self.conf.get(u'http', u'sslverify'), u'true')
+
+    def test_set_multivar(self):
+        '''
+        Test setting a multivar and then writing the resulting file
+        '''
+        orig_refspec = u'+refs/heads/*:refs/remotes/origin/*'
+        new_refspec = u'+refs/tags/*:refs/tags/*'
+        # Make sure that the original value is a string
+        self.assertEqual(
+            self.conf.get(self.remote, u'fetch'),
+            orig_refspec
+        )
+        # Add another refspec
+        self.conf.set_multivar(self.remote, u'fetch', new_refspec)
+        # The value should now be a list
+        self.assertEqual(
+            self.conf.get(self.remote, u'fetch'),
+            [orig_refspec, new_refspec]
+        )
+        # Write the config object to a file
+        with salt.utils.files.fopen(self.new_config, u'w') as fp_:
+            self.conf.write(fp_)
+        # Confirm that the new file was written correctly
+        expected = self.fix_indent(ORIG_CONFIG)
+        expected.insert(6, u'\tfetch = %s' % new_refspec)  # pylint: disable=string-substitution-usage-error
+        self.assertEqual(self.get_lines(self.new_config), expected)
+
+    def test_remove_option(self):
+        '''
+        test removing an option, including all items from a multivar
+        '''
+        for item in (u'fetch', u'pushurl'):
+            self.conf.remove_option(self.remote, item)
+            # To confirm that the option is now gone, a get should raise an
+            # NoOptionError exception.
+            self.assertRaises(
+                salt.utils.configparser.NoOptionError,
+                self.conf.get,
+                self.remote,
+                item)
+
+    def test_remove_option_regexp(self):
+        '''
+        test removing an option, including all items from a multivar
+        '''
+        orig_refspec = u'+refs/heads/*:refs/remotes/origin/*'
+        new_refspec_1 = u'+refs/tags/*:refs/tags/*'
+        new_refspec_2 = u'+refs/foo/*:refs/foo/*'
+        # First, add both refspecs
+        self.conf.set_multivar(self.remote, u'fetch', new_refspec_1)
+        self.conf.set_multivar(self.remote, u'fetch', new_refspec_2)
+        # Make sure that all three values are there
+        self.assertEqual(
+            self.conf.get(self.remote, u'fetch'),
+            [orig_refspec, new_refspec_1, new_refspec_2]
+        )
+        # If the regex doesn't match, no items should be removed
+        self.assertFalse(
+            self.conf.remove_option_regexp(
+                self.remote,
+                u'fetch',
+                salt.utils.stringutils.to_unicode(r'\d{7,10}')  # future lint: disable=non-unicode-string
+            )
+        )
+        # Make sure that all three values are still there (since none should
+        # have been removed)
+        self.assertEqual(
+            self.conf.get(self.remote, u'fetch'),
+            [orig_refspec, new_refspec_1, new_refspec_2]
+        )
+        # Remove one of the values
+        self.assertTrue(
+            self.conf.remove_option_regexp(self.remote, u'fetch', u'tags'))
+        # Confirm that the value is gone
+        self.assertEqual(
+            self.conf.get(self.remote, u'fetch'),
+            [orig_refspec, new_refspec_2]
+        )
+        # Remove the other one we added earlier
+        self.assertTrue(
+            self.conf.remove_option_regexp(self.remote, u'fetch', u'foo'))
+        # Since the option now only has one value, it should be a string
+        self.assertEqual(self.conf.get(self.remote, u'fetch'), orig_refspec)
+        # Remove the last remaining option
+        self.assertTrue(
+            self.conf.remove_option_regexp(self.remote, u'fetch', u'heads'))
+        # Trying to do a get now should raise an exception
+        self.assertRaises(
+            salt.utils.configparser.NoOptionError,
+            self.conf.get,
+            self.remote,
+            u'fetch')
+
+    def test_write(self):
+        '''
+        Test writing using non-binary filehandle
+        '''
+        self._test_write(mode=u'w')
+
+    def test_write_binary(self):
+        '''
+        Test writing using binary filehandle
+        '''
+        self._test_write(mode=u'wb')