Merge branch 'nitrogen' into 'develop'

Conflicts:
  - salt/runners/saltutil.py

doc/topics/troubleshooting/yaml_idiosyncrasies.rst

@@ -129,44 +129,42 @@ string literal:
         - source: salt://ssh_keys/chease.pub
         - config: '%h/.ssh/authorized_keys'
 
-Integers are Parsed as Integers
+Time Expressions
-===============================
+================
 
-NOTE: This has been fixed in salt 0.10.0, as of this release passing an
+PyYAML will load a time expression as the integer value of that, assuming
-integer that is preceded by a 0 will be correctly parsed
+``HH:MM``. So for example, ``12:00`` is loaded by PyYAML as ``720``. An
+excellent explanation for why can be found here__.
 
-When passing :func:`integers <python2:int>` into an SLS file, they are
+To keep time expressions like this from being loaded as integers, always quote
-passed as integers. This means that if a state accepts a string value
+them.
-and an integer is passed, that an integer will be sent. The solution here
-is to send the integer as a string.
 
-This is best explained when setting the mode for a file:
+.. note::
+    When using a jinja ``load_yaml`` map, items must be quoted twice. For
+    example:
 
-.. code-block:: yaml
+    .. code-block:: yaml
 
-    /etc/vimrc:
+        {% load_yaml as wsus_schedule %}
-      file:
-        - managed
-        - source: salt://edit/vimrc
-        - user: root
-        - group: root
-        - mode: 644
 
-Salt manages this well, since the mode is passed as 644, but if the mode is
+        FRI_10:
-zero padded as 0644, then it is read by YAML as an integer and evaluated as
+          time: '"23:00"'
-an octal value, 0644 becomes 420. Therefore, if the file mode is
+          day: 6 - Every Friday
-preceded by a 0 then it needs to be passed as a string:
+        SAT_10:
-
+          time: '"06:00"'
-.. code-block:: yaml
+          day: 7 - Every Saturday
-
+        SAT_20:
-    /etc/vimrc:
+          time: '"14:00"'
-      file:
+          day: 7 - Every Saturday
-        - managed
+        SAT_30:
-        - source: salt://edit/vimrc
+          time: '"22:00"'
-        - user: root
+          day: 7 - Every Saturday
-        - group: root
+        SUN_10:
-        - mode: '0644'
+          time: '"06:00"'
+          day: 1 - Every Sunday
+        {% endload %}
 
+.. __: http://stackoverflow.com/a/31007425
 
 YAML does not like "Double Short Decs"
 ======================================

pkg/windows/modules/get-settings.psm1

@@ -60,7 +60,7 @@ Function Get-Settings {
         # Filenames for 64 bit Windows
         $64bitPrograms = @{
             "PyCrypto2" = "pycrypto-2.6.1-cp27-none-win_amd64.whl"
-            "Python2"   = "python-2.7.12.amd64.msi"
+            "Python2"   = "python-2.7.13.amd64.msi"
             "PyYAML2"   = "PyYAML-3.11.win-amd64-py2.7.exe"
             "Python3"   = "python-3.5.3-amd64.exe"
             "PyWin323"  = "pywin32-220.1-cp35-cp35m-win_amd64.whl"
@@ -70,7 +70,7 @@ Function Get-Settings {
         # Filenames for 32 bit Windows
         $32bitPrograms = @{
             "PyCrypto2" = "pycrypto-2.6.1-cp27-none-win32.whl"
-            "Python2"   = "python-2.7.12.msi"
+            "Python2"   = "python-2.7.13.msi"
             "PyYAML2"   = "PyYAML-3.11.win32-py2.7.exe"
             "Python3"   = "python-3.5.3.exe"
             "PyWin323"  = "pywin32-220.1-cp35-cp35m-win32.whl"

pkg/windows/req_base.txt

@@ -1,36 +1,36 @@
-backports-abc==0.4
+backports-abc==0.5
 backports.ssl-match-hostname==3.5.0.1
 certifi
 cffi==1.10.0
-CherryPy==7.1.0
+CherryPy==10.2.1
 cryptography==1.8.1
 enum34==1.1.6
-futures==3.0.5
+futures==3.1.1
 gitdb==0.6.4
-GitPython==2.0.8
+GitPython==2.1.3
 idna==2.5
-ioflo==1.5.5
+ioflo==1.6.7
 ioloop==0.1a0
 ipaddress==1.0.18
-Jinja2==2.9.4
+Jinja2==2.9.6
-libnacl==1.4.5
+libnacl==1.5.0
-Mako==1.0.4
+Mako==1.0.6
-MarkupSafe==0.23
+MarkupSafe==1.0
 msgpack-python==0.4.8
-psutil==4.3.0
+psutil==5.2.2
-pyasn1==0.1.9
+pyasn1==0.2.3
 pycparser==2.17
 pycurl==7.43.0
-PyMySQL==0.7.6
+PyMySQL==0.7.11
-pyOpenSSL==16.2.0
+pyOpenSSL==17.0.0
-python-dateutil==2.5.3
+python-dateutil==2.6.0
-python-gnupg==0.3.8
+python-gnupg==0.4.0
-pyzmq==16.0.1
+pyzmq==16.0.2
-requests==2.10.0
+requests==2.13.0
 singledispatch==3.4.0.3
 six==1.10.0
 smmap==0.9.0
 timelib==0.2.4
-tornado==4.4.1
+tornado==4.5.1
-wheel==0.29.0
+wheel==0.30.0a0
 WMI==1.4.9

pkg/windows/req_pip.txt

@@ -1,2 +1,2 @@
 pip==9.0.1
-setuptools==34.3.1
+setuptools==35.0.2

salt/modules/pip.py

@@ -87,8 +87,7 @@ import salt.utils
 import tempfile
 import salt.utils.locales
 import salt.utils.url
-import salt.ext.six as six
+from salt.ext import six
-from salt.ext.six import string_types, iteritems
 from salt.exceptions import CommandExecutionError, CommandNotFoundError
 
 
@@ -217,7 +216,7 @@ def _resolve_requirements_chain(requirements):
 
     chain = []
 
-    if isinstance(requirements, string_types):
+    if isinstance(requirements, six.string_types):
         requirements = [requirements]
 
     for req_file in requirements:
@@ -234,7 +233,7 @@ def _process_requirements(requirements, cmd, cwd, saltenv, user):
     cleanup_requirements = []
 
     if requirements is not None:
-        if isinstance(requirements, string_types):
+        if isinstance(requirements, six.string_types):
             requirements = [r.strip() for r in requirements.split(',')]
         elif not isinstance(requirements, list):
             raise TypeError('requirements must be a string or list')
@@ -621,7 +620,7 @@ def install(pkgs=None,  # pylint: disable=R0912,R0913,R0914
         cmd.extend(['--timeout', timeout])
 
     if find_links:
-        if isinstance(find_links, string_types):
+        if isinstance(find_links, six.string_types):
             find_links = [l.strip() for l in find_links.split(',')]
 
         for link in find_links:
@@ -663,7 +662,7 @@ def install(pkgs=None,  # pylint: disable=R0912,R0913,R0914
                     ' use index_url and/or extra_index_url instead'
             )
 
-        if isinstance(mirrors, string_types):
+        if isinstance(mirrors, six.string_types):
             mirrors = [m.strip() for m in mirrors.split(',')]
 
         cmd.append('--use-mirrors')
@@ -732,21 +731,21 @@ def install(pkgs=None,  # pylint: disable=R0912,R0913,R0914
         cmd.extend(['--cert', cert])
 
     if global_options:
-        if isinstance(global_options, string_types):
+        if isinstance(global_options, six.string_types):
             global_options = [go.strip() for go in global_options.split(',')]
 
         for opt in global_options:
             cmd.extend(['--global-option', opt])
 
     if install_options:
-        if isinstance(install_options, string_types):
+        if isinstance(install_options, six.string_types):
             install_options = [io.strip() for io in install_options.split(',')]
 
         for opt in install_options:
             cmd.extend(['--install-option', opt])
 
     if pkgs:
-        if isinstance(pkgs, string_types):
+        if isinstance(pkgs, six.string_types):
             pkgs = [p.strip() for p in pkgs.split(',')]
 
         # It's possible we replaced version-range commas with semicolons so
@@ -757,7 +756,7 @@ def install(pkgs=None,  # pylint: disable=R0912,R0913,R0914
 
     if editable:
         egg_match = re.compile(r'(?:#|#.*?&)egg=([^&]*)')
-        if isinstance(editable, string_types):
+        if isinstance(editable, six.string_types):
             editable = [e.strip() for e in editable.split(',')]
 
         for entry in editable:
@@ -776,14 +775,14 @@ def install(pkgs=None,  # pylint: disable=R0912,R0913,R0914
         cmd.append('--allow-all-external')
 
     if allow_external:
-        if isinstance(allow_external, string_types):
+        if isinstance(allow_external, six.string_types):
             allow_external = [p.strip() for p in allow_external.split(',')]
 
         for pkg in allow_external:
             cmd.extend(['--allow-external', pkg])
 
     if allow_unverified:
-        if isinstance(allow_unverified, string_types):
+        if isinstance(allow_unverified, six.string_types):
             allow_unverified = \
                 [p.strip() for p in allow_unverified.split(',')]
 
@@ -793,27 +792,27 @@ def install(pkgs=None,  # pylint: disable=R0912,R0913,R0914
     if process_dependency_links:
         cmd.append('--process-dependency-links')
 
+    if trusted_host:
+        cmd.extend(['--trusted-host', trusted_host])
+
+    cmd_kwargs = dict(saltenv=saltenv, use_vt=use_vt, runas=user)
+
     if env_vars:
         if isinstance(env_vars, dict):
-            for k, v in iteritems(env_vars):
+            for key, val in six.iteritems(env_vars):
-                if not isinstance(v, string_types):
+                if not isinstance(val, six.string_types):
-                    env_vars[k] = str(v)
+                    val = str(val)
-            os.environ.update(env_vars)
+                cmd_kwargs.setdefault('env', {})[key] = val
         else:
             raise CommandExecutionError(
                 'env_vars {0} is not a dictionary'.format(env_vars))
 
-    if trusted_host:
-        cmd.extend(['--trusted-host', trusted_host])
-
     try:
-        cmd_kwargs = dict(saltenv=saltenv, use_vt=use_vt, runas=user)
-
         if cwd:
             cmd_kwargs['cwd'] = cwd
 
         if bin_env and os.path.isdir(bin_env):
-            cmd_kwargs['env'] = {'VIRTUAL_ENV': bin_env}
+            cmd_kwargs.setdefault('env', {})['VIRTUAL_ENV'] = bin_env
 
         logger.debug(
             'TRY BLOCK: end of pip.install -- cmd: %s, cmd_kwargs: %s',
@@ -928,7 +927,7 @@ def uninstall(pkgs=None,
         cmd.extend(['--timeout', timeout])
 
     if pkgs:
-        if isinstance(pkgs, string_types):
+        if isinstance(pkgs, six.string_types):
             pkgs = [p.strip() for p in pkgs.split(',')]
         if requirements:
             for requirement in requirements:

salt/modules/saltutil.py

@@ -579,6 +579,41 @@ def sync_output(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blackl
 sync_outputters = salt.utils.alias_function(sync_output, 'sync_outputters')
 
 
+def sync_clouds(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None):
+    '''
+    .. versionadded:: Nitrogen
+
+    Sync utility modules from ``salt://_cloud`` to the minion
+
+    saltenv : base
+        The fileserver environment from which to sync. To sync from more than
+        one environment, pass a comma-separated list.
+
+    refresh : True
+        If ``True``, refresh the available execution modules on the minion.
+        This refresh will be performed even if no new utility modules are
+        synced. Set to ``False`` to prevent this refresh.
+
+    extmod_whitelist : None
+        comma-seperated list of modules to sync
+
+    extmod_blacklist : None
+        comma-seperated list of modules to blacklist based on type
+
+    CLI Examples:
+
+    .. code-block:: bash
+
+        salt '*' saltutil.sync_clouds
+        salt '*' saltutil.sync_clouds saltenv=dev
+        salt '*' saltutil.sync_clouds saltenv=base,dev
+    '''
+    ret = _sync('clouds', saltenv, extmod_whitelist, extmod_blacklist)
+    if refresh:
+        refresh_modules()
+    return ret
+
+
 def sync_utils(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None):
     '''
     .. versionadded:: 2014.7.0
@@ -763,6 +798,7 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist
     '''
     log.debug('Syncing all')
     ret = {}
+    ret['clouds'] = sync_clouds(saltenv, False, extmod_whitelist, extmod_blacklist)
     ret['beacons'] = sync_beacons(saltenv, False, extmod_whitelist, extmod_blacklist)
     ret['modules'] = sync_modules(saltenv, False, extmod_whitelist, extmod_blacklist)
     ret['states'] = sync_states(saltenv, False, extmod_whitelist, extmod_blacklist)

salt/pillar/file_tree.py

@@ -56,10 +56,12 @@ intended to be used to deploy a file using ``contents_pillar`` with a
     files would not affected by the ``keep_newline`` configuration.  However,
     this module does not actually distinguish between binary and text files.
 
-.. versionchanged:: develop
+.. versionchanged:: Nitrogen
     Templating/rendering has been added. You can now specify a default render
     pipeline and a black- and whitelist of (dis)allowed renderers.
 
+    ``template`` must be set to ``True`` for templating to happen.
+
     .. code-block:: yaml
 
         ext_pillar:
@@ -71,6 +73,7 @@ intended to be used to deploy a file using ``contents_pillar`` with a
             renderer_whitelist:
               - jinja
               - yaml
+            template: True
 
 Assigning Pillar Data to Individual Hosts
 -----------------------------------------
@@ -220,7 +223,8 @@ def _construct_pillar(top_dir,
                       keep_newline=False,
                       render_default=None,
                       renderer_blacklist=None,
-                      renderer_whitelist=None):
+                      renderer_whitelist=None,
+                      template=False):
     '''
     Construct pillar from file tree.
     '''
@@ -272,11 +276,13 @@ def _construct_pillar(top_dir,
                           file_path,
                           exc.strerror)
             else:
-                data = salt.template.compile_template_str(template=contents,
+                data = contents
-                                                          renderers=renderers,
+                if template is True:
-                                                          default=render_default,
+                    data = salt.template.compile_template_str(template=contents,
-                                                          blacklist=renderer_blacklist,
+                                                              renderers=renderers,
-                                                          whitelist=renderer_whitelist)
+                                                              default=render_default,
+                                                              blacklist=renderer_blacklist,
+                                                              whitelist=renderer_whitelist)
                 if salt.utils.stringio.is_readable(data):
                     pillar_node[file_name] = data.getvalue()
                 else:
@@ -293,7 +299,8 @@ def ext_pillar(minion_id,
                keep_newline=False,
                render_default=None,
                renderer_blacklist=None,
-               renderer_whitelist=None):
+               renderer_whitelist=None,
+               template=False):
     '''
     Compile pillar data for the specified minion ID
     '''
@@ -341,7 +348,8 @@ def ext_pillar(minion_id,
                                               keep_newline,
                                               render_default,
                                               renderer_blacklist,
-                                              renderer_whitelist)
+                                              renderer_whitelist,
+                                              template)
                         )
         else:
             if debug is True:
@@ -367,7 +375,8 @@ def ext_pillar(minion_id,
                                     keep_newline,
                                     render_default,
                                     renderer_blacklist,
-                                    renderer_whitelist)
+                                    renderer_whitelist,
+                                    template)
     return salt.utils.dictupdate.merge(ngroup_pillar,
                                        host_pillar,
                                        strategy='recurse')

salt/runners/saltutil.py

@@ -470,3 +470,29 @@ def sync_fileserver(saltenv='base', extmod_whitelist=None, extmod_blacklist=None
     '''
     return salt.utils.extmods.sync(__opts__, 'fileserver', saltenv=saltenv, extmod_whitelist=extmod_whitelist,
                                    extmod_blacklist=extmod_blacklist)[0]
+
+
+def sync_clouds(saltenv='base', extmod_whitelist=None, extmod_blacklist=None):
+    '''
+    .. versionadded:: Nitrogen
+
+    Sync utils modules from ``salt://_cloud`` to the master
+
+    saltenv : base
+        The fileserver environment from which to sync. To sync from more than
+        one environment, pass a comma-separated list.
+
+    extmod_whitelist : None
+        comma-seperated list of modules to sync
+
+    extmod_blacklist : None
+        comma-seperated list of modules to blacklist based on type
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt-run saltutil.sync_cloud
+    '''
+    return salt.utils.extmods.sync(__opts__, 'cloud', saltenv=saltenv, extmod_whitelist=extmod_whitelist,
+                                   extmod_blacklist=extmod_blacklist)[0]

salt/states/saltmod.py

@@ -329,7 +329,7 @@ def state(name,
             except KeyError:
                 m_state = False
             if m_state:
-                m_state = salt.utils.check_state_result(m_ret)
+                m_state = salt.utils.check_state_result(m_ret, recurse=True)
 
         if not m_state:
             if minion not in fail_minions:
@@ -338,9 +338,10 @@ def state(name,
             continue
         try:
             for state_item in six.itervalues(m_ret):
-                if 'changes' in state_item and state_item['changes']:
+                if isinstance(state_item, dict):
-                    changes[minion] = m_ret
+                    if 'changes' in state_item and state_item['changes']:
-                    break
+                        changes[minion] = m_ret
+                        break
             else:
                 no_change.add(minion)
         except AttributeError:

salt/utils/gitfs.py

@@ -1403,7 +1403,7 @@ class Pygit2(GitProvider):
             try:
                 try:
                     self.repo = pygit2.Repository(self.cachedir)
-                except pygit2.GitError as exc:
+                except GitError as exc:
                     import pwd
                     # https://github.com/libgit2/pygit2/issues/339
                     # https://github.com/libgit2/libgit2/issues/2122

tests/integration/files/conf/master

@@ -31,14 +31,7 @@ peer:
   '.*':
     - 'test.*'
 
-git_pillar_verify_config: False
-
 ext_pillar:
-  - git:
-    - master https://github.com/saltstack/pillar1.git
-    - master https://github.com/saltstack/pillar2.git
-    - dev https://github.com/saltstack/pillar1.git:
-      - env: testing
   - test_ext_pillar_opts:
     - test_issue_5951_actual_file_roots_in_opts
 
@@ -102,4 +95,4 @@ libcloud_dns:
     driver: godaddy
     key: 12345
     secret: mysecret
-    shopper_id: 12345
+    shopper_id: 12345

tests/integration/files/file/base/file-pillargit.sls

@@ -1,11 +0,0 @@
-{% if grains['kernel'] == 'Windows' %}
-  {% set TMP = "C:\\Windows\\Temp\\" %}
-{% else %}
-  {% set TMP = "/tmp/" %}
-{% endif %}
-
-{% set file = salt['pillar.get']('info', '') %}
-
-create_file:
-  file.managed:
-    - name: {{ TMP }}filepillar-{{ file }}

tests/integration/files/file/base/git_pillar/http/files/nginx.conf

@@ -0,0 +1,40 @@
+worker_processes 1;
+error_log {{ pillar['git_pillar']['config_dir'] }}/error.log;
+pid {{ pillar['git_pillar']['config_dir'] }}/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    access_log {{ pillar['git_pillar']['config_dir'] }}/git_access.log;
+    error_log {{ pillar['git_pillar']['config_dir'] }}/git_error.log;
+
+    #sendfile on;
+    #keepalive_timeout 65;
+
+    server {
+        listen {{ pillar['git_pillar']['nginx_port'] }} default_server;
+        server_name git.local;
+        root {{ pillar['git_pillar']['git_dir'] }}/repos;
+
+        location / {
+{%- if salt['pillar.get']('git_pillar:auth_enabled', False) %}
+            auth_basic "YOU... SHALL NOT... PASS!!!";
+            auth_basic_user_file {{ pillar['git_pillar']['git_dir'] }}/users;
+{%- endif %}
+
+            include /etc/nginx/uwsgi_params;
+
+            uwsgi_param GIT_HTTP_EXPORT_ALL "";
+            uwsgi_param GIT_PROJECT_ROOT    {{ pillar['git_pillar']['git_dir'] }}/repos;
+            uwsgi_param REMOTE_USER         $remote_user;
+
+            uwsgi_modifier1 9;
+            uwsgi_pass  127.0.0.1:{{ pillar['git_pillar']['uwsgi_port'] }};
+        }
+    }
+}

tests/integration/files/file/base/git_pillar/http/files/users

@@ -0,0 +1 @@
+gitpillaruser:$apr1$H2XscjOs$EY8ZeOFX2NqR3XVsOEUM71

tests/integration/files/file/base/git_pillar/http/files/uwsgi.yml

@@ -0,0 +1,13 @@
+uwsgi:
+  socket: 127.0.0.1:{{ pillar['git_pillar']['uwsgi_port'] }}
+  cgi: {{ salt['pillar.get']('git_pillar:libexec_dir', '/usr/libexec') }}/git-core/git-http-backend
+  chdir: %d
+  daemonize: {{ pillar['git_pillar']['config_dir'] }}/uwsgi.log
+  pidfile: {{ pillar['git_pillar']['config_dir'] }}/uwsgi.pid
+  # This is required to work around a bug in git-http-backend, introduced in
+  # git 2.4.4 and worked around with cgi-close-stdin-on-eof in uwsgi >= 2.0.13.
+  #
+  # See:
+  #   https://github.com/git/git/commit/6bc0cb5
+  #   https://github.com/unbit/uwsgi/commit/ac1e354
+  cgi-close-stdin-on-eof: 1

tests/integration/files/file/base/git_pillar/http/init.sls

@@ -0,0 +1,62 @@
+{%- set config_dir = pillar['git_pillar']['config_dir'] %}
+{%- set git_dir = pillar['git_pillar']['git_dir'] %}
+{%- set venv_dir = pillar['git_pillar']['venv_dir'] %}
+{%- set root_dir = pillar['git_pillar']['root_dir'] %}
+
+{{ config_dir }}/nginx.conf:
+  file.managed:
+    - source: salt://git_pillar/http/files/nginx.conf
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: True
+    - template: jinja
+
+{{ config_dir }}/uwsgi.yml:
+  file.managed:
+    - source: salt://git_pillar/http/files/uwsgi.yml
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: True
+    - template: jinja
+
+{{ root_dir }}:
+  file.directory:
+    - user: root
+    - group: root
+    - mode: 755
+
+{{ git_dir }}/users:
+  file.managed:
+    - source: salt://git_pillar/http/files/users
+    - user: root
+    - group: root
+    - makedirs: True
+    - mode: 644
+
+{{ venv_dir }}:
+  virtualenv.managed:
+    - system_site_packages: False
+
+uwsgi:
+  pip.installed:
+    - name: 'uwsgi >= 2.0.13'
+    - bin_env: {{ venv_dir }}
+    - env_vars:
+        UWSGI_PROFILE: cgi
+    - require:
+      - virtualenv: {{ venv_dir }}
+
+start_uwsgi:
+  cmd.run:
+    - name: '{{ venv_dir }}/bin/uwsgi --yaml {{ config_dir }}/uwsgi.yml'
+    - require:
+      - pip: uwsgi
+      - file: {{ config_dir }}/uwsgi.yml
+
+start_nginx:
+  cmd.run:
+    - name: 'nginx -c {{ config_dir }}/nginx.conf'
+    - require:
+      - file: {{ config_dir }}/nginx.conf

tests/integration/files/file/base/git_pillar/ssh/init.sls

@@ -0,0 +1,3 @@
+include:
+  - git_pillar.ssh.server
+  - git_pillar.ssh.user

tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAv7u9K4Ns5YxN/QLYkQ6HSP9jMw/3haGDqv3ryG5n6hglsOco
+Jb0eapmBA0vc9C1VAMzyIuksBOqmDXIgX4NVkz57H4BwvKnXA4Rh5Ij/AEB2GNRo
+QQzFPgBItPXGAWEkE4euI+r2pf1yBgILOWqZZ6xThT8J9aAbuFOaeZNWzfUOD9b/
+gnmPzRSazoh7YKydsG1uheKX078BCE02OSnWPj4GCc6OKuj2ufpmI8gY6x+GJbyh
+ZG+BMmmcKtdsd6WcKZsranjY1oye3y8gWRT5mmdc98oqZEvARaHHZKotGNJo/Grt
+rgG9oo3dUF/JdHU6rzqZetX4OeTSnON4JryCIwIDAQABAoIBAQCc3cnotu86Y29P
+KKvtChjvRVtw5IhbwYhLNtJYutOz+CumL4luTut6xbqC6ueMsyYPsJ4Op/0GzMAs
+0gnge0BhZsYvQNN71+z9iKra4qbXGuZEbEwbpIofrvXNcCOe704n2GNGKa/AoLpQ
+Zg2u3SNDaf8vTiMk3eiwB16kR0LG32H5cWdBqvKlWI8LcchpBoNWRuA4XpRnfar4
+73pMnoehnb1ePwDqSOg38y4HYh4iD24XvBGzZDtHKmcGU47oRvIHwCOMjqu16tHk
+k7LQwZoxM5z1APyw1qzV/yHGvlomf9K3ZQKViFOqm1CoN2uCjq/MnsVlCszjPe48
+NrCoFSypAoGBAPdJc7d9Vrzpxfhn+niIxgneYP4CwChqTyAvAHLV97/RtSL3HeNj
+QmINKjW7MUx7ObmjF7ITkkY9hpsplX3P76ZytSBV3V9Be5rqfedGsQpWJb50MKoa
+PfSvtWphK4hdAg6VN1YifbLWrD1h6bwUrzlfhYsvaqHqVnprwCTn8nFXAoGBAMZ9
+MHRHVlaeI5nBDM+fzTHhAEbTV1Ak0FfpHYnLCRVoBC6PwSNbTjQmsWJ0yEEgxe3I
+U4YkgVuKR7NRFUuF9gf6Y5v57AhVuLuAg2kSXoj2DVkXSE8F1crzw8qVI1LTBIle
+PcG/kmD622XKbtWpU8yTi1sdtAX7F67yhsligLoVAoGBAMsSc8e8U02iAKRk5wiy
+8UbLawVNxvWpj78TOiAT3HeWxFSpcM76BVq2CvLC/dIb46Sx7VScw+OQxQiI1q3R
+47DhxCKAwOFnyhTG+ovBvsOJSUek7Q3TrQtSe/2XPIOoNXc6TI4clvMVXa6uyJ5e
+siLAcc+CKeQ7p7ay48CrBarTAoGAT4qlm0NnNwjibWAumRmJ6l4ndTqGN+i40THr
+E2gY+MoZOuuC039ohH+pADKaeXb/un1X8163tA5jE1n/9ab2ZFYUCtKJowFvKTyj
+7Lxew/YOfVBWOsy000MCiDFh2XQU0lPA3d6+czy0JUONTPQxT78kzlvF48uuvv4T
+w6pEuc0CgYEA5CCrlOYrn4pt6unCm6K0yuGfmk9MTaChTdbpqAbgfp4uG3YLKHfb
+G7WAoFCAsS8pWQZ7EVVj8COKMSYJT67CGaYFUUORjqPrguqv7SG4ZWhSzdzjls6X
+zcg/f1GimEPP0QeZjdfF+wfrO2a1GGFAHCO6pbh0EEfmFuXGFo0DEG4=
+-----END RSA PRIVATE KEY-----

tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/u70rg2zljE39AtiRDodI/2MzD/eFoYOq/evIbmfqGCWw5yglvR5qmYEDS9z0LVUAzPIi6SwE6qYNciBfg1WTPnsfgHC8qdcDhGHkiP8AQHYY1GhBDMU+AEi09cYBYSQTh64j6val/XIGAgs5aplnrFOFPwn1oBu4U5p5k1bN9Q4P1v+CeY/NFJrOiHtgrJ2wbW6F4pfTvwEITTY5KdY+PgYJzo4q6Pa5+mYjyBjrH4YlvKFkb4EyaZwq12x3pZwpmytqeNjWjJ7fLyBZFPmaZ1z3yipkS8BFocdkqi0Y0mj8au2uAb2ijd1QX8l0dTqvOpl61fg55NKc43gmvIIj root@tardis

tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config

@@ -0,0 +1,9 @@
+Port {{ pillar['git_pillar']['sshd_port'] }}
+ListenAddress 127.0.0.1
+PermitRootLogin no
+ChallengeResponseAuthentication no
+PasswordAuthentication no
+PubkeyAuthentication yes
+PrintMotd no
+
+HostKey {{ pillar['git_pillar']['sshd_config_dir'] }}/ssh_host_rsa_key

tests/integration/files/file/base/git_pillar/ssh/server/init.sls

@@ -0,0 +1,44 @@
+{%- set sshd_config_dir = pillar['git_pillar']['sshd_config_dir'] %}
+
+{{ sshd_config_dir }}/sshd_config:
+  file.managed:
+    - source: salt://git_pillar/ssh/server/files/sshd_config
+    - user: root
+    - group: root
+    - mode: 644
+    - template: jinja
+
+{{ sshd_config_dir }}/ssh_host_rsa_key:
+  file.managed:
+    - source: salt://git_pillar/ssh/server/files/ssh_host_rsa_key
+    - user: root
+    - group: root
+    - mode: 600
+    - template: jinja
+
+{{ sshd_config_dir }}/ssh_host_rsa_key.pub:
+  file.managed:
+    - source: salt://git_pillar/ssh/server/files/ssh_host_rsa_key.pub
+    - user: root
+    - group: root
+    - mode: 644
+    - template: jinja
+
+{%- if grains['os_family'] == 'Debian' %}
+/var/run/sshd:
+  file.directory:
+    - user: root
+    - group: root
+    - mode: 755
+{%- endif %}
+
+start_sshd:
+  cmd.run:
+    - name: '{{ pillar['git_pillar']['sshd_bin'] }} -f {{ sshd_config_dir }}/sshd_config'
+    - require:
+      - file: {{ sshd_config_dir }}/sshd_config
+      - file: {{ sshd_config_dir }}/ssh_host_rsa_key
+      - file: {{ sshd_config_dir }}/ssh_host_rsa_key.pub
+{%- if grains['os_family'] == 'Debian' %}
+      - file: /var/run/sshd
+{%- endif %}

tests/integration/files/file/base/git_pillar/ssh/user/files/authorized_keys

@@ -0,0 +1,2 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXo/n7jE3xaZRaJo6/ghm6sldUabg1G4DmZNWJx7By5XVnei6dWU3Qs2ZCKrgfDkq7lCPcujQpkOEX4Yei+k39A1kjlwkULXdi7aOquJOGVu32B0RjMQGG44SulAItmuXyu9dmmmVbXKPu35fGJtB4Ew9B5WNu1tzj4T9C8x5INgTJe0blKxTUaAeWD3/YCvAhKOhteLxMNHlcFdqzF/NnPIDzPXue6YHiPkxMZk/yiPJ49TKmC2L77ogxhJmOu2fGMj91Lh151EH7kmstDKzkuD/sU1jn21Y1bq8pYGiK3rNgb+NklF3BmjRl2fZod5nSmhYyqyaYUR3oyuyfqcpsaYbwLso92wnFAuhHA8M+iqgMJcyv13ur3rR81mH4PFw6viShBVUUDQ/jKad7PDeSA9OoO6+tjUbsmyUWqjCe89W0vmUJsGKoCX/0mpWW4+j8teoIfaS/jDfxGrFo5u6Sa2UB8DLxcfingyO9RW3ubkKAd/Saa/0H+8vLWdQYz8j+TDqZOn+eUjKuPMulcLiPpDAPtD5eewIG5su0Cs4rqRZr0qSRUMQ0Iqa+KuWfT9abh8LtXKRhiJFLVrtD5mRnnuR9vnJVyzD6S6VHsEHRFIrXJy8yBhEyzJlSE2iRh0aJ/tgfZ4aeiq8SO1SVu3jvW4uRf9j8gx3qOCMD0gSsVQ== nopass
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClaM74C8D9DeonGmOZDylJ83bxID+R82fB4YT+wSDpA6ewdCBLZT5PcB/FmNZ3itBewPmimz3RUdKe6ms/rQ76tarzxm8msHt3BknqxSXqcv/Gq90ilHsdgNK8TMWRTOlp1+rX2LnV58Wf5pdFNfIqru8SGcf2c5ZhtOMVAY6dVVZiovgSoqnFPU9Asu8uMnLK/dar+JjwImULkzAeBCTus49Xk5Ss4RQ3yzbd6vucYsbAxJKiwlBcXbwlGCxYZ9wUEZqbktKRmVbFbZboisW5NKG0ks9OBAickRpFumYGBXMv8iItnugni/lpkyzBfsD8tfGg/BzlaAO1MAQER4qZYHaqMm+ZAmkGhFQ82oMNiRumrzWL0sfzJ18EEbY5dUPvYtEj+bn8uDvRVTIQcPzoppWnOsheV7YBnRNRUdsUgGITZGHoBFobwc006YJ2+S+ASLMFMlT9HNy5WuvvmiIaCZaqeZH55dlDBdu+r8qNn79sYm4FUIQ65f6TiReU8/D6s3Lc2AfakbB3n6IBFIRCFWDJlSYyzXXvTHman8tTesL2TmXrgCPiQtMAagVtU6uHYFDeRcyKkm1WDxsCAK8z6Oc9tINK1ZgtBaQLGq5HTlejLTK89iED5TPXFM0BW3w/aRfT6e62oq3GpFDka6hreR8fKkVOCVyibnWk+hqX7Q== withpass

tests/integration/files/file/base/git_pillar/ssh/user/files/git_ssh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/bin/ssh -p {{ pillar['git_pillar']['sshd_port'] }} -i /root/.ssh/{{ pillar['git_pillar']['id_rsa_nopass'] }} "$@"

tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKQIBAAKCAgEA16P5+4xN8WmUWiaOv4IZurJXVGm4NRuA5mTVicewcuV1Z3ou
+nVlN0LNmQiq4Hw5Ku5Qj3Lo0KZDhF+GHovpN/QNZI5cJFC13Yu2jqriThlbt9gdE
+YzEBhuOErpQCLZrl8rvXZpplW1yj7t+XxibQeBMPQeVjbtbc4+E/QvMeSDYEyXtG
+5SsU1GgHlg9/2ArwISjobXi8TDR5XBXasxfzZzyA8z17numB4j5MTGZP8ojyePUy
+pgti++6IMYSZjrtnxjI/dS4dedRB+5JrLQys5Lg/7FNY59tWNW6vKWBoit6zYG/j
+ZJRdwZo0Zdn2aHeZ0poWMqsmmFEd6Mrsn6nKbGmG8C7KPdsJxQLoRwPDPoqoDCXM
+r9d7q960fNZh+DxcOr4koQVVFA0P4ymnezw3kgPTqDuvrY1G7JslFqownvPVtL5l
+CbBiqAl/9JqVluPo/LXqCH2kv4w38RqxaObukmtlAfAy8XH4p4MjvUVt7m5CgHf0
+mmv9B/vLy1nUGM/I/kw6mTp/nlIyrjzLpXC4j6QwD7Q+XnsCBubLtArOK6kWa9Kk
+kVDENCKmvirln0/Wm4fC7VykYYiRS1a7Q+ZkZ57kfb5yVcsw+kulR7BB0RSK1ycv
+MgYRMsyZUhNokYdGif7YH2eGnoqvEjtUlbt471uLkX/Y/IMd6jgjA9IErFUCAwEA
+AQKCAgBF7hpSVhSstkVy2tAuEL3RSqaBbGtdZZbuoEKTlNuG1xy0uu3E/0H57UO7
+L2lYQOVBYXAj04q49A/bE7tNwghqhZxxqzg5f+kYfuI1qffFeAlhYMfvtuO836mW
+h88RBQuPJRVcY7N85lUPURlCHDI8zkmDYCVXu3wUtmYyiu8GEeaJhF3gUZFGtJnJ
+MyuNXzayOjbt0VqXB+lXUIsEyz6W+wsCVqzxQt5pBTTvDbrdd0XSrgmHyWeHNbqa
+/Fpj7ChiIMdtc9ABQzFGqRvylwq2fX3VYM4TGpEhcMyDCY29gyz+mCpQ4sBo2V4m
+rYF4LVkH8ApE0jYI7T1a0fvcZ06KMwISF9NsDlSrJc6MS6GQyUyA0hYtFi79paVM
+pRJ1qPoKb6On78Hiqzj4xe/WUKc+OCCDSO8n9oEXnEw4vl7sDmR8dxlhq6Mkr4nt
+2DK6Cg7ITJXBcn0hD/FjJHRXR5NirL3tfNuan/BcLzuQU0xCMYIn8VG8Ajb/qJ7F
+l6vy6lbyRxRigVSDWMFr+9HblaYoxdxQjgrmVwVNYmgIucOoOT6DQl9uTC9EQtcL
+ubzMEaW+AdMo0th90i/b6WGro4+Gb2rq1THkA/ObvjjWQW56WZ+JBw2memYNRheG
+dFLhH5ugq/AoVjcFxXha2sYQWESuzhHv6lVva6lEkSJ19Li1YQKCAQEA/HZde8qQ
+ysuARW4gPLBKCByPzg2c36x0pO3xZOpKzC1MMMqarI/lCcx5gre/PlZ629zTJYgK
+fOYnNpQBolotecjtffES7McYhymrIVNwMdgu0RRWT2clfV2Fx79M+EoYqNdcDdsH
+7r7hvUyu+9/DRTiB0N4ZjZY+7FmFKt9c4tuica18mtPc5aGXg9oD4qxZZb+pYicj
+ezxXq3wR8aLBxR9xP57lz5FjsVZATSjA0zCL+iCXVM+V5QDtKTgVs3V9v5agWn58
+2527qOU2ILXfBtfDrTH2NYtILHPOH2QXTcWOBsGhUciVWMFZLUqAPEg/G/lzIkzE
+pp7naFw4dq2EqQKCAQEA2qmGFsa6HI6RtEpLRd17Ef1TME9W2e+eqd7M8QCi7e2t
+Wr986pHE1/oFOBBKN6d0d5AqMZOpNhE43fMnq3EWpqW4TvgwR8uwoCp/XPHm8Sxl
+WQkhGneeHc7g9fxjErXXjj6/hO44akq+M4EIfsPDt7M27EGZBTMTgVeUPnnXG7/I
+pabuFXlsAdkAsEXrbEaN9T0Fb4ACKHQlA8PPIFT7eMzl91DG1xlkpb6B1OoNth6N
+/I7NfxOUc3JTWJqMV7rXbniWMGCsuB900VvbhaL0bNv2VOrewGNTgLXomySPllQQ
+UaWsIT3G8It9My0cRj1JevUenXfdULB9lcozvVuJzQKCAQAYYm5hGI2nqMQ48IwY
+kIZ2Bhw1sMboK8YQcBMSxjZ3RiDHzanm5PcgXSmXYJwOL1gqiEe0plEtAyXidaU6
+wy8FRkz6DyDe0dQiqfmnfGGnztOmyioT/Uh3tWLIikeq6606EaMIi5FWlAVFvXRh
+S5mWxAB15h3duRdWyMa9/1j/aGtmQ3V3luMNIvB5gcNCT5dK5po7qsAYlRl6rL8m
+8at5mLHdjUFxLP/ODyCi0z7cpyG+BQvY2zwFJHPDuXEPJlgA+1F9rB3vMGsBwzHZ
+MvfZt1llDyBSx6Mu9/h+u7IshtpS+LzWI2OZcQNmBn4gVHIUB6IBPBz6YvrC77Vc
+cSIRAoIBAQCp6OK93gwOVqZXrwdQsaqJLwyuVGhLjsv+eZdMik8QjQiQpI0/hKet
+n6TgjJ/vIRr6MTboMTJiRf2nUeN4b7bHJazTCD4T++4ydvNi2MG4k+PozJRBicN+
+rBvYaRbfGhf2e0G83JNP3OZxBQoB3sK9gu/ho5NxG+BDODeEWI7TDDKwrccBPsmz
+odjMIHiwOR7j+le37YM/xghhJY1UNVT26Fil1cm8qQmxVRhzxq+C3bk9EAYUgbVw
+A91J00XMge4W9HLYArcTl7XhXPx2mkpOMJn4IE2Yt1XShQfLThyZFpdbql3XsrZc
+gjd2Rc5bshHgDoqMl/CMW6gqdeXAdVndAoIBAQDdSFdm3LrMGda3NHqTqyuo6Oa8
+AvmsL7MG4g4A3jGmr1KP6VomBBtI+Nvt9ZqAFhhYsVOAtU6TAvGohE21uSLrkmzR
+nwCxNU9lgnUZHq00r1mFn9mXwJVjYpTLlQj+s3amt641+QNDQ5paWj06z6j/2nlQ
+zYZ/BjNXZri+hsFCFcuSS1W0NwYzd6vHQJmRWIL4puXl2vK0ZIWcAX9l6pFJoWWC
+uNIRjeyszvlav6pUVE5jBOKSt/nfgGiir353ArU6uG27z/h/ruqPIvNFeeWpmaWp
+Ii8D2AAgG3mF61dvoEAtx1moB+795KajYvu0izw4gd5rsPUZRG350zVutvL/
+-----END RSA PRIVATE KEY-----

tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXo/n7jE3xaZRaJo6/ghm6sldUabg1G4DmZNWJx7By5XVnei6dWU3Qs2ZCKrgfDkq7lCPcujQpkOEX4Yei+k39A1kjlwkULXdi7aOquJOGVu32B0RjMQGG44SulAItmuXyu9dmmmVbXKPu35fGJtB4Ew9B5WNu1tzj4T9C8x5INgTJe0blKxTUaAeWD3/YCvAhKOhteLxMNHlcFdqzF/NnPIDzPXue6YHiPkxMZk/yiPJ49TKmC2L77ogxhJmOu2fGMj91Lh151EH7kmstDKzkuD/sU1jn21Y1bq8pYGiK3rNgb+NklF3BmjRl2fZod5nSmhYyqyaYUR3oyuyfqcpsaYbwLso92wnFAuhHA8M+iqgMJcyv13ur3rR81mH4PFw6viShBVUUDQ/jKad7PDeSA9OoO6+tjUbsmyUWqjCe89W0vmUJsGKoCX/0mpWW4+j8teoIfaS/jDfxGrFo5u6Sa2UB8DLxcfingyO9RW3ubkKAd/Saa/0H+8vLWdQYz8j+TDqZOn+eUjKuPMulcLiPpDAPtD5eewIG5su0Cs4rqRZr0qSRUMQ0Iqa+KuWfT9abh8LtXKRhiJFLVrtD5mRnnuR9vnJVyzD6S6VHsEHRFIrXJy8yBhEyzJlSE2iRh0aJ/tgfZ4aeiq8SO1SVu3jvW4uRf9j8gx3qOCMD0gSsVQ== nopass

tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass

@@ -0,0 +1,54 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,660A940DCC281B9A88A618C326B9B153
+
+CC06yN5RjARlX6sRsbM3WL4D2IVnseOpjSULoA2Gcq/jAgm3hOjGADNJu8vRKvrH
+TahYOE/ABQBiuJFxy45hp7wDF4ArFeAPN1cYhdHTSZYmx58BorBlYwszlKq6rTqc
+mQYTtAVBiCP6KmXI6Uc1GEoD/v+t/XlFbbhkN08rsy7+WXIKuzff4ThmKfD9Bfsh
+PXlJn1UHNPFSSZdLdPI2JwILAZSlFbfp7v/PosY4aeQn9fezOVBhpZ8AWlO+C5ZL
+UJsUYmCvmP+L8UjXz/UVxFQxPVYFJrPe/QmxiKSjx7F42Tb/2dKYxCIc7IH2gaOM
+qoPU2kFRDS8aUu6ctQw1q5h/uE/rggArueTf/zjTbOm0Jq95HsXmhPtvNNMFNlt+
+hOfPl8El+w9zRi+9Eduvm0l8IV4IryiPTgYDW/dQdgtniH5KNnqElZIlEUH2q0gd
+5rC5y7du4rEBX3mwVvOWlyVQyuCjyEBiJ34BECNnSS+ZKTQPCyFuct5Ck9Fngcwk
+WOeJKkFTw7IuBckdCRDg2UrSx6duTJQOF8yotJEtZkOgR3Igg7JhSOVC2UvVUGJ9
+rL8ILyc/BRgp6MPWhWIWE0eMAiVqGtwkp5Mwj3cwM7FQoKlwZymKJSeqPMUAjT7H
+M+wJoMoIYEAPuBLYKO8QCUU7MxHlGyX9Eg+sryoIBCas9M0ES+q5P5nuxtPZLH8w
+JmW3GSD6OLVuUnUfrF7IL9fo2LcBIRliz86JD3rkTu4PSE/lcMTu07bTtM1Ker3E
+EXqmOGdwIc9HPT+4cZScuv4YAlqHQqMMcrhYPSqyLt8OwJTWMCWkTVSU9VSOEgzj
+JubgiVciyco/0+qAdQSXDk9/2EApNjrGG7LKYJTExa7Bp+fYm5tp+RVxHUsLSQcA
+kF3RfMC9+ab5wQDEKTDMj76n298i1GWQbAQXbqmncvW6AIIm+Q4TzILP5cyZtSiK
+NJxWgkceGJNCBMNwGzlmCZc6Et3qtxy9Y0/KsTCdk+RXGdh2pFyxNy4tP9bls/TC
+U1N4l+y3gaHoZ9afV0ijF10VF3dvia22e1znt2H1IpYtZi1oAk8VIuiztiJyfab8
+1jwTH0SN9Tu6YsC3Ma8EB+4/0kDFjxlQl+clzDgexXBG2ochckudSgUX7lpugVud
+5jb+ujZnsaRkT1W4Qi8T/AweXCwZrSbRcilXzDdjEXIqK2Dd6e4TsUiuoHmvHwlH
+xcZ+T1ytApRS3ztRV8sWGs5xqXXldWCPU+ArGY2L2Zshj61v40ON+3S3iZ5ponUP
+XEOFSaZq1Bk5JZRZDRgp7evL7UPSMXQWHPRAsFFPhZejLBsk3uwh6tUOVXo0aRhw
+d/xYqHFA/t9zP45ljPrjV0Ca4XIek3YFUxc8Tid6r9J5ymQRYSq8uZOcXL7x/b7E
+h+hTI91DxpCCM+/FpiJBf2ffGUbTYgg0qV0qbSC6Wsfie8GdtDxBMh10ZGfQ40wB
+BllbjeU0KvvSNGA3PjkwgesUMdZv2Jx6/shwcYiOSJvVPzhzxiiwxvXWA/o6QHem
+K6b6zpqOagSbBSwymh9uNe6F2io4Olab+PoU6N+DVVv/kApycbovXS/0GJ4O6pDt
+SFzsAE8mFKDiQEx7bei9BZIx/eg3vr64kw6GnKhzkncr/VVomYCBgHCEoag3y6Jj
+YzdI7QvPzCj3kM4bGckpl845O/p13vqxeD/Vi6skRAiLv5AOOItaHQPB/ZoBeY/L
+2+zy0AAtRc+o2E+CkrLPcdLv2WR3uQSSktgsW9suDNtDfPujvkFijhjidHoRfpN7
+aSzkJg/DkMwoxTOWp+Vx7wpsnE6iCmx9teazKBpGC1ngz8aLxxQGHZqQ9rt8DijO
+MBbsYW3bwIWkwsTX3PmlH9nTHbxVz518xlBY1gKp5s0i4rdMB67l2duln/ffpfOR
+N42GqO9Y+zQiA8LGrKu14EJZnsQUrycfbdY3/4t9mO4dXr1nPXrwPWyOQcxDEYF5
+R3JegGf+HSt4rKBYPTPmRiEfKL7eeLkLAiduHblCLILXfMkajGUZIKfSZboBFMnP
+ewIgFUux8Yn95Z2/k/NSuGlOWv9YUvGmlx0GlaqW3jDqsenR4svV4c9CvnCpJdVd
+z5EhUQmQVnZWoA8QhR0CuWDUQC3vRj2bCPROgtcbI+Is+8FFwzoWAcy7cIHKEAha
+enBK4/0jpGx52+RNWq2Q17fdLNouSl+dbNB1bRHu8MXfzdZvez9HPhcwXJvqnh5/
+zffWzX6XW9fl/fLcTZIpIJfuDek43j5D4huhApwzbMp4952N4Hd2bkfzCsiJ7m/E
+3//PWlhJ1nokU8wnXDQYkd7mmoyC5fNrqmqIeo5BtcGL4pkL3dkjK7PtRUkZ0Vt3
+mX/yUamrrSVF/i6voz5M+sr6SZAnZHDPZdqPJ+BN89KyuiqApr3QwdqupnALQmRO
+eWNNusNasLCW4wto9R7MWW6W5WgltwqEIQARrtiCIRzNaAp12YFXDkoBrzUpzPps
+4aOPBxOeu7QIpVhU1AvKNgCvaNgfmImJ1wyEHRmnGOpluJelhOKXVwYRDa6zRwVL
+75cxY7cY+fFPGm5rqk2CjLouYsL1BJ2NA02Cr2n33Yp+wzZPQgvRHqPyGyQlqyf9
+zeKDfZf23xYsojxLevFoePppALndep2Gr2ofaBnLxV9cbsi5t3ZtgoO8QXUGfTPT
+MUHs3/YQw1MNn0tZCMinPxF3On/VQB2E0VMKzbPjaWZ4Utes8p1x+fVbSEecxttv
+90K4wab0Qz29yZdOZ6LAZVrJXoHn/tB2D7Qo7dhmrcv4Z98iqLgadFcQ01Z0zkvu
+/rWfvSUQSNdEB/gEXHrgpdO5vRQ04E4Hz+yAnLksrLOwaznc0d3oRvWun7GUxRa9
+Rio4JhamfPIReVNvAPh7LMJKsHjVjWncsbnKnpcgMygNrWQ/gNe75D3Kdfsau7pc
+PNpNsrcj9K3cr7hU08VAHgkwn6rx0TwXC8ohURErw1v22pFjX7mkaJVLezRuXlKX
+gfICrlD/L0BoEy37dYgSoKI3mxkoUHtLkZttgrjvzfmFLsYANjvG+TEHxFSNte4w
+V1C9fYc0IT44+sLg26AchJ1YwiRW50tkI/IPWJl4YAb4EUR0GI2PWxGVy9134Y2l
+-----END RSA PRIVATE KEY-----

tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClaM74C8D9DeonGmOZDylJ83bxID+R82fB4YT+wSDpA6ewdCBLZT5PcB/FmNZ3itBewPmimz3RUdKe6ms/rQ76tarzxm8msHt3BknqxSXqcv/Gq90ilHsdgNK8TMWRTOlp1+rX2LnV58Wf5pdFNfIqru8SGcf2c5ZhtOMVAY6dVVZiovgSoqnFPU9Asu8uMnLK/dar+JjwImULkzAeBCTus49Xk5Ss4RQ3yzbd6vucYsbAxJKiwlBcXbwlGCxYZ9wUEZqbktKRmVbFbZboisW5NKG0ks9OBAickRpFumYGBXMv8iItnugni/lpkyzBfsD8tfGg/BzlaAO1MAQER4qZYHaqMm+ZAmkGhFQ82oMNiRumrzWL0sfzJ18EEbY5dUPvYtEj+bn8uDvRVTIQcPzoppWnOsheV7YBnRNRUdsUgGITZGHoBFobwc006YJ2+S+ASLMFMlT9HNy5WuvvmiIaCZaqeZH55dlDBdu+r8qNn79sYm4FUIQ65f6TiReU8/D6s3Lc2AfakbB3n6IBFIRCFWDJlSYyzXXvTHman8tTesL2TmXrgCPiQtMAagVtU6uHYFDeRcyKkm1WDxsCAK8z6Oc9tINK1ZgtBaQLGq5HTlejLTK89iED5TPXFM0BW3w/aRfT6e62oq3GpFDka6hreR8fKkVOCVyibnWk+hqX7Q== withpass

tests/integration/files/file/base/git_pillar/ssh/user/init.sls

@@ -0,0 +1,67 @@
+{%- set user = pillar['git_pillar']['user'] %}
+
+{{ user }}:
+  user.present:
+    - gid_from_name: True
+    - password: '$6$saYbZFw2$rtmvt2LOYchvlM22y34mCs7FiIN4Fq27rmv/whr/M.oPrgfCDhP5uJqnfe6uwFj90FvwA45rhZplnRNMgiY.J.'
+    - require:
+      - group: {{ user }}
+  group.present: []
+
+/home/{{ user }}/.ssh:
+  file.directory:
+    - user: {{ user }}
+    - group: {{ user }}
+    - dir_mode: 700
+    - require:
+      - user: {{ user }}
+      - group: {{ user }}
+
+/home/{{ user }}/.ssh/authorized_keys:
+  file.managed:
+    - source: salt://git_pillar/ssh/user/files/authorized_keys
+    - user: {{ user }}
+    - group: {{ user }}
+    - mode: 600
+
+# Custom SSH command
+{{ pillar['git_pillar']['git_ssh'] }}:
+  file.managed:
+    - source: salt://git_pillar/ssh/user/files/git_ssh
+    - user: {{ user }}
+    - group: {{ user }}
+    - mode: 755
+    - template: jinja
+
+/root/.ssh:
+  file.directory:
+    - dir_mode: 700
+    - user: root
+
+/root/.ssh/{{ pillar['git_pillar']['id_rsa_nopass'] }}:
+  file.managed:
+    - source: salt://git_pillar/ssh/user/files/id_rsa_nopass
+    - user: root
+    - group: root
+    - mode: 600
+
+/root/.ssh/{{ pillar['git_pillar']['id_rsa_nopass'] }}.pub:
+  file.managed:
+    - source: salt://git_pillar/ssh/user/files/id_rsa_nopass.pub
+    - user: root
+    - group: root
+    - mode: 644
+
+/root/.ssh/{{ pillar['git_pillar']['id_rsa_withpass'] }}:
+  file.managed:
+    - source: salt://git_pillar/ssh/user/files/id_rsa_withpass
+    - user: root
+    - group: root
+    - mode: 600
+
+/root/.ssh/{{ pillar['git_pillar']['id_rsa_withpass'] }}.pub:
+  file.managed:
+    - source: salt://git_pillar/ssh/user/files/id_rsa_withpass.pub
+    - user: root
+    - group: root
+    - mode: 644

tests/integration/modules/test_pillar.py

@@ -5,22 +5,7 @@ from __future__ import absolute_import
 
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
-from tests.support.unit import skipIf
 from tests.support.paths import TMP_STATE_TREE
-from tests.support.helpers import requires_network
-
-# Import salt libs
-from salt.utils.versions import LooseVersion
-
-GIT_PYTHON = '0.3.2'
-HAS_GIT_PYTHON = False
-
-try:
-    import git
-    if LooseVersion(git.__version__) >= LooseVersion(GIT_PYTHON):
-        HAS_GIT_PYTHON = True
-except ImportError:
-    pass
 
 
 class PillarModuleTest(ModuleCase):
@@ -40,32 +25,6 @@ class PillarModuleTest(ModuleCase):
         else:
             self.assertEqual(pillar['class'], 'other')
 
-    @requires_network()
-    @skipIf(HAS_GIT_PYTHON is False,
-            'GitPython must be installed and >= version {0}'.format(GIT_PYTHON))
-    def test_two_ext_pillar_sources_override(self):
-        '''
-        https://github.com/saltstack/salt/issues/12647
-        '''
-
-        self.assertEqual(
-            self.run_function('pillar.data')['info'],
-            'bar'
-        )
-
-    @requires_network()
-    @skipIf(HAS_GIT_PYTHON is False,
-            'GitPython must be installed and >= version {0}'.format(GIT_PYTHON))
-    def test_two_ext_pillar_sources(self):
-        '''
-        https://github.com/saltstack/salt/issues/12647
-        '''
-
-        self.assertEqual(
-            self.run_function('pillar.data')['abc'],
-            'def'
-        )
-
     def test_issue_5449_report_actual_file_roots_in_pillar(self):
         '''
         pillar['master']['file_roots'] is overwritten by the master
@@ -92,24 +51,6 @@ class PillarModuleTest(ModuleCase):
             self.run_function('pillar.data')['test_ext_pillar_opts']['file_roots']['base']
         )
 
-    def no_test_issue_10408_ext_pillar_gitfs_url_update(self):
-        import os
-        from salt.pillar import git_pillar
-        original_url = 'git+ssh://original@example.com/home/git/test'
-        changed_url = 'git+ssh://changed@example.com/home/git/test'
-        rp_location = os.path.join(self.master_opts['cachedir'], 'pillar_gitfs/0/.git')
-        opts = {
-            'ext_pillar': [{'git': 'master {0}'.format(original_url)}],
-            'cachedir': self.master_opts['cachedir'],
-        }
-
-        git_pillar._LegacyGitPillar('master', original_url, opts)
-        opts['ext_pillar'] = [{'git': 'master {0}'.format(changed_url)}]
-        grepo = git_pillar._LegacyGitPillar('master', changed_url, opts)
-        repo = git.Repo(rp_location)
-
-        self.assertEqual(grepo.rp_location, repo.remotes.origin.url)
-
     def test_pillar_items(self):
         '''
         Test to ensure we get expected output

tests/integration/modules/test_saltutil.py

@@ -73,6 +73,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
         Test syncing all ModuleCase
         '''
         expected_return = {'engines': [],
+                           'clouds': [],
                            'grains': [],
                            'beacons': [],
                            'utils': [],
@@ -95,6 +96,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
         Test syncing all ModuleCase with whitelist
         '''
         expected_return = {'engines': [],
+                           'clouds': [],
                            'grains': [],
                            'beacons': [],
                            'utils': [],
@@ -114,6 +116,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
         Test syncing all ModuleCase with blacklist
         '''
         expected_return = {'engines': [],
+                           'clouds': [],
                            'grains': [],
                            'beacons': [],
                            'utils': [],
@@ -135,6 +138,7 @@ class SaltUtilSyncModuleTest(ModuleCase):
         Test syncing all ModuleCase with whitelist and blacklist
         '''
         expected_return = {'engines': [],
+                           'clouds': [],
                            'grains': [],
                            'beacons': [],
                            'utils': [],

tests/integration/pillar/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

tests/integration/states/test_archive.py

@@ -7,16 +7,10 @@ from __future__ import absolute_import
 import errno
 import logging
 import os
-import socket
-import threading
-import tornado.httpserver
-import tornado.ioloop
-import tornado.web
 
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
-from tests.support.paths import FILES
+from tests.support.helpers import skip_if_not_root, Webserver
-from tests.support.helpers import get_unused_localhost_port, skip_if_not_root
 from tests.support.mixins import SaltReturnAssertsMixin
 
 # Import salt libs
@@ -32,54 +26,21 @@ else:
 
 UNTAR_FILE = os.path.join(ARCHIVE_DIR, 'custom/README')
 ARCHIVE_TAR_HASH = 'md5=7643861ac07c30fe7d2310e9f25ca514'
-STATE_DIR = os.path.join(FILES, 'file', 'base')
 
 
 class ArchiveTest(ModuleCase, SaltReturnAssertsMixin):
     '''
     Validate the archive state
     '''
-    @classmethod
-    def webserver(cls):
-        '''
-        method to start tornado
-        static web app
-        '''
-        cls._ioloop = tornado.ioloop.IOLoop()
-        cls._ioloop.make_current()
-        cls._application = tornado.web.Application([(r'/(.*)', tornado.web.StaticFileHandler,
-                                                    {'path': STATE_DIR})])
-        cls._application.listen(cls.server_port)
-        cls._ioloop.start()
-
     @classmethod
     def setUpClass(cls):
-        '''
+        cls.webserver = Webserver()
-        start tornado app on thread
+        cls.webserver.start()
-        and wait till its running
+        cls.archive_tar_source = cls.webserver.url('custom.tar.gz')
-        '''
-        cls.server_port = get_unused_localhost_port()
-        cls.server_thread = threading.Thread(target=cls.webserver)
-        cls.server_thread.daemon = True
-        cls.server_thread.start()
-        cls.archive_tar_source = 'http://localhost:{0}/custom.tar.gz'.format(cls.server_port)
-        # check if tornado app is up
-        port_closed = True
-        while port_closed:
-            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            result = sock.connect_ex(('127.0.0.1', cls.server_port))
-            if result == 0:
-                port_closed = False
 
     @classmethod
     def tearDownClass(cls):
-        cls._ioloop.add_callback(cls._ioloop.stop)
+        cls.webserver.stop()
-        cls.server_thread.join()
-        for attrname in ('_ioloop', '_application', 'server_thread'):
-            try:
-                delattr(cls, attrname)
-            except AttributeError:
-                continue
 
     def setUp(self):
         self._clear_archive_dir()

tests/integration/states/test_file.py

@@ -13,14 +13,9 @@ import os
 import re
 import sys
 import shutil
-import socket
 import stat
 import tempfile
 import textwrap
-import threading
-import tornado.httpserver
-import tornado.ioloop
-import tornado.web
 import filecmp
 
 log = logging.getLogger(__name__)
@@ -29,12 +24,15 @@ log = logging.getLogger(__name__)
 from tests.support.case import ModuleCase
 from tests.support.unit import skipIf
 from tests.support.paths import FILES, TMP, TMP_STATE_TREE
-from tests.support.helpers import skip_if_not_root, with_system_user_and_group
+from tests.support.helpers import (
+    skip_if_not_root,
+    with_system_user_and_group,
+    Webserver,
+)
 from tests.support.mixins import SaltReturnAssertsMixin
 
 # Import salt libs
 import salt.utils
-from salt.utils.versions import LooseVersion
 
 HAS_PWD = True
 try:
@@ -53,15 +51,6 @@ import salt.ext.six as six
 from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin
 
 IS_WINDOWS = salt.utils.is_windows()
-GIT_PYTHON = '0.3.2'
-HAS_GIT_PYTHON = False
-
-try:
-    import git
-    if LooseVersion(git.__version__) >= LooseVersion(GIT_PYTHON):
-        HAS_GIT_PYTHON = True
-except ImportError:
-    HAS_GIT_PYTHON = False
 
 STATE_DIR = os.path.join(FILES, 'file', 'base')
 if IS_WINDOWS:
@@ -389,21 +378,6 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
         check_file = self.run_function('file.file_exists', [FILEPILLARDEF])
         self.assertTrue(check_file)
 
-    @skipIf(not HAS_GIT_PYTHON, "GitFS could not be loaded. Skipping test")
-    def test_managed_file_with_gitpillar_sls(self):
-        '''
-        Test to ensure git pillar data in sls
-        file is rendered properly and is created.
-        '''
-        state_name = 'file-pillargit'
-
-        ret = self.run_function('state.sls', [state_name])
-        self.assertSaltTrueReturn(ret)
-
-        # Check to make sure the file was created
-        check_file = self.run_function('file.file_exists', [FILEPILLARGIT])
-        self.assertTrue(check_file)
-
     @skip_if_not_root
     def test_managed_dir_mode(self):
         '''
@@ -2427,49 +2401,22 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
             if check_file:
                 self.run_function('file.remove', [file])
 
-PORT = 9999
-FILE_SOURCE = 'http://localhost:{0}/grail/scene33'.format(PORT)
-FILE_HASH = 'd2feb3beb323c79fc7a0f44f1408b4a3'
-
 
 class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin):
     '''
     Uses a local tornado webserver to test http(s) file.managed states with and
     without skip_verify
     '''
-    @classmethod
-    def webserver(cls):
-        '''
-        method to start tornado static web app
-        '''
-        application = tornado.web.Application([
-            (r'/(.*)', tornado.web.StaticFileHandler, {'path': STATE_DIR})
-        ])
-        cls.server = tornado.httpserver.HTTPServer(application)
-        cls.server.listen(PORT)
-        tornado.ioloop.IOLoop.instance().start()
-
     @classmethod
     def setUpClass(cls):
-        '''
+        cls.webserver = Webserver()
-        start tornado app on thread and wait until it is running
+        cls.webserver.start()
-        '''
+        cls.source = cls.webserver.url('grail/scene33')
-        cls.server_thread = threading.Thread(target=cls.webserver)
+        cls.source_hash = 'd2feb3beb323c79fc7a0f44f1408b4a3'
-        cls.server_thread.daemon = True
-        cls.server_thread.start()
-        # check if tornado app is up
-        port_closed = True
-        while port_closed:
-            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            result = sock.connect_ex(('127.0.0.1', PORT))
-            if result == 0:
-                port_closed = False
 
     @classmethod
     def tearDownClass(cls):
-        tornado.ioloop.IOLoop.instance().stop()
+        cls.webserver.stop()
-        cls.server_thread.join()
-        cls.server.stop()
 
     def setUp(self):
         fd_, self.name = tempfile.mkstemp(dir=TMP)
@@ -2495,7 +2442,7 @@ class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin):
         '''
         ret = self.run_state('file.managed',
                              name=self.name,
-                             source=FILE_SOURCE,
+                             source=self.source,
                              skip_verify=False)
         log.debug('ret = %s', ret)
         # This should fail because no hash was provided
@@ -2507,8 +2454,8 @@ class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin):
         '''
         ret = self.run_state('file.managed',
                              name=self.name,
-                             source=FILE_SOURCE,
+                             source=self.source,
-                             source_hash=FILE_HASH,
+                             source_hash=self.source_hash,
                              skip_verify=False)
         log.debug('ret = %s', ret)
         self.assertSaltTrueReturn(ret)
@@ -2519,7 +2466,7 @@ class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin):
         '''
         ret = self.run_state('file.managed',
                              name=self.name,
-                             source=FILE_SOURCE,
+                             source=self.source,
                              skip_verify=True)
         log.debug('ret = %s', ret)
         self.assertSaltTrueReturn(ret)

tests/runtests.py

@@ -112,6 +112,9 @@ TEST_SUITES = {
     'client':
        {'display_name': 'Client',
         'path': 'integration/client'},
+    'ext_pillar':
+       {'display_name': 'External Pillar',
+        'path': 'integration/pillar'},
     'grains':
        {'display_name': 'Grains',
         'path': 'integration/grains'},
@@ -255,6 +258,15 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
             action='store_true',
             help='Run tests for client'
         )
+        self.test_selection_group.add_option(
+            '-I',
+            '--ext-pillar',
+            '--ext-pillar-tests',
+            dest='ext_pillar',
+            default=False,
+            action='store_true',
+            help='Run ext_pillar tests'
+        )
         self.test_selection_group.add_option(
             '-G',
             '--grains',

tests/support/gitfs.py

@@ -0,0 +1,557 @@
+# -*- coding: utf-8 -*-
+'''
+Base classes for gitfs/git_pillar integration tests
+'''
+
+# Import python libs
+from __future__ import absolute_import
+import copy
+import errno
+import logging
+import os
+import psutil
+import shutil
+import signal
+import tempfile
+import textwrap
+import time
+import yaml
+
+# Import Salt libs
+import salt.utils
+from salt.fileserver import gitfs
+from salt.pillar import git_pillar
+from salt.ext.six.moves import range  # pylint: disable=redefined-builtin
+
+# Import Salt Testing libs
+from tests.support.case import ModuleCase
+from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin
+from tests.support.paths import TMP
+from tests.support.helpers import (
+    get_unused_localhost_port,
+    requires_system_grains,
+)
+from tests.support.mock import patch
+
+log = logging.getLogger(__name__)
+
+USERNAME = 'gitpillaruser'
+PASSWORD = 'saltrules'
+
+_OPTS = {
+    '__role': 'minion',
+    'environment': None,
+    'pillarenv': None,
+    'hash_type': 'sha256',
+    'file_roots': {},
+    'state_top': 'top.sls',
+    'state_top_saltenv': None,
+    'renderer': 'yaml_jinja',
+    'renderer_whitelist': [],
+    'renderer_blacklist': [],
+    'pillar_merge_lists': False,
+    'git_pillar_base': 'master',
+    'git_pillar_branch': 'master',
+    'git_pillar_env': '',
+    'git_pillar_root': '',
+    'git_pillar_ssl_verify': True,
+    'git_pillar_global_lock': True,
+    'git_pillar_user': '',
+    'git_pillar_password': '',
+    'git_pillar_insecure_auth': False,
+    'git_pillar_privkey': '',
+    'git_pillar_pubkey': '',
+    'git_pillar_passphrase': '',
+    'git_pillar_refspecs': [
+        '+refs/heads/*:refs/remotes/origin/*',
+        '+refs/tags/*:refs/tags/*',
+    ],
+    'git_pillar_includes': True,
+}
+PROC_TIMEOUT = 10
+NOTSET = object()
+
+
+class ProcessManager(object):
+    '''
+    Functions used both to set up self-contained SSH/HTTP servers for testing
+    '''
+    wait = 10
+
+    def find_proc(self, name=None, search=None):
+        def _search(proc):
+            return any([search in x for x in proc.cmdline()])
+        if name is None and search is None:
+            raise ValueError('one of name or search is required')
+        for proc in psutil.process_iter():
+            if name is not None:
+                if search is None:
+                    if name in proc.name():
+                        return proc
+                elif name in proc.name() and _search(proc):
+                    return proc
+            else:
+                if _search(proc):
+                    return proc
+        return None
+
+    def wait_proc(self, name=None, search=None, timeout=PROC_TIMEOUT):
+        for idx in range(1, self.wait + 1):
+            proc = self.find_proc(name=name, search=search)
+            if proc is not None:
+                return proc
+            else:
+                if idx != self.wait:
+                    log.debug(
+                        'Waiting for %s process (%d of %d)',
+                        name, idx, self.wait
+                    )
+                    time.sleep(1)
+                else:
+                    log.debug(
+                        'Failed fo find %s process after %d seconds',
+                        name, self.wait
+                    )
+        raise Exception(
+            'Unable to find {0} process running from temp config file {1} '
+            'using psutil'.format(name, search)
+        )
+
+
+class SSHDMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin):
+    '''
+    Functions to stand up an SSHD server to serve up git repos for tests.
+    '''
+    sshd_proc = None
+
+    @classmethod
+    def prep_server(cls):
+        cls.sshd_config_dir = tempfile.mkdtemp(dir=TMP)
+        cls.sshd_config = os.path.join(cls.sshd_config_dir, 'sshd_config')
+        cls.sshd_port = get_unused_localhost_port()
+        cls.url = 'ssh://{username}@127.0.0.1:{port}/~/repo.git'.format(
+            username=cls.username,
+            port=cls.sshd_port)
+        home = '/root/.ssh'
+        cls.ext_opts = {
+            'url': cls.url,
+            'privkey_nopass': os.path.join(home, cls.id_rsa_nopass),
+            'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'),
+            'privkey_withpass': os.path.join(home, cls.id_rsa_withpass),
+            'pubkey_withpass': os.path.join(home, cls.id_rsa_withpass + '.pub'),
+            'passphrase': cls.passphrase}
+
+    def spawn_server(self):
+        ret = self.run_function(
+            'state.apply',
+            mods='git_pillar.ssh',
+            pillar={'git_pillar': {'git_ssh': self.git_ssh,
+                                   'id_rsa_nopass': self.id_rsa_nopass,
+                                   'id_rsa_withpass': self.id_rsa_withpass,
+                                   'sshd_bin': self.sshd_bin,
+                                   'sshd_port': self.sshd_port,
+                                   'sshd_config_dir': self.sshd_config_dir,
+                                   'master_user': self.master_opts['user'],
+                                   'user': self.username}}
+        )
+
+        try:
+            self.sshd_proc = self.wait_proc(name='sshd',
+                                            search=self.sshd_config)
+        finally:
+            # Do the assert after we check for the PID so that we can track
+            # it regardless of whether or not something else in the SLS
+            # failed (but the SSH server still started).
+            self.assertSaltTrueReturn(ret)
+
+
+class WebserverMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin):
+    '''
+    Functions to stand up an nginx + uWSGI + git-http-backend webserver to
+    serve up git repos for tests.
+    '''
+    nginx_proc = uwsgi_proc = None
+
+    @classmethod
+    def prep_server(cls):
+        '''
+        Set up all the webserver paths. Designed to be run once in a
+        setUpClass function.
+        '''
+        cls.root_dir = tempfile.mkdtemp(dir=TMP)
+        cls.config_dir = os.path.join(cls.root_dir, 'config')
+        cls.nginx_conf = os.path.join(cls.config_dir, 'nginx.conf')
+        cls.uwsgi_conf = os.path.join(cls.config_dir, 'uwsgi.yml')
+        cls.git_dir = os.path.join(cls.root_dir, 'git')
+        cls.repo_dir = os.path.join(cls.git_dir, 'repos')
+        cls.venv_dir = os.path.join(cls.root_dir, 'venv')
+        cls.uwsgi_bin = os.path.join(cls.venv_dir, 'bin', 'uwsgi')
+        cls.nginx_port = cls.uwsgi_port = get_unused_localhost_port()
+        while cls.uwsgi_port == cls.nginx_port:
+            # Ensure we don't hit a corner case in which two sucessive calls to
+            # get_unused_localhost_port() return identical port numbers.
+            cls.uwsgi_port = get_unused_localhost_port()
+        cls.url = 'http://127.0.0.1:{port}/repo.git'.format(port=cls.nginx_port)
+        cls.ext_opts = {'url': cls.url}
+        # Add auth params if present (if so this will trigger the spawned
+        # server to turn on HTTP basic auth).
+        for credential_param in ('user', 'password'):
+            if hasattr(cls, credential_param):
+                cls.ext_opts[credential_param] = getattr(cls, credential_param)
+
+    @requires_system_grains
+    def spawn_server(self, grains):
+        auth_enabled = hasattr(self, 'username') and hasattr(self, 'password')
+        pillar = {'git_pillar': {'config_dir': self.config_dir,
+                                 'git_dir': self.git_dir,
+                                 'venv_dir': self.venv_dir,
+                                 'root_dir': self.root_dir,
+                                 'nginx_port': self.nginx_port,
+                                 'uwsgi_port': self.uwsgi_port,
+                                 'auth_enabled': auth_enabled}}
+
+        if grains['os_family'] in ('Debian',):
+            # Different libexec dir for git backend on Debian-based systems
+            pillar['git_pillar']['libexec_dir'] = '/usr/lib'
+
+        ret = self.run_function(
+            'state.apply',
+            mods='git_pillar.http',
+            pillar=pillar)
+
+        try:
+            self.nginx_proc = self.wait_proc(name='nginx',
+                                             search=self.nginx_conf)
+            self.uwsgi_proc = self.wait_proc(name='uwsgi',
+                                             search=self.uwsgi_conf)
+        finally:
+            # Do the assert after we check for the PID so that we can track
+            # it regardless of whether or not something else in the SLS
+            # failed (but the webserver still started).
+            self.assertSaltTrueReturn(ret)
+
+
+class GitTestBase(ModuleCase):
+    '''
+    Base class for all gitfs/git_pillar tests. Must be subclassed and paired
+    with either SSHDMixin or WebserverMixin to provide the server.
+    '''
+    case = port = bare_repo = admin_repo = None
+    maxDiff = None
+    git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com'
+    ext_opts = {}
+
+    # We need to temporarily skip pygit2 tests on EL7 until the EPEL packager
+    # updates pygit2 to bring it up-to-date with libgit2.
+    @requires_system_grains
+    def is_el7(self, grains):
+        return grains['os_family'] == 'RedHat' and grains['osmajorrelease'] == 7
+
+    @classmethod
+    def setUpClass(cls):
+        cls.prep_server()
+
+    def setUp(self):
+        # Make the test class available to the tearDownClass so we can clean up
+        # after ourselves. This (and the gated block below) prevent us from
+        # needing to spend the extra time creating an ssh server and user and
+        # then tear them down separately for each test.
+        self.update_class(self)
+
+    @classmethod
+    def update_class(cls, case):
+        '''
+        Make the test class available to the tearDownClass. Note that this
+        cannot be defined in a parent class and inherited, as this will cause
+        the parent class to be modified.
+        '''
+        if getattr(cls, 'case') is None:
+            setattr(cls, 'case', case)
+
+    def make_repo(self, root_dir, user='root'):
+        raise NotImplementedError()
+
+
+class GitFSTestBase(GitTestBase, LoaderModuleMockMixin):
+    '''
+    Base class for all gitfs tests
+    '''
+    @requires_system_grains
+    def setup_loader_modules(self, grains):  # pylint: disable=W0221
+        return {
+            gitfs: {
+                '__opts__': copy.copy(_OPTS),
+                '__grains__': grains,
+            }
+        }
+
+    def make_repo(self, root_dir, user='root'):
+        raise NotImplementedError()
+
+
+class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
+    '''
+    Base class for all git_pillar tests
+    '''
+    @requires_system_grains
+    def setup_loader_modules(self, grains):  # pylint: disable=W0221
+        return {
+            git_pillar: {
+                '__opts__': copy.copy(_OPTS),
+                '__grains__': grains,
+            }
+        }
+
+    def get_pillar(self, ext_pillar_conf):
+        '''
+        Run git_pillar with the specified configuration
+        '''
+        cachedir = tempfile.mkdtemp(dir=TMP)
+        #self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True)
+        ext_pillar_opts = yaml.safe_load(
+            ext_pillar_conf.format(
+                cachedir=cachedir,
+                extmods=os.path.join(cachedir, 'extmods'),
+                **self.ext_opts
+            )
+        )
+        with patch.dict(git_pillar.__opts__, ext_pillar_opts):
+            return git_pillar.ext_pillar(
+                'minion',
+                ext_pillar_opts['ext_pillar'][0]['git'],
+                {}
+            )
+
+    def make_repo(self, root_dir, user='root'):
+        self.bare_repo = os.path.join(root_dir, 'repo.git')
+        self.admin_repo = os.path.join(root_dir, 'admin')
+
+        for dirname in (self.bare_repo, self.admin_repo):
+            shutil.rmtree(dirname, ignore_errors=True)
+
+        # Create bare repo
+        self.run_function(
+            'git.init',
+            [self.bare_repo],
+            user=user,
+            bare=True)
+
+        # Clone bare repo
+        self.run_function(
+            'git.clone',
+            [self.admin_repo],
+            url=self.bare_repo,
+            user=user)
+
+        def _push(branch, message):
+            self.run_function(
+                'git.add',
+                [self.admin_repo, '.'],
+                user=user)
+            self.run_function(
+                'git.commit',
+                [self.admin_repo, message],
+                user=user,
+                git_opts=self.git_opts,
+            )
+            self.run_function(
+                'git.push',
+                [self.admin_repo],
+                remote='origin',
+                ref=branch,
+                user=user,
+            )
+
+        with salt.utils.fopen(
+                os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_:
+            fp_.write(textwrap.dedent('''\
+            base:
+              '*':
+                - foo
+            '''))
+        with salt.utils.fopen(
+                os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_:
+            fp_.write(textwrap.dedent('''\
+            branch: master
+            mylist:
+              - master
+            mydict:
+              master: True
+              nested_list:
+                - master
+              nested_dict:
+                master: True
+            '''))
+        # Add another file to be referenced using git_pillar_includes
+        with salt.utils.fopen(
+                os.path.join(self.admin_repo, 'bar.sls'), 'w') as fp_:
+            fp_.write('included_pillar: True\n')
+        _push('master', 'initial commit')
+
+        # Do the same with different values for "dev" branch
+        self.run_function(
+            'git.checkout',
+            [self.admin_repo],
+            user=user,
+            opts='-b dev')
+        # The bar.sls shouldn't be in any branch but master
+        self.run_function(
+            'git.rm',
+            [self.admin_repo, 'bar.sls'],
+            user=user)
+        with salt.utils.fopen(
+                os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_:
+            fp_.write(textwrap.dedent('''\
+            dev:
+              '*':
+                - foo
+            '''))
+        with salt.utils.fopen(
+                os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_:
+            fp_.write(textwrap.dedent('''\
+            branch: dev
+            mylist:
+              - dev
+            mydict:
+              dev: True
+              nested_list:
+                - dev
+              nested_dict:
+                dev: True
+            '''))
+        _push('dev', 'add dev branch')
+
+        # Create just a top file in a separate repo, to be mapped to the base
+        # env and referenced using git_pillar_includes
+        self.run_function(
+            'git.checkout',
+            [self.admin_repo],
+            user=user,
+            opts='-b top_only')
+        # The top.sls should be the only file in this branch
+        self.run_function(
+            'git.rm',
+            [self.admin_repo, 'foo.sls'],
+            user=user)
+        with salt.utils.fopen(
+                os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_:
+            fp_.write(textwrap.dedent('''\
+            base:
+              '*':
+                - bar
+            '''))
+        _push('top_only', 'add top_only branch')
+
+
+class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin):
+    '''
+    Base class for GitPython and Pygit2 SSH tests
+    '''
+    id_rsa_nopass = id_rsa_withpass = None
+    git_ssh = '/tmp/git_ssh'
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.case.sshd_proc is not None:
+            cls.case.sshd_proc.send_signal(signal.SIGTERM)
+        cls.case.run_state('user.absent', name=cls.username, purge=True)
+        for dirname in (cls.sshd_config_dir, cls.case.admin_repo,
+                        cls.case.bare_repo):
+            if dirname is not None:
+                shutil.rmtree(dirname, ignore_errors=True)
+        ssh_dir = os.path.expanduser('~/.ssh')
+        for filename in (cls.id_rsa_nopass,
+                         cls.id_rsa_nopass + '.pub',
+                         cls.id_rsa_withpass,
+                         cls.id_rsa_withpass + '.pub',
+                         cls.git_ssh):
+            try:
+                os.remove(os.path.join(ssh_dir, filename))
+            except OSError as exc:
+                if exc.errno != errno.ENOENT:
+                    raise
+
+    def setUp(self):
+        '''
+        Create the SSH server and user, and create the git repo
+        '''
+        super(GitPillarSSHTestBase, self).setUp()
+        self.sshd_proc = self.find_proc(name='sshd',
+                                        search=self.sshd_config)
+        self.sshd_bin = salt.utils.which('sshd')
+
+        if self.sshd_proc is None:
+            self.spawn_server()
+
+            known_hosts_ret = self.run_function(
+                'ssh.set_known_host',
+                user=self.master_opts['user'],
+                hostname='127.0.0.1',
+                port=self.sshd_port,
+                enc='ssh-rsa',
+                fingerprint='fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46',
+                hash_known_hosts=False,
+            )
+            if 'error' in known_hosts_ret:
+                raise Exception(
+                    'Failed to add key to {0} user\'s known_hosts '
+                    'file: {1}'.format(
+                        self.master_opts['user'],
+                        known_hosts_ret['error']
+                    )
+                )
+
+        root_dir = os.path.expanduser('~{0}'.format(self.username))
+        if root_dir.startswith('~'):
+            self.fail(
+                'Unable to resolve homedir for user \'{0}\''.format(
+                    self.username
+                )
+            )
+        self.make_repo(root_dir, user=self.username)
+
+    def get_pillar(self, ext_pillar_conf):
+        '''
+        Wrap the parent class' get_pillar() func in logic that temporarily
+        changes the GIT_SSH to use our custom script, ensuring that the
+        passphraselsess key is used to auth without needing to modify the root
+        user's ssh config file.
+        '''
+        orig_git_ssh = os.environ.pop('GIT_SSH', NOTSET)
+        os.environ['GIT_SSH'] = self.git_ssh
+        try:
+            return super(GitPillarSSHTestBase, self).get_pillar(ext_pillar_conf)
+        finally:
+            os.environ.pop('GIT_SSH', None)
+            if orig_git_ssh is not NOTSET:
+                os.environ['GIT_SSH'] = orig_git_ssh
+
+
+class GitPillarHTTPTestBase(GitPillarTestBase, WebserverMixin):
+    '''
+    Base class for GitPython and Pygit2 HTTP tests
+    '''
+    @classmethod
+    def tearDownClass(cls):
+        for proc in (cls.case.nginx_proc, cls.case.uwsgi_proc):
+            if proc is not None:
+                try:
+                    proc.send_signal(signal.SIGQUIT)
+                except psutil.NoSuchProcess:
+                    pass
+        shutil.rmtree(cls.root_dir, ignore_errors=True)
+
+    def setUp(self):
+        '''
+        Create and start the webserver, and create the git repo
+        '''
+        super(GitPillarHTTPTestBase, self).setUp()
+        self.nginx_proc = self.find_proc(name='nginx',
+                                         search=self.nginx_conf)
+        self.uwsgi_proc = self.find_proc(name='uwsgi',
+                                         search=self.uwsgi_conf)
+
+        if self.nginx_proc is None and self.uwsgi_proc is None:
+            self.spawn_server()  # pylint: disable=E1120
+
+        self.make_repo(self.repo_dir)

tests/support/helpers.py

@@ -13,16 +13,20 @@
 
 # Import Python libs
 from __future__ import absolute_import
-import os
+import base64
-import sys
-import time
 import errno
-import types
+import functools
-import signal
-import socket
 import inspect
 import logging
-import functools
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+import tornado.ioloop
+import tornado.web
+import types
 
 # Import 3rd-party libs
 import psutil  # pylint: disable=3rd-party-module-not-gated
@@ -44,6 +48,7 @@ except ImportError:
 # Import Salt Tests Support libs
 from tests.support.unit import skip, _id
 from tests.support.mock import patch
+from tests.support.paths import FILES
 
 log = logging.getLogger(__name__)
 
@@ -1272,3 +1277,180 @@ def repeat(caller=None, condition=True, times=5):
             caller(cls)
         return cls
     return wrap
+
+
+def http_basic_auth(login_cb=lambda username, password: False):
+    '''
+    A crude decorator to force a handler to request HTTP Basic Authentication
+
+    Example usage:
+
+    .. code-block:: python
+
+        @http_basic_auth(lambda u, p: u == 'foo' and p == 'bar')
+        class AuthenticatedHandler(tornado.web.RequestHandler):
+            pass
+    '''
+    def wrapper(handler_class):
+        def wrap_execute(handler_execute):
+            def check_auth(handler, kwargs):
+
+                auth = handler.request.headers.get('Authorization')
+
+                if auth is None or not auth.startswith('Basic '):
+                    # No username/password entered yet, we need to return a 401
+                    # and set the WWW-Authenticate header to request login.
+                    handler.set_status(401)
+                    handler.set_header(
+                        'WWW-Authenticate', 'Basic realm=Restricted')
+
+                else:
+                    # Strip the 'Basic ' from the beginning of the auth header
+                    # leaving the base64-encoded secret
+                    username, password = \
+                        base64.b64decode(auth[6:]).split(':', 1)
+
+                    if login_cb(username, password):
+                        # Authentication successful
+                        return
+                    else:
+                        # Authentication failed
+                        handler.set_status(403)
+
+                handler._transforms = []
+                handler.finish()
+
+            def _execute(self, transforms, *args, **kwargs):
+                check_auth(self, kwargs)
+                return handler_execute(self, transforms, *args, **kwargs)
+
+            return _execute
+
+        handler_class._execute = wrap_execute(handler_class._execute)
+        return handler_class
+    return wrapper
+
+
+class Webserver(object):
+    '''
+    Starts a tornado webserver on 127.0.0.1 on a random available port
+
+    USAGE:
+
+    .. code-block:: python
+
+        from tests.support.helpers import Webserver
+
+        webserver = Webserver('/path/to/web/root')
+        webserver.start()
+        webserver.stop()
+    '''
+    def __init__(self,
+                 root=None,
+                 port=None,
+                 wait=5,
+                 handler=None):
+        '''
+        root
+            Root directory of webserver. If not passed, it will default to the
+            location of the base environment of the integration suite's file
+            roots (tests/integration/files/file/base/)
+
+        port
+            Port on which to listen. If not passed, a random one will be chosen
+            at the time the start() function is invoked.
+
+        wait : 5
+            Number of seconds to wait for the socket to be open before raising
+            an exception
+
+        handler
+            Can be used to use a subclass of tornado.web.StaticFileHandler,
+            such as when enforcing authentication with the http_basic_auth
+            decorator.
+        '''
+        if port is not None and not isinstance(port, six.integer_types):
+            raise ValueError('port must be an integer')
+
+        if root is None:
+            root = os.path.join(FILES, 'file', 'base')
+        try:
+            self.root = os.path.realpath(root)
+        except AttributeError:
+            raise ValueError('root must be a string')
+
+        self.port = port
+        self.wait = wait
+        self.handler = handler \
+            if handler is not None \
+            else tornado.web.StaticFileHandler
+        self.web_root = None
+
+    def target(self):
+        '''
+        Threading target which stands up the tornado application
+        '''
+        self.ioloop = tornado.ioloop.IOLoop()
+        self.ioloop.make_current()
+        self.application = tornado.web.Application(
+            [(r'/(.*)', self.handler, {'path': self.root})])
+        self.application.listen(self.port)
+        self.ioloop.start()
+
+    @property
+    def listening(self):
+        if self.port is None:
+            return False
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        return sock.connect_ex(('127.0.0.1', self.port)) == 0
+
+    def url(self, path):
+        '''
+        Convenience function which, given a file path, will return a URL that
+        points to that path. If the path is relative, it will just be appended
+        to self.web_root.
+        '''
+        if self.web_root is None:
+            raise RuntimeError('Webserver instance has not been started')
+        err_msg = 'invalid path, must be either a relative path or a path ' \
+                  'within {0}'.format(self.root)
+        try:
+            relpath = path \
+                if not os.path.isabs(path) \
+                else os.path.relpath(path, self.root)
+            if relpath.startswith('..' + os.sep):
+                raise ValueError(err_msg)
+            return '/'.join((self.web_root, relpath))
+        except AttributeError:
+            raise ValueError(err_msg)
+
+    def start(self):
+        '''
+        Starts the webserver
+        '''
+        if self.port is None:
+            self.port = get_unused_localhost_port()
+
+        self.web_root = 'http://127.0.0.1:{0}'.format(self.port)
+
+        self.server_thread = threading.Thread(target=self.target)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+        for idx in range(self.wait + 1):
+            if self.listening:
+                break
+            if idx != self.wait:
+                time.sleep(1)
+        else:
+            raise Exception(
+                'Failed to start tornado webserver on 127.0.0.1:{0} within '
+                '{1} seconds'.format(self.port, self.wait)
+            )
+
+    def stop(self):
+        '''
+        Stops the webserver
+        '''
+        self.ioloop.add_callback(self.ioloop.stop)
+        self.server_thread.join()