Merge branch '2017.7' into '2018.3'

Conflicts:
  - salt/modules/cmdmod.py
  - salt/modules/reg.py
  - salt/modules/win_lgpo.py
  - salt/modules/win_path.py
  - salt/modules/win_pkg.py
  - salt/pillar/file_tree.py
  - salt/states/boto3_route53.py
  - salt/states/reg.py
  - salt/utils/win_functions.py
  - tests/unit/modules/test_kubernetes.py
  - tests/unit/modules/test_win_path.py

doc/topics/development/contributing.rst

@@ -223,7 +223,7 @@ branches, and dot release branches.
 .. note::
 
     GitHub will open pull requests against Salt's main branch, ``develop``,
-    byndefault. Be sure to check which branch is selected when creating the
+    by default. Be sure to check which branch is selected when creating the
     pull request.
 
 The Develop Branch

doc/topics/development/conventions/formulas.rst

@@ -1258,7 +1258,7 @@ target platform, and any other installation or usage instructions or tips.
 
 A sample skeleton for the ``README.rst`` file:
 
-.. code-block:: rest
+.. code-block:: restructuredtext
 
     ===
     foo
@@ -1269,7 +1269,7 @@ A sample skeleton for the ``README.rst`` file:
     .. note::
 
         See the full `Salt Formulas installation and usage instructions
-        <http://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html>`_.
+        <https://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html>`_.
 
     Available states
     ================
@@ -1298,7 +1298,7 @@ A sample skeleton for the `CHANGELOG.rst` file:
 
 :file:`CHANGELOG.rst`:
 
-.. code-block:: rest
+.. code-block:: restructuredtext
 
     foo formula
     ===========

doc/topics/jinja/index.rst

@@ -1384,7 +1384,7 @@ Example:
 
 .. code-block:: jinja
 
-  {{ 'www.google.com' | dns_check }}
+  {{ 'www.google.com' | dns_check(port=443) }}
 
 Returns:
 

doc/topics/windows/windows-package-manager.rst

@@ -302,25 +302,30 @@ can define multiple versions for the same piece of software. The lines following
 the version are indented two more spaces and contain all the information needed
 to install that package.
 
-.. warning:: The package name and the ``full_name`` must be unique to all
+.. warning::
-    other packages in the software repository.
+    The package name and the ``full_name`` must be unique to all other packages
+    in the software repository.
 
 The version line is the version for the package to be installed. It is used when
 you need to install a specific version of a piece of software.
 
-.. warning:: The version must be enclosed in quotes, otherwise the yaml parser
+.. warning::
-    will remove trailing zeros.
+    The version must be enclosed in quotes, otherwise the yaml parser will
+    remove trailing zeros.
+
+.. note::
+    There are unique situations where previous versions are unavailable. Take
+    Google Chrome for example. There is only one url provided for a standalone
+    installation of Google Chrome.
 
-.. note:: There are unique situations where previous versions are unavailable.
-    Take Google Chrome for example. There is only one url provided for a
-    standalone installation of Google Chrome.
     (https://dl.google.com/edgedl/chrome/install/GoogleChromeStandaloneEnterprise.msi)
+
     When a new version is released, the url just points to the new version. To
     handle situations such as these, set the version to `latest`. Salt will
     install the version of Chrome at the URL and report that version. Here's an
     example:
 
-.. code-block:: bash
+.. code-block:: yaml
 
     chrome:
       latest:
@@ -335,200 +340,237 @@ you need to install a specific version of a piece of software.
 
 Available parameters are as follows:
 
-:param str full_name: The Full Name for the software as shown in "Programs and
+:param str full_name:
-    Features" in the control panel. You can also get this information by
+    The Full Name for the software as shown in "Programs and Features" in the
-    installing the package manually and then running ``pkg.list_pkgs``. Here's
+    control panel. You can also get this information by installing the package
-    an example of the output from ``pkg.list_pkgs``:
+    manually and then running ``pkg.list_pkgs``. Here's an example of the output
+    from ``pkg.list_pkgs``:
 
-.. code-block:: bash
+    .. code-block:: bash
 
-    salt 'test-2008' pkg.list_pkgs
+        salt 'test-2008' pkg.list_pkgs
-    test-2008
+        test-2008
-        ----------
+            ----------
-        7-Zip 9.20 (x64 edition):
+            7-Zip 9.20 (x64 edition):
-            9.20.00.0
+                9.20.00.0
-        Microsoft .NET Framework 4 Client Profile:
+            Microsoft .NET Framework 4 Client Profile:
-            4.0.30319,4.0.30319
+                4.0.30319,4.0.30319
-        Microsoft .NET Framework 4 Extended:
+            Microsoft .NET Framework 4 Extended:
-            4.0.30319,4.0.30319
+                4.0.30319,4.0.30319
-        Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022:
+            Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022:
-            9.0.21022
+                9.0.21022
-        Mozilla Firefox 17.0.1 (x86 en-US):
+            Mozilla Firefox 17.0.1 (x86 en-US):
-            17.0.1
+                17.0.1
-        Mozilla Maintenance Service:
+            Mozilla Maintenance Service:
-            17.0.1
+                17.0.1
-        NSClient++ (x64):
+            NSClient++ (x64):
-            0.3.8.76
+                0.3.8.76
-        Notepad++:
+            Notepad++:
-            6.4.2
+                6.4.2
-        Salt Minion 0.16.0:
+            Salt Minion 0.16.0:
-            0.16.0
+                0.16.0
 
-Notice the Full Name for Firefox: Mozilla Firefox 17.0.0 (x86 en-US). That's
+    Notice the Full Name for Firefox: ``Mozilla Firefox 17.0.0 (x86 en-US)``.
-exactly what's in the ``full_name`` parameter in the software definition file.
+    That's exactly what's in the ``full_name`` parameter in the software
+    definition file.
 
-If any of the software insalled on the machine matches one of the software
+    If any of the software installed on the machine matches one of the software
-definition files in the repository the full_name will be automatically renamed
+    definition files in the repository, the full_name will be automatically
-to the package name. The example below shows the ``pkg.list_pkgs`` for a
+    renamed to the package name. The example below shows the ``pkg.list_pkgs``
-machine that already has Mozilla Firefox 17.0.1 installed.
+    for a machine that already has Mozilla Firefox 17.0.1 installed.
 
-.. code-block:: bash
+    .. code-block:: bash
+
+        test-2008:
+            ----------
+            7zip:
+                9.20.00.0
+            Microsoft .NET Framework 4 Client Profile:
+                4.0.30319,4.0.30319
+            Microsoft .NET Framework 4 Extended:
+                4.0.30319,4.0.30319
+            Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022:
+                9.0.21022
+            Mozilla Maintenance Service:
+                17.0.1
+            Notepad++:
+                6.4.2
+            Salt Minion 0.16.0:
+                0.16.0
+            firefox:
+                17.0.1
+            nsclient:
+                0.3.9.328
+
+    .. important::
+        The version number and ``full_name`` need to match the output from
+        ``pkg.list_pkgs`` so that the status can be verified when running a
+        highstate.
+
+    .. note::
+        It is still possible to successfully install packages using
+        ``pkg.install``, even if the ``full_name`` or the version number don't
+        match. However, this can make troubleshooting issues difficult, so be
+        careful.
+
+    .. tip::
+        To force salt to display the full name when there's already an existing
+        package definition file on the system, you can pass a bogus ``saltenv``
+        parameter to the command like so: ``pkg.list_pkgs saltenv=NotARealEnv``
+
+:param str installer:
+    The path to the ``.exe`` or ``.msi`` to use to install the package. This can
+    be a path or a URL. If it is a URL or a salt path (``salt://``), the package
+    will be cached locally and then executed. If it is a path to a file on disk
+    or a file share, it will be executed directly.
+
+    .. note::
+        If storing software in the same location as the winrepo it is best
+        practice to place each installer in its own directory rather than the
+        root of winrepo. Then you can place your package definition file in the
+        same directory. It is best practice to name the file ``init.sls``. This
+        will be picked up by ``pkg.refresh_db`` and processed properly.
+
+:param str install_flags:
+    Any flags that need to be passed to the installer to make it perform a
+    silent install. These can often be found by adding ``/?`` or ``/h`` when
+    running the installer from the command-line. A great resource for finding
+    these silent install flags can be found on the WPKG project's wiki_:
+
+    .. warning::
+        Salt will not return if the installer is waiting for user input so it is
+        imperative that the software package being installed has the ability to
+        install silently.
+
+:param str uninstaller:
+    The path to the program used to uninstall this software. This can be the
+    path to the same `exe` or `msi` used to install the software. It can also be
+    a GUID. You can find this value in the registry under the following keys:
+
+        - Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall
+        - Software\\Wow6432None\\Microsoft\\Windows\\CurrentVersion\\Uninstall
+
+:param str uninstall_flags:
+    Any flags that need to be passed to the uninstaller to make it perform a
+    silent uninstall. These can often be found by adding ``/?`` or ``/h`` when
+    running the uninstaller from the command-line. A great resource for finding
+    these silent install flags can be found on the WPKG project's wiki_:
+
+    .. warning::
+        Salt will not return if the uninstaller is waiting for user input so it
+        is imperative that the software package being uninstalled has the
+        ability to uninstall silently.
+
+    Here are some examples of installer and uninstaller settings:
+
+    .. code-block:: yaml
 
-    test-2008:
-        ----------
         7zip:
-            9.20.00.0
+          '9.20.00.0':
-        Microsoft .NET Framework 4 Client Profile:
+            installer: salt://win/repo/7zip/7z920-x64.msi
-            4.0.30319,4.0.30319
+            full_name: 7-Zip 9.20 (x64 edition)
-        Microsoft .NET Framework 4 Extended:
+            reboot: False
-            4.0.30319,4.0.30319
+            install_flags: '/qn /norestart'
-        Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022:
+            msiexec: True
-            9.0.21022
+            uninstaller: '{23170F69-40C1-2702-0920-000001000000}'
-        Mozilla Maintenance Service:
+            uninstall_flags: '/qn /norestart'
-            17.0.1
-        Notepad++:
-            6.4.2
-        Salt Minion 0.16.0:
-            0.16.0
-        firefox:
-            17.0.1
-        nsclient:
-            0.3.9.328
 
-.. important:: The version number and ``full_name`` need to match the output
+    Alternatively the ``uninstaller`` can also simply repeat the URL of an msi
-    from ``pkg.list_pkgs`` so that the status can be verified when running
+    file:
-    highstate.
 
-.. note:: It is still possible to successfully install packages using
+    .. code-block:: yaml
-    ``pkg.install`` even if they don't match. This can make troubleshooting
-    difficult so be careful.
 
-:param str installer: The path to the ``.exe`` or ``.msi`` to use to install the
+        7zip:
-    package. This can be a path or a URL. If it is a URL or a salt path
+          '9.20.00.0':
-    (salt://), the package will be cached locally and then executed. If it is a
+            installer: salt://win/repo/7zip/7z920-x64.msi
-    path to a file on disk or a file share, it will be executed directly.
+            full_name: 7-Zip 9.20 (x64 edition)
+            reboot: False
+            install_flags: '/qn /norestart'
+            msiexec: True
+            uninstaller: salt://win/repo/7zip/7z920-x64.msi
+            uninstall_flags: '/qn /norestart'
 
-:param str install_flags: Any flags that need to be passed to the installer to
+:param msiexec:
-    make it perform a silent install. These can often be found by adding ``/?``
+    This tells salt to use ``msiexec /i`` to install the package and
-    or ``/h`` when running the installer from the command-line. A great resource
+    ``msiexec /x`` to uninstall. This is for ``.msi`` installations. Possible
-    for finding these silent install flags can be found on the WPKG project's wiki_:
+    options are: True, False or the path to ``msiexec.exe`` on your system
 
-Salt will not return if the installer is waiting for user input so these are
+    .. code-block:: yaml
-important.
 
-:param str uninstaller: The path to the program used to uninstall this software.
+        7zip:
-    This can be the path to the same `exe` or `msi` used to install the
+          '9.20.00.0':
-    software. It can also be a GUID. You can find this value in the registry
+            installer: salt://win/repo/7zip/7z920-x64.msi
-    under the following keys:
+            full_name: 7-Zip 9.20 (x64 edition)
+            reboot: False
+            install_flags: '/qn /norestart'
+            msiexec: 'C:\Windows\System32\msiexec.exe'
+            uninstaller: salt://win/repo/7zip/7z920-x64.msi
+            uninstall_flags: '/qn /norestart'
 
-    - Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall
+:param bool allusers:
-    - Software\\Wow6432None\\Microsoft\\Windows\\CurrentVersion\\Uninstall
+    This parameter is specific to ``.msi`` installations. It tells ``msiexec``
+    to install the software for all users. The default is ``True``.
 
-:param str uninstall_flags: Any flags that need to be passed to the uninstaller
+:param bool cache_dir:
-    to make it perform a silent uninstall. These can often be found by adding
+    If ``True`` and the installer URL begins with ``salt://``, the entire
-    ``/?`` or ``/h`` when running the uninstaller from the command-line. A great
+    directory where the installer resides will be recursively cached. This is
-    resource for finding these silent install flags can be found on the WPKG
+    useful for installers that depend on other files in the same directory for
-    project's wiki_:
+    installation.
 
-Salt will not return if the uninstaller is waiting for user input so these are
+    .. warning::
-important.
+        Be aware that all files and directories in the same location as the
-
+        installer file will be copied down to the minion. If you place your
-Here are some examples of installer and uninstaller settings:
+        installer file in the root of winrepo (``/srv/salt/win/repo-ng``) and
-
+        ``cache_dir: True`` the entire contents of winrepo will be cached to
-.. code-block:: yaml
+        the minion. Therefore, it is best practice to place your installer files
-
+        in a subdirectory if they are to be stored in winrepo.
-    7zip:
-      '9.20.00.0':
-        installer: salt://win/repo/7zip/7z920-x64.msi
-        full_name: 7-Zip 9.20 (x64 edition)
-        reboot: False
-        install_flags: '/qn /norestart'
-        msiexec: True
-        uninstaller: '{23170F69-40C1-2702-0920-000001000000}'
-        uninstall_flags: '/qn /norestart'
-
-Alternatively the ``uninstaller`` can also simply repeat the URL of the msi file.
-
-.. code-block:: yaml
-
-    7zip:
-      '9.20.00.0':
-        installer: salt://win/repo/7zip/7z920-x64.msi
-        full_name: 7-Zip 9.20 (x64 edition)
-        reboot: False
-        install_flags: '/qn /norestart'
-        msiexec: True
-        uninstaller: salt://win/repo/7zip/7z920-x64.msi
-        uninstall_flags: '/qn /norestart'
-
-:param msiexec: This tells salt to use ``msiexec /i`` to install the
-    package and ``msiexec /x`` to uninstall. This is for `.msi` installations.
-    Possible options are: True, False or path to msiexec on your system
-
-    7zip:
-      '9.20.00.0':
-        installer: salt://win/repo/7zip/7z920-x64.msi
-        full_name: 7-Zip 9.20 (x64 edition)
-        reboot: False
-        install_flags: '/qn /norestart'
-        msiexec: 'C:\Windows\System32\msiexec.exe'
-        uninstaller: salt://win/repo/7zip/7z920-x64.msi
-        uninstall_flags: '/qn /norestart'
-
-:param str arch: This selects which ``msiexec.exe`` to use. Possible values:
-    ``x86``, ``x64``
-
-:param bool allusers: This parameter is specific to `.msi` installations. It
-    tells `msiexec` to install the software for all users. The default is True.
-
-:param bool cache_dir: If true when installer URL begins with salt://, the
-    entire directory where the installer resides will be recursively cached.
-    This is useful for installers that depend on other files in the same
-    directory for installation.
 
 :param str cache_file:
-    When installer URL begins with salt://, this indicates single file to copy
+    When the installer URL begins with ``salt://``, this indicates a single file
-    down for use with the installer. Copied to the same location as the
+    to copy down for use with the installer. It is copied to the same location
-    installer. Use this over ``cache_dir`` if there are many files in the
+    as the installer. Use this over ``cache_dir`` if there are many files in the
     directory and you only need a specific file and don't want to cache
     additional files that may reside in the installer directory.
 
-Here's an example for a software package that has dependent files:
+    Here's an example for a software package that has dependent files:
 
-.. code-block:: yaml
+    .. code-block:: yaml
 
-    sqlexpress:
+        sqlexpress:
-      '12.0.2000.8':
+          '12.0.2000.8':
-        installer: 'salt://win/repo/sqlexpress/setup.exe'
+            installer: 'salt://win/repo/sqlexpress/setup.exe'
-        full_name: Microsoft SQL Server 2014 Setup (English)
+            full_name: Microsoft SQL Server 2014 Setup (English)
-        reboot: False
+            reboot: False
-        install_flags: '/ACTION=install /IACCEPTSQLSERVERLICENSETERMS /Q'
+            install_flags: '/ACTION=install /IACCEPTSQLSERVERLICENSETERMS /Q'
-        cache_dir: True
+            cache_dir: True
 
-:param bool use_scheduler: If true, windows will use the task scheduler to run
+:param bool use_scheduler:
-    the installation. This is useful for running the salt installation itself as
+    If ``True``, Windows will use the task scheduler to run the installation.
-    the installation process kills any currently running instances of salt.
+    This is useful for running the Salt installation itself as the installation
+    process kills any currently running instances of Salt.
 
-:param str source_hash: This tells salt to compare a hash sum of the installer
+:param str source_hash:
-to the provided hash sum before execution. The value can be formatted as
+    This tells Salt to compare a hash sum of the installer to the provided hash
-``hash_algorithm=hash_sum``, or it can be a URI to a file containing the hash
+    sum before execution. The value can be formatted as
-sum.
+    ``<hash_algorithm>=<hash_sum>``, or it can be a URI to a file containing the
-For a list of supported algorithms, see the `hashlib documentation
+    hash sum.
-<https://docs.python.org/2/library/hashlib.html>`_.
 
-Here's an example of source_hash usage:
+    For a list of supported algorithms, see the `hashlib documentation
+    <https://docs.python.org/2/library/hashlib.html>`_.
 
-.. code-block:: yaml
+    Here's an example of source_hash usage:
 
-    messageanalyzer:
+    .. code-block:: yaml
-      '4.0.7551.0':
+
-        full_name: 'Microsoft Message Analyzer'
+        messageanalyzer:
-        installer: 'salt://win/repo/messageanalyzer/MessageAnalyzer64.msi'
+          '4.0.7551.0':
-        install_flags: '/quiet /norestart'
+            full_name: 'Microsoft Message Analyzer'
-        uninstaller: '{1CC02C23-8FCD-487E-860C-311EC0A0C933}'
+            installer: 'salt://win/repo/messageanalyzer/MessageAnalyzer64.msi'
-        uninstall_flags: '/quiet /norestart'
+            install_flags: '/quiet /norestart'
-        msiexec: True
+            uninstaller: '{1CC02C23-8FCD-487E-860C-311EC0A0C933}'
-        source_hash: 'sha1=62875ff451f13b10a8ff988f2943e76a4735d3d4'
+            uninstall_flags: '/quiet /norestart'
+            msiexec: True
+            source_hash: 'sha1=62875ff451f13b10a8ff988f2943e76a4735d3d4'
 
 :param bool reboot: Not implemented
 

requirements/base.txt

@@ -3,6 +3,6 @@ msgpack-python>0.3
 PyYAML
 MarkupSafe
 requests>=1.0.0
-tornado>=4.2.1
+tornado>=4.2.1,<5.0
 # Required by Tornado to handle threads stuff.
 futures>=2.0

salt/cloud/clouds/digitalocean.py

@@ -161,7 +161,7 @@ def avail_sizes(call=None):
             '-f or --function, or with the --list-sizes option'
         )
 
-    items = query(method='sizes')
+    items = query(method='sizes', command='?per_page=100')
     ret = {}
     for size in items['sizes']:
         ret[size['slug']] = {}

salt/modules/boto_vpc.py

@@ -2586,6 +2586,7 @@ def describe_route_tables(route_table_id=None, route_table_name=None,
                       'instance_id': 'Instance',
                       'interface_id': 'NetworkInterfaceId',
                       'nat_gateway_id': 'NatGatewayId',
+                      'vpc_peering_connection_id': 'VpcPeeringConnectionId',
                       }
         assoc_keys = {'id': 'RouteTableAssociationId',
                       'main': 'Main',

salt/modules/cmdmod.py

@@ -285,6 +285,7 @@ def _run(cmd,
             shell
         )
 
+    output_loglevel = _check_loglevel(output_loglevel)
     log_callback = _check_cb(log_callback)
 
     if runas is None and '__context__' in globals():
@@ -312,6 +313,10 @@ def _run(cmd,
         # yaml-ified into non-string types
         cwd = six.text_type(cwd)
 
+    if bg:
+        ignore_retcode = True
+        use_vt = False
+
     if not salt.utils.platform.is_windows():
         if not os.path.isfile(shell) or not os.access(shell, os.X_OK):
             msg = 'The shell {0} is not available'.format(shell)
@@ -368,7 +373,7 @@ def _run(cmd,
         else:
             return cmd
 
-    if _check_loglevel(output_loglevel) is not None:
+    if output_loglevel is not None:
         # Always log the shell commands at INFO unless quiet logging is
         # requested. The command output is what will be controlled by the
         # 'loglevel' parameter.
@@ -442,6 +447,10 @@ def _run(cmd,
                 for k, v in six.iteritems(env_runas)
             )
             env_runas.update(env)
+            # Fix platforms like Solaris that don't set a USER env var in the
+            # user's default environment as obtained above.
+            if env_runas.get('USER') != runas:
+                env_runas['USER'] = runas
             env = env_runas
         except ValueError:
             raise CommandExecutionError(
@@ -542,7 +551,7 @@ def _run(cmd,
             msg = (
                 'Unable to run command \'{0}\' with the context \'{1}\', '
                 'reason: '.format(
-                    cmd if _check_loglevel(output_loglevel) is not None
+                    cmd if output_loglevel is not None
                         else 'REDACTED',
                     kwargs
                 )
@@ -621,7 +630,7 @@ def _run(cmd,
         to = ''
         if timeout:
             to = ' (timeout: {0}s)'.format(timeout)
-        if _check_loglevel(output_loglevel) is not None:
+        if output_loglevel is not None:
             msg = 'Running {0} in VT{1}'.format(cmd, to)
             log.debug(log_callback(msg))
         stdout, stderr = '', ''
@@ -694,6 +703,26 @@ def _run(cmd,
     except NameError:
         # Ignore the context error during grain generation
         pass
+
+    # Log the output
+    if output_loglevel is not None:
+        if not ignore_retcode and ret['retcode'] != 0:
+            if output_loglevel < LOG_LEVELS['error']:
+                output_loglevel = LOG_LEVELS['error']
+            msg = (
+                'Command \'{0}\' failed with return code: {1}'.format(
+                    cmd,
+                    ret['retcode']
+                )
+            )
+            log.error(log_callback(msg))
+        if ret['stdout']:
+            log.log(output_loglevel, 'stdout: {0}'.format(log_callback(ret['stdout'])))
+        if ret['stderr']:
+            log.log(output_loglevel, 'stderr: {0}'.format(log_callback(ret['stderr'])))
+        if ret['retcode']:
+            log.log(output_loglevel, 'retcode: {0}'.format(ret['retcode']))
+
     return ret
 
 
@@ -3578,7 +3607,6 @@ def run_bg(cmd,
         output_loglevel='debug',
         log_callback=None,
         reset_system_locale=True,
-        ignore_retcode=False,
         saltenv='base',
         password=None,
         prepend_path=None,
@@ -3743,7 +3771,6 @@ def run_bg(cmd,
                log_callback=log_callback,
                timeout=timeout,
                reset_system_locale=reset_system_locale,
-               ignore_retcode=ignore_retcode,
                saltenv=saltenv,
                password=password,
                **kwargs

salt/modules/reg.py

@@ -34,7 +34,6 @@ from salt.ext.six.moves import range  # pylint: disable=W0622,import-error
 
 # Import third party libs
 try:
-    import win32gui
     import win32api
     import win32con
     import pywintypes
@@ -45,6 +44,7 @@ except ImportError:
 # Import Salt libs
 import salt.utils.platform
 import salt.utils.stringutils
+import salt.utils.win_functions
 from salt.exceptions import CommandExecutionError
 
 PY2 = sys.version_info[0] == 2
@@ -65,7 +65,7 @@ def __virtual__():
     if not HAS_WINDOWS_MODULES:
         return (False, 'reg execution module failed to load: '
                        'One of the following libraries did not load: '
-                       + 'win32gui, win32con, win32api')
+                       'win32con, win32api, pywintypes')
 
     return __virtualname__
 
@@ -190,11 +190,7 @@ def broadcast_change():
 
         salt '*' reg.broadcast_change
     '''
-    # https://msdn.microsoft.com/en-us/library/windows/desktop/ms644952(v=vs.85).aspx
+    return salt.utils.win_functions.broadcast_setting_change('Environment')
-    _, res = win32gui.SendMessageTimeout(
-        win32con.HWND_BROADCAST, win32con.WM_SETTINGCHANGE, 0, 0,
-        win32con.SMTO_ABORTIFHUNG, 5000)
-    return not bool(res)
 
 
 def list_keys(hive, key=None, use_32bit_registry=False):

salt/modules/win_lgpo.py

@@ -2442,7 +2442,7 @@ class _policy_info(object):
                 elif ord(val) == 1:
                     return 'Enabled'
                 else:
-                    return 'Invalid Value'
+                    return 'Invalid Value: {0!r}'.format(val)  # pylint: disable=repr-flag-used-in-string
             else:
                 return 'Not Defined'
         except TypeError:
@@ -5066,7 +5066,10 @@ def get(policy_class=None, return_full_policy_names=True,
                     class_vals[policy_name] = __salt__['reg.read_value'](_pol['Registry']['Hive'],
                                                                          _pol['Registry']['Path'],
                                                                          _pol['Registry']['Value'])['vdata']
-                    log.debug('Value %s found for reg policy %s', class_vals[policy_name], policy_name)
+                    log.debug(
+                        'Value %r found for reg policy %s',
+                        class_vals[policy_name], policy_name
+                    )
                 elif 'Secedit' in _pol:
                     # get value from secedit
                     _ret, _val = _findOptionValueInSeceditFile(_pol['Secedit']['Option'])

salt/modules/win_path.py

@@ -18,12 +18,11 @@ import salt.utils.args
 import salt.utils.data
 import salt.utils.platform
 import salt.utils.stringutils
+import salt.utils.win_functions
 
 # Import 3rd-party libs
 from salt.ext.six.moves import map
 try:
-    from win32con import HWND_BROADCAST, WM_SETTINGCHANGE
-    from win32api import SendMessage
     HAS_WIN32 = True
 except ImportError:
     HAS_WIN32 = False
@@ -57,7 +56,14 @@ def _normalize_dir(string_):
 def rehash():
     '''
     Send a WM_SETTINGCHANGE Broadcast to Windows to refresh the Environment
-    variables
+    variables for new processes.
+
+    .. note::
+        This will only affect new processes that aren't launched by services. To
+        apply changes to the path to services, the host must be restarted. The
+        ``salt-minion``, if running as a service, will not see changes to the
+        environment until the system is restarted. See
+        `MSDN Documentation <https://support.microsoft.com/en-us/help/821761/changes-that-you-make-to-environment-variables-do-not-affect-services>`_
 
     CLI Example:
 
@@ -65,7 +71,7 @@ def rehash():
 
         salt '*' win_path.rehash
     '''
-    return bool(SendMessage(HWND_BROADCAST, WM_SETTINGCHANGE, 0, 'Environment'))
+    return salt.utils.win_functions.broadcast_setting_change('Environment')
 
 
 def get_path():

salt/modules/win_pkg.py

@@ -581,28 +581,77 @@ def _refresh_db_conditional(saltenv, **kwargs):
 
 
 def refresh_db(**kwargs):
-    '''
+    r'''
-    Fetches metadata files and calls :py:func:`pkg.genrepo
+    Generates the local software metadata database (`winrepo.p`) on the minion.
-    <salt.modules.win_pkg.genrepo>` to compile updated repository metadata.
+    The database is stored in a serialized format located by default at the
+    following location:
+
+    `C:\salt\var\cache\salt\minion\files\base\win\repo-ng\winrepo.p`
+
+    This module performs the following steps to generate the software metadata
+    database:
+
+    - Fetch the package definition files (.sls) from `winrepo_source_dir`
+      (default `salt://win/repo-ng`) and cache them in
+      `<cachedir>\files\<saltenv>\<winrepo_source_dir>`
+      (default: `C:\salt\var\cache\salt\minion\files\base\win\repo-ng`)
+    - Call :py:func:`pkg.genrepo <salt.modules.win_pkg.genrepo>` to parse the
+      package definition files and generate the repository metadata database
+      file (`winrepo.p`)
+    - Return the report received from
+      :py:func:`pkg.genrepo <salt.modules.win_pkg.genrepo>`
+
+    The default winrepo directory on the master is `/srv/salt/win/repo-ng`. All
+    files that end with `.sls` in this and all subdirectories will be used to
+    generate the repository metadata database (`winrepo.p`).
+
+    .. note::
+        - Hidden directories (directories beginning with '`.`', such as
+          '`.git`') will be ignored.
+
+    .. note::
+        There is no need to call `pkg.refresh_db` every time you work with the
+        pkg module. Automatic refresh will occur based on the following minion
+        configuration settings:
+            - `winrepo_cache_expire_min`
+            - `winrepo_cache_expire_max`
+        However, if the package definition files have changed, as would be the
+        case if you are developing a new package definition, this function
+        should be called to ensure the minion has the latest information about
+        packages available to it.
+
+    .. warning::
+        Directories and files fetched from <winrepo_source_dir>
+        (`/srv/salt/win/repo-ng`) will be processed in alphabetical order. If
+        two or more software definition files contain the same name, the last
+        one processed replaces all data from the files processed before it.
+
+    For more information see
+    :ref:`Windows Software Repository <windows-package-manager>`
 
     Kwargs:
 
         saltenv (str): Salt environment. Default: ``base``
 
         verbose (bool):
-            Return verbose data structure which includes 'success_list', a list
+            Return a verbose data structure which includes 'success_list', a
-            of all sls files and the package names contained within. Default
+            list of all sls files and the package names contained within.
-            'False'
+            Default is 'False'
 
         failhard (bool):
-            If ``True``, an error will be raised if any repo SLS files failed to
+            If ``True``, an error will be raised if any repo SLS files fails to
             process. If ``False``, no error will be raised, and a dictionary
             containing the full results will be returned.
 
     Returns:
         dict: A dictionary containing the results of the database refresh.
 
-    .. Warning::
+    .. note::
+        A result with a `total: 0` generally means that the files are in the
+        wrong location on the master. Try running the following command on the
+        minion: `salt-call -l debug pkg.refresh saltenv=base`
+
+    .. warning::
         When calling this command from a state using `module.run` be sure to
         pass `failhard: False`. Otherwise the state will report failure if it
         encounters a bad software definition file.
@@ -648,10 +697,12 @@ def refresh_db(**kwargs):
         )
 
     # Cache repo-ng locally
+    log.info('Fetching *.sls files from {0}'.format(repo_details.winrepo_source_dir))
     __salt__['cp.cache_dir'](
-        repo_details.winrepo_source_dir,
+        path=repo_details.winrepo_source_dir,
-        saltenv,
+        saltenv=saltenv,
-        include_pat='*.sls'
+        include_pat='*.sls',
+        exclude_pat=r'E@\/\..*?\/'  # Exclude all hidden directories (.git)
     )
 
     return genrepo(saltenv=saltenv, verbose=verbose, failhard=failhard)
@@ -754,6 +805,10 @@ def genrepo(**kwargs):
             to process. If ``False``, no error will be raised, and a dictionary
             containing the full results will be returned.
 
+    .. note::
+        - Hidden directories (directories beginning with '`.`', such as
+          '`.git`') will be ignored.
+
     Returns:
         dict: A dictionary of the results of the command
 
@@ -777,9 +832,16 @@ def genrepo(**kwargs):
     repo_details = _get_repo_details(saltenv)
 
     for root, _, files in salt.utils.path.os_walk(repo_details.local_dest, followlinks=False):
+
+        # Skip hidden directories (.git)
+        if re.search(r'[\\/]\..*', root):
+            log.debug('Skipping files in directory: {0}'.format(root))
+            continue
+
         short_path = os.path.relpath(root, repo_details.local_dest)
         if short_path == '.':
             short_path = ''
+
         for name in files:
             if name.endswith('.sls'):
                 total_files_processed += 1
@@ -1209,11 +1271,11 @@ def install(name=None, refresh=False, pkgs=None, **kwargs):
             # single files
             if cache_dir and installer.startswith('salt:'):
                 path, _ = os.path.split(installer)
-                __salt__['cp.cache_dir'](path,
+                __salt__['cp.cache_dir'](path=path,
-                                         saltenv,
+                                         saltenv=saltenv,
-                                         False,
+                                         include_empty=False,
-                                         None,
+                                         include_pat=None,
-                                         'E@init.sls$')
+                                         exclude_pat='E@init.sls$')
 
             # Check to see if the cache_file is cached... if passed
             if cache_file and cache_file.startswith('salt:'):

salt/modules/yumpkg.py

@@ -1070,8 +1070,8 @@ def refresh_db(**kwargs):
 
     options = _get_options(**kwargs)
 
-    clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache']
+    clean_cmd = [_yum(), '--quiet', '--assumeyes', 'clean', 'expire-cache']
-    update_cmd = [_yum(), '--quiet', 'check-update']
+    update_cmd = [_yum(), '--quiet', '--assumeyes', 'check-update']
 
     if __grains__.get('os_family') == 'RedHat' \
             and __grains__.get('osmajorrelease') == 7:
@@ -1525,8 +1525,8 @@ def install(name=None,
                             break
                     else:
                         if pkgname is not None:
-                            if re.match('kernel(-.+)?', pkgname):
+                            if re.match('^kernel(|-devel)$', pkgname):
-                                # kernel and its subpackages support multiple
+                                # kernel and kernel-devel support multiple
                                 # installs as their paths do not conflict.
                                 # Performing a yum/dnf downgrade will be a
                                 # no-op so just do an install instead. It will

salt/netapi/rest_tornado/saltnado.py

@@ -248,28 +248,6 @@ def _json_dumps(obj, **kwargs):
 #  - "wheel" (need async api...)
 
 
-class SaltClientsMixIn(object):
-    '''
-    MixIn class to container all of the salt clients that the API needs
-    '''
-    # TODO: load this proactively, instead of waiting for a request
-    __saltclients = None
-
-    @property
-    def saltclients(self):
-        if SaltClientsMixIn.__saltclients is None:
-            local_client = salt.client.get_local_client(mopts=self.application.opts)
-            # TODO: refreshing clients using cachedict
-            SaltClientsMixIn.__saltclients = {
-                'local': local_client.run_job_async,
-                # not the actual client we'll use.. but its what we'll use to get args
-                'local_async': local_client.run_job_async,
-                'runner': salt.runner.RunnerClient(opts=self.application.opts).cmd_async,
-                'runner_async': None,  # empty, since we use the same client as `runner`
-                }
-        return SaltClientsMixIn.__saltclients
-
-
 AUTH_TOKEN_HEADER = 'X-Auth-Token'
 AUTH_COOKIE_NAME = 'session_id'
 
@@ -400,7 +378,7 @@ class EventListener(object):
                         del self.timeout_map[future]
 
 
-class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn):  # pylint: disable=W0223
+class BaseSaltAPIHandler(tornado.web.RequestHandler):  # pylint: disable=W0223
     ct_out_map = (
         ('application/json', _json_dumps),
         ('application/x-yaml', salt.utils.yaml.safe_dump),
@@ -428,6 +406,16 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn):  # pylin
                 self.application.opts,
             )
 
+        if not hasattr(self, 'saltclients'):
+            local_client = salt.client.get_local_client(mopts=self.application.opts)
+            self.saltclients = {
+                'local': local_client.run_job_async,
+                # not the actual client we'll use.. but its what we'll use to get args
+                'local_async': local_client.run_job_async,
+                'runner': salt.runner.RunnerClient(opts=self.application.opts).cmd_async,
+                'runner_async': None,  # empty, since we use the same client as `runner`
+                }
+
     @property
     def token(self):
         '''
@@ -759,7 +747,7 @@ class SaltAuthHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
         self.write(self.serialize(ret))
 
 
-class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn):  # pylint: disable=W0223
+class SaltAPIHandler(BaseSaltAPIHandler):  # pylint: disable=W0223
     '''
     Main API handler for base "/"
     '''

salt/pillar/file_tree.py

@@ -1,182 +1,97 @@
 # -*- coding: utf-8 -*-
 '''
+The ``file_tree`` external pillar allows values from all files in a directory
+tree to be imported as Pillar data.
 
-``File_tree`` is an external pillar that allows
+.. note::
-values from all files in a directory tree to be imported as Pillar data.
 
-Note this is an external pillar, and is subject to the rules and constraints
+    This is an external pillar and is subject to the :ref:`rules and
-governing external pillars detailed here: :ref:`external-pillars`.
+    constraints <external-pillars>` governing external pillars.
 
 .. versionadded:: 2015.5.0
 
-Example Configuration
+In this pillar, data is organized by either Minion ID or Nodegroup name.  To
----------------------
+setup pillar data for a specific Minion, place it in
+``<root_dir>/hosts/<minion_id>``.  To setup pillar data for an entire
+Nodegroup, place it in ``<root_dir>/nodegroups/<node_group>`` where
+``<node_group>`` is the Nodegroup's name.
+
+Example ``file_tree`` Pillar
+============================
+
+Master Configuration
+--------------------
 
 .. code-block:: yaml
 
     ext_pillar:
       - file_tree:
-          root_dir: /path/to/root/directory
+          root_dir: /srv/ext_pillar
           follow_dir_links: False
           keep_newline: True
 
-The ``root_dir`` parameter is required and points to the directory where files
+    node_groups:
-for each host are stored. The ``follow_dir_links`` parameter is optional and
+      internal_servers: 'L@bob,stuart,kevin'
-defaults to False. If ``follow_dir_links`` is set to True, this external pillar
-will follow symbolic links to other directories.
 
-.. warning::
+Pillar Configuration
-    Be careful when using ``follow_dir_links``, as a recursive symlink chain
+--------------------
-    will result in unexpected results.
 
-.. versionchanged:: Oxygen
+.. code-block:: bash
-    If ``root_dir`` is a relative path, it will be treated as relative to the
-    :conf_master:`pillar_roots` of the environment specified by
-    :conf_minion:`pillarenv`. If an environment specifies multiple
-    roots, this module will search for files relative to all of them, in order,
-    merging the results.
 
-If ``keep_newline`` is set to ``True``, then the pillar values for files ending
+    (salt-master) # tree /srv/ext_pillar
-in newlines will keep that newline. The default behavior is to remove the
+    /srv/ext_pillar/
-end-of-file newline. ``keep_newline`` should be turned on if the pillar data is
+    |-- hosts
-intended to be used to deploy a file using ``contents_pillar`` with a
+    |   |-- bob
-:py:func:`file.managed <salt.states.file.managed>` state.
+    |   |   |-- apache
+    |   |   |   `-- config.d
+    |   |   |       |-- 00_important.conf
+    |   |   |       `-- 20_bob_extra.conf
+    |   |   `-- corporate_app
+    |   |       `-- settings
+    |   |           `-- bob_settings.cfg
+    |   `-- kevin
+    |       |-- apache
+    |       |   `-- config.d
+    |       |       `-- 00_important.conf
+    |       `-- corporate_app
+    |           `-- settings
+    |               `-- kevin_settings.cfg
+    `-- nodegroups
+        `-- internal_servers
+            `-- corporate_app
+                `-- settings
+                    `-- common_settings.cfg
 
-.. versionchanged:: 2015.8.4
+Verify Pillar Data
-    The ``raw_data`` parameter has been renamed to ``keep_newline``. In earlier
+------------------
-    releases, ``raw_data`` must be used. Also, this parameter can now be a list
-    of globs, allowing for more granular control over which pillar values keep
-    their end-of-file newline. The globs match paths relative to the
-    directories named for minion IDs and nodegroups underneath the ``root_dir``
-    (see the layout examples in the below sections).
 
-    .. code-block:: yaml
+.. code-block:: bash
 
-        ext_pillar:
+    (salt-master) # salt bob pillar.items
-          - file_tree:
+    bob:
-              root_dir: /path/to/root/directory
+        ----------
-              keep_newline:
+        apache:
-                - files/testdir/*
+            ----------
+            config.d:
+                ----------
+                00_important.conf:
+                    <important_config important_setting="yes" />
+                20_bob_extra.conf:
+                    <bob_specific_cfg has_freeze_ray="yes" />
+        corporate_app:
+            ----------
+            settings:
+                ----------
+                common_settings:
+                    // This is the main settings file for the corporate
+                    // internal web app
+                    main_setting: probably
+                bob_settings:
+                    role: bob
 
 .. note::
-    In earlier releases, this documentation incorrectly stated that binary
-    files would not affected by the ``keep_newline`` configuration.  However,
-    this module does not actually distinguish between binary and text files.
 
-.. versionchanged:: 2017.7.0
+    The leaf data in the example shown is the contents of the pillar files.
-    Templating/rendering has been added. You can now specify a default render
-    pipeline and a black- and whitelist of (dis)allowed renderers.
-
-    ``template`` must be set to ``True`` for templating to happen.
-
-    .. code-block:: yaml
-
-        ext_pillar:
-          - file_tree:
-            root_dir: /path/to/root/directory
-            render_default: jinja|yaml
-            renderer_blacklist:
-              - gpg
-            renderer_whitelist:
-              - jinja
-              - yaml
-            template: True
-
-Assigning Pillar Data to Individual Hosts
------------------------------------------
-
-To configure pillar data for each host, this external pillar will recursively
-iterate over ``root_dir``/hosts/``id`` (where ``id`` is a minion ID), and
-compile pillar data with each subdirectory as a dictionary key and each file
-as a value.
-
-For example, the following ``root_dir`` tree:
-
-.. code-block:: text
-
-    ./hosts/
-    ./hosts/test-host/
-    ./hosts/test-host/files/
-    ./hosts/test-host/files/testdir/
-    ./hosts/test-host/files/testdir/file1.txt
-    ./hosts/test-host/files/testdir/file2.txt
-    ./hosts/test-host/files/another-testdir/
-    ./hosts/test-host/files/another-testdir/symlink-to-file1.txt
-
-will result in the following pillar tree for minion with ID ``test-host``:
-
-.. code-block:: text
-
-    test-host:
-        ----------
-        files:
-            ----------
-            another-testdir:
-                ----------
-                symlink-to-file1.txt:
-                    Contents of file #1.
-
-            testdir:
-                ----------
-                file1.txt:
-                    Contents of file #1.
-
-                file2.txt:
-                    Contents of file #2.
-
-.. note::
-    Subdirectories underneath ``root_dir``/hosts/``id`` become nested
-    dictionaries, as shown above.
-
-
-Assigning Pillar Data to Entire Nodegroups
-------------------------------------------
-
-To assign Pillar data to all minions in a given nodegroup, this external pillar
-recursively iterates over ``root_dir``/nodegroups/``nodegroup`` (where
-``nodegroup`` is the name of a nodegroup), and like for individual hosts,
-compiles pillar data with each subdirectory as a dictionary key and each file
-as a value.
-
-.. important::
-    If the same Pillar key is set for a minion both by nodegroup and by
-    individual host, then the value set for the individual host will take
-    precedence.
-
-For example, the following ``root_dir`` tree:
-
-.. code-block:: text
-
-    ./nodegroups/
-    ./nodegroups/test-group/
-    ./nodegroups/test-group/files/
-    ./nodegroups/test-group/files/testdir/
-    ./nodegroups/test-group/files/testdir/file1.txt
-    ./nodegroups/test-group/files/testdir/file2.txt
-    ./nodegroups/test-group/files/another-testdir/
-    ./nodegroups/test-group/files/another-testdir/symlink-to-file1.txt
-
-will result in the following pillar data for minions in the node group
-``test-group``:
-
-.. code-block:: text
-
-    test-host:
-        ----------
-        files:
-            ----------
-            another-testdir:
-                ----------
-                symlink-to-file1.txt:
-                    Contents of file #1.
-
-            testdir:
-                ----------
-                file1.txt:
-                    Contents of file #1.
-
-                file2.txt:
-                    Contents of file #2.
 '''
 from __future__ import absolute_import, print_function, unicode_literals
 
@@ -311,7 +226,130 @@ def ext_pillar(minion_id,
                renderer_whitelist=None,
                template=False):
     '''
-    Compile pillar data for the specified minion ID
+    Compile pillar data from the given ``root_dir`` specific to Nodegroup names
+    and Minion IDs.
+
+    If a Minion's ID is not found at ``<root_dir>/host/<minion_id>`` or if it
+    is not included in any Nodegroups named at
+    ``<root_dir>/nodegroups/<node_group>``, no pillar data provided by this
+    pillar module will be available for that Minion.
+
+    .. versionchanged:: 2017.7.0
+        Templating/rendering has been added. You can now specify a default
+        render pipeline and a black- and whitelist of (dis)allowed renderers.
+
+        :param:`template` must be set to ``True`` for templating to happen.
+
+        .. code-block:: yaml
+
+            ext_pillar:
+              - file_tree:
+                root_dir: /path/to/root/directory
+                render_default: jinja|yaml
+                renderer_blacklist:
+                  - gpg
+                renderer_whitelist:
+                  - jinja
+                  - yaml
+                template: True
+
+    :param minion_id:
+        The ID of the Minion whose pillar data is to be collected
+
+    :param pillar:
+        Unused by the ``file_tree`` pillar module
+
+    :param root_dir:
+        Filesystem directory used as the root for pillar data (e.g.
+        ``/srv/ext_pillar``)
+
+        .. versionchanged:: Oxygen
+            If ``root_dir`` is a relative path, it will be treated as relative to the
+            :conf_master:`pillar_roots` of the environment specified by
+            :conf_minion:`pillarenv`. If an environment specifies multiple
+            roots, this module will search for files relative to all of them, in order,
+            merging the results.
+
+    :param follow_dir_links:
+        Follow symbolic links to directories while collecting pillar files.
+        Defaults to ``False``.
+
+        .. warning::
+
+            Care should be exercised when enabling this option as it will
+            follow links that point outside of :param:`root_dir`.
+
+        .. warning::
+
+            Symbolic links that lead to infinite recursion are not filtered.
+
+    :param debug:
+        Enable debug information at log level ``debug``.  Defaults to
+        ``False``.  This option may be useful to help debug errors when setting
+        up the ``file_tree`` pillar module.
+
+    :param keep_newline:
+        Preserve the end-of-file newline in files.  Defaults to ``False``.
+        This option may either be a boolean or a list of file globs (as defined
+        by the `Python fnmatch package
+        <https://docs.python.org/library/fnmatch.html>`_) for which end-of-file
+        newlines are to be kept.
+
+        ``keep_newline`` should be turned on if the pillar data is intended to
+        be used to deploy a file using ``contents_pillar`` with a
+        :py:func:`file.managed <salt.states.file.managed>` state.
+
+        .. versionchanged:: 2015.8.4
+            The ``raw_data`` parameter has been renamed to ``keep_newline``. In
+            earlier releases, ``raw_data`` must be used. Also, this parameter
+            can now be a list of globs, allowing for more granular control over
+            which pillar values keep their end-of-file newline. The globs match
+            paths relative to the directories named for Minion IDs and
+            Nodegroup namess underneath the :param:`root_dir`.
+
+            .. code-block:: yaml
+
+                ext_pillar:
+                  - file_tree:
+                      root_dir: /srv/ext_pillar
+                      keep_newline:
+                        - apache/config.d/*
+                        - corporate_app/settings/*
+
+        .. note::
+            In earlier releases, this documentation incorrectly stated that
+            binary files would not affected by the ``keep_newline``.  However,
+            this module does not actually distinguish between binary and text
+            files.
+
+
+    :param render_default:
+        Override Salt's :conf_master:`default global renderer <renderer>` for
+        the ``file_tree`` pillar.
+
+        .. code-block:: yaml
+
+            render_default: jinja
+
+    :param renderer_blacklist:
+        Disallow renderers for pillar files.
+
+        .. code-block:: yaml
+
+            renderer_blacklist:
+              - json
+
+    :param renderer_whitelist:
+        Allow renderers for pillar files.
+
+        .. code-block:: yaml
+
+            renderer_whitelist:
+              - yaml
+              - jinja
+
+    :param template:
+        Enable templating of pillar files.  Defaults to ``False``.
     '''
     # Not used
     del pillar
@@ -391,7 +429,7 @@ def _ext_pillar(minion_id,
 
     ngroup_pillar = {}
     nodegroups_dir = os.path.join(root_dir, 'nodegroups')
-    if os.path.exists(nodegroups_dir) and len(__opts__['nodegroups']) > 0:
+    if os.path.exists(nodegroups_dir) and len(__opts__.get('nodegroups', ())) > 0:
         master_ngroups = __opts__['nodegroups']
         ext_pillar_dirs = os.listdir(nodegroups_dir)
         if len(ext_pillar_dirs) > 0:
@@ -419,8 +457,8 @@ def _ext_pillar(minion_id,
         else:
             if debug is True:
                 log.debug(
-                    'file_tree: no nodegroups found in file tree directory '
+                    'file_tree: no nodegroups found in file tree directory %s, skipping...',
-                    'ext_pillar_dirs, skipping...'
+                    ext_pillar_dirs
                 )
     else:
         if debug is True:
@@ -428,7 +466,12 @@ def _ext_pillar(minion_id,
 
     host_dir = os.path.join(root_dir, 'hosts', minion_id)
     if not os.path.exists(host_dir):
-        # No data for host with this ID
+        if debug is True:
+            log.debug(
+                'file_tree: no pillar data for minion %s found in file tree directory %s',
+                minion_id,
+                host_dir
+            )
         return ngroup_pillar
 
     if not os.path.isdir(host_dir):

salt/states/boto3_route53.py

@@ -138,7 +138,7 @@ def _from_aws_encoding(string):  # XXX TODO
 
 
 def hosted_zone_present(name, Name=None, PrivateZone=False,
-                        CallerReference=None, Comment='', VPCs=None,
+                        CallerReference=None, Comment=None, VPCs=None,
                         region=None, key=None, keyid=None, profile=None):
     '''
     Ensure a hosted zone exists with the given attributes.
@@ -595,54 +595,60 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name
 
     # Convert any magic RR values to something AWS will understand, and otherwise clean them up.
     fixed_rrs = []
-    for rr in ResourceRecords:
+    if ResourceRecords:
-        if rr.startswith('magic:'):
+        for rr in ResourceRecords:
-            fields = rr.split(':')
+            if rr.startswith('magic:'):
-            if fields[1] == 'ec2_instance_tag':
+                fields = rr.split(':')
-                if len(fields) != 5:
+                if fields[1] == 'ec2_instance_tag':
-                    log.warning("Invalid magic RR value seen: '%s'.  Passing as-is.", rr)
+                    if len(fields) != 5:
-                    fixed_rrs += [rr]
+                        log.warning("Invalid magic RR value seen: '%s'.  Passing as-is.", rr)
-                    continue
+                        fixed_rrs += [rr]
-                tag_name = fields[2]
+                        continue
-                tag_value = fields[3]
+                    tag_name = fields[2]
-                instance_attr = fields[4]
+                    tag_value = fields[3]
-                good_states = ('pending', 'rebooting', 'running', 'stopping', 'stopped')
+                    instance_attr = fields[4]
-                r = __salt__['boto_ec2.find_instances'](
+                    good_states = ('pending', 'rebooting', 'running', 'stopping', 'stopped')
-                        tags={tag_name: tag_value}, return_objs=True, in_states=good_states,
+                    r = __salt__['boto_ec2.find_instances'](
-                        region=region, key=key, keyid=keyid, profile=profile)
+                            tags={tag_name: tag_value}, return_objs=True, in_states=good_states,
-                if len(r) < 1:
+                            region=region, key=key, keyid=keyid, profile=profile)
-                    ret['comment'] = 'No EC2 instance with tag {} == {} found'.format(tag_name,
+                    if len(r) < 1:
-                            tag_value)
+                        ret['comment'] = 'No EC2 instance with tag {} == {} found'.format(tag_name,
-                    log.error(ret['comment'])
+                                tag_value)
-                    ret['result'] = False
+                        log.error(ret['comment'])
-                    return ret
+                        ret['result'] = False
-                if len(r) > 1:
+                        return ret
-                    ret['comment'] = 'Multiple EC2 instances with tag {} == {} found'.format(
+                    if len(r) > 1:
-                            tag_name, tag_value)
+                        ret['comment'] = 'Multiple EC2 instances with tag {} == {} found'.format(
-                    log.error(ret['comment'])
+                                tag_name, tag_value)
-                    ret['result'] = False
+                        log.error(ret['comment'])
-                    return ret
+                        ret['result'] = False
-                instance = r[0]
+                        return ret
-                res = getattr(instance, instance_attr, None)
+                    instance = r[0]
-                if res:
+                    res = getattr(instance, instance_attr, None)
-                    log.debug('Found %s %s for instance %s', instance_attr, res, instance.id)
+                    if res:
-                    fixed_rrs += [_to_aws_encoding(res)]
+                        log.debug('Found %s %s for instance %s', instance_attr, res, instance.id)
+                        fixed_rrs += [_to_aws_encoding(res)]
+                    else:
+                        ret['comment'] = 'Attribute {} not found on instance {}'.format(instance_attr,
+                                instance.id)
+                        log.error(ret['comment'])
+                        ret['result'] = False
+                        return ret
                 else:
-                    ret['comment'] = 'Attribute {} not found on instance {}'.format(instance_attr,
+                    ret['comment'] = ('Unknown RR magic value seen: {}.  Please extend the '
-                            instance.id)
+                                      'boto3_route53 state module to add support for your preferred '
+                                      'incantation.'.format(fields[1]))
                     log.error(ret['comment'])
                     ret['result'] = False
                     return ret
             else:
-                ret['comment'] = ('Unknown RR magic value seen: {}.  Please extend the '
+                # for TXT records the entry must be encapsulated in quotes as required by the API
-                                  'boto3_route53 state module to add support for your preferred '
+                # this appears to be incredibly difficult with the jinja templating engine
-                                  'incantation.'.format(fields[1]))
+                # so inject the quotations here to make a viable ChangeBatch
-                log.error(ret['comment'])
+                if Type == 'TXT':
-                ret['result'] = False
+                    rr = '"{}"'.format(rr)
-                return ret
+                fixed_rrs += [rr]
-        else:
+        ResourceRecords = [{'Value': rr} for rr in sorted(fixed_rrs)]
-            fixed_rrs += [rr]
-    ResourceRecords = [{'Value': rr} for rr in sorted(fixed_rrs)]
 
     recordsets = __salt__['boto3_route53.get_resource_records'](HostedZoneId=HostedZoneId,
             StartRecordName=Name, StartRecordType=Type, region=region, key=key, keyid=keyid,
@@ -691,9 +697,10 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name
             return ret
         ResourceRecordSet = {
             'Name': Name,
-            'Type': Type,
+            'Type': Type
-            'ResourceRecords': ResourceRecords
         }
+        if ResourceRecords:
+            ResourceRecordSet['ResourceRecords'] = ResourceRecords
         for u in updatable:
             ResourceRecordSet.update({u: locals().get(u)}) if locals().get(u) else None
 

salt/states/reg.py

@@ -192,9 +192,14 @@ def present(name,
                                    salt.utils.stringutils.to_unicode(name, 'utf-8'))
         return ret
 
+    try:
+        vdata_decoded = salt.utils.to_unicode(vdata, 'utf-8')
+    except UnicodeDecodeError:
+        # vdata contains binary data that can't be decoded
+        vdata_decoded = vdata
     add_change = {'Key': r'{0}\{1}'.format(hive, key),
                   'Entry': '{0}'.format(salt.utils.stringutils.to_unicode(vname, 'utf-8') if vname else '(Default)'),
-                  'Value': salt.utils.stringutils.to_unicode(vdata, 'utf-8')}
+                  'Value': vdata_decoded}
 
     # Check for test option
     if __opts__['test']:

salt/transport/ipc.py

@@ -558,6 +558,11 @@ class IPCMessagePublisher(object):
                     io_loop=self.io_loop
                 )
             self.streams.add(stream)
+
+            def discard_after_closed():
+                self.streams.discard(stream)
+
+            stream.set_close_callback(discard_after_closed)
         except Exception as exc:
             log.error('IPC streaming error: %s', exc)
 

salt/transport/mixins/auth.py

@@ -420,6 +420,10 @@ class AESReqServerMixin(object):
                 log.debug('Host key change detected in open mode.')
                 with salt.utils.files.fopen(pubfn, 'w+') as fp_:
                     fp_.write(load['pub'])
+            elif not load['pub']:
+                log.error('Public key is empty: {0}'.format(load['id']))
+                return {'enc': 'clear',
+                        'load': {'ret': False}}
 
         pub = None
 

salt/utils/win_functions.py

@@ -6,6 +6,7 @@ missing functions in other modules
 from __future__ import absolute_import, print_function, unicode_literals
 import platform
 import re
+import ctypes
 
 # Import Salt Libs
 from salt.exceptions import CommandExecutionError
@@ -17,6 +18,7 @@ try:
     import win32api
     import win32net
     import win32security
+    from win32con import HWND_BROADCAST, WM_SETTINGCHANGE, SMTO_ABORTIFHUNG
     HAS_WIN32 = True
 except ImportError:
     HAS_WIN32 = False
@@ -219,3 +221,72 @@ def escape_for_cmd_exe(arg):
         return meta_map[char]
 
     return meta_re.sub(escape_meta_chars, arg)
+
+
+def broadcast_setting_change(message='Environment'):
+    '''
+    Send a WM_SETTINGCHANGE Broadcast to all Windows
+
+    Args:
+
+        message (str):
+            A string value representing the portion of the system that has been
+            updated and needs to be refreshed. Default is ``Environment``. These
+            are some common values:
+
+            - "Environment" : to effect a change in the environment variables
+            - "intl" : to effect a change in locale settings
+            - "Policy" : to effect a change in Group Policy Settings
+            - a leaf node in the registry
+            - the name of a section in the ``Win.ini`` file
+
+            See lParam within msdn docs for
+            `WM_SETTINGCHANGE <https://msdn.microsoft.com/en-us/library/ms725497%28VS.85%29.aspx>`_
+            for more information on Broadcasting Messages.
+
+            See GWL_WNDPROC within msdn docs for
+            `SetWindowLong <https://msdn.microsoft.com/en-us/library/windows/desktop/ms633591(v=vs.85).aspx>`_
+            for information on how to retrieve those messages.
+
+    .. note::
+        This will only affect new processes that aren't launched by services. To
+        apply changes to the path or registry to services, the host must be
+        restarted. The ``salt-minion``, if running as a service, will not see
+        changes to the environment until the system is restarted. Services
+        inherit their environment from ``services.exe`` which does not respond
+        to messaging events. See
+        `MSDN Documentation <https://support.microsoft.com/en-us/help/821761/changes-that-you-make-to-environment-variables-do-not-affect-services>`_
+        for more information.
+
+    CLI Example:
+
+    ... code-block:: python
+
+        import salt.utils.win_functions
+        salt.utils.win_functions.broadcast_setting_change('Environment')
+    '''
+    # Listen for messages sent by this would involve working with the
+    # SetWindowLong function. This can be accessed via win32gui or through
+    # ctypes. You can find examples on how to do this by searching for
+    # `Accessing WGL_WNDPROC` on the internet. Here are some examples of how
+    # this might work:
+    #
+    # # using win32gui
+    # import win32con
+    # import win32gui
+    # old_function = win32gui.SetWindowLong(window_handle, win32con.GWL_WNDPROC, new_function)
+    #
+    # # using ctypes
+    # import ctypes
+    # import win32con
+    # from ctypes import c_long, c_int
+    # user32 = ctypes.WinDLL('user32', use_last_error=True)
+    # WndProcType = ctypes.WINFUNCTYPE(c_int, c_long, c_int, c_int)
+    # new_function = WndProcType
+    # old_function = user32.SetWindowLongW(window_handle, win32con.GWL_WNDPROC, new_function)
+    broadcast_message = ctypes.create_unicode_buffer(message)
+    user32 = ctypes.WinDLL('user32', use_last_error=True)
+    result = user32.SendMessageTimeoutW(HWND_BROADCAST, WM_SETTINGCHANGE, 0,
+                                        broadcast_message, SMTO_ABORTIFHUNG,
+                                        5000, 0)
+    return result == 1

tests/unit/modules/test_kubernetes.py

@@ -18,12 +18,7 @@ from tests.support.mock import (
     NO_MOCK_REASON
 )
 
-try:
+from salt.modules import kubernetes
-    from salt.modules import kubernetes
-except ImportError:
-    kubernetes = False
-if not kubernetes.HAS_LIBS:
-    kubernetes = False
 
 
 @contextmanager
@@ -41,8 +36,8 @@ def mock_kubernetes_library():
 
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
-@skipIf(kubernetes is False, "Probably Kubernetes client lib is not installed. \
+@skipIf(not kubernetes.HAS_LIBS, "Kubernetes client lib is not installed. "
-                              Skipping test_kubernetes.py")
+                                 "Skipping test_kubernetes.py")
 class KubernetesTestCase(TestCase, LoaderModuleMockMixin):
     '''
     Test cases for salt.modules.kubernetes

tests/unit/modules/test_win_path.py

@@ -22,43 +22,13 @@ import salt.modules.win_path as win_path
 import salt.utils.stringutils
 
 
-class MockWin32API(object):
-    '''
-        Mock class for win32api
-    '''
-    def __init__(self):
-        pass
-
-    @staticmethod
-    def SendMessage(*args):
-        '''
-            Mock method for SendMessage
-        '''
-        return [args[0]]
-
-
-class MockWin32Con(object):
-    '''
-        Mock class for win32con
-    '''
-    HWND_BROADCAST = 1
-    WM_SETTINGCHANGE = 1
-
-    def __init__(self):
-        pass
-
-
 @skipIf(NO_MOCK, NO_MOCK_REASON)
 class WinPathTestCase(TestCase, LoaderModuleMockMixin):
     '''
         Test cases for salt.modules.win_path
     '''
     def setup_loader_modules(self):
-        return {win_path: {'win32api': MockWin32API,
+        return {win_path: {}}
-                           'win32con': MockWin32Con,
-                           'SendMessage': MagicMock,
-                           'HWND_BROADCAST': MagicMock,
-                           'WM_SETTINGCHANGE': MagicMock}}
 
     def __init__(self, *args, **kwargs):
         super(WinPathTestCase, self).__init__(*args, **kwargs)
@@ -79,11 +49,6 @@ class WinPathTestCase(TestCase, LoaderModuleMockMixin):
             salt.utils.stringutils.to_str(self.pathsep.join(new_path))
         )
 
-    def test_rehash(self):
-        '''
-        Test to rehash the Environment variables
-        '''
-        self.assertTrue(win_path.rehash())
 
     def test_get_path(self):
         '''

tests/unit/modules/test_yumpkg.py

@@ -473,11 +473,11 @@ class YumTestCase(TestCase, LoaderModuleMockMixin):
                     fromrepo='good',
                     branch='foo')
                 clean_cmd.assert_called_once_with(
-                    ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=*',
+                    ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=*',
                      '--enablerepo=good', '--branch=foo'],
                     python_shell=False)
                 update_cmd.assert_called_once_with(
-                    ['yum', '--quiet', 'check-update',
+                    ['yum', '--quiet', '--assumeyes', 'check-update',
                      '--setopt=autocheck_running_kernel=false', '--disablerepo=*',
                      '--enablerepo=good', '--branch=foo'],
                     output_loglevel='trace',
@@ -495,11 +495,11 @@ class YumTestCase(TestCase, LoaderModuleMockMixin):
                     disablerepo='bad',
                     branch='foo')
                 clean_cmd.assert_called_once_with(
-                    ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=bad',
+                    ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=bad',
                      '--enablerepo=good', '--branch=foo'],
                     python_shell=False)
                 update_cmd.assert_called_once_with(
-                    ['yum', '--quiet', 'check-update',
+                    ['yum', '--quiet', '--assumeyes', 'check-update',
                      '--setopt=autocheck_running_kernel=false', '--disablerepo=bad',
                      '--enablerepo=good', '--branch=foo'],
                     output_loglevel='trace',
@@ -516,7 +516,7 @@ class YumTestCase(TestCase, LoaderModuleMockMixin):
                     fromrepo='good',
                     branch='foo')
                 clean_cmd.assert_called_once_with(
-                    ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=*',
+                    ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=*',
                      '--enablerepo=good', '--branch=foo'],
                     python_shell=False)
 
@@ -529,7 +529,7 @@ class YumTestCase(TestCase, LoaderModuleMockMixin):
                     disablerepo='bad',
                     branch='foo')
                 clean_cmd.assert_called_once_with(
-                    ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=bad',
+                    ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=bad',
                      '--enablerepo=good', '--branch=foo'],
                     python_shell=False)
 

tests/unit/utils/test_win_functions.py

@@ -12,6 +12,7 @@ from tests.support.mock import (
 
 # Import Salt Libs
 import salt.utils.win_functions as win_functions
+import salt.utils
 
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
@@ -51,3 +52,10 @@ class WinFunctionsTestCase(TestCase):
         encoded = win_functions.escape_argument('C:\\Some Path\\With Spaces')
 
         self.assertEqual(encoded, '^"C:\\Some Path\\With Spaces^"')
+
+    @skipIf(not salt.utils.is_windows(), 'WinDLL only available on Windows')
+    def test_broadcast_setting_change(self):
+        '''
+            Test to rehash the Environment variables
+        '''
+        self.assertTrue(win_functions.broadcast_setting_change())