Merge branch 'develop' into ec2_image_name

.github/stale.yml

@@ -1,8 +1,8 @@
 # Probot Stale configuration file
 
 # Number of days of inactivity before an issue becomes stale
-# 760 is approximately 2 years and 1 month
-daysUntilStale: 760
+# 750 is approximately 2 years and 1 month
+daysUntilStale: 750
 
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7

.kitchen.yml

@@ -31,7 +31,7 @@ provisioner:
   salt_version: latest
   salt_bootstrap_url: https://bootstrap.saltstack.com
   salt_bootstrap_options: -X -p rsync stable <%= version %>
-  log_level: info
+  log_level: debug
   sudo: true
   require_chef: false
   retry_on_exit_code:
@@ -189,7 +189,6 @@ suites:
 verifier:
   name: runtests
   sudo: true
-  verbose: true
   run_destructive: true
   transport: zeromq
   types:

doc/conf.py

@@ -263,8 +263,8 @@ if on_saltstack:
     copyright = time.strftime("%Y")
 
 # < --- START do not merge these settings to other branches START ---> #
-build_type = 'develop'  # latest, previous, develop, next
-release = version  # version, latest_release, previous_release
+build_type = 'latest'  # latest, previous, develop, next
+release = latest_release  # version, latest_release, previous_release
 # < --- END do not merge these settings to other branches END ---> #
 
 # Set google custom search engine

doc/faq.rst

@@ -254,17 +254,19 @@ specifying the pillar variable is the same one used for :py:func:`pillar.get
     <salt.states.file.managed>` state is only supported in Salt 2015.8.4 and
     newer.
 
+.. _faq-restart-salt-minion:
+
 What is the best way to restart a Salt Minion daemon using Salt after upgrade?
 ------------------------------------------------------------------------------
 
 Updating the ``salt-minion`` package requires a restart of the ``salt-minion``
 service. But restarting the service while in the middle of a state run
 interrupts the process of the Minion running states and sending results back to
-the Master. A common way to workaround that is to schedule restarting of the
-Minion service using :ref:`masterless mode <masterless-quickstart>` after all
-other states have been applied. This allows the minion to keep Minion to Master
-connection alive for the Minion to report the final results to the Master, while
-the service is restarting in the background.
+the Master. A common way to workaround that is to schedule restarting the
+Minion service in the background by issuing a ``salt-call`` command calling
+``service.restart`` function. This prevents the Minion being disconnected from
+the Master immediately. Otherwise you would get
+``Minion did not return. [Not connected]`` message as the result of a state run.
 
 Upgrade without automatic restart
 *********************************
@@ -328,7 +330,7 @@ The following example works on UNIX-like operating systems:
     {%- if grains['os'] != 'Windows' %}
     Restart Salt Minion:
       cmd.run:
-        - name: 'salt-call --local service.restart salt-minion'
+        - name: 'salt-call service.restart salt-minion'
         - bg: True
         - onchanges:
           - pkg: Upgrade Salt Minion
@@ -348,9 +350,9 @@ as follows:
     Restart Salt Minion:
       cmd.run:
     {%- if grains['kernel'] == 'Windows' %}
-        - name: 'C:\salt\salt-call.bat --local service.restart salt-minion'
+        - name: 'C:\salt\salt-call.bat service.restart salt-minion'
     {%- else %}
-        - name: 'salt-call --local service.restart salt-minion'
+        - name: 'salt-call service.restart salt-minion'
     {%- endif %}
         - bg: True
         - onchanges:
@@ -358,7 +360,13 @@ as follows:
 
 However, it requires more advanced tricks to upgrade from legacy version of
 Salt (before ``2016.3.0``) on UNIX-like operating systems, where executing
-commands in the background is not supported:
+commands in the background is not supported. You also may need to schedule
+restarting the Minion service using :ref:`masterless mode
+<masterless-quickstart>` after all other states have been applied for Salt
+versions earlier than ``2016.11.0``. This allows the Minion to keep the
+connection to the Master alive for being able to report the final results back
+to the Master, while the service is restarting in the background. This state
+should run last or watch for the ``pkg`` state changes:
 
 .. code-block:: jinja
 
@@ -382,8 +390,8 @@ Restart the Minion from the command line:
 
 .. code-block:: bash
 
-    salt -G kernel:Windows cmd.run_bg 'C:\salt\salt-call.bat --local service.restart salt-minion'
-    salt -C 'not G@kernel:Windows' cmd.run_bg 'salt-call --local service.restart salt-minion'
+    salt -G kernel:Windows cmd.run_bg 'C:\salt\salt-call.bat service.restart salt-minion'
+    salt -C 'not G@kernel:Windows' cmd.run_bg 'salt-call service.restart salt-minion'
 
 Salting the Salt Master
 -----------------------
@@ -409,6 +417,10 @@ for salt itself:
 
 https://github.com/saltstack-formulas/salt-formula
 
+Restarting the ``salt-master`` service using execution module or application of
+state could be done the same way as for the Salt minion described :ref:`above
+<faq-restart-salt-minion>`.
+
 .. _faq-grain-security:
 
 Is Targeting using Grain Data Secure?
@@ -443,4 +455,3 @@ the grain and values that you want to change / set.)
 
 You should also `file an issue <https://github.com/saltstack/salt/issues>`_
 describing the change so it can be fixed in Salt.
-

doc/ref/clouds/all/index.rst

@@ -1,8 +1,8 @@
 .. _all-salt.clouds:
 
-===============================
-Full list of Salt Cloud modules
-===============================
+=============
+cloud modules
+=============
 
 .. currentmodule:: salt.cloud.clouds
 

doc/ref/configuration/master.rst

@@ -4515,6 +4515,25 @@ Recursively merge lists by aggregating them instead of replacing them.
 
     pillar_merge_lists: False
 
+.. conf_master:: pillar_includes_override_sls
+
+``pillar_includes_override_sls``
+********************************
+
+.. versionadded:: 2017.7.6,2018.3.1
+
+Default: ``False``
+
+Prior to version 2017.7.3, keys from :ref:`pillar includes <pillar-include>`
+would be merged on top of the pillar SLS. Since 2017.7.3, the includes are
+merged together and then the pillar SLS is merged on top of that.
+
+Set this option to ``True`` to return to the old behavior.
+
+.. code-block:: yaml
+
+    pillar_includes_override_sls: True
+
 .. _pillar-cache-opts:
 
 Pillar Cache Options

doc/ref/proxy/all/index.rst

@@ -12,6 +12,7 @@ proxy modules
 
     cimc
     chronos
+    cimc
     cisconso
     dummy
     esxi

doc/ref/proxy/all/salt.proxy.cimc.rst

@@ -1,6 +1,5 @@
-===================
-salt.proxy.cimc
-===================
+salt.proxy.cimc module
+======================
 
 .. automodule:: salt.proxy.cimc
     :members:

doc/ref/proxy/all/salt.proxy.panos.rst

@@ -1,5 +1,5 @@
 salt.proxy.panos module
-=========================
+=======================
 
 .. automodule:: salt.proxy.panos
     :members:

doc/topics/installation/rhel.rst

@@ -49,17 +49,13 @@ the SaltStack Repository.
 Installation from the Community-Maintained Repository
 =====================================================
 
-Beginning with version 0.9.4, Salt has been available in `EPEL`_. For
-RHEL/CentOS 5, `Fedora COPR`_ is a single community repository that provides
-Salt packages due to the removal from EPEL5.
+Beginning with version 0.9.4, Salt has been available in `EPEL`_.
 
 .. note::
-   Packages in these repositories are built by community, and it can
-   take a little while until the latest stable SaltStack release become
-   available.
+   Packages in this repository are built by community, and it can take a little
+   while until the latest stable SaltStack release become available.
 
 .. _`EPEL`: http://fedoraproject.org/wiki/EPEL
-.. _`Fedora COPR`: https://copr.fedorainfracloud.org/coprs/saltstack/salt-el5/
 
 RHEL/CentOS 6 and 7, Scientific Linux, etc.
 -------------------------------------------
@@ -146,26 +142,13 @@ ZeroMQ 4
 ========
 
 We recommend using ZeroMQ 4 where available. SaltStack provides ZeroMQ 4.0.5
-and pyzmq 14.5.0 in the :ref:`SaltStack Repository <installation-rhel-repo>`
-as well as a separate `zeromq4 COPR`_ repository.
-
-.. _`zeromq4 COPR`: http://copr.fedorainfracloud.org/coprs/saltstack/zeromq4/
+and ``pyzmq`` 14.5.0 in the :ref:`SaltStack Repository
+<installation-rhel-repo>`.
 
 If this repository is added *before* Salt is installed, then installing either
 ``salt-master`` or ``salt-minion`` will automatically pull in ZeroMQ 4.0.5, and
 additional steps to upgrade ZeroMQ and pyzmq are unnecessary.
 
-.. warning:: RHEL/CentOS 5 Users
-    Using COPR repos on RHEL/CentOS 5 requires that the ``python-hashlib``
-    package be installed. Not having it present will result in checksum errors
-    because YUM will not be able to process the SHA256 checksums used by COPR.
-
-.. note::
-    For RHEL/CentOS 5 installations, if using the SaltStack repo or Fedora COPR
-    to install Salt (as described :ref:`above <installation-rhel-repo>`),
-    then it is not necessary to enable the `zeromq4 COPR`_, because those
-    repositories already include ZeroMQ 4.
-
 Package Management
 ==================
 

doc/topics/pillar/index.rst

@@ -312,6 +312,8 @@ Since both pillar SLS files contained a ``bind`` key which contained a nested
 dictionary, the pillar dictionary's ``bind`` key contains the combined contents
 of both SLS files' ``bind`` keys.
 
+.. _pillar-include:
+
 Including Other Pillars
 =======================
 

doc/topics/releases/2017.7.6.rst

@@ -0,0 +1,15 @@
+===========================
+Salt 2017.7.6 Release Notes
+===========================
+
+Version 2017.7.6 is a bugfix release for :ref:`2017.7.0 <release-2017-7-0>`.
+
+Option to Return to Previous Pillar Include Behavior
+----------------------------------------------------
+
+Prior to version 2017.7.3, keys from :ref:`pillar includes <pillar-include>`
+would be merged on top of the pillar SLS. Since 2017.7.3, the includes are
+merged together and then the pillar SLS is merged on top of that.
+
+The :conf_master:`pillar_includes_override_sls` option has been added allow
+users to switch back to the pre-2017.7.3 behavior.

doc/topics/releases/2018.3.0.rst

@@ -30,7 +30,7 @@ The :py:func:`docker_network.present <salt.states.docker_network.present>`
 state has undergone a full rewrite, which includes the following improvements:
 
 Full API Support for Network Management
----------------------------------------
+=======================================
 
 The improvements made to input handling in the
 :py:func:`docker_container.running <salt.states.docker_container.running>`
@@ -39,14 +39,14 @@ state for 2017.7.0 have now been expanded to :py:func:`docker_network.present
 tunable configuration arguments.
 
 Custom Subnets
---------------
+==============
 
 Custom subnets can now be configured. Both IPv4 and mixed IPv4/IPv6 networks
 are supported. See :ref:`here <salt-states-docker-network-present-ipam>` for
 more information.
 
 Network Configuration in :py:func:`docker_container.running` States
--------------------------------------------------------------------
+===================================================================
 
 A long-requested feature has finally been added! It is now possible to
 configure static IPv4/IPv6 addresses, as well as links and labels. See
@@ -85,174 +85,6 @@ Additionally, the ``tag`` argument must now be explicitly passed to the
 :py:func:`docker_image.present <salt.states.docker_image.present>` state,
 unless the image is being pulled from a docker registry.
 
-``utils`` functions moved into separate modules
-===============================================
-
-The Salt utility functions from ``salt.utils`` have been moved into different
-modules, grouped logically based on their functionality. This change is
-backwards compatible, but the old imports will no longer be supported starting
-with release Neon.
-
-The functions have been moved as follows:
-
-- ``salt.utils.appendproctitle``: use ``salt.utils.process.appendproctitle``
-  instead.
-- ``salt.utils.daemonize``: use ``salt.utils.process.daemonize`` instead.
-- ``salt.utils.daemonize_if``: use ``salt.utils.process.daemonize_if`` instead.
-- ``salt.utils.reinit_crypto``: use ``salt.utils.crypt.reinit_crypto`` instead.
-- ``salt.utils.pem_finger``: use ``salt.utils.crypt.pem_finger`` instead.
-- ``salt.utils.to_bytes``: use ``salt.utils.stringutils.to_bytes`` instead.
-- ``salt.utils.to_str``: use ``salt.utils.stringutils.to_str`` instead.
-- ``salt.utils.to_unicode``: use ``salt.utils.stringutils.to_unicode`` instead.
-- ``salt.utils.str_to_num``: use ``salt.utils.stringutils.to_num`` instead.
-- ``salt.utils.is_quoted``: use ``salt.utils.stringutils.is_quoted`` instead.
-- ``salt.utils.dequote``: use ``salt.utils.stringutils.dequote`` instead.
-- ``salt.utils.is_hex``: use ``salt.utils.stringutils.is_hex`` instead.
-- ``salt.utils.is_bin_str``: use ``salt.utils.stringutils.is_bin_str`` instead.
-- ``salt.utils.rand_string``: use ``salt.utils.stringutils.random`` instead.
-- ``salt.utils.contains_whitespace``: use
-  ``salt.utils.stringutils.contains_whitespace`` instead.
-- ``salt.utils.build_whitespace_split_regex``: use
-  ``salt.utils.stringutils.build_whitespace_split_regex`` instead.
-- ``salt.utils.expr_match``: use ``salt.utils.stringutils.expr_match`` instead.
-- ``salt.utils.check_whitelist_blacklist``: use
-  ``salt.utils.stringutils.check_whitelist_blacklist`` instead.
-- ``salt.utils.check_include_exclude``: use
-  ``salt.utils.stringutils.check_include_exclude`` instead.
-- ``salt.utils.print_cli``: use ``salt.utils.stringutils.print_cli`` instead.
-- ``salt.utils.clean_kwargs``: use ``salt.utils.args.clean_kwargs`` instead.
-- ``salt.utils.invalid_kwargs``: use ``salt.utils.args.invalid_kwargs``
-  instead.
-- ``salt.utils.shlex_split``: use ``salt.utils.args.shlex_split`` instead.
-- ``salt.utils.arg_lookup``: use ``salt.utils.args.arg_lookup`` instead.
-- ``salt.utils.argspec_report``: use ``salt.utils.args.argspec_report``
-  instead.
-- ``salt.utils.split_input``: use ``salt.utils.args.split_input`` instead.
-- ``salt.utils.test_mode``: use ``salt.utils.args.test_mode`` instead.
-- ``salt.utils.format_call``: use ``salt.utils.args.format_call`` instead.
-- ``salt.utils.which``: use ``salt.utils.path.which`` instead.
-- ``salt.utils.which_bin``: use ``salt.utils.path.which_bin`` instead.
-- ``salt.utils.path_join``: use ``salt.utils.path.join`` instead.
-- ``salt.utils.check_or_die``: use ``salt.utils.path.check_or_die`` instead.
-- ``salt.utils.sanitize_win_path_string``: use
-  ``salt.utils.path.sanitize_win_path`` instead.
-- ``salt.utils.rand_str``: use ``salt.utils.hashutils.random_hash`` instead.
-- ``salt.utils.get_hash``: use ``salt.utils.hashutils.get_hash`` instead.
-- ``salt.utils.is_windows``: use ``salt.utils.platform.is_windows`` instead.
-- ``salt.utils.is_proxy``: use ``salt.utils.platform.is_proxy`` instead.
-- ``salt.utils.is_linux``: use ``salt.utils.platform.is_linux`` instead.
-- ``salt.utils.is_darwin``: use ``salt.utils.platform.is_darwin`` instead.
-- ``salt.utils.is_sunos``: use ``salt.utils.platform.is_sunos`` instead.
-- ``salt.utils.is_smartos``: use ``salt.utils.platform.is_smartos`` instead.
-- ``salt.utils.is_smartos_globalzone``: use
-  ``salt.utils.platform.is_smartos_globalzone`` instead.
-- ``salt.utils.is_smartos_zone``: use ``salt.utils.platform.is_smartos_zone``
-  instead.
-- ``salt.utils.is_freebsd``: use ``salt.utils.platform.is_freebsd`` instead.
-- ``salt.utils.is_netbsd``: use ``salt.utils.platform.is_netbsd`` instead.
-- ``salt.utils.is_openbsd``: use ``salt.utils.platform.is_openbsd`` instead.
-- ``salt.utils.is_aix``: use ``salt.utils.platform.is_aix`` instead.
-- ``salt.utils.safe_rm``: use ``salt.utils.files.safe_rm`` instead.
-- ``salt.utils.is_empty``: use ``salt.utils.files.is_empty`` instead.
-- ``salt.utils.fopen``: use ``salt.utils.files.fopen`` instead.
-- ``salt.utils.flopen``: use ``salt.utils.files.flopen`` instead.
-- ``salt.utils.fpopen``: use ``salt.utils.files.fpopen`` instead.
-- ``salt.utils.rm_rf``: use ``salt.utils.files.rm_rf`` instead.
-- ``salt.utils.mkstemp``: use ``salt.utils.files.mkstemp`` instead.
-- ``salt.utils.istextfile``: use ``salt.utils.files.is_text_file`` instead.
-- ``salt.utils.is_bin_file``: use ``salt.utils.files.is_binary`` instead.
-- ``salt.utils.list_files``: use ``salt.utils.files.list_files`` instead.
-- ``salt.utils.safe_walk``: use ``salt.utils.files.safe_walk`` instead.
-- ``salt.utils.st_mode_to_octal``: use ``salt.utils.files.st_mode_to_octal``
-  instead.
-- ``salt.utils.normalize_mode``: use ``salt.utils.files.normalize_mode``
-  instead.
-- ``salt.utils.human_size_to_bytes``: use
-  ``salt.utils.files.human_size_to_bytes`` instead.
-- ``salt.utils.backup_minion``: use ``salt.utils.files.backup_minion`` instead.
-- ``salt.utils.str_version_to_evr``: use ``salt.utils.pkg.rpm.version_to_evr``
-  instead.
-- ``salt.utils.parse_docstring``: use ``salt.utils.doc.parse_docstring``
-  instead.
-- ``salt.utils.compare_versions``: use ``salt.utils.versions.compare`` instead.
-- ``salt.utils.version_cmp``: use ``salt.utils.versions.version_cmp`` instead.
-- ``salt.utils.warn_until``: use ``salt.utils.versions.warn_until`` instead.
-- ``salt.utils.kwargs_warn_until``: use
-  ``salt.utils.versions.kwargs_warn_until`` instead.
-- ``salt.utils.get_color_theme``: use ``salt.utils.color.get_color_theme``
-  instead.
-- ``salt.utils.get_colors``: use ``salt.utils.color.get_colors`` instead.
-- ``salt.utils.gen_state_tag``: use ``salt.utils.state.gen_tag`` instead.
-- ``salt.utils.search_onfail_requisites``: use
-  ``salt.utils.state.search_onfail_requisites`` instead.
-- ``salt.utils.check_state_result``: use ``salt.utils.state.check_result``
-  instead.
-- ``salt.utils.get_user``: use ``salt.utils.user.get_user`` instead.
-- ``salt.utils.get_uid``: use ``salt.utils.user.get_uid`` instead.
-- ``salt.utils.get_specific_user``: use ``salt.utils.user.get_specific_user``
-  instead.
-- ``salt.utils.chugid``: use ``salt.utils.user.chugid`` instead.
-- ``salt.utils.chugid_and_umask``: use ``salt.utils.user.chugid_and_umask``
-  instead.
-- ``salt.utils.get_default_group``: use ``salt.utils.user.get_default_group``
-  instead.
-- ``salt.utils.get_group_list``: use ``salt.utils.user.get_group_list``
-  instead.
-- ``salt.utils.get_group_dict``: use ``salt.utils.user.get_group_dict``
-  instead.
-- ``salt.utils.get_gid_list``: use ``salt.utils.user.get_gid_list`` instead.
-- ``salt.utils.get_gid``: use ``salt.utils.user.get_gid`` instead.
-- ``salt.utils.enable_ctrl_logoff_handler``: use
-  ``salt.utils.win_functions.enable_ctrl_logoff_handler`` instead.
-- ``salt.utils.traverse_dict``: use ``salt.utils.data.traverse_dict`` instead.
-- ``salt.utils.traverse_dict_and_list``: use
-  ``salt.utils.data.traverse_dict_and_list`` instead.
-- ``salt.utils.filter_by``: use ``salt.utils.data.filter_by`` instead.
-- ``salt.utils.subdict_match``: use ``salt.utils.data.subdict_match`` instead.
-- ``salt.utils.substr_in_list``: use ``salt.utils.data.substr_in_list`` instead.
-- ``salt.utils.is_dictlist``: use ``salt.utils.data.is_dictlist``.
-- ``salt.utils.repack_dictlist``: use ``salt.utils.data.repack_dictlist``
-  instead.
-- ``salt.utils.compare_dicts``: use ``salt.utils.data.compare_dicts`` instead.
-- ``salt.utils.compare_lists``: use ``salt.utils.data.compare_lists`` instead.
-- ``salt.utils.decode_dict``: use ``salt.utils.data.encode_dict`` instead.
-- ``salt.utils.decode_list``: use ``salt.utils.data.encode_list`` instead.
-- ``salt.utils.exactly_n``: use ``salt.utils.data.exactly_n`` instead.
-- ``salt.utils.exactly_one``: use ``salt.utils.data.exactly_one`` instead.
-- ``salt.utils.is_list``: use ``salt.utils.data.is_list`` instead.
-- ``salt.utils.is_iter``: use ``salt.utils.data.is_iter`` instead.
-- ``salt.utils.isorted``: use ``salt.utils.data.sorted_ignorecase`` instead.
-- ``salt.utils.is_true``: use ``salt.utils.data.is_true`` instead.
-- ``salt.utils.mysql_to_dict``: use ``salt.utils.data.mysql_to_dict`` instead.
-- ``salt.utils.simple_types_filter``: use
-  ``salt.utils.data.simple_types_filter`` instead.
-- ``salt.utils.ip_bracket``: use ``salt.utils.zeromq.ip_bracket`` instead.
-- ``salt.utils.gen_mac``: use ``salt.utils.network.gen_mac`` instead.
-- ``salt.utils.mac_str_to_bytes``: use ``salt.utils.network.mac_str_to_bytes``
-  instead.
-- ``salt.utils.refresh_dns``: use ``salt.utils.network.refresh_dns`` instead.
-- ``salt.utils.dns_check``: use ``salt.utils.network.dns_check`` instead.
-- ``salt.utils.get_context``: use ``salt.utils.stringutils.get_context`` instead.
-- ``salt.utils.get_master_key``: use ``salt.utils.master.get_master_key``
-  instead.
-- ``salt.utils.get_values_of_matching_keys``: use
-  ``salt.utils.master.get_values_of_matching_keys`` instead.
-- ``salt.utils.date_cast``: use ``salt.utils.dateutils.date_cast`` instead.
-- ``salt.utils.date_format``: use ``salt.utils.dateutils.strftime`` instead.
-- ``salt.utils.total_seconds``: use ``salt.utils.dateutils.total_seconds``
-  instead.
-- ``salt.utils.find_json``: use ``salt.utils.json.find_json`` instead.
-- ``salt.utils.import_json``: use ``salt.utils.json.import_json`` instead.
-- ``salt.utils.namespaced_function``: use
-  ``salt.utils.functools.namespaced_function`` instead.
-- ``salt.utils.alias_function``: use ``salt.utils.functools.alias_function``
-  instead.
-- ``salt.utils.profile_func``: use ``salt.utils.profile.profile_func`` instead.
-- ``salt.utils.activate_profile``: use ``salt.utils.profile.activate_profile``
-  instead.
-- ``salt.utils.output_profile``: use ``salt.utils.profile.output_profile``
-  instead.
-
 State and Execution Module Support for ``docker run`` Functionality
 ===================================================================
 
@@ -456,16 +288,14 @@ programatically <orchestrate-runner-parsing-results-programatically>`. The
 failed returns in these cases are now included in the changes dictionary,
 making for much easier parsing.
 
-New Grains
-----------
-
-New core grains have been added to expose any storage inititator setting.
-
-The new grains added are:
-
-* ``fc_wwn``: Show all fibre channel world wide port names for a host
+Grains
+------
+* ``fc_wwn``: Show all fibre channel world wide port names for a host, must be enabled with `fibre_channel_grains`
 * ``iscsi_iqn``: Show the iSCSI IQN name for a host
 * ``swap_total``: Show the configured swap_total for Linux, \*BSD, OS X and Solaris/SunOS
+* ``virtual``:
+    * identifies reports KVM and VMM hypervisors when running an OpenBSD guest
+    * for detecting Solaris Logical Domains (LDOMs) running on T-Series SPARC hardware. The ``virtual_subtype`` grain is populated as a list of domain roles.
 
 Salt Minion Auto-discovery
 ------------------------
@@ -482,7 +312,7 @@ Configuration
 By default, automatic discovery is disabled.
 
 .. warning::
-    Due to the current limitations that will be changing in a future, before you turn on auto-discovery,
+    Due to the current limitations that will be changing in a future release, before you turn on auto-discovery,
     make sure your network is secured and trusted.
 
 Auto-discovery is configured on Master and Minion. Both of them are configured via the ``discovery`` option
@@ -589,12 +419,6 @@ This feature has a couple of _temporary_ limitations that are subject to change
   security applied (priv/pub key check, signature, fingerprint etc). That implies that administrator
   is expected to know his network and make sure it is clean.
 
-Grains Changes
---------------
-
-* The ``virtual`` grain identifies reports KVM and VMM hypervisors when running
-  an OpenBSD guest
-
 New Modules
 -----------
 
@@ -655,31 +479,96 @@ The ``state_output`` parameter now supports ``full_id``, ``changes_id`` and ``te
 Just like ``mixed_id``, these use the state ID as name in the highstate output.
 For more information on these output modes, see the docs for the :mod:`Highstate Outputter <salt.output.highstate>`.
 
-Windows Installer: Changes to existing config handling
-------------------------------------------------------
-Behavior with existing configuration has changed. With previous installers the
+Windows
+-------
+Python Version
+==============
+Python 2 Windows API was design when Windows did not support Unicode. Windows now supports
+Unicode however to keep backwards compatibility Python 2 Windows API has not been changed.
+Python 3 Windows API supports Unicode. Salt Python 3 installer is the recommend choice for
+users who need characters other than Non-ASCII (7bit) characters.
+
+Execution module changes
+========================
+pkg
+***
+Significate changes have been made to the :mod:`win_pkg <salt.modules.win_pkg>` execution module. Users should test this release against their existing package sls definition files. These changes are also in 2016.11.9 & 2017.7.3.
+
+- ``pkg.list_available`` no longer defaults to refreshing the winrepo meta database.
+- ``pkg.install`` without a ``version`` parameter no longer upgrades software if the software is already installed. Use ``pkg.install version=latest`` or in a state use ``pkg.latest`` to get the old behavior. 
+- ``pkg.list_pkgs`` now returns multiple versions if software installed more than once.
+- ``pkg.list_pkgs`` now returns 'Not Found' when the version is not found instead of '(value not set)' which matches the contents of the sls definitions.
+- ``pkg.remove()`` will wait upto 3 seconds (normally about a second) to detect changes in the registry after removing software, improving reporting of version changes.
+- ``pkg.remove()`` can remove ``latest`` software, if ``latest`` is defined in sls definition.
+- Documentation was update for the execution module to match the style in new versions, some corrections as well.
+- All install/remove commands are prefix with cmd.exe shell and cmdmod is called with a command line string instead of a list. Some sls files in saltstack/salt-winrepo-ng expected the commands to be prefixed with cmd.exe (i.e. the use of ``&``).
+- Some execution module functions results, now behavour more like their Unix/Linux versions.
+
+cmdmod
+******
+Linux/Unix OS command & arguments requires a python list. Windows was being treated
+the same. Windows requires commands & arguments to be a string. These changes are
+also in 2016.11.9 & 2017.7.3.
+
+Installer
+=========
+Changes to config handling
+**************************
+Behavior with existing configuration has changed. With previous windows installers the
 existing config was used and the master and minion id could be modified via the
 installer. It was problematic in that it didn't account for configuration that
 may be defined in the ``minion.d`` directory. This change gives you the option
-via a checkbox to either use the existing config with out changes or the default
-config using values you pass to the installer. If you choose to use the existing
-config then no changes are made. If not, the existing config is deleted, to
-include the ``minion.d`` directory, and the default config is used. A
-command-line switch (``/use-existing-config``) has also been added to control
-this behavior.
+via a drop-down list to use one of the following:
 
-Windows Installer: Multi-master configuration
----------------------------------------------
+- Default Config: Use the config that comes with the installer
+- Existing Config: Use the current config without changes
+- Custom Config: Select a custom config using the file picker
+
+The existing config option will only be available if the installer detects an
+existing config. If there is an existing config, and you choose ``Default`` or
+``Custom``, the existing config will be deleted, including the ``minion.d``
+directory, and replaced by your selection.
+
+The ``Default Config`` and ``Custom Config`` options will allow you to modify
+the Master and the Minion ID. ``Existing Config`` will leave the existing
+configuration unchanged.
+
+These settings can be defined on the command line using the following switches:
+
+- ``/default-config``
+- ``/custom-config=C:\Path\To\Custom\Config\minion``
+
+If neither option is passed and there is an existing config, the default is to
+use the existing config. If there is no existing config (new install) the
+default config will be used.
+
+Multi-master configuration
+**************************
 The installer now has the ability to apply a multi-master configuration either
 from the gui or the command line. The ``master`` field in the gui can accept
 either a single master or a comma-separated list of masters. The command-line
 switch (``/master=``) can accept the same.
 
-Windows Installer: Command-line help
-------------------------------------
+Command-line help
+*****************
 The Windows installer will now display command-line help when a help switch
 (``/?``) is passed.
 
+utils.pkg.win preview
+=====================
+A new utils python module has been added, which gathers information about windows
+installed software. This is currently not used by any salt execution module or state.
+Users are encource to run this and report any issues. Running the command with the
+``detail`` option will be useful for anyone developing windows package definitions.
+With salt installed in the default location the following command will print the help
+message.
+
+.. code-block:: text
+
+    chcp 65001
+    c:\salt\bin\python.exe c:\salt\bin\lib\site-packages\salt\utils\pkg\win.py
+    c:\salt\bin\python.exe c:\salt\bin\lib\site-packages\salt\utils\pkg\win.py detail system
+
 Salt Cloud Features
 -------------------
 
@@ -784,6 +673,21 @@ which designate a ``Vagrantfile`` on the host machine.
 The master can be a very limited machine, such as a Raspberry Pi, or a small
 VagrantBox VM.
 
+Python PyWinRM Module
+=====================
+Versions of ``pywinrm>=0.2.1`` are finally able to disable validation of self
+signed certificates.  :ref:`Here<new-pywinrm>` for more information.
+
+DigitalOcean
+============
+The DigitalOcean driver has been renamed to conform to the company name.  The
+new driver name is ``digitalocean``.  The old name ``digital_ocean`` and a
+short one ``do`` will still be supported through virtual aliases, this is mostly
+cosmetic.
+
+Azure Cloud
+===========
+The azure sdk used for the ``azurearm`` cloud driver now depends on ``azure-cli>=2.0.12``
 
 New pillar/master_tops module called saltclass
 ----------------------------------------------
@@ -973,27 +877,6 @@ Not using ``^`` as the first entry will simply merge the lists
 
 Currently you can't have both a variable and an escaped variable in the same string as the escaped one will not be correctly rendered - '\${xx}' will stay as is instead of being rendered as '${xx}'
 
-Newer PyWinRM Versions
-----------------------
-
-Versions of ``pywinrm>=0.2.1`` are finally able to disable validation of self
-signed certificates.  :ref:`Here<new-pywinrm>` for more information.
-
-DigitalOcean
-------------
-
-The DigitalOcean driver has been renamed to conform to the company name.  The
-new driver name is ``digitalocean``.  The old name ``digital_ocean`` and a
-short one ``do`` will still be supported through virtual aliases, this is mostly
-cosmetic.
-
-Solaris Logical Domains In Virtual Grain
-----------------------------------------
-
-Support has been added to the ``virtual`` grain for detecting Solaris LDOMs
-running on T-Series SPARC hardware.  The ``virtual_subtype`` grain is
-populated as a list of domain roles.
-
 Lists of comments in state returns
 ----------------------------------
 
@@ -1610,9 +1493,177 @@ NaCL Module and Runner changes
 ------------------------------
 
 In addition to argument changes in both the NaCL module and runner for future
-deprecation in the Fluorine release, the default box_type has changed from
-`secretbox` to `sealedbox`.  SecretBox is data encrypted using private key
-`sk` and Sealedbox is encrypted using public key `pk`
+removal in the Neon release, the default "box_type" has changed from
+``secretbox`` to ``sealedbox``.  SecretBox is data encrypted using private key
+``sk`` and Sealedbox is encrypted using public key ``pk``.
+
+``utils`` functions moved into separate modules
+-----------------------------------------------
+
+The Salt utility functions from ``salt.utils`` have been moved into different
+modules, grouped logically based on their functionality. This change is
+backwards compatible, but the old imports will no longer be supported starting
+with release Neon.
+
+The functions have been moved as follows:
+
+- ``salt.utils.appendproctitle``: use ``salt.utils.process.appendproctitle``
+  instead.
+- ``salt.utils.daemonize``: use ``salt.utils.process.daemonize`` instead.
+- ``salt.utils.daemonize_if``: use ``salt.utils.process.daemonize_if`` instead.
+- ``salt.utils.reinit_crypto``: use ``salt.utils.crypt.reinit_crypto`` instead.
+- ``salt.utils.pem_finger``: use ``salt.utils.crypt.pem_finger`` instead.
+- ``salt.utils.to_bytes``: use ``salt.utils.stringutils.to_bytes`` instead.
+- ``salt.utils.to_str``: use ``salt.utils.stringutils.to_str`` instead.
+- ``salt.utils.to_unicode``: use ``salt.utils.stringutils.to_unicode`` instead.
+- ``salt.utils.str_to_num``: use ``salt.utils.stringutils.to_num`` instead.
+- ``salt.utils.is_quoted``: use ``salt.utils.stringutils.is_quoted`` instead.
+- ``salt.utils.dequote``: use ``salt.utils.stringutils.dequote`` instead.
+- ``salt.utils.is_hex``: use ``salt.utils.stringutils.is_hex`` instead.
+- ``salt.utils.is_bin_str``: use ``salt.utils.stringutils.is_bin_str`` instead.
+- ``salt.utils.rand_string``: use ``salt.utils.stringutils.random`` instead.
+- ``salt.utils.contains_whitespace``: use
+  ``salt.utils.stringutils.contains_whitespace`` instead.
+- ``salt.utils.build_whitespace_split_regex``: use
+  ``salt.utils.stringutils.build_whitespace_split_regex`` instead.
+- ``salt.utils.expr_match``: use ``salt.utils.stringutils.expr_match`` instead.
+- ``salt.utils.check_whitelist_blacklist``: use
+  ``salt.utils.stringutils.check_whitelist_blacklist`` instead.
+- ``salt.utils.check_include_exclude``: use
+  ``salt.utils.stringutils.check_include_exclude`` instead.
+- ``salt.utils.print_cli``: use ``salt.utils.stringutils.print_cli`` instead.
+- ``salt.utils.clean_kwargs``: use ``salt.utils.args.clean_kwargs`` instead.
+- ``salt.utils.invalid_kwargs``: use ``salt.utils.args.invalid_kwargs``
+  instead.
+- ``salt.utils.shlex_split``: use ``salt.utils.args.shlex_split`` instead.
+- ``salt.utils.arg_lookup``: use ``salt.utils.args.arg_lookup`` instead.
+- ``salt.utils.argspec_report``: use ``salt.utils.args.argspec_report``
+  instead.
+- ``salt.utils.split_input``: use ``salt.utils.args.split_input`` instead.
+- ``salt.utils.test_mode``: use ``salt.utils.args.test_mode`` instead.
+- ``salt.utils.format_call``: use ``salt.utils.args.format_call`` instead.
+- ``salt.utils.which``: use ``salt.utils.path.which`` instead.
+- ``salt.utils.which_bin``: use ``salt.utils.path.which_bin`` instead.
+- ``salt.utils.path_join``: use ``salt.utils.path.join`` instead.
+- ``salt.utils.check_or_die``: use ``salt.utils.path.check_or_die`` instead.
+- ``salt.utils.sanitize_win_path_string``: use
+  ``salt.utils.path.sanitize_win_path`` instead.
+- ``salt.utils.rand_str``: use ``salt.utils.hashutils.random_hash`` instead.
+- ``salt.utils.get_hash``: use ``salt.utils.hashutils.get_hash`` instead.
+- ``salt.utils.is_windows``: use ``salt.utils.platform.is_windows`` instead.
+- ``salt.utils.is_proxy``: use ``salt.utils.platform.is_proxy`` instead.
+- ``salt.utils.is_linux``: use ``salt.utils.platform.is_linux`` instead.
+- ``salt.utils.is_darwin``: use ``salt.utils.platform.is_darwin`` instead.
+- ``salt.utils.is_sunos``: use ``salt.utils.platform.is_sunos`` instead.
+- ``salt.utils.is_smartos``: use ``salt.utils.platform.is_smartos`` instead.
+- ``salt.utils.is_smartos_globalzone``: use
+  ``salt.utils.platform.is_smartos_globalzone`` instead.
+- ``salt.utils.is_smartos_zone``: use ``salt.utils.platform.is_smartos_zone``
+  instead.
+- ``salt.utils.is_freebsd``: use ``salt.utils.platform.is_freebsd`` instead.
+- ``salt.utils.is_netbsd``: use ``salt.utils.platform.is_netbsd`` instead.
+- ``salt.utils.is_openbsd``: use ``salt.utils.platform.is_openbsd`` instead.
+- ``salt.utils.is_aix``: use ``salt.utils.platform.is_aix`` instead.
+- ``salt.utils.safe_rm``: use ``salt.utils.files.safe_rm`` instead.
+- ``salt.utils.is_empty``: use ``salt.utils.files.is_empty`` instead.
+- ``salt.utils.fopen``: use ``salt.utils.files.fopen`` instead.
+- ``salt.utils.flopen``: use ``salt.utils.files.flopen`` instead.
+- ``salt.utils.fpopen``: use ``salt.utils.files.fpopen`` instead.
+- ``salt.utils.rm_rf``: use ``salt.utils.files.rm_rf`` instead.
+- ``salt.utils.mkstemp``: use ``salt.utils.files.mkstemp`` instead.
+- ``salt.utils.istextfile``: use ``salt.utils.files.is_text_file`` instead.
+- ``salt.utils.is_bin_file``: use ``salt.utils.files.is_binary`` instead.
+- ``salt.utils.list_files``: use ``salt.utils.files.list_files`` instead.
+- ``salt.utils.safe_walk``: use ``salt.utils.files.safe_walk`` instead.
+- ``salt.utils.st_mode_to_octal``: use ``salt.utils.files.st_mode_to_octal``
+  instead.
+- ``salt.utils.normalize_mode``: use ``salt.utils.files.normalize_mode``
+  instead.
+- ``salt.utils.human_size_to_bytes``: use
+  ``salt.utils.files.human_size_to_bytes`` instead.
+- ``salt.utils.backup_minion``: use ``salt.utils.files.backup_minion`` instead.
+- ``salt.utils.str_version_to_evr``: use ``salt.utils.pkg.rpm.version_to_evr``
+  instead.
+- ``salt.utils.parse_docstring``: use ``salt.utils.doc.parse_docstring``
+  instead.
+- ``salt.utils.compare_versions``: use ``salt.utils.versions.compare`` instead.
+- ``salt.utils.version_cmp``: use ``salt.utils.versions.version_cmp`` instead.
+- ``salt.utils.warn_until``: use ``salt.utils.versions.warn_until`` instead.
+- ``salt.utils.kwargs_warn_until``: use
+  ``salt.utils.versions.kwargs_warn_until`` instead.
+- ``salt.utils.get_color_theme``: use ``salt.utils.color.get_color_theme``
+  instead.
+- ``salt.utils.get_colors``: use ``salt.utils.color.get_colors`` instead.
+- ``salt.utils.gen_state_tag``: use ``salt.utils.state.gen_tag`` instead.
+- ``salt.utils.search_onfail_requisites``: use
+  ``salt.utils.state.search_onfail_requisites`` instead.
+- ``salt.utils.check_state_result``: use ``salt.utils.state.check_result``
+  instead.
+- ``salt.utils.get_user``: use ``salt.utils.user.get_user`` instead.
+- ``salt.utils.get_uid``: use ``salt.utils.user.get_uid`` instead.
+- ``salt.utils.get_specific_user``: use ``salt.utils.user.get_specific_user``
+  instead.
+- ``salt.utils.chugid``: use ``salt.utils.user.chugid`` instead.
+- ``salt.utils.chugid_and_umask``: use ``salt.utils.user.chugid_and_umask``
+  instead.
+- ``salt.utils.get_default_group``: use ``salt.utils.user.get_default_group``
+  instead.
+- ``salt.utils.get_group_list``: use ``salt.utils.user.get_group_list``
+  instead.
+- ``salt.utils.get_group_dict``: use ``salt.utils.user.get_group_dict``
+  instead.
+- ``salt.utils.get_gid_list``: use ``salt.utils.user.get_gid_list`` instead.
+- ``salt.utils.get_gid``: use ``salt.utils.user.get_gid`` instead.
+- ``salt.utils.enable_ctrl_logoff_handler``: use
+  ``salt.utils.win_functions.enable_ctrl_logoff_handler`` instead.
+- ``salt.utils.traverse_dict``: use ``salt.utils.data.traverse_dict`` instead.
+- ``salt.utils.traverse_dict_and_list``: use
+  ``salt.utils.data.traverse_dict_and_list`` instead.
+- ``salt.utils.filter_by``: use ``salt.utils.data.filter_by`` instead.
+- ``salt.utils.subdict_match``: use ``salt.utils.data.subdict_match`` instead.
+- ``salt.utils.substr_in_list``: use ``salt.utils.data.substr_in_list`` instead.
+- ``salt.utils.is_dictlist``: use ``salt.utils.data.is_dictlist``.
+- ``salt.utils.repack_dictlist``: use ``salt.utils.data.repack_dictlist``
+  instead.
+- ``salt.utils.compare_dicts``: use ``salt.utils.data.compare_dicts`` instead.
+- ``salt.utils.compare_lists``: use ``salt.utils.data.compare_lists`` instead.
+- ``salt.utils.decode_dict``: use ``salt.utils.data.encode_dict`` instead.
+- ``salt.utils.decode_list``: use ``salt.utils.data.encode_list`` instead.
+- ``salt.utils.exactly_n``: use ``salt.utils.data.exactly_n`` instead.
+- ``salt.utils.exactly_one``: use ``salt.utils.data.exactly_one`` instead.
+- ``salt.utils.is_list``: use ``salt.utils.data.is_list`` instead.
+- ``salt.utils.is_iter``: use ``salt.utils.data.is_iter`` instead.
+- ``salt.utils.isorted``: use ``salt.utils.data.sorted_ignorecase`` instead.
+- ``salt.utils.is_true``: use ``salt.utils.data.is_true`` instead.
+- ``salt.utils.mysql_to_dict``: use ``salt.utils.data.mysql_to_dict`` instead.
+- ``salt.utils.simple_types_filter``: use
+  ``salt.utils.data.simple_types_filter`` instead.
+- ``salt.utils.ip_bracket``: use ``salt.utils.zeromq.ip_bracket`` instead.
+- ``salt.utils.gen_mac``: use ``salt.utils.network.gen_mac`` instead.
+- ``salt.utils.mac_str_to_bytes``: use ``salt.utils.network.mac_str_to_bytes``
+  instead.
+- ``salt.utils.refresh_dns``: use ``salt.utils.network.refresh_dns`` instead.
+- ``salt.utils.dns_check``: use ``salt.utils.network.dns_check`` instead.
+- ``salt.utils.get_context``: use ``salt.utils.stringutils.get_context`` instead.
+- ``salt.utils.get_master_key``: use ``salt.utils.master.get_master_key``
+  instead.
+- ``salt.utils.get_values_of_matching_keys``: use
+  ``salt.utils.master.get_values_of_matching_keys`` instead.
+- ``salt.utils.date_cast``: use ``salt.utils.dateutils.date_cast`` instead.
+- ``salt.utils.date_format``: use ``salt.utils.dateutils.strftime`` instead.
+- ``salt.utils.total_seconds``: use ``salt.utils.dateutils.total_seconds``
+  instead.
+- ``salt.utils.find_json``: use ``salt.utils.json.find_json`` instead.
+- ``salt.utils.import_json``: use ``salt.utils.json.import_json`` instead.
+- ``salt.utils.namespaced_function``: use
+  ``salt.utils.functools.namespaced_function`` instead.
+- ``salt.utils.alias_function``: use ``salt.utils.functools.alias_function``
+  instead.
+- ``salt.utils.profile_func``: use ``salt.utils.profile.profile_func`` instead.
+- ``salt.utils.activate_profile``: use ``salt.utils.profile.activate_profile``
+  instead.
+- ``salt.utils.output_profile``: use ``salt.utils.profile.output_profile``
+  instead.
 
 Deprecations
 ------------
@@ -1629,10 +1680,7 @@ Profitbricks Cloud Updated Dependency
 The minimum version of the ``profitbrick`` python package for the ``profitbricks``
 cloud driver has changed from 3.0.0 to 3.1.0.
 
-Azure Cloud Updated Dependency
-------------------------------
 
-The azure sdk used for the ``azurearm`` cloud driver now depends on ``azure-cli>=2.0.12``
 
 Module Deprecations
 ===================

pkg/osx/req.txt

@@ -21,7 +21,7 @@ pycrypto==2.6.1
 python-dateutil==2.6.1
 python-gnupg==0.4.1
 PyYAML==3.12
-pyzmq==17.0.0b3
+pyzmq==17.0.0
 requests==2.18.4
 singledispatch==3.4.0.3
 six==1.11.0

pkg/windows/installer/Salt-Minion-Setup.nsi

@@ -1056,7 +1056,7 @@ Function AddToPath
 
     # Make sure the new length isn't over the NSIS_MAX_STRLEN
     IntCmp $2 ${NSIS_MAX_STRLEN} +4 +4 0
-        DetailPrint "AddToPath: new length $2 > ${NSIS_MAX_STRLEN}"
+        DetailPrint "AddToPath Failed: new length $2 > ${NSIS_MAX_STRLEN}"
         MessageBox MB_OK \
             "You may add C:\salt to the %PATH% for convenience when issuing local salt commands from the command line." \
             /SD IDOK

pkg/windows/req.txt

@@ -3,7 +3,7 @@ backports.ssl-match-hostname==3.5.0.1
 certifi
 cffi==1.10.0
 CherryPy==10.2.1
-cryptography==1.8.1
+cryptography==2.1.4
 enum34==1.1.6
 futures==3.1.1
 gitdb==0.6.4
@@ -23,13 +23,14 @@ pycparser==2.17
 pycrypto==2.6.1
 pycurl==7.43.0
 PyMySQL==0.7.11
-pyOpenSSL==17.0.0
-python-dateutil==2.6.0
-python-gnupg==0.4.0
+pyOpenSSL==17.5.0
+python-dateutil==2.6.1
+python-gnupg==0.4.1
+pythonnet==2.3.0
 pywin32==223
 PyYAML==3.12
-pyzmq==16.0.2
-requests==2.13.0
+pyzmq==16.0.3
+requests==2.18.4
 singledispatch==3.4.0.3
 smmap==0.9.0
 timelib==0.2.4

salt/auth/sharedsecret.py

@@ -37,8 +37,8 @@ import logging
 log = logging.getLogger(__name__)
 
 
-def auth(username, sharedsecret, **kwargs):
+def auth(username, password):
     '''
     Shared secret authentication
     '''
-    return sharedsecret == __opts__.get('sharedsecret')
+    return password == __opts__.get('sharedsecret')

salt/client/ssh/__init__.py

@@ -57,7 +57,7 @@ from salt.ext import six
 from salt.ext.six.moves import input  # pylint: disable=import-error,redefined-builtin
 try:
     import saltwinshell
-    HAS_WINSHELL = False
+    HAS_WINSHELL = True
 except ImportError:
     HAS_WINSHELL = False
 from salt.utils.zeromq import zmq
@@ -560,6 +560,19 @@ class SSH(object):
                         self.targets[host][default] = self.defaults[default]
                 if 'host' not in self.targets[host]:
                     self.targets[host]['host'] = host
+                if self.targets[host].get('winrm') and not HAS_WINSHELL:
+                    returned.add(host)
+                    rets.add(host)
+                    log_msg = 'Please contact sales@saltstack.com for access to the enterprise saltwinshell module.'
+                    log.debug(log_msg)
+                    no_ret = {'fun_args': [],
+                              'jid': None,
+                              'return': log_msg,
+                              'retcode': 1,
+                              'fun': '',
+                              'id': host}
+                    yield {host: no_ret}
+                    continue
                 args = (
                         que,
                         self.opts,

salt/cloud/clouds/openstack.py

@@ -708,7 +708,7 @@ def create(vm_):
     try:
         ip_address = __utils__['cloud.wait_for_fun'](
             __query_node,
-            update_args=(vm_,)
+            vm_=vm_
         )
     except (SaltCloudExecutionTimeout, SaltCloudExecutionFailure) as exc:
         try:

salt/config/__init__.py

@@ -739,6 +739,9 @@ VALID_OPTS = {
     # Recursively merge lists by aggregating them instead of replacing them.
     'pillar_merge_lists': bool,
 
+    # If True, values from included pillar SLS targets will override
+    'pillar_includes_override_sls': bool,
+
     # How to merge multiple top files from multiple salt environments
     # (saltenvs); can be 'merge' or 'same'
     'top_file_merging_strategy': six.string_types,
@@ -1238,6 +1241,9 @@ DEFAULT_MINION_OPTS = {
     'pillarenv': None,
     'pillarenv_from_saltenv': False,
     'pillar_opts': False,
+    'pillar_source_merging_strategy': 'smart',
+    'pillar_merge_lists': False,
+    'pillar_includes_override_sls': False,
     # ``pillar_cache``, ``pillar_cache_ttl`` and ``pillar_cache_backend``
     # are not used on the minion but are unavoidably in the code path
     'pillar_cache': False,
@@ -1616,6 +1622,7 @@ DEFAULT_MASTER_OPTS = {
     'pillar_safe_render_error': True,
     'pillar_source_merging_strategy': 'smart',
     'pillar_merge_lists': False,
+    'pillar_includes_override_sls': False,
     'pillar_cache': False,
     'pillar_cache_ttl': 3600,
     'pillar_cache_backend': 'disk',

salt/engines/libvirt_events.py

@@ -214,7 +214,8 @@ def _salt_send_domain_event(opaque, conn, domain, event, event_data):
     data = {
         'domain': {
             'name': domain.name(),
-            'id': domain.ID()
+            'id': domain.ID(),
+            'uuid': domain.UUIDString()
         },
         'event': event
     }
@@ -228,7 +229,8 @@ def _domain_event_lifecycle_cb(conn, domain, event, detail, opaque):
     '''
     event_str, detail_str = _get_domain_event_detail(event, detail)
 
-    _salt_send_domain_event(opaque, conn, domain, event_str, {
+    _salt_send_domain_event(opaque, conn, domain, opaque['event'], {
+        'event':  event_str,
         'detail': detail_str
     })
 
@@ -468,7 +470,10 @@ def _network_event_lifecycle_cb(conn, net, event, detail, opaque):
     '''
 
     _salt_send_event(opaque, conn, {
-        'network': net.name(),
+        'network': {
+            'name': net.name(),
+            'uuid': net.UUIDString()
+        },
         'event': _get_libvirt_enum_string('VIR_NETWORK_EVENT_', event),
         'detail': 'unknown'  # currently unused
     })
@@ -479,7 +484,10 @@ def _pool_event_lifecycle_cb(conn, pool, event, detail, opaque):
     Storage pool lifecycle events handler
     '''
     _salt_send_event(opaque, conn, {
-        'pool': pool.name(),
+        'pool': {
+            'name': pool.name(),
+            'uuid': pool.UUIDString()
+        },
         'event': _get_libvirt_enum_string('VIR_STORAGE_POOL_EVENT_', event),
         'detail': 'unknown'  # currently unused
     })
@@ -490,7 +498,10 @@ def _pool_event_refresh_cb(conn, pool, opaque):
     Storage pool refresh events handler
     '''
     _salt_send_event(opaque, conn, {
-        'pool': pool.name(),
+        'pool': {
+            'name': pool.name(),
+            'uuid': pool.UUIDString()
+        },
         'event': opaque['event']
     })
 
@@ -500,7 +511,9 @@ def _nodedev_event_lifecycle_cb(conn, dev, event, detail, opaque):
     Node device lifecycle events handler
     '''
     _salt_send_event(opaque, conn, {
-        'nodedev': dev.name(),
+        'nodedev': {
+            'name': dev.name()
+        },
         'event': _get_libvirt_enum_string('VIR_NODE_DEVICE_EVENT_', event),
         'detail': 'unknown'  # currently unused
     })
@@ -511,7 +524,9 @@ def _nodedev_event_update_cb(conn, dev, opaque):
     Node device update events handler
     '''
     _salt_send_event(opaque, conn, {
-        'nodedev': dev.name(),
+        'nodedev': {
+            'name': dev.name()
+        },
         'event': opaque['event']
     })
 
@@ -521,7 +536,9 @@ def _secret_event_lifecycle_cb(conn, secret, event, detail, opaque):
     Secret lifecycle events handler
     '''
     _salt_send_event(opaque, conn, {
-        'secret': secret.UUIDString(),
+        'secret': {
+            'uuid': secret.UUIDString()
+        },
         'event': _get_libvirt_enum_string('VIR_SECRET_EVENT_', event),
         'detail': 'unknown'  # currently unused
     })
@@ -532,7 +549,9 @@ def _secret_event_value_changed_cb(conn, secret, opaque):
     Secret value change events handler
     '''
     _salt_send_event(opaque, conn, {
-        'secret': secret.UUIDString(),
+        'secret': {
+            'uuid': secret.UUIDString()
+        },
         'event': opaque['event']
     })
 

salt/grains/core.py

@@ -2449,7 +2449,30 @@ def get_server_id():
 
     if salt.utils.platform.is_proxy():
         return {}
-    return {'server_id': abs(hash(__opts__.get('id', '')) % (2 ** 31))}
+    id_ = __opts__.get('id', '')
+    id_hash = None
+    py_ver = sys.version_info[:2]
+    if py_ver >= (3, 3):
+        # Python 3.3 enabled hash randomization, so we need to shell out to get
+        # a reliable hash.
+        py_bin = 'python{0}.{1}'.format(*py_ver)
+        id_hash = __salt__['cmd.run'](
+            [py_bin, '-c', 'print(hash("{0}"))'.format(id_)],
+            env={'PYTHONHASHSEED': '0'}
+        )
+        try:
+            id_hash = int(id_hash)
+        except (TypeError, ValueError):
+            log.debug(
+                'Failed to hash the ID to get the server_id grain. Result of '
+                'hash command: %s', id_hash
+            )
+            id_hash = None
+    if id_hash is None:
+        # Python < 3.3 or error encountered above
+        id_hash = hash(id_)
+
+    return {'server_id': abs(id_hash % (2 ** 31))}
 
 
 def get_master():

salt/grains/metadata.py

@@ -48,9 +48,11 @@ def _search(prefix="latest/"):
     Recursively look up all grains in the metadata server
     '''
     ret = {}
-    linedata = http.query(os.path.join(HOST, prefix))
+    linedata = http.query(os.path.join(HOST, prefix), headers=True)
     if 'body' not in linedata:
         return ret
+    if linedata['headers'].get('Content-Type', 'text/plain') == 'application/octet-stream':
+        return linedata['body']
     for line in linedata['body'].split('\n'):
         if line.endswith('/'):
             ret[line[:-1]] = _search(prefix=os.path.join(prefix, line))

salt/loader.py

@@ -14,6 +14,7 @@ import logging
 import inspect
 import tempfile
 import functools
+import threading
 import types
 from collections import MutableMapping
 from zipimport import zipimporter
@@ -1124,7 +1125,8 @@ class LazyLoader(salt.utils.lazy.LazyDict):
             )
         )
 
-        self.refresh_file_mapping()
+        self._lock = threading.RLock()
+        self._refresh_file_mapping()
 
         super(LazyLoader, self).__init__()  # late init the lazy loader
         # create all of the import namespaces
@@ -1191,7 +1193,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
                 else:
                     return '\'{0}\' __virtual__ returned False'.format(mod_name)
 
-    def refresh_file_mapping(self):
+    def _refresh_file_mapping(self):
         '''
         refresh the mapping of the FS on disk
         '''
@@ -1308,15 +1310,16 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         '''
         Clear the dict
         '''
-        super(LazyLoader, self).clear()  # clear the lazy loader
-        self.loaded_files = set()
-        self.missing_modules = {}
-        self.loaded_modules = {}
-        # if we have been loaded before, lets clear the file mapping since
-        # we obviously want a re-do
-        if hasattr(self, 'opts'):
-            self.refresh_file_mapping()
-        self.initial_load = False
+        with self._lock:
+            super(LazyLoader, self).clear()  # clear the lazy loader
+            self.loaded_files = set()
+            self.missing_modules = {}
+            self.loaded_modules = {}
+            # if we have been loaded before, lets clear the file mapping since
+            # we obviously want a re-do
+            if hasattr(self, 'opts'):
+                self._refresh_file_mapping()
+            self.initial_load = False
 
     def __prep_mod_opts(self, opts):
         '''
@@ -1520,14 +1523,14 @@ class LazyLoader(salt.utils.lazy.LazyDict):
             virtual_funcs_to_process = ['__virtual__'] + self.virtual_funcs
             for virtual_func in virtual_funcs_to_process:
                 virtual_ret, module_name, virtual_err, virtual_aliases = \
-                    self.process_virtual(mod, module_name, virtual_func)
+                    self._process_virtual(mod, module_name, virtual_func)
                 if virtual_err is not None:
                     log.trace(
                         'Error loading %s.%s: %s',
                         self.tag, module_name, virtual_err
                     )
 
-                # if process_virtual returned a non-True value then we are
+                # if _process_virtual returned a non-True value then we are
                 # supposed to not process this module
                 if virtual_ret is not True and module_name not in self.missing_modules:
                     # If a module has information about why it could not be loaded, record it
@@ -1619,39 +1622,42 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         if '.' not in key:
             raise KeyError('The key \'%s\' should contain a \'.\'', key)
         mod_name, _ = key.split('.', 1)
-        if mod_name in self.missing_modules:
-            return True
-        # if the modulename isn't in the whitelist, don't bother
-        if self.whitelist and mod_name not in self.whitelist:
-            raise KeyError
+        with self._lock:
+            # It is possible that the key is in the dictionary after
+            # acquiring the lock due to another thread loading it.
+            if mod_name in self.missing_modules or key in self._dict:
+                return True
+            # if the modulename isn't in the whitelist, don't bother
+            if self.whitelist and mod_name not in self.whitelist:
+                raise KeyError
 
-        def _inner_load(mod_name):
-            for name in self._iter_files(mod_name):
-                if name in self.loaded_files:
-                    continue
-                # if we got what we wanted, we are done
-                if self._load_module(name) and key in self._dict:
-                    return True
-            return False
+            def _inner_load(mod_name):
+                for name in self._iter_files(mod_name):
+                    if name in self.loaded_files:
+                        continue
+                    # if we got what we wanted, we are done
+                    if self._load_module(name) and key in self._dict:
+                        return True
+                return False
 
-        # try to load the module
-        ret = None
-        reloaded = False
-        # re-scan up to once, IOErrors or a failed load cause re-scans of the
-        # filesystem
-        while True:
-            try:
-                ret = _inner_load(mod_name)
-                if not reloaded and ret is not True:
-                    self.refresh_file_mapping()
-                    reloaded = True
+            # try to load the module
+            ret = None
+            reloaded = False
+            # re-scan up to once, IOErrors or a failed load cause re-scans of the
+            # filesystem
+            while True:
+                try:
+                    ret = _inner_load(mod_name)
+                    if not reloaded and ret is not True:
+                        self._refresh_file_mapping()
+                        reloaded = True
+                        continue
+                    break
+                except IOError:
+                    if not reloaded:
+                        self._refresh_file_mapping()
+                        reloaded = True
                     continue
-                break
-            except IOError:
-                if not reloaded:
-                    self.refresh_file_mapping()
-                    reloaded = True
-                continue
 
         return ret
 
@@ -1659,16 +1665,18 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         '''
         Load all of them
         '''
-        for name in self.file_mapping:
-            if name in self.loaded_files or name in self.missing_modules:
-                continue
-            self._load_module(name)
+        with self._lock:
+            for name in self.file_mapping:
+                if name in self.loaded_files or name in self.missing_modules:
+                    continue
+                self._load_module(name)
 
-        self.loaded = True
+            self.loaded = True
 
     def reload_modules(self):
-        self.loaded_files = set()
-        self._load_all()
+        with self._lock:
+            self.loaded_files = set()
+            self._load_all()
 
     def _apply_outputter(self, func, mod):
         '''
@@ -1679,7 +1687,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
             if func.__name__ in outp:
                 func.__outputter__ = outp[func.__name__]
 
-    def process_virtual(self, mod, module_name, virtual_func='__virtual__'):
+    def _process_virtual(self, mod, module_name, virtual_func='__virtual__'):
         '''
         Given a loaded module and its default name determine its virtual name
 

salt/minion.py

@@ -2524,6 +2524,7 @@ class Minion(MinionBase):
                     self.opts,
                     self.functions,
                     self.returners,
+                    utils=self.utils,
                     cleanup=[master_event(type='alive')])
 
             try:

salt/modules/cmdmod.py

@@ -437,10 +437,16 @@ def _run(cmd,
     if runas or group:
         try:
             # Getting the environment for the runas user
+            # Use markers to thwart any stdout noise
             # There must be a better way to do this.
+            import uuid
+            marker = '<<<' + str(uuid.uuid4()) + '>>>'
+            marker_b = marker.encode(__salt_system_encoding__)
             py_code = (
                 'import sys, os, itertools; '
-                'sys.stdout.write(\"\\0\".join(itertools.chain(*os.environ.items())))'
+                'sys.stdout.write(\"' + marker + '\"); '
+                'sys.stdout.write(\"\\0\".join(itertools.chain(*os.environ.items()))); '
+                'sys.stdout.write(\"' + marker + '\");'
             )
 
             if use_sudo or __grains__['os'] in ['MacOS', 'Darwin']:
@@ -466,11 +472,34 @@ def _run(cmd,
                 env_cmd = ('su', '-s', shell, '-', runas, '-c', sys.executable)
             msg = 'env command: {0}'.format(env_cmd)
             log.debug(log_callback(msg))
-            env_bytes = salt.utils.stringutils.to_bytes(subprocess.Popen(
+
+            env_bytes, env_encoded_err = subprocess.Popen(
                 env_cmd,
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE
-            ).communicate(salt.utils.stringutils.to_bytes(py_code))[0])
+                stderr=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stdin=subprocess.PIPE
+            ).communicate(salt.utils.stringutils.to_bytes(py_code))
+            marker_count = env_bytes.count(marker_b)
+            if marker_count == 0:
+                # Possibly PAM prevented the login
+                log.error(
+                    'Environment could not be retrieved for user \'%s\': '
+                    'stderr=%r stdout=%r',
+                    runas, env_encoded_err, env_bytes
+                )
+                # Ensure that we get an empty env_runas dict below since we
+                # were not able to get the environment.
+                env_bytes = b''
+            elif marker_count != 2:
+                raise CommandExecutionError(
+                    'Environment could not be retrieved for user \'{0}\'',
+                    info={'stderr': repr(env_encoded_err),
+                          'stdout': repr(env_bytes)}
+                )
+            else:
+                # Strip the marker
+                env_bytes = env_bytes.split(marker_b)[1]
+
             if six.PY2:
                 import itertools
                 env_runas = dict(itertools.izip(*[iter(env_bytes.split(b'\0'))]*2))
@@ -488,10 +517,11 @@ def _run(cmd,
             if env_runas.get('USER') != runas:
                 env_runas['USER'] = runas
             env = env_runas
-        except ValueError:
+        except ValueError as exc:
+            log.exception('Error raised retrieving environment for user %s', runas)
             raise CommandExecutionError(
-                'Environment could not be retrieved for User \'{0}\''.format(
-                    runas
+                'Environment could not be retrieved for user \'{0}\': {1}'.format(
+                    runas, exc
                 )
             )
 
@@ -630,7 +660,7 @@ def _run(cmd,
             ret['retcode'] = 1
             return ret
 
-        if output_encoding is not None:
+        if output_loglevel != 'quiet' and output_encoding is not None:
             log.debug('Decoding output from command %s using %s encoding',
                       cmd, output_encoding)
 
@@ -646,10 +676,11 @@ def _run(cmd,
                 proc.stdout,
                 encoding=output_encoding,
                 errors='replace')
-            log.error(
-                'Failed to decode stdout from command %s, non-decodable '
-                'characters have been replaced', cmd
-            )
+            if output_loglevel != 'quiet':
+                log.error(
+                    'Failed to decode stdout from command %s, non-decodable '
+                    'characters have been replaced', cmd
+                )
 
         try:
             err = salt.utils.stringutils.to_unicode(
@@ -663,10 +694,11 @@ def _run(cmd,
                 proc.stderr,
                 encoding=output_encoding,
                 errors='replace')
-            log.error(
-                'Failed to decode stderr from command %s, non-decodable '
-                'characters have been replaced', cmd
-            )
+            if output_loglevel != 'quiet':
+                log.error(
+                    'Failed to decode stderr from command %s, non-decodable '
+                    'characters have been replaced', cmd
+                )
 
         if rstrip:
             if out is not None:

salt/modules/debbuild.py

@@ -426,6 +426,9 @@ def build(runas,
     # use default /var/cache/pbuilder/result
     results_dir = '/var/cache/pbuilder/result'
 
+    ## ensure clean
+    __salt__['cmd.run']('rm -fR {0}'.format(results_dir))
+
     # dscs should only contain salt orig and debian tarballs and dsc file
     for dsc in dscs:
         afile = os.path.basename(dsc)
@@ -436,10 +439,10 @@ def build(runas,
             try:
                 __salt__['cmd.run']('chown {0} -R {1}'.format(runas, dbase))
 
-                cmd = 'pbuilder --update --override-config'
+                cmd = 'pbuilder update --override-config'
                 __salt__['cmd.run'](cmd, runas=runas, python_shell=True)
 
-                cmd = 'pbuilder --build {0}'.format(dsc)
+                cmd = 'pbuilder build --debbuildopts "-sa" {0}'.format(dsc)
                 __salt__['cmd.run'](cmd, runas=runas, python_shell=True)
 
                 # ignore local deps generated package file

salt/modules/localemod.py

@@ -87,7 +87,7 @@ def _localectl_status():
                         ret[ctl_key] = {}
                     ret[ctl_key][loc_set[0]] = loc_set[1]
             else:
-                ret[ctl_key] = ctl_data
+                ret[ctl_key] = {'data': ctl_data}
     if not ret:
         log.debug("Unable to find any locale information inside the following data:\n%s", locale_ctl_out)
         raise CommandExecutionError('Unable to parse result of "localectl"')

salt/modules/mysql.py

@@ -1691,11 +1691,11 @@ def __grant_generate(grant,
     table = db_part[2]
 
     if escape:
-        if dbc is not '*':
+        if dbc != '*':
             # _ and % are authorized on GRANT queries and should get escaped
             # on the db name, but only if not requesting a table level grant
-            dbc = quote_identifier(dbc, for_grants=(table is '*'))
-        if table is not '*':
+            dbc = quote_identifier(dbc, for_grants=(table == '*'))
+        if table != '*':
             table = quote_identifier(table)
     # identifiers cannot be used as values, and same thing for grants
     qry = 'GRANT {0} ON {1}.{2} TO %(user)s@%(host)s'.format(grant, dbc, table)
@@ -1790,8 +1790,10 @@ def grant_exists(grant,
             if not target_tokens:  # Avoid the overhead of re-calc in loop
                 target_tokens = _grant_to_tokens(target)
             grant_tokens = _grant_to_tokens(grant)
+            grant_tokens_database = grant_tokens['database'].replace('"', '').replace('\\', '').replace('`', '')
+            target_tokens_database = target_tokens['database'].replace('"', '').replace('\\', '').replace('`', '')
             if grant_tokens['user'] == target_tokens['user'] and \
-                    grant_tokens['database'] == target_tokens['database'] and \
+                    grant_tokens_database == target_tokens_database and \
                     grant_tokens['host'] == target_tokens['host'] and \
                     set(grant_tokens['grant']) >= set(target_tokens['grant']):
                 return True
@@ -1893,16 +1895,16 @@ def grant_revoke(grant,
     db_part = database.rpartition('.')
     dbc = db_part[0]
     table = db_part[2]
-    if dbc is not '*':
+    if dbc != '*':
         # _ and % are authorized on GRANT queries and should get escaped
         # on the db name, but only if not requesting a table level grant
-        s_database = quote_identifier(dbc, for_grants=(table is '*'))
-    if dbc is '*':
+        s_database = quote_identifier(dbc, for_grants=(table == '*'))
+    if dbc == '*':
         # add revoke for *.*
         # before the modification query send to mysql will looks like
         # REVOKE SELECT ON `*`.* FROM %(user)s@%(host)s
         s_database = dbc
-    if table is not '*':
+    if table != '*':
         table = quote_identifier(table)
     # identifiers cannot be used as values, same thing for grants
     qry = 'REVOKE {0} ON {1}.{2} FROM %(user)s@%(host)s;'.format(

salt/modules/nacl.py

@@ -161,6 +161,7 @@ import salt.syspaths
 import salt.utils.files
 import salt.utils.platform
 import salt.utils.stringutils
+import salt.utils.versions
 import salt.utils.win_functions
 import salt.utils.win_dacl
 
@@ -254,7 +255,7 @@ def keygen(sk_file=None, pk_file=None, **kwargs):
     '''
     if 'keyfile' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'keyfile\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk_file\' argument instead.'
         )
@@ -322,7 +323,7 @@ def enc(data, **kwargs):
     '''
     if 'keyfile' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'keyfile\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk_file\' argument instead.'
         )
@@ -330,7 +331,7 @@ def enc(data, **kwargs):
 
     if 'key' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'key\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk\' argument instead.'
         )
@@ -388,7 +389,7 @@ def dec(data, **kwargs):
     '''
     if 'keyfile' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'keyfile\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk_file\' argument instead.'
         )
@@ -399,7 +400,7 @@ def dec(data, **kwargs):
 
     if 'key' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'key\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk\' argument instead.'
         )

salt/modules/npm.py

@@ -159,7 +159,7 @@ def install(pkg=None,
     if runas:
         uid = salt.utils.user.get_uid(runas)
         if uid:
-            env.update({'SUDO_UID': b'{0}'.format(uid), 'SUDO_USER': b''})
+            env.update({'SUDO_UID': uid, 'SUDO_USER': ''})
 
     cmd = ' '.join(cmd)
     result = __salt__['cmd.run_all'](cmd, python_shell=True, cwd=dir, runas=runas, env=env)
@@ -238,7 +238,7 @@ def uninstall(pkg, dir=None, runas=None, env=None):
     if runas:
         uid = salt.utils.user.get_uid(runas)
         if uid:
-            env.update({'SUDO_UID': b'{0}'.format(uid), 'SUDO_USER': b''})
+            env.update({'SUDO_UID': uid, 'SUDO_USER': ''})
 
     cmd = ['npm', 'uninstall', '"{0}"'.format(pkg)]
     if not dir:
@@ -297,7 +297,7 @@ def list_(pkg=None, dir=None, runas=None, env=None, depth=None):
     if runas:
         uid = salt.utils.user.get_uid(runas)
         if uid:
-            env.update({'SUDO_UID': b'{0}'.format(uid), 'SUDO_USER': b''})
+            env.update({'SUDO_UID': uid, 'SUDO_USER': ''})
 
     cmd = ['npm', 'list', '--json', '--silent']
 
@@ -360,7 +360,7 @@ def cache_clean(path=None, runas=None, env=None, force=False):
     if runas:
         uid = salt.utils.user.get_uid(runas)
         if uid:
-            env.update({'SUDO_UID': b'{0}'.format(uid), 'SUDO_USER': b''})
+            env.update({'SUDO_UID': uid, 'SUDO_USER': ''})
 
     cmd = ['npm', 'cache', 'clean']
     if path:
@@ -407,7 +407,7 @@ def cache_list(path=None, runas=None, env=None):
     if runas:
         uid = salt.utils.user.get_uid(runas)
         if uid:
-            env.update({'SUDO_UID': b'{0}'.format(uid), 'SUDO_USER': b''})
+            env.update({'SUDO_UID': uid, 'SUDO_USER': ''})
 
     cmd = ['npm', 'cache', 'ls']
     if path:
@@ -447,7 +447,7 @@ def cache_path(runas=None, env=None):
     if runas:
         uid = salt.utils.user.get_uid(runas)
         if uid:
-            env.update({'SUDO_UID': b'{0}'.format(uid), 'SUDO_USER': b''})
+            env.update({'SUDO_UID': uid, 'SUDO_USER': ''})
 
     cmd = 'npm config get cache'
 

salt/modules/pillar.py

@@ -366,6 +366,28 @@ def item(*args, **kwargs):
 
         .. versionadded:: 2015.8.0
 
+    pillarenv
+        If specified, this function will query the master to generate fresh
+        pillar data on the fly, specifically from the requested pillar
+        environment. Note that this can produce different pillar data than
+        executing this function without an environment, as its normal behavior
+        is just to return a value from minion's pillar data in memory (which
+        can be sourced from more than one pillar environment).
+
+        Using this argument will not affect the pillar data in memory. It will
+        however be slightly slower and use more resources on the master due to
+        the need for the master to generate and send the minion fresh pillar
+        data. This tradeoff in performance however allows for the use case
+        where pillar data is desired only from a single environment.
+
+        .. versionadded:: 2017.7.6,2018.3.1
+
+    saltenv
+        Included only for compatibility with
+        :conf_minion:`pillarenv_from_saltenv`, and is otherwise ignored.
+
+        .. versionadded:: 2017.7.6,2018.3.1
+
     CLI Examples:
 
     .. code-block:: bash
@@ -377,11 +399,17 @@ def item(*args, **kwargs):
     ret = {}
     default = kwargs.get('default', '')
     delimiter = kwargs.get('delimiter', DEFAULT_TARGET_DELIM)
+    pillarenv = kwargs.get('pillarenv', None)
+    saltenv = kwargs.get('saltenv', None)
+
+    pillar_dict = __pillar__ \
+        if all(x is None for x in (saltenv, pillarenv)) \
+        else items(saltenv=saltenv, pillarenv=pillarenv)
 
     try:
         for arg in args:
             ret[arg] = salt.utils.data.traverse_dict_and_list(
-                __pillar__,
+                pillar_dict,
                 arg,
                 default,
                 delimiter)

salt/modules/sensehat.py

@@ -25,7 +25,6 @@ import logging
 
 try:
     from sense_hat import SenseHat
-    _sensehat = SenseHat()
     has_sense_hat = True
 except (ImportError, NameError):
     _sensehat = None
@@ -39,15 +38,20 @@ def __virtual__():
     Only load the module if SenseHat is available
     '''
     if has_sense_hat:
+        try:
+            _sensehat = SenseHat()
+        except OSError:
+            return False, 'This module can only be used on a Raspberry Pi with a SenseHat.'
+
         rotation = __salt__['pillar.get']('sensehat:rotation', 0)
         if rotation in [0, 90, 180, 270]:
             _sensehat.set_rotation(rotation, False)
         else:
-            log.error("%s is not a valid rotation. Using default rotation.",
+            log.error('%s is not a valid rotation. Using default rotation.',
                       rotation)
         return True
-    else:
-        return False, "The SenseHat excecution module can not be loaded: SenseHat unavailable.\nThis module can only be used on a Raspberry Pi with a SenseHat. Also make sure that the sense_hat python library is installed!"
+
+    return False, 'The SenseHat execution module cannot be loaded: \'sense_hat\' python library unavailable.'
 
 
 def set_pixels(pixels):

salt/modules/state.py

@@ -2344,9 +2344,10 @@ def event(tagmatch='*',
                 )
                 sys.stdout.flush()
 
-            if count > -1:
+            if count > 0:
                 count -= 1
                 log.debug('Remaining event matches: %s', count)
+
             if count == 0:
                 break
         else:

salt/modules/vault.py

@@ -108,10 +108,6 @@ Functions to interact with Hashicorp Vault.
 from __future__ import absolute_import, print_function, unicode_literals
 import logging
 
-# Import Salt libs
-import salt.crypt
-import salt.exceptions
-
 
 log = logging.getLogger(__name__)
 
@@ -146,7 +142,7 @@ def read_secret(path, key=None):
         return data
     except Exception as err:
         log.error('Failed to read secret! %s: %s', type(err).__name__, err)
-        raise salt.exceptions.CommandExecutionError(err)
+        return None
 
 
 def write_secret(path, **kwargs):
@@ -169,7 +165,7 @@ def write_secret(path, **kwargs):
         return True
     except Exception as err:
         log.error('Failed to write secret! %s: %s', type(err).__name__, err)
-        raise salt.exceptions.CommandExecutionError(err)
+        return False
 
 
 def delete_secret(path):
@@ -191,7 +187,7 @@ def delete_secret(path):
         return True
     except Exception as err:
         log.error('Failed to delete secret! %s: %s', type(err).__name__, err)
-        raise salt.exceptions.CommandExecutionError(err)
+        return False
 
 
 def list_secrets(path):
@@ -214,4 +210,4 @@ def list_secrets(path):
         return response.json()['data']
     except Exception as err:
         log.error('Failed to list secrets! %s: %s', type(err).__name__, err)
-        raise salt.exceptions.CommandExecutionError(err)
+        return None

salt/modules/win_dism.py

@@ -430,16 +430,27 @@ def add_package(package,
     Install a package using DISM
 
     Args:
-        package (str): The package to install. Can be a .cab file, a .msu file,
-            or a folder
-        ignore_check (Optional[bool]): Skip installation of the package if the
-            applicability checks fail
-        prevent_pending (Optional[bool]): Skip the installation of the package
-            if there are pending online actions
-        image (Optional[str]): The path to the root directory of an offline
-            Windows image. If `None` is passed, the running operating system is
-            targeted. Default is None.
-        restart (Optional[bool]): Reboot the machine if required by the install
+        package (str):
+            The package to install. Can be a .cab file, a .msu file, or a folder
+
+            .. note::
+                An `.msu` package is supported only when the target image is
+                offline, either mounted or applied.
+
+        ignore_check (Optional[bool]):
+            Skip installation of the package if the applicability checks fail
+
+        prevent_pending (Optional[bool]):
+            Skip the installation of the package if there are pending online
+            actions
+
+        image (Optional[str]):
+            The path to the root directory of an offline Windows image. If
+            ``None`` is passed, the running operating system is targeted.
+            Default is None.
+
+        restart (Optional[bool]):
+            Reboot the machine if required by the install
 
     Returns:
         dict: A dictionary containing the results of the command

salt/modules/win_lgpo.py

@@ -3440,7 +3440,7 @@ def _processValueItem(element, reg_key, reg_valuename, policy, parent_element,
                 this_element_value = b''.join([this_element_value.encode('utf-16-le'),
                                                encoded_null])
         elif etree.QName(element).localname == 'multiText':
-            this_vtype = 'REG_MULTI_SZ'
+            this_vtype = 'REG_MULTI_SZ' if not check_deleted else 'REG_SZ'
             if this_element_value is not None:
                 this_element_value = '{0}{1}{1}'.format(chr(0).join(this_element_value), chr(0))
         elif etree.QName(element).localname == 'list':
@@ -3449,7 +3449,7 @@ def _processValueItem(element, reg_key, reg_valuename, policy, parent_element,
             element_valuenames = []
             element_values = this_element_value
             if this_element_value is not None:
-                element_valuenames = list(range(1, len(this_element_value) + 1))
+                element_valuenames = list([str(z) for z in range(1, len(this_element_value) + 1)])
             if 'additive' in element.attrib:
                 if element.attrib['additive'].lower() == 'false':
                     # a delete values will be added before all the other
@@ -3474,11 +3474,18 @@ def _processValueItem(element, reg_key, reg_valuename, policy, parent_element,
                 if this_element_value is not None:
                     element_valuenames = this_element_value.keys()
                     element_values = this_element_value.values()
-
-            if 'valuePrefix' in element.attrib and element.attrib['valuePrefix'] != '':
-                if this_element_value is not None:
-                    element_valuenames = ['{0}{1}'.format(element.attrib['valuePrefix'],
-                                                          k) for k in element_valuenames]
+            if 'valuePrefix' in element.attrib:
+                # if the valuePrefix attribute exists, the valuenames are <prefix><number>
+                # most prefixes attributes are empty in the admx files, so the valuenames
+                # end up being just numbers
+                if element.attrib['valuePrefix'] != '':
+                    if this_element_value is not None:
+                        element_valuenames = ['{0}{1}'.format(element.attrib['valuePrefix'],
+                                                              k) for k in element_valuenames]
+            else:
+                # if there is no valuePrefix attribute, the valuename is the value
+                if element_values is not None:
+                    element_valuenames = [str(z) for z in element_values]
             if not check_deleted:
                 if this_element_value is not None:
                     log.debug('_processValueItem has an explicit '

salt/modules/win_pkg.py

@@ -1882,7 +1882,7 @@ def get_repo_data(saltenv='base'):
         serial = salt.payload.Serial(__opts__)
         with salt.utils.files.fopen(repo_details.winrepo_file, 'rb') as repofile:
             try:
-                repodata = salt.utils.data.decode(serial.loads(repofile.read(), encoding='utf-8') or {})
+                repodata = salt.utils.data.decode(serial.loads(repofile.read()) or {})
                 __context__['winrepo.data'] = repodata
                 return repodata
             except Exception as exc:

salt/modules/x509.py

@@ -20,6 +20,7 @@ import tempfile
 import re
 import datetime
 import ast
+import sys
 
 # Import salt libs
 import salt.utils.files
@@ -132,8 +133,8 @@ def _new_extension(name, value, critical=0, issuer=None, _pyfree=1):
             'value must be precomputed hash')
 
     # ensure name and value are bytes
-    name = salt.utils.stringutils.to_bytes(name)
-    value = salt.utils.stringutils.to_bytes(value)
+    name = salt.utils.stringutils.to_str(name)
+    value = salt.utils.stringutils.to_str(value)
 
     try:
         ctx = M2Crypto.m2.x509v3_set_nconf()
@@ -320,9 +321,9 @@ def _text_or_file(input_):
     '''
     if os.path.isfile(input_):
         with salt.utils.files.fopen(input_) as fp_:
-            return salt.utils.stringutils.to_bytes(fp_.read())
+            return salt.utils.stringutils.to_str(fp_.read())
     else:
-        return input_
+        return salt.utils.stringutils.to_str(input_)
 
 
 def _parse_subject(subject):
@@ -497,7 +498,7 @@ def get_pem_entry(text, pem_type=None):
         ret += pem_body[i:i + 64] + '\n'
     ret += pem_footer + '\n'
 
-    return ret.encode('ascii')
+    return salt.utils.stringutils.to_bytes(ret, encoding='ascii')
 
 
 def get_pem_entries(glob_path):
@@ -682,12 +683,12 @@ def get_public_key(key, passphrase=None, asObj=False):
 
     if isinstance(key, M2Crypto.X509.X509):
         rsa = key.get_pubkey().get_rsa()
-        text = ''
+        text = b''
     else:
         text = _text_or_file(key)
         text = get_pem_entry(text)
 
-    if text.startswith('-----BEGIN PUBLIC KEY-----'):
+    if text.startswith(b'-----BEGIN PUBLIC KEY-----'):
         if not asObj:
             return text
         bio = M2Crypto.BIO.MemoryBuffer()
@@ -695,14 +696,14 @@ def get_public_key(key, passphrase=None, asObj=False):
         rsa = M2Crypto.RSA.load_pub_key_bio(bio)
 
     bio = M2Crypto.BIO.MemoryBuffer()
-    if text.startswith('-----BEGIN CERTIFICATE-----'):
+    if text.startswith(b'-----BEGIN CERTIFICATE-----'):
         cert = M2Crypto.X509.load_cert_string(text)
         rsa = cert.get_pubkey().get_rsa()
-    if text.startswith('-----BEGIN CERTIFICATE REQUEST-----'):
+    if text.startswith(b'-----BEGIN CERTIFICATE REQUEST-----'):
         csr = M2Crypto.X509.load_request_string(text)
         rsa = csr.get_pubkey().get_rsa()
-    if (text.startswith('-----BEGIN PRIVATE KEY-----') or
-            text.startswith('-----BEGIN RSA PRIVATE KEY-----')):
+    if (text.startswith(b'-----BEGIN PRIVATE KEY-----') or
+            text.startswith(b'-----BEGIN RSA PRIVATE KEY-----')):
         rsa = M2Crypto.RSA.load_key_string(
             text, callback=_passphrase_callback(passphrase))
 
@@ -851,7 +852,7 @@ def create_private_key(path=None,
             pem_type='(?:RSA )?PRIVATE KEY'
         )
     else:
-        return bio.read_all()
+        return salt.utils.stringutils.to_str(bio.read_all())
 
 
 def create_crl(  # pylint: disable=too-many-arguments,too-many-locals
@@ -1432,7 +1433,14 @@ def create_certificate(
     if 'serial_number' not in kwargs:
         kwargs['serial_number'] = _dec2hex(
             random.getrandbits(kwargs['serial_bits']))
-    cert.set_serial_number(int(kwargs['serial_number'].replace(':', ''), 16))
+    serial_number = int(kwargs['serial_number'].replace(':', ''), 16)
+    # With Python3 we occasionally end up with an INT
+    # that is too large because Python3 no longer supports long INTs.
+    # If we're larger than the maxsize value
+    # then we adjust the serial number.
+    if serial_number > sys.maxsize:
+        serial_number = serial_number - sys.maxsize
+    cert.set_serial_number(serial_number)
 
     # Set validity dates
     # pylint: disable=no-member

salt/modules/yumpkg.py

@@ -2829,20 +2829,18 @@ def mod_repo(repo, basedir=None, **kwargs):
     filerepos[repo].update(repo_opts)
     content = header
     for stanza in six.iterkeys(filerepos):
-        comments = ''
-        if 'comments' in six.iterkeys(filerepos[stanza]):
-            comments = salt.utils.pkg.rpm.combine_comments(
-                    filerepos[stanza]['comments'])
-            del filerepos[stanza]['comments']
-        content += '\n[{0}]'.format(stanza)
+        comments = salt.utils.pkg.rpm.combine_comments(
+            filerepos[stanza].pop('comments', [])
+        )
+        content += '[{0}]\n'.format(stanza)
         for line in six.iterkeys(filerepos[stanza]):
-            content += '\n{0}={1}'.format(
+            content += '{0}={1}\n'.format(
                 line,
                 filerepos[stanza][line]
                     if not isinstance(filerepos[stanza][line], bool)
                     else _bool_to_str(filerepos[stanza][line])
             )
-        content += '\n{0}\n'.format(comments)
+        content += comments + '\n'
 
     with salt.utils.files.fopen(repofile, 'w') as fileout:
         fileout.write(salt.utils.stringutils.to_str(content))
@@ -2871,15 +2869,30 @@ def _parse_repo_file(filename):
         section_dict.pop('__name__', None)
         config[section] = section_dict
 
-    # Try to extract leading comments
+    # Try to extract header comments, as well as comments for each repo. Read
+    # from the beginning of the file and assume any leading comments are
+    # header comments. Continue to read each section header and then find the
+    # comments for each repo.
     headers = ''
-    with salt.utils.files.fopen(filename, 'r') as rawfile:
-        for line in rawfile:
+    section = None
+    with salt.utils.files.fopen(filename, 'r') as repofile:
+        for line in repofile:
             line = salt.utils.stringutils.to_unicode(line)
-            if line.strip().startswith('#'):
-                headers += '{0}\n'.format(line.strip())
-            else:
-                break
+            line = line.strip()
+            if line.startswith('#'):
+                if section is None:
+                    headers += line + '\n'
+                else:
+                    try:
+                        comments = config[section].setdefault('comments', [])
+                        comments.append(line[1:].lstrip())
+                    except KeyError:
+                        log.debug(
+                            'Found comment in %s which does not appear to '
+                            'belong to any repo section: %s', filename, line
+                        )
+            elif line.startswith('[') and line.endswith(']'):
+                section = line[1:-1]
 
     return (headers, salt.utils.data.decode(config))
 

salt/modules/zypper.py

@@ -309,7 +309,11 @@ class _Zypper(object):
         if self.error_msg and not self.__no_raise and not self.__ignore_repo_failure:
             raise CommandExecutionError('Zypper command failure: {0}'.format(self.error_msg))
 
-        return self._is_xml_mode() and dom.parseString(self.__call_result['stdout']) or self.__call_result['stdout']
+        return (
+            self._is_xml_mode() and
+            dom.parseString(salt.utils.stringutils.to_str(self.__call_result['stdout'])) or
+            self.__call_result['stdout']
+        )
 
 
 __zypper__ = _Zypper()
@@ -726,6 +730,123 @@ def list_pkgs(versions_as_list=False, **kwargs):
         attr)
 
 
+def list_repo_pkgs(*args, **kwargs):
+    '''
+    .. versionadded:: 2017.7.5,2018.3.1
+
+    Returns all available packages. Optionally, package names (and name globs)
+    can be passed and the results will be filtered to packages matching those
+    names. This is recommended as it speeds up the function considerably.
+
+    This function can be helpful in discovering the version or repo to specify
+    in a :mod:`pkg.installed <salt.states.pkg.installed>` state.
+
+    The return data will be a dictionary mapping package names to a list of
+    version numbers, ordered from newest to oldest. If ``byrepo`` is set to
+    ``True``, then the return dictionary will contain repository names at the
+    top level, and each repository will map packages to lists of version
+    numbers. For example:
+
+    .. code-block:: python
+
+        # With byrepo=False (default)
+        {
+            'bash': ['4.3-83.3.1',
+                     '4.3-82.6'],
+            'vim': ['7.4.326-12.1']
+        }
+        {
+            'OSS': {
+                'bash': ['4.3-82.6'],
+                'vim': ['7.4.326-12.1']
+            },
+            'OSS Update': {
+                'bash': ['4.3-83.3.1']
+            }
+        }
+
+    fromrepo : None
+        Only include results from the specified repo(s). Multiple repos can be
+        specified, comma-separated.
+
+    byrepo : False
+        When ``True``, the return data for each package will be organized by
+        repository.
+
+    CLI Examples:
+
+    .. code-block:: bash
+
+        salt '*' pkg.list_repo_pkgs
+        salt '*' pkg.list_repo_pkgs foo bar baz
+        salt '*' pkg.list_repo_pkgs 'python2-*' byrepo=True
+        salt '*' pkg.list_repo_pkgs 'python2-*' fromrepo='OSS Updates'
+    '''
+    byrepo = kwargs.pop('byrepo', False)
+    fromrepo = kwargs.pop('fromrepo', '') or ''
+    ret = {}
+
+    targets = [
+        arg if isinstance(arg, six.string_types) else six.text_type(arg)
+        for arg in args
+    ]
+
+    def _is_match(pkgname):
+        '''
+        When package names are passed to a zypper search, they will be matched
+        anywhere in the package name. This makes sure that only exact or
+        fnmatch matches are identified.
+        '''
+        if not args:
+            # No package names passed, everyone's a winner!
+            return True
+        for target in targets:
+            if fnmatch.fnmatch(pkgname, target):
+                return True
+        return False
+
+    for node in __zypper__.xml.call('se', '-s', *targets).getElementsByTagName('solvable'):
+        pkginfo = dict(node.attributes.items())
+        try:
+            if pkginfo['kind'] != 'package':
+                continue
+            reponame = pkginfo['repository']
+            if fromrepo and reponame != fromrepo:
+                continue
+            pkgname = pkginfo['name']
+            pkgversion = pkginfo['edition']
+        except KeyError:
+            continue
+        else:
+            if _is_match(pkgname):
+                repo_dict = ret.setdefault(reponame, {})
+                version_list = repo_dict.setdefault(pkgname, set())
+                version_list.add(pkgversion)
+
+    if byrepo:
+        for reponame in ret:
+            # Sort versions newest to oldest
+            for pkgname in ret[reponame]:
+                sorted_versions = sorted(
+                    [LooseVersion(x) for x in ret[reponame][pkgname]],
+                    reverse=True
+                )
+                ret[reponame][pkgname] = [x.vstring for x in sorted_versions]
+        return ret
+    else:
+        byrepo_ret = {}
+        for reponame in ret:
+            for pkgname in ret[reponame]:
+                byrepo_ret.setdefault(pkgname, []).extend(ret[reponame][pkgname])
+        for pkgname in byrepo_ret:
+            sorted_versions = sorted(
+                [LooseVersion(x) for x in byrepo_ret[pkgname]],
+                reverse=True
+            )
+            byrepo_ret[pkgname] = [x.vstring for x in sorted_versions]
+        return byrepo_ret
+
+
 def _get_configured_repos():
     '''
     Get all the info about repositories from the configurations.
@@ -1144,6 +1265,15 @@ def install(name=None,
         return {}
 
     version_num = Wildcard(__zypper__)(name, version)
+
+    if version_num:
+        if pkgs is None and sources is None:
+            # Allow "version" to work for single package target
+            pkg_params = {name: version_num}
+        else:
+            log.warning('"version" parameter will be ignored for multiple '
+                        'package targets')
+
     if pkg_type == 'repository':
         targets = []
         for param, version_num in six.iteritems(pkg_params):

salt/pillar/__init__.py

@@ -776,11 +776,22 @@ class Pillar(object):
                                             nstate = {
                                                 key_fragment: nstate
                                             }
-                                    include_states.append(nstate)
+                                    if not self.opts.get('pillar_includes_override_sls', False):
+                                        include_states.append(nstate)
+                                    else:
+                                        state = merge(
+                                            state,
+                                            nstate,
+                                            self.merge_strategy,
+                                            self.opts.get('renderer', 'yaml'),
+                                            self.opts.get('pillar_merge_lists', False))
                                 if err:
                                     errors += err
-                        if include_states:
-                            # merge included state(s) with the current state merged last
+
+                        if not self.opts.get('pillar_includes_override_sls', False):
+                            # merge included state(s) with the current state
+                            # merged last to ensure that its values are
+                            # authoritative.
                             include_states.append(state)
                             state = None
                             for s in include_states:

salt/proxy/cimc.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 '''
+Proxy Minion interface module for managing Cisco Integrated Management Controller devices
+=========================================================================================
 
-Proxy Minion interface module for managing Cisco Integrated Management Controller devices.
+.. versionadded:: 2018.3.0
 
 :codeauthor: :email:`Spencer Ervin <spencer_ervin@hotmail.com>`
 :maturity:   new
@@ -21,6 +23,7 @@ documentation.
 
 Configuration
 =============
+
 To use this integration proxy module, please configure the following:
 
 Pillar
@@ -40,6 +43,7 @@ the ID.
 
 proxytype
 ^^^^^^^^^
+
 The ``proxytype`` key and value pair is critical, as it tells Salt which
 interface to load from the ``proxy`` directory in Salt's install hierarchy,
 or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your
@@ -48,16 +52,18 @@ own proxy module, for example). To use this cimc Proxy Module, set this to
 
 host
 ^^^^
+
 The location, or ip/dns, of the cimc host. Required.
 
 username
 ^^^^^^^^
+
 The username used to login to the cimc host. Required.
 
 password
 ^^^^^^^^
-The password used to login to the cimc host. Required.
 
+The password used to login to the cimc host. Required.
 '''
 
 from __future__ import absolute_import, print_function, unicode_literals

salt/proxy/panos.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 '''
+Proxy Minion interface module for managing Palo Alto firewall devices
+=====================================================================
 
-Proxy Minion interface module for managing Palo Alto firewall devices.
+.. versionadded:: 2018.3.0
 
 :codeauthor: :email:`Spencer Ervin <spencer_ervin@hotmail.com>`
 :maturity:   new
@@ -22,6 +24,7 @@ documentation.
 
 Configuration
 =============
+
 To use this integration proxy module, please configure the following:
 
 Pillar
@@ -53,6 +56,7 @@ the device with username and password.
 
 proxytype
 ^^^^^^^^^
+
 The ``proxytype`` key and value pair is critical, as it tells Salt which
 interface to load from the ``proxy`` directory in Salt's install hierarchy,
 or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your
@@ -61,14 +65,17 @@ own proxy module, for example). To use this panos Proxy Module, set this to
 
 host
 ^^^^
+
 The location, or ip/dns, of the panos host. Required.
 
 username
 ^^^^^^^^
+
 The username used to login to the panos host. Required.
 
 password
 ^^^^^^^^
+
 The password used to login to the panos host. Required.
 
 Direct Device (API Key)
@@ -88,6 +95,7 @@ instead of username and password.
 
 proxytype
 ^^^^^^^^^
+
 The ``proxytype`` key and value pair is critical, as it tells Salt which
 interface to load from the ``proxy`` directory in Salt's install hierarchy,
 or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your
@@ -96,14 +104,16 @@ own proxy module, for example). To use this panos Proxy Module, set this to
 
 host
 ^^^^
+
 The location, or ip/dns, of the panos host. Required.
 
 apikey
-^^^^^^^^
+^^^^^^
+
 The generated XML API key for the panos host. Required.
 
 Panorama Pass-Through (Password)
-------------------------
+--------------------------------
 
 The Panorama pass-through method sends all connections through the Panorama
 management system. It passes the connections to the appropriate device using
@@ -126,6 +136,7 @@ not the panos device.
 
 proxytype
 ^^^^^^^^^
+
 The ``proxytype`` key and value pair is critical, as it tells Salt which
 interface to load from the ``proxy`` directory in Salt's install hierarchy,
 or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your
@@ -134,22 +145,26 @@ own proxy module, for example). To use this panos Proxy Module, set this to
 
 serial
 ^^^^^^
+
 The serial number of the panos host. Required.
 
 host
 ^^^^
+
 The location, or ip/dns, of the Panorama server. Required.
 
 username
 ^^^^^^^^
+
 The username used to login to the Panorama server. Required.
 
 password
 ^^^^^^^^
+
 The password used to login to the Panorama server. Required.
 
 Panorama Pass-Through (API Key)
-------------------------
+-------------------------------
 
 The Panorama server can also utilize a generated 'API key'_ for authentication.
 
@@ -165,6 +180,7 @@ The Panorama server can also utilize a generated 'API key'_ for authentication.
 
 proxytype
 ^^^^^^^^^
+
 The ``proxytype`` key and value pair is critical, as it tells Salt which
 interface to load from the ``proxy`` directory in Salt's install hierarchy,
 or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your
@@ -173,16 +189,18 @@ own proxy module, for example). To use this panos Proxy Module, set this to
 
 serial
 ^^^^^^
+
 The serial number of the panos host. Required.
 
 host
 ^^^^
+
 The location, or ip/dns, of the Panorama server. Required.
 
 apikey
 ^^^^^^^^
-The generated XML API key for the Panorama server. Required.
 
+The generated XML API key for the Panorama server. Required.
 '''
 
 from __future__ import absolute_import, print_function, unicode_literals

salt/returners/local_cache.py

@@ -441,7 +441,7 @@ def clean_old_jobs():
                     shutil.rmtree(f_path)
                 elif os.path.isfile(jid_file):
                     jid_ctime = os.stat(jid_file).st_ctime
-                    hours_difference = (time.time()- jid_ctime) / 3600.0
+                    hours_difference = (time.time() - jid_ctime) / 3600.0
                     if hours_difference > __opts__['keep_jobs'] and os.path.exists(t_path):
                         # Remove the entire f_path from the original JID dir
                         shutil.rmtree(f_path)

salt/runners/nacl.py

@@ -121,6 +121,7 @@ import os
 import salt.utils.files
 import salt.utils.platform
 import salt.utils.stringutils
+import salt.utils.versions
 import salt.utils.win_functions
 import salt.utils.win_dacl
 import salt.syspaths
@@ -219,7 +220,7 @@ def keygen(sk_file=None, pk_file=None, **kwargs):
 
     if 'keyfile' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'keyfile\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk_file\' argument instead.'
         )
@@ -288,7 +289,7 @@ def enc(data, **kwargs):
 
     if 'keyfile' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'keyfile\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk_file\' argument instead.'
         )
@@ -296,7 +297,7 @@ def enc(data, **kwargs):
 
     if 'key' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'key\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk\' argument instead.'
         )
@@ -353,7 +354,7 @@ def dec(data, **kwargs):
     '''
     if 'keyfile' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'keyfile\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk_file\' argument instead.'
         )
@@ -364,7 +365,7 @@ def dec(data, **kwargs):
 
     if 'key' in kwargs:
         salt.utils.versions.warn_until(
-            'Fluorine',
+            'Neon',
             'The \'key\' argument has been deprecated and will be removed in Salt '
             '{version}. Please use \'sk\' argument instead.'
         )

salt/runners/vault.py

@@ -40,7 +40,7 @@ def generate_token(minion_id, signature, impersonated_by_master=False):
         True. This happens when the master generates minion pillars.
     '''
     log.debug(
-        'Token generation request for %s (impersonated by master: %s)'.
+        'Token generation request for %s (impersonated by master: %s)',
         minion_id, impersonated_by_master
     )
     _validate_signature(minion_id, signature, impersonated_by_master)

salt/state.py

@@ -2371,7 +2371,8 @@ class State(object):
                     if not r_state.startswith('prerequired'):
                         req_stats.add('pre')
                 else:
-                    req_stats.add('met')
+                    if run_dict[tag].get('__state_ran__', True):
+                        req_stats.add('met')
             if r_state.endswith('_any'):
                 if 'met' in req_stats or 'change' in req_stats:
                     if 'fail' in req_stats:
@@ -2620,6 +2621,7 @@ class State(object):
                     '__run_num__': self.__run_num,
                     '__sls__': low['__sls__']
                 }
+                self.pre[tag] = running[tag]
             self.__run_num += 1
         elif status == 'change' and not low.get('__prereq__'):
             ret = self.call(low, chunks, running)
@@ -2649,6 +2651,7 @@ class State(object):
                             'duration': duration,
                             'start_time': start_time,
                             'comment': 'State was not run because onfail req did not change',
+                            '__state_ran__': False,
                             '__run_num__': self.__run_num,
                             '__sls__': low['__sls__']}
             self.__run_num += 1
@@ -2659,6 +2662,7 @@ class State(object):
                             'duration': duration,
                             'start_time': start_time,
                             'comment': 'State was not run because none of the onchanges reqs changed',
+                            '__state_ran__': False,
                             '__run_num__': self.__run_num,
                             '__sls__': low['__sls__']}
             self.__run_num += 1
@@ -3014,6 +3018,7 @@ class BaseHighState(object):
                     'top_file_merging_strategy set to \'same\', but no '
                     'default_top configuration option was set'
                 )
+            self.opts['environment'] = self.opts['default_top']
 
         if self.opts['saltenv']:
             contents = self.client.cache_file(

salt/states/environ.py

@@ -22,6 +22,15 @@ def __virtual__():
     return True
 
 
+def _norm_key(key):
+    '''
+    Normalize windows environment keys
+    '''
+    if salt.utils.platform.is_windows():
+        return key.upper()
+    return key
+
+
 def setenv(name,
            value,
            false_unsets=False,
@@ -126,12 +135,11 @@ def setenv(name,
                         permanent_hive = 'HKLM'
                         permanent_key = r'SYSTEM\CurrentControlSet\Control\Session Manager\Environment'
 
-                    out = __salt__['reg.read_value'](permanent_hive, permanent_key, key)
+                    out = __salt__['reg.read_value'](permanent_hive, permanent_key, _norm_key(key))
                     return out['success'] is True
                 else:
                     return False
-
-            if current_environ.get(key, None) is None and not key_exists():
+            if current_environ.get(_norm_key(key), None) is None and not key_exists():
                 # The key does not exist in environment
                 if false_unsets is not True:
                     # This key will be added with value ''
@@ -140,13 +148,13 @@ def setenv(name,
                 # The key exists.
                 if false_unsets is not True:
                     # Check to see if the value will change
-                    if current_environ.get(key, None) != '':
+                    if current_environ.get(_norm_key(key), None) != '':
                         # This key value will change to ''
                         ret['changes'].update({key: ''})
                 else:
                     # We're going to delete the key
                     ret['changes'].update({key: None})
-        elif current_environ.get(key, '') == val:
+        elif current_environ.get(_norm_key(key), '') == val:
             already_set.append(key)
         else:
             ret['changes'].update({key: val})

salt/states/file.py

@@ -619,28 +619,34 @@ def _check_file(name):
     return ret, msg
 
 
-def _clean_dir(root, keep, exclude_pat):
+def _find_keep_files(root, keep):
     '''
-    Clean out all of the files and directories in a directory (root) while
-    preserving the files in a list (keep) and part of exclude_pat
+    Compile a list of valid keep files (and directories).
     '''
-    removed = set()
     real_keep = set()
     real_keep.add(root)
     if isinstance(keep, list):
         for fn_ in keep:
             if not os.path.isabs(fn_):
                 continue
+            fn_ = os.path.normcase(os.path.abspath(fn_))
             real_keep.add(fn_)
             while True:
-                fn_ = os.path.dirname(fn_)
+                fn_ = os.path.abspath(os.path.dirname(fn_))
                 real_keep.add(fn_)
-                if fn_ in [
-                           os.sep,
-                           ''.join([os.path.splitdrive(fn_)[0], os.sep]),
-                           ''.join([os.path.splitdrive(fn_)[0], os.sep, os.sep])
-                          ]:
+                drive, path = os.path.splitdrive(fn_)
+                if not path.lstrip(os.sep):
                     break
+    return real_keep
+
+
+def _clean_dir(root, keep, exclude_pat):
+    '''
+    Clean out all of the files and directories in a directory (root) while
+    preserving the files in a list (keep) and part of exclude_pat
+    '''
+    real_keep = _find_keep_files(root, keep)
+    removed = set()
 
     def _delete_not_kept(nfn):
         if nfn not in real_keep:
@@ -5351,9 +5357,14 @@ def copy(
         subdir=False,
         **kwargs):
     '''
-    If the source file exists on the system, copy it to the named file. The
-    named file will not be overwritten if it already exists unless the force
-    option is set to True.
+    If the file defined by the ``source`` option exists on the minion, copy it
+    to the named path. The file will not be overwritten if it already exists,
+    unless the ``force`` option is set to ``True``.
+
+    .. note::
+        This state only copies files from one location on a minion to another
+        location on the same minion. For copying files from the master, use a
+        :py:func:`file.managed <salt.states.file.managed>` state.
 
     name
         The location of the file to copy to

salt/states/git.py

@@ -152,8 +152,10 @@ def _strip_exc(exc):
 def _uptodate(ret, target, comments=None, local_changes=False):
     ret['comment'] = 'Repository {0} is up-to-date'.format(target)
     if local_changes:
-        ret['comment'] += ', but with local changes. Set \'force_reset\' to ' \
-                          'True to purge local changes.'
+        ret['comment'] += (
+            ', but with uncommitted changes. Set \'force_reset\' to True to '
+            'purge uncommitted changes.'
+        )
     if comments:
         # Shouldn't be making any changes if the repo was up to date, but
         # report on them so we are alerted to potential problems with our
@@ -223,7 +225,7 @@ def _not_fast_forward(ret, rev, pre, post, branch, local_branch,
     return _fail(
         ret,
         'Repository would be updated {0}{1}, but {2}. Set \'force_reset\' to '
-        'True to force this update{3}.{4}'.format(
+        'True{3} to force this update{4}.{5}'.format(
             'from {0} to {1}'.format(pre, post)
                 if local_changes and pre != post
                 else 'to {0}'.format(post),
@@ -233,6 +235,7 @@ def _not_fast_forward(ret, rev, pre, post, branch, local_branch,
             'this is not a fast-forward merge'
                 if not local_changes
                 else 'there are uncommitted changes',
+            ' (or \'remote-changes\')' if local_changes else '',
             ' and discard these changes' if local_changes else '',
             branch_msg,
         ),
@@ -421,6 +424,11 @@ def latest(name,
         argument to ``True`` to force a hard-reset to the remote revision in
         these cases.
 
+        .. versionchanged:: Fluorine
+            This option can now be set to ``remote-changes``, which will
+            instruct Salt not to discard local changes if the repo is
+            up-to-date with the remote repository.
+
     submodules : False
         Update submodules on clone or branch change
 
@@ -532,21 +540,9 @@ def latest(name,
         cases.
 
         .. note::
-
-            On Windows, this option works slightly differently in the git state
-            and execution module than it does in the :mod:`"cmd" execution
-            module <salt.modules.cmdmod>`. The filenames in most git
-            repositories are created using a UTF-8 locale, and the system
-            encoding on Windows (CP1252) will successfully (but incorrectly)
-            decode many UTF-8 characters. This makes interacting with
-            repositories containing UTF-8 filenames on Windows unreliable.
-            Therefore, Windows will default to decoding the output from git
-            commands using UTF-8 unless this option is explicitly used to
-            specify the encoding.
-
-            On non-Windows platforms, the default output decoding behavior will
-            be observed (i.e. the encoding specified by the locale will be
-            tried first, and if that fails, UTF-8 will be used as a fallback).
+            This should only be needed if the files in the repository were
+            created with filenames using an encoding other than UTF-8 to handle
+            Unicode characters.
 
         .. versionadded:: 2018.3.1
 
@@ -622,6 +618,12 @@ def latest(name,
             '\'{0}\' is not a valid value for the \'rev\' argument'.format(rev)
         )
 
+    if force_reset not in (True, False, 'remote-changes'):
+        return _fail(
+            ret,
+            '\'force_reset\' must be one of True, False, or \'remote-changes\''
+        )
+
     # Ensure that certain arguments are strings to ensure that comparisons work
     if not isinstance(rev, six.string_types):
         rev = six.text_type(rev)
@@ -936,11 +938,13 @@ def latest(name,
                 local_changes = False
 
             if local_changes and revs_match:
-                if force_reset:
+                if force_reset is True:
                     msg = (
-                        '{0} is up-to-date, but with local changes. Since '
-                        '\'force_reset\' is enabled, these local changes '
-                        'would be reset.'.format(target)
+                        '{0} is up-to-date, but with uncommitted changes. '
+                        'Since \'force_reset\' is set to True, these local '
+                        'changes would be reset. To only reset when there are '
+                        'changes in the remote repository, set '
+                        '\'force_reset\' to \'remote-changes\'.'.format(target)
                     )
                     if __opts__['test']:
                         ret['changes']['forced update'] = True
@@ -950,9 +954,9 @@ def latest(name,
                     log.debug(msg.replace('would', 'will'))
                 else:
                     log.debug(
-                        '%s up-to-date, but with local changes. Since '
-                        '\'force_reset\' is disabled, no changes will be '
-                        'made.', target
+                        '%s up-to-date, but with uncommitted changes. Since '
+                        '\'force_reset\' is set to %s, no changes will be '
+                        'made.', target, force_reset
                     )
                     return _uptodate(ret,
                                      target,
@@ -1054,20 +1058,23 @@ def latest(name,
                         elif remote_rev_type == 'sha1':
                             has_remote_rev = True
 
-            # If fast_forward is not boolean, then we don't know if this will
-            # be a fast forward or not, because a fetch is required.
-            fast_forward = None if not local_changes else False
+            # If fast_forward is not boolean, then we don't yet know if this
+            # will be a fast forward or not, because a fetch is required.
+            fast_forward = False \
+                if (local_changes and force_reset != 'remote-changes') \
+                else None
 
             if has_remote_rev:
                 if (not revs_match and not update_head) \
                         and (branch is None or branch == local_branch):
-                    ret['comment'] = remote_loc.capitalize() \
-                        if rev == 'HEAD' \
-                        else remote_loc
-                    ret['comment'] += (
-                        ' is already present and local HEAD ({0}) does not '
+                    ret['comment'] = (
+                        '{0} is already present and local HEAD ({1}) does not '
                         'match, but update_head=False. HEAD has not been '
-                        'updated locally.'.format(local_rev[:7])
+                        'updated locally.'.format(
+                            remote_loc.capitalize() if rev == 'HEAD'
+                                else remote_loc,
+                            local_rev[:7]
+                        )
                     )
                     return ret
 
@@ -1081,9 +1088,8 @@ def latest(name,
                         # existed there and a remote was added and fetched, but
                         # the repository was not fast-forwarded. Regardless,
                         # going from no HEAD to a locally-present rev is
-                        # considered a fast-forward update, unless there are
-                        # local changes.
-                        fast_forward = not bool(local_changes)
+                        # considered a fast-forward update.
+                        fast_forward = True
                     else:
                         fast_forward = __salt__['git.merge_base'](
                             target,
@@ -1095,7 +1101,7 @@ def latest(name,
                             output_encoding=output_encoding)
 
             if fast_forward is False:
-                if not force_reset:
+                if force_reset is False:
                     return _not_fast_forward(
                         ret,
                         rev,
@@ -1439,7 +1445,10 @@ def latest(name,
                             password=password,
                             output_encoding=output_encoding)
 
-                    if fast_forward is False and not force_reset:
+                    if fast_forward is force_reset is False \
+                            or (fast_forward is True
+                                and local_changes
+                                and force_reset is False):
                         return _not_fast_forward(
                             ret,
                             rev,
@@ -1495,9 +1504,6 @@ def latest(name,
                             '\'{0}\' was checked out'.format(checkout_rev)
                         )
 
-                if local_changes:
-                    comments.append('Local changes were discarded')
-
                 if fast_forward is False:
                     __salt__['git.reset'](
                         target,
@@ -1506,9 +1512,20 @@ def latest(name,
                         password=password,
                         output_encoding=output_encoding)
                     ret['changes']['forced update'] = True
+                    if local_changes:
+                        comments.append('Uncommitted changes were discarded')
                     comments.append(
                         'Repository was hard-reset to {0}'.format(remote_loc)
                     )
+                elif fast_forward is True \
+                        and local_changes \
+                        and force_reset is not False:
+                    __salt__['git.discard_local_changes'](
+                        target,
+                        user=user,
+                        password=password,
+                        output_encoding=output_encoding)
+                    comments.append('Uncommitted changes were discarded')
 
                 if branch_opts is not None:
                     __salt__['git.branch'](
@@ -2032,21 +2049,9 @@ def present(name,
         cases.
 
         .. note::
-
-            On Windows, this option works slightly differently in the git state
-            and execution module than it does in the :mod:`"cmd" execution
-            module <salt.modules.cmdmod>`. The filenames in most git
-            repositories are created using a UTF-8 locale, and the system
-            encoding on Windows (CP1252) will successfully (but incorrectly)
-            decode many UTF-8 characters. This makes interacting with
-            repositories containing UTF-8 filenames on Windows unreliable.
-            Therefore, Windows will default to decoding the output from git
-            commands using UTF-8 unless this option is explicitly used to
-            specify the encoding.
-
-            On non-Windows platforms, the default output decoding behavior will
-            be observed (i.e. the encoding specified by the locale will be
-            tried first, and if that fails, UTF-8 will be used as a fallback).
+            This should only be needed if the files in the repository were
+            created with filenames using an encoding other than UTF-8 to handle
+            Unicode characters.
 
         .. versionadded:: 2018.3.1
 
@@ -2246,21 +2251,9 @@ def detached(name,
         cases.
 
         .. note::
-
-            On Windows, this option works slightly differently in the git state
-            and execution module than it does in the :mod:`"cmd" execution
-            module <salt.modules.cmdmod>`. The filenames in most git
-            repositories are created using a UTF-8 locale, and the system
-            encoding on Windows (CP1252) will successfully (but incorrectly)
-            decode many UTF-8 characters. This makes interacting with
-            repositories containing UTF-8 filenames on Windows unreliable.
-            Therefore, Windows will default to decoding the output from git
-            commands using UTF-8 unless this option is explicitly used to
-            specify the encoding.
-
-            On non-Windows platforms, the default output decoding behavior will
-            be observed (i.e. the encoding specified by the locale will be
-            tried first, and if that fails, UTF-8 will be used as a fallback).
+            This should only be needed if the files in the repository were
+            created with filenames using an encoding other than UTF-8 to handle
+            Unicode characters.
 
         .. versionadded:: 2018.3.1
     '''
@@ -2736,21 +2729,9 @@ def config_unset(name,
         cases.
 
         .. note::
-
-            On Windows, this option works slightly differently in the git state
-            and execution module than it does in the :mod:`"cmd" execution
-            module <salt.modules.cmdmod>`. The filenames in most git
-            repositories are created using a UTF-8 locale, and the system
-            encoding on Windows (CP1252) will successfully (but incorrectly)
-            decode many UTF-8 characters. This makes interacting with
-            repositories containing UTF-8 filenames on Windows unreliable.
-            Therefore, Windows will default to decoding the output from git
-            commands using UTF-8 unless this option is explicitly used to
-            specify the encoding.
-
-            On non-Windows platforms, the default output decoding behavior will
-            be observed (i.e. the encoding specified by the locale will be
-            tried first, and if that fails, UTF-8 will be used as a fallback).
+            This should only be needed if the files in the repository were
+            created with filenames using an encoding other than UTF-8 to handle
+            Unicode characters.
 
         .. versionadded:: 2018.3.1
 
@@ -3009,21 +2990,9 @@ def config_set(name,
         cases.
 
         .. note::
-
-            On Windows, this option works slightly differently in the git state
-            and execution module than it does in the :mod:`"cmd" execution
-            module <salt.modules.cmdmod>`. The filenames in most git
-            repositories are created using a UTF-8 locale, and the system
-            encoding on Windows (CP1252) will successfully (but incorrectly)
-            decode many UTF-8 characters. This makes interacting with
-            repositories containing UTF-8 filenames on Windows unreliable.
-            Therefore, Windows will default to decoding the output from git
-            commands using UTF-8 unless this option is explicitly used to
-            specify the encoding.
-
-            On non-Windows platforms, the default output decoding behavior will
-            be observed (i.e. the encoding specified by the locale will be
-            tried first, and if that fails, UTF-8 will be used as a fallback).
+            This should only be needed if the files in the repository were
+            created with filenames using an encoding other than UTF-8 to handle
+            Unicode characters.
 
         .. versionadded:: 2018.3.1
 

salt/states/mysql_query.py

@@ -57,6 +57,7 @@ def run_file(name,
         grain=None,
         key=None,
         overwrite=True,
+        check_db_exists=True,
         **connection_args):
     '''
     Execute an arbitrary query on the specified database
@@ -85,6 +86,10 @@ def run_file(name,
     overwrite:
         The file or grain will be overwritten if it already exists (default)
 
+    check_db_exists:
+        The state run will check that the specified database exists (default=True)
+        before running any queries
+
     .. versionadded:: 2017.7.0
     '''
     ret = {'name': name,
@@ -98,7 +103,7 @@ def run_file(name,
         return ret
 
     # check if database exists
-    if not __salt__['mysql.db_exists'](database, **connection_args):
+    if check_db_exists and not __salt__['mysql.db_exists'](database, **connection_args):
         err = _get_mysql_error()
         if err is not None:
             ret['comment'] = err
@@ -216,6 +221,7 @@ def run(name,
         grain=None,
         key=None,
         overwrite=True,
+        check_db_exists=False,
         **connection_args):
     '''
     Execute an arbitrary query on the specified database
@@ -243,13 +249,17 @@ def run(name,
 
     overwrite:
         The file or grain will be overwritten if it already exists (default)
+
+    check_db_exists:
+        The state run will check that the specified database exists (default=True)
+        before running any queries
     '''
     ret = {'name': name,
            'changes': {},
            'result': True,
            'comment': 'Database {0} is already present'.format(database)}
     # check if database exists
-    if not __salt__['mysql.db_exists'](database, **connection_args):
+    if check_db_exists and not __salt__['mysql.db_exists'](database, **connection_args):
         err = _get_mysql_error()
         if err is not None:
             ret['comment'] = err

salt/states/pkg.py

@@ -1009,11 +1009,11 @@ def installed(
         **WILDCARD VERSIONS**
 
         As of the 2017.7.0 release, this state now supports wildcards in
-        package versions for SUSE SLES/Leap/Tumbleweed, Debian/Ubuntu, RHEL/CentOS,
-        Arch Linux, and their derivatives. Using wildcards can be useful for
-        packages where the release name is built into the version in some way,
-        such as for RHEL/CentOS which typically has version numbers like
-        ``1.2.34-5.el7``. An example of the usage for this would be:
+        package versions for SUSE SLES/Leap/Tumbleweed, Debian/Ubuntu,
+        RHEL/CentOS, Arch Linux, and their derivatives. Using wildcards can be
+        useful for packages where the release name is built into the version in
+        some way, such as for RHEL/CentOS which typically has version numbers
+        like ``1.2.34-5.el7``. An example of the usage for this would be:
 
         .. code-block:: yaml
 
@@ -1021,6 +1021,11 @@ def installed(
               pkg.installed:
                 - version: '1.2.34*'
 
+        Keep in mind that using wildcard versions will result in a slower state
+        run since Salt must gather the available versions of the specified
+        packages and figure out which of them match the specified wildcard
+        expression.
+
     :param bool refresh:
         This parameter controls whether or not the package repo database is
         updated prior to installing the requested package(s).
@@ -2668,7 +2673,7 @@ def removed(name,
             .. code-block:: yaml
 
                 vim-enhanced:
-                  pkg.installed:
+                  pkg.removed:
                     - version: 2:7.4.160-1.el7
 
             In version 2015.8.9, an **ignore_epoch** argument has been added to
@@ -2774,7 +2779,7 @@ def purged(name,
             .. code-block:: yaml
 
                 vim-enhanced:
-                  pkg.installed:
+                  pkg.purged:
                     - version: 2:7.4.160-1.el7
 
             In version 2015.8.9, an **ignore_epoch** argument has been added to

salt/states/win_network.py

@@ -267,8 +267,13 @@ def managed(name,
         ret['comment'] = ' '.join(errors)
         return ret
 
+    try:
+        currently_enabled = __salt__['ip.is_enabled'](name)
+    except CommandExecutionError:
+        currently_enabled = False
+
     if not enabled:
-        if __salt__['ip.is_enabled'](name):
+        if currently_enabled:
             if __opts__['test']:
                 ret['result'] = None
                 ret['comment'] = ('Interface \'{0}\' will be disabled'
@@ -282,18 +287,13 @@ def managed(name,
             ret['comment'] += ' (already disabled)'
         return ret
     else:
-        try:
-            currently_enabled = __salt__['ip.is_disabled'](name)
-        except CommandExecutionError:
-            currently_enabled = False
         if not currently_enabled:
             if __opts__['test']:
                 ret['result'] = None
                 ret['comment'] = ('Interface \'{0}\' will be enabled'
                                   .format(name))
             else:
-                result = __salt__['ip.enable'](name)
-                if not result:
+                if not __salt__['ip.enable'](name):
                     ret['result'] = False
                     ret['comment'] = ('Failed to enable interface \'{0}\' to '
                                       'make changes'.format(name))

salt/states/x509.py

@@ -308,6 +308,9 @@ def private_key_managed(name,
         file_args['contents'] = __salt__['x509.create_private_key'](
             text=True, bits=bits, passphrase=passphrase, cipher=cipher, verbose=verbose)
 
+    # Ensure the key contents are a string before passing it along
+    file_args['contents'] = salt.utils.stringutils.to_str(file_args['contents'])
+
     ret = __states__['file.managed'](**file_args)
     if ret['changes'] and new_key:
         ret['changes'] = {'new': 'New private key generated'}

salt/templates/debian_ip/route_eth.jinja

@@ -2,5 +2,5 @@
 # {{route_type}}
 test "${IFACE}" = "{{iface}}" || exit 0
 {% for route in routes %}{% if route.name %}# {{route.name}}
-{%endif%}ip route {{route_type}} {% if route.ipaddr %}{{route.ipaddr}}{%endif%}{% if route.netmask %}/{{route.netmask}}{%endif%} {% if route.gateway %}via {{route.gateway}}{%endif%} dev {{iface}}
+{%endif%}ip route {{route_type}} {% if route.ipaddr %}{{route.ipaddr}}{%endif%}{% if route.netmask %}/{{route.netmask}}{%endif%} {% if route.gateway %}via {{route.gateway}}{%endif%} dev {{iface}}{% if route.metric %} metric {{route.metric}}{%endif%}
 {% endfor %}

salt/templates/rh_ip/rh6_route_eth.jinja

@@ -7,4 +7,6 @@
 {%- if route.gateway %} via {{route.gateway}}
 {%- else %} dev {{iface}}
 {%- endif %}
+{%- if route.metric %} metric {{route.metric}}
+{%- endif %}
 {% endfor -%}

salt/utils/event.py

@@ -439,7 +439,7 @@ class SaltEvent(object):
 
         if six.PY2:
             mtag, sep, mdata = raw.partition(TAGEND)  # split tag from data
-            data = serial.loads(mdata)
+            data = serial.loads(mdata, encoding='utf-8')
         else:
             mtag, sep, mdata = raw.partition(salt.utils.stringutils.to_bytes(TAGEND))  # split tag from data
             mtag = salt.utils.stringutils.to_str(mtag)

salt/utils/parsers.py

@@ -38,6 +38,7 @@ import salt.utils.platform
 import salt.utils.process
 import salt.utils.stringutils
 import salt.utils.user
+import salt.utils.win_functions
 import salt.utils.xdg
 import salt.utils.yaml
 from salt.defaults import DEFAULT_TARGET_DELIM
@@ -1020,11 +1021,11 @@ class DaemonMixIn(six.with_metaclass(MixInMeta, object)):
         if self.check_pidfile():
             pid = self.get_pidfile()
             if not salt.utils.platform.is_windows():
-                if self.check_pidfile() and self.is_daemonized(pid) and not os.getppid() == pid:
+                if self.check_pidfile() and self.is_daemonized(pid) and os.getppid() != pid:
                     return True
             else:
-                # We have no os.getppid() on Windows. Best effort.
-                if self.check_pidfile() and self.is_daemonized(pid):
+                # We have no os.getppid() on Windows. Use salt.utils.win_functions.get_parent_pid
+                if self.check_pidfile() and self.is_daemonized(pid) and salt.utils.win_functions.get_parent_pid() != pid:
                     return True
         return False
 

salt/utils/pkg/rpm.py

@@ -12,7 +12,6 @@ import subprocess
 
 # Import 3rd-party libs
 from salt.ext import six
-from salt.ext.six.moves import range  # pylint: disable=redefined-builtin
 
 log = logging.getLogger(__name__)
 
@@ -122,13 +121,13 @@ def combine_comments(comments):
     '''
     if not isinstance(comments, list):
         comments = [comments]
-    for idx in range(len(comments)):
-        if not isinstance(comments[idx], six.string_types):
-            comments[idx] = six.text_type(comments[idx])
-        comments[idx] = comments[idx].strip()
-        if not comments[idx].startswith('#'):
-            comments[idx] = '#' + comments[idx]
-    return '\n'.join(comments)
+    ret = []
+    for comment in comments:
+        if not isinstance(comment, six.string_types):
+            comment = str(comment)
+        # Normalize for any spaces (or lack thereof) after the #
+        ret.append('# {0}\n'.format(comment.lstrip('#').lstrip()))
+    return ''.join(ret)
 
 
 def version_to_evr(verstring):

salt/utils/process.py

@@ -231,7 +231,7 @@ def get_pidfile(pidfile):
             pid = pdf.read().strip()
         return int(pid)
     except (OSError, IOError, TypeError, ValueError):
-        return None
+        return -1
 
 
 def clean_proc(proc, wait_for_kill=10):

salt/utils/schedule.py

@@ -110,13 +110,15 @@ class Schedule(object):
         pass
 
     # an init for the singleton instance to call
-    def __singleton_init__(self, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None, standalone=False):
+    def __singleton_init__(self, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None, utils=None, standalone=False):
         self.opts = opts
         self.proxy = proxy
         self.functions = functions
+        self.utils = utils
         self.standalone = standalone
         self.skip_function = None
         self.skip_during_range = None
+        self.splay = None
         self.enabled = True
         if isinstance(intervals, dict):
             self.intervals = intervals
@@ -283,6 +285,7 @@ class Schedule(object):
         self.skip_function = None
         self.skip_during_range = None
         self.enabled = True
+        self.splay = None
         self.opts['schedule'] = {}
 
     def delete_job_prefix(self, name, persist=True):
@@ -595,10 +598,11 @@ class Schedule(object):
             # This also needed for ZeroMQ transport to reset all functions
             # context data that could keep paretns connections. ZeroMQ will
             # hang on polling parents connections from the child process.
+            utils = self.utils or salt.loader.utils(self.opts)
             if self.opts['__role'] == 'master':
-                self.functions = salt.loader.runner(self.opts)
+                self.functions = salt.loader.runner(self.opts, utils=utils)
             else:
-                self.functions = salt.loader.minion_mods(self.opts, proxy=self.proxy)
+                self.functions = salt.loader.minion_mods(self.opts, proxy=self.proxy, utils=utils)
             self.returners = salt.loader.returners(self.opts, self.functions, proxy=self.proxy)
         ret = {'id': self.opts.get('id', 'master'),
                'fun': func,
@@ -1297,10 +1301,13 @@ class Schedule(object):
             self.skip_during_range = schedule['skip_during_range']
         if 'enabled' in schedule:
             self.enabled = schedule['enabled']
+        if 'splay' in schedule:
+            self.splay = schedule['splay']
 
         _hidden = ['enabled',
                    'skip_function',
-                   'skip_during_range']
+                   'skip_during_range',
+                   'splay']
         for job, data in six.iteritems(schedule):
 
             # Skip anything that is a global setting
@@ -1404,7 +1411,12 @@ class Schedule(object):
 
             seconds = int((data['_next_fire_time'] - now).total_seconds())
 
-            if 'splay' in data:
+            # If there is no job specific splay available,
+            # grab the global which defaults to None.
+            if 'splay' not in data:
+                data['splay'] = self.splay
+
+            if 'splay' in data and data['splay']:
                 # Got "splay" configured, make decision to run a job based on that
                 if not data['_splay']:
                     # Try to add "splay" time only if next job fire time is
@@ -1613,6 +1625,7 @@ class Schedule(object):
                 # Restore our function references.
                 self.functions = functions
                 self.returners = returners
+                self.utils = utils
 
 
 def clean_proc_dir(opts):

salt/utils/stringutils.py

@@ -41,10 +41,11 @@ def to_bytes(s, encoding=None, errors='strict'):
                 return s.encode(encoding, errors)
             else:
                 try:
-                    return s.encode(__salt_system_encoding__, errors)
-                except UnicodeEncodeError:
-                    # Fall back to UTF-8
+                    # Try UTF-8 first
                     return s.encode('utf-8', errors)
+                except UnicodeEncodeError:
+                    # Fall back to detected encoding
+                    return s.encode(__salt_system_encoding__, errors)
         raise TypeError('expected bytes, bytearray, or str')
     else:
         return to_str(s, encoding, errors)
@@ -64,10 +65,11 @@ def to_str(s, encoding=None, errors='strict'):
                 return s.decode(encoding, errors)
             else:
                 try:
-                    return s.decode(__salt_system_encoding__, errors)
-                except UnicodeDecodeError:
-                    # Fall back to UTF-8
+                    # Try UTF-8 first
                     return s.decode('utf-8', errors)
+                except UnicodeDecodeError:
+                    # Fall back to detected encoding
+                    return s.decode(__salt_system_encoding__, errors)
         raise TypeError('expected str, bytes, or bytearray not {}'.format(type(s)))
     else:
         if isinstance(s, bytearray):
@@ -77,10 +79,11 @@ def to_str(s, encoding=None, errors='strict'):
                 return s.encode(encoding, errors)
             else:
                 try:
-                    return s.encode(__salt_system_encoding__, errors)
-                except UnicodeEncodeError:
-                    # Fall back to UTF-8
+                    # Try UTF-8 first
                     return s.encode('utf-8', errors)
+                except UnicodeEncodeError:
+                    # Fall back to detected encoding
+                    return s.encode(__salt_system_encoding__, errors)
         raise TypeError('expected str, bytearray, or unicode')
 
 
@@ -108,10 +111,11 @@ def to_unicode(s, encoding=None, errors='strict', normalize=False):
                 return _normalize(s.decode(encoding, errors))
             else:
                 try:
-                    return _normalize(s.decode(__salt_system_encoding__, errors))
-                except UnicodeDecodeError:
-                    # Fall back to UTF-8
+                    # Try UTF-8 first
                     return _normalize(s.decode('utf-8', errors))
+                except UnicodeDecodeError:
+                    # Fall back to detected encoding
+                    return _normalize(s.decode(__salt_system_encoding__, errors))
         raise TypeError('expected str or bytearray')
 
 
@@ -294,20 +298,29 @@ def build_whitespace_split_regex(text):
 
 def expr_match(line, expr):
     '''
-    Evaluate a line of text against an expression. First try a full-string
-    match, next try globbing, and then try to match assuming expr is a regular
-    expression. Originally designed to match minion IDs for
-    whitelists/blacklists.
+    Checks whether or not the passed value matches the specified expression.
+    Tries to match expr first as a glob using fnmatch.fnmatch(), and then tries
+    to match expr as a regular expression. Originally designed to match minion
+    IDs for whitelists/blacklists.
+
+    Note that this also does exact matches, as fnmatch.fnmatch() will return
+    ``True`` when no glob characters are used and the string is an exact match:
+
+    .. code-block:: python
+
+        >>> fnmatch.fnmatch('foo', 'foo')
+        True
     '''
-    if line == expr:
-        return True
-    if fnmatch.fnmatch(line, expr):
-        return True
     try:
-        if re.match(r'\A{0}\Z'.format(expr), line):
+        if fnmatch.fnmatch(line, expr):
             return True
-    except re.error:
-        pass
+        try:
+            if re.match(r'\A{0}\Z'.format(expr), line):
+                return True
+        except re.error:
+            pass
+    except TypeError:
+        log.exception('Value %r or expression %r is not a string', line, expr)
     return False
 
 
@@ -337,22 +350,16 @@ def check_whitelist_blacklist(value, whitelist=None, blacklist=None):
     if blacklist is not None:
         if not hasattr(blacklist, '__iter__'):
             blacklist = [blacklist]
-        try:
-            for expr in blacklist:
-                if expr_match(value, expr):
-                    return False
-        except TypeError:
-            log.error('Non-iterable blacklist %s', blacklist)
+        for expr in blacklist:
+            if expr_match(value, expr):
+                return False
 
     if whitelist:
         if not hasattr(whitelist, '__iter__'):
             whitelist = [whitelist]
-        try:
-            for expr in whitelist:
-                if expr_match(value, expr):
-                    return True
-        except TypeError:
-            log.error('Non-iterable whitelist %s', whitelist)
+        for expr in whitelist:
+            if expr_match(value, expr):
+                return True
     else:
         return True
 

salt/utils/templates.py

@@ -28,6 +28,7 @@ else:
 
 # Import Salt libs
 import salt.utils.data
+import salt.utils.dateutils
 import salt.utils.http
 import salt.utils.files
 import salt.utils.platform
@@ -372,7 +373,14 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
             decoded_context[key] = value
             continue
 
-        decoded_context[key] = salt.utils.locales.sdecode(value)
+        try:
+            decoded_context[key] = salt.utils.stringutils.to_unicode(value, encoding=SLS_ENCODING)
+        except UnicodeDecodeError as ex:
+            log.debug(
+                "Failed to decode using default encoding (%s), trying system encoding",
+                SLS_ENCODING,
+            )
+            decoded_context[key] = salt.utils.locales.sdecode(value)
 
     try:
         template = jinja_env.from_string(tmplstr)

tests/integration/cli/test_batch.py

@@ -18,7 +18,7 @@ class BatchTest(ShellCase):
         '''
         Tests executing a simple batch command to help catch regressions
         '''
-        ret = 'Executing run on [\'sub_minion\']'
+        ret = 'Executing run on [{0}]'.format(repr('sub_minion'))
 
         cmd = self.run_salt('\'*minion\' test.echo \'batch testing\' -b 50%')
         self.assertIn(ret, cmd)
@@ -28,7 +28,7 @@ class BatchTest(ShellCase):
         Tests executing a simple batch command using a number division instead of
         a percentage with full batch CLI call.
         '''
-        ret = "Executing run on ['minion', 'sub_minion']"
+        ret = "Executing run on [{0}, {1}]".format(repr('minion'), repr('sub_minion'))
         cmd = self.run_salt('\'*minion\' test.ping --batch-size 2')
         self.assertIn(ret, cmd)
 
@@ -38,8 +38,8 @@ class BatchTest(ShellCase):
         targeting.
         '''
         os_grain = ''
-        sub_min_ret = "Executing run on ['sub_minion']"
-        min_ret = "Executing run on ['minion']"
+        sub_min_ret = "Executing run on [{0}]".format(repr('sub_minion'))
+        min_ret = "Executing run on [{0}]".format(repr('minion'))
 
         for item in self.run_salt('minion grains.get os'):
             if item != 'minion':

tests/integration/cloud/providers/test_ec2.py

@@ -6,14 +6,18 @@
 # Import Python Libs
 from __future__ import absolute_import, print_function, unicode_literals
 import os
+import yaml
 
 # Import Salt Libs
 from salt.config import cloud_providers_config
+import salt.utils.files
 
 # Import Salt Testing Libs
 from tests.support.case import ShellCase
 from tests.support.paths import FILES
 from tests.support.helpers import expensiveTest, generate_random_name
+from tests.support.unit import expectedFailure
+from tests.support import win_installer
 
 # Create the cloud instance name to be used throughout the tests
 INSTANCE_NAME = generate_random_name('CLOUD-TEST-')
@@ -26,6 +30,38 @@ class EC2Test(ShellCase):
     '''
     Integration tests for the EC2 cloud provider in Salt-Cloud
     '''
+    TIMEOUT = 1000
+
+    def _installer_name(self):
+        '''
+        Determine the downloaded installer name by searching the files
+        directory for the firt file that loosk like an installer.
+        '''
+        for path, dirs, files in os.walk(FILES):
+            for file in files:
+                if file.startswith(win_installer.PREFIX):
+                    return file
+            break
+        return
+
+    def _fetch_latest_installer(self):
+        '''
+        Download the latest Windows installer executable
+        '''
+        name = win_installer.latest_installer_name()
+        path = os.path.join(FILES, name)
+        with salt.utils.files.fopen(path, 'wb') as fp:
+            win_installer.download_and_verify(fp, name)
+        return name
+
+    def _ensure_installer(self):
+        '''
+        Make sure the testing environment has a Windows installer executbale.
+        '''
+        name = self._installer_name()
+        if name:
+            return name
+        return self._fetch_latest_installer()
 
     @expensiveTest
     def setUp(self):
@@ -77,27 +113,51 @@ class EC2Test(ShellCase):
                 'missing. Check tests/integration/files/conf/cloud.providers.d/{0}.conf'
                 .format(PROVIDER_NAME)
             )
+        self.INSTALLER = self._ensure_installer()
 
-    def test_instance(self):
+    def override_profile_config(self, name, data):
+        conf_path = os.path.join(self.get_config_dir(), 'cloud.profiles.d', 'ec2.conf')
+        with salt.utils.files.fopen(conf_path, 'r') as fp:
+            conf = yaml.safe_load(fp)
+        conf[name].update(data)
+        with salt.utils.files.fopen(conf_path, 'w') as fp:
+            yaml.dump(conf, fp)
+
+    def copy_file(self, name):
+        '''
+        Copy a file from tests/integration/files to a test's temporary
+        configuration directory. The path to the file which is created will be
+        returned.
+        '''
+        src = os.path.join(FILES, name)
+        dst = os.path.join(self.get_config_dir(), name)
+        with salt.utils.files.fopen(src, 'rb') as sfp:
+            with salt.utils.files.fopen(dst, 'wb') as dfp:
+                dfp.write(sfp.read())
+        return dst
+
+    def _test_instance(self, profile='ec2-test', debug=False, timeout=TIMEOUT):
         '''
         Tests creating and deleting an instance on EC2 (classic)
         '''
+
         # create the instance
-        instance = self.run_cloud('-p ec2-test {0}'.format(INSTANCE_NAME),
-                                  timeout=EC2_TIMEOUT)
+        cmd = '-p {0}'.format(profile)
+        if debug:
+            cmd += ' -l debug'
+        cmd += ' {0}'.format(INSTANCE_NAME)
+        instance = self.run_cloud(cmd, timeout=timeout)
         ret_str = '{0}:'.format(INSTANCE_NAME)
 
         # check if instance returned with salt installed
         try:
             self.assertIn(ret_str, instance)
         except AssertionError:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME),
-                           timeout=EC2_TIMEOUT)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=timeout)
             raise
 
         # delete the instance
-        delete = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME),
-                                timeout=EC2_TIMEOUT)
+        delete = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=timeout)
         ret_str = '                    shutting-down'
 
         # check if deletion was performed appropriately
@@ -145,6 +205,80 @@ class EC2Test(ShellCase):
         # check if deletion was performed appropriately
         self.assertIn(ret_str, delete)
 
+    def test_instance(self):
+        '''
+        Tests creating and deleting an instance on EC2 (classic)
+        '''
+        self._test_instance('ec2-test')
+
+    @expectedFailure
+    def test_win2012r2_winexe(self):
+        '''
+        Tests creating and deleting a Windows 2012r2instance on EC2 using
+        winexe (classic)
+        '''
+        # TODO: winexe calls hang and the test fails by timing out. The same
+        # same calls succeed when run outside of the test environment.
+        self.override_profile_config(
+            'ec2-win2012-test',
+            {
+                'use_winrm': False,
+                'user_data': self.copy_file('windows-firewall-winexe.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+            },
+        )
+        self._test_instance('ec2-win2012r2-test', debug=True, timeout=500)
+
+    def test_win2012r2_winrm(self):
+        '''
+        Tests creating and deleting a Windows 2012r2 instance on EC2 using
+        winrm (classic)
+        '''
+        self.override_profile_config(
+            'ec2-win2016-test',
+            {
+                'user_data': self.copy_file('windows-firewall.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+                'winrm_ssl_verify': False,
+            }
+
+        )
+        self._test_instance('ec2-win2012r2-test', debug=True, timeout=500)
+
+    @expectedFailure
+    def test_win2016_winexe(self):
+        '''
+        Tests creating and deleting a Windows 2016 instance on EC2 using winrm
+        (classic)
+        '''
+        # TODO: winexe calls hang and the test fails by timing out. The same
+        # same calls succeed when run outside of the test environment.
+        self.override_profile_config(
+            'ec2-win2016-test',
+            {
+                'use_winrm': False,
+                'user_data': self.copy_file('windows-firewall-winexe.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+            },
+        )
+        self._test_instance('ec2-win2016-test', debug=True, timeout=500)
+
+    def test_win2016_winrm(self):
+        '''
+        Tests creating and deleting a Windows 2016 instance on EC2 using winrm
+        (classic)
+        '''
+        self.override_profile_config(
+            'ec2-win2016-test',
+            {
+                'user_data': self.copy_file('windows-firewall.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+                'winrm_ssl_verify': False,
+            }
+
+        )
+        self._test_instance('ec2-win2016-test', debug=True, timeout=500)
+
     def tearDown(self):
         '''
         Clean up after tests
@@ -154,5 +288,4 @@ class EC2Test(ShellCase):
 
         # if test instance is still present, delete it
         if ret_str in query:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME),
-                           timeout=EC2_TIMEOUT)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=self.TIMEOUT)

tests/integration/doc/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

tests/integration/doc/test_man.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+'''
+Tests for existence of manpages
+'''
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+import shutil
+
+# Import Salt libs
+import salt.utils.platform
+
+# Import Salt Testing libs
+from tests.support.case import ModuleCase
+from tests.support.paths import TMP
+from tests.support.unit import skipIf
+
+
+@skipIf(salt.utils.platform.is_windows(), 'minion is windows')
+class ManTest(ModuleCase):
+    rootdir = os.path.join(TMP, 'mantest')
+    # Map filenames to search strings which should be in the manpage
+    manpages = {
+        'salt-cp.1': [
+            'salt-cp Documentation',
+            'copies files from the master',
+        ],
+        'salt-cloud.1': [
+            'Salt Cloud Command',
+            'Provision virtual machines in the cloud',
+        ],
+        'salt-call.1': [
+            'salt-call Documentation',
+            'run module functions locally',
+        ],
+        'salt-api.1': [
+            'salt-api Command',
+            'Start interfaces used to remotely connect',
+        ],
+        'salt-unity.1': [
+            'salt-unity Command',
+            'unified invocation wrapper',
+        ],
+        'salt-syndic.1': [
+            'salt-syndic Documentation',
+            'Salt syndic daemon',
+        ],
+        'salt-ssh.1': [
+            'salt-ssh Documentation',
+            'executed using only SSH',
+        ],
+        'salt-run.1': [
+            'salt-run Documentation',
+            'frontend command for executing',
+        ],
+        'salt-proxy.1': [
+            'salt-proxy Documentation',
+            'proxies these commands',
+        ],
+        'salt-minion.1': [
+            'salt-minion Documentation',
+            'Salt minion daemon',
+        ],
+        'salt-master.1': [
+            'salt-master Documentation',
+            'Salt master daemon',
+        ],
+        'salt-key.1': [
+            'salt-key Documentation',
+            'management of Salt server public keys',
+        ],
+        'salt.1': [
+            'allows for commands to be executed',
+        ],
+        'salt.7': [
+            'Salt Documentation',
+        ],
+        'spm.1': [
+            'Salt Package Manager Command',
+            'command for managing Salt packages',
+        ],
+    }
+
+    def setUp(self):
+        if not self.run_function('mantest.install', [self.rootdir]):
+            self.fail('Failed to install salt to {0}'.format(self.rootdir))
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            shutil.rmtree(cls.rootdir)
+        except OSError:
+            pass
+
+    def test_man(self):
+        '''
+        Make sure that man pages are installed
+        '''
+        ret = self.run_function('mantest.search', [self.manpages, self.rootdir])
+        # The above function returns True if successful and an exception (which
+        # will manifest in the return as a stringified exception) if
+        # unsuccessful. Therefore, a simple assertTrue is not sufficient.
+        if ret is not True:
+            self.fail(ret)

tests/integration/files/conf/cloud.profiles.d/ec2.conf

@@ -4,3 +4,31 @@ ec2-test:
   size: t1.micro
   sh_username: ec2-user
   script_args: '-P -Z'
+ec2-win2012r2-test:
+  provider: ec2-config
+  size: t2.micro
+  image: ami-eb1ecd96
+  smb_port: 445
+  win_installer: ''
+  win_username: Administrator
+  win_password: auto
+  userdata_file: ''
+  userdata_template: False
+  use_winrm: True
+  winrm_verify_ssl: False
+  ssh_interface: private_ips
+  deploy: True
+ec2-win2016-test:
+  provider: ec2-config
+  size: t2.micro
+  image: ami-ed14c790
+  smb_port: 445
+  win_installer: ''
+  win_username: Administrator
+  win_password: auto
+  userdata_file: ''
+  userdata_template: False
+  use_winrm: True
+  winrm_verify_ssl: False
+  ssh_interface: private_ips
+  deploy: True

tests/integration/files/file/base/_modules/mantest.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+'''
+Helpers for testing man pages
+'''
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+import sys
+
+# Import Salt libs
+import salt.utils.files
+import salt.utils.path
+import salt.utils.stringutils
+from salt.exceptions import CommandExecutionError
+
+# Import Salt Tesing libs
+from tests.support.paths import CODE_DIR
+
+
+def install(rootdir):
+    if not os.path.exists(rootdir):
+        os.makedirs(rootdir)
+        return __salt__['cmd.retcode'](
+            [sys.executable,
+             os.path.join(CODE_DIR, 'setup.py'),
+             'install', '--root={0}'.format(rootdir)]) == 0
+    return True
+
+
+def search(manpages, rootdir):
+    manpage_fns = set(manpages)
+    manpage_paths = {}
+    for root, _, files in os.walk(rootdir):
+        if not manpage_fns:
+            # All manpages found, no need to keep walking
+            break
+        # Using list because we will be modifying the set during iteration
+        for manpage_fn in list(manpage_fns):
+            if manpage_fn in files:
+                manpage_path = salt.utils.path.join(root, manpage_fn)
+                manpage_paths[manpage_fn] = manpage_path
+                manpage_fns.remove(manpage_fn)
+
+    if manpage_fns:
+        raise CommandExecutionError(
+            'The following manpages were not found under {0}: {1}'.format(
+                rootdir,
+                ', '.join(sorted(manpage_fns))
+            )
+        )
+
+    failed = {}
+    for manpage in sorted(manpages):
+        with salt.utils.files.fopen(manpage_paths[manpage]) as fp_:
+            contents = salt.utils.stringutils.to_unicode(fp_.read())
+        # Check for search string in contents
+        for search_string in manpages[manpage]:
+            if search_string not in contents:
+                failed.setdefault(manpage, []).append(
+                    'No match for search string \'{0}\' found in {1}'.format(
+                        search_string, manpage_paths[manpage]
+                    )
+                )
+        # Check for correct install dir
+        path = '/man{0}/'.format(manpage.rsplit('.', 1)[-1])
+        if path not in manpage_paths[manpage]:
+            failed.setdefault(manpage, []).append(
+                '{0} not found in manpage path {1}'.format(
+                    path, manpage_paths[manpage]
+                )
+            )
+
+    if failed:
+        raise CommandExecutionError('One or more manpages failed', info=failed)
+
+    return True

tests/integration/files/file/base/_modules/runtests_helpers.py

@@ -66,6 +66,8 @@ def get_invalid_docs():
         'log.warning',
         'lowpkg.bin_pkg_info',
         'lxc.run_cmd',
+        'mantest.install',
+        'mantest.search',
         'nspawn.restart',
         'nspawn.stop',
         'pkg.expand_repo_def',

tests/integration/files/file/base/issue-46672.sls

@@ -0,0 +1,3 @@
+echo1:
+  cmd.run:
+    - name: "echo 'This is Æ test!'"

tests/integration/files/file/base/requisites/onfail_multiple_required.sls

@@ -0,0 +1,25 @@
+a:
+  cmd.run:
+    - name: exit 1
+
+b:
+  cmd.run:
+    - name: echo b
+    - onfail:
+      - cmd: a
+
+c:
+  cmd.run:
+    - name: echo c
+    - onfail:
+      - cmd: a
+    - require:
+      - cmd: b
+
+d:
+  cmd.run:
+    - name: echo d
+    - onfail:
+      - cmd: a
+    - require:
+      - cmd: c

tests/integration/files/file/base/requisites/onfail_multiple_required_no_run.sls

@@ -0,0 +1,25 @@
+a:
+  cmd.run:
+    - name: exit 0
+
+b:
+  cmd.run:
+    - name: echo b
+    - onfail:
+      - cmd: a
+
+c:
+  cmd.run:
+    - name: echo c
+    - onfail:
+      - cmd: a
+    - require:
+      - cmd: b
+
+d:
+  cmd.run:
+    - name: echo d
+    - onfail:
+      - cmd: a
+    - require:
+      - cmd: c

tests/integration/files/windows-firewall-winexe.ps1

@@ -0,0 +1,5 @@
+<powershell>
+New-NetFirewallRule -Name "SMB445" -DisplayName "SMB445" -Protocol TCP -LocalPort 445
+Set-Item (dir wsman:\localhost\Listener\*\Port -Recurse).pspath 445 -Force
+Restart-Service winrm
+</powershell>

tests/integration/files/windows-firewall.ps1

@@ -0,0 +1,33 @@
+<powershell>
+New-NetFirewallRule -Name "SMB445" -DisplayName "SMB445" -Protocol TCP -LocalPort 445
+New-NetFirewallRule -Name "WINRM5986" -DisplayName "WINRM5986" -Protocol TCP -LocalPort 5986
+
+winrm quickconfig -q
+winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="300"}'
+winrm set winrm/config '@{MaxTimeoutms="1800000"}'
+winrm set winrm/config/service/auth '@{Basic="true"}'
+
+$SourceStoreScope = 'LocalMachine'
+$SourceStorename = 'Remote Desktop'
+
+$SourceStore = New-Object  -TypeName System.Security.Cryptography.X509Certificates.X509Store  -ArgumentList $SourceStorename, $SourceStoreScope
+$SourceStore.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadOnly)
+
+$cert = $SourceStore.Certificates | Where-Object  -FilterScript {
+    $_.subject -like '*'
+}
+
+$DestStoreScope = 'LocalMachine'
+$DestStoreName = 'My'
+
+$DestStore = New-Object  -TypeName System.Security.Cryptography.X509Certificates.X509Store  -ArgumentList $DestStoreName, $DestStoreScope
+$DestStore.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite)
+$DestStore.Add($cert)
+
+$SourceStore.Close()
+$DestStore.Close()
+
+winrm create winrm/config/listener?Address=*+Transport=HTTPS  `@`{Hostname=`"($certId)`"`;CertificateThumbprint=`"($cert.Thumbprint)`"`}
+
+Restart-Service winrm
+</powershell>

tests/integration/modules/test_cmdmod.py

@@ -15,9 +15,11 @@ from tests.support.helpers import (
     skip_if_not_root
 )
 from tests.support.paths import TMP
+from tests.support.unit import skipIf
 
 # Import salt libs
 import salt.utils.path
+import salt.utils.platform
 
 # Import 3rd-party libs
 from salt.ext import six
@@ -289,6 +291,15 @@ class CMDModuleTest(ModuleCase):
                                    runas=runas).strip()
         self.assertEqual(result, expected_result)
 
+    @skipIf(salt.utils.platform.is_windows(), 'minion is windows')
+    @skip_if_not_root
+    def test_runas(self):
+        '''
+        Ensure that the env is the runas user's
+        '''
+        out = self.run_function('cmd.run', ['env'], runas='nobody').splitlines()
+        self.assertIn('USER=nobody', out)
+
     def test_timeout(self):
         '''
         cmd.run trigger timeout

tests/integration/modules/test_cp.py

@@ -14,7 +14,10 @@ import textwrap
 
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
-from tests.support.helpers import get_unused_localhost_port, skip_if_not_root
+from tests.support.helpers import (
+    get_unused_localhost_port,
+    skip_if_not_root,
+    with_tempfile)
 from tests.support.unit import skipIf
 import tests.support.paths as paths
 
@@ -41,11 +44,11 @@ class CPModuleTest(ModuleCase):
             super(CPModuleTest, self).run_function(*args, **kwargs)
         )
 
-    def test_get_file(self):
+    @with_tempfile
+    def test_get_file(self, tgt):
         '''
         cp.get_file
         '''
-        tgt = os.path.join(paths.TMP, 'scene33')
         self.run_function(
                 'cp.get_file',
                 [
@@ -73,11 +76,11 @@ class CPModuleTest(ModuleCase):
         self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
         self.assertNotIn('bacon', data)
 
-    def test_get_file_templated_paths(self):
+    @with_tempfile
+    def test_get_file_templated_paths(self, tgt):
         '''
         cp.get_file
         '''
-        tgt = os.path.join(paths.TMP, 'cheese')
         self.run_function(
             'cp.get_file',
             [
@@ -91,11 +94,11 @@ class CPModuleTest(ModuleCase):
         self.assertIn('Gromit', data)
         self.assertNotIn('bacon', data)
 
-    def test_get_file_gzipped(self):
+    @with_tempfile
+    def test_get_file_gzipped(self, tgt):
         '''
         cp.get_file
         '''
-        tgt = os.path.join(paths.TMP, 'file.big')
         src = os.path.join(paths.FILES, 'file', 'base', 'file.big')
         with salt.utils.files.fopen(src, 'rb') as fp_:
             hash_str = hashlib.md5(fp_.read()).hexdigest()
@@ -134,11 +137,11 @@ class CPModuleTest(ModuleCase):
         self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
         self.assertNotIn('bacon', data)
 
-    def test_get_template(self):
+    @with_tempfile
+    def test_get_template(self, tgt):
         '''
         cp.get_template
         '''
-        tgt = os.path.join(paths.TMP, 'scene33')
         self.run_function(
                 'cp.get_template',
                 ['salt://grail/scene33', tgt],
@@ -183,11 +186,11 @@ class CPModuleTest(ModuleCase):
 
     # cp.get_url tests
 
-    def test_get_url(self):
+    @with_tempfile
+    def test_get_url(self, tgt):
         '''
         cp.get_url with salt:// source given
         '''
-        tgt = os.path.join(paths.TMP, 'scene33')
         self.run_function(
             'cp.get_url',
             [
@@ -274,11 +277,11 @@ class CPModuleTest(ModuleCase):
         self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
         self.assertNotIn('bacon', data)
 
-    def test_get_url_https(self):
+    @with_tempfile
+    def test_get_url_https(self, tgt):
         '''
         cp.get_url with https:// source given
         '''
-        tgt = os.path.join(paths.TMP, 'test_get_url_https')
         self.run_function(
             'cp.get_url',
             [
@@ -616,7 +619,8 @@ class CPModuleTest(ModuleCase):
             self.assertEqual(
                 sha256_hash['hsum'], hashlib.sha256(data).hexdigest())
 
-    def test_get_file_from_env_predefined(self):
+    @with_tempfile
+    def test_get_file_from_env_predefined(self, tgt):
         '''
         cp.get_file
         '''
@@ -630,7 +634,8 @@ class CPModuleTest(ModuleCase):
         finally:
             os.unlink(tgt)
 
-    def test_get_file_from_env_in_url(self):
+    @with_tempfile
+    def test_get_file_from_env_in_url(self, tgt):
         tgt = os.path.join(paths.TMP, 'cheese')
         try:
             self.run_function('cp.get_file', ['salt://cheese?saltenv=prod', tgt])

tests/integration/modules/test_git.py

@@ -147,8 +147,7 @@ class GitModuleTest(ModuleCase):
         TODO: maybe move this behavior to ModuleCase itself?
         '''
         return salt.utils.data.decode(
-            super(GitModuleTest, self).run_function(*args, **kwargs),
-            encoding='utf-8'
+            super(GitModuleTest, self).run_function(*args, **kwargs)
         )
 
     def tearDown(self):
@@ -207,8 +206,7 @@ class GitModuleTest(ModuleCase):
             self.run_function('cmd.run', ['cp ' + tar_archive + ' /root/'])
             with closing(tarfile.open(tar_archive, 'r')) as tar_obj:
                 self.assertEqual(
-                    sorted(salt.utils.data.decode(tar_obj.getnames(),
-                                                  encoding='utf-8')),
+                    sorted(salt.utils.data.decode(tar_obj.getnames())),
                     sorted([
                         'foo', 'foo/bar', 'foo/baz', 'foo/foo', 'foo/питон',
                         'foo/qux', 'foo/qux/bar', 'foo/qux/baz', 'foo/qux/foo',
@@ -238,8 +236,7 @@ class GitModuleTest(ModuleCase):
             self.assertTrue(tarfile.is_tarfile(tar_archive))
             with closing(tarfile.open(tar_archive, 'r')) as tar_obj:
                 self.assertEqual(
-                    sorted(salt.utils.data.decode(tar_obj.getnames(),
-                                                  encoding='utf-8')),
+                    sorted(salt.utils.data.decode(tar_obj.getnames())),
                     sorted(['foo', 'foo/bar', 'foo/baz', 'foo/foo', 'foo/питон'])
                 )
         finally:

tests/integration/modules/test_groupadd.py

@@ -9,11 +9,13 @@ import string
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
 from tests.support.helpers import destructiveTest, skip_if_not_root
+from tests.support.unit import skipIf
 
 # Import Salt libs
 from salt.ext import six
 from salt.ext.six.moves import range
 import salt.utils.files
+import salt.utils.platform
 import salt.utils.stringutils
 
 
@@ -34,13 +36,13 @@ class GroupModuleTest(ModuleCase):
         self._no_user = self.__random_string()
         self._group = self.__random_string()
         self._no_group = self.__random_string()
-        self._gid = 64989
-        self._new_gid = 64998
-        os_grain = self.run_function('grains.item', ['kernel'])
-        if os_grain['kernel'] not in 'Linux':
+        self.os_grain = self.run_function('grains.item', ['kernel'])
+        self._gid = 64989 if 'Windows' not in self.os_grain['kernel'] else None
+        self._new_gid = 64998 if 'Windows' not in self.os_grain['kernel'] else None
+        if self.os_grain['kernel'] not in ('Linux', 'Windows'):
             self.skipTest(
                 'Test not applicable to \'{kernel}\' kernel'.format(
-                    **os_grain
+                    **self.os_grain
                 )
             )
 
@@ -110,12 +112,18 @@ class GroupModuleTest(ModuleCase):
         Test the add group function
         '''
         # add a new group
-        self.assertTrue(self.run_function('group.add', [self._group, self._gid]))
+        self.assertTrue(self.run_function('group.add', [self._group], gid=self._gid))
         group_info = self.run_function('group.info', [self._group])
-        self.assertEqual(group_info['name'], self._group)
         self.assertEqual(group_info['gid'], self._gid)
+        self.assertEqual(group_info['name'], self._group)
         # try adding the group again
-        self.assertFalse(self.run_function('group.add', [self._group, self._gid]))
+        if self.os_grain['kernel'] == 'Windows':
+            add_group = self.run_function('group.add', [self._group], gid=self._gid)
+            self.assertEqual(add_group['result'], None)
+            self.assertEqual(add_group['comment'], 'The group {0} already exists.'.format(self._group))
+            self.assertEqual(add_group['changes'], [])
+        else:
+            self.assertFalse(self.run_function('group.add', [self._group], gid=self._gid))
 
     @destructiveTest
     def test_add_system_group(self):
@@ -164,26 +172,32 @@ class GroupModuleTest(ModuleCase):
         self.assertTrue(self.run_function('group.delete', [self._group]))
 
         # group does not exist
-        self.assertFalse(self.run_function('group.delete', [self._no_group]))
+        if self.os_grain['kernel'] == 'Windows':
+            del_group = self.run_function('group.delete', [self._no_group])
+            self.assertEqual(del_group['changes'], [])
+            self.assertEqual(del_group['comment'], 'The group {0} does not exists.'.format(self._no_group))
+        else:
+            self.assertFalse(self.run_function('group.delete', [self._no_group]))
 
     def test_info(self):
         '''
         Test the info group function
         '''
-        self.run_function('group.add', [self._group, self._gid])
+        self.run_function('group.add', [self._group], gid=self._gid)
         self.run_function('user.add', [self._user])
         self.run_function('group.adduser', [self._group, self._user])
         group_info = self.run_function('group.info', [self._group])
 
         self.assertEqual(group_info['name'], self._group)
         self.assertEqual(group_info['gid'], self._gid)
-        self.assertIn(self._user, group_info['members'])
+        self.assertIn(self._user, str(group_info['members']))
 
+    @skipIf(salt.utils.platform.is_windows(), 'gid test skipped on windows')
     def test_chgid(self):
         '''
         Test the change gid function
         '''
-        self.run_function('group.add', [self._group, self._gid])
+        self.run_function('group.add', [self._group], gid=self._gid)
         self.assertTrue(self.run_function('group.chgid', [self._group, self._new_gid]))
         group_info = self.run_function('group.info', [self._group])
         self.assertEqual(group_info['gid'], self._new_gid)
@@ -192,47 +206,55 @@ class GroupModuleTest(ModuleCase):
         '''
         Test the add user to group function
         '''
-        self.run_function('group.add', [self._group, self._gid])
+        self.run_function('group.add', [self._group], gid=self._gid)
         self.run_function('user.add', [self._user])
         self.assertTrue(self.run_function('group.adduser', [self._group, self._user]))
         group_info = self.run_function('group.info', [self._group])
-        self.assertIn(self._user, group_info['members'])
-        # try to add a non existing user
-        self.assertFalse(self.run_function('group.adduser', [self._group, self._no_user]))
-        # try to add a user to non existing group
-        self.assertFalse(self.run_function('group.adduser', [self._no_group, self._user]))
-        # try to add a non existing user to a non existing group
-        self.assertFalse(self.run_function('group.adduser', [self._no_group, self._no_user]))
+        self.assertIn(self._user, str(group_info['members']))
+        if self.os_grain['kernel'] == 'Windows':
+            no_group = self.run_function('group.adduser', [self._no_group, self._no_user])
+            no_user = self.run_function('group.adduser', [self._group, self._no_user])
+            funcs = [no_group, no_user]
+            for func in funcs:
+                self.assertIn('Fail', func['comment'])
+                self.assertFalse(func['result'])
+        else:
+            # try add a non existing user
+            self.assertFalse(self.run_function('group.adduser', [self._group, self._no_user]))
+            # try add a user to non existing group
+            self.assertFalse(self.run_function('group.adduser', [self._no_group, self._user]))
+            # try add a non existing user to a non existing group
+            self.assertFalse(self.run_function('group.adduser', [self._no_group, self._no_user]))
 
     def test_deluser(self):
         '''
         Test the delete user from group function
         '''
-        self.run_function('group.add', [self._group, self._gid])
+        self.run_function('group.add', [self._group], gid=self._gid)
         self.run_function('user.add', [self._user])
         self.run_function('group.adduser', [self._group, self._user])
         self.assertTrue(self.run_function('group.deluser', [self._group, self._user]))
         group_info = self.run_function('group.info', [self._group])
-        self.assertNotIn(self._user, group_info['members'])
+        self.assertNotIn(self._user, str(group_info['members']))
 
     def test_members(self):
         '''
         Test the members function
         '''
-        self.run_function('group.add', [self._group, self._gid])
+        self.run_function('group.add', [self._group], gid=self._gid)
         self.run_function('user.add', [self._user])
         self.run_function('user.add', [self._user1])
         m = '{0},{1}'.format(self._user, self._user1)
         self.assertTrue(self.run_function('group.members', [self._group, m]))
         group_info = self.run_function('group.info', [self._group])
-        self.assertIn(self._user, group_info['members'])
-        self.assertIn(self._user1, group_info['members'])
+        self.assertIn(self._user, str(group_info['members']))
+        self.assertIn(self._user1, str(group_info['members']))
 
     def test_getent(self):
         '''
         Test the getent function
         '''
-        self.run_function('group.add', [self._group, self._gid])
+        self.run_function('group.add', [self._group], gid=self._gid)
         self.run_function('user.add', [self._user])
         self.run_function('group.adduser', [self._group, self._user])
         ginfo = self.run_function('user.getent')

tests/integration/modules/test_pkg.py

@@ -272,38 +272,53 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
         self.run_function('pkg.refresh_db')
 
         if os_family == 'Suse':
-            # pkg.latest version returns empty if the latest version is already installed
-            vim_version_dict = self.run_function('pkg.latest_version', ['vim'])
-            vim_info = self.run_function('pkg.info_available', ['vim'])['vim']
-            if vim_version_dict == {}:
-                # Latest version is installed, get its version and construct
-                # a version selector so the immediately previous version is selected
-                vim_version = 'version=<'+vim_info['version']
-            else:
-                # Vim was not installed, so pkg.latest_version returns the latest one.
-                # Construct a version selector so immediately previous version is selected
-                vim_version = 'version=<'+vim_version_dict
+            # This test assumes that there are multiple possible versions of a
+            # package available. That makes it brittle if you pick just one
+            # target, as changes in the available packages will break the test.
+            # Therefore, we'll choose from several packages to make sure we get
+            # one that is suitable for this test.
+            packages = ('hwinfo', 'avrdude', 'diffoscope', 'vim')
 
-            # Only install a new version of vim if vim is up-to-date, otherwise we don't
-            # need this check. (And the test will fail when we check for the empty dict
-            # since vim gets upgraded in the install step.)
-            if 'out-of-date' not in vim_info['status']:
-                # Install a version of vim that should need upgrading
-                ret = self.run_function('pkg.install', ['vim', vim_version])
-                if not isinstance(ret, dict):
-                    if ret.startswith('ERROR'):
-                        self.skipTest('Could not install earlier vim to complete test.')
+            available = self.run_function('pkg.list_repo_pkgs', packages)
+            versions = self.run_function('pkg.version', packages)
+
+            for package in packages:
+                try:
+                    new, old = available[package][:2]
+                except (KeyError, ValueError):
+                    # Package not available, or less than 2 versions
+                    # available. This is not a suitable target.
+                    continue
                 else:
-                    self.assertNotEqual(ret, {})
+                    target = package
+                    current = versions[target]
+                    break
+            else:
+                # None of the packages have more than one version available, so
+                # we need to find new package(s). pkg.list_repo_pkgs can be
+                # used to get an overview of the available packages. We should
+                # try to find packages with few dependencies and small download
+                # sizes, to keep this test from taking longer than necessary.
+                self.fail('No suitable package found for this test')
 
-            # Run a system upgrade, which should catch the fact that Vim needs upgrading, and upgrade it.
+            # Make sure we have the 2nd-oldest available version installed
+            ret = self.run_function('pkg.install', [target], version=old)
+            if not isinstance(ret, dict):
+                if ret.startswith('ERROR'):
+                    self.skipTest(
+                        'Could not install older {0} to complete '
+                        'test.'.format(target)
+                    )
+
+            # Run a system upgrade, which should catch the fact that the
+            # targeted package needs upgrading, and upgrade it.
             ret = self.run_function(func)
 
             # The changes dictionary should not be empty.
             if 'changes' in ret:
-                self.assertIn('vim', ret['changes'])
+                self.assertIn(target, ret['changes'])
             else:
-                self.assertIn('vim', ret)
+                self.assertIn(target, ret)
         else:
             ret = self.run_function('pkg.list_upgrades')
             if ret == '' or ret == {}:

tests/integration/modules/test_saltutil.py

@@ -84,7 +84,8 @@ class SaltUtilSyncModuleTest(ModuleCase):
                            'beacons': [],
                            'utils': [],
                            'returners': [],
-                           'modules': ['modules.override_test',
+                           'modules': ['modules.mantest',
+                                       'modules.override_test',
                                        'modules.runtests_decorators',
                                        'modules.runtests_helpers',
                                        'modules.salttest'],
@@ -131,7 +132,8 @@ class SaltUtilSyncModuleTest(ModuleCase):
                            'beacons': [],
                            'utils': [],
                            'returners': [],
-                           'modules': ['modules.override_test',
+                           'modules': ['modules.mantest',
+                                       'modules.override_test',
                                        'modules.runtests_helpers',
                                        'modules.salttest'],
                            'renderers': [],

tests/integration/modules/test_state.py

@@ -1476,6 +1476,56 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin):
         test_data = state_run['cmd_|-test_non_failing_state_|-echo "Should not run"_|-run']
         self.assertIn('duration', test_data)
 
+    def test_multiple_onfail_requisite_with_required(self):
+        '''
+        test to ensure multiple states are run
+        when specified as onfails for a single state.
+        This is a test for the issue:
+        https://github.com/saltstack/salt/issues/46552
+        '''
+
+        state_run = self.run_function('state.sls', mods='requisites.onfail_multiple_required')
+
+        retcode = state_run['cmd_|-b_|-echo b_|-run']['changes']['retcode']
+        self.assertEqual(retcode, 0)
+
+        retcode = state_run['cmd_|-c_|-echo c_|-run']['changes']['retcode']
+        self.assertEqual(retcode, 0)
+
+        retcode = state_run['cmd_|-d_|-echo d_|-run']['changes']['retcode']
+        self.assertEqual(retcode, 0)
+
+        stdout = state_run['cmd_|-b_|-echo b_|-run']['changes']['stdout']
+        self.assertEqual(stdout, 'b')
+
+        stdout = state_run['cmd_|-c_|-echo c_|-run']['changes']['stdout']
+        self.assertEqual(stdout, 'c')
+
+        stdout = state_run['cmd_|-d_|-echo d_|-run']['changes']['stdout']
+        self.assertEqual(stdout, 'd')
+
+    def test_multiple_onfail_requisite_with_required_no_run(self):
+        '''
+        test to ensure multiple states are not run
+        when specified as onfails for a single state
+        which fails.
+        This is a test for the issue:
+        https://github.com/saltstack/salt/issues/46552
+        '''
+
+        state_run = self.run_function('state.sls', mods='requisites.onfail_multiple_required_no_run')
+
+        expected = 'State was not run because onfail req did not change'
+
+        stdout = state_run['cmd_|-b_|-echo b_|-run']['comment']
+        self.assertEqual(stdout, expected)
+
+        stdout = state_run['cmd_|-c_|-echo c_|-run']['comment']
+        self.assertEqual(stdout, expected)
+
+        stdout = state_run['cmd_|-d_|-echo d_|-run']['comment']
+        self.assertEqual(stdout, expected)
+
     # listen tests
 
     def test_listen_requisite(self):
@@ -1855,6 +1905,16 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin):
         for id in _expected:
             self.assertEqual(sls[id]['comment'], _expected[id]['comment'])
 
+    def test_state_sls_unicode_characters(self):
+        '''
+        test state.sls when state file contains non-ascii characters
+        '''
+        ret = self.run_function('state.sls', ['issue-46672'])
+        log.debug('== ret %s ==', type(ret))
+
+        _expected = "cmd_|-echo1_|-echo 'This is Æ test!'_|-run"
+        self.assertIn(_expected, ret)
+
     def tearDown(self):
         nonbase_file = os.path.join(TMP, 'nonbase_env')
         if os.path.isfile(nonbase_file):

tests/integration/scheduler/test_eval.py

@@ -469,4 +469,90 @@ class SchedulerEvalTest(ModuleCase, SaltReturnAssertsMixin):
         run_time = dateutil_parser.parse('11/29/2017 3:00pm')
         self.schedule.eval(now=run_time)
         ret = self.schedule.job_status('job1')
-        self.assertEqual(ret['_last_run'], run_time)
+
+    def test_eval_splay(self):
+        '''
+        verify that scheduled job runs with splayed time
+        '''
+        job = {
+          'schedule': {
+            'job_eval_splay': {
+              'function': 'test.ping',
+              'seconds': '30',
+              'splay': '10'
+            }
+          }
+        }
+
+        # Add job to schedule
+        self.schedule.opts.update(job)
+
+        with patch('random.randint', MagicMock(return_value=10)):
+            # eval at 2:00pm to prime, simulate minion start up.
+            run_time = dateutil_parser.parse('11/29/2017 2:00pm')
+            self.schedule.eval(now=run_time)
+            ret = self.schedule.job_status('job_eval_splay')
+
+            # eval at 2:00:40pm, will run.
+            run_time = dateutil_parser.parse('11/29/2017 2:00:40pm')
+            self.schedule.eval(now=run_time)
+            ret = self.schedule.job_status('job_eval_splay')
+            self.assertEqual(ret['_last_run'], run_time)
+
+    def test_eval_splay_range(self):
+        '''
+        verify that scheduled job runs with splayed time
+        '''
+        job = {
+          'schedule': {
+            'job_eval_splay': {
+              'function': 'test.ping',
+              'seconds': '30',
+              'splay': {'start': 5, 'end': 10}
+            }
+          }
+        }
+
+        # Add job to schedule
+        self.schedule.opts.update(job)
+
+        with patch('random.randint', MagicMock(return_value=10)):
+            # eval at 2:00pm to prime, simulate minion start up.
+            run_time = dateutil_parser.parse('11/29/2017 2:00pm')
+            self.schedule.eval(now=run_time)
+            ret = self.schedule.job_status('job_eval_splay')
+
+            # eval at 2:00:40pm, will run.
+            run_time = dateutil_parser.parse('11/29/2017 2:00:40pm')
+            self.schedule.eval(now=run_time)
+            ret = self.schedule.job_status('job_eval_splay')
+            self.assertEqual(ret['_last_run'], run_time)
+
+    def test_eval_splay_global(self):
+        '''
+        verify that scheduled job runs with splayed time
+        '''
+        job = {
+          'schedule': {
+            'splay': {'start': 5, 'end': 10},
+            'job_eval_splay': {
+              'function': 'test.ping',
+              'seconds': '30',
+            }
+          }
+        }
+
+        # Add job to schedule
+        self.schedule.opts.update(job)
+
+        with patch('random.randint', MagicMock(return_value=10)):
+            # eval at 2:00pm to prime, simulate minion start up.
+            run_time = dateutil_parser.parse('11/29/2017 2:00pm')
+            self.schedule.eval(now=run_time)
+            ret = self.schedule.job_status('job_eval_splay')
+
+            # eval at 2:00:40pm, will run.
+            run_time = dateutil_parser.parse('11/29/2017 2:00:40pm')
+            self.schedule.eval(now=run_time)
+            ret = self.schedule.job_status('job_eval_splay')
+            self.assertEqual(ret['_last_run'], run_time)

tests/integration/shell/test_auth.py

@@ -59,12 +59,18 @@ class AuthTest(ShellCase):
     def setUp(self):
         for user in (self.userA, self.userB):
             try:
+                if salt.utils.platform.is_darwin() and user not in str(self.run_call('user.list_users')):
+                    # workaround for https://github.com/saltstack/salt-jenkins/issues/504
+                    raise KeyError
                 pwd.getpwnam(user)
             except KeyError:
                 self.run_call('user.add {0} createhome=False'.format(user))
 
         # only put userB into the group for the group auth test
         try:
+            if salt.utils.platform.is_darwin() and self.group not in str(self.run_call('group.info {0}'.format(self.group))):
+                # workaround for https://github.com/saltstack/salt-jenkins/issues/504
+                raise KeyError
             grp.getgrnam(self.group)
         except KeyError:
             self.run_call('group.add {0}'.format(self.group))

tests/integration/spm/test_man_spm.py

@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-'''
-Tests man spm
-'''
-# Import python libs
-from __future__ import absolute_import
-import os
-import shutil
-import sys
-import tempfile
-
-# Import Salt Testing libs
-from tests.support.case import ModuleCase
-from tests.support.helpers import destructiveTest, flaky
-from tests.support.paths import CODE_DIR
-
-
-@destructiveTest
-class SPMManTest(ModuleCase):
-    '''
-    Validate man spm
-    '''
-
-    def setUp(self):
-        self.tmpdir = tempfile.mktemp()
-        os.mkdir(self.tmpdir)
-        self.run_function('cmd.run', ['{0} {1} install --root={2}'.format(
-            sys.executable,
-            os.path.join(CODE_DIR, 'setup.py'),
-            self.tmpdir
-        )])
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir)
-
-    @flaky
-    def test_man_spm(self):
-        '''
-        test man spm
-        '''
-        manpath = self.run_function('cmd.run', ['find {0} -name spm.1'.format(self.tmpdir)])
-        self.assertIn('/man1/', manpath)
-        cmd = self.run_function('cmd.run', ['man {0}'.format(manpath)])
-        self.assertIn('Salt Package Manager', cmd)
-        self.assertIn('command for managing Salt packages', cmd)

tests/integration/states/test_file.py

@@ -2182,7 +2182,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
             'salt_utf8_tests',
             '{0}.txt'.format(korean_1)
         )
-        test_file_encoded = salt.utils.stringutils.to_str(test_file)
+        test_file_encoded = test_file
         template_path = os.path.join(TMP_STATE_TREE, 'issue-8947.sls')
         # create the sls template
         template_lines = [
@@ -2247,46 +2247,45 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
                 ' 한국어 시험\n'
                 '+마지막 행\n'
             )
-            diff = salt.utils.stringutils.to_str(diff)
             # future_lint: disable=blacklisted-function
             expected = {
-                str('file_|-some-utf8-file-create_|-{0}_|-managed').format(test_file_encoded): {
+                'file_|-some-utf8-file-create_|-{0}_|-managed'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 0,
-                    'comment': str('File {0} updated').format(test_file_encoded),
+                    'comment': 'File {0} updated'.format(test_file_encoded),
                     'diff': 'New file'
                 },
-                str('file_|-some-utf8-file-create2_|-{0}_|-managed').format(test_file_encoded): {
+                'file_|-some-utf8-file-create2_|-{0}_|-managed'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 1,
-                    'comment': str('File {0} updated').format(test_file_encoded),
+                    'comment': 'File {0} updated'.format(test_file_encoded),
                     'diff': diff
                 },
-                str('file_|-some-utf8-file-exists_|-{0}_|-exists').format(test_file_encoded): {
+                'file_|-some-utf8-file-exists_|-{0}_|-exists'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 2,
-                    'comment': str('Path {0} exists').format(test_file_encoded)
+                    'comment': 'Path {0} exists'.format(test_file_encoded)
                 },
-                str('cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run').format(test_file_encoded): {
-                    'name': str('cat "{0}"').format(test_file_encoded),
+                'cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run'.format(test_file_encoded): {
+                    'name': 'cat "{0}"'.format(test_file_encoded),
                     '__run_num__': 3,
-                    'comment': str('Command "cat "{0}"" run').format(test_file_encoded),
-                    'stdout': str('{0}\n{1}\n{2}').format(
-                        salt.utils.stringutils.to_str(korean_2),
-                        salt.utils.stringutils.to_str(korean_1),
-                        salt.utils.stringutils.to_str(korean_3),
+                    'comment': 'Command "cat "{0}"" run'.format(test_file_encoded),
+                    'stdout': '{0}\n{1}\n{2}'.format(
+                        korean_2,
+                        korean_1,
+                        korean_3,
                     )
                 },
-                str('cmd_|-some-utf8-file-content-remove_|-rm -f "{0}"_|-run').format(test_file_encoded): {
-                    'name': str('rm -f "{0}"').format(test_file_encoded),
+                'cmd_|-some-utf8-file-content-remove_|-rm -f "{0}"_|-run'.format(test_file_encoded): {
+                    'name': 'rm -f "{0}"'.format(test_file_encoded),
                     '__run_num__': 4,
-                    'comment': str('Command "rm -f "{0}"" run').format(test_file_encoded),
+                    'comment': 'Command "rm -f "{0}"" run'.format(test_file_encoded),
                     'stdout': ''
                 },
-                str('file_|-some-utf8-file-removed_|-{0}_|-missing').format(test_file_encoded): {
+                'file_|-some-utf8-file-removed_|-{0}_|-missing'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 5,
-                    'comment': str('Path {0} is missing').format(test_file_encoded),
+                    'comment': 'Path {0} is missing'.format(test_file_encoded),
                 }
             }
             # future_lint: enable=blacklisted-function
@@ -2308,7 +2307,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
 
             self.assertEqual(expected, result)
             # future_lint: disable=blacklisted-function
-            cat_id = str('cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run').format(test_file_encoded)
+            cat_id = 'cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run'.format(test_file_encoded)
             # future_lint: enable=blacklisted-function
             self.assertEqual(
                 salt.utils.stringutils.to_unicode(result[cat_id]['stdout']),

tests/integration/states/test_git.py

@@ -244,8 +244,9 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin):
             self.assertSaltTrueReturn(ret)
             self.assertEqual(
                 ret[next(iter(ret))]['comment'],
-                ('Repository {0} is up-to-date, but with local changes. Set '
-                 '\'force_reset\' to True to purge local changes.'.format(name))
+                ('Repository {0} is up-to-date, but with uncommitted changes. '
+                 'Set \'force_reset\' to True to purge uncommitted changes.'
+                 .format(name))
             )
 
             # Now run the state with force_reset=True
@@ -560,43 +561,228 @@ class LocalRepoGitTest(ModuleCase, SaltReturnAssertsMixin):
     '''
     Tests which do no require connectivity to github.com
     '''
+    def setUp(self):
+        self.repo = tempfile.mkdtemp(dir=TMP)
+        self.admin = tempfile.mkdtemp(dir=TMP)
+        self.target = tempfile.mkdtemp(dir=TMP)
+        for dirname in (self.repo, self.admin, self.target):
+            self.addCleanup(shutil.rmtree, dirname, ignore_errors=True)
+
+        # Create bare repo
+        self.run_function('git.init', [self.repo], bare=True)
+        # Clone bare repo
+        self.run_function('git.clone', [self.admin], url=self.repo)
+        self._commit(self.admin, '', message='initial commit')
+        self._push(self.admin)
+
+    def _commit(self, repo_path, content, message):
+        with salt.utils.files.fopen(os.path.join(repo_path, 'foo'), 'a') as fp_:
+            fp_.write(content)
+        self.run_function('git.add', [repo_path, '.'])
+        self.run_function(
+            'git.commit', [repo_path, message],
+            git_opts='-c user.name="Foo Bar" -c user.email=foo@bar.com',
+        )
+
+    def _push(self, repo_path, remote='origin', ref='master'):
+        self.run_function('git.push', [repo_path], remote=remote, ref=ref)
+
+    def _test_latest_force_reset_setup(self):
+        # Perform the initial clone
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target)
+        self.assertSaltTrueReturn(ret)
+
+        # Make and push changes to remote repo
+        self._commit(self.admin,
+                     content='Hello world!\n',
+                     message='added a line')
+        self._push(self.admin)
+
+        # Make local changes to clone, but don't commit them
+        with salt.utils.files.fopen(os.path.join(self.target, 'foo'), 'a') as fp_:
+            fp_.write('Local changes!\n')
+
+    def test_latest_force_reset_remote_changes(self):
+        '''
+        This tests that an otherwise fast-forward change with local chanegs
+        will not reset local changes when force_reset='remote_changes'
+        '''
+        self._test_latest_force_reset_setup()
+
+        # This should fail because of the local changes
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target)
+        self.assertSaltFalseReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('there are uncommitted changes', ret['comment'])
+        self.assertIn(
+            'Set \'force_reset\' to True (or \'remote-changes\')',
+            ret['comment']
+        )
+        self.assertEqual(ret['changes'], {})
+
+        # Now run again with force_reset='remote_changes', the state should
+        # succeed and discard the local changes
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target,
+            force_reset='remote-changes')
+        self.assertSaltTrueReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('Uncommitted changes were discarded', ret['comment'])
+        self.assertIn('Repository was fast-forwarded', ret['comment'])
+        self.assertNotIn('forced update', ret['changes'])
+        self.assertIn('revision', ret['changes'])
+
+        # Add new local changes, but don't commit them
+        with salt.utils.files.fopen(os.path.join(self.target, 'foo'), 'a') as fp_:
+            fp_.write('More local changes!\n')
+
+        # Now run again with force_reset='remote_changes', the state should
+        # succeed with an up-to-date message and mention that there are local
+        # changes, telling the user how to discard them.
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target,
+            force_reset='remote-changes')
+        self.assertSaltTrueReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('up-to-date, but with uncommitted changes', ret['comment'])
+        self.assertIn(
+            'Set \'force_reset\' to True to purge uncommitted changes',
+            ret['comment']
+        )
+        self.assertEqual(ret['changes'], {})
+
+    def test_latest_force_reset_true_fast_forward(self):
+        '''
+        This tests that an otherwise fast-forward change with local chanegs
+        does reset local changes when force_reset=True
+        '''
+        self._test_latest_force_reset_setup()
+
+        # Test that local changes are discarded and that we fast-forward
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target,
+            force_reset=True)
+        self.assertSaltTrueReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('Uncommitted changes were discarded', ret['comment'])
+        self.assertIn('Repository was fast-forwarded', ret['comment'])
+
+        # Add new local changes
+        with salt.utils.files.fopen(os.path.join(self.target, 'foo'), 'a') as fp_:
+            fp_.write('More local changes!\n')
+
+        # Running without setting force_reset should mention uncommitted changes
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target)
+        self.assertSaltTrueReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('up-to-date, but with uncommitted changes', ret['comment'])
+        self.assertIn(
+            'Set \'force_reset\' to True to purge uncommitted changes',
+            ret['comment']
+        )
+        self.assertEqual(ret['changes'], {})
+
+        # Test that local changes are discarded
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target,
+            force_reset=True)
+        self.assertSaltTrueReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('Uncommitted changes were discarded', ret['comment'])
+        self.assertIn('Repository was hard-reset', ret['comment'])
+        self.assertIn('forced update', ret['changes'])
+
+    def test_latest_force_reset_true_non_fast_forward(self):
+        '''
+        This tests that a non fast-forward change with divergent commits fails
+        unless force_reset=True.
+        '''
+        self._test_latest_force_reset_setup()
+
+        # Reset to remote HEAD
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target,
+            force_reset=True)
+        self.assertSaltTrueReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('Uncommitted changes were discarded', ret['comment'])
+        self.assertIn('Repository was fast-forwarded', ret['comment'])
+
+        # Make and push changes to remote repo
+        self._commit(self.admin,
+                     content='New line\n',
+                     message='added another line')
+        self._push(self.admin)
+
+        # Make different changes to local file and commit locally
+        self._commit(self.target,
+                     content='Different new line\n',
+                     message='added a different line')
+
+        # This should fail since the local clone has diverged and cannot
+        # fast-forward to the remote rev
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target)
+        self.assertSaltFalseReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('this is not a fast-forward merge', ret['comment'])
+        self.assertIn(
+            'Set \'force_reset\' to True to force this update',
+            ret['comment']
+        )
+        self.assertEqual(ret['changes'], {})
+
+        # Repeat the state with force_reset=True and confirm that the hard
+        # reset was performed
+        ret = self.run_state(
+            'git.latest',
+            name=self.repo,
+            target=self.target,
+            force_reset=True)
+        self.assertSaltTrueReturn(ret)
+        ret = ret[next(iter(ret))]
+        self.assertIn('Repository was hard-reset', ret['comment'])
+        self.assertIn('forced update', ret['changes'])
+        self.assertIn('revision', ret['changes'])
+
     def test_renamed_default_branch(self):
         '''
         Test the case where the remote branch has been removed
         https://github.com/saltstack/salt/issues/36242
         '''
-        repo = tempfile.mkdtemp(dir=TMP)
-        admin = tempfile.mkdtemp(dir=TMP)
-        name = tempfile.mkdtemp(dir=TMP)
-        for dirname in (repo, admin, name):
-            self.addCleanup(shutil.rmtree, dirname, ignore_errors=True)
-
-        # Create bare repo
-        self.run_function('git.init', [repo], bare=True)
-        # Clone bare repo
-        self.run_function('git.clone', [admin], url=repo)
-        # Create, add, commit, and push file
-        with salt.utils.files.fopen(os.path.join(admin, 'foo'), 'w'):
-            pass
-        self.run_function('git.add', [admin, '.'])
-        self.run_function(
-            'git.commit', [admin, 'initial commit'],
-            git_opts='-c user.name="Foo Bar" -c user.email=foo@bar.com',
-        )
-        self.run_function('git.push', [admin], remote='origin', ref='master')
-
         # Rename remote 'master' branch to 'develop'
         os.rename(
-            os.path.join(repo, 'refs', 'heads', 'master'),
-            os.path.join(repo, 'refs', 'heads', 'develop')
+            os.path.join(self.repo, 'refs', 'heads', 'master'),
+            os.path.join(self.repo, 'refs', 'heads', 'develop')
         )
 
         # Run git.latest state. This should successfully clone and fail with a
         # specific error in the comment field.
         ret = self.run_state(
             'git.latest',
-            name=repo,
-            target=name,
+            name=self.repo,
+            target=self.target,
             rev='develop',
         )
         self.assertSaltFalseReturn(ret)
@@ -610,19 +796,19 @@ class LocalRepoGitTest(ModuleCase, SaltReturnAssertsMixin):
             '(which will ensure that the named branch is created '
             'if it does not already exist).\n\n'
             'Changes already made: {0} cloned to {1}'
-            .format(repo, name)
+            .format(self.repo, self.target)
         )
         self.assertEqual(
             ret[next(iter(ret))]['changes'],
-            {'new': '{0} => {1}'.format(repo, name)}
+            {'new': '{0} => {1}'.format(self.repo, self.target)}
         )
 
         # Run git.latest state again. This should fail again, with a different
         # error in the comment field, and should not change anything.
         ret = self.run_state(
             'git.latest',
-            name=repo,
-            target=name,
+            name=self.repo,
+            target=self.target,
             rev='develop',
         )
         self.assertSaltFalseReturn(ret)
@@ -643,8 +829,8 @@ class LocalRepoGitTest(ModuleCase, SaltReturnAssertsMixin):
         # checkout a new branch and the state should pass.
         ret = self.run_state(
             'git.latest',
-            name=repo,
-            target=name,
+            name=self.repo,
+            target=self.target,
             rev='develop',
             branch='develop',
         )

tests/integration/states/test_npm.py

@@ -6,12 +6,14 @@
 '''
 # Import Python libs
 from __future__ import absolute_import, unicode_literals, print_function
+import os
 
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
 from tests.support.unit import skipIf
 from tests.support.helpers import destructiveTest, requires_network
 from tests.support.mixins import SaltReturnAssertsMixin
+from tests.support.runtests import RUNTIME_VARS
 
 # Import salt libs
 import salt.modules.cmdmod as cmd
@@ -31,7 +33,7 @@ class NpmStateTest(ModuleCase, SaltReturnAssertsMixin):
         Basic test to determine if NPM module was successfully installed and
         removed.
         '''
-        ret = self.run_state('npm.installed', name='pm2')
+        ret = self.run_state('npm.installed', name='pm2', registry="http://registry.npmjs.org/")
         self.assertSaltTrueReturn(ret)
         ret = self.run_state('npm.removed', name='pm2')
         self.assertSaltTrueReturn(ret)
@@ -42,10 +44,23 @@ class NpmStateTest(ModuleCase, SaltReturnAssertsMixin):
         '''
         Determine if URL-referenced NPM module can be successfully installed.
         '''
-        ret = self.run_state('npm.installed', name='request/request#v2.81.1')
+        if LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION):
+            user = os.environ.get('SUDO_USER', 'root')
+            npm_dir = os.path.join(RUNTIME_VARS.TMP, 'git-install-npm')
+            self.run_state('file.directory', name=npm_dir, user=user, dir_mode='755')
+        else:
+            user = None
+            npm_dir = None
+        ret = self.run_state('npm.installed',
+                             name='request/request#v2.81.1',
+                             runas=user,
+                             dir=npm_dir,
+                             registry="http://registry.npmjs.org/")
         self.assertSaltTrueReturn(ret)
-        ret = self.run_state('npm.removed', name='git://github.com/request/request')
+        ret = self.run_state('npm.removed', name='git://github.com/request/request', runas=user, dir=npm_dir)
         self.assertSaltTrueReturn(ret)
+        if npm_dir is not None:
+            self.run_state('file.absent', name=npm_dir)
 
     @requires_network()
     @destructiveTest
@@ -54,7 +69,7 @@ class NpmStateTest(ModuleCase, SaltReturnAssertsMixin):
         Basic test to determine if NPM module successfully installs multiple
         packages.
         '''
-        ret = self.run_state('npm.installed', name='unused', pkgs=['pm2', 'grunt'])
+        ret = self.run_state('npm.installed', name='unused', pkgs=['pm2', 'grunt'], registry="http://registry.npmjs.org/")
         self.assertSaltTrueReturn(ret)
 
     @skipIf(salt.utils.path.which('npm') and LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION),

tests/integration/states/test_pip.py

@@ -14,6 +14,7 @@ import os
 import pwd
 import glob
 import shutil
+import sys
 
 # Import Salt Testing libs
 from tests.support.mixins import SaltReturnAssertsMixin
@@ -525,6 +526,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
             if os.path.isdir(venv_dir):
                 shutil.rmtree(venv_dir)
 
+    @skipIf(sys.version_info[:2] >= (3, 6), 'Old version of virtualenv too old for python3.6')
     def test_46127_pip_env_vars(self):
         '''
         Test that checks if env_vars passed to pip.installed are also passed

tests/integration/states/test_pkgrepo.py

@@ -22,16 +22,16 @@ import salt.utils.platform
 from salt.ext import six
 
 
+@destructiveTest
+@skipIf(salt.utils.platform.is_windows(), 'minion is windows')
 class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin):
     '''
     pkgrepo state tests
     '''
-    @destructiveTest
-    @skipIf(salt.utils.platform.is_windows(), 'minion is windows')
     @requires_system_grains
     def test_pkgrepo_01_managed(self, grains):
         '''
-        This is a destructive test as it adds a repository.
+        Test adding a repo
         '''
         os_grain = self.run_function('grains.item', ['os'])['os']
         os_release_info = tuple(self.run_function('grains.item', ['osrelease_info'])['osrelease_info'])
@@ -56,12 +56,9 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin):
         for state_id, state_result in six.iteritems(ret):
             self.assertSaltTrueReturn(dict([(state_id, state_result)]))
 
-    @destructiveTest
-    @skipIf(salt.utils.platform.is_windows(), 'minion is windows')
     def test_pkgrepo_02_absent(self):
         '''
-        This is a destructive test as it removes the repository added in the
-        above test.
+        Test removing the repo from the above test
         '''
         os_grain = self.run_function('grains.item', ['os'])['os']
         os_release_info = tuple(self.run_function('grains.item', ['osrelease_info'])['osrelease_info'])
@@ -78,3 +75,56 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin):
         self.assertReturnNonEmptySaltType(ret)
         for state_id, state_result in six.iteritems(ret):
             self.assertSaltTrueReturn(dict([(state_id, state_result)]))
+
+    @requires_system_grains
+    def test_pkgrepo_03_with_comments(self, grains):
+        '''
+        Test adding a repo with comments
+        '''
+        os_family = grains['os_family'].lower()
+
+        if os_family in ('redhat', 'suse'):
+            kwargs = {
+                'name': 'examplerepo',
+                'baseurl': 'http://example.com/repo',
+                'enabled': False,
+                'comments': ['This is a comment']
+            }
+        elif os_family in ('debian',):
+            self.skipTest('Debian/Ubuntu test case needed')
+        else:
+            self.skipTest("No test case for os_family '{0}'".format(os_family))
+
+        try:
+            # Run the state to add the repo
+            ret = self.run_state('pkgrepo.managed', **kwargs)
+            self.assertSaltTrueReturn(ret)
+
+            # Run again with modified comments
+            kwargs['comments'].append('This is another comment')
+            ret = self.run_state('pkgrepo.managed', **kwargs)
+            self.assertSaltTrueReturn(ret)
+            ret = ret[next(iter(ret))]
+            self.assertEqual(
+                ret['changes'],
+                {
+                    'comments': {
+                        'old': ['This is a comment'],
+                        'new': ['This is a comment',
+                                'This is another comment']
+                    }
+                }
+            )
+
+            # Run a third time, no changes should be made
+            ret = self.run_state('pkgrepo.managed', **kwargs)
+            self.assertSaltTrueReturn(ret)
+            ret = ret[next(iter(ret))]
+            self.assertFalse(ret['changes'])
+            self.assertEqual(
+                ret['comment'],
+                "Package repo '{0}' already configured".format(kwargs['name'])
+            )
+        finally:
+            # Clean up
+            self.run_state('pkgrepo.absent', name=kwargs['name'])

tests/runtests.py

@@ -115,6 +115,9 @@ TEST_SUITES = {
     'client':
        {'display_name': 'Client',
         'path': 'integration/client'},
+    'doc':
+       {'display_name': 'Documentation',
+        'path': 'integration/doc'},
     'ext_pillar':
        {'display_name': 'External Pillar',
         'path': 'integration/pillar'},
@@ -292,6 +295,15 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
             action='store_true',
             help='Run tests for client'
         )
+        self.test_selection_group.add_option(
+            '-d',
+            '--doc',
+            '--doc-tests',
+            dest='doc',
+            default=False,
+            action='store_true',
+            help='Run tests for documentation'
+        )
         self.test_selection_group.add_option(
             '-I',
             '--ext-pillar',
@@ -743,6 +755,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
                         failfast=self.options.failfast,
                     )
                     status.append(results)
+                return status
             for suite in TEST_SUITES:
                 if suite != 'unit' and getattr(self.options, suite):
                     status.append(self.run_integration_suite(**TEST_SUITES[suite]))

tests/support/win_installer.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+'''
+    :copyright: Copyright 2013-2017 by the SaltStack Team, see AUTHORS for more details.
+    :license: Apache 2.0, see LICENSE for more details.
+
+
+    tests.support.win_installer
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Fetches the binary Windows installer
+'''
+from __future__ import absolute_import
+import hashlib
+import requests
+import re
+
+PREFIX = 'Salt-Minion-'
+REPO = "https://repo.saltstack.com/windows"
+
+
+def iter_installers(content):
+    '''
+    Parse a list of windows installer links and their corresponding md5
+    checksum links.
+    '''
+    HREF_RE = "<a href=\"(.*?)\">"
+    installer, md5 = None, None
+    for m in re.finditer(HREF_RE, content):
+        x = m.groups()[0]
+        if not x.startswith(PREFIX):
+            continue
+        if x.endswith('zip'):
+            continue
+        if installer:
+            if x != installer + '.md5':
+                raise Exception("Unable to parse response")
+            md5 = x
+            yield installer, md5
+            installer, md5 = None, None
+        else:
+            installer = x
+
+
+def split_installer(name):
+    '''
+    Return a tuple of the salt version, python verison and architecture from an
+    installer name.
+    '''
+    x = name[len(PREFIX):]
+    return x.split('-')[:3]
+
+
+def latest_version(repo=REPO):
+    '''
+    Return the latest version found on the salt repository webpage.
+    '''
+    for name, md5 in iter_installers(requests.get(repo).content):
+        pass
+    return split_installer(name)[0]
+
+
+def installer_name(salt_ver, py_ver='Py2', arch='AMD64'):
+    '''
+    Create an installer file name
+    '''
+    return "Salt-Minion-{}-{}-{}-Setup.exe".format(salt_ver, py_ver, arch)
+
+
+def latest_installer_name(repo=REPO, **kwargs):
+    '''
+    Fetch the latest installer name
+    '''
+    return installer_name(latest_version(repo), **kwargs)
+
+
+def download_and_verify(fp, name, repo=REPO):
+    '''
+    Download an installer and verify it's contents.
+    '''
+    md5 = "{}.md5".format(name)
+    url = lambda x: "{}/{}".format(repo, x)
+    resp = requests.get(url(md5))
+    if resp.status_code != 200:
+        raise Exception("Unable to fetch installer md5")
+    installer_md5 = resp.text.strip().split()[0].lower()
+    resp = requests.get(url(name), stream=True)
+    if resp.status_code != 200:
+        raise Exception("Unable to fetch installer")
+    md5hsh = hashlib.md5()
+    for chunk in resp.iter_content(chunk_size=1024):
+        md5hsh.update(chunk)
+        fp.write(chunk)
+    if md5hsh.hexdigest() != installer_md5:
+        raise Exception("Installer's hash does not match {} != {}".format(
+            md5hsh.hexdigest(), installer_md5
+        ))

tests/unit/modules/test_cmdmod.py

@@ -303,7 +303,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin):
                 environment = os.environ.copy()
 
                 popen_mock.return_value = Mock(
-                    communicate=lambda *args, **kwags: ['{}', None],
+                    communicate=lambda *args, **kwags: [b'', None],
                     pid=lambda: 1,
                     retcode=0
                 )

tests/unit/modules/test_inspect_collector.py

@@ -19,6 +19,8 @@
 # Import Python Libs
 from __future__ import absolute_import, print_function, unicode_literals
 import os
+import errno
+import subprocess
 
 # Import Salt Testing Libs
 from tests.support.unit import TestCase, skipIf
@@ -33,7 +35,33 @@ from tests.support.mock import (
 from salt.modules.inspectlib.collector import Inspector
 
 
+HAS_SYMLINKS = None
+
+
+def no_symlinks():
+    '''
+    Check if git is installed and has symlinks enabled in the configuration.
+    '''
+    global HAS_SYMLINKS
+    if HAS_SYMLINKS is not None:
+        return not HAS_SYMLINKS
+    output = ''
+    try:
+        output = subprocess.check_output('git config --get core.symlinks', shell=True)
+    except OSError as exc:
+        if exc.errno != errno.ENOENT:
+            raise
+    except subprocess.CalledProcessError:
+        # git returned non-zero status
+        pass
+    HAS_SYMLINKS = False
+    if output.strip() == 'true':
+        HAS_SYMLINKS = True
+    return not HAS_SYMLINKS
+
+
 @skipIf(NO_MOCK, NO_MOCK_REASON)
+@skipIf(no_symlinks(), "Git missing 'core.symlinks=true' config")
 class InspectorCollectorTestCase(TestCase):
     '''
     Test inspectlib:collector:Inspector

tests/unit/modules/test_localemod.py

@@ -40,6 +40,11 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin):
       X11 Layout: us
        X11 Model: pc105
     '''
+    locale_ctl_notset = '''
+   System Locale: n/a
+       VC Keymap: n/a
+      X11 Layout: n/a
+    '''
     locale_ctl_out_empty = ''
     locale_ctl_out_broken = '''
     System error:Recursive traversal of loopback mount points
@@ -79,9 +84,36 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin):
         assert 'LANG' in out['system_locale']
         assert 'LANGUAGE' in out['system_locale']
         assert out['system_locale']['LANG'] == out['system_locale']['LANGUAGE'] == 'de_DE.utf8'
-        assert out['vc_keymap'] == 'n/a'
-        assert out['x11_layout'] == 'us'
-        assert out['x11_model'] == 'pc105'
+        assert isinstance(out['vc_keymap'], dict)
+        assert 'data' in out['vc_keymap']
+        assert out['vc_keymap']['data'] == 'n/a'
+        assert isinstance(out['x11_layout'], dict)
+        assert 'data' in out['x11_layout']
+        assert out['x11_layout']['data'] == 'us'
+        assert isinstance(out['x11_model'], dict)
+        assert 'data' in out['x11_model']
+        assert out['x11_model']['data'] == 'pc105'
+
+    @patch('salt.utils.path.which', MagicMock(return_value="/usr/bin/localctl"))
+    @patch('salt.modules.localemod.__salt__', {'cmd.run': MagicMock(return_value=locale_ctl_notset)})
+    def test_localectl_status_parser_notset(self):
+        '''
+        Test localectl status parser.
+        :return:
+        '''
+        out = localemod._localectl_status()
+        assert isinstance(out, dict)
+        for key in ['system_locale', 'vc_keymap', 'x11_layout']:
+            assert key in out
+        assert isinstance(out['system_locale'], dict)
+        assert 'data' in out['system_locale']
+        assert out['system_locale']['data'] == 'n/a'
+        assert isinstance(out['vc_keymap'], dict)
+        assert 'data' in out['vc_keymap']
+        assert out['vc_keymap']['data'] == 'n/a'
+        assert isinstance(out['x11_layout'], dict)
+        assert 'data' in out['x11_layout']
+        assert out['x11_layout']['data'] == 'n/a'
 
     @patch('salt.modules.localemod.dbus', MagicMock())
     def test_dbus_locale_parser_matches(self):
@@ -154,7 +186,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin):
             assert isinstance(out[key], dict)
             for in_key in out[key]:
                 assert isinstance(out[key][in_key], six.text_type)
-        assert isinstance(out['reason'], six.text_type)
+        assert isinstance(out['reason']['data'], six.text_type)
 
     @patch('salt.utils.path.which', MagicMock(return_value="/usr/bin/localctl"))
     @patch('salt.modules.localemod.__grains__', {'os_family': 'Ubuntu', 'osmajorrelease': 42})

tests/unit/modules/test_x509.py

@@ -78,7 +78,7 @@ class X509TestCase(TestCase, LoaderModuleMockMixin):
         Test private function _parse_subject(subject) it handles a missing fields
         :return:
         '''
-        ca_key = '''-----BEGIN RSA PRIVATE KEY-----
+        ca_key = b'''-----BEGIN RSA PRIVATE KEY-----
 MIICWwIBAAKBgQCjdjbgL4kQ8Lu73xeRRM1q3C3K3ptfCLpyfw38LRnymxaoJ6ls
 pNSx2dU1uJ89YKFlYLo1QcEk4rJ2fdIjarV0kuNCY3rC8jYUp9BpAU5Z6p9HKeT1
 2rTPH81JyjbQDR5PyfCyzYOQtpwpB4zIUUK/Go7tTm409xGKbbUFugJNgQIDAQAB
@@ -163,4 +163,4 @@ c9bcgp7D7xD+TxWWNj4CSXEccJgGr91StV+gFg4ARQ==
                                       authorityKeyIdentifier='keyid,issuer:always',
                                       days_valid=3650,
                                       days_remaining=0)
-        self.assertIn('BEGIN CERTIFICATE', ret)
+        self.assertIn(b'BEGIN CERTIFICATE', ret)