Merge pull request #51581 from garethgreenaway/merge-2019.2.0rc1

[2019.2] Merge forward from 2019.2.0rc1 to 2019.2

Gemfile

@@ -2,6 +2,7 @@
 
 source 'https://rubygems.org'
 
+# Point this back at the test-kitchen package after 1.23.3 is relased
 gem 'test-kitchen', '~>1.23.3'
 gem 'kitchen-salt', '~>0.4.1'
 gem 'kitchen-sync'

pkg/osx/build_pkg.sh

@@ -158,13 +158,20 @@ echo -n -e "\033]0;Build_Pkg: Add Version to .xml\007"
 if [ "$PYVER" == "2" ]; then
     TITLE="Salt $VERSION"
     DESC="Salt $VERSION with Python 2"
+    SEDSTR="s/@PY2@/_py2/g"
 else
     TITLE="Salt $VERSION (Python 3)"
     DESC="Salt $VERSION with Python 3"
+    SEDSTR="s/@PY2@//g"
 fi
 
 cd $PKGRESOURCES
 cp distribution.xml.dist distribution.xml
+
+# Select the appropriate welcome text
+# This is only necessary until Sodium, then this can be removed
+sed -E -i '' "$SEDSTR" distribution.xml
+
 SEDSTR="s/@TITLE@/$TITLE/g"
 sed -E -i '' "$SEDSTR" distribution.xml
 

pkg/osx/distribution.xml.dist

@@ -12,11 +12,12 @@
              hostArchitectures="@CPUARCH@" />
     <domains enable_localSystem="true" />
     <!-- Define background image -->
-    <background file="saltstack.png"
+    <background file="logo.png"
                 mime-type="image/png"
-                scaling="proportional" />
+                scaling="proportional"
+                alignment="bottomleft" />
     <!-- Define documents displayed at various steps -->
-    <welcome file="welcome.rtf"
+    <welcome file="welcome@PY2@.rtf"
              mime-type="text/rtf" />
     <license file="license.rtf"
              mime-type="text/rtf" />

pkg/osx/pkg-resources/welcome_py2.rtf

@@ -0,0 +1,32 @@
+{\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340
+{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\froman\fcharset0 Times-Roman;\f2\fnil\fcharset0 PTMono-Regular;
+}
+{\colortbl;\red255\green255\blue255;\red0\green0\blue233;}
+\vieww28300\viewh14680\viewkind0
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
+
+\f0\fs24 \cf0 WARNING: Python 2 Support will be discontinued in Sodium. Salt will only ship Python 3 installers starting with the Sodium release.\
+\
+\pard\pardeftab720\sl280\sa240\partightenfactor0
+{\field{\*\fldinst{HYPERLINK "http://saltstack.com/"}}{\fldrslt 
+\f1 \cf2 \expnd0\expndtw0\kerning0
+\ul \ulc2 SaltStack}}
+\f1 \expnd0\expndtw0\kerning0
+ is extremely fast and scalable systems and configuration management software for predictive orchestration, cloud and data center automation, server provisioning, application deployment and more. \
+Documentation for Salt is available at {\field{\*\fldinst{HYPERLINK "http://docs.saltstack.com/"}}{\fldrslt \cf2 \ul \ulc2 http://docs.saltstack.com}} \
+This package will install Salt on your Mac. Salt installs into 
+\f2\fs22 /opt/salt
+\f1\fs24 .\
+Sample configuration files (
+\f2\fs22 master.dist
+\f1\fs24  and 
+\f2\fs22 minion.dist
+\f1\fs24 ) will be installed to 
+\f2\fs22 /etc/salt
+\f1\fs24 . Create copies of them without the '
+\f2\fs22 .dist
+\f1\fs24 ' in the filename and edit as you see fit.\
+This Salt package uses a custom-built Python. To install additional Python modules for Salt, use the associated 'pip' binary. For example, if you need LDAP support in Salt you will need the 'python-ldap' module. Install it with 
+\f2\fs22 /opt/salt/bin/pip install python-ldap
+\f1\fs24 \
+Note that some Python modules need a compiler available. Installing Apple's Xcode Command Line Tools should provide the necessary utilities. Alternatively, {\field{\*\fldinst{HYPERLINK "http://macports.org/"}}{\fldrslt \cf2 \ul \ulc2 MacPorts}}, {\field{\*\fldinst{HYPERLINK "http://finkproject.org/"}}{\fldrslt \cf2 \ul \ulc2 Fink}}, or {\field{\*\fldinst{HYPERLINK "http://brew.sh/"}}{\fldrslt \cf2 \ul \ulc2 Homebrew}} may provide what you need.}

pkg/windows/installer/Salt-Minion-Setup.nsi

@@ -70,6 +70,20 @@ ${StrStrAdv}
 !define MUI_ICON "salt.ico"
 !define MUI_UNICON "salt.ico"
 !define MUI_WELCOMEFINISHPAGE_BITMAP "panel.bmp"
+!define MUI_UNWELCOMEFINISHPAGE_BITMAP "panel.bmp"
+
+!if "${PYTHON_VERSION}" == "2"
+    !define MUI_WELCOMEPAGE_TEXT "\
+        WARNING: Python 2 Support will be discontinued in Sodium. Salt will only ship Python 3 \
+        installers starting with the Sodium release.$\r$\n\
+        $\r$\n\
+        Setup will guide you through the installation of ${PRODUCT_NAME} ${PRODUCT_VERSION}.$\r$\n\
+        $\r$\n\
+        It is recommended that you close all other applications before starting Setup. This will make it possible to \
+        update relevant system files without having to reboot your computer.$\r$\n\
+        $\r$\n\
+        Click Next to continue."
+!endif
 
 # Welcome page
 !insertmacro MUI_PAGE_WELCOME

salt/_compat.py

@@ -144,6 +144,12 @@ class IPv6AddressScoped(ipaddress.IPv6Address):
 
         :param address:
         '''
+        # pylint: disable-all
+        if not hasattr(self, '_is_packed_binary'):
+            # This method (below) won't be around for some Python 3 versions
+            # and we need check this differently anyway
+            self._is_packed_binary = lambda p: isinstance(p, bytes)
+        # pylint: enable-all
         if isinstance(address, string_types) and '%' in address:
             buff = address.split('%')
             if len(buff) != 2:

salt/modules/win_lgpo.py

@@ -5083,8 +5083,8 @@ def _findOptionValueAdvAudit(option):
                 field_names = _get_audit_defaults('fieldnames')
                 # If the file doesn't exist anywhere, create it with default
                 # fieldnames
-                __salt__['file.touch'](f_audit)
-                __salt__['file.append'](f_audit, ','.join(field_names))
+                __salt__['file.mkdir'](os.path.dirname(f_audit))
+                __salt__['file.write'](f_audit, ','.join(field_names))
 
         audit_settings = {}
         with salt.utils.files.fopen(f_audit, mode='r') as csv_file:
@@ -5187,6 +5187,7 @@ def _set_audit_file_data(option, value):
             # Copy the temporary csv file over the existing audit.csv in both
             # locations if a value was written
             __salt__['file.copy'](f_temp.name, f_audit, remove_existing=True)
+            __salt__['file.mkdir'](os.path.dirname(f_audit_gpo))
             __salt__['file.copy'](f_temp.name, f_audit_gpo, remove_existing=True)
     finally:
         f_temp.close()

salt/modules/win_timezone.py

@@ -205,17 +205,22 @@ def get_zone():
     Returns:
         str: Timezone in unix format
 
+    Raises:
+        CommandExecutionError: If timezone could not be gathered
+
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' timezone.get_zone
     '''
-    win_zone = __utils__['reg.read_value'](
-        hive='HKLM',
-        key='SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation',
-        vname='TimeZoneKeyName')['vdata']
-    return mapper.get_unix(win_zone.lower(), 'Unknown')
+    cmd = ['tzutil', '/g']
+    res = __salt__['cmd.run_all'](cmd, python_shell=False)
+    if res['retcode'] or not res['stdout']:
+        raise CommandExecutionError('tzutil encountered an error getting '
+                                    'timezone',
+                                    info=res)
+    return mapper.get_unix(res['stdout'].lower(), 'Unknown')
 
 
 def get_offset():

salt/scripts.py

@@ -20,6 +20,7 @@ from random import randint
 # Import salt libs
 from salt.exceptions import SaltSystemExit, SaltClientError, SaltReqTimeoutError
 import salt.defaults.exitcodes  # pylint: disable=unused-import
+import salt.ext.six as six
 
 log = logging.getLogger(__name__)
 
@@ -93,6 +94,16 @@ def salt_master():
     Start the salt master.
     '''
     import salt.cli.daemons
+# REMOVEME after Python 2.7 support is dropped (also the six import)
+    if six.PY2:
+        from salt.utils.versions import warn_until
+        # Message borrowed from pip's deprecation warning
+        warn_until('Sodium',
+                   'Python 2.7 will reach the end of its life on January 1st,'
+                   ' 2020. Please upgrade your Python as Python 2.7 won\'t be'
+                   ' maintained after that date.  Salt will drop support for'
+                   ' Python 2.7 in the Sodium release or later.')
+# END REMOVEME
     master = salt.cli.daemons.Master()
     master.start()
 
@@ -179,6 +190,16 @@ def salt_minion():
         minion = salt.cli.daemons.Minion()
         minion.start()
         return
+# REMOVEME after Python 2.7 support is dropped (also the six import)
+    elif six.PY2:
+        from salt.utils.versions import warn_until
+        # Message borrowed from pip's deprecation warning
+        warn_until('Sodium',
+                   'Python 2.7 will reach the end of its life on January 1st,'
+                   ' 2020. Please upgrade your Python as Python 2.7 won\'t be'
+                   ' maintained after that date.  Salt will drop support for'
+                   ' Python 2.7 in the Sodium release or later.')
+# END REMOVEME
 
     if '--disable-keepalive' in sys.argv:
         sys.argv.remove('--disable-keepalive')

salt/transport/ipc.py

@@ -219,7 +219,12 @@ class IPCServer(object):
             self.sock.close()
 
     def __del__(self):
+        try:
             self.close()
+        except TypeError:
+            # This is raised when Python's GC has collected objects which
+            # would be needed when calling self.close()
+            pass
 
 
 class IPCClient(object):
@@ -360,6 +365,7 @@ class IPCClient(object):
                 yield tornado.gen.sleep(1)
 
     def __del__(self):
+        try:
             with self._refcount_lock:
                 # Make sure we actually close no matter if something
                 # went wrong with our ref counting
@@ -370,6 +376,10 @@ class IPCClient(object):
                 if exc.errno != errno.EBADF:
                     # If its not a bad file descriptor error, raise
                     raise
+        except TypeError:
+            # This is raised when Python's GC has collected objects which
+            # would be needed when calling self.close()
+            pass
 
     def close(self):
         '''
@@ -610,7 +620,12 @@ class IPCMessagePublisher(object):
             self.sock.close()
 
     def __del__(self):
+        try:
             self.close()
+        except TypeError:
+            # This is raised when Python's GC has collected objects which
+            # would be needed when calling self.close()
+            pass
 
 
 class IPCMessageSubscriber(IPCClient):

tests/integration/modules/test_win_dns_client.py

@@ -22,8 +22,12 @@ class WinDNSTest(ModuleCase):
         '''
         Test add and removing a dns server
         '''
+        # Get a list of interfaces on the system
+        interfaces = self.run_function('network.interfaces_names')
+        skipIf(interfaces.count == 0, 'This test requires a network interface')
+
+        interface = interfaces[0]
         dns = '8.8.8.8'
-        interface = 'Ethernet'
         # add dns server
         self.assertTrue(self.run_function('win_dns_client.add_dns', [dns, interface], index=42))
 

tests/integration/states/test_file.py

@@ -148,7 +148,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
         remove files created in previous tests
         '''
         user = 'salt'
-        if user in str(self.run_function('user.list_users', [user])):
+        if user in str(self.run_function('user.list_users')):
             self.run_function('user.delete', [user])
 
         for path in (FILEPILLAR, FILEPILLARDEF, FILEPILLARGIT):

tests/unit/grains/test_core.py

@@ -601,7 +601,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin):
         self.assertIn(returned_grains['windowsdomaintype'], valid_types)
         valid_releases = ['Vista', '7', '8', '8.1', '10', '2008Server',
                           '2008ServerR2', '2012Server', '2012ServerR2',
-                          '2016Server']
+                          '2016Server', '2019Server']
         self.assertIn(returned_grains['osrelease'], valid_releases)
 
     @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux')

tests/unit/modules/test_win_timezone.py

@@ -3,18 +3,15 @@
     :codeauthor: Jayesh Kariya <jayeshk@saltstack.com>
 '''
 # Import Python Libs
-from __future__ import absolute_import, unicode_literals, print_function
-
-# Import Salt Testing Libs
-from tests.support.mixins import LoaderModuleMockMixin
-from tests.support.unit import TestCase, skipIf
-from tests.support.mock import (
-    MagicMock,
-    patch
-)
+from __future__ import absolute_import, print_function, unicode_literals
 
 # Import Salt Libs
 import salt.modules.win_timezone as win_timezone
+from salt.exceptions import CommandExecutionError
+# Import Salt Testing Libs
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.mock import MagicMock, patch
+from tests.support.unit import TestCase, skipIf
 
 
 @skipIf(not win_timezone.HAS_PYTZ, 'This test requires pytz')
@@ -25,20 +22,36 @@ class WinTimezoneTestCase(TestCase, LoaderModuleMockMixin):
     def setup_loader_modules(self):
         return {win_timezone: {}}
 
-    # 'get_zone' function tests: 1
+    # 'get_zone' function tests: 3
 
     def test_get_zone(self):
         '''
         Test if it get current timezone (i.e. Asia/Calcutta)
         '''
-        mock_read = MagicMock(side_effect=[{'vdata': 'India Standard Time'},
-                                           {'vdata': 'Indian Standard Time'}])
+        mock_read_ok = MagicMock(return_value={'pid': 78,
+                                            'retcode': 0,
+                                            'stderr': '',
+                                            'stdout': 'India Standard Time'})
 
-        with patch.dict(win_timezone.__utils__, {'reg.read_value': mock_read}):
+        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_read_ok}):
             self.assertEqual(win_timezone.get_zone(), 'Asia/Calcutta')
 
+        mock_read_error = MagicMock(return_value={'pid': 78,
+                                               'retcode': 0,
+                                               'stderr': '',
+                                               'stdout': 'Indian Standard Time'})
+
+        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_read_error}):
             self.assertEqual(win_timezone.get_zone(), 'Unknown')
 
+        mock_read_fatal = MagicMock(return_value={'pid': 78,
+                                                  'retcode': 1,
+                                                  'stderr': '',
+                                                  'stdout': ''})
+
+        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_read_fatal}):
+            self.assertRaises(CommandExecutionError, win_timezone.get_zone)
+
     # 'get_offset' function tests: 1
 
     def test_get_offset(self):
@@ -49,10 +62,12 @@ class WinTimezoneTestCase(TestCase, LoaderModuleMockMixin):
         # New Delhi\nIndia Standard Time')
         # mock_cmd = MagicMock(side_effect=['India Standard Time', time])
         # with patch.dict(win_timezone.__salt__, {'cmd.run': mock_cmd}):
+        mock_read = MagicMock(return_value={'pid': 78,
+                                               'retcode': 0,
+                                               'stderr': '',
+                                               'stdout': 'India Standard Time'})
 
-        mock_read = MagicMock(return_value={'vdata': 'India Standard Time'})
-
-        with patch.dict(win_timezone.__utils__, {'reg.read_value': mock_read}):
+        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_read}):
             self.assertEqual(win_timezone.get_offset(), '+0530')
 
     # 'get_zonecode' function tests: 1
@@ -61,9 +76,12 @@ class WinTimezoneTestCase(TestCase, LoaderModuleMockMixin):
         '''
         Test if it get current timezone (i.e. PST, MDT, etc)
         '''
-        mock_read = MagicMock(return_value={'vdata': 'India Standard Time'})
+        mock_read = MagicMock(return_value={'pid': 78,
+                                               'retcode': 0,
+                                               'stderr': '',
+                                               'stdout': 'India Standard Time'})
 
-        with patch.dict(win_timezone.__utils__, {'reg.read_value': mock_read}):
+        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_read}):
             self.assertEqual(win_timezone.get_zonecode(), 'IST')
 
     # 'set_zone' function tests: 1
@@ -72,14 +90,17 @@ class WinTimezoneTestCase(TestCase, LoaderModuleMockMixin):
         '''
         Test if it unlinks, then symlinks /etc/localtime to the set timezone.
         '''
-        mock_cmd = MagicMock(return_value={'pid': 78,
+        mock_write = MagicMock(return_value={'pid': 78,
                                                'retcode': 0,
                                                'stderr': '',
                                                'stdout': ''})
-        mock_read = MagicMock(return_value={'vdata': 'India Standard Time'})
+        mock_read = MagicMock(return_value={'pid': 78,
+                                               'retcode': 0,
+                                               'stderr': '',
+                                               'stdout': 'India Standard Time'})
 
-        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_cmd}), \
-                patch.dict(win_timezone.__utils__, {'reg.read_value': mock_read}):
+        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_write}), \
+                patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_read}):
 
             self.assertTrue(win_timezone.set_zone('Asia/Calcutta'))
 
@@ -91,9 +112,12 @@ class WinTimezoneTestCase(TestCase, LoaderModuleMockMixin):
         the one set in /etc/localtime. Returns True if they match,
         and False if not. Mostly useful for running state checks.
         '''
-        mock_read = MagicMock(return_value={'vdata': 'India Standard Time'})
+        mock_read = MagicMock(return_value={'pid': 78,
+                                           'retcode': 0,
+                                           'stderr': '',
+                                           'stdout': 'India Standard Time'})
 
-        with patch.dict(win_timezone.__utils__, {'reg.read_value': mock_read}):
+        with patch.dict(win_timezone.__salt__, {'cmd.run_all': mock_read}):
             self.assertTrue(win_timezone.zone_compare('Asia/Calcutta'))
 
     # 'get_hwclock' function tests: 1