Merge pull request #46733 from rallytime/merge-2018.3

[2018.3] Merge forward from 2018.3.0rc1 to 2018.3

doc/topics/releases/2018.3.0.rst

@@ -655,18 +655,35 @@ The ``state_output`` parameter now supports ``full_id``, ``changes_id`` and ``te
 Just like ``mixed_id``, these use the state ID as name in the highstate output.
 For more information on these output modes, see the docs for the :mod:`Highstate Outputter <salt.output.highstate>`.
 
-Windows Installer: Changes to existing config handling
-------------------------------------------------------
+Windows Installer: Changes to config handling
+---------------------------------------------
 Behavior with existing configuration has changed. With previous installers the
 existing config was used and the master and minion id could be modified via the
 installer. It was problematic in that it didn't account for configuration that
 may be defined in the ``minion.d`` directory. This change gives you the option
-via a checkbox to either use the existing config with out changes or the default
-config using values you pass to the installer. If you choose to use the existing
-config then no changes are made. If not, the existing config is deleted, to
-include the ``minion.d`` directory, and the default config is used. A
-command-line switch (``/use-existing-config``) has also been added to control
-this behavior.
+via a drop-down list to use one of the following:
+
+- Default Config: Use the config that comes with the installer
+- Existing Config: Use the current config without changes
+- Custom Config: Select a custom config using the file picker
+
+The existing config option will only be available if the installer detects an
+existing config. If there is an existing config, and you choose ``Default`` or
+``Custom``, the existing config will be deleted, including the ``minion.d``
+directory, and replaced by your selection.
+
+The ``Default Config`` and ``Custom Config`` options will allow you to modify
+the Master and the Minion ID. ``Existing Config`` will leave the existing
+configuration unchanged.
+
+These settings can be defined on the command line using the following switches:
+
+- ``/default-config``
+- ``/custom-config=C:\Path\To\Custom\Config\minion``
+
+If neither option is passed and there is an existing config, the default is to
+use the existing config. If there is no existing config (new install) the
+default config will be used.
 
 Windows Installer: Multi-master configuration
 ---------------------------------------------

pkg/windows/req.txt

@@ -3,7 +3,7 @@ backports.ssl-match-hostname==3.5.0.1
 certifi
 cffi==1.10.0
 CherryPy==10.2.1
-cryptography==1.8.1
+cryptography==2.1.4
 enum34==1.1.6
 futures==3.1.1
 gitdb==0.6.4
@@ -23,13 +23,14 @@ pycparser==2.17
 pycrypto==2.6.1
 pycurl==7.43.0
 PyMySQL==0.7.11
-pyOpenSSL==17.0.0
-python-dateutil==2.6.0
-python-gnupg==0.4.0
+pyOpenSSL==17.5.0
+python-dateutil==2.6.1
+python-gnupg==0.4.1
+pythonnet==2.3.0
 pywin32==223
 PyYAML==3.12
-pyzmq==16.0.2
-requests==2.13.0
+pyzmq==16.0.3
+requests==2.18.4
 singledispatch==3.4.0.3
 smmap==0.9.0
 timelib==0.2.4

salt/cloud/clouds/openstack.py

@@ -708,7 +708,7 @@ def create(vm_):
     try:
         ip_address = __utils__['cloud.wait_for_fun'](
             __query_node,
-            update_args=(vm_,)
+            vm_=vm_
         )
     except (SaltCloudExecutionTimeout, SaltCloudExecutionFailure) as exc:
         try:

salt/modules/x509.py

@@ -20,6 +20,7 @@ import tempfile
 import re
 import datetime
 import ast
+import sys
 
 # Import salt libs
 import salt.utils.files
@@ -132,8 +133,8 @@ def _new_extension(name, value, critical=0, issuer=None, _pyfree=1):
             'value must be precomputed hash')
 
     # ensure name and value are bytes
-    name = salt.utils.stringutils.to_bytes(name)
-    value = salt.utils.stringutils.to_bytes(value)
+    name = salt.utils.stringutils.to_str(name)
+    value = salt.utils.stringutils.to_str(value)
 
     try:
         ctx = M2Crypto.m2.x509v3_set_nconf()
@@ -320,9 +321,9 @@ def _text_or_file(input_):
     '''
     if os.path.isfile(input_):
         with salt.utils.files.fopen(input_) as fp_:
-            return salt.utils.stringutils.to_bytes(fp_.read())
+            return salt.utils.stringutils.to_str(fp_.read())
     else:
-        return input_
+        return salt.utils.stringutils.to_str(input_)
 
 
 def _parse_subject(subject):
@@ -497,7 +498,7 @@ def get_pem_entry(text, pem_type=None):
         ret += pem_body[i:i + 64] + '\n'
     ret += pem_footer + '\n'
 
-    return ret.encode('ascii')
+    return salt.utils.stringutils.to_bytes(ret, encoding='ascii')
 
 
 def get_pem_entries(glob_path):
@@ -682,12 +683,12 @@ def get_public_key(key, passphrase=None, asObj=False):
 
     if isinstance(key, M2Crypto.X509.X509):
         rsa = key.get_pubkey().get_rsa()
-        text = ''
+        text = b''
     else:
         text = _text_or_file(key)
         text = get_pem_entry(text)
 
-    if text.startswith('-----BEGIN PUBLIC KEY-----'):
+    if text.startswith(b'-----BEGIN PUBLIC KEY-----'):
         if not asObj:
             return text
         bio = M2Crypto.BIO.MemoryBuffer()
@@ -695,14 +696,14 @@ def get_public_key(key, passphrase=None, asObj=False):
         rsa = M2Crypto.RSA.load_pub_key_bio(bio)
 
     bio = M2Crypto.BIO.MemoryBuffer()
-    if text.startswith('-----BEGIN CERTIFICATE-----'):
+    if text.startswith(b'-----BEGIN CERTIFICATE-----'):
         cert = M2Crypto.X509.load_cert_string(text)
         rsa = cert.get_pubkey().get_rsa()
-    if text.startswith('-----BEGIN CERTIFICATE REQUEST-----'):
+    if text.startswith(b'-----BEGIN CERTIFICATE REQUEST-----'):
         csr = M2Crypto.X509.load_request_string(text)
         rsa = csr.get_pubkey().get_rsa()
-    if (text.startswith('-----BEGIN PRIVATE KEY-----') or
-            text.startswith('-----BEGIN RSA PRIVATE KEY-----')):
+    if (text.startswith(b'-----BEGIN PRIVATE KEY-----') or
+            text.startswith(b'-----BEGIN RSA PRIVATE KEY-----')):
         rsa = M2Crypto.RSA.load_key_string(
             text, callback=_passphrase_callback(passphrase))
 
@@ -851,7 +852,7 @@ def create_private_key(path=None,
             pem_type='(?:RSA )?PRIVATE KEY'
         )
     else:
-        return bio.read_all()
+        return salt.utils.stringutils.to_str(bio.read_all())
 
 
 def create_crl(  # pylint: disable=too-many-arguments,too-many-locals
@@ -1429,7 +1430,14 @@ def create_certificate(
     if 'serial_number' not in kwargs:
         kwargs['serial_number'] = _dec2hex(
             random.getrandbits(kwargs['serial_bits']))
-    cert.set_serial_number(int(kwargs['serial_number'].replace(':', ''), 16))
+    serial_number = int(kwargs['serial_number'].replace(':', ''), 16)
+    # With Python3 we occasionally end up with an INT
+    # that is too large because Python3 no longer supports long INTs.
+    # If we're larger than the maxsize value
+    # then we adjust the serial number.
+    if serial_number > sys.maxsize:
+        serial_number = serial_number - sys.maxsize
+    cert.set_serial_number(serial_number)
 
     # Set validity dates
     # pylint: disable=no-member

salt/states/x509.py

@@ -308,6 +308,9 @@ def private_key_managed(name,
         file_args['contents'] = __salt__['x509.create_private_key'](
             text=True, bits=bits, passphrase=passphrase, cipher=cipher, verbose=verbose)
 
+    # Ensure the key contents are a string before passing it along
+    file_args['contents'] = salt.utils.stringutils.to_str(file_args['contents'])
+
     ret = __states__['file.managed'](**file_args)
     if ret['changes'] and new_key:
         ret['changes'] = {'new': 'New private key generated'}

salt/utils/event.py

@@ -433,7 +433,7 @@ class SaltEvent(object):
 
         if six.PY2:
             mtag, sep, mdata = raw.partition(TAGEND)  # split tag from data
-            data = serial.loads(mdata)
+            data = serial.loads(mdata, encoding='utf-8')
         else:
             mtag, sep, mdata = raw.partition(salt.utils.stringutils.to_bytes(TAGEND))  # split tag from data
             mtag = salt.utils.stringutils.to_str(mtag)

tests/integration/cli/test_batch.py

@@ -18,7 +18,7 @@ class BatchTest(ShellCase):
         '''
         Tests executing a simple batch command to help catch regressions
         '''
-        ret = 'Executing run on [\'sub_minion\']'
+        ret = 'Executing run on [{0}]'.format(repr('sub_minion'))
 
         cmd = self.run_salt('\'*minion\' test.echo \'batch testing\' -b 50%')
         self.assertIn(ret, cmd)
@@ -28,7 +28,7 @@ class BatchTest(ShellCase):
         Tests executing a simple batch command using a number division instead of
         a percentage with full batch CLI call.
         '''
-        ret = "Executing run on ['minion', 'sub_minion']"
+        ret = "Executing run on [{0}, {1}]".format(repr('minion'), repr('sub_minion'))
         cmd = self.run_salt('\'*minion\' test.ping --batch-size 2')
         self.assertIn(ret, cmd)
 
@@ -38,8 +38,8 @@ class BatchTest(ShellCase):
         targeting.
         '''
         os_grain = ''
-        sub_min_ret = "Executing run on ['sub_minion']"
-        min_ret = "Executing run on ['minion']"
+        sub_min_ret = "Executing run on [{0}]".format(repr('sub_minion'))
+        min_ret = "Executing run on [{0}]".format(repr('minion'))
 
         for item in self.run_salt('minion grains.get os'):
             if item != 'minion':

tests/integration/files/file/base/issue-46672.sls

@@ -0,0 +1,3 @@
+echo1:
+  cmd.run:
+    - name: "echo 'This is Æ test!'"

tests/integration/modules/test_state.py

@@ -1801,6 +1801,16 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin):
                              'File {0} updated'.format(file_name))
             self.assertEqual(val['changes']['diff'], 'New file')
 
+    def test_state_sls_unicode_characters(self):
+        '''
+        test state.sls when state file contains non-ascii characters
+        '''
+        ret = self.run_function('state.sls', ['issue-46672'])
+        log.debug('== ret %s ==', type(ret))
+
+        _expected = "cmd_|-echo1_|-echo 'This is Æ test!'_|-run"
+        self.assertIn(_expected, ret)
+
     def tearDown(self):
         nonbase_file = os.path.join(TMP, 'nonbase_env')
         if os.path.isfile(nonbase_file):

tests/integration/states/test_file.py

@@ -2182,7 +2182,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
             'salt_utf8_tests',
             '{0}.txt'.format(korean_1)
         )
-        test_file_encoded = salt.utils.stringutils.to_str(test_file)
+        test_file_encoded = test_file
         template_path = os.path.join(TMP_STATE_TREE, 'issue-8947.sls')
         # create the sls template
         template_lines = [
@@ -2247,46 +2247,45 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
                 ' 한국어 시험\n'
                 '+마지막 행\n'
             )
-            diff = salt.utils.stringutils.to_str(diff)
             # future_lint: disable=blacklisted-function
             expected = {
-                str('file_|-some-utf8-file-create_|-{0}_|-managed').format(test_file_encoded): {
+                'file_|-some-utf8-file-create_|-{0}_|-managed'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 0,
-                    'comment': str('File {0} updated').format(test_file_encoded),
+                    'comment': 'File {0} updated'.format(test_file_encoded),
                     'diff': 'New file'
                 },
-                str('file_|-some-utf8-file-create2_|-{0}_|-managed').format(test_file_encoded): {
+                'file_|-some-utf8-file-create2_|-{0}_|-managed'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 1,
-                    'comment': str('File {0} updated').format(test_file_encoded),
+                    'comment': 'File {0} updated'.format(test_file_encoded),
                     'diff': diff
                 },
-                str('file_|-some-utf8-file-exists_|-{0}_|-exists').format(test_file_encoded): {
+                'file_|-some-utf8-file-exists_|-{0}_|-exists'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 2,
-                    'comment': str('Path {0} exists').format(test_file_encoded)
+                    'comment': 'Path {0} exists'.format(test_file_encoded)
                 },
-                str('cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run').format(test_file_encoded): {
-                    'name': str('cat "{0}"').format(test_file_encoded),
+                'cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run'.format(test_file_encoded): {
+                    'name': 'cat "{0}"'.format(test_file_encoded),
                     '__run_num__': 3,
-                    'comment': str('Command "cat "{0}"" run').format(test_file_encoded),
-                    'stdout': str('{0}\n{1}\n{2}').format(
-                        salt.utils.stringutils.to_str(korean_2),
-                        salt.utils.stringutils.to_str(korean_1),
-                        salt.utils.stringutils.to_str(korean_3),
+                    'comment': 'Command "cat "{0}"" run'.format(test_file_encoded),
+                    'stdout': '{0}\n{1}\n{2}'.format(
+                        korean_2,
+                        korean_1,
+                        korean_3,
                     )
                 },
-                str('cmd_|-some-utf8-file-content-remove_|-rm -f "{0}"_|-run').format(test_file_encoded): {
-                    'name': str('rm -f "{0}"').format(test_file_encoded),
+                'cmd_|-some-utf8-file-content-remove_|-rm -f "{0}"_|-run'.format(test_file_encoded): {
+                    'name': 'rm -f "{0}"'.format(test_file_encoded),
                     '__run_num__': 4,
-                    'comment': str('Command "rm -f "{0}"" run').format(test_file_encoded),
+                    'comment': 'Command "rm -f "{0}"" run'.format(test_file_encoded),
                     'stdout': ''
                 },
-                str('file_|-some-utf8-file-removed_|-{0}_|-missing').format(test_file_encoded): {
+                'file_|-some-utf8-file-removed_|-{0}_|-missing'.format(test_file_encoded): {
                     'name': test_file_encoded,
                     '__run_num__': 5,
-                    'comment': str('Path {0} is missing').format(test_file_encoded),
+                    'comment': 'Path {0} is missing'.format(test_file_encoded),
                 }
             }
             # future_lint: enable=blacklisted-function
@@ -2308,7 +2307,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
 
             self.assertEqual(expected, result)
             # future_lint: disable=blacklisted-function
-            cat_id = str('cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run').format(test_file_encoded)
+            cat_id = 'cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run'.format(test_file_encoded)
             # future_lint: enable=blacklisted-function
             self.assertEqual(
                 salt.utils.stringutils.to_unicode(result[cat_id]['stdout']),

tests/unit/modules/test_x509.py

@@ -78,7 +78,7 @@ class X509TestCase(TestCase, LoaderModuleMockMixin):
         Test private function _parse_subject(subject) it handles a missing fields
         :return:
         '''
-        ca_key = '''-----BEGIN RSA PRIVATE KEY-----
+        ca_key = b'''-----BEGIN RSA PRIVATE KEY-----
 MIICWwIBAAKBgQCjdjbgL4kQ8Lu73xeRRM1q3C3K3ptfCLpyfw38LRnymxaoJ6ls
 pNSx2dU1uJ89YKFlYLo1QcEk4rJ2fdIjarV0kuNCY3rC8jYUp9BpAU5Z6p9HKeT1
 2rTPH81JyjbQDR5PyfCyzYOQtpwpB4zIUUK/Go7tTm409xGKbbUFugJNgQIDAQAB
@@ -163,4 +163,4 @@ c9bcgp7D7xD+TxWWNj4CSXEccJgGr91StV+gFg4ARQ==
                                       authorityKeyIdentifier='keyid,issuer:always',
                                       days_valid=3650,
                                       days_remaining=0)
-        self.assertIn('BEGIN CERTIFICATE', ret)
+        self.assertIn(b'BEGIN CERTIFICATE', ret)