Merge pull request #44985 from terminalmage/salt-ext-unicode

[PY3] Add unicode_literals to files in root salt/ dir (and associated tests)

salt/__init__.py

@@ -4,7 +4,7 @@ Salt package
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import warnings
 
 # All salt related deprecation warnings should be shown once each!

salt/crypt.py

@@ -5,6 +5,8 @@ masters, encrypting and decrypting payloads, preparing messages, and
 authenticating peers
 '''
 # Import python libs
+# NOTE: We can't use unicode_literals because this module implicitly uses
+# the Array class, which has incompatibilities with it.
 from __future__ import absolute_import, print_function
 import os
 import sys
@@ -171,7 +173,7 @@ def _get_rsa_key(path, passphrase):
     retrieve the key from disk.
     '''
     log.debug('salt.crypt._get_rsa_key: Loading private key')
-    return _get_key_with_evict(path, str(os.path.getmtime(path)), passphrase)
+    return _get_key_with_evict(path, six.text_type(os.path.getmtime(path)), passphrase)
 
 
 def sign_message(privkey_path, message, passphrase=None):

salt/exceptions.py

@@ -54,7 +54,7 @@ class SaltException(Exception):
             # Some non-string input was passed. Run the parent dunder init with
             # a str version, and convert the passed value to unicode for the
             # message/strerror attributes.
-            super(SaltException, self).__init__(str(message))
+            super(SaltException, self).__init__(str(message))  # future lint: blacklisted-function
             self.message = self.strerror = unicode(message)  # pylint: disable=incompatible-py3-code
 
     def __unicode__(self):
@@ -66,8 +66,7 @@ class SaltException(Exception):
         transport via msgpack
         '''
         if six.PY3:
-            # The message should be a str type, not a unicode
-            return {'message': str(self), 'args': self.args}
+            return {'message': six.text_type(self), 'args': self.args}
         return dict(message=self.__unicode__(), args=self.args)
 
 

salt/fileclient.py

@@ -2,7 +2,7 @@
 '''
 Classes that manage file clients
 '''
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import python libs
 import contextlib
@@ -504,7 +504,7 @@ class Client(object):
                     'Path \'{0}\' is not absolute'.format(url_path)
                 )
             if dest is None:
-                with salt.utils.files.fopen(url_path, 'r') as fp_:
+                with salt.utils.files.fopen(url_path, 'rb') as fp_:
                     data = fp_.read()
                 return data
             return url_path
@@ -512,7 +512,7 @@ class Client(object):
         if url_scheme == 'salt':
             result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
             if result and dest is None:
-                with salt.utils.files.fopen(result, 'r') as fp_:
+                with salt.utils.files.fopen(result, 'rb') as fp_:
                     data = fp_.read()
                 return data
             return result
@@ -1232,7 +1232,7 @@ class RemoteClient(Client):
                     data_type = type(data).__name__
                 except AttributeError:
                     # Shouldn't happen, but don't let this cause a traceback.
-                    data_type = str(type(data))
+                    data_type = six.text_type(type(data))
                 transport_tries += 1
                 log.warning(
                     'Data transport is broken, got: %s, type: %s, '

salt/loader.py

@@ -6,7 +6,7 @@ plugin interfaces used by Salt.
 '''
 
 # Import python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import os
 import sys
 import time
@@ -989,7 +989,7 @@ def _generate_module(name):
 
     code = u"'''Salt loaded {0} parent module'''".format(name.split('.')[-1])
     # ModuleType can't accept a unicode type on PY2
-    module = types.ModuleType(str(name))
+    module = types.ModuleType(str(name))  # future lint: disable=blacklisted-function
     exec(code, module.__dict__)
     sys.modules[name] = module
 
@@ -1436,7 +1436,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         except IOError:
             raise
         except ImportError as exc:
-            if 'magic number' in str(exc):
+            if 'magic number' in six.text_type(exc):
                 error_msg = 'Failed to import {0} {1}. Bad magic number. If migrating from Python2 to Python3, remove all .pyc files and try again.'.format(self.tag, name)
                 log.warning(error_msg)
                 self.missing_modules[name] = error_msg

salt/payload.py

@@ -6,7 +6,7 @@ in here
 '''
 
 # Import python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 # import sys  # Use if sys is commented out below
 import logging
 import gc
@@ -201,9 +201,9 @@ class Serial(object):
                 # This is a spurious lint failure as we are gating this check
                 # behind a check for six.PY2.
                 if six.PY2 and isinstance(obj, long) and long > pow(2, 64):  # pylint: disable=incompatible-py3-code
-                    return str(obj)
+                    return six.text_type(obj)
                 elif six.PY3 and isinstance(obj, int) and int > pow(2, 64):
-                    return str(obj)
+                    return six.text_type(obj)
                 else:
                     return obj
             if msgpack.version >= (0, 4, 0):
@@ -252,12 +252,12 @@ class Serial(object):
                 if isinstance(obj, immutabletypes.ImmutableSet):
                     return set(obj)
 
-            if "datetime.datetime" in str(e):
+            if "datetime.datetime" in six.text_type(e):
                 if msgpack.version >= (0, 4, 0):
                     return msgpack.dumps(datetime_encoder(msg), use_bin_type=use_bin_type)
                 else:
                     return msgpack.dumps(datetime_encoder(msg))
-            elif "Immutable" in str(e):
+            elif "Immutable" in six.text_type(e):
                 if msgpack.version >= (0, 4, 0):
                     return msgpack.dumps(msg, default=immutable_encoder, use_bin_type=use_bin_type)
                 else:

salt/scripts.py

@@ -4,7 +4,7 @@ This module contains the function calls to execute command line scripts
 '''
 
 # Import python libs
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import, print_function, unicode_literals
 import os
 import sys
 import time

salt/syspaths.py

@@ -18,7 +18,7 @@
 '''
 
 # Import python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import sys
 import os.path
 
@@ -31,7 +31,7 @@ try:
     import salt._syspaths as __generated_syspaths  # pylint: disable=no-name-in-module
 except ImportError:
     import types
-    __generated_syspaths = types.ModuleType('salt._syspaths')
+    __generated_syspaths = types.ModuleType(str('salt._syspaths'))  # future lint: blacklisted-function
     for key in ('ROOT_DIR', 'CONFIG_DIR', 'CACHE_DIR', 'SOCK_DIR',
                 'SRV_ROOT_DIR', 'BASE_FILE_ROOTS_DIR', 'HOME_DIR',
                 'BASE_PILLAR_ROOTS_DIR', 'BASE_THORIUM_ROOTS_DIR',

salt/template.py

@@ -3,7 +3,7 @@
 Manage basic template commands
 '''
 
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import Python libs
 import time

salt/textformat.py

@@ -3,7 +3,7 @@
 ANSI escape code utilities, see
 http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf
 '''
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import 3rd-party libs
 from salt.ext import six

salt/version.py

@@ -730,7 +730,7 @@ def msi_conformant_version():
 
     Note that the commit count for tags is 0(zero)
     '''
-    year2 = int(str(__saltstack_version__.major)[2:])
+    year2 = int(six.text_type(__saltstack_version__.major)[2:])
     month = __saltstack_version__.minor
     minor = __saltstack_version__.bugfix
     commi = __saltstack_version__.noc

tests/integration/loader/test_ext_grains.py

@@ -7,7 +7,7 @@
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import os
 import time
 

tests/integration/loader/test_ext_modules.py

@@ -10,7 +10,7 @@
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import os
 import time
 

tests/unit/loader/test_globals.py

@@ -7,7 +7,7 @@
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import Salt Testing libs
 from tests.support.case import ModuleCase

tests/unit/loader/test_interfaces.py

@@ -7,7 +7,7 @@
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import Salt Testing libs
 from tests.support.unit import TestCase

tests/unit/loader/test_loader.py

@@ -7,7 +7,7 @@
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import inspect
 import logging
 import tempfile
@@ -83,7 +83,7 @@ class LazyLoaderTest(TestCase):
         self.module_file = os.path.join(self.module_dir,
                                         '{0}.py'.format(self.module_name))
         with salt.utils.files.fopen(self.module_file, 'w') as fh:
-            fh.write(loader_template)
+            fh.write(salt.utils.stringutils.to_str(loader_template))
             fh.flush()
             os.fsync(fh.fileno())
 
@@ -306,8 +306,10 @@ class LazyLoaderSingleItem(TestCase):
             self.assertEqual(err.exception[0],
                              'The key \'%s\' should contain a \'.\'')
         else:
-            self.assertEqual(str(err.exception),
-                             str(("The key '%s' should contain a '.'", 'testing_no_dot')))
+            self.assertEqual(
+                six.text_type(err.exception),
+                six.text_type(("The key '%s' should contain a '.'", 'testing_no_dot'))
+            )
 
 
 module_template = '''
@@ -863,7 +865,11 @@ class LazyLoaderDeepSubmodReloadingTest(TestCase):
         with salt.utils.files.fopen(os.path.join(self.module_dir, '__init__.py'), 'w') as fh:
             # No .decode() needed here as deep_init_base is defined as str and
             # not bytes.
-            fh.write(deep_init_base.format(self.module_name))
+            fh.write(
+                salt.utils.stringutils.to_str(
+                    deep_init_base.format(self.module_name)
+                )
+            )
             fh.flush()
             os.fsync(fh.fileno())  # flush to disk
 

tests/unit/templates/test_jinja.py

@@ -39,6 +39,8 @@ from salt.utils.templates import (
     JINJA,
     render_jinja_tmpl
 )
+# dateutils is needed so that the strftime jinja filter is loaded
+import salt.utils.dateutils  # pylint: disable=unused-import
 import salt.utils.files
 import salt.utils.stringutils
 
@@ -103,7 +105,7 @@ class TestSaltCacheLoader(TestCase):
         res = loader.get_source(None, 'hello_simple')
         assert len(res) == 3
         # res[0] on Windows is unicode and use os.linesep so it works cross OS
-        self.assertEqual(str(res[0]), 'world' + os.linesep)
+        self.assertEqual(six.text_type(res[0]), 'world' + os.linesep)
         tmpl_dir = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_simple')
         self.assertEqual(res[1], tmpl_dir)
         assert res[2](), 'Template up to date?'
@@ -181,7 +183,7 @@ class TestGetTemplate(TestCase):
         fn_ = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_simple')
         with salt.utils.files.fopen(fn_) as fp_:
             out = render_jinja_tmpl(
-                fp_.read(),
+                salt.utils.stringutils.to_unicode(fp_.read()),
                 dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)
             )
         self.assertEqual(out, 'world' + os.linesep)
@@ -194,7 +196,7 @@ class TestGetTemplate(TestCase):
         filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import')
         with salt.utils.files.fopen(filename) as fp_:
             out = render_jinja_tmpl(
-                fp_.read(),
+                salt.utils.stringutils.to_unicode(fp_.read()),
                 dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)
             )
         self.assertEqual(out, 'Hey world !a b !' + os.linesep)
@@ -211,7 +213,7 @@ class TestGetTemplate(TestCase):
             filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import')
             with salt.utils.files.fopen(filename) as fp_:
                 out = render_jinja_tmpl(
-                    fp_.read(),
+                    salt.utils.stringutils.to_unicode(fp_.read()),
                     dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote',
                                'file_roots': self.local_opts['file_roots'],
                                'pillar_roots': self.local_opts['pillar_roots']},
@@ -240,7 +242,7 @@ class TestGetTemplate(TestCase):
                     SaltRenderError,
                     expected,
                     render_jinja_tmpl,
-                    fp_.read(),
+                    salt.utils.stringutils.to_unicode(fp_.read()),
                     dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
 
     def test_macro_additional_log_for_undefined(self):
@@ -264,7 +266,7 @@ class TestGetTemplate(TestCase):
                     SaltRenderError,
                     expected,
                     render_jinja_tmpl,
-                    fp_.read(),
+                    salt.utils.stringutils.to_unicode(fp_.read()),
                     dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
 
     def test_macro_additional_log_syntaxerror(self):
@@ -288,7 +290,7 @@ class TestGetTemplate(TestCase):
                     SaltRenderError,
                     expected,
                     render_jinja_tmpl,
-                    fp_.read(),
+                    salt.utils.stringutils.to_unicode(fp_.read()),
                     dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
 
     def test_non_ascii_encoding(self):
@@ -297,7 +299,7 @@ class TestGetTemplate(TestCase):
             filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import')
             with salt.utils.files.fopen(filename) as fp_:
                 out = render_jinja_tmpl(
-                    fp_.read(),
+                    salt.utils.stringutils.to_unicode(fp_.read()),
                     dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote',
                                'file_roots': self.local_opts['file_roots'],
                                'pillar_roots': self.local_opts['pillar_roots']},
@@ -308,7 +310,7 @@ class TestGetTemplate(TestCase):
             filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'non_ascii')
             with salt.utils.files.fopen(filename) as fp_:
                 out = render_jinja_tmpl(
-                    fp_.read(),
+                    salt.utils.stringutils.to_unicode(fp_.read()),
                     dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote',
                                'file_roots': self.local_opts['file_roots'],
                                'pillar_roots': self.local_opts['pillar_roots']},
@@ -374,7 +376,7 @@ class TestGetTemplate(TestCase):
             salt=self.local_salt
         )
         with salt.utils.files.fopen(out['data']) as fp:
-            result = fp.read().decode(__salt_system_encoding__)
+            result = salt.utils.stringutils.to_unicode(fp.read())
             self.assertEqual(salt.utils.stringutils.to_unicode('Assunção' + os.linesep), result)
 
     def test_get_context_has_enough_context(self):

tests/unit/test_payload.py

@@ -8,7 +8,7 @@
 '''
 
 # Import Salt libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import time
 import errno
 import threading
@@ -84,9 +84,10 @@ class SREQTestCase(TestCase):
                     #  Wait for next request from client
                     message = socket.recv(zmq.NOBLOCK)
                     msg_deserialized = payload.loads(message)
-                    log.info('Echo server received message: {0}'.format(msg_deserialized))
+                    log.info('Echo server received message: %s', msg_deserialized)
                     if isinstance(msg_deserialized['load'], dict) and msg_deserialized['load'].get('sleep'):
-                        log.info('Test echo server sleeping for {0} seconds'.format(msg_deserialized['load']['sleep']))
+                        log.info('Test echo server sleeping for %s seconds',
+                                 msg_deserialized['load']['sleep'])
                         time.sleep(msg_deserialized['load']['sleep'])
                     socket.send(message)
                 except zmq.ZMQError as exc:

tests/unit/test_template.py

@@ -4,7 +4,7 @@
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 # Import Salt Testing libs
 from tests.support.unit import skipIf, TestCase

tests/unit/utils/test_templates.py

@@ -4,7 +4,7 @@ Tests for salt.utils.data
 '''
 
 # Import Python libs
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 import textwrap
 
 # Import Salt libs