From b6181b5ed6e2880747d9e6f8c8f573dbc13d2a03 Mon Sep 17 00:00:00 2001
From: Benjamin Drung <benjamin.drung@profitbricks.com>
Date: Fri, 26 Jan 2018 13:00:13 +0100
Subject: [PATCH] Fix Unicode tests when run with LC_ALL=POSIX

When running the unit tests with the locale set to POSIX, some Unicode
tests fail:

$ LC_ALL=POSIX python3 ./tests/runtests.py --unit
[...]
======================================================================
ERROR: test_list_products (unit.modules.test_zypper.ZypperTestCase)
[CPU:0.0%|MEM:73.2%]
----------------------------------------------------------------------
Traceback (most recent call last):
  File "tests/unit/modules/test_zypper.py", line 236, in
test_list_products
    'stdout': get_test_data(filename)
  File "tests/unit/modules/test_zypper.py", line 53, in get_test_data
    return rfh.read()
  File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position
828: ordinal not in range(128)

======================================================================
ERROR: test_non_ascii (unit.templates.test_jinja.TestGetTemplate)
[CPU:0.0%|MEM:73.2%]
----------------------------------------------------------------------
Traceback (most recent call last):
  File "tests/unit/templates/test_jinja.py", line 341, in test_non_ascii
    result = fp.read()
  File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 5:
ordinal not in range(128)

======================================================================
ERROR: test_non_ascii_encoding
(unit.templates.test_jinja.TestGetTemplate)
[CPU:0.0%|MEM:73.2%]
----------------------------------------------------------------------
Traceback (most recent call last):
  File "tests/unit/templates/test_jinja.py", line 303, in
test_non_ascii_encoding
    fp_.read(),
  File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 5:
ordinal not in range(128)
----------------------------------------------------------------------

Therefore open files in binary mode and explicitly decode them with
utf-8 instead of their default locale.
---
 tests/unit/modules/test_zypper.py  |  4 ++--
 tests/unit/templates/test_jinja.py | 14 ++++++--------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/unit/modules/test_zypper.py b/tests/unit/modules/test_zypper.py
index ac8621307a..8d5e52aa69 100644
--- a/tests/unit/modules/test_zypper.py
+++ b/tests/unit/modules/test_zypper.py
@@ -49,8 +49,8 @@ def get_test_data(filename):
     with salt.utils.fopen(
             os.path.join(
                 os.path.join(
-                    os.path.dirname(os.path.abspath(__file__)), 'zypp'), filename)) as rfh:
-        return rfh.read()
+                    os.path.dirname(os.path.abspath(__file__)), 'zypp'), filename), mode='rb') as rfh:
+        return salt.utils.to_str(rfh.read(), 'utf-8')
 
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py
index 8516e974ec..0e69ea80b3 100644
--- a/tests/unit/templates/test_jinja.py
+++ b/tests/unit/templates/test_jinja.py
@@ -294,13 +294,13 @@ class TestGetTemplate(TestCase):
                                'file_roots': self.local_opts['file_roots'],
                                'pillar_roots': self.local_opts['pillar_roots']},
                          a='Hi', b='Sàlt', saltenv='test', salt=self.local_salt))
-            self.assertEqual(out, salt.utils.to_unicode('Hey world !Hi Sàlt !' + os.linesep))
+            self.assertEqual(out, u'Hey world !Hi Sàlt !' + os.linesep)
             self.assertEqual(fc.requests[0]['path'], 'salt://macro')
 
             filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'non_ascii')
-            with salt.utils.fopen(filename) as fp_:
+            with salt.utils.fopen(filename, mode='rb') as fp_:
                 out = render_jinja_tmpl(
-                    fp_.read(),
+                    salt.utils.to_unicode(fp_.read(), 'utf-8'),
                     dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote',
                                'file_roots': self.local_opts['file_roots'],
                                'pillar_roots': self.local_opts['pillar_roots']},
@@ -337,11 +337,9 @@ class TestGetTemplate(TestCase):
     def test_non_ascii(self):
         fn = os.path.join(TEMPLATES_DIR, 'files', 'test', 'non_ascii')
         out = JINJA(fn, opts=self.local_opts, saltenv='test')
-        with salt.utils.fopen(out['data']) as fp:
-            result = fp.read()
-            if six.PY2:
-                result = salt.utils.to_unicode(result)
-            self.assertEqual(salt.utils.to_unicode('Assunção' + os.linesep), result)
+        with salt.utils.fopen(out['data'], mode='rb') as fp:
+            result = salt.utils.to_unicode(fp.read(), 'utf-8')
+            self.assertEqual(u'Assunção' + os.linesep, result)
 
     def test_get_context_has_enough_context(self):
         template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf'