Merge pull request #45024 from terminalmage/salt.client-unicode

[PY3] Add unicode literals to the clients and associated files/tests
This commit is contained in:
Nicole Thomas 2017-12-15 11:01:16 -05:00 committed by GitHub
commit 3e33276d13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
44 changed files with 104 additions and 77 deletions

View File

@ -19,7 +19,7 @@ The data structure needs to be:
# 4. How long do we wait for all of the replies?
#
# Import python libs
from __future__ import absolute_import, print_function
from __future__ import absolute_import, print_function, unicode_literals
import os
import time
import random
@ -38,6 +38,7 @@ import salt.utils.files
import salt.utils.jid
import salt.utils.minions
import salt.utils.platform
import salt.utils.stringutils
import salt.utils.user
import salt.utils.verify
import salt.utils.versions
@ -196,7 +197,7 @@ class LocalClient(object):
key_user,
self.skip_perm_errors)
with salt.utils.files.fopen(keyfile, 'r') as key:
return key.read()
return salt.utils.stringutils.to_unicode(key.read())
except (OSError, IOError, SaltClientError):
# Fall back to eauth
return ''
@ -1795,7 +1796,7 @@ class LocalClient(object):
**kwargs)
master_uri = 'tcp://' + salt.utils.zeromq.ip_bracket(self.opts['interface']) + \
':' + str(self.opts['ret_port'])
':' + six.text_type(self.opts['ret_port'])
channel = salt.transport.Channel.factory(self.opts,
crypt='clear',
master_uri=master_uri)
@ -1903,7 +1904,7 @@ class LocalClient(object):
**kwargs)
master_uri = 'tcp://' + salt.utils.zeromq.ip_bracket(self.opts['interface']) + \
':' + str(self.opts['ret_port'])
':' + six.text_type(self.opts['ret_port'])
channel = salt.transport.client.AsyncReqChannel.factory(self.opts,
io_loop=io_loop,
crypt='clear',

View File

@ -16,7 +16,7 @@ client applications.
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
# Import Salt libs

View File

@ -4,7 +4,7 @@ A collection of mixins useful for the various *Client interfaces
'''
# Import Python libs
from __future__ import absolute_import, print_function, with_statement
from __future__ import absolute_import, print_function, with_statement, unicode_literals
import fnmatch
import signal
import logging
@ -395,7 +395,7 @@ class SyncClientMixin(object):
data['success'] = salt.utils.state.check_result(data['return']['data'])
except (Exception, SystemExit) as ex:
if isinstance(ex, salt.exceptions.NotImplemented):
data['return'] = str(ex)
data['return'] = six.text_type(ex)
else:
data['return'] = 'Exception occurred in {0} {1}: {2}'.format(
self.client,

View File

@ -3,7 +3,7 @@
The main entry point for salt-api
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import signal
import logging

View File

@ -2,7 +2,7 @@
'''
The client libs to communicate with the salt master when running raet
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import python libs
import os

View File

@ -3,7 +3,7 @@
Create ssh executor system
'''
# Import python libs
from __future__ import absolute_import, print_function
from __future__ import absolute_import, print_function, unicode_literals
import base64
import copy
import getpass

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
import copy
import logging

View File

@ -2,7 +2,7 @@
'''
Manage transport commands via ssh
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import python libs
import re
@ -18,6 +18,8 @@ import salt.defaults.exitcodes
import salt.utils.nb_popen
import salt.utils.vt
from salt.ext import six
log = logging.getLogger(__name__)
SSH_PASSWORD_PROMPT_RE = re.compile(r'(?:.*)[Pp]assword(?: for .*)?:', re.M)
@ -88,7 +90,7 @@ class Shell(object):
self.host = host.strip('[]')
self.user = user
self.port = port
self.passwd = str(passwd) if passwd else passwd
self.passwd = six.text_type(passwd) if passwd else passwd
self.priv = priv
self.timeout = timeout
self.sudo = sudo

View File

@ -8,7 +8,7 @@ helper script used by salt.client.ssh.Single. It is here, in a
separate file, for convenience of development.
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import hashlib
import tarfile

View File

@ -2,7 +2,7 @@
'''
Create ssh executor system
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import python libs
import logging
import os
@ -17,6 +17,7 @@ import salt.client.ssh.shell
import salt.client.ssh
import salt.utils.files
import salt.utils.path
import salt.utils.stringutils
import salt.utils.thin
import salt.utils.url
import salt.utils.verify
@ -180,13 +181,13 @@ def prep_trans_tar(opts, file_client, chunks, file_refs, pillar=None, id_=None,
[salt.utils.url.create('_utils')],
]
with salt.utils.files.fopen(lowfn, 'w+') as fp_:
fp_.write(json.dumps(chunks))
fp_.write(salt.utils.stringutils.to_str(json.dumps(chunks)))
if pillar:
with salt.utils.files.fopen(pillarfn, 'w+') as fp_:
fp_.write(json.dumps(pillar))
fp_.write(salt.utils.stringutils.to_str(json.dumps(pillar)))
if roster_grains:
with salt.utils.files.fopen(roster_grainsfn, 'w+') as fp_:
fp_.write(json.dumps(roster_grains))
fp_.write(salt.utils.stringutils.to_str(json.dumps(roster_grains)))
if id_ is None:
id_ = ''
@ -194,7 +195,7 @@ def prep_trans_tar(opts, file_client, chunks, file_refs, pillar=None, id_=None,
cachedir = os.path.join('salt-ssh', id_).rstrip(os.sep)
except AttributeError:
# Minion ID should always be a str, but don't let an int break this
cachedir = os.path.join('salt-ssh', str(id_)).rstrip(os.sep)
cachedir = os.path.join('salt-ssh', six.string_types(id_)).rstrip(os.sep)
for saltenv in file_refs:
# Location where files in this saltenv will be cached

View File

@ -7,7 +7,7 @@ as ZeroMQ salt, but via ssh.
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import json
import copy

View File

@ -4,7 +4,7 @@ Return config information
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import re
import os

View File

@ -3,13 +3,14 @@
Wrap the cp module allowing for managed ssh file transfers
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import logging
import os
# Import salt libs
import salt.client.ssh
import salt.utils.files
import salt.utils.stringutils
import salt.utils.templates
from salt.exceptions import CommandExecutionError
@ -141,7 +142,7 @@ def _render_filenames(path, dest, saltenv, template):
# write out path to temp file
tmp_path_fn = salt.utils.files.mkstemp()
with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_:
fp_.write(contents)
fp_.write(salt.utils.stringutils.to_str(contents))
data = salt.utils.templates.TEMPLATE_REGISTRY[template](
tmp_path_fn,
to_str=True,

View File

@ -4,7 +4,7 @@ Return/control aspects of the grains data
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import collections
import copy
import math

View File

@ -7,7 +7,7 @@ Wrapper function for mine operations for salt-ssh
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import copy
# Import salt libs

View File

@ -2,7 +2,7 @@
'''
Extract the pillar data for this minion
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import python libs
import collections

View File

@ -10,7 +10,7 @@ salt-ssh calls and return the data from them.
No access control is needed because calls cannot originate from the minions.
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import copy
import logging

View File

@ -3,7 +3,7 @@
Create ssh executor system
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import python libs
import os
import time
@ -177,7 +177,7 @@ def sls(mods, saltenv='base', test=None, exclude=None, **kwargs):
return json.loads(stdout, object_hook=salt.utils.data.decode_dict)
except Exception as e:
log.error("JSON Render failed for: %s\n%s", stdout, stderr)
log.error(str(e))
log.error(six.text_type(e))
# If for some reason the json load fails, return the stdout
return stdout
@ -347,7 +347,7 @@ def low(data, **kwargs):
return json.loads(stdout, object_hook=salt.utils.data.decode_dict)
except Exception as e:
log.error("JSON Render failed for: %s\n%s", stdout, stderr)
log.error(str(e))
log.error(six.text_type(e))
# If for some reason the json load fails, return the stdout
return stdout
@ -435,7 +435,7 @@ def high(data, **kwargs):
return json.loads(stdout, object_hook=salt.utils.data.decode_dict)
except Exception as e:
log.error("JSON Render failed for: %s\n%s", stdout, stderr)
log.error(str(e))
log.error(six.text_type(e))
# If for some reason the json load fails, return the stdout
return stdout
@ -521,6 +521,8 @@ def check_request(name=None):
serial = salt.payload.Serial(__opts__)
if os.path.isfile(notify_path):
with salt.utils.files.fopen(notify_path, 'rb') as fp_:
# Not sure if this needs to be decoded since it is being returned,
# and msgpack serialization will encode it to bytes anyway.
req = serial.load(fp_)
if name:
return req[name]
@ -676,7 +678,7 @@ def highstate(test=None, **kwargs):
return json.loads(stdout, object_hook=salt.utils.data.decode_dict)
except Exception as e:
log.error("JSON Render failed for: %s\n%s", stdout, stderr)
log.error(str(e))
log.error(six.text_type(e))
# If for some reason the json load fails, return the stdout
return stdout
@ -756,7 +758,7 @@ def top(topfn, test=None, **kwargs):
return json.loads(stdout, object_hook=salt.utils.data.decode_dict)
except Exception as e:
log.error("JSON Render failed for: %s\n%s", stdout, stderr)
log.error(str(e))
log.error(six.text_type(e))
# If for some reason the json load fails, return the stdout
return stdout
@ -1119,7 +1121,7 @@ def single(fun, name, test=None, **kwargs):
return json.loads(stdout, object_hook=salt.utils.data.decode_dict)
except Exception as e:
log.error("JSON Render failed for: %s\n%s", stdout, stderr)
log.error(str(e))
log.error(six.text_type(e))
# If for some reason the json load fails, return the stdout
return stdout

View File

@ -5,7 +5,7 @@ used to manage salt keys directly without interfacing with the CLI.
'''
# Import python libs
from __future__ import absolute_import, print_function
from __future__ import absolute_import, print_function, unicode_literals
import os
import copy
import json
@ -30,6 +30,7 @@ import salt.utils.files
import salt.utils.kinds
import salt.utils.master
import salt.utils.sdb
import salt.utils.stringutils
import salt.utils.user
# pylint: disable=import-error,no-name-in-module,redefined-builtin
@ -125,7 +126,8 @@ class KeyCLI(object):
if 'token' in self.opts:
try:
with salt.utils.files.fopen(os.path.join(self.opts['key_dir'], '.root_key'), 'r') as fp_:
low['key'] = fp_.readline()
low['key'] = \
salt.utils.stringutils.to_unicode(fp_.readline())
except IOError:
low['token'] = self.opts['token']
#
@ -606,7 +608,9 @@ class Key(object):
for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(dir_)):
if not fn_.startswith('.'):
if os.path.isfile(os.path.join(dir_, fn_)):
ret[os.path.basename(dir_)].append(fn_)
ret[os.path.basename(dir_)].append(
salt.utils.stringutils.to_unicode(fn_)
)
except (OSError, IOError):
# key dir kind is not created yet, just skip
continue
@ -664,7 +668,8 @@ class Key(object):
for key in salt.utils.data.sorted_ignorecase(keys):
path = os.path.join(self.opts['pki_dir'], status, key)
with salt.utils.files.fopen(path, 'r') as fp_:
ret[status][key] = fp_.read()
ret[status][key] = \
salt.utils.stringutils.to_unicode(fp_.read())
return ret
def key_str_all(self):
@ -677,7 +682,8 @@ class Key(object):
for key in salt.utils.data.sorted_ignorecase(keys):
path = os.path.join(self.opts['pki_dir'], status, key)
with salt.utils.files.fopen(path, 'r') as fp_:
ret[status][key] = fp_.read()
ret[status][key] = \
salt.utils.stringutils.to_unicode(fp_.read())
return ret
def accept(self, match=None, match_dict=None, include_rejected=False, include_denied=False):
@ -1026,11 +1032,14 @@ class RaetKey(Key):
continue
path = os.path.join(road_cache, road)
with salt.utils.files.fopen(path, 'rb') as fp_:
# Do not use to_unicode to decode this. It needs to stay as
# bytes to be deserialized.
if ext == '.json':
data = json.load(fp_)
elif ext == '.msgpack':
data = msgpack.load(fp_)
if data['role'] not in minions:
role = salt.utils.stringutils.to_unicode(data['role'])
if role not in minions:
os.remove(path)
def gen_keys(self, keydir=None, keyname=None, keysize=None, user=None):

View File

@ -4,7 +4,7 @@ Execute salt convenience routines
'''
# Import python libs
from __future__ import absolute_import, print_function
from __future__ import absolute_import, print_function, unicode_literals
import os
import logging
@ -206,7 +206,7 @@ class Runner(RunnerClient):
if 'token' in self.opts:
try:
with salt.utils.files.fopen(os.path.join(self.opts['key_dir'], '.root_key'), 'r') as fp_:
low['key'] = fp_.readline()
low['key'] = salt.utils.stringutils.to_unicode(fp_.readline())
except IOError:
low['token'] = self.opts['token']

View File

@ -4,7 +4,7 @@ Modules used to control the master itself
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import collections
# Import salt libs
@ -15,6 +15,9 @@ import salt.transport
import salt.utils.error
import salt.utils.zeromq
# Import 3rd-party libs
from salt.ext import six
class WheelClient(salt.client.mixins.SyncClientMixin,
salt.client.mixins.AsyncClientMixin, object):
@ -65,7 +68,7 @@ class WheelClient(salt.client.mixins.SyncClientMixin,
if interface == '0.0.0.0':
interface = '127.0.0.1'
master_uri = 'tcp://' + salt.utils.zeromq.ip_bracket(interface) + \
':' + str(self.opts['ret_port'])
':' + six.text_type(self.opts['ret_port'])
channel = salt.transport.Channel.factory(self.opts,
crypt='clear',
master_uri=master_uri,

View File

@ -2,7 +2,7 @@
'''
Manage the master configuration file
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import python libs
import logging
@ -16,6 +16,9 @@ import salt.config
import salt.utils.files
from salt.utils.yamldumper import SafeOrderedDumper
# Import 3rd-party libs
from salt.ext import six
log = logging.getLogger(__name__)
@ -84,7 +87,7 @@ def update_config(file_name, yaml_contents):
yaml_out = yaml.safe_dump(yaml_contents, default_flow_style=False)
if not os.path.exists(dir_path):
log.debug('Creating directory {0}'.format(dir_path))
log.debug('Creating directory %s', dir_path)
os.makedirs(dir_path, 0o755)
file_path = os.path.join(dir_path, file_name)
@ -93,4 +96,4 @@ def update_config(file_name, yaml_contents):
return 'Wrote {0}'.format(file_name)
except (IOError, OSError, yaml.YAMLError, ValueError) as err:
return str(err)
return six.text_type(err)

View File

@ -3,7 +3,7 @@
Error generator to enable integration testing of salt wheel error handling
'''
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import python libs

View File

@ -4,7 +4,7 @@ Read in files from the file_root and save files to the file root
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
# Import salt libs
@ -88,7 +88,9 @@ def read(path, saltenv='base'):
form = fn_[full]
if form == 'txt':
with salt.utils.files.fopen(full, 'rb') as fp_:
ret.append({full: fp_.read()})
ret.append(
{full: salt.utils.stringutils.to_unicode(fp_.read())}
)
return ret
@ -110,5 +112,5 @@ def write(data, path, saltenv='base', index=0):
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
with salt.utils.files.fopen(dest, 'w+') as fp_:
fp_.write(data)
fp_.write(salt.utils.stringutils.to_str(data))
return 'Wrote data to file {0}'.format(dest)

View File

@ -28,7 +28,7 @@ using the :ref:`saltutil execution module <salt.modules.saltutil>`.
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
import hashlib
import logging
@ -356,9 +356,9 @@ def gen(id_=None, keysize=2048):
priv = salt.crypt.gen_keys(__opts__['pki_dir'], id_, keysize)
pub = '{0}.pub'.format(priv[:priv.rindex('.')])
with salt.utils.files.fopen(priv) as fp_:
ret['priv'] = fp_.read()
ret['priv'] = salt.utils.stringutils.to_unicode(fp_.read())
with salt.utils.files.fopen(pub) as fp_:
ret['pub'] = fp_.read()
ret['pub'] = salt.utils.stringutils.to_unicode(fp_.read())
# The priv key is given the Read-Only attribute. The causes `os.remove` to
# fail in Windows.
@ -416,7 +416,7 @@ def gen_accept(id_, keysize=2048, force=False):
if os.path.isfile(acc_path) and not force:
return {}
with salt.utils.files.fopen(acc_path, 'w+') as fp_:
fp_.write(ret['pub'])
fp_.write(salt.utils.stringutils.to_str(ret['pub']))
return ret

View File

@ -4,7 +4,7 @@ Wheel system wrapper for connected minions
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt libs
from salt.utils.cache import CacheCli

View File

@ -5,7 +5,7 @@ directories on the master server.
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
# Import salt libs
@ -89,7 +89,9 @@ def read(path, saltenv='base'):
form = fn_[full]
if form == 'txt':
with salt.utils.files.fopen(full, 'rb') as fp_:
ret.append({full: fp_.read()})
ret.append(
{full: salt.utils.stringutils.to_unicode(fp_.read())}
)
return ret
@ -111,5 +113,5 @@ def write(data, path, saltenv='base', index=0):
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
with salt.utils.files.fopen(dest, 'w+') as fp_:
fp_.write(data)
fp_.write(salt.utils.stringutils.to_str(data))
return 'Wrote data to file {0}'.format(dest)

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing libs
from tests.support.case import ModuleCase

View File

@ -1,7 +1,7 @@
# coding: utf-8
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing libs
from tests.support.unit import TestCase

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
# Import Salt Testing libs

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing libs
from tests.support.case import SyndicCase

View File

@ -4,7 +4,7 @@ Wheel functions for integration tests
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
def failure():

View File

@ -1,7 +1,7 @@
# encoding: utf-8
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
# Import Salt Testing libs

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
import shutil
import tempfile
@ -14,6 +14,7 @@ from tests.support.mixins import ShellCaseCommonTestsMixin
# Import 3rd-party libs
import yaml
from salt.ext import six
# Import Salt libs
import salt.utils.files
@ -42,7 +43,7 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin):
self.assertTrue(add_user)
self.assertTrue(add_pwd)
user_list = self.run_call('user.list_users')
self.assertIn(USERA, str(user_list))
self.assertIn(USERA, six.text_type(user_list))
except AssertionError:
self.run_call('user.delete {0} remove=True'.format(USERA))
self.skipTest(

View File

@ -3,7 +3,7 @@
salt-ssh testing
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
import shutil

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing Libs
from tests.support.case import SSHCase

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
import shutil

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing Libs
from tests.support.case import SSHCase

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing Libs
from tests.support.case import SSHCase

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os
import shutil

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing libs
from tests.support.unit import TestCase, skipIf

View File

@ -1,7 +1,7 @@
# coding: utf-8
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing libs
from tests.support.unit import TestCase

View File

@ -4,7 +4,7 @@
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
import os.path
# Import Salt Testing Libs

View File

@ -4,7 +4,7 @@
'''
# Import python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals
# Import Salt Testing libs
import tests.integration as integration