From c27bb7c1162dc6cf7937c5ea032be4c1d1c057b1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 9 Jan 2018 22:34:27 -0600 Subject: [PATCH 1/2] [PY3] Update fileserver for Unicode/PY3 compatibility This adds a unicode_literals import to the fileserver code, and also ensures that we return unicode paths from the serialized file/env list caches (which when deserialized are still encoded str types on PY2). It also adds a test and modifies a few other existing tests to confirm that unicode paths/envs are returned *as unicode* in the return data. --- salt/fileserver/__init__.py | 63 +++++++------ salt/fileserver/azurefs.py | 35 +++---- salt/fileserver/gitfs.py | 2 +- salt/fileserver/hgfs.py | 93 ++++++++++--------- salt/fileserver/minionfs.py | 29 +++--- salt/fileserver/roots.py | 29 +++--- salt/fileserver/s3fs.py | 31 ++++--- salt/fileserver/svnfs.py | 87 ++++++++--------- salt/utils/gitfs.py | 71 ++++++-------- tests/integration/files/file/base/питон.txt | 3 + .../integration/files/file/base/соль/foo.txt | 1 + tests/unit/fileserver/test_gitfs.py | 16 +++- tests/unit/fileserver/test_map.py | 2 +- tests/unit/fileserver/test_roots.py | 38 +++++--- 14 files changed, 270 insertions(+), 230 deletions(-) create mode 100644 tests/integration/files/file/base/питон.txt create mode 100644 tests/integration/files/file/base/соль/foo.txt diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index 3bf0d28b69..052ad82710 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -4,7 +4,7 @@ File server pluggable modules and generic backend functions ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import collections import errno import fnmatch @@ -15,6 +15,7 @@ import time # Import salt libs import salt.loader +import salt.utils.data import salt.utils.files import salt.utils.locales import salt.utils.path @@ -41,7 +42,7 @@ def _unlock_cache(w_lock): elif os.path.isfile(w_lock): os.unlink(w_lock) except (OSError, IOError) as exc: - log.trace('Error removing lockfile {0}: {1}'.format(w_lock, exc)) + log.trace('Error removing lockfile %s: %s', w_lock, exc) def _lock_cache(w_lock): @@ -52,7 +53,7 @@ def _lock_cache(w_lock): raise return False else: - log.trace('Lockfile {0} created'.format(w_lock)) + log.trace('Lockfile %s created', w_lock) return True @@ -95,9 +96,10 @@ def wait_lock(lk_fn, dest, wait_timeout=0): if timeout: if time.time() > timeout: raise ValueError( - 'Timeout({0}s) for {1} ' - '(lock: {2}) elapsed'.format( - wait_timeout, dest, lk_fn)) + 'Timeout({0}s) for {1} (lock: {2}) elapsed'.format( + wait_timeout, dest, lk_fn + ) + ) return False @@ -130,9 +132,11 @@ def check_file_list_cache(opts, form, list_cache, w_lock): if age < opts.get('fileserver_list_cache_time', 20): # Young enough! Load this sucker up! with salt.utils.files.fopen(list_cache, 'rb') as fp_: - log.trace('Returning file_lists cache data from ' - '{0}'.format(list_cache)) - return serial.load(fp_).get(form, []), False, False + log.trace( + 'Returning file_lists cache data from %s', + list_cache + ) + return salt.utils.data.decode(serial.load(fp_).get(form, [])), False, False elif _lock_cache(w_lock): # Set the w_lock and go refresh_cache = True @@ -157,7 +161,7 @@ def write_file_list_cache(opts, data, list_cache, w_lock): with salt.utils.files.fopen(list_cache, 'w+b') as fp_: fp_.write(serial.dumps(data)) _unlock_cache(w_lock) - log.trace('Lockfile {0} removed'.format(w_lock)) + log.trace('Lockfile %s removed', w_lock) def check_env_cache(opts, env_cache): @@ -168,9 +172,9 @@ def check_env_cache(opts, env_cache): return None try: with salt.utils.files.fopen(env_cache, 'rb') as fp_: - log.trace('Returning env cache data from {0}'.format(env_cache)) + log.trace('Returning env cache data from %s', env_cache) serial = salt.payload.Serial(opts) - return serial.load(fp_) + return salt.utils.data.decode(serial.load(fp_)) except (IOError, OSError): pass return None @@ -193,8 +197,9 @@ def generate_mtime_map(opts, path_map): except (OSError, IOError): # skip dangling symlinks log.info( - 'Failed to get mtime on {0}, ' - 'dangling symlink ?'.format(file_path)) + 'Failed to get mtime on %s, dangling symlink?', + file_path + ) continue return file_map @@ -242,10 +247,10 @@ def reap_fileserver_cache_dir(cache_base, find_func): try: filename, _, hash_type = file_rel_path.rsplit('.', 2) except ValueError: - log.warning(( - 'Found invalid hash file [{0}] when attempting to reap' - ' cache directory.' - ).format(file_)) + log.warning( + 'Found invalid hash file [%s] when attempting to reap ' + 'cache directory', file_ + ) continue # do we have the file? ret = find_func(filename, saltenv=saltenv) @@ -265,9 +270,8 @@ def is_file_ignored(opts, fname): for regex in opts['file_ignore_regex']: if re.search(regex, fname): log.debug( - 'File matching file_ignore_regex. Skipping: {0}'.format( - fname - ) + 'File matching file_ignore_regex. Skipping: %s', + fname ) return True @@ -275,9 +279,8 @@ def is_file_ignored(opts, fname): for glob in opts['file_ignore_glob']: if fnmatch.fnmatch(fname, glob): log.debug( - 'File matching file_ignore_glob. Skipping: {0}'.format( - fname - ) + 'File matching file_ignore_glob. Skipping: %s', + fname ) return True return False @@ -395,7 +398,7 @@ class Fileserver(object): for fsb in back: fstr = '{0}.clear_cache'.format(fsb) if fstr in self.servers: - log.debug('Clearing {0} fileserver cache'.format(fsb)) + log.debug('Clearing %s fileserver cache', fsb) failed = self.servers[fstr]() if failed: errors.extend(failed) @@ -468,7 +471,7 @@ class Fileserver(object): for fsb in back: fstr = '{0}.update'.format(fsb) if fstr in self.servers: - log.debug('Updating {0} fileserver cache'.format(fsb)) + log.debug('Updating %s fileserver cache', fsb) self.servers[fstr]() def update_intervals(self, back=None): @@ -543,6 +546,8 @@ class Fileserver(object): Find the path and return the fnd structure, this structure is passed to other backend interfaces. ''' + path = salt.utils.locales.sdecode(path) + saltenv = salt.utils.locales.sdecode(saltenv) back = self.backends(back) kwargs = {} fnd = {'path': '', @@ -663,7 +668,7 @@ class Fileserver(object): try: saltenv = [x.strip() for x in saltenv.split(',')] except AttributeError: - saltenv = [x.strip() for x in str(saltenv).split(',')] + saltenv = [x.strip() for x in six.text_type(saltenv).split(',')] for idx, val in enumerate(saltenv): if not isinstance(val, six.string_types): @@ -858,7 +863,7 @@ class FSChan(object): Emulate the channel send method, the tries and timeout are not used ''' if 'cmd' not in load: - log.error('Malformed request, no cmd: {0}'.format(load)) + log.error('Malformed request, no cmd: %s', load) return {} cmd = load['cmd'].lstrip('_') if cmd in self.cmd_stub: @@ -866,6 +871,6 @@ class FSChan(object): if cmd == 'file_envs': return self.fs.envs() if not hasattr(self.fs, cmd): - log.error('Malformed request, invalid cmd: {0}'.format(load)) + log.error('Malformed request, invalid cmd: %s', load) return {} return getattr(self.fs, cmd)(load) diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py index eae9969a84..a60f2ec336 100644 --- a/salt/fileserver/azurefs.py +++ b/salt/fileserver/azurefs.py @@ -47,7 +47,7 @@ permissions. ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import base64 import logging import os @@ -60,6 +60,7 @@ import salt.utils.gzip_util import salt.utils.hashutils import salt.utils.json import salt.utils.path +import salt.utils.stringutils from salt.utils.versions import LooseVersion try: @@ -153,10 +154,8 @@ def serve_file(load, fnd): required_load_keys = set(['path', 'loc', 'saltenv']) if not all(x in load for x in required_load_keys): log.debug( - 'Not all of the required keys present in payload. ' - 'Missing: {0}'.format( - ', '.join(required_load_keys.difference(load)) - ) + 'Not all of the required keys present in payload. Missing: %s', + ', '.join(required_load_keys.difference(load)) ) return ret if not fnd['path']: @@ -239,8 +238,8 @@ def update(): # Lock writes lk_fn = fname + '.lk' salt.fileserver.wait_lock(lk_fn, fname) - with salt.utils.files.fopen(lk_fn, 'w+') as fp_: - fp_.write('') + with salt.utils.files.fopen(lk_fn, 'w'): + pass try: blob_service.get_blob_to_path(name, blob.name, fname) @@ -258,8 +257,8 @@ def update(): container_list = path + '.list' lk_fn = container_list + '.lk' salt.fileserver.wait_lock(lk_fn, container_list) - with salt.utils.files.fopen(lk_fn, 'w+') as fp_: - fp_.write('') + with salt.utils.files.fopen(lk_fn, 'w'): + pass with salt.utils.files.fopen(container_list, 'w') as fp_: salt.utils.json.dump(blob_names, fp_) try: @@ -292,11 +291,11 @@ def file_hash(load, fnd): os.makedirs(os.path.dirname(hashdest)) ret['hsum'] = salt.utils.hashutils.get_hash(path, __opts__['hash_type']) with salt.utils.files.fopen(hashdest, 'w+') as fp_: - fp_.write(ret['hsum']) + fp_.write(salt.utils.stringutils.to_str(ret['hsum'])) return ret else: with salt.utils.files.fopen(hashdest, 'rb') as fp_: - ret['hsum'] = fp_.read() + ret['hsum'] = salt.utils.stringutils.to_unicode(fp_.read()) return ret @@ -378,13 +377,15 @@ def _validate_config(): return False for container in __opts__['azurefs']: if not isinstance(container, dict): - log.error('One or more entries in the azurefs configuration list ' - 'are not formed as a dict. Skipping azurefs: {0}' - .format(container)) + log.error( + 'One or more entries in the azurefs configuration list are ' + 'not formed as a dict. Skipping azurefs: %s', container + ) return False if 'account_name' not in container or 'container_name' not in container: - log.error('An azurefs container configuration is missing either ' - 'an account_name or a container_name: {0}' - .format(container)) + log.error( + 'An azurefs container configuration is missing either an ' + 'account_name or a container_name: %s', container + ) return False return True diff --git a/salt/fileserver/gitfs.py b/salt/fileserver/gitfs.py index c7eef46ba2..bc89b63269 100644 --- a/salt/fileserver/gitfs.py +++ b/salt/fileserver/gitfs.py @@ -49,7 +49,7 @@ Walkthrough `. ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging PER_REMOTE_OVERRIDES = ( diff --git a/salt/fileserver/hgfs.py b/salt/fileserver/hgfs.py index a632774886..2d04d65e9f 100644 --- a/salt/fileserver/hgfs.py +++ b/salt/fileserver/hgfs.py @@ -37,7 +37,7 @@ will set the desired branch method. Possible values are: ``branches``, ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import copy import errno import fnmatch @@ -91,8 +91,8 @@ def __virtual__(): return False if __opts__['hgfs_branch_method'] not in VALID_BRANCH_METHODS: log.error( - 'Invalid hgfs_branch_method \'{0}\'. Valid methods are: {1}' - .format(__opts__['hgfs_branch_method'], VALID_BRANCH_METHODS) + 'Invalid hgfs_branch_method \'%s\'. Valid methods are: %s', + __opts__['hgfs_branch_method'], VALID_BRANCH_METHODS ) return False return __virtualname__ @@ -213,11 +213,10 @@ def init(): ) if not per_remote_conf: log.error( - 'Invalid per-remote configuration for hgfs remote {0}. If ' + 'Invalid per-remote configuration for hgfs remote %s. If ' 'no per-remote parameters are being specified, there may ' 'be a trailing colon after the URL, which should be ' - 'removed. Check the master configuration file.' - .format(repo_url) + 'removed. Check the master configuration file.', repo_url ) _failhard() @@ -226,10 +225,9 @@ def init(): per_remote_defaults['branch_method']) if branch_method not in VALID_BRANCH_METHODS: log.error( - 'Invalid branch_method \'{0}\' for remote {1}. Valid ' - 'branch methods are: {2}. This remote will be ignored.' - .format(branch_method, repo_url, - ', '.join(VALID_BRANCH_METHODS)) + 'Invalid branch_method \'%s\' for remote %s. Valid ' + 'branch methods are: %s. This remote will be ignored.', + branch_method, repo_url, ', '.join(VALID_BRANCH_METHODS) ) _failhard() @@ -237,11 +235,10 @@ def init(): for param in (x for x in per_remote_conf if x not in PER_REMOTE_OVERRIDES): log.error( - 'Invalid configuration parameter \'{0}\' for remote {1}. ' - 'Valid parameters are: {2}. See the documentation for ' - 'further information.'.format( - param, repo_url, ', '.join(PER_REMOTE_OVERRIDES) - ) + 'Invalid configuration parameter \'%s\' for remote %s. ' + 'Valid parameters are: %s. See the documentation for ' + 'further information.', + param, repo_url, ', '.join(PER_REMOTE_OVERRIDES) ) per_remote_errors = True if per_remote_errors: @@ -253,8 +250,8 @@ def init(): if not isinstance(repo_url, six.string_types): log.error( - 'Invalid hgfs remote {0}. Remotes must be strings, you may ' - 'need to enclose the URL in quotes'.format(repo_url) + 'Invalid hgfs remote %s. Remotes must be strings, you may ' + 'need to enclose the URL in quotes', repo_url ) _failhard() @@ -280,16 +277,16 @@ def init(): repo = hglib.open(rp_) except hglib.error.ServerError: log.error( - 'Cache path {0} (corresponding remote: {1}) exists but is not ' + 'Cache path %s (corresponding remote: %s) exists but is not ' 'a valid mercurial repository. You will need to manually ' 'delete this directory on the master to continue to use this ' - 'hgfs remote.'.format(rp_, repo_url) + 'hgfs remote.', rp_, repo_url ) _failhard() except Exception as exc: log.error( - 'Exception \'{0}\' encountered while initializing hgfs remote ' - '{1}'.format(exc, repo_url) + 'Exception \'%s\' encountered while initializing hgfs ' + 'remote %s', exc, repo_url ) _failhard() @@ -306,7 +303,11 @@ def init(): hgconfpath = os.path.join(rp_, '.hg', 'hgrc') with salt.utils.files.fopen(hgconfpath, 'w+') as hgconfig: hgconfig.write('[paths]\n') - hgconfig.write('default = {0}\n'.format(repo_url)) + hgconfig.write( + salt.utils.stringutils.to_str( + 'default = {0}\n'.format(repo_url) + ) + ) repo_conf.update({ 'repo': repo, @@ -327,11 +328,15 @@ def init(): timestamp = datetime.now().strftime('%d %b %Y %H:%M:%S.%f') fp_.write('# hgfs_remote map as of {0}\n'.format(timestamp)) for repo in repos: - fp_.write('{0} = {1}\n'.format(repo['hash'], repo['url'])) + fp_.write( + salt.utils.stringutils.to_str( + '{0} = {1}\n'.format(repo['hash'], repo['url']) + ) + ) except OSError: pass else: - log.info('Wrote new hgfs_remote map to {0}'.format(remote_map)) + log.info('Wrote new hgfs_remote map to %s', remote_map) return repos @@ -366,12 +371,12 @@ def _clear_old_remotes(): shutil.rmtree(rdir) except OSError as exc: log.error( - 'Unable to remove old hgfs remote cachedir {0}: {1}' - .format(rdir, exc) + 'Unable to remove old hgfs remote cachedir %s: %s', + rdir, exc ) failed.append(rdir) else: - log.debug('hgfs removed old cachedir {0}'.format(rdir)) + log.debug('hgfs removed old cachedir %s', rdir) for fdir in failed: to_remove.remove(fdir) return bool(to_remove), repos @@ -462,8 +467,8 @@ def lock(remote=None): failed = [] if not os.path.exists(repo['lockfile']): try: - with salt.utils.files.fopen(repo['lockfile'], 'w+') as fp_: - fp_.write('') + with salt.utils.files.fopen(repo['lockfile'], 'w'): + pass except (IOError, OSError) as exc: msg = ('Unable to set update lock for {0} ({1}): {2} ' .format(repo['url'], repo['lockfile'], exc)) @@ -509,29 +514,29 @@ def update(): for repo in repos: if os.path.exists(repo['lockfile']): log.warning( - 'Update lockfile is present for hgfs remote {0}, skipping. ' + 'Update lockfile is present for hgfs remote %s, skipping. ' 'If this warning persists, it is possible that the update ' - 'process was interrupted. Removing {1} or running ' + 'process was interrupted. Removing %s or running ' '\'salt-run fileserver.clear_lock hgfs\' will allow updates ' - 'to continue for this remote.' - .format(repo['url'], repo['lockfile']) + 'to continue for this remote.', repo['url'], repo['lockfile'] ) continue _, errors = lock(repo) if errors: - log.error('Unable to set update lock for hgfs remote {0}, ' - 'skipping.'.format(repo['url'])) + log.error( + 'Unable to set update lock for hgfs remote %s, skipping.', + repo['url'] + ) continue - log.debug('hgfs is fetching from {0}'.format(repo['url'])) + log.debug('hgfs is fetching from %s', repo['url']) repo['repo'].open() curtip = repo['repo'].tip() try: repo['repo'].pull() except Exception as exc: log.error( - 'Exception {0} caught while updating hgfs remote {1}' - .format(exc, repo['url']), - exc_info_on_loglevel=logging.DEBUG + 'Exception %s caught while updating hgfs remote %s', + exc, repo['url'], exc_info_on_loglevel=logging.DEBUG ) else: newtip = repo['repo'].tip() @@ -549,7 +554,7 @@ def update(): serial = salt.payload.Serial(__opts__) with salt.utils.files.fopen(env_cache, 'wb+') as fp_: fp_.write(serial.dumps(new_envs)) - log.trace('Wrote env cache data to {0}'.format(env_cache)) + log.trace('Wrote env cache data to %s', env_cache) # if there is a change, fire an event if __opts__.get('fileserver_events', False): @@ -701,8 +706,8 @@ def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 except hglib.error.CommandError: repo['repo'].close() continue - with salt.utils.files.fopen(lk_fn, 'w+') as fp_: - fp_.write('') + with salt.utils.files.fopen(lk_fn, 'w'): + pass for filename in glob.glob(hashes_glob): try: os.remove(filename) @@ -791,7 +796,7 @@ def file_hash(load, fnd): return ret else: with salt.utils.files.fopen(hashdest, 'rb') as fp_: - ret['hsum'] = fp_.read() + ret['hsum'] = salt.utils.stringutils.to_unicode(fp_.read()) return ret @@ -808,7 +813,7 @@ def _file_lists(load, form): try: os.makedirs(list_cachedir) except os.error: - log.critical('Unable to make cachedir {0}'.format(list_cachedir)) + log.critical('Unable to make cachedir %s', list_cachedir) return [] list_cache = os.path.join(list_cachedir, '{0}.p'.format(load['saltenv'])) w_lock = os.path.join(list_cachedir, '.{0}.w'.format(load['saltenv'])) diff --git a/salt/fileserver/minionfs.py b/salt/fileserver/minionfs.py index 6d9be0b5a6..573ddf0715 100644 --- a/salt/fileserver/minionfs.py +++ b/salt/fileserver/minionfs.py @@ -27,7 +27,7 @@ Other minionfs settings include: :conf_master:`minionfs_whitelist`, .. seealso:: :ref:`tutorial-minionfs` ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import os @@ -83,8 +83,10 @@ def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 if tgt_env not in envs(): return fnd if os.path.basename(path) == 'top.sls': - log.debug('minionfs will NOT serve top.sls ' - 'for security reasons (path requested: {0})'.format(path)) + log.debug( + 'minionfs will NOT serve top.sls ' + 'for security reasons (path requested: %s)', path + ) return fnd mountpoint = salt.utils.url.strip_proto(__opts__['minionfs_mountpoint']) @@ -188,7 +190,8 @@ def file_hash(load, fnd): # cache file's contents should be "hash:mtime" cache_path = os.path.join( __opts__['cachedir'], - 'minionfs/hash', + 'minionfs', + 'hash', load['saltenv'], '{0}.hash.{1}'.format(fnd['rel'], __opts__['hash_type']) ) @@ -197,7 +200,7 @@ def file_hash(load, fnd): try: with salt.utils.files.fopen(cache_path, 'rb') as fp_: try: - hsum, mtime = fp_.read().split(':') + hsum, mtime = salt.utils.stringutils.to_unicode(fp_.read()).split(':') except ValueError: log.debug( 'Fileserver attempted to read incomplete cache file. ' @@ -261,8 +264,8 @@ def file_list(load): # pushed files if tgt_minion not in minion_dirs: log.warning( - 'No files found in minionfs cache for minion ID \'{0}\'' - .format(tgt_minion) + 'No files found in minionfs cache for minion ID \'%s\'', + tgt_minion ) return [] minion_dirs = [tgt_minion] @@ -274,8 +277,8 @@ def file_list(load): minion_files_dir = os.path.join(minions_cache_dir, minion, 'files') if not os.path.isdir(minion_files_dir): log.debug( - 'minionfs: could not find files directory under {0}!' - .format(os.path.join(minions_cache_dir, minion)) + 'minionfs: could not find files directory under %s!', + os.path.join(minions_cache_dir, minion) ) continue walk_dir = os.path.join(minion_files_dir, prefix) @@ -340,8 +343,8 @@ def dir_list(load): # pushed files if tgt_minion not in minion_dirs: log.warning( - 'No files found in minionfs cache for minion ID \'{0}\'' - .format(tgt_minion) + 'No files found in minionfs cache for minion ID \'%s\'', + tgt_minion ) return [] minion_dirs = [tgt_minion] @@ -353,8 +356,8 @@ def dir_list(load): minion_files_dir = os.path.join(minions_cache_dir, minion, 'files') if not os.path.isdir(minion_files_dir): log.warning( - 'minionfs: could not find files directory under {0}!' - .format(os.path.join(minions_cache_dir, minion)) + 'minionfs: could not find files directory under %s!', + os.path.join(minions_cache_dir, minion) ) continue walk_dir = os.path.join(minion_files_dir, prefix) diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index 0486df7989..a100e83808 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -106,7 +106,7 @@ def envs(): ''' Return the file server environments ''' - return list(__opts__['file_roots'].keys()) + return sorted(__opts__['file_roots']) def serve_file(load, fnd): @@ -144,14 +144,14 @@ def update(): ''' try: salt.fileserver.reap_fileserver_cache_dir( - os.path.join(__opts__['cachedir'], 'roots/hash'), + os.path.join(__opts__['cachedir'], 'roots', 'hash'), find_file ) except (IOError, OSError): # Hash file won't exist if no files have yet been served up pass - mtime_map_path = os.path.join(__opts__['cachedir'], 'roots/mtime_map') + mtime_map_path = os.path.join(__opts__['cachedir'], 'roots', 'mtime_map') # data to send on event data = {'changed': False, 'files': {'changed': []}, @@ -165,15 +165,18 @@ def update(): if os.path.exists(mtime_map_path): with salt.utils.files.fopen(mtime_map_path, 'r') as fp_: for line in fp_: + line = salt.utils.stringutils.to_unicode(line) try: file_path, mtime = line.replace('\n', '').split(':', 1) old_mtime_map[file_path] = mtime - if mtime != str(new_mtime_map.get(file_path, mtime)): + if mtime != new_mtime_map.get(file_path, mtime): data['files']['changed'].append(file_path) except ValueError: # Document the invalid entry in the log - log.warning('Skipped invalid cache mtime entry in {0}: {1}' - .format(mtime_map_path, line)) + log.warning( + 'Skipped invalid cache mtime entry in %s: %s', + mtime_map_path, line + ) # compare the maps, set changed to the return value data['changed'] = salt.fileserver.diff_mtime_map(old_mtime_map, new_mtime_map) @@ -190,8 +193,11 @@ def update(): os.makedirs(mtime_map_path_dir) with salt.utils.files.fopen(mtime_map_path, 'w') as fp_: for file_path, mtime in six.iteritems(new_mtime_map): - fp_.write('{file_path}:{mtime}\n'.format(file_path=file_path, - mtime=mtime)) + fp_.write( + salt.utils.stringutils.to_str( + '{0}:{1}\n'.format(file_path, mtime) + ) + ) if __opts__.get('fileserver_events', False): # if there is a change, fire an event @@ -228,7 +234,8 @@ def file_hash(load, fnd): # check if the hash is cached # cache file's contents should be "hash:mtime" cache_path = os.path.join(__opts__['cachedir'], - 'roots/hash', + 'roots', + 'hash', load['saltenv'], '{0}.hash.{1}'.format(fnd['rel'], __opts__['hash_type'])) @@ -237,7 +244,7 @@ def file_hash(load, fnd): try: with salt.utils.files.fopen(cache_path, 'r') as fp_: try: - hsum, mtime = fp_.read().split(':') + hsum, mtime = salt.utils.stringutils.to_unicode(fp_.read()).split(':') except ValueError: log.debug('Fileserver attempted to read incomplete cache file. Retrying.') # Delete the file since its incomplete (either corrupted or incomplete) @@ -296,7 +303,7 @@ def _file_lists(load, form): try: os.makedirs(list_cachedir) except os.error: - log.critical('Unable to make cachedir {0}'.format(list_cachedir)) + log.critical('Unable to make cachedir %s', list_cachedir) return [] list_cache = os.path.join(list_cachedir, '{0}.p'.format(load['saltenv'])) w_lock = os.path.join(list_cachedir, '.{0}.w'.format(load['saltenv'])) diff --git a/salt/fileserver/s3fs.py b/salt/fileserver/s3fs.py index f96f62ece1..c7fb49e552 100644 --- a/salt/fileserver/s3fs.py +++ b/salt/fileserver/s3fs.py @@ -60,7 +60,7 @@ structure:: ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import datetime import os import time @@ -113,7 +113,7 @@ def update(): for bucket, files in six.iteritems(_find_files(env_meta)): for file_path in files: cached_file_path = _get_cached_file_name(bucket, saltenv, file_path) - log.info('{0} - {1} : {2}'.format(bucket, saltenv, file_path)) + log.info('%s - %s : %s', bucket, saltenv, file_path) # load the file from S3 if it's not in the cache or it's old _get_file_from_s3(metadata, saltenv, bucket, file_path, cached_file_path) @@ -438,15 +438,15 @@ def _refresh_buckets_cache_file(cache_file): meta_response.update(k) # attempt use of human readable output first. try: - log.warning("'{0}' response for bucket '{1}'".format(meta_response['Message'], bucket_name)) + log.warning("'%s' response for bucket '%s'", meta_response['Message'], bucket_name) continue except KeyError: # no human readable error message provided if 'Code' in meta_response: log.warning( - ("'{0}' response for " - "bucket '{1}'").format(meta_response['Code'], - bucket_name)) + "'%s' response for bucket '%s'", + meta_response['Code'], bucket_name + ) continue else: log.warning( @@ -477,15 +477,15 @@ def _refresh_buckets_cache_file(cache_file): meta_response.update(k) # attempt use of human readable output first. try: - log.warning("'{0}' response for bucket '{1}'".format(meta_response['Message'], bucket_name)) + log.warning("'%s' response for bucket '%s'", meta_response['Message'], bucket_name) continue except KeyError: # no human readable error message provided if 'Code' in meta_response: log.warning( - ("'{0}' response for " - "bucket '{1}'").format(meta_response['Code'], - bucket_name)) + "'%s' response for bucket '%s'", + meta_response['Code'], bucket_name + ) continue else: log.warning( @@ -658,17 +658,18 @@ def _get_file_from_s3(metadata, saltenv, bucket_name, path, cached_file_path): for header_name, header_value in ret['headers'].items(): name = header_name.strip() value = header_value.strip() - if str(name).lower() == 'last-modified': + if six.text_type(name).lower() == 'last-modified': s3_file_mtime = datetime.datetime.strptime( value, '%a, %d %b %Y %H:%M:%S %Z') - elif str(name).lower() == 'content-length': + elif six.text_type(name).lower() == 'content-length': s3_file_size = int(value) if (cached_file_size == s3_file_size and cached_file_mtime > s3_file_mtime): log.info( - '{0} - {1} : {2} skipped download since cached file size ' - 'equal to and mtime after s3 values'.format( - bucket_name, saltenv, path)) + '%s - %s : %s skipped download since cached file size ' + 'equal to and mtime after s3 values', + bucket_name, saltenv, path + ) return # ... or get the file from S3 diff --git a/salt/fileserver/svnfs.py b/salt/fileserver/svnfs.py index 7ecd4a474b..e354ea33c3 100644 --- a/salt/fileserver/svnfs.py +++ b/salt/fileserver/svnfs.py @@ -32,7 +32,7 @@ This backend assumes a standard svn layout with directories for ``branches``, ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import copy import errno import fnmatch @@ -109,9 +109,9 @@ def _rev(repo): except (pysvn._pysvn.ClientError, TypeError, KeyError, AttributeError) as exc: log.error( - 'Error retrieving revision ID for svnfs remote {0} ' - '(cachedir: {1}): {2}' - .format(repo['url'], repo['repo'], exc) + 'Error retrieving revision ID for svnfs remote %s ' + '(cachedir: %s): %s', + repo['url'], repo['repo'], exc ) else: return repo_info['revision'].number @@ -150,10 +150,10 @@ def init(): ) if not per_remote_conf: log.error( - 'Invalid per-remote configuration for remote {0}. If no ' + 'Invalid per-remote configuration for remote %s. If no ' 'per-remote parameters are being specified, there may be ' 'a trailing colon after the URL, which should be removed. ' - 'Check the master configuration file.'.format(repo_url) + 'Check the master configuration file.', repo_url ) _failhard() @@ -161,11 +161,10 @@ def init(): for param in (x for x in per_remote_conf if x not in PER_REMOTE_OVERRIDES): log.error( - 'Invalid configuration parameter \'{0}\' for remote {1}. ' - 'Valid parameters are: {2}. See the documentation for ' - 'further information.'.format( - param, repo_url, ', '.join(PER_REMOTE_OVERRIDES) - ) + 'Invalid configuration parameter \'%s\' for remote %s. ' + 'Valid parameters are: %s. See the documentation for ' + 'further information.', + param, repo_url, ', '.join(PER_REMOTE_OVERRIDES) ) per_remote_errors = True if per_remote_errors: @@ -177,8 +176,8 @@ def init(): if not isinstance(repo_url, six.string_types): log.error( - 'Invalid svnfs remote {0}. Remotes must be strings, you may ' - 'need to enclose the URL in quotes'.format(repo_url) + 'Invalid svnfs remote %s. Remotes must be strings, you may ' + 'need to enclose the URL in quotes', repo_url ) _failhard() @@ -204,8 +203,8 @@ def init(): new_remote = True except pysvn._pysvn.ClientError as exc: log.error( - 'Failed to initialize svnfs remote \'{0}\': {1}' - .format(repo_url, exc) + 'Failed to initialize svnfs remote \'%s\': %s', + repo_url, exc ) _failhard() else: @@ -215,10 +214,10 @@ def init(): CLIENT.status(rp_) except pysvn._pysvn.ClientError as exc: log.error( - 'Cache path {0} (corresponding remote: {1}) exists but is ' + 'Cache path %s (corresponding remote: %s) exists but is ' 'not a valid subversion checkout. You will need to ' 'manually delete this directory on the master to continue ' - 'to use this svnfs remote.'.format(rp_, repo_url) + 'to use this svnfs remote.', rp_, repo_url ) _failhard() @@ -239,14 +238,16 @@ def init(): fp_.write('# svnfs_remote map as of {0}\n'.format(timestamp)) for repo_conf in repos: fp_.write( - '{0} = {1}\n'.format( - repo_conf['hash'], repo_conf['url'] + salt.utils.stringutils.to_str( + '{0} = {1}\n'.format( + repo_conf['hash'], repo_conf['url'] + ) ) ) except OSError: pass else: - log.info('Wrote new svnfs_remote map to {0}'.format(remote_map)) + log.info('Wrote new svnfs_remote map to %s', remote_map) return repos @@ -281,12 +282,12 @@ def _clear_old_remotes(): shutil.rmtree(rdir) except OSError as exc: log.error( - 'Unable to remove old svnfs remote cachedir {0}: {1}' - .format(rdir, exc) + 'Unable to remove old svnfs remote cachedir %s: %s', + rdir, exc ) failed.append(rdir) else: - log.debug('svnfs removed old cachedir {0}'.format(rdir)) + log.debug('svnfs removed old cachedir %s', rdir) for fdir in failed: to_remove.remove(fdir) return bool(to_remove), repos @@ -424,27 +425,28 @@ def update(): for repo in repos: if os.path.exists(repo['lockfile']): log.warning( - 'Update lockfile is present for svnfs remote {0}, skipping. ' + 'Update lockfile is present for svnfs remote %s, skipping. ' 'If this warning persists, it is possible that the update ' - 'process was interrupted. Removing {1} or running ' + 'process was interrupted. Removing %s or running ' '\'salt-run fileserver.clear_lock svnfs\' will allow updates ' - 'to continue for this remote.' - .format(repo['url'], repo['lockfile']) + 'to continue for this remote.', repo['url'], repo['lockfile'] ) continue _, errors = lock(repo) if errors: - log.error('Unable to set update lock for svnfs remote {0}, ' - 'skipping.'.format(repo['url'])) + log.error( + 'Unable to set update lock for svnfs remote %s, skipping.', + repo['url'] + ) continue - log.debug('svnfs is fetching from {0}'.format(repo['url'])) + log.debug('svnfs is fetching from %s', repo['url']) old_rev = _rev(repo) try: CLIENT.update(repo['repo']) except pysvn._pysvn.ClientError as exc: log.error( - 'Error updating svnfs remote {0} (cachedir: {1}): {2}' - .format(repo['url'], repo['cachedir'], exc) + 'Error updating svnfs remote %s (cachedir: %s): %s', + repo['url'], repo['cachedir'], exc ) new_rev = _rev(repo) @@ -465,7 +467,7 @@ def update(): serial = salt.payload.Serial(__opts__) with salt.utils.files.fopen(env_cache, 'wb+') as fp_: fp_.write(serial.dumps(new_envs)) - log.trace('Wrote env cache data to {0}'.format(env_cache)) + log.trace('Wrote env cache data to %s', env_cache) # if there is a change, fire an event if __opts__.get('fileserver_events', False): @@ -535,9 +537,9 @@ def envs(ignore_cache=False): ret.add('base') else: log.error( - 'svnfs trunk path \'{0}\' does not exist in repo {1}, no base ' - 'environment will be provided by this remote' - .format(repo['trunk'], repo['url']) + 'svnfs trunk path \'%s\' does not exist in repo %s, no base ' + 'environment will be provided by this remote', + repo['trunk'], repo['url'] ) branches = os.path.join(repo['repo'], repo['branches']) @@ -545,8 +547,8 @@ def envs(ignore_cache=False): ret.update(os.listdir(branches)) else: log.error( - 'svnfs branches path \'{0}\' does not exist in repo {1}' - .format(repo['branches'], repo['url']) + 'svnfs branches path \'%s\' does not exist in repo %s', + repo['branches'], repo['url'] ) tags = os.path.join(repo['repo'], repo['tags']) @@ -554,8 +556,8 @@ def envs(ignore_cache=False): ret.update(os.listdir(tags)) else: log.error( - 'svnfs tags path \'{0}\' does not exist in repo {1}' - .format(repo['tags'], repo['url']) + 'svnfs tags path \'%s\' does not exist in repo %s', + repo['tags'], repo['url'] ) return [x for x in sorted(ret) if _env_is_exposed(x)] @@ -689,7 +691,8 @@ def file_hash(load, fnd): # Check if the hash is cached # Cache file's contents should be "hash:mtime" cache_path = os.path.join(__opts__['cachedir'], - 'svnfs/hash', + 'svnfs', + 'hash', saltenv, '{0}.hash.{1}'.format(relpath, __opts__['hash_type'])) @@ -731,7 +734,7 @@ def _file_lists(load, form): try: os.makedirs(list_cachedir) except os.error: - log.critical('Unable to make cachedir {0}'.format(list_cachedir)) + log.critical('Unable to make cachedir %s', list_cachedir) return [] list_cache = os.path.join(list_cachedir, '{0}.p'.format(load['saltenv'])) w_lock = os.path.join(list_cachedir, '.{0}.w'.format(load['saltenv'])) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index eabac99961..b7c5d2ccf8 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -4,7 +4,7 @@ Classes which provide the shared base for GitFS, git_pillar, and winrepo ''' # Import python libs -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals import copy import contextlib import distutils @@ -14,7 +14,6 @@ import glob import hashlib import logging import os -import re import shlex import shutil import stat @@ -70,19 +69,19 @@ AUTH_PARAMS = ('user', 'password', 'pubkey', 'privkey', 'passphrase', PER_SALTENV_PARAMS = ('mountpoint', 'root', 'ref') _RECOMMEND_GITPYTHON = ( - 'GitPython is installed, you may wish to set {0}_provider to ' - '\'gitpython\' to use GitPython for {0} support.' + 'GitPython is installed, you may wish to set %s_provider to ' + '\'gitpython\' to use GitPython for %s support.' ) _RECOMMEND_PYGIT2 = ( - 'pygit2 is installed, you may wish to set {0}_provider to ' - '\'pygit2\' to use pygit2 for for {0} support.' + 'pygit2 is installed, you may wish to set %s_provider to ' + '\'pygit2\' to use pygit2 for for %s support.' ) _INVALID_REPO = ( - 'Cache path {0} (corresponding remote: {1}) exists but is not a valid ' + 'Cache path %s (corresponding remote: %s) exists but is not a valid ' 'git repository. You will need to manually delete this directory on the ' - 'master to continue to use this {2} remote.' + 'master to continue to use this %s remote.' ) log = logging.getLogger(__name__) @@ -221,7 +220,7 @@ class GitProvider(object): 'The following parameter names are restricted to per-remote ' 'use only: %s. This is a bug, please report it.', ', '.join(per_remote_collisions) - ) + ) try: valid_per_remote_params = override_params + per_remote_only @@ -448,7 +447,8 @@ class GitProvider(object): ret = set() for ref in refs: - ref = re.sub('^refs/', '', ref) + if ref.startswith('refs/'): + ref = ref[5:] rtype, rname = ref.split('/', 1) if rtype == 'remotes' and use_branches: parted = rname.partition('/') @@ -744,7 +744,7 @@ class GitProvider(object): try: fh_ = os.open(self._get_lock_file(lock_type), os.O_CREAT | os.O_EXCL | os.O_WRONLY) - with os.fdopen(fh_, 'w'): + with os.fdopen(fh_, 'wb'): # Write the lock file and close the filehandle os.write(fh_, six.b(six.text_type(os.getpid()))) except (OSError, IOError) as exc: @@ -1153,7 +1153,7 @@ class GitPython(GitProvider): try: self.repo = git.Repo(self.cachedir) except git.exc.InvalidGitRepositoryError: - log.error(_INVALID_REPO.format(self.cachedir, self.url, self.role)) + log.error(_INVALID_REPO, self.cachedir, self.url, self.role) return new self.gitdir = salt.utils.path.join(self.repo.working_dir, '.git') @@ -1583,7 +1583,7 @@ class Pygit2(GitProvider): pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = home self.repo = pygit2.Repository(self.cachedir) except KeyError: - log.error(_INVALID_REPO.format(self.cachedir, self.url, self.role)) + log.error(_INVALID_REPO, self.cachedir, self.url, self.role) return new self.gitdir = salt.utils.path.join(self.repo.workdir, '.git') @@ -2181,8 +2181,7 @@ class GitBase(object): log.critical( 'The following %s remotes have conflicting cachedirs: ' '%s. Resolve this using a per-remote parameter called ' - '\'name\'.', - self.role, ', '.join(cachedir_map[dirname]) + '\'name\'.', self.role, ', '.join(cachedir_map[dirname]) ) failhard(self.role) @@ -2222,9 +2221,7 @@ class GitBase(object): ) failed.append(rdir) else: - log.debug( - '%s removed old cachedir %s', self.role, rdir - ) + log.debug('%s removed old cachedir %s', self.role, rdir) for fdir in failed: to_remove.remove(fdir) ret = bool(to_remove) @@ -2429,8 +2426,7 @@ class GitBase(object): self.provider = 'gitpython' if not hasattr(self, 'provider'): log.critical( - 'No suitable %s provider module is installed.', - self.role + 'No suitable %s provider module is installed.', self.role ) failhard(self.role) @@ -2440,7 +2436,7 @@ class GitBase(object): ''' def _recommend(): if HAS_PYGIT2 and 'pygit2' in self.git_providers: - log.error(_RECOMMEND_PYGIT2.format(self.role)) + log.error(_RECOMMEND_PYGIT2, self.role, self.role) if not HAS_GITPYTHON: if not quiet: @@ -2491,7 +2487,7 @@ class GitBase(object): ''' def _recommend(): if HAS_GITPYTHON and 'gitpython' in self.git_providers: - log.error(_RECOMMEND_GITPYTHON.format(self.role)) + log.error(_RECOMMEND_GITPYTHON, self.role, self.role) if not HAS_PYGIT2: if not quiet: @@ -2565,19 +2561,17 @@ class GitBase(object): ) for repo in self.remotes: fp_.write( - '{0} = {1}\n'.format( - repo.cachedir_basename, - repo.id + salt.utils.stringutils.to_str( + '{0} = {1}\n'.format( + repo.cachedir_basename, + repo.id + ) ) ) except OSError: pass else: - log.info( - 'Wrote new %s remote map to %s', - self.role, - remote_map - ) + log.info('Wrote new %s remote map to %s', self.role, remote_map) def do_checkout(self, repo): ''' @@ -2757,13 +2751,13 @@ class GitFS(GitBase): salt.fileserver.wait_lock(lk_fn, dest) if os.path.isfile(blobshadest) and os.path.isfile(dest): with salt.utils.files.fopen(blobshadest, 'r') as fp_: - sha = fp_.read() + sha = salt.utils.stringutils.to_unicode(fp_.read()) if sha == blob_hexsha: fnd['rel'] = path fnd['path'] = dest return _add_file_stat(fnd, blob_mode) - with salt.utils.files.fopen(lk_fn, 'w+') as fp_: - fp_.write('') + with salt.utils.files.fopen(lk_fn, 'w'): + pass for filename in glob.glob(hashes_glob): try: os.remove(filename) @@ -2798,10 +2792,8 @@ class GitFS(GitBase): required_load_keys = set(['path', 'loc', 'saltenv']) if not all(x in load for x in required_load_keys): log.debug( - 'Not all of the required keys present in payload. ' - 'Missing: {0}'.format( - ', '.join(required_load_keys.difference(load)) - ) + 'Not all of the required keys present in payload. Missing: %s', + ', '.join(required_load_keys.difference(load)) ) return ret if not fnd['path']: @@ -2861,10 +2853,7 @@ class GitFS(GitBase): try: os.makedirs(self.file_list_cachedir) except os.error: - log.error( - 'Unable to make cachedir %s', - self.file_list_cachedir - ) + log.error('Unable to make cachedir %s', self.file_list_cachedir) return [] list_cache = salt.utils.path.join( self.file_list_cachedir, diff --git a/tests/integration/files/file/base/питон.txt b/tests/integration/files/file/base/питон.txt new file mode 100644 index 0000000000..ecf7cf0dca --- /dev/null +++ b/tests/integration/files/file/base/питон.txt @@ -0,0 +1,3 @@ +{{ grains['id'] }} + +спам, спам, спам, яйца и спам diff --git a/tests/integration/files/file/base/соль/foo.txt b/tests/integration/files/file/base/соль/foo.txt new file mode 100644 index 0000000000..5716ca5987 --- /dev/null +++ b/tests/integration/files/file/base/соль/foo.txt @@ -0,0 +1 @@ +bar diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index 6a671cf540..cc8bdbac3b 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -4,7 +4,7 @@ ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import errno import os import shutil @@ -35,6 +35,7 @@ from tests.support.paths import TMP, FILES # Import salt libs import salt.fileserver.gitfs as gitfs +import salt.utils.files import salt.utils.gitfs import salt.utils.platform import salt.utils.win_functions @@ -45,6 +46,8 @@ log = logging.getLogger(__name__) TMP_SOCK_DIR = tempfile.mkdtemp(dir=TMP) TMP_REPO_DIR = os.path.join(TMP, 'gitfs_root') INTEGRATION_BASE_FILES = os.path.join(FILES, 'file', 'base') +UNICODE_FILENAME = 'питон.txt' +UNICODE_DIRNAME = UNICODE_ENVNAME = 'соль' def _rmtree_error(func, path, excinfo): @@ -261,6 +264,9 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): if x != '.git']) repo.index.commit('Test') + # Add another branch with unicode characters in the name + repo.create_head(UNICODE_ENVNAME, 'HEAD') + def setUp(self): ''' We don't want to check in another .git dir into GH because that just @@ -286,11 +292,17 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): def test_file_list(self): ret = gitfs.file_list(LOAD) self.assertIn('testfile', ret) + self.assertIn(UNICODE_FILENAME, ret) + # This function does not use os.sep, the Salt fileserver uses the + # forward slash, hence it being explicitly used to join here. + self.assertIn('/'.join((UNICODE_DIRNAME, 'foo.txt')), ret) def test_dir_list(self): ret = gitfs.dir_list(LOAD) self.assertIn('grail', ret) + self.assertIn(UNICODE_DIRNAME, ret) def test_envs(self): - ret = gitfs.envs() + ret = gitfs.envs(ignore_cache=True) self.assertIn('base', ret) + self.assertIn(UNICODE_ENVNAME, ret) diff --git a/tests/unit/fileserver/test_map.py b/tests/unit/fileserver/test_map.py index c7f5588c1b..148fdc7359 100644 --- a/tests/unit/fileserver/test_map.py +++ b/tests/unit/fileserver/test_map.py @@ -4,7 +4,7 @@ ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.unit import TestCase diff --git a/tests/unit/fileserver/test_roots.py b/tests/unit/fileserver/test_roots.py index 4956fdac7a..97d028cf17 100644 --- a/tests/unit/fileserver/test_roots.py +++ b/tests/unit/fileserver/test_roots.py @@ -4,7 +4,8 @@ ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals +import copy import os import tempfile @@ -16,7 +17,7 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON # Import Salt libs -import salt.fileserver.roots +import salt.fileserver.roots as roots import salt.fileclient import salt.utils.files import salt.utils.platform @@ -26,6 +27,9 @@ try: except ImportError: pass +UNICODE_FILENAME = 'питон.txt' +UNICODE_DIRNAME = UNICODE_ENVNAME = 'соль' + @skipIf(NO_MOCK, NO_MOCK_REASON) class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMixin): @@ -37,7 +41,7 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix empty_dir = os.path.join(TMP_STATE_TREE, 'empty_dir') if not os.path.isdir(empty_dir): os.makedirs(empty_dir) - return {salt.fileserver.roots: {'__opts__': self.opts}} + return {roots: {'__opts__': self.opts}} @classmethod def setUpClass(cls): @@ -74,25 +78,26 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix del self.opts def test_file_list(self): - ret = salt.fileserver.roots.file_list({'saltenv': 'base'}) + ret = roots.file_list({'saltenv': 'base'}) self.assertIn('testfile', ret) + self.assertIn(UNICODE_FILENAME, ret) def test_find_file(self): - ret = salt.fileserver.roots.find_file('testfile') + ret = roots.find_file('testfile') self.assertEqual('testfile', ret['rel']) full_path_to_file = os.path.join(FILES, 'file', 'base', 'testfile') self.assertEqual(full_path_to_file, ret['path']) def test_serve_file(self): - with patch.dict(salt.fileserver.roots.__opts__, {'file_buffer_size': 262144}): + with patch.dict(roots.__opts__, {'file_buffer_size': 262144}): load = {'saltenv': 'base', 'path': os.path.join(FILES, 'file', 'base', 'testfile'), 'loc': 0 } fnd = {'path': os.path.join(FILES, 'file', 'base', 'testfile'), 'rel': 'testfile'} - ret = salt.fileserver.roots.serve_file(load, fnd) + ret = roots.serve_file(load, fnd) data = 'Scene 24\n\n \n OLD MAN: Ah, hee he he ha!\n ' \ 'ARTHUR: And this enchanter of whom you speak, he ' \ @@ -128,9 +133,13 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix {'data': data, 'dest': 'testfile'}) - @skipIf(True, "Update test not yet implemented") - def test_update(self): - pass + def test_envs(self): + opts = {'file_roots': copy.copy(self.opts['file_roots'])} + opts['file_roots'][UNICODE_ENVNAME] = opts['file_roots']['base'] + with patch.dict(roots.__opts__, opts): + ret = roots.envs() + self.assertIn('base', ret) + self.assertIn(UNICODE_ENVNAME, ret) def test_file_hash(self): load = { @@ -141,7 +150,7 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix 'path': os.path.join(FILES, 'file', 'base', 'testfile'), 'rel': 'testfile' } - ret = salt.fileserver.roots.file_hash(load, fnd) + ret = roots.file_hash(load, fnd) # Hashes are different in Windows. May be how git translates line # endings @@ -158,17 +167,18 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix ) def test_file_list_emptydirs(self): - ret = salt.fileserver.roots.file_list_emptydirs({'saltenv': 'base'}) + ret = roots.file_list_emptydirs({'saltenv': 'base'}) self.assertIn('empty_dir', ret) def test_dir_list(self): - ret = salt.fileserver.roots.dir_list({'saltenv': 'base'}) + ret = roots.dir_list({'saltenv': 'base'}) self.assertIn('empty_dir', ret) + self.assertIn(UNICODE_DIRNAME, ret) def test_symlink_list(self): if self.test_symlink_list_file_roots: self.opts['file_roots'] = self.test_symlink_list_file_roots - ret = salt.fileserver.roots.symlink_list({'saltenv': 'base'}) + ret = roots.symlink_list({'saltenv': 'base'}) self.assertDictEqual(ret, {'dest_sym': 'source_sym'}) From 07d37ed94428a967ba233c8a8040d41855acb058 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 16 Jan 2018 10:55:46 -0600 Subject: [PATCH 2/2] Add unicode_literals to salt.utils.url to fix str/unicode mismatch in fileclient --- salt/utils/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/url.py b/salt/utils/url.py index 7ec056f5ca..4c84b39441 100644 --- a/salt/utils/url.py +++ b/salt/utils/url.py @@ -4,7 +4,7 @@ URL utils ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import re import sys