diff --git a/salt/cli/batch.py b/salt/cli/batch.py index 0bd4a7ace6..6e0c5b7ac6 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -11,7 +11,7 @@ import copy from datetime import datetime, timedelta # Import salt libs -import salt.utils # Can be removed once print_cli is moved +import salt.utils # TODO: Remove this once print_cli is moved import salt.client import salt.output import salt.exceptions diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 5c04bab11c..dca0b70edf 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -20,7 +20,7 @@ import salt.minion import salt.output import salt.payload import salt.transport -import salt.utils # Can be removed once print_cli, activate_profile, and output_profile are moved +import salt.utils # TODO: Remove this once print_cli, activate_profile, and output_profile are moved import salt.utils.args import salt.utils.files import salt.utils.jid diff --git a/salt/cli/run.py b/salt/cli/run.py index c780681368..f54e984252 100644 --- a/salt/cli/run.py +++ b/salt/cli/run.py @@ -2,7 +2,7 @@ from __future__ import print_function from __future__ import absolute_import -import salt.utils # Can be removed once activate_profile and output_profile are moved +import salt.utils # TODO: Remove this once activate_profile and output_profile are moved import salt.utils.parsers from salt.utils.verify import check_user, verify_log from salt.exceptions import SaltClientError diff --git a/salt/cli/salt.py b/salt/cli/salt.py index 24e3a2ecd4..814140ff12 100644 --- a/salt/cli/salt.py +++ b/salt/cli/salt.py @@ -7,7 +7,7 @@ sys.modules['pkg_resources'] = None import os # Import Salt libs -import salt.utils # Can be removed once print_cli is moved +import salt.utils # TODO: Remove this once print_cli is moved import salt.utils.parsers from salt.utils.args import yamlify_arg from salt.utils.verify import verify_log diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 0df7cf7dd1..9b851c6ad9 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -16,7 +16,7 @@ import copy as pycopy # Import Salt libs import salt.exceptions import salt.minion -import salt.utils # Can be removed once format_call is moved +import salt.utils # TODO: Remove this once format_call is moved import salt.utils.args import salt.utils.doc import salt.utils.error diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index cd3e53f0ea..96f20ba023 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -41,6 +41,7 @@ import salt.utils.args import salt.utils.atomicfile import salt.utils.event import salt.utils.files +import salt.utils.hashutils import salt.utils.network import salt.utils.path import salt.utils.stringutils @@ -1507,7 +1508,7 @@ def mod_data(fsclient): if not os.path.isfile(mod_path): continue mods_data[os.path.basename(fn_)] = mod_path - chunk = salt.utils.get_hash(mod_path) + chunk = salt.utils.hashutils.get_hash(mod_path) ver_base += chunk if mods_data: if ref in ret: diff --git a/salt/client/ssh/ssh_py_shim.py b/salt/client/ssh/ssh_py_shim.py index ea2d78f3c4..bff348ab43 100644 --- a/salt/client/ssh/ssh_py_shim.py +++ b/salt/client/ssh/ssh_py_shim.py @@ -137,7 +137,7 @@ def need_deployment(): sys.exit(EX_THIN_DEPLOY) -# Adapted from salt.utils.get_hash() +# Adapted from salt.utils.hashutils.get_hash() def get_hash(path, form=u'sha1', chunk_size=4096): ''' Generate a hash digest string for a file. diff --git a/salt/client/ssh/wrapper/config.py b/salt/client/ssh/wrapper/config.py index cf97316928..fd02eb7b83 100644 --- a/salt/client/ssh/wrapper/config.py +++ b/salt/client/ssh/wrapper/config.py @@ -9,8 +9,8 @@ import re import os # Import salt libs -import salt.utils # Can be removed once normalize_mode is moved import salt.utils.data +import salt.utils.files import salt.syspaths as syspaths # Import 3rd-party libs @@ -82,7 +82,7 @@ def manage_mode(mode): # config.manage_mode should no longer be invoked from the __salt__ dunder # in Salt code, this function is only being left here for backwards # compatibility. - return salt.utils.normalize_mode(mode) + return salt.utils.files.normalize_mode(mode) def valid_fileproto(uri): diff --git a/salt/client/ssh/wrapper/grains.py b/salt/client/ssh/wrapper/grains.py index 1d610da52d..599f7fc13f 100644 --- a/salt/client/ssh/wrapper/grains.py +++ b/salt/client/ssh/wrapper/grains.py @@ -11,7 +11,6 @@ import math import json # Import salt libs -import salt.utils # Can be removed once is_true is moved import salt.utils.data import salt.utils.dictupdate from salt.defaults import DEFAULT_TARGET_DELIM @@ -122,7 +121,7 @@ def items(sanitize=False): salt '*' grains.items sanitize=True ''' - if salt.utils.is_true(sanitize): + if salt.utils.data.is_true(sanitize): out = dict(__grains__) for key, func in six.iteritems(_SANITIZERS): if key in out: @@ -155,7 +154,7 @@ def item(*args, **kwargs): ret[arg] = __grains__[arg] except KeyError: pass - if salt.utils.is_true(kwargs.get(u'sanitize')): + if salt.utils.data.is_true(kwargs.get(u'sanitize')): for arg, func in six.iteritems(_SANITIZERS): if arg in ret: ret[arg] = func(ret[arg]) diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index bc4c610a28..cac5a5cef2 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -13,8 +13,9 @@ import logging # Import salt libs import salt.client.ssh.shell import salt.client.ssh.state -import salt.utils # Can be removed once get_hash, test_mode are moved +import salt.utils.args import salt.utils.data +import salt.utils.hashutils import salt.utils.thin import salt.roster import salt.state @@ -101,7 +102,7 @@ def sls(mods, saltenv=u'base', test=None, exclude=None, **kwargs): __pillar__, st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__[u'hash_type']) cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__[u'thin_dir'], test, @@ -178,7 +179,7 @@ def low(data, **kwargs): __pillar__, st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__[u'hash_type']) cmd = u'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( __opts__[u'thin_dir'], trans_tar_sum, @@ -251,7 +252,7 @@ def high(data, **kwargs): __pillar__, st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__[u'hash_type']) cmd = u'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( __opts__[u'thin_dir'], trans_tar_sum, @@ -354,7 +355,7 @@ def highstate(test=None, **kwargs): __pillar__, st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__[u'hash_type']) cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__[u'thin_dir'], test, @@ -403,7 +404,7 @@ def top(topfn, test=None, **kwargs): __pillar__.update(kwargs.get(u'pillar', {})) st_kwargs = __salt__.kwargs __opts__[u'grains'] = __grains__ - if salt.utils.test_mode(test=test, **kwargs): + if salt.utils.args.test_mode(test=test, **kwargs): __opts__[u'test'] = True else: __opts__[u'test'] = __opts__.get(u'test', None) @@ -434,7 +435,7 @@ def top(topfn, test=None, **kwargs): __pillar__, st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__[u'hash_type']) cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( __opts__[u'thin_dir'], test, @@ -520,7 +521,7 @@ def show_sls(mods, saltenv=u'base', test=None, **kwargs): __pillar__.update(kwargs.get(u'pillar', {})) __opts__[u'grains'] = __grains__ opts = copy.copy(__opts__) - if salt.utils.test_mode(test=test, **kwargs): + if salt.utils.args.test_mode(test=test, **kwargs): opts[u'test'] = True else: opts[u'test'] = __opts__.get(u'test', None) @@ -563,7 +564,7 @@ def show_low_sls(mods, saltenv=u'base', test=None, **kwargs): __opts__[u'grains'] = __grains__ opts = copy.copy(__opts__) - if salt.utils.test_mode(test=test, **kwargs): + if salt.utils.args.test_mode(test=test, **kwargs): opts[u'test'] = True else: opts[u'test'] = __opts__.get(u'test', None) @@ -652,7 +653,7 @@ def single(fun, name, test=None, **kwargs): opts = copy.deepcopy(__opts__) # Set test mode - if salt.utils.test_mode(test=test, **kwargs): + if salt.utils.args.test_mode(test=test, **kwargs): opts[u'test'] = True else: opts[u'test'] = __opts__.get(u'test', None) @@ -696,7 +697,7 @@ def single(fun, name, test=None, **kwargs): roster_grains) # Create a hash so we can verify the tar on the target system - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__[u'hash_type']) # We use state.pkg to execute the "state package" cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 04b84a1759..fc48740255 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -2099,7 +2099,7 @@ class Map(Cloud): master_temp_pub = salt.utils.files.mkstemp() with salt.utils.files.fopen(master_temp_pub, 'w') as mtp: mtp.write(pub) - master_finger = salt.utils.pem_finger(master_temp_pub, sum_type=self.opts['hash_type']) + master_finger = salt.utils.crypt.pem_finger(master_temp_pub, sum_type=self.opts['hash_type']) os.unlink(master_temp_pub) if master_profile.get('make_minion', True) is True: @@ -2184,7 +2184,7 @@ class Map(Cloud): # mitigate man-in-the-middle attacks master_pub = os.path.join(self.opts['pki_dir'], 'master.pub') if os.path.isfile(master_pub): - master_finger = salt.utils.pem_finger(master_pub, sum_type=self.opts['hash_type']) + master_finger = salt.utils.crypt.pem_finger(master_pub, sum_type=self.opts['hash_type']) opts = self.opts.copy() if self.opts['parallel']: diff --git a/salt/cloud/clouds/opennebula.py b/salt/cloud/clouds/opennebula.py index 55aa531d0d..9d03202f17 100644 --- a/salt/cloud/clouds/opennebula.py +++ b/salt/cloud/clouds/opennebula.py @@ -76,7 +76,7 @@ from salt.exceptions import ( SaltCloudNotFound, SaltCloudSystemExit ) -import salt.utils +import salt.utils.data import salt.utils.files # Import Third Party Libs @@ -1575,7 +1575,7 @@ def image_persistent(call=None, kwargs=None): server, user, password = _get_xml_rpc() auth = ':'.join([user, password]) - response = server.one.image.persistent(auth, int(image_id), salt.utils.is_true(persist)) + response = server.one.image.persistent(auth, int(image_id), salt.utils.data.is_true(persist)) data = { 'action': 'image.persistent', @@ -2794,7 +2794,7 @@ def vm_allocate(call=None, kwargs=None): server, user, password = _get_xml_rpc() auth = ':'.join([user, password]) - response = server.one.vm.allocate(auth, data, salt.utils.is_true(hold)) + response = server.one.vm.allocate(auth, data, salt.utils.data.is_true(hold)) ret = { 'action': 'vm.allocate', @@ -3026,7 +3026,7 @@ def vm_deploy(name, kwargs=None, call=None): response = server.one.vm.deploy(auth, int(vm_id), int(host_id), - salt.utils.is_true(capacity_maintained), + salt.utils.data.is_true(capacity_maintained), int(datastore_id)) data = { @@ -3495,8 +3495,8 @@ def vm_migrate(name, kwargs=None, call=None): response = server.one.vm.migrate(auth, vm_id, int(host_id), - salt.utils.is_true(live_migration), - salt.utils.is_true(capacity_maintained), + salt.utils.data.is_true(live_migration), + salt.utils.data.is_true(capacity_maintained), int(datastore_id)) data = { @@ -3615,7 +3615,7 @@ def vm_resize(name, kwargs=None, call=None): server, user, password = _get_xml_rpc() auth = ':'.join([user, password]) vm_id = int(get_vm_id(kwargs={'name': name})) - response = server.one.vm.resize(auth, vm_id, data, salt.utils.is_true(capacity_maintained)) + response = server.one.vm.resize(auth, vm_id, data, salt.utils.data.is_true(capacity_maintained)) ret = { 'action': 'vm.resize', diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index b26a36a47c..a073ba29e9 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -185,7 +185,7 @@ except Exception: from salt.cloud.libcloudfuncs import * # pylint: disable=W0614,W0401 # Import salt libs -import salt.utils # Can be removed once namespaced_function has been moved +import salt.utils # TODO: Remove this once namespaced_function has been moved import salt.utils.cloud import salt.utils.files import salt.utils.pycrypto diff --git a/salt/crypt.py b/salt/crypt.py index 764015e480..241b9b02b7 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -50,7 +50,6 @@ import salt.defaults.exitcodes import salt.payload import salt.transport.client import salt.transport.frame -import salt.utils # Can be removed when pem_finger is moved import salt.utils.crypt import salt.utils.decorators import salt.utils.event @@ -681,11 +680,11 @@ class AsyncAuth(object): if self.opts.get(u'syndic_master', False): # Is syndic syndic_finger = self.opts.get(u'syndic_finger', self.opts.get(u'master_finger', False)) if syndic_finger: - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != syndic_finger: + if salt.utils.crypt.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != syndic_finger: self._finger_fail(syndic_finger, m_pub_fn) else: if self.opts.get(u'master_finger', False): - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != self.opts[u'master_finger']: + if salt.utils.crypt.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != self.opts[u'master_finger']: self._finger_fail(self.opts[u'master_finger'], m_pub_fn) auth[u'publish_port'] = payload[u'publish_port'] raise tornado.gen.Return(auth) @@ -1030,7 +1029,7 @@ class AsyncAuth(object): u'matches the fingerprint of the correct master and that ' u'this minion is not subject to a man-in-the-middle attack.', finger, - salt.utils.pem_finger(master_key, sum_type=self.opts[u'hash_type']) + salt.utils.crypt.pem_finger(master_key, sum_type=self.opts[u'hash_type']) ) sys.exit(42) @@ -1237,11 +1236,11 @@ class SAuth(AsyncAuth): if self.opts.get(u'syndic_master', False): # Is syndic syndic_finger = self.opts.get(u'syndic_finger', self.opts.get(u'master_finger', False)) if syndic_finger: - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != syndic_finger: + if salt.utils.crypt.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != syndic_finger: self._finger_fail(syndic_finger, m_pub_fn) else: if self.opts.get(u'master_finger', False): - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != self.opts[u'master_finger']: + if salt.utils.crypt.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != self.opts[u'master_finger']: self._finger_fail(self.opts[u'master_finger'], m_pub_fn) auth[u'publish_port'] = payload[u'publish_port'] return auth diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index ad63e43e56..4d5a8a1c06 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -15,7 +15,7 @@ import stat # Import salt libs import salt.crypt -import salt.utils # Can be removed once check_whitelist_blacklist, expr_match, get_values_of_matching_keys are moved +import salt.utils # TODO: Remove this once check_whitelist_blacklist, expr_match, get_values_of_matching_keys are moved import salt.cache import salt.client import salt.payload diff --git a/salt/fileclient.py b/salt/fileclient.py index 35c63b2cb1..0836df7230 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -27,6 +27,7 @@ import salt.fileserver import salt.utils import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.http import salt.utils.path import salt.utils.platform @@ -530,7 +531,7 @@ class Client(object): try: source_hash = source_hash.split('=')[-1] form = salt.utils.files.HASHES_REVMAP[len(source_hash)] - if salt.utils.get_hash(dest, form) == source_hash: + if salt.utils.hashutils.get_hash(dest, form) == source_hash: log.debug( 'Cached copy of %s (%s) matches source_hash %s, ' 'skipping download', url, dest, source_hash @@ -968,7 +969,7 @@ class LocalClient(Client): fnd_path = fnd hash_type = self.opts.get(u'hash_type', u'md5') - ret[u'hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) + ret[u'hsum'] = salt.utils.hashutils.get_hash(fnd_path, form=hash_type) ret[u'hash_type'] = hash_type return ret @@ -999,7 +1000,7 @@ class LocalClient(Client): fnd_stat = None hash_type = self.opts.get(u'hash_type', u'md5') - ret[u'hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) + ret[u'hsum'] = salt.utils.hashutils.get_hash(fnd_path, form=hash_type) ret[u'hash_type'] = hash_type return ret, fnd_stat @@ -1194,7 +1195,7 @@ class RemoteClient(Client): # Master has prompted a file verification, if the # verification fails, re-download the file. Try 3 times d_tries += 1 - hsum = salt.utils.get_hash(dest, salt.utils.stringutils.to_str(data.get(u'hash_type', b'md5'))) # future lint: disable=non-unicode-string + hsum = salt.utils.hashutils.get_hash(dest, salt.utils.stringutils.to_str(data.get(u'hash_type', b'md5'))) # future lint: disable=non-unicode-string if hsum != data[u'hsum']: log.warning( u'Bad download of file %s, attempt %d of 3', @@ -1308,7 +1309,7 @@ class RemoteClient(Client): else: ret = {} hash_type = self.opts.get(u'hash_type', u'md5') - ret[u'hsum'] = salt.utils.get_hash(path, form=hash_type) + ret[u'hsum'] = salt.utils.hashutils.get_hash(path, form=hash_type) ret[u'hash_type'] = hash_type return ret load = {u'path': path, diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py index 7eb919fa7d..560455779b 100644 --- a/salt/fileserver/azurefs.py +++ b/salt/fileserver/azurefs.py @@ -58,6 +58,7 @@ import salt.fileserver import salt.utils import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.path from salt.utils.versions import LooseVersion @@ -166,7 +167,7 @@ def serve_file(load, fnd): with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) - if data and six.PY3 and not salt.utils.is_bin_file(fpath): + if data and six.PY3 and not salt.utils.files.is_binary(fpath): data = data.decode(__salt_system_encoding__) if gzip and data: data = salt.utils.gzip_util.compress(data, gzip) @@ -226,7 +227,7 @@ def update(): if os.path.exists(fname): # File exists, check the hashes source_md5 = blob.properties.content_settings.content_md5 - local_md5 = base64.b64encode(salt.utils.get_hash(fname, 'md5').decode('hex')) + local_md5 = base64.b64encode(salt.utils.hashutils.get_hash(fname, 'md5').decode('hex')) if local_md5 != source_md5: update = True else: @@ -289,7 +290,7 @@ def file_hash(load, fnd): if not os.path.isfile(hashdest): if not os.path.exists(os.path.dirname(hashdest)): os.makedirs(os.path.dirname(hashdest)) - ret['hsum'] = salt.utils.get_hash(path, __opts__['hash_type']) + ret['hsum'] = salt.utils.hashutils.get_hash(path, __opts__['hash_type']) with salt.utils.files.fopen(hashdest, 'w+') as fp_: fp_.write(ret['hsum']) return ret diff --git a/salt/fileserver/hgfs.py b/salt/fileserver/hgfs.py index 42c830c528..495021dfd5 100644 --- a/salt/fileserver/hgfs.py +++ b/salt/fileserver/hgfs.py @@ -63,6 +63,7 @@ import salt.utils import salt.utils.data import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.url import salt.utils.versions import salt.fileserver @@ -752,7 +753,7 @@ def serve_file(load, fnd): with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) - if data and six.PY3 and not salt.utils.is_bin_file(fpath): + if data and six.PY3 and not salt.utils.files.is_binary(fpath): data = data.decode(__salt_system_encoding__) if gzip and data: data = salt.utils.gzip_util.compress(data, gzip) @@ -780,7 +781,7 @@ def file_hash(load, fnd): '{0}.hash.{1}'.format(relpath, __opts__['hash_type'])) if not os.path.isfile(hashdest): - ret['hsum'] = salt.utils.get_hash(path, __opts__['hash_type']) + ret['hsum'] = salt.utils.hashutils.get_hash(path, __opts__['hash_type']) with salt.utils.files.fopen(hashdest, 'w+') as fp_: fp_.write(ret['hsum']) return ret diff --git a/salt/fileserver/minionfs.py b/salt/fileserver/minionfs.py index 292bf0f85e..fd5dd2ed23 100644 --- a/salt/fileserver/minionfs.py +++ b/salt/fileserver/minionfs.py @@ -34,6 +34,7 @@ import salt.fileserver import salt.utils import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.url import salt.utils.versions @@ -135,7 +136,7 @@ def serve_file(load, fnd): with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) - if data and six.PY3 and not salt.utils.is_bin_file(fpath): + if data and six.PY3 and not salt.utils.files.is_binary(fpath): data = data.decode(__salt_system_encoding__) if gzip and data: data = salt.utils.gzip_util.compress(data, gzip) @@ -213,7 +214,7 @@ def file_hash(load, fnd): return ret # if we don't have a cache entry-- lets make one - ret['hsum'] = salt.utils.get_hash(path, __opts__['hash_type']) + ret['hsum'] = salt.utils.hashutils.get_hash(path, __opts__['hash_type']) cache_dir = os.path.dirname(cache_path) # make cache directory if it doesn't exist if not os.path.exists(cache_dir): diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index 0547ff0e62..a92537fa14 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -24,10 +24,10 @@ import logging # Import salt libs import salt.fileserver -import salt.utils # Can be removed once is_bin_file and get_hash are moved import salt.utils.event import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.path import salt.utils.versions from salt.ext import six @@ -127,7 +127,7 @@ def serve_file(load, fnd): with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) - if data and six.PY3 and not salt.utils.is_bin_file(fpath): + if data and six.PY3 and not salt.utils.files.is_binary(fpath): data = data.decode(__salt_system_encoding__) if gzip and data: data = salt.utils.gzip_util.compress(data, gzip) @@ -258,7 +258,7 @@ def file_hash(load, fnd): return file_hash(load, fnd) # if we don't have a cache entry-- lets make one - ret['hsum'] = salt.utils.get_hash(path, __opts__['hash_type']) + ret['hsum'] = salt.utils.hashutils.get_hash(path, __opts__['hash_type']) cache_dir = os.path.dirname(cache_path) # make cache directory if it doesn't exist if not os.path.exists(cache_dir): diff --git a/salt/fileserver/s3fs.py b/salt/fileserver/s3fs.py index 04f0b5e51c..5883408d94 100644 --- a/salt/fileserver/s3fs.py +++ b/salt/fileserver/s3fs.py @@ -71,6 +71,7 @@ import salt.modules import salt.utils import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.versions # Import 3rd-party libs @@ -180,7 +181,7 @@ def file_hash(load, fnd): fnd['path']) if os.path.isfile(cached_file_path): - ret['hsum'] = salt.utils.get_hash(cached_file_path) + ret['hsum'] = salt.utils.hashutils.get_hash(cached_file_path) ret['hash_type'] = 'md5' return ret @@ -216,7 +217,7 @@ def serve_file(load, fnd): with salt.utils.files.fopen(cached_file_path, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) - if data and six.PY3 and not salt.utils.is_bin_file(cached_file_path): + if data and six.PY3 and not salt.utils.files.is_binary(cached_file_path): data = data.decode(__salt_system_encoding__) if gzip and data: data = salt.utils.gzip_util.compress(data, gzip) @@ -619,7 +620,7 @@ def _get_file_from_s3(metadata, saltenv, bucket_name, path, cached_file_path): if file_etag.find('-') == -1: file_md5 = file_etag - cached_md5 = salt.utils.get_hash(cached_file_path, 'md5') + cached_md5 = salt.utils.hashutils.get_hash(cached_file_path, 'md5') # hashes match we have a cache hit if cached_md5 == file_md5: diff --git a/salt/fileserver/svnfs.py b/salt/fileserver/svnfs.py index cc60fbb312..c2ff3d373a 100644 --- a/salt/fileserver/svnfs.py +++ b/salt/fileserver/svnfs.py @@ -58,6 +58,7 @@ import salt.utils import salt.utils.data import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.url import salt.utils.versions import salt.fileserver @@ -647,7 +648,7 @@ def serve_file(load, fnd): with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) - if data and six.PY3 and not salt.utils.is_bin_file(fpath): + if data and six.PY3 and not salt.utils.files.is_binary(fpath): data = data.decode(__salt_system_encoding__) if gzip and data: data = salt.utils.gzip_util.compress(data, gzip) @@ -697,7 +698,7 @@ def file_hash(load, fnd): return ret # if we don't have a cache entry-- lets make one - ret['hsum'] = salt.utils.get_hash(path, __opts__['hash_type']) + ret['hsum'] = salt.utils.hashutils.get_hash(path, __opts__['hash_type']) cache_dir = os.path.dirname(cache_path) # make cache directory if it doesn't exist if not os.path.exists(cache_dir): diff --git a/salt/key.py b/salt/key.py index 4d315a9e74..a4f1eefa0f 100644 --- a/salt/key.py +++ b/salt/key.py @@ -22,8 +22,10 @@ import salt.crypt import salt.daemons.masterapi import salt.exceptions import salt.minion -import salt.utils +import salt.utils # TODO: Remove this once get_master_key is moved import salt.utils.args +import salt.utils.crypt +import salt.utils.data import salt.utils.event import salt.utils.files import salt.utils.kinds @@ -416,7 +418,7 @@ class Key(object): keydir, keyname, keysize, user = self._get_key_attrs(keydir, keyname, keysize, user) salt.crypt.gen_keys(keydir, keyname, keysize, user, self.passphrase) - return salt.utils.pem_finger(os.path.join(keydir, keyname + u'.pub')) + return salt.utils.crypt.pem_finger(os.path.join(keydir, keyname + u'.pub')) def gen_signature(self, privkey, pubkey, sig_path): ''' @@ -544,7 +546,7 @@ class Key(object): if u',' in match and isinstance(match, six.string_types): match = match.split(u',') for status, keys in six.iteritems(matches): - for key in salt.utils.isorted(keys): + for key in salt.utils.data.sorted_ignorecase(keys): if isinstance(match, list): for match_item in match: if fnmatch.fnmatch(key, match_item): @@ -566,7 +568,7 @@ class Key(object): ret = {} cur_keys = self.list_keys() for status, keys in six.iteritems(match_dict): - for key in salt.utils.isorted(keys): + for key in salt.utils.data.sorted_ignorecase(keys): for keydir in (self.ACC, self.PEND, self.REJ, self.DEN): if keydir and fnmatch.filter(cur_keys.get(keydir, []), key): ret.setdefault(keydir, []).append(key) @@ -577,7 +579,7 @@ class Key(object): Return a dict of local keys ''' ret = {u'local': []} - for fn_ in salt.utils.isorted(os.listdir(self.opts[u'pki_dir'])): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(self.opts[u'pki_dir'])): if fn_.endswith(u'.pub') or fn_.endswith(u'.pem'): path = os.path.join(self.opts[u'pki_dir'], fn_) if os.path.isfile(path): @@ -603,7 +605,7 @@ class Key(object): continue ret[os.path.basename(dir_)] = [] try: - for fn_ in salt.utils.isorted(os.listdir(dir_)): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(dir_)): if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(dir_, fn_)): ret[os.path.basename(dir_)].append(fn_) @@ -628,25 +630,25 @@ class Key(object): ret = {} if match.startswith(u'acc'): ret[os.path.basename(acc)] = [] - for fn_ in salt.utils.isorted(os.listdir(acc)): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(acc)): if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(acc, fn_)): ret[os.path.basename(acc)].append(fn_) elif match.startswith(u'pre') or match.startswith(u'un'): ret[os.path.basename(pre)] = [] - for fn_ in salt.utils.isorted(os.listdir(pre)): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(pre)): if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(pre, fn_)): ret[os.path.basename(pre)].append(fn_) elif match.startswith(u'rej'): ret[os.path.basename(rej)] = [] - for fn_ in salt.utils.isorted(os.listdir(rej)): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(rej)): if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(rej, fn_)): ret[os.path.basename(rej)].append(fn_) elif match.startswith(u'den') and den is not None: ret[os.path.basename(den)] = [] - for fn_ in salt.utils.isorted(os.listdir(den)): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(den)): if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(den, fn_)): ret[os.path.basename(den)].append(fn_) @@ -661,7 +663,7 @@ class Key(object): ret = {} for status, keys in six.iteritems(self.name_match(match)): ret[status] = {} - for key in salt.utils.isorted(keys): + for key in salt.utils.data.sorted_ignorecase(keys): path = os.path.join(self.opts[u'pki_dir'], status, key) with salt.utils.files.fopen(path, u'r') as fp_: ret[status][key] = fp_.read() @@ -674,7 +676,7 @@ class Key(object): ret = {} for status, keys in six.iteritems(self.list_keys()): ret[status] = {} - for key in salt.utils.isorted(keys): + for key in salt.utils.data.sorted_ignorecase(keys): path = os.path.join(self.opts[u'pki_dir'], status, key) with salt.utils.files.fopen(path, u'r') as fp_: ret[status][key] = fp_.read() @@ -927,7 +929,7 @@ class Key(object): path = os.path.join(self.opts[u'pki_dir'], key) else: path = os.path.join(self.opts[u'pki_dir'], status, key) - ret[status][key] = salt.utils.pem_finger(path, sum_type=hash_type) + ret[status][key] = salt.utils.crypt.pem_finger(path, sum_type=hash_type) return ret def finger_all(self, hash_type=None): @@ -945,7 +947,7 @@ class Key(object): path = os.path.join(self.opts[u'pki_dir'], key) else: path = os.path.join(self.opts[u'pki_dir'], status, key) - ret[status][key] = salt.utils.pem_finger(path, sum_type=hash_type) + ret[status][key] = salt.utils.crypt.pem_finger(path, sum_type=hash_type) return ret @@ -1165,7 +1167,7 @@ class RaetKey(Key): ret = {} for status, keys in six.iteritems(self.name_match(match)): ret[status] = {} - for key in salt.utils.isorted(keys): + for key in salt.utils.data.sorted_ignorecase(keys): ret[status][key] = self._get_key_str(key, status) return ret @@ -1176,7 +1178,7 @@ class RaetKey(Key): ret = {} for status, keys in six.iteritems(self.list_keys()): ret[status] = {} - for key in salt.utils.isorted(keys): + for key in salt.utils.data.sorted_ignorecase(keys): ret[status][key] = self._get_key_str(key, status) return ret diff --git a/salt/master.py b/salt/master.py index f24f78330c..05c0cc704b 100644 --- a/salt/master.py +++ b/salt/master.py @@ -47,7 +47,7 @@ import tornado.gen # pylint: disable=F0401 # Import salt libs import salt.crypt -import salt.utils # Can be removed once get_values_of_matching_keys is moved +import salt.utils # TODO: Remove this once get_values_of_matching_keys is moved import salt.client import salt.payload import salt.pillar diff --git a/salt/modules/apk.py b/salt/modules/apk.py index 51c1ad6627..13975fe8d6 100644 --- a/salt/modules/apk.py +++ b/salt/modules/apk.py @@ -18,7 +18,6 @@ import copy import logging # Import salt libs -import salt.utils import salt.utils.data import salt.utils.itertools @@ -131,9 +130,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs salt '*' pkg.list_pkgs versions_as_list=True ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -177,7 +176,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version salt '*' pkg.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) if len(names) == 0: return '' @@ -290,7 +289,7 @@ def install(name=None, {'': {'old': '', 'new': ''}} ''' - refreshdb = salt.utils.is_true(refresh) + refreshdb = salt.utils.data.is_true(refresh) pkg_to_install = [] old = list_pkgs() @@ -447,7 +446,7 @@ def upgrade(name=None, pkgs=None, refresh=True): 'comment': '', } - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() old = list_pkgs() @@ -497,7 +496,7 @@ def list_upgrades(refresh=True): salt '*' pkg.list_upgrades ''' ret = {} - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() cmd = ['apk', 'upgrade', '-s'] diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 2ee9223cc2..591b7ab2dd 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -38,7 +38,7 @@ from salt.ext.six.moves.urllib.request import Request as _Request, urlopen as _u import salt.config import salt.syspaths from salt.modules.cmdmod import _parse_env -import salt.utils +import salt.utils # TODO: Remove this when alias_function is moved import salt.utils.args import salt.utils.data import salt.utils.files @@ -240,8 +240,8 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version fromrepo=unstable salt '*' pkg.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) - show_installed = salt.utils.is_true(kwargs.pop('show_installed', False)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) + show_installed = salt.utils.data.is_true(kwargs.pop('show_installed', False)) if 'repo' in kwargs: raise SaltInvocationError( 'The \'repo\' argument is invalid, use \'fromrepo\' instead' @@ -371,7 +371,7 @@ def refresh_db(cache_valid_time=0, failhard=False): ''' # Remove rtag file to keep multiple refreshes from happening in pkg states salt.utils.pkg.clear_rtag(__opts__) - failhard = salt.utils.is_true(failhard) + failhard = salt.utils.data.is_true(failhard) ret = {} error_repos = list() @@ -576,7 +576,7 @@ def install(name=None, 'new': ''}} ''' _refresh_db = False - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): _refresh_db = True if 'version' in kwargs and kwargs['version']: _refresh_db = False @@ -1099,7 +1099,7 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): salt '*' pkg.upgrade ''' cache_valid_time = kwargs.pop('cache_valid_time', 0) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db(cache_valid_time) old = list_pkgs() @@ -1198,7 +1198,7 @@ def hold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W0613 if not state: ret[target]['comment'] = ('Package {0} not currently held.' .format(target)) - elif not salt.utils.is_true(state.get('hold', False)): + elif not salt.utils.data.is_true(state.get('hold', False)): if 'test' in __opts__ and __opts__['test']: ret[target].update(result=None) ret[target]['comment'] = ('Package {0} is set to be held.' @@ -1272,7 +1272,7 @@ def unhold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W06 if not state: ret[target]['comment'] = ('Package {0} does not have a state.' .format(target)) - elif salt.utils.is_true(state.get('hold', False)): + elif salt.utils.data.is_true(state.get('hold', False)): if 'test' in __opts__ and __opts__['test']: ret[target].update(result=None) ret[target]['comment'] = ('Package {0} is set not to be ' @@ -1340,9 +1340,9 @@ def list_pkgs(versions_as_list=False, salt '*' pkg.list_pkgs salt '*' pkg.list_pkgs versions_as_list=True ''' - versions_as_list = salt.utils.is_true(versions_as_list) - removed = salt.utils.is_true(removed) - purge_desired = salt.utils.is_true(purge_desired) + versions_as_list = salt.utils.data.is_true(versions_as_list) + removed = salt.utils.data.is_true(removed) + purge_desired = salt.utils.data.is_true(purge_desired) if 'pkg.list_pkgs' in __context__: if removed: @@ -1502,7 +1502,7 @@ def list_upgrades(refresh=True, dist_upgrade=True, **kwargs): salt '*' pkg.list_upgrades ''' cache_valid_time = kwargs.pop('cache_valid_time', 0) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db(cache_valid_time) return _get_upgradable(dist_upgrade, **kwargs) @@ -2389,7 +2389,7 @@ def mod_repo(repo, saltenv='base', **kwargs): kwargs['architectures'] = kwargs['architectures'].split(',') if 'disabled' in kwargs: - kwargs['disabled'] = salt.utils.is_true(kwargs['disabled']) + kwargs['disabled'] = salt.utils.data.is_true(kwargs['disabled']) kw_type = kwargs.get('type') kw_dist = kwargs.get('dist') diff --git a/salt/modules/chocolatey.py b/salt/modules/chocolatey.py index 1226505f47..5f19196447 100644 --- a/salt/modules/chocolatey.py +++ b/salt/modules/chocolatey.py @@ -14,7 +14,7 @@ import re import tempfile # Import salt libs -import salt.utils +import salt.utils.data import salt.utils.platform from salt.utils.versions import LooseVersion as _LooseVersion from salt.exceptions import CommandExecutionError, CommandNotFoundError, \ @@ -274,9 +274,9 @@ def list_(narrow=None, cmd = [choc_path, 'list'] if narrow: cmd.append(narrow) - if salt.utils.is_true(all_versions): + if salt.utils.data.is_true(all_versions): cmd.append('--allversions') - if salt.utils.is_true(pre_versions): + if salt.utils.data.is_true(pre_versions): cmd.append('--prerelease') if source: cmd.extend(['--source', source]) @@ -452,9 +452,9 @@ def install(name, cmd.extend(['--version', version]) if source: cmd.extend(['--source', source]) - if salt.utils.is_true(force): + if salt.utils.data.is_true(force): cmd.append('--force') - if salt.utils.is_true(pre_versions): + if salt.utils.data.is_true(pre_versions): cmd.append('--prerelease') if install_args: cmd.extend(['--installarguments', install_args]) @@ -802,9 +802,9 @@ def upgrade(name, cmd.extend(['-version', version]) if source: cmd.extend(['--source', source]) - if salt.utils.is_true(force): + if salt.utils.data.is_true(force): cmd.append('--force') - if salt.utils.is_true(pre_versions): + if salt.utils.data.is_true(pre_versions): cmd.append('--prerelease') if install_args: cmd.extend(['--installarguments', install_args]) @@ -862,7 +862,7 @@ def update(name, source=None, pre_versions=False, no_progress=False): cmd = [choc_path, 'update', name] if source: cmd.extend(['--source', source]) - if salt.utils.is_true(pre_versions): + if salt.utils.data.is_true(pre_versions): cmd.append('--prerelease') if no_progress: cmd.append(_no_progress(__context__)) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index de4f503019..8a03bacd7e 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -24,7 +24,6 @@ import re import tempfile # Import salt libs -import salt.utils import salt.utils.args import salt.utils.data import salt.utils.files @@ -34,6 +33,7 @@ import salt.utils.powershell import salt.utils.stringutils import salt.utils.templates import salt.utils.timed_subprocess +import salt.utils.user import salt.utils.versions import salt.utils.vt import salt.grains.extra @@ -188,7 +188,7 @@ def _check_loglevel(level='info', quiet=False): ) return LOG_LEVELS['info'] - if salt.utils.is_true(quiet) or str(level).lower() == 'quiet': + if salt.utils.data.is_true(quiet) or str(level).lower() == 'quiet': return None try: @@ -525,7 +525,7 @@ def _run(cmd, if runas or umask: kwargs['preexec_fn'] = functools.partial( - salt.utils.chugid_and_umask, + salt.utils.user.chugid_and_umask, runas, _umask) diff --git a/salt/modules/config.py b/salt/modules/config.py index d9dfeafbaf..3b641f8e87 100644 --- a/salt/modules/config.py +++ b/salt/modules/config.py @@ -12,8 +12,9 @@ import logging # Import salt libs import salt.config -import salt.utils import salt.utils.data +import salt.utils.dictupdate +import salt.utils.files import salt.utils.platform try: # Gated for salt-ssh (salt.utils.cloud imports msgpack) @@ -108,7 +109,7 @@ def manage_mode(mode): # config.manage_mode should no longer be invoked from the __salt__ dunder # in Salt code, this function is only being left here for backwards # compatibility. - return salt.utils.normalize_mode(mode) + return salt.utils.files.normalize_mode(mode) def valid_fileproto(uri): diff --git a/salt/modules/cp.py b/salt/modules/cp.py index cdbeb4434e..9c2ea4e3c1 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -748,7 +748,7 @@ def stat_file(path, saltenv='base', octal=True): stat = _client().hash_and_stat_file(path, saltenv)[1] if stat is None: return stat - return salt.utils.st_mode_to_octal(stat[0]) if octal is True else stat[0] + return salt.utils.files.st_mode_to_octal(stat[0]) if octal is True else stat[0] def push(path, keep_symlinks=False, upload_path=None, remove_source=False): diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 5e9ef7d3b4..58cc1f0d25 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -206,6 +206,7 @@ import salt.utils.args import salt.utils.decorators import salt.utils.docker import salt.utils.files +import salt.utils.hashutils import salt.utils.path import salt.utils.stringutils import salt.utils.thin @@ -5431,7 +5432,7 @@ def sls(name, mods=None, saltenv='base', **kwargs): ret = None try: - trans_tar_sha256 = salt.utils.get_hash(trans_tar, 'sha256') + trans_tar_sha256 = salt.utils.hashutils.get_hash(trans_tar, 'sha256') copy_to(name, trans_tar, os.path.join(trans_dest_path, 'salt_state.tgz'), diff --git a/salt/modules/ebuild.py b/salt/modules/ebuild.py index 354a37732d..facde056bd 100644 --- a/salt/modules/ebuild.py +++ b/salt/modules/ebuild.py @@ -21,7 +21,7 @@ import logging import re # Import salt libs -import salt.utils +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.args import salt.utils.data import salt.utils.path @@ -237,7 +237,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version salt '*' pkg.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) if len(names) == 0: return '' @@ -334,7 +334,7 @@ def list_upgrades(refresh=True, backtrack=3, **kwargs): # pylint: disable=W0613 salt '*' pkg.list_upgrades ''' - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() return _get_upgradable(backtrack) @@ -394,9 +394,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -597,7 +597,7 @@ def install(name=None, 'binhost': binhost, } )) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() try: @@ -764,7 +764,7 @@ def update(pkg, slot=None, fromrepo=None, refresh=False, binhost=None): salt '*' pkg.update ''' - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() full_atom = pkg @@ -864,7 +864,7 @@ def upgrade(refresh=True, binhost=None, backtrack=3): 'result': True, 'comment': ''} - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() if binhost == 'try': diff --git a/salt/modules/file.py b/salt/modules/file.py index 0ca2a16dc6..1e91d73552 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -52,6 +52,7 @@ import salt.utils.atomicfile import salt.utils.filebuffer import salt.utils.files import salt.utils.find +import salt.utils.hashutils import salt.utils.itertools import salt.utils.locales import salt.utils.path @@ -118,8 +119,8 @@ def _binary_replace(old, new): This function should only be run AFTER it has been determined that the files differ. ''' - old_isbin = not __utils__['files.is_text_file'](old) - new_isbin = not __utils__['files.is_text_file'](new) + old_isbin = not __utils__['files.is_text'](old) + new_isbin = not __utils__['files.is_text'](new) if any((old_isbin, new_isbin)): if all((old_isbin, new_isbin)): return u'Replace binary file' @@ -660,7 +661,7 @@ def get_sum(path, form='sha256'): if not os.path.isfile(path): return 'File not found' - return salt.utils.get_hash(path, form, 4096) + return salt.utils.hashutils.get_hash(path, form, 4096) def get_hash(path, form='sha256', chunk_size=65536): @@ -688,7 +689,7 @@ def get_hash(path, form='sha256', chunk_size=65536): salt '*' file.get_hash /etc/shadow ''' - return salt.utils.get_hash(os.path.expanduser(path), form, chunk_size) + return salt.utils.hashutils.get_hash(os.path.expanduser(path), form, chunk_size) def get_source_sum(file_name='', @@ -1428,7 +1429,7 @@ def comment_line(path, raise SaltInvocationError('File not found: {0}'.format(path)) # Make sure it is a text file - if not __utils__['files.is_text_file'](path): + if not __utils__['files.is_text'](path): raise SaltInvocationError( 'Cannot perform string replacements on a binary file: {0}'.format(path)) @@ -1471,7 +1472,7 @@ def comment_line(path, if not salt.utils.platform.is_windows(): pre_user = get_user(path) pre_group = get_group(path) - pre_mode = salt.utils.normalize_mode(get_mode(path)) + pre_mode = salt.utils.files.normalize_mode(get_mode(path)) # Create a copy to read from and to use as a backup later try: @@ -2172,7 +2173,7 @@ def replace(path, else: raise SaltInvocationError('File not found: {0}'.format(path)) - if not __utils__['files.is_text_file'](path): + if not __utils__['files.is_text'](path): raise SaltInvocationError( 'Cannot perform string replacements on a binary file: {0}' .format(path) @@ -2201,7 +2202,7 @@ def replace(path, if not salt.utils.platform.is_windows(): pre_user = get_user(path) pre_group = get_group(path) - pre_mode = salt.utils.normalize_mode(get_mode(path)) + pre_mode = salt.utils.files.normalize_mode(get_mode(path)) # Avoid TypeErrors by forcing repl to be bytearray related to mmap # Replacement text may contains integer: 123 for example @@ -2489,7 +2490,7 @@ def blockreplace(path, 'Only one of append and prepend_if_not_found is permitted' ) - if not __utils__['files.is_text_file'](path): + if not __utils__['files.is_text'](path): raise SaltInvocationError( 'Cannot perform string replacements on a binary file: {0}' .format(path) @@ -2604,7 +2605,7 @@ def blockreplace(path, perms = {} perms['user'] = get_user(path) perms['group'] = get_group(path) - perms['mode'] = salt.utils.normalize_mode(get_mode(path)) + perms['mode'] = salt.utils.files.normalize_mode(get_mode(path)) # backup old content if backup is not False: @@ -3300,7 +3301,7 @@ def copy(src, dst, recurse=False, remove_existing=False): if not salt.utils.platform.is_windows(): pre_user = get_user(src) pre_group = get_group(src) - pre_mode = salt.utils.normalize_mode(get_mode(src)) + pre_mode = salt.utils.files.normalize_mode(get_mode(src)) try: if (os.path.exists(dst) and os.path.isdir(dst)) or os.path.isdir(src): @@ -4354,7 +4355,7 @@ def check_perms(name, ret, user, group, mode, attrs=None, follow_symlinks=False) raise CommandExecutionError('{0} does not exist'.format(name)) perms['luser'] = cur['user'] perms['lgroup'] = cur['group'] - perms['lmode'] = salt.utils.normalize_mode(cur['mode']) + perms['lmode'] = salt.utils.files.normalize_mode(cur['mode']) is_dir = os.path.isdir(name) if not salt.utils.platform.is_windows() and not is_dir and lsattr_cmd: @@ -4371,13 +4372,13 @@ def check_perms(name, ret, user, group, mode, attrs=None, follow_symlinks=False) if os.path.islink(name) and not follow_symlinks: pass else: - mode = salt.utils.normalize_mode(mode) + mode = salt.utils.files.normalize_mode(mode) if mode != perms['lmode']: if __opts__['test'] is True: ret['changes']['mode'] = mode else: set_mode(name, mode) - if mode != salt.utils.normalize_mode(get_mode(name)): + if mode != salt.utils.files.normalize_mode(get_mode(name)): ret['result'] = False ret['comment'].append( 'Failed to change mode to {0}'.format(mode) @@ -4758,8 +4759,8 @@ def check_file_meta( changes['group'] = group # Normalize the file mode - smode = salt.utils.normalize_mode(lstats['mode']) - mode = salt.utils.normalize_mode(mode) + smode = salt.utils.files.normalize_mode(lstats['mode']) + mode = salt.utils.files.normalize_mode(mode) if mode is not None and mode != smode: changes['mode'] = mode @@ -5441,7 +5442,7 @@ def makedirs_(path, path = os.path.expanduser(path) if mode: - mode = salt.utils.normalize_mode(mode) + mode = salt.utils.files.normalize_mode(mode) # walk up the directory structure until we find the first existing # directory diff --git a/salt/modules/freebsdpkg.py b/salt/modules/freebsdpkg.py index ebec22ef7f..de37a4218e 100644 --- a/salt/modules/freebsdpkg.py +++ b/salt/modules/freebsdpkg.py @@ -80,7 +80,7 @@ import logging import re # Import salt libs -import salt.utils +import salt.utils # TODO: Remove this when alias_function is moved import salt.utils.data import salt.utils.pkg from salt.exceptions import CommandExecutionError, MinionError @@ -225,7 +225,7 @@ def version(*names, **kwargs): ''' with_origin = kwargs.pop('with_origin', False) ret = __salt__['pkg_resource.version'](*names, **kwargs) - if not salt.utils.is_true(with_origin): + if not salt.utils.data.is_true(with_origin): return ret # Put the return value back into a dict since we're adding a subdict if len(names) == 1: @@ -271,9 +271,9 @@ def list_pkgs(versions_as_list=False, with_origin=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -281,7 +281,7 @@ def list_pkgs(versions_as_list=False, with_origin=False, **kwargs): ret = copy.deepcopy(__context__['pkg.list_pkgs']) if not versions_as_list: __salt__['pkg_resource.stringify'](ret) - if salt.utils.is_true(with_origin): + if salt.utils.data.is_true(with_origin): origins = __context__.get('pkg.origin', {}) return dict([ (x, {'origin': origins.get(x, ''), 'version': y}) @@ -310,7 +310,7 @@ def list_pkgs(versions_as_list=False, with_origin=False, **kwargs): __context__['pkg.origin'] = origins if not versions_as_list: __salt__['pkg_resource.stringify'](ret) - if salt.utils.is_true(with_origin): + if salt.utils.data.is_true(with_origin): return dict([ (x, {'origin': origins.get(x, ''), 'version': y}) for x, y in six.iteritems(ret) diff --git a/salt/modules/grains.py b/salt/modules/grains.py index 0b21a43672..a4822477b1 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -17,7 +17,6 @@ from functools import reduce # pylint: disable=redefined-builtin # Import Salt libs from salt.ext import six -import salt.utils # Can be removed once is_true is moved import salt.utils.compat import salt.utils.data import salt.utils.files @@ -157,7 +156,7 @@ def items(sanitize=False): salt '*' grains.items sanitize=True ''' - if salt.utils.is_true(sanitize): + if salt.utils.data.is_true(sanitize): out = dict(__grains__) for key, func in six.iteritems(_SANITIZERS): if key in out: @@ -198,7 +197,7 @@ def item(*args, **kwargs): except KeyError: pass - if salt.utils.is_true(kwargs.get('sanitize')): + if salt.utils.data.is_true(kwargs.get('sanitize')): for arg, func in six.iteritems(_SANITIZERS): if arg in ret: ret[arg] = func(ret[arg]) diff --git a/salt/modules/hg.py b/salt/modules/hg.py index d4992eb34a..bb0282cb98 100644 --- a/salt/modules/hg.py +++ b/salt/modules/hg.py @@ -9,7 +9,7 @@ import logging # Import salt libs from salt.exceptions import CommandExecutionError -import salt.utils +import salt.utils.data import salt.utils.path log = logging.getLogger(__name__) @@ -323,7 +323,7 @@ def status(cwd, opts=None, user=None): ret[t].append(f) return ret - if salt.utils.is_iter(cwd): + if salt.utils.data.is_iter(cwd): return dict((cwd, _status(cwd)) for cwd in cwd) else: return _status(cwd) diff --git a/salt/modules/key.py b/salt/modules/key.py index c68871cbc6..73fe04fdd9 100644 --- a/salt/modules/key.py +++ b/salt/modules/key.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import os # Import Salt libs -import salt.utils +import salt.utils.crypt def finger(hash_type=None): @@ -27,7 +27,7 @@ def finger(hash_type=None): if hash_type is None: hash_type = __opts__['hash_type'] - return salt.utils.pem_finger( + return salt.utils.crypt.pem_finger( os.path.join(__opts__['pki_dir'], 'minion.pub'), sum_type=hash_type) @@ -48,6 +48,6 @@ def finger_master(hash_type=None): if hash_type is None: hash_type = __opts__['hash_type'] - return salt.utils.pem_finger( + return salt.utils.crypt.pem_finger( os.path.join(__opts__['pki_dir'], 'minion_master.pub'), sum_type=hash_type) diff --git a/salt/modules/lxc.py b/salt/modules/lxc.py index a72430c348..f9c69b5404 100644 --- a/salt/modules/lxc.py +++ b/salt/modules/lxc.py @@ -30,6 +30,7 @@ import salt.utils.args import salt.utils.cloud import salt.utils.dictupdate import salt.utils.files +import salt.utils.hashutils import salt.utils.network import salt.utils.odict import salt.utils.path @@ -1094,7 +1095,7 @@ def _get_base(**kwargs): proto = _urlparse(image).scheme img_tar = __salt__['cp.cache_file'](image) img_name = os.path.basename(img_tar) - hash_ = salt.utils.get_hash( + hash_ = salt.utils.hashutils.get_hash( img_tar, __salt__['config.get']('hash_type')) name = '__base_{0}_{1}_{2}'.format(proto, img_name, hash_) diff --git a/salt/modules/mac_brew.py b/salt/modules/mac_brew.py index 076f2873b6..08fe63105a 100644 --- a/salt/modules/mac_brew.py +++ b/salt/modules/mac_brew.py @@ -17,7 +17,7 @@ import json import logging # Import salt libs -import salt.utils # Can be removed when alias_function, is_true are moved +import salt.utils # TODO: Remove this when alias_function is moved import salt.utils.data import salt.utils.path import salt.utils.pkg @@ -108,9 +108,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -174,7 +174,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version salt '*' pkg.latest_version ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) if refresh: refresh_db() @@ -475,7 +475,7 @@ def upgrade(refresh=True): old = list_pkgs() - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() result = _call_brew('brew upgrade', failhard=False) diff --git a/salt/modules/mac_ports.py b/salt/modules/mac_ports.py index 07bf2c8f90..80e5952395 100644 --- a/salt/modules/mac_ports.py +++ b/salt/modules/mac_ports.py @@ -37,7 +37,7 @@ import logging import re # Import salt libs -import salt.utils # Can be removed when alias_function, is_true are removed +import salt.utils # TODO: Remove this when alias_function is removed import salt.utils.data import salt.utils.path import salt.utils.pkg @@ -96,9 +96,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # 'removed', 'purge_desired' not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -163,7 +163,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version ''' - if salt.utils.is_true(kwargs.get('refresh', True)): + if salt.utils.data.is_true(kwargs.get('refresh', True)): refresh_db() available = _list(' '.join(names)) or {} @@ -299,11 +299,9 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): salt '*' pkg.install 'package package package' ''' pkg_params, pkg_type = \ - __salt__['pkg_resource.parse_targets'](name, - pkgs, - {}) + __salt__['pkg_resource.parse_targets'](name, pkgs, {}) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() # Handle version kwarg for a single package target diff --git a/salt/modules/mac_softwareupdate.py b/salt/modules/mac_softwareupdate.py index 0f598ed0bf..3bc45200d4 100644 --- a/salt/modules/mac_softwareupdate.py +++ b/salt/modules/mac_softwareupdate.py @@ -10,7 +10,7 @@ import re import os # import salt libs -import salt.utils +import salt.utils.data import salt.utils.files import salt.utils.mac_utils import salt.utils.platform @@ -48,7 +48,7 @@ def _get_available(recommended=False, restart=False): rexp = re.compile('(?m)^ [*|-] ' r'([^ ].*)[\r\n].*\(([^\)]+)') - if salt.utils.is_true(recommended): + if salt.utils.data.is_true(recommended): # rexp parses lines that look like the following: # * Safari6.1.2MountainLion-6.1.2 # Safari (6.1.2), 51679K [recommended] @@ -66,7 +66,7 @@ def _get_available(recommended=False, restart=False): version_num = _get(line, 'version') ret[name] = version_num - if not salt.utils.is_true(restart): + if not salt.utils.data.is_true(restart): return ret # rexp parses lines that look like the following: diff --git a/salt/modules/minion.py b/salt/modules/minion.py index 94bf948b0b..9cf348a5c6 100644 --- a/salt/modules/minion.py +++ b/salt/modules/minion.py @@ -10,7 +10,7 @@ import sys import time # Import Salt libs -import salt.utils +import salt.utils.data import salt.key # Import third party libs @@ -55,7 +55,7 @@ def list_(): for dir_ in key_dirs: ret[os.path.basename(dir_)] = [] try: - for fn_ in salt.utils.isorted(os.listdir(dir_)): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(dir_)): if not fn_.startswith('.'): if os.path.isfile(os.path.join(dir_, fn_)): ret[os.path.basename(dir_)].append(fn_) diff --git a/salt/modules/mount.py b/salt/modules/mount.py index 9463283a4c..30fc955877 100644 --- a/salt/modules/mount.py +++ b/salt/modules/mount.py @@ -10,7 +10,7 @@ import re import logging # Import salt libs -import salt.utils # Can be removed once test_mode is moved +import salt.utils.args import salt.utils.files import salt.utils.path import salt.utils.platform @@ -625,7 +625,7 @@ def set_fstab( ret = 'new' if ret != 'present': # ret in ['new', 'change']: - if not salt.utils.test_mode(test=test, **kwargs): + if not salt.utils.args.test_mode(test=test, **kwargs): try: with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! @@ -753,7 +753,7 @@ def set_vfstab( ret = 'new' if ret != 'present': # ret in ['new', 'change']: - if not salt.utils.test_mode(test=test, **kwargs): + if not salt.utils.args.test_mode(test=test, **kwargs): try: with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! @@ -908,7 +908,7 @@ def set_automaster( raise CommandExecutionError(msg.format(config, str(exc))) if change: - if not salt.utils.test_mode(test=test, **kwargs): + if not salt.utils.args.test_mode(test=test, **kwargs): try: with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! @@ -924,7 +924,7 @@ def set_automaster( # The right entry is already here return 'present' else: - if not salt.utils.test_mode(test=test, **kwargs): + if not salt.utils.args.test_mode(test=test, **kwargs): # The entry is new, add it to the end of the fstab newline = ( '{0}\t{1}\t{2}\n'.format( diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index 5dcb7957dd..bc0bbe7928 100644 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -43,7 +43,7 @@ import shlex import os # Import salt libs -import salt.utils +import salt.utils.data import salt.utils.files # Import third party libs @@ -1228,8 +1228,8 @@ def user_exists(user, args['user'] = user args['host'] = host - if salt.utils.is_true(passwordless): - if salt.utils.is_true(unix_socket): + if salt.utils.data.is_true(passwordless): + if salt.utils.data.is_true(unix_socket): qry += ' AND plugin=%(unix_socket)s' args['unix_socket'] = 'unix_socket' else: @@ -1356,8 +1356,8 @@ def user_create(user, elif password_hash is not None: qry += ' IDENTIFIED BY PASSWORD %(password)s' args['password'] = password_hash - elif salt.utils.is_true(allow_passwordless): - if salt.utils.is_true(unix_socket): + elif salt.utils.data.is_true(allow_passwordless): + if salt.utils.data.is_true(unix_socket): if host == 'localhost': qry += ' IDENTIFIED VIA unix_socket' else: @@ -1441,7 +1441,7 @@ def user_chpass(user, elif password_hash is not None: password_sql = '%(password)s' args['password'] = password_hash - elif not salt.utils.is_true(allow_passwordless): + elif not salt.utils.data.is_true(allow_passwordless): log.error('password or password_hash must be specified, unless ' 'allow_passwordless=True') return False @@ -1461,8 +1461,8 @@ def user_chpass(user, ' WHERE User=%(user)s AND Host = %(host)s;') args['user'] = user args['host'] = host - if salt.utils.is_true(allow_passwordless) and \ - salt.utils.is_true(unix_socket): + if salt.utils.data.is_true(allow_passwordless) and \ + salt.utils.data.is_true(unix_socket): if host == 'localhost': qry = ('UPDATE mysql.user SET ' + password_column + '=' + password_sql + ', plugin=%(unix_socket)s' + @@ -1715,7 +1715,7 @@ def __grant_generate(grant, args['host'] = host if isinstance(ssl_option, list) and len(ssl_option): qry += __ssl_option_sanitize(ssl_option) - if salt.utils.is_true(grant_option): + if salt.utils.data.is_true(grant_option): qry += ' WITH GRANT OPTION' log.debug('Grant Query generated: {0} args {1}'.format(qry, repr(args))) return {'qry': qry, 'args': args} @@ -1903,7 +1903,7 @@ def grant_revoke(grant, grant = __grant_normalize(grant) - if salt.utils.is_true(grant_option): + if salt.utils.data.is_true(grant_option): grant += ', GRANT OPTION' db_part = database.rpartition('.') diff --git a/salt/modules/network.py b/salt/modules/network.py index 2346a72026..ec0927eccf 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -13,7 +13,7 @@ import os import socket # Import salt libs -import salt.utils # Can be removed when alias_function mac_str_to_bytes are moved +import salt.utils # TODO: Remove this when alias_function mac_str_to_bytes are moved import salt.utils.decorators.path import salt.utils.files import salt.utils.network diff --git a/salt/modules/openbsdpkg.py b/salt/modules/openbsdpkg.py index bcf1123fea..b6e8d2478b 100644 --- a/salt/modules/openbsdpkg.py +++ b/salt/modules/openbsdpkg.py @@ -29,7 +29,6 @@ import re import logging # Import Salt libs -import salt.utils # Can be removed when is_true is moved import salt.utils.data import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError @@ -66,9 +65,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} diff --git a/salt/modules/opkg.py b/salt/modules/opkg.py index 2b583b3cf9..04fa9abfa7 100644 --- a/salt/modules/opkg.py +++ b/salt/modules/opkg.py @@ -24,7 +24,6 @@ import re import logging # Import salt libs -import salt.utils # Can be removed when is_true is moved import salt.utils.args import salt.utils.data import salt.utils.files @@ -82,7 +81,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version salt '*' pkg.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) if len(names) == 0: return '' @@ -283,7 +282,7 @@ def install(name=None, {'': {'old': '', 'new': ''}} ''' - refreshdb = salt.utils.is_true(refresh) + refreshdb = salt.utils.data.is_true(refresh) try: pkg_params, pkg_type = __salt__['pkg_resource.parse_targets']( @@ -538,7 +537,7 @@ def upgrade(refresh=True): 'comment': '', } - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() old = list_pkgs() @@ -770,9 +769,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs salt '*' pkg.list_pkgs versions_as_list=True ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -815,7 +814,7 @@ def list_upgrades(refresh=True): salt '*' pkg.list_upgrades ''' ret = {} - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() cmd = ['opkg', 'list-upgradable'] diff --git a/salt/modules/pacman.py b/salt/modules/pacman.py index 757387160b..5088cc9020 100644 --- a/salt/modules/pacman.py +++ b/salt/modules/pacman.py @@ -18,7 +18,7 @@ import logging import os.path # Import salt libs -import salt.utils +import salt.utils # TODO: Remove this once alias_function, fnmatch_multiple are moved import salt.utils.args import salt.utils.data import salt.utils.pkg @@ -68,7 +68,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version salt '*' pkg.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', False)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', False)) if len(names) == 0: return '' @@ -201,9 +201,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -517,8 +517,8 @@ def install(name=None, {'': {'old': '', 'new': ''}} ''' - refresh = salt.utils.is_true(refresh) - sysupgrade = salt.utils.is_true(sysupgrade) + refresh = salt.utils.data.is_true(refresh) + sysupgrade = salt.utils.data.is_true(sysupgrade) try: pkg_params, pkg_type = __salt__['pkg_resource.parse_targets']( @@ -678,7 +678,7 @@ def upgrade(refresh=False, root=None, **kwargs): and __salt__['config.get']('systemd.scope', True): cmd.extend(['systemd-run', '--scope']) cmd.extend(['pacman', '-Su', '--noprogressbar', '--noconfirm']) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): cmd.append('-y') if root is not None: diff --git a/salt/modules/pecl.py b/salt/modules/pecl.py index 284ce74e49..e80b608a90 100644 --- a/salt/modules/pecl.py +++ b/salt/modules/pecl.py @@ -14,7 +14,7 @@ except ImportError: from pipes import quote as _cmd_quote # Import salt libs -import salt.utils +import salt.utils.data import salt.utils.path # Import 3rd-party libs @@ -42,7 +42,7 @@ def _pecl(command, defaults=False): Execute the command passed with pecl ''' cmdline = 'pecl {0}'.format(command) - if salt.utils.is_true(defaults): + if salt.utils.data.is_true(defaults): cmdline = 'yes ' "''" + ' | ' + cmdline ret = __salt__['cmd.run_all'](cmdline, python_shell=True) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index fa1b091e3b..990e0045b8 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -17,7 +17,7 @@ from salt.ext import six # Import salt libs import salt.pillar -import salt.utils # Can be removed once alias_function is moved +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.crypt import salt.utils.data import salt.utils.dictupdate diff --git a/salt/modules/pkg_resource.py b/salt/modules/pkg_resource.py index d72829830b..a59f858cb7 100644 --- a/salt/modules/pkg_resource.py +++ b/salt/modules/pkg_resource.py @@ -16,7 +16,6 @@ import yaml from salt.ext import six # Import salt libs -import salt.utils # Can be removed once is_true is moved import salt.utils.data import salt.utils.versions from salt.exceptions import SaltInvocationError @@ -192,7 +191,7 @@ def version(*names, **kwargs): ''' ret = {} versions_as_list = \ - salt.utils.is_true(kwargs.pop('versions_as_list', False)) + salt.utils.data.is_true(kwargs.pop('versions_as_list', False)) pkg_glob = False if len(names) != 0: pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True, **kwargs) diff --git a/salt/modules/pkgin.py b/salt/modules/pkgin.py index fdd8ed4b8e..f4a302b0c5 100644 --- a/salt/modules/pkgin.py +++ b/salt/modules/pkgin.py @@ -17,7 +17,7 @@ import os import re # Import salt libs -import salt.utils # Can be removed when alias_function, is_true are moved +import salt.utils # TODO: Remove this when alias_function is moved import salt.utils.data import salt.utils.path import salt.utils.pkg @@ -152,7 +152,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) pkglist = {} pkgin = _check_pkgin() @@ -256,9 +256,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} diff --git a/salt/modules/pkgng.py b/salt/modules/pkgng.py index ceac214a80..b7afdb199f 100644 --- a/salt/modules/pkgng.py +++ b/salt/modules/pkgng.py @@ -44,7 +44,7 @@ import logging import os # Import salt libs -import salt.utils # Can be removed once alias_function, is_true are moved +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.data import salt.utils.files import salt.utils.itertools @@ -205,7 +205,7 @@ def version(*names, **kwargs): ''' with_origin = kwargs.pop('with_origin', False) ret = __salt__['pkg_resource.version'](*names, **kwargs) - if not salt.utils.is_true(with_origin): + if not salt.utils.data.is_true(with_origin): return ret # Put the return value back into a dict since we're adding a subdict if len(names) == 1: @@ -313,7 +313,7 @@ def latest_version(*names, **kwargs): cmd = _pkg(jail, chroot, root) + ['search', '-S', 'name', '-Q', 'version', '-e'] if quiet: cmd.append('-q') - if not salt.utils.is_true(refresh): + if not salt.utils.data.is_true(refresh): cmd.append('-U') cmd.append(name) @@ -385,11 +385,11 @@ def list_pkgs(versions_as_list=False, salt '*' pkg.list_pkgs chroot=/path/to/chroot ''' # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) contextkey_pkg = _contextkey(jail, chroot, root) contextkey_origins = _contextkey(jail, chroot, root, prefix='pkg.origin') @@ -397,7 +397,7 @@ def list_pkgs(versions_as_list=False, ret = copy.deepcopy(__context__[contextkey_pkg]) if not versions_as_list: __salt__['pkg_resource.stringify'](ret) - if salt.utils.is_true(with_origin): + if salt.utils.data.is_true(with_origin): origins = __context__.get(contextkey_origins, {}) return dict([ (x, {'origin': origins.get(x, ''), 'version': y}) @@ -427,7 +427,7 @@ def list_pkgs(versions_as_list=False, __context__[contextkey_origins] = origins if not versions_as_list: __salt__['pkg_resource.stringify'](ret) - if salt.utils.is_true(with_origin): + if salt.utils.data.is_true(with_origin): return dict([ (x, {'origin': origins.get(x, ''), 'version': y}) for x, y in six.iteritems(ret) @@ -811,23 +811,23 @@ def install(name=None, return {} opts = 'y' - if salt.utils.is_true(orphan): + if salt.utils.data.is_true(orphan): opts += 'A' - if salt.utils.is_true(force): + if salt.utils.data.is_true(force): opts += 'f' - if salt.utils.is_true(glob): + if salt.utils.data.is_true(glob): opts += 'g' - if salt.utils.is_true(local): + if salt.utils.data.is_true(local): opts += 'U' - if salt.utils.is_true(dryrun): + if salt.utils.data.is_true(dryrun): opts += 'n' - if salt.utils.is_true(quiet): + if salt.utils.data.is_true(quiet): opts += 'q' - if salt.utils.is_true(reinstall_requires): + if salt.utils.data.is_true(reinstall_requires): opts += 'R' - if salt.utils.is_true(regex): + if salt.utils.data.is_true(regex): opts += 'x' - if salt.utils.is_true(pcre): + if salt.utils.data.is_true(pcre): opts += 'X' old = list_pkgs(jail=jail, chroot=chroot, root=root) @@ -858,7 +858,7 @@ def install(name=None, cmd.append('-' + opts) cmd.extend(targets) - if pkg_cmd == 'add' and salt.utils.is_true(dryrun): + if pkg_cmd == 'add' and salt.utils.data.is_true(dryrun): # pkg add doesn't have a dryrun mode, so echo out what will be run return ' '.join(cmd) @@ -1014,21 +1014,21 @@ def remove(name=None, return {} opts = '' - if salt.utils.is_true(all_installed): + if salt.utils.data.is_true(all_installed): opts += 'a' - if salt.utils.is_true(force): + if salt.utils.data.is_true(force): opts += 'f' - if salt.utils.is_true(glob): + if salt.utils.data.is_true(glob): opts += 'g' - if salt.utils.is_true(dryrun): + if salt.utils.data.is_true(dryrun): opts += 'n' - if not salt.utils.is_true(dryrun): + if not salt.utils.data.is_true(dryrun): opts += 'y' - if salt.utils.is_true(recurse): + if salt.utils.data.is_true(recurse): opts += 'R' - if salt.utils.is_true(regex): + if salt.utils.data.is_true(regex): opts += 'x' - if salt.utils.is_true(pcre): + if salt.utils.data.is_true(pcre): opts += 'X' cmd = _pkg(jail, chroot, root) diff --git a/salt/modules/pkgutil.py b/salt/modules/pkgutil.py index 8c8bc95750..ce0f7de66f 100644 --- a/salt/modules/pkgutil.py +++ b/salt/modules/pkgutil.py @@ -14,7 +14,7 @@ from __future__ import absolute_import import copy # Import salt libs -import salt.utils # Can be removed once alias_function, is_true are moved +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.data import salt.utils.pkg import salt.utils.versions @@ -84,7 +84,7 @@ def list_upgrades(refresh=True, **kwargs): # pylint: disable=W0613 salt '*' pkgutil.list_upgrades ''' - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() upgrades = {} lines = __salt__['cmd.run_stdout']( @@ -114,7 +114,7 @@ def upgrade(refresh=True): salt '*' pkgutil.upgrade ''' - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() old = list_pkgs() @@ -141,9 +141,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs salt '*' pkg.list_pkgs versions_as_list=True ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # 'removed' not yet implemented or not applicable - if salt.utils.is_true(kwargs.get('removed')): + if salt.utils.data.is_true(kwargs.get('removed')): return {} if 'pkg.list_pkgs' in __context__: @@ -204,7 +204,7 @@ def latest_version(*names, **kwargs): salt '*' pkgutil.latest_version CSWpython salt '*' pkgutil.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) if not names: return '' diff --git a/salt/modules/pw_group.py b/salt/modules/pw_group.py index 143b3a1495..7f3e8beee1 100644 --- a/salt/modules/pw_group.py +++ b/salt/modules/pw_group.py @@ -14,8 +14,8 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils import salt.utils.args +import salt.utils.data log = logging.getLogger(__name__) @@ -51,7 +51,7 @@ def add(name, gid=None, **kwargs): salt '*' group.add foo 3456 ''' kwargs = salt.utils.args.clean_kwargs(**kwargs) - if salt.utils.is_true(kwargs.pop('system', False)): + if salt.utils.data.is_true(kwargs.pop('system', False)): log.warning('pw_group module does not support the \'system\' argument') if kwargs: log.warning('Invalid kwargs passed to group.add') diff --git a/salt/modules/pw_user.py b/salt/modules/pw_user.py index 3d1ca97856..f3362b65e8 100644 --- a/salt/modules/pw_user.py +++ b/salt/modules/pw_user.py @@ -47,8 +47,8 @@ except ImportError: from salt.ext import six # Import salt libs -import salt.utils # Can be removed once is_true is moved import salt.utils.args +import salt.utils.data import salt.utils.locales import salt.utils.user from salt.exceptions import CommandExecutionError @@ -144,7 +144,7 @@ def add(name, salt '*' user.add name ''' kwargs = salt.utils.args.clean_kwargs(**kwargs) - if salt.utils.is_true(kwargs.pop('system', False)): + if salt.utils.data.is_true(kwargs.pop('system', False)): log.warning('pw_user module does not support the \'system\' argument') if kwargs: log.warning('Invalid kwargs passed to user.add') @@ -166,7 +166,7 @@ def add(name, cmd.extend(['-L', loginclass]) if shell: cmd.extend(['-s', shell]) - if not salt.utils.is_true(unique): + if not salt.utils.data.is_true(unique): cmd.append('-o') gecos_field = _build_gecos({'fullname': fullname, 'roomnumber': roomnumber, @@ -187,7 +187,7 @@ def delete(name, remove=False, force=False): salt '*' user.delete name remove=True force=True ''' - if salt.utils.is_true(force): + if salt.utils.data.is_true(force): log.error('pw userdel does not support force-deleting user while ' 'user is logged in') cmd = ['pw', 'userdel'] diff --git a/salt/modules/rpmbuild.py b/salt/modules/rpmbuild.py index a6687db524..6482553875 100644 --- a/salt/modules/rpmbuild.py +++ b/salt/modules/rpmbuild.py @@ -24,7 +24,7 @@ import functools # Import salt libs from salt.exceptions import SaltInvocationError -import salt.utils # Can be removed when chugid_and_umask is moved +import salt.utils # TODO: Remove this when chugid_and_umask is moved import salt.utils.files import salt.utils.path import salt.utils.vt diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 869fb1ac39..337c5de7c0 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -46,7 +46,7 @@ import salt.payload import salt.runner import salt.state import salt.transport -import salt.utils +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.args import salt.utils.event import salt.utils.extmods @@ -988,7 +988,7 @@ def clear_cache(): salt '*' saltutil.clear_cache ''' - for root, dirs, files in salt.utils.safe_walk(__opts__['cachedir'], followlinks=False): + for root, dirs, files in salt.utils.files.safe_walk(__opts__['cachedir'], followlinks=False): for name in files: try: os.remove(os.path.join(root, name)) @@ -1014,7 +1014,7 @@ def clear_job_cache(hours=24): salt '*' saltutil.clear_job_cache hours=12 ''' threshold = time.time() - hours * 3600 - for root, dirs, files in salt.utils.safe_walk(os.path.join(__opts__['cachedir'], 'minion_jobs'), + for root, dirs, files in salt.utils.files.safe_walk(os.path.join(__opts__['cachedir'], 'minion_jobs'), followlinks=False): for name in dirs: try: diff --git a/salt/modules/shadow.py b/salt/modules/shadow.py index 785af50dd3..e93435b89d 100644 --- a/salt/modules/shadow.py +++ b/salt/modules/shadow.py @@ -19,7 +19,7 @@ except ImportError: pass # Import salt libs -import salt.utils # Can be removed when is_true is moved +import salt.utils.data import salt.utils.files import salt.utils.stringutils from salt.exceptions import CommandExecutionError @@ -278,7 +278,7 @@ def set_password(name, password, use_usermod=False): salt '*' shadow.set_password root '$1$UYCIxa628.9qXjpQCjM4a..' ''' - if not salt.utils.is_true(use_usermod): + if not salt.utils.data.is_true(use_usermod): # Edit the shadow file directly # ALT Linux uses tcb to store password hashes. More information found # in manpage (http://docs.altlinux.org/manpages/tcb.5.html) diff --git a/salt/modules/solaris_group.py b/salt/modules/solaris_group.py index 51392efbbb..623f429b77 100644 --- a/salt/modules/solaris_group.py +++ b/salt/modules/solaris_group.py @@ -14,7 +14,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.data log = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def add(name, gid=None, **kwargs): salt '*' group.add foo 3456 ''' - if salt.utils.is_true(kwargs.pop('system', False)): + if salt.utils.data.is_true(kwargs.pop('system', False)): log.warning('solaris_group module does not support the \'system\' ' 'argument') if kwargs: diff --git a/salt/modules/solaris_user.py b/salt/modules/solaris_user.py index 276dcbe916..9db8b0b474 100644 --- a/salt/modules/solaris_user.py +++ b/salt/modules/solaris_user.py @@ -22,7 +22,7 @@ import copy import logging # Import salt libs -import salt.utils # Can be removed once is_true is moved +import salt.utils.data import salt.utils.user from salt.ext import six from salt.exceptions import CommandExecutionError @@ -112,7 +112,7 @@ def add(name, salt '*' user.add name ''' - if salt.utils.is_true(kwargs.pop('system', False)): + if salt.utils.data.is_true(kwargs.pop('system', False)): log.warning('solaris_user module does not support the \'system\' ' 'argument') if kwargs: @@ -169,7 +169,7 @@ def delete(name, remove=False, force=False): salt '*' user.delete name remove=True force=True ''' - if salt.utils.is_true(force): + if salt.utils.data.is_true(force): log.warning( 'userdel does not support force-deleting user while user is ' 'logged in' diff --git a/salt/modules/solarisips.py b/salt/modules/solarisips.py index 106f4d5f83..be992325d0 100644 --- a/salt/modules/solarisips.py +++ b/salt/modules/solarisips.py @@ -43,7 +43,7 @@ import logging # Import salt libs -import salt.utils # Can be removed once alias_function, is_true are moved +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.data import salt.utils.path import salt.utils.pkg @@ -182,7 +182,7 @@ def list_upgrades(refresh=True, **kwargs): # pylint: disable=W0613 salt '*' pkg.list_upgrades salt '*' pkg.list_upgrades refresh=False ''' - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db(full=True) upgrades = {} # awk is in core-os package so we can use it without checking @@ -216,7 +216,7 @@ def upgrade(refresh=False, **kwargs): salt '*' pkg.upgrade ''' - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db() # Get a list of the packages before install so we can diff after to see @@ -257,7 +257,7 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} diff --git a/salt/modules/solarispkg.py b/salt/modules/solarispkg.py index e3928e8d3c..bab2c4e30a 100644 --- a/salt/modules/solarispkg.py +++ b/salt/modules/solarispkg.py @@ -16,7 +16,7 @@ import os import logging # Import salt libs -import salt.utils # Can be removed once alias_function, is_true are moved +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.data import salt.utils.files from salt.exceptions import CommandExecutionError, MinionError @@ -90,9 +90,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -317,7 +317,7 @@ def install(name=None, sources=None, saltenv='base', **kwargs): The ID declaration is ignored, as the package name is read from the ``sources`` parameter. ''' - if salt.utils.is_true(kwargs.get('refresh')): + if salt.utils.data.is_true(kwargs.get('refresh')): log.warning('\'refresh\' argument not implemented for solarispkg ' 'module') diff --git a/salt/modules/state.py b/salt/modules/state.py index e5616af7d7..2f20cdaa45 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -28,10 +28,12 @@ import time import salt.config import salt.payload import salt.state -import salt.utils +import salt.utils # TODO: Remove this once namespaced_function is moved +import salt.utils.args import salt.utils.data import salt.utils.event import salt.utils.files +import salt.utils.hashutils import salt.utils.jid import salt.utils.platform import salt.utils.url @@ -327,7 +329,7 @@ def _get_test_value(test=None, **kwargs): ''' ret = True if test is None: - if salt.utils.test_mode(test=test, **kwargs): + if salt.utils.args.test_mode(test=test, **kwargs): ret = True else: ret = __opts__.get('test', None) @@ -1775,7 +1777,7 @@ def pkg(pkg_path, # TODO - Add ability to download from salt master or other source if not os.path.isfile(pkg_path): return {} - if not salt.utils.get_hash(pkg_path, hash_type) == pkg_sum: + if not salt.utils.hashutils.get_hash(pkg_path, hash_type) == pkg_sum: return {} root = tempfile.mkdtemp() s_pkg = tarfile.open(pkg_path, 'r:gz') diff --git a/salt/modules/timezone.py b/salt/modules/timezone.py index d3f4b645b4..1f10dbde43 100644 --- a/salt/modules/timezone.py +++ b/salt/modules/timezone.py @@ -13,8 +13,8 @@ import re import string # Import salt libs -import salt.utils import salt.utils.files +import salt.utils.hashutils import salt.utils.itertools import salt.utils.path import salt.utils.platform @@ -118,7 +118,7 @@ def _get_zone_etc_localtime(): ) # Regular file. Try to match the hash. hash_type = __opts__.get('hash_type', 'md5') - tzfile_hash = salt.utils.get_hash(tzfile, hash_type) + tzfile_hash = salt.utils.hashutils.get_hash(tzfile, hash_type) # Not a link, just a copy of the tzdata file for root, dirs, files in os.walk(tzdir): for filename in files: @@ -127,7 +127,7 @@ def _get_zone_etc_localtime(): if olson_name[0] in string.ascii_lowercase: continue if tzfile_hash == \ - salt.utils.get_hash(full_path, hash_type): + salt.utils.hashutils.get_hash(full_path, hash_type): return olson_name raise CommandExecutionError('Unable to determine timezone') diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index ff23435800..3cae80a152 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -50,10 +50,10 @@ from salt.ext.six.moves.urllib.parse import urlparse as _urlparse from salt.exceptions import (CommandExecutionError, SaltInvocationError, SaltRenderError) -import salt.utils # Can be removed once is_true, get_hash are moved import salt.utils.args import salt.utils.data import salt.utils.files +import salt.utils.hashutils import salt.utils.pkg import salt.utils.platform import salt.utils.versions @@ -113,7 +113,7 @@ def latest_version(*names, **kwargs): saltenv = kwargs.get('saltenv', 'base') # Refresh before looking for the latest version available - refresh = salt.utils.is_true(kwargs.get('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.get('refresh', True)) # no need to call _refresh_db_conditional as list_pkgs will do it installed_pkgs = list_pkgs( @@ -190,7 +190,7 @@ def upgrade_available(name, **kwargs): saltenv = kwargs.get('saltenv', 'base') # Refresh before looking for the latest version available, # same default as latest_version - refresh = salt.utils.is_true(kwargs.get('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.get('refresh', True)) current = version(name, saltenv=saltenv, refresh=refresh).get(name) latest = latest_version(name, saltenv=saltenv, refresh=False) @@ -218,7 +218,7 @@ def list_upgrades(refresh=True, **kwargs): salt '*' pkg.list_upgrades ''' saltenv = kwargs.get('saltenv', 'base') - refresh = salt.utils.is_true(refresh) + refresh = salt.utils.data.is_true(refresh) _refresh_db_conditional(saltenv, force=refresh) installed_pkgs = list_pkgs(refresh=False, saltenv=saltenv) @@ -268,9 +268,9 @@ def list_available(*names, **kwargs): return '' saltenv = kwargs.get('saltenv', 'base') - refresh = salt.utils.is_true(kwargs.get('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.get('refresh', True)) return_dict_always = \ - salt.utils.is_true(kwargs.get('return_dict_always', False)) + salt.utils.data.is_true(kwargs.get('return_dict_always', False)) _refresh_db_conditional(saltenv, force=refresh) if len(names) == 1 and not return_dict_always: @@ -360,13 +360,13 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs salt '*' pkg.list_pkgs versions_as_list=True ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} saltenv = kwargs.get('saltenv', 'base') - refresh = salt.utils.is_true(kwargs.get('refresh', False)) + refresh = salt.utils.data.is_true(kwargs.get('refresh', False)) _refresh_db_conditional(saltenv, force=refresh) ret = {} @@ -488,8 +488,8 @@ def _refresh_db_conditional(saltenv, **kwargs): :codeauthor: Damon Atkins ''' - force = salt.utils.is_true(kwargs.pop('force', False)) - failhard = salt.utils.is_true(kwargs.pop('failhard', False)) + force = salt.utils.data.is_true(kwargs.pop('force', False)) + failhard = salt.utils.data.is_true(kwargs.pop('failhard', False)) expired_max = __opts__['winrepo_cache_expire_max'] expired_min = __opts__['winrepo_cache_expire_min'] @@ -570,8 +570,8 @@ def refresh_db(**kwargs): # Remove rtag file to keep multiple refreshes from happening in pkg states salt.utils.pkg.clear_rtag(__opts__) saltenv = kwargs.pop('saltenv', 'base') - verbose = salt.utils.is_true(kwargs.pop('verbose', False)) - failhard = salt.utils.is_true(kwargs.pop('failhard', True)) + verbose = salt.utils.data.is_true(kwargs.pop('verbose', False)) + failhard = salt.utils.data.is_true(kwargs.pop('failhard', True)) __context__.pop('winrepo.data', None) repo_details = _get_repo_details(saltenv) @@ -742,8 +742,8 @@ def genrepo(**kwargs): salt -G 'os:windows' pkg.genrepo saltenv=base ''' saltenv = kwargs.pop('saltenv', 'base') - verbose = salt.utils.is_true(kwargs.pop('verbose', False)) - failhard = salt.utils.is_true(kwargs.pop('failhard', True)) + verbose = salt.utils.data.is_true(kwargs.pop('verbose', False)) + failhard = salt.utils.data.is_true(kwargs.pop('failhard', True)) ret = {} successful_verbose = {} @@ -1079,7 +1079,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ret = {} saltenv = kwargs.pop('saltenv', 'base') - refresh = salt.utils.is_true(refresh) + refresh = salt.utils.data.is_true(refresh) # no need to call _refresh_db_conditional as list_pkgs will do it # Make sure name or pkgs is passed @@ -1251,8 +1251,8 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): log.debug('Source {0} hash: {1}'.format(source_sum['hash_type'], source_sum['hsum'])) - cached_pkg_sum = salt.utils.get_hash(cached_pkg, - source_sum['hash_type']) + cached_pkg_sum = salt.utils.hashutils.get_hash(cached_pkg, + source_sum['hash_type']) log.debug('Package {0} hash: {1}'.format(source_sum['hash_type'], cached_pkg_sum)) @@ -1419,11 +1419,11 @@ def upgrade(**kwargs): salt '*' pkg.upgrade ''' log.warning('pkg.upgrade not implemented on Windows yet') - refresh = salt.utils.is_true(kwargs.get('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.get('refresh', True)) saltenv = kwargs.get('saltenv', 'base') # Uncomment the below once pkg.upgrade has been implemented - # if salt.utils.is_true(refresh): + # if salt.utils.data.is_true(refresh): # refresh_db() return {} @@ -1472,7 +1472,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): salt '*' pkg.remove pkgs='["foo", "bar"]' ''' saltenv = kwargs.get('saltenv', 'base') - refresh = salt.utils.is_true(kwargs.get('refresh', False)) + refresh = salt.utils.data.is_true(kwargs.get('refresh', False)) # no need to call _refresh_db_conditional as list_pkgs will do it ret = {} diff --git a/salt/modules/xbpspkg.py b/salt/modules/xbpspkg.py index 33e80be532..fd079dd79d 100644 --- a/salt/modules/xbpspkg.py +++ b/salt/modules/xbpspkg.py @@ -16,7 +16,6 @@ import logging import glob # Import salt libs -import salt.utils # Can be removed when is_true is moved import salt.utils.data import salt.utils.files import salt.utils.path @@ -91,9 +90,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} cmd = 'xbps-query -l' @@ -135,7 +134,7 @@ def list_upgrades(refresh=True): # fuse-2.9.4_4 update i686 http://repo.voidlinux.eu/current 298133 91688 # xtools-0.34_1 update noarch http://repo.voidlinux.eu/current 21424 10752 - refresh = salt.utils.is_true(refresh) + refresh = salt.utils.data.is_true(refresh) # Refresh repo index before checking for latest version available if refresh: @@ -196,7 +195,7 @@ def latest_version(*names, **kwargs): # xtools-0.34_1 update noarch http://repo.voidlinux.eu/current 21424 10752 # Package 'vim' is up to date. - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) if len(names) == 0: return '' diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 4d30a5a785..16cd9860ee 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -41,7 +41,7 @@ from salt.ext.six.moves import configparser # pylint: enable=import-error,redefined-builtin # Import Salt libs -import salt.utils # Can be removed once alias_function, is_true, and fnmatch_multiple are moved +import salt.utils # TODO: Remove this once alias_function, fnmatch_multiple are moved import salt.utils.args import salt.utils.data import salt.utils.decorators.path @@ -443,7 +443,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version disableexcludes=main salt '*' pkg.latest_version ... ''' - refresh = salt.utils.is_true(kwargs.pop('refresh', True)) + refresh = salt.utils.data.is_true(kwargs.pop('refresh', True)) if len(names) == 0: return '' @@ -631,9 +631,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs salt '*' pkg.list_pkgs attr='["version", "arch"]' ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} @@ -949,7 +949,7 @@ def list_upgrades(refresh=True, **kwargs): repo_arg = _get_repo_options(**kwargs) exclude_arg = _get_excludes_option(**kwargs) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db(check_update=False, **kwargs) cmd = [_yum(), '--quiet'] @@ -1323,9 +1323,9 @@ def install(name=None, exclude_arg = _get_excludes_option(**kwargs) branch_arg = _get_branch_option(**kwargs) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db(**kwargs) - reinstall = salt.utils.is_true(reinstall) + reinstall = salt.utils.data.is_true(reinstall) try: pkg_params, pkg_type = __salt__['pkg_resource.parse_targets']( @@ -1833,7 +1833,7 @@ def upgrade(name=None, branch_arg = _get_branch_option(**kwargs) extra_args = _get_extra_options(**kwargs) - if salt.utils.is_true(refresh): + if salt.utils.data.is_true(refresh): refresh_db(**kwargs) old = list_pkgs() diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 0e0a72f7d4..12251fb33a 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -32,7 +32,7 @@ from xml.dom import minidom as dom from xml.parsers.expat import ExpatError # Import salt libs -import salt.utils # Can be removed once alias_function, is_true are moved +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.data import salt.utils.event import salt.utils.files @@ -687,9 +687,9 @@ def list_pkgs(versions_as_list=False, **kwargs): salt '*' pkg.list_pkgs salt '*' pkg.list_pkgs attr='["version", "arch"]' ''' - versions_as_list = salt.utils.is_true(versions_as_list) + versions_as_list = salt.utils.data.is_true(versions_as_list) # not yet implemented or not applicable - if any([salt.utils.is_true(kwargs.get(x)) + if any([salt.utils.data.is_true(kwargs.get(x)) for x in ('removed', 'purge_desired')]): return {} diff --git a/salt/pillar/s3.py b/salt/pillar/s3.py index e65dfd8b36..a0183f5723 100644 --- a/salt/pillar/s3.py +++ b/salt/pillar/s3.py @@ -105,8 +105,8 @@ from salt.ext.six.moves.urllib.parse import quote as _quote # Import salt libs from salt.pillar import Pillar -import salt.utils import salt.utils.files +import salt.utils.hashutils # Set up logging log = logging.getLogger(__name__) @@ -403,7 +403,7 @@ def _get_file_from_s3(creds, metadata, saltenv, bucket, path, file_md5 = "".join(list(filter(str.isalnum, file_meta['ETag']))) \ if file_meta else None - cached_md5 = salt.utils.get_hash(cached_file_path, 'md5') + cached_md5 = salt.utils.hashutils.get_hash(cached_file_path, 'md5') log.debug("Cached file: path={0}, md5={1}, etag={2}".format(cached_file_path, cached_md5, file_md5)) diff --git a/salt/runners/cache.py b/salt/runners/cache.py index a821d2d7a6..459ff325ea 100644 --- a/salt/runners/cache.py +++ b/salt/runners/cache.py @@ -12,8 +12,8 @@ import os import salt.config from salt.ext import six import salt.log -import salt.utils import salt.utils.args +import salt.utils.gitfs import salt.utils.master import salt.utils.versions import salt.payload @@ -323,7 +323,7 @@ def clear_git_lock(role, remote=None, **kwargs): salt-run cache.clear_git_lock git_pillar ''' kwargs = salt.utils.args.clean_kwargs(**kwargs) - type_ = salt.utils.split_input(kwargs.pop('type', ['update', 'checkout'])) + type_ = salt.utils.args.split_input(kwargs.pop('type', ['update', 'checkout'])) if kwargs: salt.utils.args.invalid_kwargs(kwargs) diff --git a/salt/runners/jobs.py b/salt/runners/jobs.py index 605839eee4..0ce3565bf3 100644 --- a/salt/runners/jobs.py +++ b/salt/runners/jobs.py @@ -12,7 +12,7 @@ import os # Import salt libs import salt.client import salt.payload -import salt.utils +import salt.utils.args import salt.utils.files import salt.utils.jid import salt.minion @@ -326,14 +326,14 @@ def list_jobs(ext_source=None, if isinstance(targets, six.string_types): targets = [targets] for target in targets: - for key in salt.utils.split_input(search_target): + for key in salt.utils.args.split_input(search_target): if fnmatch.fnmatch(target, key): _match = True if search_function and _match: _match = False if 'Function' in ret[item]: - for key in salt.utils.split_input(search_function): + for key in salt.utils.args.split_input(search_function): if fnmatch.fnmatch(ret[item]['Function'], key): _match = True diff --git a/salt/states/aptpkg.py b/salt/states/aptpkg.py index f72be3f57c..5264649e26 100644 --- a/salt/states/aptpkg.py +++ b/salt/states/aptpkg.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.data log = logging.getLogger(__name__) @@ -40,7 +40,7 @@ def held(name): ) if not state: ret.update(comment='Package {0} does not have a state'.format(name)) - elif not salt.utils.is_true(state.get('hold', False)): + elif not salt.utils.data.is_true(state.get('hold', False)): if not __opts__['test']: result = __salt__['pkg.set_selections']( selection={'hold': [name]} diff --git a/salt/states/archive.py b/salt/states/archive.py index 8bea1e17b2..65ee7550db 100644 --- a/salt/states/archive.py +++ b/salt/states/archive.py @@ -22,9 +22,9 @@ from salt.ext.six.moves import shlex_quote as _cmd_quote from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module # Import Salt libs -import salt.utils import salt.utils.args import salt.utils.files +import salt.utils.hashutils import salt.utils.path import salt.utils.platform import salt.utils.url @@ -60,7 +60,7 @@ def _add_explanation(ret, source_hash_trigger, contents_missing): def _gen_checksum(path): - return {'hsum': salt.utils.get_hash(path, form=__opts__['hash_type']), + return {'hsum': salt.utils.hashutils.get_hash(path, form=__opts__['hash_type']), 'hash_type': __opts__['hash_type']} diff --git a/salt/states/file.py b/salt/states/file.py index 7724acf5c4..06011f3fb9 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -285,6 +285,7 @@ import salt.payload import salt.utils import salt.utils.dictupdate import salt.utils.files +import salt.utils.hashutils import salt.utils.platform import salt.utils.templates import salt.utils.url @@ -902,8 +903,8 @@ def _check_dir_meta(name, and group != stats.get('gid')): changes['group'] = group # Normalize the dir mode - smode = salt.utils.normalize_mode(stats['mode']) - mode = salt.utils.normalize_mode(mode) + smode = salt.utils.files.normalize_mode(stats['mode']) + mode = salt.utils.files.normalize_mode(mode) if mode is not None and mode != smode: changes['mode'] = mode return changes @@ -1256,7 +1257,7 @@ def symlink( name = os.path.expanduser(name) # Make sure that leading zeros stripped by YAML loader are added back - mode = salt.utils.normalize_mode(mode) + mode = salt.utils.files.normalize_mode(mode) user = _test_owner(kwargs, user=user) ret = {'name': name, @@ -2121,7 +2122,7 @@ def managed(name, keep_mode = False # Make sure that any leading zeros stripped by YAML loader are added back - mode = salt.utils.normalize_mode(mode) + mode = salt.utils.files.normalize_mode(mode) contents_count = len( [x for x in (contents, contents_pillar, contents_grains) @@ -2814,8 +2815,8 @@ def directory(name, file_mode = dir_mode # Make sure that leading zeros stripped by YAML loader are added back - dir_mode = salt.utils.normalize_mode(dir_mode) - file_mode = salt.utils.normalize_mode(file_mode) + dir_mode = salt.utils.files.normalize_mode(dir_mode) + file_mode = salt.utils.files.normalize_mode(file_mode) if salt.utils.platform.is_windows(): # Verify win_owner is valid on the target system @@ -3268,7 +3269,7 @@ def recurse(name, return _error(ret, 'mode management is not supported on Windows') # Make sure that leading zeros stripped by YAML loader are added back - dir_mode = salt.utils.normalize_mode(dir_mode) + dir_mode = salt.utils.files.normalize_mode(dir_mode) try: keep_mode = file_mode.lower() == 'keep' @@ -3278,7 +3279,7 @@ def recurse(name, except AttributeError: keep_mode = False - file_mode = salt.utils.normalize_mode(file_mode) + file_mode = salt.utils.files.normalize_mode(file_mode) u_check = _check_user(user, group) if u_check: @@ -4385,7 +4386,7 @@ def comment(name, regex, char='#', backup='.bak'): ret['result'] = __salt__['file.search'](name, unanchor_regex, multiline=True) if slines != nlines: - if not __utils__['files.is_text_file'](name): + if not __utils__['files.is_text'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4497,7 +4498,7 @@ def uncomment(name, regex, char='#', backup='.bak'): ) if slines != nlines: - if not __utils__['files.is_text_file'](name): + if not __utils__['files.is_text'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4740,7 +4741,7 @@ def append(name, nlines = list(slines) nlines.extend(append_lines) if slines != nlines: - if not __utils__['files.is_text_file'](name): + if not __utils__['files.is_text'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4765,7 +4766,7 @@ def append(name, nlines = nlines.splitlines() if slines != nlines: - if not __utils__['files.is_text_file'](name): + if not __utils__['files.is_text'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4933,7 +4934,7 @@ def prepend(name, if __opts__['test']: nlines = test_lines + slines if slines != nlines: - if not __utils__['files.is_text_file'](name): + if not __utils__['files.is_text'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4976,7 +4977,7 @@ def prepend(name, nlines = nlines.splitlines(True) if slines != nlines: - if not __utils__['files.is_text_file'](name): + if not __utils__['files.is_text'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -5314,8 +5315,8 @@ def copy( if os.path.lexists(source) and os.path.lexists(name): # if this is a file which did not change, do not update if force and os.path.isfile(name): - hash1 = salt.utils.get_hash(name) - hash2 = salt.utils.get_hash(source) + hash1 = salt.utils.hashutils.get_hash(name) + hash2 = salt.utils.hashutils.get_hash(source) if hash1 == hash2: changed = True ret['comment'] = ' '.join([ret['comment'], '- files are identical but force flag is set']) @@ -5789,7 +5790,7 @@ def serialize(name, contents += '\n' # Make sure that any leading zeros stripped by YAML loader are added back - mode = salt.utils.normalize_mode(mode) + mode = salt.utils.files.normalize_mode(mode) if __opts__['test']: ret['changes'] = __salt__['file.check_managed_changes']( @@ -6653,7 +6654,7 @@ def cached(name, # it cause any trouble, and just return True. return True try: - return salt.utils.get_hash(path, form=form) != checksum + return salt.utils.hashutils.get_hash(path, form=form) != checksum except (IOError, OSError, ValueError): # Again, shouldn't happen, but don't let invalid input/permissions # in the call to get_hash blow this up. diff --git a/salt/states/mysql_user.py b/salt/states/mysql_user.py index 0a14b83af9..adcf81764f 100644 --- a/salt/states/mysql_user.py +++ b/salt/states/mysql_user.py @@ -46,7 +46,7 @@ from __future__ import absolute_import import sys # Import salt libs -import salt.utils +import salt.utils.data def __virtual__(): @@ -121,7 +121,7 @@ def present(name, # check if user exists with the same password (or passwordless login) if passwordless: - if not salt.utils.is_true(allow_passwordless): + if not salt.utils.data.is_true(allow_passwordless): ret['comment'] = 'Either password or password_hash must be ' \ 'specified, unless allow_passwordless is True' ret['result'] = False @@ -161,7 +161,7 @@ def present(name, ret['result'] = None if passwordless: ret['comment'] += 'cleared' - if not salt.utils.is_true(allow_passwordless): + if not salt.utils.data.is_true(allow_passwordless): ret['comment'] += ', but allow_passwordless != True' ret['result'] = False else: @@ -185,7 +185,7 @@ def present(name, err = _get_mysql_error() if err is not None: ret['comment'] += ' ({0})'.format(err) - if passwordless and not salt.utils.is_true(allow_passwordless): + if passwordless and not salt.utils.data.is_true(allow_passwordless): ret['comment'] += '. Note: allow_passwordless must be True ' \ 'to permit passwordless login.' ret['result'] = False @@ -204,7 +204,7 @@ def present(name, ret['result'] = None if passwordless: ret['comment'] += ' with passwordless login' - if not salt.utils.is_true(allow_passwordless): + if not salt.utils.data.is_true(allow_passwordless): ret['comment'] += ', but allow_passwordless != True' ret['result'] = False return ret diff --git a/salt/states/pkgrepo.py b/salt/states/pkgrepo.py index dd19a30bb3..e7efca0dc1 100644 --- a/salt/states/pkgrepo.py +++ b/salt/states/pkgrepo.py @@ -93,7 +93,7 @@ import sys from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.modules.aptpkg import _strip_uri from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS -import salt.utils +import salt.utils.data import salt.utils.files import salt.utils.pkg.deb import salt.utils.pkg.rpm @@ -361,9 +361,9 @@ def managed(name, ppa=None, **kwargs): except TypeError: repo = ':'.join(('ppa', str(ppa))) - kwargs['disabled'] = not salt.utils.is_true(enabled) \ + kwargs['disabled'] = not salt.utils.data.is_true(enabled) \ if enabled is not None \ - else salt.utils.is_true(disabled) + else salt.utils.data.is_true(disabled) elif os_family in ('redhat', 'suse'): if 'humanname' in kwargs: @@ -372,15 +372,15 @@ def managed(name, ppa=None, **kwargs): # Fall back to the repo name if humanname not provided kwargs['name'] = repo - kwargs['enabled'] = not salt.utils.is_true(disabled) \ + kwargs['enabled'] = not salt.utils.data.is_true(disabled) \ if disabled is not None \ - else salt.utils.is_true(enabled) + else salt.utils.data.is_true(enabled) elif os_family == 'nilinuxrt': # opkg is the pkg virtual - kwargs['enabled'] = not salt.utils.is_true(disabled) \ + kwargs['enabled'] = not salt.utils.data.is_true(disabled) \ if disabled is not None \ - else salt.utils.is_true(enabled) + else salt.utils.data.is_true(enabled) for kwarg in _STATE_INTERNAL_KEYWORDS: kwargs.pop(kwarg, None) @@ -417,7 +417,7 @@ def managed(name, ppa=None, **kwargs): # if it's desired to be enabled and the 'enabled' key is # missing from the repo definition if os_family == 'redhat': - if not salt.utils.is_true(sanitizedkwargs[kwarg]): + if not salt.utils.data.is_true(sanitizedkwargs[kwarg]): break else: break @@ -452,8 +452,8 @@ def managed(name, ppa=None, **kwargs): and any(isinstance(x, bool) for x in (sanitizedkwargs[kwarg], pre[kwarg])): # This check disambiguates 1/0 from True/False - if salt.utils.is_true(sanitizedkwargs[kwarg]) != \ - salt.utils.is_true(pre[kwarg]): + if salt.utils.data.is_true(sanitizedkwargs[kwarg]) != \ + salt.utils.data.is_true(pre[kwarg]): break else: if str(sanitizedkwargs[kwarg]) != str(pre[kwarg]): diff --git a/salt/states/service.py b/salt/states/service.py index 0b0850d9a3..3b0b3318d5 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -62,7 +62,7 @@ from __future__ import absolute_import import time # Import Salt libs -import salt.utils +import salt.utils.data import salt.utils.platform from salt.utils.args import get_function_argspec as _argspec from salt.exceptions import CommandExecutionError @@ -395,7 +395,7 @@ def running(name, # Convert enable to boolean in case user passed a string value if isinstance(enable, six.string_types): - enable = salt.utils.is_true(enable) + enable = salt.utils.data.is_true(enable) # Check if the service is available try: @@ -534,7 +534,7 @@ def dead(name, # Convert enable to boolean in case user passed a string value if isinstance(enable, six.string_types): - enable = salt.utils.is_true(enable) + enable = salt.utils.data.is_true(enable) # Check if the service is available try: diff --git a/salt/states/user.py b/salt/states/user.py index 1328960675..03a2ac1590 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -29,7 +29,7 @@ import os import logging # Import Salt libs -import salt.utils # Can be removed once date_format is moved +import salt.utils # TODO: Remove this once date_format is moved import salt.utils.platform import salt.utils.user from salt.utils.locales import sdecode, sdecode_if_string diff --git a/salt/states/virtualenv_mod.py b/salt/states/virtualenv_mod.py index 364699051f..bd394bfd0b 100644 --- a/salt/states/virtualenv_mod.py +++ b/salt/states/virtualenv_mod.py @@ -12,7 +12,7 @@ import os # Import Salt libs import salt.version -import salt.utils # Can be removed once alias_function is moved +import salt.utils # TODO: Remove this once alias_function is moved import salt.utils.platform import salt.utils.versions from salt.exceptions import CommandExecutionError, CommandNotFoundError diff --git a/salt/states/win_wua.py b/salt/states/win_wua.py index 798853d5ca..8f60934595 100644 --- a/salt/states/win_wua.py +++ b/salt/states/win_wua.py @@ -55,7 +55,7 @@ import logging # Import Salt libs from salt.ext import six -import salt.utils +import salt.utils.data import salt.utils.platform import salt.utils.win_update @@ -151,14 +151,14 @@ def installed(name, updates=None): # List of updates to download download = salt.utils.win_update.Updates() for item in install_list.updates: - if not salt.utils.is_true(item.IsDownloaded): + if not salt.utils.data.is_true(item.IsDownloaded): download.updates.Add(item) # List of updates to install install = salt.utils.win_update.Updates() installed_updates = [] for item in install_list.updates: - if not salt.utils.is_true(item.IsInstalled): + if not salt.utils.data.is_true(item.IsInstalled): install.updates.Add(item) else: installed_updates.extend('KB' + kb for kb in item.KBArticleIDs) @@ -190,7 +190,7 @@ def installed(name, updates=None): # Verify the installation for item in install.list(): - if not salt.utils.is_true(post_info[item]['Installed']): + if not salt.utils.data.is_true(post_info[item]['Installed']): ret['changes']['failed'] = { item: {'Title': post_info[item]['Title'][:40] + '...', 'KBs': post_info[item]['KBs']} @@ -285,7 +285,7 @@ def removed(name, updates=None): uninstall = salt.utils.win_update.Updates() removed_updates = [] for item in updates.updates: - if salt.utils.is_true(item.IsInstalled): + if salt.utils.data.is_true(item.IsInstalled): uninstall.updates.Add(item) else: removed_updates.extend('KB' + kb for kb in item.KBArticleIDs) @@ -314,7 +314,7 @@ def removed(name, updates=None): # Verify the installation for item in uninstall.list(): - if salt.utils.is_true(post_info[item]['Installed']): + if salt.utils.data.is_true(post_info[item]['Installed']): ret['changes']['failed'] = { item: {'Title': post_info[item]['Title'][:40] + '...', 'KBs': post_info[item]['KBs']} @@ -451,13 +451,13 @@ def uptodate(name, # List of updates to download download = salt.utils.win_update.Updates() for item in install_list.updates: - if not salt.utils.is_true(item.IsDownloaded): + if not salt.utils.data.is_true(item.IsDownloaded): download.updates.Add(item) # List of updates to install install = salt.utils.win_update.Updates() for item in install_list.updates: - if not salt.utils.is_true(item.IsInstalled): + if not salt.utils.data.is_true(item.IsInstalled): install.updates.Add(item) # Return comment of changes if test. @@ -483,7 +483,7 @@ def uptodate(name, # Verify the installation for item in install.list(): - if not salt.utils.is_true(post_info[item]['Installed']): + if not salt.utils.data.is_true(post_info[item]['Installed']): ret['changes']['failed'] = { item: {'Title': post_info[item]['Title'][:40] + '...', 'KBs': post_info[item]['KBs']} diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 85da33b01c..0fa4232bbf 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -235,22 +235,6 @@ def output_profile(pr, stats_path='/tmp/stats', stop=False, id_=None): return pr -@jinja_filter('list_files') -def list_files(directory): - ''' - Return a list of all files found under directory - ''' - ret = set() - ret.add(directory) - for root, dirs, files in safe_walk(directory): - for name in files: - ret.add(os.path.join(root, name)) - for name in dirs: - ret.add(os.path.join(root, name)) - - return list(ret) - - def required_module_list(docstring=None): ''' Return a list of python modules required by a salt module that aren't @@ -346,35 +330,6 @@ def backup_minion(path, bkroot): os.chmod(bkpath, fstat.st_mode) -def pem_finger(path=None, key=None, sum_type='sha256'): - ''' - Pass in either a raw pem string, or the path on disk to the location of a - pem file, and the type of cryptographic hash to use. The default is SHA256. - The fingerprint of the pem will be returned. - - If neither a key nor a path are passed in, a blank string will be returned. - ''' - # Late import to avoid circular import. - import salt.utils.files - - if not key: - if not os.path.isfile(path): - return '' - - with salt.utils.files.fopen(path, 'rb') as fp_: - key = b''.join([x for x in fp_.readlines() if x.strip()][1:-1]) - - pre = getattr(hashlib, sum_type)(key).hexdigest() - finger = '' - for ind in range(len(pre)): - if ind % 2: - # Is odd - finger += '{0}:'.format(pre[ind]) - else: - finger += pre[ind] - return finger.rstrip(':') - - def build_whitespace_split_regex(text): ''' Create a regular expression at runtime which should match ignoring the @@ -565,21 +520,6 @@ def format_call(fun, return ret -@jinja_filter('sorted_ignorecase') -def isorted(to_sort): - ''' - Sort a list of strings ignoring case. - - >>> L = ['foo', 'Foo', 'bar', 'Bar'] - >>> sorted(L) - ['Bar', 'Foo', 'bar', 'foo'] - >>> sorted(L, key=lambda x: x.lower()) - ['bar', 'Bar', 'foo', 'Foo'] - >>> - ''' - return sorted(to_sort, key=lambda x: x.lower()) - - @jinja_filter('mysql_to_dict') def mysql_to_dict(data, key): ''' @@ -752,79 +692,6 @@ def check_include_exclude(path_str, include_pat=None, exclude_pat=None): return ret -def st_mode_to_octal(mode): - ''' - Convert the st_mode value from a stat(2) call (as returned from os.stat()) - to an octal mode. - ''' - try: - return oct(mode)[-4:] - except (TypeError, IndexError): - return '' - - -def normalize_mode(mode): - ''' - Return a mode value, normalized to a string and containing a leading zero - if it does not have one. - - Allow "keep" as a valid mode (used by file state/module to preserve mode - from the Salt fileserver in file states). - ''' - if mode is None: - return None - if not isinstance(mode, six.string_types): - mode = str(mode) - if six.PY3: - mode = mode.replace('0o', '0') - # Strip any quotes any initial zeroes, then though zero-pad it up to 4. - # This ensures that somethign like '00644' is normalized to '0644' - return mode.strip('"').strip('\'').lstrip('0').zfill(4) - - -def test_mode(**kwargs): - ''' - Examines the kwargs passed and returns True if any kwarg which matching - "Test" in any variation on capitalization (i.e. "TEST", "Test", "TeSt", - etc) contains a True value (as determined by salt.utils.is_true). - ''' - for arg, value in six.iteritems(kwargs): - try: - if arg.lower() == 'test' and is_true(value): - return True - except AttributeError: - continue - return False - - -def is_true(value=None): - ''' - Returns a boolean value representing the "truth" of the value passed. The - rules for what is a "True" value are: - - 1. Integer/float values greater than 0 - 2. The string values "True" and "true" - 3. Any object for which bool(obj) returns True - ''' - # First, try int/float conversion - try: - value = int(value) - except (ValueError, TypeError): - pass - try: - value = float(value) - except (ValueError, TypeError): - pass - - # Now check for truthiness - if isinstance(value, (six.integer_types, float)): - return value > 0 - elif isinstance(value, six.string_types): - return str(value).lower() == 'true' - else: - return bool(value) - - def option(value, default='', opts=None, pillar=None): ''' Pass in a generic option and receive the value that will be assigned @@ -872,83 +739,6 @@ def print_cli(msg, retries=10, step=0.01): break -def safe_walk(top, topdown=True, onerror=None, followlinks=True, _seen=None): - ''' - A clone of the python os.walk function with some checks for recursive - symlinks. Unlike os.walk this follows symlinks by default. - ''' - islink, join, isdir = os.path.islink, os.path.join, os.path.isdir - if _seen is None: - _seen = set() - - # We may not have read permission for top, in which case we can't - # get a list of the files the directory contains. os.path.walk - # always suppressed the exception then, rather than blow up for a - # minor reason when (say) a thousand readable directories are still - # left to visit. That logic is copied here. - try: - # Note that listdir and error are globals in this module due - # to earlier import-*. - names = os.listdir(top) - except os.error as err: - if onerror is not None: - onerror(err) - return - - if followlinks: - status = os.stat(top) - # st_ino is always 0 on some filesystems (FAT, NTFS); ignore them - if status.st_ino != 0: - node = (status.st_dev, status.st_ino) - if node in _seen: - return - _seen.add(node) - - dirs, nondirs = [], [] - for name in names: - full_path = join(top, name) - if isdir(full_path): - dirs.append(name) - else: - nondirs.append(name) - - if topdown: - yield top, dirs, nondirs - for name in dirs: - new_path = join(top, name) - if followlinks or not islink(new_path): - for x in safe_walk(new_path, topdown, onerror, followlinks, _seen): - yield x - if not topdown: - yield top, dirs, nondirs - - -@jinja_filter('file_hashsum') -def get_hash(path, form='sha256', chunk_size=65536): - ''' - Get the hash sum of a file - - This is better than ``get_sum`` for the following reasons: - - It does not read the entire file into memory. - - It does not return a string on error. The returned value of - ``get_sum`` cannot really be trusted since it is vulnerable to - collisions: ``get_sum(..., 'xyz') == 'Hash xyz not supported'`` - ''' - # Late import to avoid circular import. - import salt.utils.files - - hash_type = hasattr(hashlib, form) and getattr(hashlib, form) or None - if hash_type is None: - raise ValueError('Invalid hash type: {0}'.format(form)) - - with salt.utils.files.fopen(path, 'rb') as ifile: - hash_obj = hash_type() - # read the file in in chunks, not the entire file - for chunk in iter(lambda: ifile.read(chunk_size), b''): - hash_obj.update(chunk) - return hash_obj.hexdigest() - - def namespaced_function(function, global_dict, defaults=None, preserve_context=False): ''' Redefine (clone) a function under a different globals() namespace scope @@ -1094,31 +884,6 @@ def find_json(raw): raise ValueError -@jinja_filter('is_bin_file') -def is_bin_file(path): - ''' - Detects if the file is a binary, returns bool. Returns True if the file is - a bin, False if the file is not and None if the file is not available. - ''' - # Late import to avoid circular import. - import salt.utils.files - import salt.utils.stringutils - - if not os.path.isfile(path): - return False - try: - with salt.utils.files.fopen(path, 'rb') as fp_: - try: - data = fp_.read(2048) - if six.PY3: - data = data.decode(__salt_system_encoding__) - return salt.utils.stringutils.is_binary(data) - except UnicodeDecodeError: - return True - except os.error: - return False - - def total_seconds(td): ''' Takes a timedelta and returns the total number of seconds @@ -1141,67 +906,6 @@ def import_json(): continue -def human_size_to_bytes(human_size): - ''' - Convert human-readable units to bytes - ''' - size_exp_map = {'K': 1, 'M': 2, 'G': 3, 'T': 4, 'P': 5} - human_size_str = str(human_size) - match = re.match(r'^(\d+)([KMGTP])?$', human_size_str) - if not match: - raise ValueError( - 'Size must be all digits, with an optional unit type ' - '(K, M, G, T, or P)' - ) - size_num = int(match.group(1)) - unit_multiplier = 1024 ** size_exp_map.get(match.group(2), 0) - return size_num * unit_multiplier - - -@jinja_filter('is_list') -def is_list(value): - ''' - Check if a variable is a list. - ''' - return isinstance(value, list) - - -@jinja_filter('is_iter') -def is_iter(y, ignore=six.string_types): - ''' - Test if an object is iterable, but not a string type. - - Test if an object is an iterator or is iterable itself. By default this - does not return True for string objects. - - The `ignore` argument defaults to a list of string types that are not - considered iterable. This can be used to also exclude things like - dictionaries or named tuples. - - Based on https://bitbucket.org/petershinners/yter - ''' - - if ignore and isinstance(y, ignore): - return False - try: - iter(y) - return True - except TypeError: - return False - - -def split_input(val): - ''' - Take an input value and split it into a list, returning the resulting list - ''' - if isinstance(val, list): - return val - try: - return [x.strip() for x in val.split(',')] - except AttributeError: - return [x.strip() for x in str(val).split(',')] - - def simple_types_filter(data): ''' Convert the data list, dictionary into simple types, i.e., int, float, string, @@ -1317,6 +1021,19 @@ def reinit_crypto(): return salt.utils.crypt.reinit_crypto() +def pem_finger(path=None, key=None, sum_type='sha256'): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.crypt + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.pem_finger\' detected. This function has been ' + 'moved to \'salt.utils.crypt.pem_finger\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.crypt.pem_finger(path, key, sum_type) + + def to_bytes(s, encoding=None): # Late import to avoid circular import. import salt.utils.versions @@ -1512,6 +1229,32 @@ def argspec_report(functions, module=''): return salt.utils.args.argspec_report(functions, module=module) +def split_input(val): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.args + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.split_input\' detected. This function has been ' + 'moved to \'salt.utils.args.split_input\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.args.split_input(val) + + +def test_mode(**kwargs): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.args + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.test_mode\' detected. This function has been ' + 'moved to \'salt.utils.args.test_mode\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.args.test_mode(**kwargs) + + def which(exe=None): # Late import to avoid circular import. import salt.utils.versions @@ -1564,6 +1307,19 @@ def rand_str(size=9999999999, hash_type=None): return salt.utils.hashutils.random_hash(size, hash_type) +def get_hash(path, form='sha256', chunk_size=65536): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.hashutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.get_hash\' detected. This function has been ' + 'moved to \'salt.utils.hashutils.get_hash\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.hashutils.get_hash(path, form, chunk_size) + + def is_windows(): # Late import to avoid circular import. import salt.utils.versions @@ -1814,9 +1570,9 @@ def mkstemp(*args, **kwargs): return salt.utils.files.mkstemp(*args, **kwargs) -@jinja_filter('is_text_file') def istextfile(fp_, blocksize=512): # Late import to avoid circular import. + import salt.utils.versions import salt.utils.files salt.utils.versions.warn_until( @@ -1828,6 +1584,90 @@ def istextfile(fp_, blocksize=512): return salt.utils.files.is_text_file(fp_, blocksize=blocksize) +def is_bin_file(path): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_bin_file\' detected. This function has been moved ' + 'to \'salt.utils.files.is_binary\' as of Salt Oxygen. This warning will ' + 'be removed in Salt Neon.' + ) + return salt.utils.files.is_binary(path) + + +def list_files(directory): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.list_files\' detected. This function has been moved ' + 'to \'salt.utils.files.list_files\' as of Salt Oxygen. This warning will ' + 'be removed in Salt Neon.' + ) + return salt.utils.files.list_files(directory) + + +def safe_walk(top, topdown=True, onerror=None, followlinks=True, _seen=None): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.safe_walk\' detected. This function has been moved ' + 'to \'salt.utils.files.safe_walk\' as of Salt Oxygen. This warning will ' + 'be removed in Salt Neon.' + ) + return salt.utils.files.safe_walk(top, topdown, onerror, followlinks, _seen) + + +def st_mode_to_octal(mode): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.st_mode_to_octal\' detected. This function has ' + 'been moved to \'salt.utils.files.st_mode_to_octal\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.files.st_mode_to_octal(mode) + + +def normalize_mode(mode): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.normalize_mode\' detected. This function has ' + 'been moved to \'salt.utils.files.normalize_mode\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.files.normalize_mode(mode) + + +def human_size_to_bytes(human_size): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.human_size_to_bytes\' detected. This function has ' + 'been moved to \'salt.utils.files.human_size_to_bytes\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.files.human_size_to_bytes(human_size) + + def str_version_to_evr(verstring): # Late import to avoid circular import. import salt.utils.versions @@ -1931,8 +1771,8 @@ def kwargs_warn_until(kwargs, def get_color_theme(theme): # Late import to avoid circular import. - import salt.utils.color import salt.utils.versions + import salt.utils.color salt.utils.versions.warn_until( 'Neon', @@ -1945,8 +1785,8 @@ def get_color_theme(theme): def get_colors(use=True, theme=None): # Late import to avoid circular import. - import salt.utils.color import salt.utils.versions + import salt.utils.color salt.utils.versions.warn_until( 'Neon', @@ -2328,6 +2168,58 @@ def exactly_one(l): return salt.utils.data.exactly_one(l) +def is_list(value): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.data + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_list\' detected. This function ' + 'has been moved to \'salt.utils.data.is_list\' as of ' + 'Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.data.is_list(value) + + +def is_iter(y, ignore=six.string_types): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.data + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_iter\' detected. This function ' + 'has been moved to \'salt.utils.data.is_iter\' as of ' + 'Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.data.is_iter(y, ignore) + + +def isorted(to_sort): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.data + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.isorted\' detected. This function ' + 'has been moved to \'salt.utils.data.sorted_ignorecase\' as of ' + 'Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.data.sorted_ignorecase(to_sort) + + +def is_true(value=None): + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.data + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_true\' detected. This function ' + 'has been moved to \'salt.utils.data.is_true\' as of ' + 'Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.data.is_true(value) + + def ip_bracket(addr): # Late import to avoid circular import. import salt.utils.versions diff --git a/salt/utils/args.py b/salt/utils/args.py index 7d47c52862..484359114f 100644 --- a/salt/utils/args.py +++ b/salt/utils/args.py @@ -14,6 +14,7 @@ import shlex from salt.exceptions import SaltInvocationError from salt.ext import six from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin +import salt.utils.data import salt.utils.jid @@ -327,3 +328,32 @@ def argspec_report(functions, module=''): ret[fun]['kwargs'] = True if kwargs else None return ret + + +def split_input(val): + ''' + Take an input value and split it into a list, returning the resulting list + ''' + if isinstance(val, list): + return val + try: + return [x.strip() for x in val.split(',')] + except AttributeError: + return [x.strip() for x in str(val).split(',')] + + +def test_mode(**kwargs): + ''' + Examines the kwargs passed and returns True if any kwarg which matching + "Test" in any variation on capitalization (i.e. "TEST", "Test", "TeSt", + etc) contains a True value (as determined by salt.utils.data.is_true). + ''' + # Once is_true is moved, remove this import and fix the ref below + import salt.utils + for arg, value in six.iteritems(kwargs): + try: + if arg.lower() == 'test' and salt.utils.data.is_true(value): + return True + except AttributeError: + continue + return False diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index b013c48f30..9fd3aee822 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -46,8 +46,8 @@ import salt.client import salt.config import salt.loader import salt.template -import salt.utils # Can be removed when pem_finger is moved import salt.utils.compat +import salt.utils.crypt import salt.utils.event import salt.utils.files import salt.utils.platform @@ -2675,7 +2675,7 @@ def request_minion_cachedir( base = __opts__['cachedir'] if not fingerprint and pubkey is not None: - fingerprint = salt.utils.pem_finger(key=pubkey, sum_type=(opts and opts.get('hash_type') or 'sha256')) + fingerprint = salt.utils.crypt.pem_finger(key=pubkey, sum_type=(opts and opts.get('hash_type') or 'sha256')) init_cachedir(base) diff --git a/salt/utils/crypt.py b/salt/utils/crypt.py index 10d5e800ea..1f643c8127 100644 --- a/salt/utils/crypt.py +++ b/salt/utils/crypt.py @@ -3,12 +3,15 @@ from __future__ import absolute_import # Import Python libs +import hashlib import logging +import os log = logging.getLogger(__name__) # Import Salt libs import salt.loader +import salt.utils.files from salt.exceptions import SaltInvocationError try: @@ -106,3 +109,29 @@ def reinit_crypto(): ''' if HAS_CRYPTO: Crypto.Random.atfork() + + +def pem_finger(path=None, key=None, sum_type='sha256'): + ''' + Pass in either a raw pem string, or the path on disk to the location of a + pem file, and the type of cryptographic hash to use. The default is SHA256. + The fingerprint of the pem will be returned. + + If neither a key nor a path are passed in, a blank string will be returned. + ''' + if not key: + if not os.path.isfile(path): + return '' + + with salt.utils.files.fopen(path, 'rb') as fp_: + key = b''.join([x for x in fp_.readlines() if x.strip()][1:-1]) + + pre = getattr(hashlib, sum_type)(key).hexdigest() + finger = '' + for ind, _ in enumerate(pre): + if ind % 2: + # Is odd + finger += '{0}:'.format(pre[ind]) + else: + finger += pre[ind] + return finger.rstrip(':') diff --git a/salt/utils/data.py b/salt/utils/data.py index f20173badf..286095b642 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -420,3 +420,78 @@ def repack_dictlist(data, else: ret[key_cb(key)] = val_cb(key, val) return ret + + +@jinja_filter('is_list') +def is_list(value): + ''' + Check if a variable is a list. + ''' + return isinstance(value, list) + + +@jinja_filter('is_iter') +def is_iter(y, ignore=six.string_types): + ''' + Test if an object is iterable, but not a string type. + + Test if an object is an iterator or is iterable itself. By default this + does not return True for string objects. + + The `ignore` argument defaults to a list of string types that are not + considered iterable. This can be used to also exclude things like + dictionaries or named tuples. + + Based on https://bitbucket.org/petershinners/yter + ''' + + if ignore and isinstance(y, ignore): + return False + try: + iter(y) + return True + except TypeError: + return False + + +@jinja_filter('sorted_ignorecase') +def sorted_ignorecase(to_sort): + ''' + Sort a list of strings ignoring case. + + >>> L = ['foo', 'Foo', 'bar', 'Bar'] + >>> sorted(L) + ['Bar', 'Foo', 'bar', 'foo'] + >>> sorted(L, key=lambda x: x.lower()) + ['bar', 'Bar', 'foo', 'Foo'] + >>> + ''' + return sorted(to_sort, key=lambda x: x.lower()) + + +def is_true(value=None): + ''' + Returns a boolean value representing the "truth" of the value passed. The + rules for what is a "True" value are: + + 1. Integer/float values greater than 0 + 2. The string values "True" and "true" + 3. Any object for which bool(obj) returns True + ''' + # First, try int/float conversion + try: + value = int(value) + except (ValueError, TypeError): + pass + try: + value = float(value) + except (ValueError, TypeError): + pass + + # Now check for truthiness + if isinstance(value, (six.integer_types, float)): + return value > 0 + elif isinstance(value, six.string_types): + return str(value).lower() == 'true' + else: + return bool(value) diff --git a/salt/utils/extmods.py b/salt/utils/extmods.py index 5d4263b80f..e4b2d46553 100644 --- a/salt/utils/extmods.py +++ b/salt/utils/extmods.py @@ -11,6 +11,7 @@ import shutil # Import salt libs import salt.fileclient +import salt.utils.hashutils import salt.utils.url # Import 3rd-party libs @@ -111,8 +112,8 @@ def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None) if os.path.isfile(dest): # The file is present, if the sum differs replace it hash_type = opts.get('hash_type', 'md5') - src_digest = salt.utils.get_hash(fn_, hash_type) - dst_digest = salt.utils.get_hash(dest, hash_type) + src_digest = salt.utils.hashutils.get_hash(fn_, hash_type) + dst_digest = salt.utils.hashutils.get_hash(dest, hash_type) if src_digest != dst_digest: # The downloaded file differs, replace! shutil.copyfile(fn_, dest) diff --git a/salt/utils/files.py b/salt/utils/files.py index 207e29b5ca..9a4a14eebd 100644 --- a/salt/utils/files.py +++ b/salt/utils/files.py @@ -16,9 +16,10 @@ import time import urllib # Import Salt libs -import salt.utils # Can be removed when backup_minion is moved +import salt.utils # TODO: Remove this when backup_minion is moved import salt.utils.path import salt.utils.platform +import salt.utils.stringutils import salt.modules.selinux from salt.exceptions import CommandExecutionError, FileLockError, MinionError from salt.utils.decorators.jinja import jinja_filter @@ -414,6 +415,56 @@ def fpopen(*args, **kwargs): yield f_handle +def safe_walk(top, topdown=True, onerror=None, followlinks=True, _seen=None): + ''' + A clone of the python os.walk function with some checks for recursive + symlinks. Unlike os.walk this follows symlinks by default. + ''' + if _seen is None: + _seen = set() + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. os.path.walk + # always suppressed the exception then, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. That logic is copied here. + try: + # Note that listdir and error are globals in this module due + # to earlier import-*. + names = os.listdir(top) + except os.error as err: + if onerror is not None: + onerror(err) + return + + if followlinks: + status = os.stat(top) + # st_ino is always 0 on some filesystems (FAT, NTFS); ignore them + if status.st_ino != 0: + node = (status.st_dev, status.st_ino) + if node in _seen: + return + _seen.add(node) + + dirs, nondirs = [], [] + for name in names: + full_path = os.path.join(top, name) + if os.path.isdir(full_path): + dirs.append(name) + else: + nondirs.append(name) + + if topdown: + yield top, dirs, nondirs + for name in dirs: + new_path = os.path.join(top, name) + if followlinks or not os.path.islink(new_path): + for x in safe_walk(new_path, topdown, onerror, followlinks, _seen): + yield x + if not topdown: + yield top, dirs, nondirs + + def safe_rm(tgt): ''' Safely remove a file @@ -523,7 +574,7 @@ def safe_filepath(file_path_name, dir_sep=None): @jinja_filter('is_text_file') -def is_text_file(fp_, blocksize=512): +def is_text(fp_, blocksize=512): ''' Uses heuristics to guess whether the given file is text or binary, by reading a single block of bytes from the file. @@ -561,6 +612,27 @@ def is_text_file(fp_, blocksize=512): return float(len(nontext)) / len(block) <= 0.30 +@jinja_filter('is_bin_file') +def is_binary(path): + ''' + Detects if the file is a binary, returns bool. Returns True if the file is + a bin, False if the file is not and None if the file is not available. + ''' + if not os.path.isfile(path): + return False + try: + with fopen(path, 'rb') as fp_: + try: + data = fp_.read(2048) + if six.PY3: + data = data.decode(__salt_system_encoding__) + return salt.utils.stringutils.is_binary(data) + except UnicodeDecodeError: + return True + except os.error: + return False + + def remove(path): ''' Runs os.remove(path) and suppresses the OSError if the file doesn't exist @@ -570,3 +642,66 @@ def remove(path): except OSError as exc: if exc.errno != errno.ENOENT: raise + + +@jinja_filter('list_files') +def list_files(directory): + ''' + Return a list of all files found under directory (and its subdirectories) + ''' + ret = set() + ret.add(directory) + for root, dirs, files in safe_walk(directory): + for name in files: + ret.add(os.path.join(root, name)) + for name in dirs: + ret.add(os.path.join(root, name)) + + return list(ret) + + +def st_mode_to_octal(mode): + ''' + Convert the st_mode value from a stat(2) call (as returned from os.stat()) + to an octal mode. + ''' + try: + return oct(mode)[-4:] + except (TypeError, IndexError): + return '' + + +def normalize_mode(mode): + ''' + Return a mode value, normalized to a string and containing a leading zero + if it does not have one. + + Allow "keep" as a valid mode (used by file state/module to preserve mode + from the Salt fileserver in file states). + ''' + if mode is None: + return None + if not isinstance(mode, six.string_types): + mode = str(mode) + if six.PY3: + mode = mode.replace('0o', '0') + # Strip any quotes any initial zeroes, then though zero-pad it up to 4. + # This ensures that somethign like '00644' is normalized to '0644' + return mode.strip('"').strip('\'').lstrip('0').zfill(4) + + +def human_size_to_bytes(human_size): + ''' + Convert human-readable units to bytes + ''' + size_exp_map = {'K': 1, 'M': 2, 'G': 3, 'T': 4, 'P': 5} + human_size_str = str(human_size) + match = re.match(r'^(\d+)([KMGTP])?$', human_size_str) + if not match: + raise ValueError( + 'Size must be all digits, with an optional unit type ' + '(K, M, G, T, or P)' + ) + size_num = int(match.group(1)) + unit_multiplier = 1024 ** size_exp_map.get(match.group(2), 0) + return size_num * unit_multiplier diff --git a/salt/utils/find.py b/salt/utils/find.py index 565ee996c3..ee641cd2bc 100644 --- a/salt/utils/find.py +++ b/salt/utils/find.py @@ -105,8 +105,8 @@ except ImportError: from salt.ext import six # Import salt libs -import salt.utils import salt.utils.args +import salt.utils.hashutils import salt.utils.stringutils import salt.defaults.exitcodes from salt.utils.filebuffer import BufferedReader @@ -510,7 +510,7 @@ class PrintOption(Option): result.append(gid) elif arg == 'md5': if stat.S_ISREG(fstat[stat.ST_MODE]): - md5digest = salt.utils.get_hash(fullpath, 'md5') + md5digest = salt.utils.hashutils.get_hash(fullpath, 'md5') result.append(md5digest) else: result.append('') diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index e4310a1519..ae028463a0 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -20,11 +20,12 @@ import time from datetime import datetime # Import salt libs -import salt.utils # Can be removed once check_whitelist_blacklist, get_hash, is_bin_file, repack_dictlist are moved +import salt.utils # TODO: Remove this once check_whitelist_blacklist is moved import salt.utils.configparser import salt.utils.data import salt.utils.files import salt.utils.gzip_util +import salt.utils.hashutils import salt.utils.itertools import salt.utils.path import salt.utils.platform @@ -2649,7 +2650,7 @@ class GitFS(GitBase): if not os.path.isfile(hashdest): if not os.path.exists(os.path.dirname(hashdest)): os.makedirs(os.path.dirname(hashdest)) - ret['hsum'] = salt.utils.get_hash(path, self.opts['hash_type']) + ret['hsum'] = salt.utils.hashutils.get_hash(path, self.opts['hash_type']) with salt.utils.files.fopen(hashdest, 'w+') as fp_: fp_.write(ret['hsum']) return ret diff --git a/salt/utils/hashutils.py b/salt/utils/hashutils.py index 341058e80f..f24ef42185 100644 --- a/salt/utils/hashutils.py +++ b/salt/utils/hashutils.py @@ -12,6 +12,7 @@ import random # Import Salt libs from salt.ext import six +import salt.utils.files import salt.utils.stringutils from salt.utils.decorators.jinja import jinja_filter @@ -139,3 +140,26 @@ def random_hash(size=9999999999, hash_type=None): hash_type = 'md5' hasher = getattr(hashlib, hash_type) return hasher(salt.utils.stringutils.to_bytes(str(random.SystemRandom().randint(0, size)))).hexdigest() + + +@jinja_filter('file_hashsum') +def get_hash(path, form='sha256', chunk_size=65536): + ''' + Get the hash sum of a file + + This is better than ``get_sum`` for the following reasons: + - It does not read the entire file into memory. + - It does not return a string on error. The returned value of + ``get_sum`` cannot really be trusted since it is vulnerable to + collisions: ``get_sum(..., 'xyz') == 'Hash xyz not supported'`` + ''' + hash_type = hasattr(hashlib, form) and getattr(hashlib, form) or None + if hash_type is None: + raise ValueError('Invalid hash type: {0}'.format(form)) + + with salt.utils.files.fopen(path, 'rb') as ifile: + hash_obj = hash_type() + # read the file in in chunks, not the entire file + for chunk in iter(lambda: ifile.read(chunk_size), b''): + hash_obj.update(chunk) + return hash_obj.hexdigest() diff --git a/salt/utils/minions.py b/salt/utils/minions.py index ab30e31d1c..41c6b11cef 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -13,7 +13,7 @@ import logging # Import salt libs import salt.payload -import salt.utils +import salt.utils # TODO: Remove this once expr_match is moved import salt.utils.data import salt.utils.files import salt.utils.network @@ -236,7 +236,7 @@ class CkMinions(object): with salt.utils.files.fopen(pki_cache_fn) as fn_: return self.serial.load(fn_) else: - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): minions.append(fn_) return minions @@ -263,7 +263,7 @@ class CkMinions(object): if greedy: minions = [] - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): minions.append(fn_) elif cache_enabled: @@ -420,7 +420,7 @@ class CkMinions(object): cache_enabled = self.opts.get('minion_data_cache', False) if greedy: mlist = [] - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): mlist.append(fn_) return {'minions': mlist, @@ -636,7 +636,7 @@ class CkMinions(object): Return a list of all minions that have auth'd ''' mlist = [] - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): + for fn_ in salt.utils.data.sorted_ignorecase(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): mlist.append(fn_) return {'minions': mlist, 'missing': []} diff --git a/salt/utils/pkg/__init__.py b/salt/utils/pkg/__init__.py index 68e5864676..a764560816 100644 --- a/salt/utils/pkg/__init__.py +++ b/salt/utils/pkg/__init__.py @@ -10,7 +10,7 @@ import os import re # Import Salt libs -import salt.utils # Can be removed once is_true is moved +import salt.utils.data import salt.utils.files import salt.utils.versions @@ -62,7 +62,7 @@ def check_refresh(opts, refresh=None): - A boolean if refresh is not False and the rtag file exists ''' return bool( - salt.utils.is_true(refresh) or + salt.utils.data.is_true(refresh) or (os.path.isfile(rtag(opts)) and refresh is not False) ) diff --git a/salt/utils/s3.py b/salt/utils/s3.py index 0bb198d3a6..236d8a48b6 100644 --- a/salt/utils/s3.py +++ b/salt/utils/s3.py @@ -17,9 +17,9 @@ except ImportError: HAS_REQUESTS = False # pylint: disable=W0612 # Import Salt libs -import salt.utils import salt.utils.aws import salt.utils.files +import salt.utils.hashutils import salt.utils.xmlutil as xml from salt._compat import ElementTree as ET from salt.exceptions import CommandExecutionError @@ -119,7 +119,7 @@ def query(key, keyid, method='GET', params=None, headers=None, payload_hash = None if method == 'PUT': if local_file: - payload_hash = salt.utils.get_hash(local_file, form='sha256') + payload_hash = salt.utils.hashutils.get_hash(local_file, form='sha256') if path is None: path = '' diff --git a/salt/utils/templates.py b/salt/utils/templates.py index bbb834bb41..f8e7682538 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -152,7 +152,7 @@ def wrap_tmpl_func(render_str): ValueError, OSError, IOError) as exc: - if salt.utils.is_bin_file(tmplsrc): + if salt.utils.files.is_binary(tmplsrc): # Template is a bin file, return the raw file return dict(result=True, data=tmplsrc) log.error( diff --git a/salt/utils/thin.py b/salt/utils/thin.py index 2dca207140..bd13b9f11c 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -72,6 +72,7 @@ except ImportError: import salt import salt.utils import salt.utils.files +import salt.utils.hashutils import salt.exceptions SALTCALL = ''' @@ -341,7 +342,7 @@ def thin_sum(cachedir, form='sha1'): Return the checksum of the current thin tarball ''' thintar = gen_thin(cachedir) - return salt.utils.get_hash(thintar, form) + return salt.utils.hashutils.get_hash(thintar, form) def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', @@ -625,4 +626,4 @@ def min_sum(cachedir, form='sha1'): Return the checksum of the current thin tarball ''' mintar = gen_min(cachedir) - return salt.utils.get_hash(mintar, form) + return salt.utils.hashutils.get_hash(mintar, form) diff --git a/salt/utils/win_update.py b/salt/utils/win_update.py index fac45d9276..e16bfc7b57 100644 --- a/salt/utils/win_update.py +++ b/salt/utils/win_update.py @@ -8,8 +8,8 @@ import logging import subprocess # Import Salt libs -import salt.utils import salt.utils.args +import salt.utils.data from salt.ext import six from salt.ext.six.moves import range from salt.exceptions import CommandExecutionError @@ -199,17 +199,17 @@ class Updates(object): results['Total'] += 1 # Updates available for download - if not salt.utils.is_true(update.IsDownloaded) \ - and not salt.utils.is_true(update.IsInstalled): + if not salt.utils.data.is_true(update.IsDownloaded) \ + and not salt.utils.data.is_true(update.IsInstalled): results['Available'] += 1 # Updates downloaded awaiting install - if salt.utils.is_true(update.IsDownloaded) \ - and not salt.utils.is_true(update.IsInstalled): + if salt.utils.data.is_true(update.IsDownloaded) \ + and not salt.utils.data.is_true(update.IsInstalled): results['Downloaded'] += 1 # Updates installed - if salt.utils.is_true(update.IsInstalled): + if salt.utils.data.is_true(update.IsInstalled): results['Installed'] += 1 # Add Categories and increment total for each one @@ -437,16 +437,16 @@ class WindowsUpdateAgent(object): for update in self._updates: - if salt.utils.is_true(update.IsHidden) and skip_hidden: + if salt.utils.data.is_true(update.IsHidden) and skip_hidden: continue - if salt.utils.is_true(update.IsInstalled) and skip_installed: + if salt.utils.data.is_true(update.IsInstalled) and skip_installed: continue - if salt.utils.is_true(update.IsMandatory) and skip_mandatory: + if salt.utils.data.is_true(update.IsMandatory) and skip_mandatory: continue - if salt.utils.is_true( + if salt.utils.data.is_true( update.InstallationBehavior.RebootBehavior) and skip_reboot: continue @@ -587,12 +587,12 @@ class WindowsUpdateAgent(object): bool(update.IsDownloaded) # Accept EULA - if not salt.utils.is_true(update.EulaAccepted): + if not salt.utils.data.is_true(update.EulaAccepted): log.debug('Accepting EULA: {0}'.format(update.Title)) update.AcceptEula() # pylint: disable=W0104 # Update already downloaded - if not salt.utils.is_true(update.IsDownloaded): + if not salt.utils.data.is_true(update.IsDownloaded): log.debug('To Be Downloaded: {0}'.format(uid)) log.debug('\tTitle: {0}'.format(update.Title)) download_list.Add(update) @@ -697,7 +697,7 @@ class WindowsUpdateAgent(object): ret['Updates'][uid]['AlreadyInstalled'] = bool(update.IsInstalled) # Make sure the update has actually been installed - if not salt.utils.is_true(update.IsInstalled): + if not salt.utils.data.is_true(update.IsInstalled): log.debug('To Be Installed: {0}'.format(uid)) log.debug('\tTitle: {0}'.format(update.Title)) install_list.Add(update) @@ -817,7 +817,7 @@ class WindowsUpdateAgent(object): not bool(update.IsInstalled) # Make sure the update has actually been Uninstalled - if salt.utils.is_true(update.IsInstalled): + if salt.utils.data.is_true(update.IsInstalled): log.debug('To Be Uninstalled: {0}'.format(uid)) log.debug('\tTitle: {0}'.format(update.Title)) uninstall_list.Add(update) @@ -1003,4 +1003,4 @@ def needs_reboot(): # Create an AutoUpdate object obj_sys = win32com.client.Dispatch('Microsoft.Update.SystemInfo') - return salt.utils.is_true(obj_sys.RebootRequired) + return salt.utils.data.is_true(obj_sys.RebootRequired) diff --git a/salt/wheel/file_roots.py b/salt/wheel/file_roots.py index 8df2eee4b8..1f2d7ab68d 100644 --- a/salt/wheel/file_roots.py +++ b/salt/wheel/file_roots.py @@ -27,7 +27,7 @@ def find(path, saltenv='base'): if os.path.isfile(full): # Add it to the dict with salt.utils.files.fopen(full, 'rb') as fp_: - if salt.utils.files.is_text_file(fp_): + if salt.utils.files.is_text(fp_): ret.append({full: 'txt'}) else: ret.append({full: 'bin'}) diff --git a/salt/wheel/key.py b/salt/wheel/key.py index 9f1a17bfa7..a4dcf7a787 100644 --- a/salt/wheel/key.py +++ b/salt/wheel/key.py @@ -36,7 +36,7 @@ import logging # Import salt libs from salt.key import get_key import salt.crypt -import salt.utils # Can be removed once pem_finger is moved +import salt.utils.crypt import salt.utils.files import salt.utils.platform from salt.utils.sanitizers import clean @@ -315,7 +315,7 @@ def finger_master(hash_type=None): if hash_type is None: hash_type = __opts__['hash_type'] - fingerprint = salt.utils.pem_finger( + fingerprint = salt.utils.crypt.pem_finger( os.path.join(__opts__['pki_dir'], keyname), sum_type=hash_type) return {'local': {keyname: fingerprint}} diff --git a/salt/wheel/pillar_roots.py b/salt/wheel/pillar_roots.py index 65790e17d9..b5464a23ca 100644 --- a/salt/wheel/pillar_roots.py +++ b/salt/wheel/pillar_roots.py @@ -28,7 +28,7 @@ def find(path, saltenv='base'): if os.path.isfile(full): # Add it to the dict with salt.utils.files.fopen(full, 'rb') as fp_: - if salt.utils.files.is_text_file(fp_): + if salt.utils.files.is_text(fp_): ret.append({full: 'txt'}) else: ret.append({full: 'bin'}) diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index d145efdbb1..a720a9e356 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -32,7 +32,6 @@ from tests.support.helpers import ( from tests.support.mixins import SaltReturnAssertsMixin # Import Salt libs -import salt.utils # Can be removed once normalize_mode is moved import salt.utils.files import salt.utils.path import salt.utils.platform @@ -615,7 +614,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ''' Return a string octal representation of the permissions for name ''' - return salt.utils.normalize_mode(oct(os.stat(name).st_mode & 0o777)) + return salt.utils.files.normalize_mode(oct(os.stat(name).st_mode & 0o777)) top = os.path.join(TMP, 'top_dir') sub = os.path.join(top, 'sub_dir') diff --git a/tests/support/mixins.py b/tests/support/mixins.py index 64a4bd4fbc..036c6fa3ba 100644 --- a/tests/support/mixins.py +++ b/tests/support/mixins.py @@ -31,7 +31,7 @@ from tests.support.paths import CODE_DIR # Import salt libs import salt.config -import salt.utils # Can be removed once namespaced_function is moved +import salt.utils # TODO: Remove this once namespaced_function is moved import salt.utils.event import salt.utils.files import salt.utils.path diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index d580667923..8f1b3a97a0 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -48,7 +48,7 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin): 'grains': {}, }, '__grains__': {'kernel': 'Linux'}, - '__utils__': {'files.is_text_file': MagicMock(return_value=True)}, + '__utils__': {'files.is_text': MagicMock(return_value=True)}, } } @@ -218,7 +218,7 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): 'grains': {}, }, '__grains__': {'kernel': 'Linux'}, - '__utils__': {'files.is_text_file': MagicMock(return_value=True)}, + '__utils__': {'files.is_text': MagicMock(return_value=True)}, } } diff --git a/tests/unit/modules/test_key.py b/tests/unit/modules/test_key.py index fc5cb27e66..5c540fffef 100644 --- a/tests/unit/modules/test_key.py +++ b/tests/unit/modules/test_key.py @@ -18,7 +18,7 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.utils +import salt.utils.crypt import salt.modules.key as key @@ -35,7 +35,7 @@ class KeyTestCase(TestCase, LoaderModuleMockMixin): Test for finger ''' with patch.object(os.path, 'join', return_value='A'): - with patch.object(salt.utils, + with patch.object(salt.utils.crypt, 'pem_finger', return_value='A'): with patch.dict(key.__opts__, {'pki_dir': MagicMock(return_value='A'), 'hash_type': 'sha256'}): @@ -46,7 +46,7 @@ class KeyTestCase(TestCase, LoaderModuleMockMixin): Test for finger ''' with patch.object(os.path, 'join', return_value='A'): - with patch.object(salt.utils, + with patch.object(salt.utils.crypt, 'pem_finger', return_value='A'): with patch.dict(key.__opts__, {'pki_dir': 'A', 'hash_type': 'sha256'}): diff --git a/tests/unit/modules/test_pkg_resource.py b/tests/unit/modules/test_pkg_resource.py index e00d8cda77..3ac5a6f7d8 100644 --- a/tests/unit/modules/test_pkg_resource.py +++ b/tests/unit/modules/test_pkg_resource.py @@ -18,7 +18,7 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.utils +import salt.utils.data import salt.modules.pkg_resource as pkg_resource from salt.ext import six @@ -99,7 +99,7 @@ class PkgresTestCase(TestCase, LoaderModuleMockMixin): Test to Common interface for obtaining the version of installed packages. ''' - with patch.object(salt.utils, 'is_true', return_value=True): + with patch.object(salt.utils.data, 'is_true', return_value=True): mock = MagicMock(return_value={'A': 'B'}) with patch.dict(pkg_resource.__salt__, {'pkg.list_pkgs': mock}): diff --git a/tests/unit/modules/test_state.py b/tests/unit/modules/test_state.py index 7d9b78ead1..31e0b7385c 100644 --- a/tests/unit/modules/test_state.py +++ b/tests/unit/modules/test_state.py @@ -21,7 +21,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.config import salt.loader -import salt.utils +import salt.utils.hashutils import salt.utils.odict import salt.utils.platform import salt.modules.state as state @@ -969,7 +969,7 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(state.pkg(tar_file, "", "md5"), {}) mock = MagicMock(side_effect=[False, 0, 0, 0, 0]) - with patch.object(salt.utils, 'get_hash', mock): + with patch.object(salt.utils.hashutils, 'get_hash', mock): # Verify hash self.assertDictEqual(state.pkg(tar_file, "", "md5"), {}) diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index 7aa2c0d0f7..77ffe469bd 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -1182,7 +1182,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ret.update({'name': name}) with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open(read_data=''))): - with patch.dict(filestate.__utils__, {'files.is_text_file': mock_f}): + with patch.dict(filestate.__utils__, {'files.is_text': mock_f}): with patch.dict(filestate.__opts__, {'test': True}): change = {'diff': 'Replace binary file'} comt = ('File {0} is set to be updated' diff --git a/tests/unit/states/test_mysql_user.py b/tests/unit/states/test_mysql_user.py index 862e165c18..b34f3f97bb 100644 --- a/tests/unit/states/test_mysql_user.py +++ b/tests/unit/states/test_mysql_user.py @@ -16,7 +16,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.mysql_user as mysql_user -import salt.utils +import salt.utils.data @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -49,7 +49,7 @@ class MysqlUserTestCase(TestCase, LoaderModuleMockMixin): mock_str = MagicMock(return_value='salt') mock_none = MagicMock(return_value=None) mock_sn = MagicMock(side_effect=[None, 'salt', None, None, None]) - with patch.object(salt.utils, 'is_true', mock_f): + with patch.object(salt.utils.data, 'is_true', mock_f): comt = ('Either password or password_hash must be specified,' ' unless allow_passwordless is True') ret.update({'comment': comt}) @@ -57,7 +57,7 @@ class MysqlUserTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(mysql_user.__salt__, {'mysql.user_exists': mock, 'mysql.user_chpass': mock_t}): - with patch.object(salt.utils, 'is_true', mock_t): + with patch.object(salt.utils.data, 'is_true', mock_t): comt = ('User frank@localhost is already present' ' with passwordless login') ret.update({'comment': comt, 'result': True}) diff --git a/tests/unit/utils/test_args.py b/tests/unit/utils/test_args.py index 7389978265..9fdbde7c82 100644 --- a/tests/unit/utils/test_args.py +++ b/tests/unit/utils/test_args.py @@ -94,3 +94,8 @@ class ArgsTestCase(TestCase): ret = salt.utils.args.argspec_report(test_functions, 'test_module.test_spec') self.assertDictEqual(ret, {'test_module.test_spec': {'kwargs': True, 'args': None, 'defaults': None, 'varargs': True}}) + + def test_test_mode(self): + self.assertTrue(salt.utils.args.test_mode(test=True)) + self.assertTrue(salt.utils.args.test_mode(Test=True)) + self.assertTrue(salt.utils.args.test_mode(tEsT=True)) diff --git a/tests/unit/utils/test_data.py b/tests/unit/utils/test_data.py new file mode 100644 index 0000000000..be7ddf9f52 --- /dev/null +++ b/tests/unit/utils/test_data.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +''' +Tests for salt.utils.data +''' + +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +import salt.utils.data +from tests.support.unit import TestCase + + +class DataTestCase(TestCase): + def test_sorted_ignorecase(self): + test_list = ['foo', 'Foo', 'bar', 'Bar'] + expected_list = ['bar', 'Bar', 'foo', 'Foo'] + self.assertEqual( + salt.utils.data.sorted_ignorecase(test_list), expected_list) diff --git a/tests/unit/utils/test_hashutils.py b/tests/unit/utils/test_hashutils.py new file mode 100644 index 0000000000..5d5ce2596e --- /dev/null +++ b/tests/unit/utils/test_hashutils.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# Import python libs +from __future__ import absolute_import + +# Import Salt Testing libs +from tests.support.unit import TestCase + +# Import Salt libs +import salt.utils.hashutils + + +class HashutilsTestCase(TestCase): + + def test_get_hash_exception(self): + self.assertRaises( + ValueError, + salt.utils.hashutils.get_hash, + '/tmp/foo/', + form='INVALID') diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index cfff130667..06d286eeff 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -78,11 +78,6 @@ class UtilsTestCase(TestCase): ret = salt.utils.build_whitespace_split_regex(' '.join(LOREM_IPSUM.split()[:5])) self.assertEqual(ret, expected_regex) - def test_isorted(self): - test_list = ['foo', 'Foo', 'bar', 'Bar'] - expected_list = ['bar', 'Bar', 'foo', 'Foo'] - self.assertEqual(salt.utils.isorted(test_list), expected_list) - def test_mysql_to_dict(self): test_mysql_output = ['+----+------+-----------+------+---------+------+-------+------------------+', '| Id | User | Host | db | Command | Time | State | Info |', @@ -247,11 +242,6 @@ class UtilsTestCase(TestCase): with patch('zmq.IPC_PATH_MAX_LEN', 1): self.assertRaises(SaltSystemExit, salt.utils.zeromq.check_ipc_path_max_len, '1' * 1024) - def test_test_mode(self): - self.assertTrue(salt.utils.test_mode(test=True)) - self.assertTrue(salt.utils.test_mode(Test=True)) - self.assertTrue(salt.utils.test_mode(tEsT=True)) - def test_option(self): test_two_level_dict = {'foo': {'bar': 'baz'}} @@ -260,9 +250,6 @@ class UtilsTestCase(TestCase): self.assertEqual('baz', salt.utils.option('foo:bar', {'not_found': 'nope'}, pillar={'master': test_two_level_dict})) self.assertEqual('baz', salt.utils.option('foo:bar', {'not_found': 'nope'}, pillar=test_two_level_dict)) - def test_get_hash_exception(self): - self.assertRaises(ValueError, salt.utils.get_hash, '/tmp/foo/', form='INVALID') - @skipIf(NO_MOCK, NO_MOCK_REASON) def test_date_cast(self): now = datetime.datetime.now()