mirror of
https://github.com/valitydev/salt.git
synced 2024-11-08 17:33:54 +00:00
3184168365
This PR is part of what will be an ongoing effort to use explicit unicode strings in Salt. Because Python 3 does not suport Python 2's raw unicode string syntax (i.e. `ur'\d+'`), we must use `salt.utils.locales.sdecode()` to ensure that the raw string is unicode. However, because of how `salt/utils/__init__.py` has evolved into the hulking monstrosity it is today, this means importing a large module in places where it is not needed, which could negatively impact performance. For this reason, this PR also breaks out some of the functions from `salt/utils/__init__.py` into new/existing modules under `salt/utils/`. The long term goal will be that the modules within this directory do not depend on importing `salt.utils`. A summary of the changes in this PR is as follows: * Moves the following functions from `salt.utils` to new locations (including a deprecation warning if invoked from `salt.utils`): `to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`, `dequote`, `is_hex`, `is_bin_str`, `rand_string`, `contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`, `which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`, `is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`, `is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`, `is_openbsd`, `is_aix` * Moves the functions already deprecated by @rallytime to the bottom of `salt/utils/__init__.py` for better organization, so we can keep the deprecated ones separate from the ones yet to be deprecated as we continue to break up `salt.utils` * Updates `salt/*.py` and all files under `salt/client/` to use explicit unicode string literals. * Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils import foo` becomes `import salt.utils.foo as foo`). * Renames the `test.rand_str` function to `test.random_hash` to more accurately reflect what it does * Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`) such that it returns a string matching the passed size. Previously this function would get `size` bytes from `os.urandom()`, base64-encode it, and return the result, which would in most cases not be equal to the passed size.
448 lines
19 KiB
Python
448 lines
19 KiB
Python
# -*- coding: utf-8 -*-
|
|
'''
|
|
:codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
|
|
|
|
|
|
tests.unit.modules.archive_test
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
'''
|
|
|
|
# Import Python libs
|
|
from __future__ import absolute_import
|
|
import os.path
|
|
|
|
# Import Salt Testing libs
|
|
from tests.support.mixins import LoaderModuleMockMixin
|
|
from tests.support.unit import skipIf, TestCase
|
|
from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch
|
|
|
|
# Import salt libs
|
|
import salt.modules.archive as archive
|
|
import salt.utils.path
|
|
from salt.exceptions import CommandNotFoundError
|
|
|
|
# Import 3rd-party libs
|
|
from salt.ext.six.moves import zip
|
|
|
|
|
|
class ZipFileMock(MagicMock):
|
|
def __init__(self, files=None, **kwargs): # pylint: disable=W0231
|
|
if files is None:
|
|
files = ['salt']
|
|
MagicMock.__init__(self, **kwargs)
|
|
self._files = files
|
|
self.external_attr = 0o0777 << 16
|
|
|
|
def namelist(self):
|
|
return self._files
|
|
|
|
|
|
@skipIf(NO_MOCK, NO_MOCK_REASON)
|
|
class ArchiveTestCase(TestCase, LoaderModuleMockMixin):
|
|
|
|
def setup_loader_modules(self):
|
|
return {archive: {'__grains__': {'id': 0}}}
|
|
|
|
def test_tar(self):
|
|
with patch('glob.glob', lambda pathname: [pathname]):
|
|
with patch('salt.utils.path.which', lambda exe: exe):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.tar(
|
|
'-zcvf', 'foo.tar',
|
|
['/tmp/something-to-compress-1',
|
|
'/tmp/something-to-compress-2'],
|
|
template=None
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['tar', '-zcvf', 'foo.tar', '/tmp/something-to-compress-1',
|
|
'/tmp/something-to-compress-2'],
|
|
runas=None, python_shell=False, template=None, cwd=None
|
|
)
|
|
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.tar(
|
|
'-zcvf', 'foo.tar',
|
|
'/tmp/something-to-compress-1,/tmp/something-to-compress-2',
|
|
template=None
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['tar', '-zcvf', 'foo.tar', '/tmp/something-to-compress-1',
|
|
'/tmp/something-to-compress-2'],
|
|
runas=None, python_shell=False, template=None, cwd=None
|
|
)
|
|
|
|
def test_tar_raises_exception_if_not_found(self):
|
|
with patch('salt.utils.path.which', lambda exe: None):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
self.assertRaises(
|
|
CommandNotFoundError,
|
|
archive.tar,
|
|
'zxvf',
|
|
'foo.tar',
|
|
'/tmp/something-to-compress'
|
|
)
|
|
self.assertFalse(mock.called)
|
|
|
|
def test_gzip(self):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
with patch('salt.utils.path.which', lambda exe: exe):
|
|
ret = archive.gzip('/tmp/something-to-compress')
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['gzip', '/tmp/something-to-compress'],
|
|
runas=None, python_shell=False, template=None
|
|
)
|
|
|
|
def test_gzip_raises_exception_if_not_found(self):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
with patch('salt.utils.path.which', lambda exe: None):
|
|
self.assertRaises(
|
|
CommandNotFoundError,
|
|
archive.gzip, '/tmp/something-to-compress'
|
|
)
|
|
self.assertFalse(mock.called)
|
|
|
|
def test_gunzip(self):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
with patch('salt.utils.path.which', lambda exe: exe):
|
|
ret = archive.gunzip('/tmp/something-to-decompress.tar.gz')
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['gunzip', '/tmp/something-to-decompress.tar.gz'],
|
|
runas=None, python_shell=False, template=None
|
|
)
|
|
|
|
def test_gunzip_raises_exception_if_not_found(self):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
with patch('salt.utils.path.which', lambda exe: None):
|
|
self.assertRaises(
|
|
CommandNotFoundError,
|
|
archive.gunzip,
|
|
'/tmp/something-to-decompress.tar.gz'
|
|
)
|
|
self.assertFalse(mock.called)
|
|
|
|
def test_cmd_zip(self):
|
|
with patch('glob.glob', lambda pathname: [pathname]):
|
|
with patch('salt.utils.path.which', lambda exe: exe):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.cmd_zip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/tmpePe8yO,/tmp/tmpLeSw1A',
|
|
template='jinja'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['zip', '-r', '/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
runas=None, python_shell=False, template='jinja', cwd=None
|
|
)
|
|
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.cmd_zip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
['/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
template='jinja'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['zip', '-r', '/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
runas=None, python_shell=False, template='jinja', cwd=None
|
|
)
|
|
|
|
def test_zip(self):
|
|
with patch('glob.glob', lambda pathname: [pathname]):
|
|
with patch.multiple(os.path, **{'isdir': MagicMock(return_value=False),
|
|
'exists': MagicMock(return_value=True)}):
|
|
with patch('zipfile.ZipFile', MagicMock()):
|
|
ret = archive.zip_(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/tmpePe8yO,/tmp/tmpLeSw1A',
|
|
template='jinja'
|
|
)
|
|
expected = [os.path.join('tmp', 'tmpePe8yO'),
|
|
os.path.join('tmp', 'tmpLeSw1A')]
|
|
self.assertEqual(expected, ret)
|
|
|
|
def test_zip_raises_exception_if_not_found(self):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
with patch('salt.utils.path.which', lambda exe: None):
|
|
self.assertRaises(
|
|
CommandNotFoundError,
|
|
archive.cmd_zip,
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/tmpePe8yO,/tmp/tmpLeSw1A',
|
|
template='jinja',
|
|
)
|
|
self.assertFalse(mock.called)
|
|
|
|
def test_cmd_unzip(self):
|
|
def _get_mock():
|
|
'''
|
|
Create a new MagicMock for each scenario in this test, so that
|
|
assert_called_once_with doesn't complain that the same mock object
|
|
is called more than once.
|
|
'''
|
|
return MagicMock(return_value={'stdout': 'salt',
|
|
'stderr': '',
|
|
'pid': 12345,
|
|
'retcode': 0})
|
|
|
|
with patch('salt.utils.path.which', lambda exe: exe):
|
|
|
|
mock = _get_mock()
|
|
with patch.dict(archive.__salt__, {'cmd.run_all': mock}):
|
|
ret = archive.cmd_unzip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/dest',
|
|
excludes='/tmp/tmpePe8yO,/tmp/tmpLeSw1A',
|
|
template='jinja'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['unzip', '/tmp/salt.{{grains.id}}.zip', '-d', '/tmp/dest',
|
|
'-x', '/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
output_loglevel='debug',
|
|
python_shell=False,
|
|
redirect_stderr=True,
|
|
runas=None,
|
|
template='jinja'
|
|
)
|
|
|
|
mock = _get_mock()
|
|
with patch.dict(archive.__salt__, {'cmd.run_all': mock}):
|
|
ret = archive.cmd_unzip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/dest',
|
|
excludes=['/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
template='jinja'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['unzip', '/tmp/salt.{{grains.id}}.zip', '-d', '/tmp/dest',
|
|
'-x', '/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
output_loglevel='debug',
|
|
python_shell=False,
|
|
redirect_stderr=True,
|
|
runas=None,
|
|
template='jinja'
|
|
)
|
|
|
|
mock = _get_mock()
|
|
with patch.dict(archive.__salt__, {'cmd.run_all': mock}):
|
|
ret = archive.cmd_unzip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/dest',
|
|
excludes='/tmp/tmpePe8yO,/tmp/tmpLeSw1A',
|
|
template='jinja',
|
|
options='-fo'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['unzip', '-fo', '/tmp/salt.{{grains.id}}.zip', '-d',
|
|
'/tmp/dest', '-x', '/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
output_loglevel='debug',
|
|
python_shell=False,
|
|
redirect_stderr=True,
|
|
runas=None,
|
|
template='jinja'
|
|
)
|
|
|
|
mock = _get_mock()
|
|
with patch.dict(archive.__salt__, {'cmd.run_all': mock}):
|
|
ret = archive.cmd_unzip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/dest',
|
|
excludes=['/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
template='jinja',
|
|
options='-fo'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['unzip', '-fo', '/tmp/salt.{{grains.id}}.zip', '-d',
|
|
'/tmp/dest', '-x', '/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
output_loglevel='debug',
|
|
python_shell=False,
|
|
redirect_stderr=True,
|
|
runas=None,
|
|
template='jinja'
|
|
)
|
|
|
|
mock = _get_mock()
|
|
with patch.dict(archive.__salt__, {'cmd.run_all': mock}):
|
|
ret = archive.cmd_unzip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/dest',
|
|
excludes=['/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
template='jinja',
|
|
options='-fo',
|
|
password='asdf',
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['unzip', '-P', 'asdf', '-fo', '/tmp/salt.{{grains.id}}.zip',
|
|
'-d', '/tmp/dest', '-x', '/tmp/tmpePe8yO', '/tmp/tmpLeSw1A'],
|
|
output_loglevel='quiet',
|
|
python_shell=False,
|
|
redirect_stderr=True,
|
|
runas=None,
|
|
template='jinja'
|
|
)
|
|
|
|
def test_unzip(self):
|
|
mock = ZipFileMock()
|
|
with patch('zipfile.ZipFile', mock):
|
|
ret = archive.unzip(
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/dest',
|
|
excludes='/tmp/tmpePe8yO,/tmp/tmpLeSw1A',
|
|
template='jinja',
|
|
extract_perms=False
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
|
|
def test_unzip_raises_exception_if_not_found(self):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
with patch('salt.utils.path.which', lambda exe: None):
|
|
self.assertRaises(
|
|
CommandNotFoundError,
|
|
archive.cmd_unzip,
|
|
'/tmp/salt.{{grains.id}}.zip',
|
|
'/tmp/dest',
|
|
excludes='/tmp/tmpePe8yO,/tmp/tmpLeSw1A',
|
|
template='jinja',
|
|
)
|
|
self.assertFalse(mock.called)
|
|
|
|
def test_rar(self):
|
|
with patch('glob.glob', lambda pathname: [pathname]):
|
|
with patch('salt.utils.path.which', lambda exe: exe):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.rar(
|
|
'/tmp/rarfile.rar',
|
|
'/tmp/sourcefile1,/tmp/sourcefile2'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['rar', 'a', '-idp', '/tmp/rarfile.rar',
|
|
'/tmp/sourcefile1', '/tmp/sourcefile2'],
|
|
runas=None, python_shell=False, template=None, cwd=None
|
|
)
|
|
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.rar(
|
|
'/tmp/rarfile.rar',
|
|
['/tmp/sourcefile1', '/tmp/sourcefile2']
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['rar', 'a', '-idp', '/tmp/rarfile.rar',
|
|
'/tmp/sourcefile1', '/tmp/sourcefile2'],
|
|
runas=None, python_shell=False, template=None, cwd=None
|
|
)
|
|
|
|
def test_rar_raises_exception_if_not_found(self):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
with patch('salt.utils.path.which', lambda exe: None):
|
|
self.assertRaises(
|
|
CommandNotFoundError,
|
|
archive.rar,
|
|
'/tmp/rarfile.rar',
|
|
'/tmp/sourcefile1,/tmp/sourcefile2'
|
|
)
|
|
self.assertFalse(mock.called)
|
|
|
|
@skipIf(salt.utils.path.which_bin(('unrar', 'rar')) is None, 'unrar not installed')
|
|
def test_unrar(self):
|
|
with patch('salt.utils.path.which_bin', lambda exe: exe[0] if isinstance(exe, (list, tuple)) else exe):
|
|
with patch('salt.utils.path.which', lambda exe: exe[0] if isinstance(exe, (list, tuple)) else exe):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.unrar(
|
|
'/tmp/rarfile.rar',
|
|
'/home/strongbad/',
|
|
excludes='file_1,file_2'
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['unrar', 'x', '-idp', '/tmp/rarfile.rar',
|
|
'-x', 'file_1', '-x', 'file_2', '/home/strongbad/'],
|
|
runas=None, python_shell=False, template=None
|
|
)
|
|
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
ret = archive.unrar(
|
|
'/tmp/rarfile.rar',
|
|
'/home/strongbad/',
|
|
excludes=['file_1', 'file_2']
|
|
)
|
|
self.assertEqual(['salt'], ret)
|
|
mock.assert_called_once_with(
|
|
['unrar', 'x', '-idp', '/tmp/rarfile.rar',
|
|
'-x', 'file_1', '-x', 'file_2', '/home/strongbad/'],
|
|
runas=None, python_shell=False, template=None
|
|
)
|
|
|
|
def test_unrar_raises_exception_if_not_found(self):
|
|
with patch('salt.utils.path.which_bin', lambda exe: None):
|
|
mock = MagicMock(return_value='salt')
|
|
with patch.dict(archive.__salt__, {'cmd.run': mock}):
|
|
self.assertRaises(
|
|
CommandNotFoundError,
|
|
archive.unrar,
|
|
'/tmp/rarfile.rar',
|
|
'/home/strongbad/',
|
|
)
|
|
self.assertFalse(mock.called)
|
|
|
|
def test_trim_files(self):
|
|
with patch('salt.utils.path.which_bin', lambda exe: exe):
|
|
source = 'file.tar.gz'
|
|
tmp_dir = 'temp'
|
|
files = [
|
|
'\n'.join(['file1'] * 200),
|
|
'\n'.join(['file2'] * 200),
|
|
'this\nthat\nother',
|
|
'this\nthat\nother',
|
|
'this\nthat\nother'
|
|
]
|
|
trim_opt = [
|
|
True,
|
|
False,
|
|
3,
|
|
1,
|
|
0
|
|
]
|
|
trim_100 = (['file1'] * 100)
|
|
trim_100.append("List trimmed after 100 files.")
|
|
expected = [
|
|
trim_100,
|
|
['file2'] * 200,
|
|
['this', 'that', 'other'],
|
|
['this', 'List trimmed after 1 files.'],
|
|
['List trimmed after 0 files.'],
|
|
]
|
|
|
|
for test_files, test_trim_opts, test_expected in zip(files, trim_opt, expected):
|
|
with patch.dict(archive.__salt__, {'cmd.run': MagicMock(return_value=test_files)}):
|
|
ret = archive.unrar(source, tmp_dir, trim_output=test_trim_opts)
|
|
self.assertEqual(ret, test_expected)
|