2017-03-27 15:25:09 +00:00
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
'''
|
2018-05-28 21:13:12 +00:00
|
|
|
:codeauthor: Mike Place <mp@saltstack.com>
|
2017-03-27 15:25:09 +00:00
|
|
|
'''
|
|
|
|
|
|
|
|
# Import Python libs
|
2018-01-10 04:34:27 +00:00
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import copy
|
2017-03-27 15:25:09 +00:00
|
|
|
import os
|
|
|
|
import tempfile
|
|
|
|
|
|
|
|
# Import Salt Testing libs
|
2017-04-02 15:21:06 +00:00
|
|
|
from tests.integration import AdaptedConfigurationTestCaseMixin
|
2017-03-27 15:25:09 +00:00
|
|
|
from tests.support.mixins import LoaderModuleMockMixin
|
2018-07-27 14:16:59 +00:00
|
|
|
from tests.support.paths import BASE_FILES, TMP, TMP_STATE_TREE
|
2017-03-27 15:25:09 +00:00
|
|
|
from tests.support.unit import TestCase, skipIf
|
|
|
|
from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON
|
|
|
|
|
Use explicit unicode strings + break up salt.utils
This PR is part of what will be an ongoing effort to use explicit
unicode strings in Salt. Because Python 3 does not suport Python 2's raw
unicode string syntax (i.e. `ur'\d+'`), we must use
`salt.utils.locales.sdecode()` to ensure that the raw string is unicode.
However, because of how `salt/utils/__init__.py` has evolved into the
hulking monstrosity it is today, this means importing a large module in
places where it is not needed, which could negatively impact
performance. For this reason, this PR also breaks out some of the
functions from `salt/utils/__init__.py` into new/existing modules under
`salt/utils/`. The long term goal will be that the modules within this
directory do not depend on importing `salt.utils`.
A summary of the changes in this PR is as follows:
* Moves the following functions from `salt.utils` to new locations
(including a deprecation warning if invoked from `salt.utils`):
`to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`,
`dequote`, `is_hex`, `is_bin_str`, `rand_string`,
`contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`,
`which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`,
`is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`,
`is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`,
`is_openbsd`, `is_aix`
* Moves the functions already deprecated by @rallytime to the bottom of
`salt/utils/__init__.py` for better organization, so we can keep the
deprecated ones separate from the ones yet to be deprecated as we
continue to break up `salt.utils`
* Updates `salt/*.py` and all files under `salt/client/` to use explicit
unicode string literals.
* Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils
import foo` becomes `import salt.utils.foo as foo`).
* Renames the `test.rand_str` function to `test.random_hash` to more
accurately reflect what it does
* Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`)
such that it returns a string matching the passed size. Previously
this function would get `size` bytes from `os.urandom()`,
base64-encode it, and return the result, which would in most cases not
be equal to the passed size.
2017-07-25 01:47:15 +00:00
|
|
|
# Import Salt libs
|
2018-01-10 04:34:27 +00:00
|
|
|
import salt.fileserver.roots as roots
|
Use explicit unicode strings + break up salt.utils
This PR is part of what will be an ongoing effort to use explicit
unicode strings in Salt. Because Python 3 does not suport Python 2's raw
unicode string syntax (i.e. `ur'\d+'`), we must use
`salt.utils.locales.sdecode()` to ensure that the raw string is unicode.
However, because of how `salt/utils/__init__.py` has evolved into the
hulking monstrosity it is today, this means importing a large module in
places where it is not needed, which could negatively impact
performance. For this reason, this PR also breaks out some of the
functions from `salt/utils/__init__.py` into new/existing modules under
`salt/utils/`. The long term goal will be that the modules within this
directory do not depend on importing `salt.utils`.
A summary of the changes in this PR is as follows:
* Moves the following functions from `salt.utils` to new locations
(including a deprecation warning if invoked from `salt.utils`):
`to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`,
`dequote`, `is_hex`, `is_bin_str`, `rand_string`,
`contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`,
`which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`,
`is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`,
`is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`,
`is_openbsd`, `is_aix`
* Moves the functions already deprecated by @rallytime to the bottom of
`salt/utils/__init__.py` for better organization, so we can keep the
deprecated ones separate from the ones yet to be deprecated as we
continue to break up `salt.utils`
* Updates `salt/*.py` and all files under `salt/client/` to use explicit
unicode string literals.
* Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils
import foo` becomes `import salt.utils.foo as foo`).
* Renames the `test.rand_str` function to `test.random_hash` to more
accurately reflect what it does
* Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`)
such that it returns a string matching the passed size. Previously
this function would get `size` bytes from `os.urandom()`,
base64-encode it, and return the result, which would in most cases not
be equal to the passed size.
2017-07-25 01:47:15 +00:00
|
|
|
import salt.fileclient
|
2017-07-18 16:31:01 +00:00
|
|
|
import salt.utils.files
|
2018-07-27 14:16:59 +00:00
|
|
|
import salt.utils.hashutils
|
Use explicit unicode strings + break up salt.utils
This PR is part of what will be an ongoing effort to use explicit
unicode strings in Salt. Because Python 3 does not suport Python 2's raw
unicode string syntax (i.e. `ur'\d+'`), we must use
`salt.utils.locales.sdecode()` to ensure that the raw string is unicode.
However, because of how `salt/utils/__init__.py` has evolved into the
hulking monstrosity it is today, this means importing a large module in
places where it is not needed, which could negatively impact
performance. For this reason, this PR also breaks out some of the
functions from `salt/utils/__init__.py` into new/existing modules under
`salt/utils/`. The long term goal will be that the modules within this
directory do not depend on importing `salt.utils`.
A summary of the changes in this PR is as follows:
* Moves the following functions from `salt.utils` to new locations
(including a deprecation warning if invoked from `salt.utils`):
`to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`,
`dequote`, `is_hex`, `is_bin_str`, `rand_string`,
`contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`,
`which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`,
`is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`,
`is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`,
`is_openbsd`, `is_aix`
* Moves the functions already deprecated by @rallytime to the bottom of
`salt/utils/__init__.py` for better organization, so we can keep the
deprecated ones separate from the ones yet to be deprecated as we
continue to break up `salt.utils`
* Updates `salt/*.py` and all files under `salt/client/` to use explicit
unicode string literals.
* Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils
import foo` becomes `import salt.utils.foo as foo`).
* Renames the `test.rand_str` function to `test.random_hash` to more
accurately reflect what it does
* Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`)
such that it returns a string matching the passed size. Previously
this function would get `size` bytes from `os.urandom()`,
base64-encode it, and return the result, which would in most cases not
be equal to the passed size.
2017-07-25 01:47:15 +00:00
|
|
|
import salt.utils.platform
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
try:
|
|
|
|
import win32file
|
|
|
|
except ImportError:
|
|
|
|
pass
|
|
|
|
|
2018-01-10 04:34:27 +00:00
|
|
|
UNICODE_FILENAME = 'питон.txt'
|
|
|
|
UNICODE_DIRNAME = UNICODE_ENVNAME = 'соль'
|
|
|
|
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
@skipIf(NO_MOCK, NO_MOCK_REASON)
|
2017-04-02 15:21:06 +00:00
|
|
|
class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMixin):
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
def setup_loader_modules(self):
|
|
|
|
self.tmp_cachedir = tempfile.mkdtemp(dir=TMP)
|
2017-04-01 17:55:48 +00:00
|
|
|
self.opts = self.get_temp_config('master')
|
2017-03-27 15:25:09 +00:00
|
|
|
self.opts['cachedir'] = self.tmp_cachedir
|
|
|
|
empty_dir = os.path.join(TMP_STATE_TREE, 'empty_dir')
|
|
|
|
if not os.path.isdir(empty_dir):
|
|
|
|
os.makedirs(empty_dir)
|
2018-01-10 04:34:27 +00:00
|
|
|
return {roots: {'__opts__': self.opts}}
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def setUpClass(cls):
|
|
|
|
'''
|
|
|
|
Create special file_roots for symlink test on Windows
|
|
|
|
'''
|
Use explicit unicode strings + break up salt.utils
This PR is part of what will be an ongoing effort to use explicit
unicode strings in Salt. Because Python 3 does not suport Python 2's raw
unicode string syntax (i.e. `ur'\d+'`), we must use
`salt.utils.locales.sdecode()` to ensure that the raw string is unicode.
However, because of how `salt/utils/__init__.py` has evolved into the
hulking monstrosity it is today, this means importing a large module in
places where it is not needed, which could negatively impact
performance. For this reason, this PR also breaks out some of the
functions from `salt/utils/__init__.py` into new/existing modules under
`salt/utils/`. The long term goal will be that the modules within this
directory do not depend on importing `salt.utils`.
A summary of the changes in this PR is as follows:
* Moves the following functions from `salt.utils` to new locations
(including a deprecation warning if invoked from `salt.utils`):
`to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`,
`dequote`, `is_hex`, `is_bin_str`, `rand_string`,
`contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`,
`which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`,
`is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`,
`is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`,
`is_openbsd`, `is_aix`
* Moves the functions already deprecated by @rallytime to the bottom of
`salt/utils/__init__.py` for better organization, so we can keep the
deprecated ones separate from the ones yet to be deprecated as we
continue to break up `salt.utils`
* Updates `salt/*.py` and all files under `salt/client/` to use explicit
unicode string literals.
* Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils
import foo` becomes `import salt.utils.foo as foo`).
* Renames the `test.rand_str` function to `test.random_hash` to more
accurately reflect what it does
* Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`)
such that it returns a string matching the passed size. Previously
this function would get `size` bytes from `os.urandom()`,
base64-encode it, and return the result, which would in most cases not
be equal to the passed size.
2017-07-25 01:47:15 +00:00
|
|
|
if salt.utils.platform.is_windows():
|
2017-03-27 15:25:09 +00:00
|
|
|
root_dir = tempfile.mkdtemp(dir=TMP)
|
|
|
|
source_sym = os.path.join(root_dir, 'source_sym')
|
2017-07-18 16:31:01 +00:00
|
|
|
with salt.utils.files.fopen(source_sym, 'w') as fp_:
|
2017-03-27 15:25:09 +00:00
|
|
|
fp_.write('hello world!\n')
|
|
|
|
cwd = os.getcwd()
|
|
|
|
try:
|
|
|
|
os.chdir(root_dir)
|
|
|
|
win32file.CreateSymbolicLink('dest_sym', 'source_sym', 0)
|
|
|
|
finally:
|
|
|
|
os.chdir(cwd)
|
|
|
|
cls.test_symlink_list_file_roots = {'base': [root_dir]}
|
|
|
|
else:
|
|
|
|
cls.test_symlink_list_file_roots = None
|
2018-04-13 17:10:35 +00:00
|
|
|
cls.tmp_dir = tempfile.mkdtemp(dir=TMP)
|
2018-07-27 14:16:59 +00:00
|
|
|
full_path_to_file = os.path.join(BASE_FILES, 'testfile')
|
2018-04-17 15:02:38 +00:00
|
|
|
with salt.utils.files.fopen(full_path_to_file, 'rb') as s_fp:
|
|
|
|
with salt.utils.files.fopen(os.path.join(cls.tmp_dir, 'testfile'), 'wb') as d_fp:
|
2018-04-13 17:10:35 +00:00
|
|
|
for line in s_fp:
|
2018-08-22 03:41:15 +00:00
|
|
|
d_fp.write(line)
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def tearDownClass(cls):
|
|
|
|
'''
|
|
|
|
Remove special file_roots for symlink test
|
|
|
|
'''
|
Use explicit unicode strings + break up salt.utils
This PR is part of what will be an ongoing effort to use explicit
unicode strings in Salt. Because Python 3 does not suport Python 2's raw
unicode string syntax (i.e. `ur'\d+'`), we must use
`salt.utils.locales.sdecode()` to ensure that the raw string is unicode.
However, because of how `salt/utils/__init__.py` has evolved into the
hulking monstrosity it is today, this means importing a large module in
places where it is not needed, which could negatively impact
performance. For this reason, this PR also breaks out some of the
functions from `salt/utils/__init__.py` into new/existing modules under
`salt/utils/`. The long term goal will be that the modules within this
directory do not depend on importing `salt.utils`.
A summary of the changes in this PR is as follows:
* Moves the following functions from `salt.utils` to new locations
(including a deprecation warning if invoked from `salt.utils`):
`to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`,
`dequote`, `is_hex`, `is_bin_str`, `rand_string`,
`contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`,
`which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`,
`is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`,
`is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`,
`is_openbsd`, `is_aix`
* Moves the functions already deprecated by @rallytime to the bottom of
`salt/utils/__init__.py` for better organization, so we can keep the
deprecated ones separate from the ones yet to be deprecated as we
continue to break up `salt.utils`
* Updates `salt/*.py` and all files under `salt/client/` to use explicit
unicode string literals.
* Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils
import foo` becomes `import salt.utils.foo as foo`).
* Renames the `test.rand_str` function to `test.random_hash` to more
accurately reflect what it does
* Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`)
such that it returns a string matching the passed size. Previously
this function would get `size` bytes from `os.urandom()`,
base64-encode it, and return the result, which would in most cases not
be equal to the passed size.
2017-07-25 01:47:15 +00:00
|
|
|
if salt.utils.platform.is_windows():
|
2017-03-27 15:25:09 +00:00
|
|
|
try:
|
2017-07-18 16:48:54 +00:00
|
|
|
salt.utils.files.rm_rf(cls.test_symlink_list_file_roots['base'][0])
|
2017-03-27 15:25:09 +00:00
|
|
|
except OSError:
|
|
|
|
pass
|
2018-04-17 15:02:38 +00:00
|
|
|
salt.utils.files.rm_rf(cls.tmp_dir)
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
def tearDown(self):
|
|
|
|
del self.opts
|
|
|
|
|
|
|
|
def test_file_list(self):
|
2018-01-10 04:34:27 +00:00
|
|
|
ret = roots.file_list({'saltenv': 'base'})
|
2017-03-27 15:25:09 +00:00
|
|
|
self.assertIn('testfile', ret)
|
2018-01-10 04:34:27 +00:00
|
|
|
self.assertIn(UNICODE_FILENAME, ret)
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
def test_find_file(self):
|
2018-01-10 04:34:27 +00:00
|
|
|
ret = roots.find_file('testfile')
|
2017-03-27 15:25:09 +00:00
|
|
|
self.assertEqual('testfile', ret['rel'])
|
|
|
|
|
2018-07-27 14:16:59 +00:00
|
|
|
full_path_to_file = os.path.join(BASE_FILES, 'testfile')
|
2017-03-27 15:25:09 +00:00
|
|
|
self.assertEqual(full_path_to_file, ret['path'])
|
|
|
|
|
|
|
|
def test_serve_file(self):
|
2018-01-10 04:34:27 +00:00
|
|
|
with patch.dict(roots.__opts__, {'file_buffer_size': 262144}):
|
2017-03-27 15:25:09 +00:00
|
|
|
load = {'saltenv': 'base',
|
2018-04-13 17:10:35 +00:00
|
|
|
'path': os.path.join(self.tmp_dir, 'testfile'),
|
2017-03-27 15:25:09 +00:00
|
|
|
'loc': 0
|
|
|
|
}
|
2018-04-13 17:10:35 +00:00
|
|
|
fnd = {'path': os.path.join(self.tmp_dir, 'testfile'),
|
2017-03-27 15:25:09 +00:00
|
|
|
'rel': 'testfile'}
|
2018-01-10 04:34:27 +00:00
|
|
|
ret = roots.serve_file(load, fnd)
|
2017-03-27 15:25:09 +00:00
|
|
|
|
2018-07-27 14:16:59 +00:00
|
|
|
with salt.utils.files.fopen(
|
|
|
|
os.path.join(BASE_FILES, 'testfile'), 'rb') as fp_:
|
|
|
|
data = fp_.read()
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
self.assertDictEqual(
|
|
|
|
ret,
|
|
|
|
{'data': data,
|
|
|
|
'dest': 'testfile'})
|
|
|
|
|
2018-01-10 04:34:27 +00:00
|
|
|
def test_envs(self):
|
|
|
|
opts = {'file_roots': copy.copy(self.opts['file_roots'])}
|
|
|
|
opts['file_roots'][UNICODE_ENVNAME] = opts['file_roots']['base']
|
|
|
|
with patch.dict(roots.__opts__, opts):
|
|
|
|
ret = roots.envs()
|
|
|
|
self.assertIn('base', ret)
|
|
|
|
self.assertIn(UNICODE_ENVNAME, ret)
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
def test_file_hash(self):
|
|
|
|
load = {
|
|
|
|
'saltenv': 'base',
|
2018-04-13 17:10:35 +00:00
|
|
|
'path': os.path.join(self.tmp_dir, 'testfile'),
|
2017-03-27 15:25:09 +00:00
|
|
|
}
|
|
|
|
fnd = {
|
2018-04-13 17:10:35 +00:00
|
|
|
'path': os.path.join(self.tmp_dir, 'testfile'),
|
2017-03-27 15:25:09 +00:00
|
|
|
'rel': 'testfile'
|
|
|
|
}
|
2018-01-10 04:34:27 +00:00
|
|
|
ret = roots.file_hash(load, fnd)
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
# Hashes are different in Windows. May be how git translates line
|
|
|
|
# endings
|
2018-07-27 14:16:59 +00:00
|
|
|
with salt.utils.files.fopen(
|
|
|
|
os.path.join(BASE_FILES, 'testfile'), 'rb') as fp_:
|
|
|
|
hsum = salt.utils.hashutils.sha256_digest(fp_.read())
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
self.assertDictEqual(
|
|
|
|
ret,
|
|
|
|
{
|
|
|
|
'hsum': hsum,
|
|
|
|
'hash_type': 'sha256'
|
|
|
|
}
|
|
|
|
)
|
|
|
|
|
|
|
|
def test_file_list_emptydirs(self):
|
2018-01-10 04:34:27 +00:00
|
|
|
ret = roots.file_list_emptydirs({'saltenv': 'base'})
|
2017-03-27 15:25:09 +00:00
|
|
|
self.assertIn('empty_dir', ret)
|
|
|
|
|
|
|
|
def test_dir_list(self):
|
2018-01-10 04:34:27 +00:00
|
|
|
ret = roots.dir_list({'saltenv': 'base'})
|
2017-03-27 15:25:09 +00:00
|
|
|
self.assertIn('empty_dir', ret)
|
2018-01-10 04:34:27 +00:00
|
|
|
self.assertIn(UNICODE_DIRNAME, ret)
|
2017-03-27 15:25:09 +00:00
|
|
|
|
|
|
|
def test_symlink_list(self):
|
2018-06-13 14:58:29 +00:00
|
|
|
orig_file_roots = self.opts['file_roots']
|
|
|
|
try:
|
|
|
|
if self.test_symlink_list_file_roots:
|
|
|
|
self.opts['file_roots'] = self.test_symlink_list_file_roots
|
|
|
|
ret = roots.symlink_list({'saltenv': 'base'})
|
|
|
|
self.assertDictEqual(ret, {'dest_sym': 'source_sym'})
|
|
|
|
finally:
|
|
|
|
if self.test_symlink_list_file_roots:
|
|
|
|
self.opts['file_roots'] = orig_file_roots
|