Merge branch '2018.3' into 'develop'

Conflicts:
  - salt/modules/nilrt_ip.py
  - salt/states/pip_state.py
  - tests/unit/modules/test_file.py
This commit is contained in:
rallytime 2018-08-03 09:27:20 -04:00
commit a8b30fed9f
No known key found for this signature in database
GPG Key ID: E8F1A4B90D0DEA19
18 changed files with 330 additions and 153 deletions

View File

@ -5106,6 +5106,21 @@ https://github.com/ytoolshed/range/wiki/%22yamlfile%22-module-file-spec
Include Configuration
=====================
Configuration can be loaded from multiple files. The order in which this is
done is:
1. The master config file itself
2. The files matching the glob in :conf_master:`default_include`
3. The files matching the glob in :conf_master:`include` (if defined)
Each successive step overrides any values defined in the previous steps.
Therefore, any config options defined in one of the
:conf_master:`default_include` files would override the same value in the
master config file, and any options defined in :conf_master:`include` would
override both.
.. conf_master:: default_include
``default_include``

View File

@ -3025,7 +3025,22 @@ at the moment a single state fails
Include Configuration
=====================
.. conf_minion:: include
Configuration can be loaded from multiple files. The order in which this is
done is:
1. The minion config file itself
2. The files matching the glob in :conf_minion:`default_include`
3. The files matching the glob in :conf_minion:`include` (if defined)
Each successive step overrides any values defined in the previous steps.
Therefore, any config options defined in one of the
:conf_minion:`default_include` files would override the same value in the
minion config file, and any options defined in :conf_minion:`include` would
override both.
.. conf_minion:: default_include
``default_include``
-------------------
@ -3043,6 +3058,7 @@ file.
files are prefixed with an underscore. A common example of this is the
``_schedule.conf`` file.
.. conf_minion:: include
``include``
-----------

View File

@ -422,7 +422,7 @@ def create(vm_):
else:
raise SaltCloudExecutionFailure("Disk type '{0}' not supported".format(disk_type))
clone_xml = ElementTree.tostring(domain_xml)
clone_xml = salt.utils.stringutils.to_str(ElementTree.tostring(domain_xml))
log.debug("Clone XML '%s'", clone_xml)
validate_flags = libvirt.VIR_DOMAIN_DEFINE_VALIDATE if validate_xml else 0
@ -615,7 +615,7 @@ def create_volume_xml(volume):
log.debug("Volume: %s", dir(volume))
volume_xml.find('capacity').text = six.text_type(volume.info()[1])
volume_xml.find('./target/path').text = volume.path()
xml_string = ElementTree.tostring(volume_xml)
xml_string = salt.utils.stringutils.to_str(ElementTree.tostring(volume_xml))
log.debug("Creating %s", xml_string)
return xml_string
@ -641,7 +641,7 @@ def create_volume_with_backing_store_xml(volume):
log.debug("volume: %s", dir(volume))
volume_xml.find('capacity').text = six.text_type(volume.info()[1])
volume_xml.find('./backingStore/path').text = volume.path()
xml_string = ElementTree.tostring(volume_xml)
xml_string = salt.utils.stringutils.to_str(ElementTree.tostring(volume_xml))
log.debug("Creating %s", xml_string)
return xml_string

View File

@ -379,20 +379,14 @@ def __within(within=None, errmsg=None, dtype=None):
def __space_delimited_list(value):
'''validate that a value contains one or more space-delimited values'''
valid, _value, errmsg = False, value, 'space-delimited string'
try:
if hasattr(value, '__iter__'):
valid = True # TODO:
else:
_value = value.split()
if _value == []:
raise ValueError
valid = True
except AttributeError:
pass
except ValueError:
pass
return (valid, _value, errmsg)
if isinstance(value, str):
value = value.strip().split()
if hasattr(value, '__iter__') and value != []:
return (True, value, 'space-delimited string')
else:
return (False, value, '{0} is not a valid space-delimited value.\n'.format(value))
SALT_ATTR_TO_DEBIAN_ATTR_MAP = {
'dns': 'dns-nameservers',

View File

@ -6216,6 +6216,16 @@ def grep(path,
'''
path = os.path.expanduser(path)
# Backup the path in case the glob returns nothing
_path = path
path = glob.glob(path)
# If the list is empty no files exist
# so we revert back to the original path
# so the result is an error.
if not path:
path = _path
split_opts = []
for opt in opts:
try:
@ -6230,7 +6240,10 @@ def grep(path,
)
split_opts.extend(split)
cmd = ['grep'] + split_opts + [pattern, path]
if isinstance(path, list):
cmd = ['grep'] + split_opts + [pattern] + path
else:
cmd = ['grep'] + split_opts + [pattern, path]
try:
ret = __salt__['cmd.run_all'](cmd, python_shell=False)
except (IOError, OSError) as exc:

View File

@ -130,20 +130,16 @@ def _space_delimited_list(value):
'''
validate that a value contains one or more space-delimited values
'''
valid, _value, errmsg = False, value, 'space-delimited string'
try:
if hasattr(value, '__iter__'):
valid = True
else:
_value = value.split()
if not _value:
raise ValueError
valid = True
except AttributeError:
errmsg = '{0} is not a valid list.\n'.format(value)
except ValueError:
errmsg = '{0} is not a valid list.\n'.format(value)
return valid, errmsg
if isinstance(value, str):
items = value.split(' ')
valid = items and all(items)
else:
valid = hasattr(value, '__iter__') and (value != [])
if valid:
return (True, 'space-delimited string')
else:
return (False, '{0} is not a valid list.\n'.format(value))
def _validate_ipv4(value):

View File

@ -173,13 +173,15 @@ def renderer(path=None, string=None, default_renderer='jinja|yaml', **kwargs):
path_or_string = ':string:'
kwargs['input_data'] = string
return salt.template.compile_template(
path_or_string,
renderers,
default_renderer,
__opts__['renderer_blacklist'],
__opts__['renderer_whitelist'],
**kwargs)
ret = salt.template.compile_template(
path_or_string,
renderers,
default_renderer,
__opts__['renderer_blacklist'],
__opts__['renderer_whitelist'],
**kwargs
)
return ret.read() if __utils__['stringio.is_readable'](ret) else ret
def _get_serialize_fn(serializer, fn_name):

View File

@ -3899,6 +3899,8 @@ def retention_schedule(name, retain, strptime_format=None, timezone=None):
return (None, None)
def get_file_time_from_mtime(f):
if f == '.' or f == '..':
return (None, None)
lstat = __salt__['file.lstat'](os.path.join(name, f))
if lstat:
mtime = lstat['st_mtime']

View File

@ -194,16 +194,32 @@ def _check_if_installed(prefix,
env_vars,
index_url,
extra_index_url,
pip_list=False,
**kwargs):
# result: None means the command failed to run
# result: True means the package is installed
# result: False means the package is not installed
'''
Takes a package name and version specification (if any) and checks it is
installed
Keyword arguments include:
pip_list: optional dict of installed pip packages, and their versions,
to search through to check if the package is installed. If not
provided, one will be generated in this function by querying the
system.
Returns:
result: None means the command failed to run
result: True means the package is installed
result: False means the package is not installed
'''
ret = {'result': False, 'comment': None}
# If we are not passed a pip list, get one:
if not pip_list:
pip_list = __salt__['pip.list'](prefix, bin_env=bin_env,
user=user, cwd=cwd,
env_vars=env_vars, **kwargs)
# Check if the requested package is already installed.
pip_list = __salt__['pip.list'](prefix, bin_env=bin_env,
user=user, cwd=cwd,
env_vars=env_vars, **kwargs)
prefix_realname = _find_key(prefix, pip_list)
# If the package was already installed, check
@ -718,6 +734,14 @@ def installed(name,
# No requirements case.
# Check pre-existence of the requested packages.
else:
# Attempt to pre-cache a the current pip list
try:
pip_list = __salt__['pip.list'](bin_env=bin_env, user=user, cwd=cwd)
# If we fail, then just send False, and we'll try again in the next function call
except Exception as exc:
log.exception(exc)
pip_list = False
for prefix, state_pkg_name, version_spec in pkgs_details:
if prefix:
@ -726,7 +750,8 @@ def installed(name,
out = _check_if_installed(prefix, state_pkg_name, version_spec,
ignore_installed, force_reinstall,
upgrade, user, cwd, bin_env, env_vars,
index_url, extra_index_url, **kwargs)
index_url, extra_index_url, pip_list,
**kwargs)
# If _check_if_installed result is None, something went wrong with
# the command running. This way we keep stateful output.
if out['result'] is None:

View File

@ -1010,8 +1010,8 @@ def installed(
In version 2015.8.9, an **ignore_epoch** argument has been added to
:py:mod:`pkg.installed <salt.states.pkg.installed>`,
:py:mod:`pkg.removed <salt.states.pkg.installed>`, and
:py:mod:`pkg.purged <salt.states.pkg.installed>` states, which
:py:mod:`pkg.removed <salt.states.pkg.removed>`, and
:py:mod:`pkg.purged <salt.states.pkg.purged>` states, which
causes the epoch to be disregarded when the state checks to see if
the desired version was installed.
@ -1571,54 +1571,46 @@ def installed(
# _find_install_targets() found no targets or encountered an error
# check that the hold function is available
if 'pkg.hold' in __salt__:
if 'hold' in kwargs:
try:
if kwargs['hold']:
hold_ret = __salt__['pkg.hold'](
name=name, pkgs=pkgs, sources=sources
)
else:
hold_ret = __salt__['pkg.unhold'](
name=name, pkgs=pkgs, sources=sources
)
except (CommandExecutionError, SaltInvocationError) as exc:
return {'name': name,
'changes': {},
'result': False,
'comment': six.text_type(exc)}
if 'pkg.hold' in __salt__ and 'hold' in kwargs:
try:
action = 'pkg.hold' if kwargs['hold'] else 'pkg.unhold'
hold_ret = __salt__[action](
name=name, pkgs=pkgs, sources=sources
)
except (CommandExecutionError, SaltInvocationError) as exc:
return {'name': name,
'changes': {},
'result': False,
'comment': six.text_type(exc)}
if 'result' in hold_ret and not hold_ret['result']:
return {'name': name,
'changes': {},
'result': False,
'comment': 'An error was encountered while '
'holding/unholding package(s): {0}'
.format(hold_ret['comment'])}
else:
modified_hold = [hold_ret[x] for x in hold_ret
if hold_ret[x]['changes']]
not_modified_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['changes']
and hold_ret[x]['result']]
failed_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['result']]
if 'result' in hold_ret and not hold_ret['result']:
return {'name': name,
'changes': {},
'result': False,
'comment': 'An error was encountered while '
'holding/unholding package(s): {0}'
.format(hold_ret['comment'])}
else:
modified_hold = [hold_ret[x] for x in hold_ret
if hold_ret[x]['changes']]
not_modified_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['changes']
and hold_ret[x]['result']]
failed_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['result']]
if modified_hold:
for i in modified_hold:
result['comment'] += '.\n{0}'.format(i['comment'])
result['result'] = i['result']
result['changes'][i['name']] = i['changes']
for i in modified_hold:
result['comment'] += '.\n{0}'.format(i['comment'])
result['result'] = i['result']
result['changes'][i['name']] = i['changes']
if not_modified_hold:
for i in not_modified_hold:
result['comment'] += '.\n{0}'.format(i['comment'])
result['result'] = i['result']
for i in not_modified_hold:
result['comment'] += '.\n{0}'.format(i['comment'])
result['result'] = i['result']
if failed_hold:
for i in failed_hold:
result['comment'] += '.\n{0}'.format(i['comment'])
result['result'] = i['result']
for i in failed_hold:
result['comment'] += '.\n{0}'.format(i['comment'])
result['result'] = i['result']
return result
if to_unpurge and 'lowpkg.unpurge' not in __salt__:
@ -1739,45 +1731,40 @@ def installed(
# checks reinstall targets works.
pkg_ret = {}
if 'pkg.hold' in __salt__:
if 'hold' in kwargs:
try:
if kwargs['hold']:
hold_ret = __salt__['pkg.hold'](
name=name, pkgs=pkgs, sources=sources
)
else:
hold_ret = __salt__['pkg.unhold'](
name=name, pkgs=pkgs, sources=sources
)
except (CommandExecutionError, SaltInvocationError) as exc:
comment.append(six.text_type(exc))
ret = {'name': name,
'changes': changes,
'result': False,
'comment': '\n'.join(comment)}
if warnings:
ret.setdefault('warnings', []).extend(warnings)
return ret
else:
if 'result' in hold_ret and not hold_ret['result']:
ret = {'name': name,
'changes': {},
'result': False,
'comment': 'An error was encountered while '
'holding/unholding package(s): {0}'
.format(hold_ret['comment'])}
if warnings:
ret.setdefault('warnings', []).extend(warnings)
return ret
else:
modified_hold = [hold_ret[x] for x in hold_ret
if hold_ret[x]['changes']]
not_modified_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['changes']
and hold_ret[x]['result']]
failed_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['result']]
if 'pkg.hold' in __salt__ and 'hold' in kwargs:
try:
action = 'pkg.hold' if kwargs['hold'] else 'pkg.unhold'
hold_ret = __salt__[action](
name=name, pkgs=desired, sources=sources
)
except (CommandExecutionError, SaltInvocationError) as exc:
comment.append(six.text_type(exc))
ret = {'name': name,
'changes': changes,
'result': False,
'comment': '\n'.join(comment)}
if warnings:
ret.setdefault('warnings', []).extend(warnings)
return ret
else:
if 'result' in hold_ret and not hold_ret['result']:
ret = {'name': name,
'changes': {},
'result': False,
'comment': 'An error was encountered while '
'holding/unholding package(s): {0}'
.format(hold_ret['comment'])}
if warnings:
ret.setdefault('warnings', []).extend(warnings)
return ret
else:
modified_hold = [hold_ret[x] for x in hold_ret
if hold_ret[x]['changes']]
not_modified_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['changes']
and hold_ret[x]['result']]
failed_hold = [hold_ret[x] for x in hold_ret
if not hold_ret[x]['result']]
if to_unpurge:
changes['purge_desired'] = __salt__['lowpkg.unpurge'](*to_unpurge)
@ -1840,6 +1827,10 @@ def installed(
if len(changes[change_name]['old']) > 0:
changes[change_name]['old'] += '\n'
changes[change_name]['old'] += '{0}'.format(i['changes']['old'])
else:
comment.append(i['comment'])
changes[change_name] = {}
changes[change_name]['new'] = '{0}'.format(i['changes']['new'])
# Any requested packages that were not targeted for install or reinstall
if not_modified:
@ -2743,8 +2734,8 @@ def removed(name,
In version 2015.8.9, an **ignore_epoch** argument has been added to
:py:mod:`pkg.installed <salt.states.pkg.installed>`,
:py:mod:`pkg.removed <salt.states.pkg.installed>`, and
:py:mod:`pkg.purged <salt.states.pkg.installed>` states, which
:py:mod:`pkg.removed <salt.states.pkg.removed>`, and
:py:mod:`pkg.purged <salt.states.pkg.purged>` states, which
causes the epoch to be disregarded when the state checks to see if
the desired version was installed. If **ignore_epoch** was not set
to ``True``, and instead of ``2:7.4.160-1.el7`` a version of
@ -2849,8 +2840,8 @@ def purged(name,
In version 2015.8.9, an **ignore_epoch** argument has been added to
:py:mod:`pkg.installed <salt.states.pkg.installed>`,
:py:mod:`pkg.removed <salt.states.pkg.installed>`, and
:py:mod:`pkg.purged <salt.states.pkg.installed>` states, which
:py:mod:`pkg.removed <salt.states.pkg.removed>`, and
:py:mod:`pkg.purged <salt.states.pkg.purged>` states, which
causes the epoch to be disregarded when the state checks to see if
the desired version was installed. If **ignore_epoch** was not set
to ``True``, and instead of ``2:7.4.160-1.el7`` a version of

View File

@ -115,10 +115,10 @@ def installed(name,
'''
if 'force' in kwargs:
salt.utils.versions.warn_until(
'Fluorine',
'Neon',
'Parameter \'force\' has been detected in the argument list. This'
'parameter is no longer used and has been replaced by \'recurse\''
'as of Salt 2018.3.0. This warning will be removed in Salt Fluorine.'
'as of Salt 2018.3.0. This warning will be removed in Salt Neon.'
)
kwargs.pop('force')

View File

@ -586,7 +586,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)):
dest=self._loglevel_config_setting_name_,
choices=list(log.LOG_LEVELS),
help='Console logging log level. One of {0}. Default: \'{1}\'.'.format(
', '.join([repr(l) for l in log.SORTED_LEVEL_NAMES]),
', '.join(["'{}'".format(n) for n in log.SORTED_LEVEL_NAMES]),
self._default_logging_level_
)
)
@ -605,7 +605,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)):
dest=self._logfile_loglevel_config_setting_name_,
choices=list(log.LOG_LEVELS),
help='Logfile logging log level. One of {0}. Default: \'{1}\'.'.format(
', '.join([repr(l) for l in log.SORTED_LEVEL_NAMES]),
', '.join(["'{}'".format(n) for n in log.SORTED_LEVEL_NAMES]),
self._default_logging_level_
)
)

View File

@ -42,6 +42,18 @@ class PipModuleTest(ModuleCase):
os.makedirs(self.pip_temp)
os.environ['PIP_SOURCE_DIR'] = os.environ['PIP_BUILD_DIR'] = ''
def tearDown(self):
super(PipModuleTest, self).tearDown()
if os.path.isdir(self.venv_test_dir):
shutil.rmtree(self.venv_test_dir, ignore_errors=True)
if os.path.isdir(self.pip_temp):
shutil.rmtree(self.pip_temp, ignore_errors=True)
del self.venv_dir
del self.venv_test_dir
del self.pip_temp
if 'PIP_SOURCE_DIR' in os.environ:
os.environ.pop('PIP_SOURCE_DIR')
def _check_download_error(self, ret):
'''
Checks to see if a download error looks transitory
@ -440,13 +452,3 @@ class PipModuleTest(ModuleCase):
ret2 = self.run_function('cmd.run', '/bin/pip3 freeze | grep lazyimport')
assert 'lazyimport==0.0.1' in ret1
assert ret2 == ''
def tearDown(self):
super(PipModuleTest, self).tearDown()
if os.path.isdir(self.venv_test_dir):
shutil.rmtree(self.venv_test_dir, ignore_errors=True)
if os.path.isdir(self.pip_temp):
shutil.rmtree(self.pip_temp, ignore_errors=True)
del self.venv_dir
del self.venv_test_dir
del self.pip_temp

View File

@ -8,8 +8,19 @@ Linux and Solaris are supported
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
try:
import tzlocal # pylint: disable=unused-import
HAS_TZLOCAL = True
except ImportError:
HAS_TZLOCAL = False
# Import Salt Testing libs
from tests.support.case import ModuleCase
from tests.support.helpers import destructiveTest
from tests.support.unit import skipIf
# Import salt libs
import salt.utils.platform
class TimezoneLinuxModuleTest(ModuleCase):
@ -42,3 +53,44 @@ class TimezoneSolarisModuleTest(ModuleCase):
timescale = ['UTC', 'localtime']
ret = self.run_function('timezone.get_hwclock')
self.assertIn(ret, timescale)
@skipIf(not salt.utils.platform.is_windows(), 'windows test only')
class TimezoneWindowsModuleTest(ModuleCase):
def setUp(self):
self.pre = self.run_function('timezone.get_zone')
def tearDown(self):
post = self.run_function('timezone.get_zone')
if self.pre != post:
self.run_function('timezone.set_zone', [self.pre])
def test_get_hwclock(self):
timescale = ['UTC', 'localtime']
ret = self.run_function('timezone.get_hwclock')
self.assertIn(ret, timescale)
@destructiveTest
def test_get_zone(self):
'''
test timezone.set_zone, get_zone and zone_compare
'''
zone = 'America/Inuvik' if not HAS_TZLOCAL else 'America/Denver'
# first set the zone
assert self.run_function('timezone.set_zone', [zone])
# check it set the correct zone
ret = self.run_function('timezone.get_zone')
assert zone in ret
# compare zones
assert self.run_function('timezone.zone_compare', [zone])
def test_get_offset(self):
'''
test timezone.get_offset
'''
ret = self.run_function('timezone.get_offset')
self.assertIn('-', ret)

View File

@ -591,8 +591,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
continue
self.assertEqual(
ret[key]['comment'],
('Python package carbon < 1.3 was already installed\n'
'All specified packages are already installed'))
('All packages were successfully installed'))
break
else:
raise Exception('Expected state did not run')

View File

@ -1051,3 +1051,52 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin):
refresh=False,
test=True)
self.assertInSaltComment("System update will be performed", ret)
@requires_salt_modules('pkg.hold', 'pkg.unhold')
@requires_system_grains
def test_pkg_015_installed_held(self, grains=None): # pylint: disable=unused-argument
'''
Tests that a package can be held even when the package is already installed.
'''
os_family = grains.get('os_family', '')
if os_family.lower() != 'redhat' and os_family.lower() != 'debian':
self.skipTest('Test only runs on RedHat or Debian family')
pkg_targets = _PKG_TARGETS.get(os_family, [])
# Make sure that we have targets that match the os_family. If this
# fails then the _PKG_TARGETS dict above needs to have an entry added,
# with two packages that are not installed before these tests are run
self.assertTrue(pkg_targets)
target = pkg_targets[0]
# First we ensure that the package is installed
ret = self.run_state(
'pkg.installed',
name=target,
refresh=False,
)
self.assertSaltTrueReturn(ret)
# Then we check that the package is now held
ret = self.run_state(
'pkg.installed',
name=target,
hold=True,
refresh=False,
)
try:
tag = 'pkg_|-{0}_|-{0}_|-installed'.format(target)
self.assertSaltTrueReturn(ret)
self.assertIn(tag, ret)
self.assertIn('changes', ret[tag])
self.assertIn(target, ret[tag]['changes'])
self.assertEqual(ret[tag]['changes'][target], {'new': 'hold', 'old': 'install'})
finally:
# Clean up, unhold package and remove
self.run_function('pkg.unhold', name=target)
ret = self.run_state('pkg.removed', name=target)
self.assertSaltTrueReturn(ret)

View File

@ -1113,7 +1113,7 @@ def requires_salt_modules(*names):
)
for name in names:
if name not in cls.run_function('sys.doc'):
if name not in cls.run_function('sys.doc', [name]):
cls.skipTest(
'Salt module {0!r} is not available'.format(name)
)

View File

@ -685,6 +685,27 @@ class FileGrepTestCase(TestCase, LoaderModuleMockMixin):
'Lorem Lorem',
'-i -b2')
def test_grep_query_exists_wildcard(self):
_file = '{0}*'.format(self.tfile.name)
result = filemod.grep(_file,
'Lorem ipsum')
self.assertTrue(result, None)
self.assertTrue(result['retcode'] == 0)
self.assertTrue(result['stdout'] == 'Lorem ipsum dolor sit amet, consectetur')
self.assertTrue(result['stderr'] == '')
def test_grep_file_not_exists_wildcard(self):
_file = '{0}-junk*'.format(self.tfile.name)
result = filemod.grep(_file,
'Lorem ipsum')
self.assertTrue(result, None)
self.assertFalse(result['retcode'] == 0)
self.assertFalse(result['stdout'] == 'Lorem ipsum dolor sit amet, consectetur')
_expected_stderr = 'grep: {0}-junk*: No such file or directory'.format(self.tfile.name)
self.assertTrue(result['stderr'] == _expected_stderr)
class FileModuleTestCase(TestCase, LoaderModuleMockMixin):
def setup_loader_modules(self):