valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 08:58:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #48023 from rallytime/merge-develop

Browse Source 
[develop] Merge forward from 2018.3 to develop
This commit is contained in:
Nicole Thomas 2018-06-08 09:55:00 -04:00 committed by GitHub
commit bec663ed24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 213 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
salt/cloud/clouds/ec2.py
View File

@ -4782,7 +4782,7 @@ def get_password_data(
rsa_key = kwargs['key']
pwdata = base64.b64decode(pwdata)
if HAS_M2:
key = RSA.load_key_string(rsa_key)
key = RSA.load_key_string(rsa_key.encode('ascii'))
password = key.private_decrypt(pwdata, RSA.pkcs1_padding)
else:
dsize = Crypto.Hash.SHA.digest_size

7
salt/modules/archive.py
View File

@ -186,12 +186,13 @@ def list_(name,
else {'fileobj': cached.stdout, 'mode': 'r|'}
with contextlib.closing(tarfile.open(**open_kwargs)) as tar_archive:
for member in tar_archive.getmembers():
_member = salt.utils.data.decode(member.name)
if member.issym():
links.append(member.name)
links.append(_member)
elif member.isdir():
dirs.append(member.name + '/')
dirs.append(_member + '/')
else:
files.append(member.name)
files.append(_member)
return dirs, files, links
except tarfile.ReadError:

16
salt/modules/pillar.py
View File

@ -40,10 +40,10 @@ def get(key,
'''
.. versionadded:: 0.14
Attempt to retrieve the named value from pillar, if the named value is not
available return the passed default. The default return is an empty string
except ``__opts__['pillar_raise_on_missing']`` is set to True, in which
case a ``KeyError`` exception will be raised.
Attempt to retrieve the named value from :ref:`in-memory pillar data
<pillar-in-memory>`. If the pillar key is not present in the in-memory
pillar, then the value specified in the ``default`` option (described
below) will be returned.
If the merge parameter is set to ``True``, the default will be recursively
merged into the returned pillar data.
@ -62,8 +62,12 @@ def get(key,
The pillar key to get value from
default
If specified, return this value in case when named pillar value does
not exist.
The value specified by this option will be returned if the desired
pillar key does not exist.
If a default value is specified, then it will be an empty string,
unless :conf_minion:`pillar_raise_on_missing` is set to ``True``, in
which case an error will be raised.
merge : ``False``
If ``True``, the retrieved values will be merged into the passed

3
salt/modules/postfix.py
View File

@ -293,9 +293,10 @@ def set_main(key, value, path=MAIN_CF):
pairs, conf_list = _parse_main(path)
new_conf = []
key_line_match = re.compile("^{0}([\\s=]|$)".format(re.escape(key)))
if key in pairs:
for line in conf_list:
if line.startswith(key):
if re.match(key_line_match, line):
new_conf.append('{0} = {1}'.format(key, value))
else:
new_conf.append(line)

2
salt/modules/win_pkg.py
View File

@ -953,7 +953,7 @@ def _repo_process_pkg_sls(filename, short_path_name, ret, successful_verbose):
else:
ret.setdefault('repo', {}).update(config)
ret.setdefault('name_map', {}).update(revmap)
successful_verbose[short_path_name] = config.keys()
successful_verbose[short_path_name] = list(config.keys())
elif config:
return _failed_compile('Compiled contents', 'not a dictionary/hash')
else:

24
salt/runners/state.py
View File

@ -15,22 +15,38 @@ from salt.exceptions import SaltInvocationError
LOGGER = logging.getLogger(__name__)
def set_pause(jid, state_id, duration=None):
def pause(jid, state_id=None, duration=None):
'''
Set up a state id pause, this instructs a running state to pause at a given
state id. This needs to pass in the jid of the running state and can
optionally pass in a duration in seconds.
'''
minion = salt.minion.MasterMinion(__opts__)
minion['state.set_pause'](jid, state_id, duration)
minion.functions['state.pause'](jid, state_id, duration)
set_pause = salt.utils.functools.alias_function(pause, 'set_pause')
def rm_pause(jid, state_id, duration=None):
def resume(jid, state_id=None):
'''
Remove a pause from a jid, allowing it to continue
'''
minion = salt.minion.MasterMinion(__opts__)
minion['state.rm_pause'](jid, state_id)
minion.functions['state.resume'](jid, state_id)
rm_pause = salt.utils.functools.alias_function(resume, 'rm_pause')
def soft_kill(jid, state_id=None):
'''
Set up a state run to die before executing the given state id,
this instructs a running state to safely exit at a given
state id. This needs to pass in the jid of the running state.
If a state_id is not passed then the jid referenced will be safely exited
at the beginning of the next state run.
'''
minion = salt.minion.MasterMinion(__opts__)
minion.functions['state.soft_kill'](jid, state_id)
def orchestrate(mods,

2
salt/states/git.py
View File

@ -2172,7 +2172,7 @@ def detached(name,
.. versionadded:: 2016.3.0
Make sure a repository is cloned to the given target directory and is
a detached HEAD checkout of the commit ID resolved from ``ref``.
a detached HEAD checkout of the commit ID resolved from ``rev``.
name
Address of the remote repository.

6
salt/states/win_wua.py
View File

@ -410,18 +410,18 @@ def uptodate(name,
# Update the system using the state defaults
update_system:
wua.up_to_date
wua.uptodate
# Update the drivers
update_drivers:
wua.up_to_date:
wua.uptodate:
- software: False
- drivers: True
- skip_reboot: False
# Apply all critical updates
update_critical:
wua.up_to_date:
wua.uptodate:
- severities:
- Critical
'''

54
setup.py
View File

@ -29,6 +29,7 @@ from distutils.command.build import build
from distutils.command.clean import clean
from distutils.command.sdist import sdist
from distutils.command.install_lib import install_lib
from distutils.version import LooseVersion # pylint: disable=blacklisted-module
from ctypes.util import find_library
# pylint: enable=E0611
@ -73,7 +74,7 @@ else:
# os.uname() not available on Windows.
IS_SMARTOS_PLATFORM = os.uname()[0] == 'SunOS' and os.uname()[3].startswith('joyent_')
# Store a reference wether if we're running under Python 3 and above
# Store a reference whether if we're running under Python 3 and above
IS_PY3 = sys.version_info > (3,)
# Use setuptools only if the user opts-in by setting the USE_SETUPTOOLS env var
@ -145,10 +146,6 @@ def _parse_requirements_file(requirements_file):
if IS_WINDOWS_PLATFORM:
if 'libcloud' in line:
continue
if 'm2crypto' in line.lower() and __saltstack_version__.info < (2015, 8): # pylint: disable=undefined-variable
# In Windows, we're installing M2CryptoWin{32,64} which comes
# compiled
continue
if IS_PY3 and 'futures' in line.lower():
# Python 3 already has futures, installing it will only break
# the current python installation whenever futures is imported
@ -313,12 +310,6 @@ if WITH_SETUPTOOLS:
def run(self):
if IS_WINDOWS_PLATFORM:
if __saltstack_version__.info < (2015, 8): # pylint: disable=undefined-variable
# Install M2Crypto first
self.distribution.salt_installing_m2crypto_windows = True
self.run_command('install-m2crypto-windows')
self.distribution.salt_installing_m2crypto_windows = None
# Download the required DLLs
self.distribution.salt_download_windows_dlls = True
self.run_command('download-windows-dlls')
@ -337,30 +328,6 @@ if WITH_SETUPTOOLS:
develop.run(self)
class InstallM2CryptoWindows(Command):
description = 'Install M2CryptoWindows'
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
if getattr(self.distribution, 'salt_installing_m2crypto_windows', None) is None:
print('This command is not meant to be called on it\'s own')
exit(1)
import platform
from pip.utils import call_subprocess
from pip.utils.logging import indent_log
platform_bits, _ = platform.architecture()
with indent_log():
call_subprocess(
['pip', 'install', '--egg', 'M2CryptoWin{0}'.format(platform_bits[:2])]
)
def uri_to_resource(resource_file):
# ## Returns the URI for a resource
# The basic case is that the resource is on saltstack.com
@ -398,12 +365,17 @@ class DownloadWindowsDlls(Command):
print('This command is not meant to be called on it\'s own')
exit(1)
import platform
from pip.utils.logging import indent_log
import pip
# pip has moved many things to `_internal` starting with pip 10
if LooseVersion(pip.__version__) < LooseVersion('10.0'):
from pip.utils.logging import indent_log
else:
from pip._internal.utils.logging import indent_log # pylint: disable=no-name-in-module
platform_bits, _ = platform.architecture()
url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll'
dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll')
with indent_log():
for fname in ('libeay32', 'ssleay32', 'libsodium', 'msvcr120'):
for fname in ('libeay32', 'ssleay32', 'msvcr120'):
# See if the library is already on the system
if find_library(fname):
continue
@ -699,12 +671,6 @@ class Install(install):
self.build_lib, 'salt', '_version.py'
)
if IS_WINDOWS_PLATFORM:
if __saltstack_version__.info < (2015, 8): # pylint: disable=undefined-variable
# Install M2Crypto first
self.distribution.salt_installing_m2crypto_windows = True
self.run_command('install-m2crypto-windows')
self.distribution.salt_installing_m2crypto_windows = None
# Download the required DLLs
self.distribution.salt_download_windows_dlls = True
self.run_command('download-windows-dlls')
@ -848,8 +814,6 @@ class SaltDistribution(distutils.dist.Distribution):
'install_lib': InstallLib})
if IS_WINDOWS_PLATFORM:
self.cmdclass.update({'download-windows-dlls': DownloadWindowsDlls})
if __saltstack_version__.info < (2015, 8): # pylint: disable=undefined-variable
self.cmdclass.update({'install-m2crypto-windows': InstallM2CryptoWindows})
if WITH_SETUPTOOLS:
self.cmdclass.update({'develop': Develop})

15
tests/integration/modules/test_archive.py
View File

@ -183,6 +183,21 @@ class ArchiveTest(ModuleCase):
self._tear_down()
@skipIf(not salt.utils.path.which('tar'), 'Cannot find tar executable')
def test_tar_list_unicode(self):
'''
Validate using the tar function to extract archives
'''
self._set_up(arch_fmt='tar', unicode_filename=True)
self.run_function('archive.tar', ['-cvf', self.arch], sources=self.src)
# Test list archive
ret = self.run_function('archive.list', name=self.arch)
self.assertTrue(isinstance(ret, list), six.text_type(ret))
self._assert_artifacts_in_ret(ret)
self._tear_down()
@skipIf(not salt.utils.path.which('gzip'), 'Cannot find gzip executable')
def test_gzip(self):
'''

64
tests/integration/runners/test_state.py
View File

@ -21,6 +21,7 @@ from tests.support.case import ShellCase
from tests.support.unit import skipIf
from tests.support.paths import TMP
from tests.support.helpers import flaky
from tests.support.mock import MagicMock, patch
# Import Salt Libs
import salt.utils.platform
@ -477,3 +478,66 @@ class OrchEventTest(ShellCase):
self.assertTrue(received)
del listener
signal.alarm(0)
def test_orchestration_soft_kill(self):
'''
Test to confirm that the parallel state requisite works in orch
we do this by running 10 test.sleep's of 10 seconds, and insure it only takes roughly 10s
'''
self.write_conf({
'fileserver_backend': ['roots'],
'file_roots': {
'base': [self.base_env],
},
})
orch_sls = os.path.join(self.base_env, 'two_stage_orch_kill.sls')
with salt.utils.files.fopen(orch_sls, 'w') as fp_:
fp_.write(textwrap.dedent('''
stage_one:
test.succeed_without_changes
stage_two:
test.fail_without_changes
'''))
listener = salt.utils.event.get_event(
'master',
sock_dir=self.master_opts['sock_dir'],
transport=self.master_opts['transport'],
opts=self.master_opts)
mock_jid = '20131219120000000000'
self.run_run('state.soft_kill {0} stage_two'.format(mock_jid))
with patch('salt.utils.jid.gen_jid', MagicMock(return_value=mock_jid)):
jid = self.run_run_plus(
'state.orchestrate',
'two_stage_orch_kill',
__reload_config=True).get('jid')
if jid is None:
raise Exception('jid missing from run_run_plus output')
signal.signal(signal.SIGALRM, self.alarm_handler)
signal.alarm(self.timeout)
received = False
try:
while True:
event = listener.get_event(full=True)
if event is None:
continue
# Ensure that stage_two of the state does not run
if event['tag'] == 'salt/run/{0}/ret'.format(jid):
received = True
# Don't wrap this in a try/except. We want to know if the
# data structure is different from what we expect!
ret = event['data']['return']['data']['master']
self.assertNotIn('test_|-stage_two_|-stage_two_|-fail_without_changes', ret)
break
finally:
self.assertTrue(received)
del listener
signal.alarm(0)

9
tests/integration/shell/test_call.py
View File

@ -233,20 +233,17 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin
with salt.utils.files.fopen(minion_config_file, 'w') as fh_:
salt.utils.yaml.safe_dump(minion_config, fh_, default_flow_style=False)
out = self.run_script(
_, timed_out = self.run_script(
'salt-call',
'--config-dir {0} cmd.run "echo foo"'.format(
config_dir
),
timeout=timeout,
catch_timeout=True,
)
try:
self.assertIn(
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout),
out
)
self.assertTrue(timed_out)
except AssertionError:
if os.path.isfile(minion_config_file):
os.unlink(minion_config_file)

56
tests/support/case.py
View File

@ -245,6 +245,7 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
arg_str,
catch_stderr=False,
with_retcode=False,
catch_timeout=False,
# FIXME A timeout of zero or disabling timeouts may not return results!
timeout=15,
raw=False,
@ -299,43 +300,47 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
popen_kwargs['preexec_fn'] = detach_from_parent_group
def format_return(retcode, stdout, stderr=None):
def format_return(retcode, stdout, stderr=None, timed_out=False):
'''
DRY helper to log script result if it failed, and then return the
desired output based on whether or not stderr was desired, and
wither or not a retcode was desired.
'''
if log_output is True or \
(log_output is None and (retcode is None or retcode != 0)):
if stderr is not None:
log.debug(
'run_script results for: %s %s\n'
'return code: %s\n'
'stdout:\n'
'%s\n\n'
'stderr:\n'
'%s',
script, arg_str, retcode, stdout, stderr
)
else:
log.debug(
'run_script results for: %s %s\n'
'return code: %s\n'
'stdout:\n'
'%s',
script, arg_str, retcode, stdout
)
log_func = log.debug
if timed_out:
log.error(
'run_script timed out after %d seconds (process killed)',
timeout
)
log_func = log.error
if log_output is True \
or timed_out \
or (log_output is None and retcode != 0):
log_func(
'run_script results for: %s %s\n'
'return code: %s\n'
'stdout:\n'
'%s\n\n'
'stderr:\n'
'%s',
script, arg_str, retcode, stdout, stderr
)
stdout = stdout or ''
stderr = stderr or ''
if not raw:
stdout = stdout.splitlines()
if stderr is not None:
stderr = stderr.splitlines()
stderr = stderr.splitlines()
ret = [stdout]
if catch_stderr:
ret.append(stderr)
if with_retcode:
ret.append(retcode)
if catch_timeout:
ret.append(timed_out)
return ret[0] if len(ret) == 1 else tuple(ret)
@ -384,9 +389,8 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
return format_return(
process.returncode,
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout),
'Process killed, unable to catch stderr output'
*process.communicate(),
timed_out=True
)
tmp_file.seek(0)

51
tests/unit/cloud/clouds/test_ec2.py
View File

@ -2,22 +2,50 @@
# Import Python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
import tempfile
# Import Salt Libs
from salt.cloud.clouds import ec2
from salt.exceptions import SaltCloudSystemExit
import salt.utils.files
# Import Salt Testing Libs
from tests.support.unit import TestCase, skipIf
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, PropertyMock
from tests.support.paths import TMP
from tests.unit.test_crypt import PRIVKEY_DATA
PASS_DATA = (
b'qOjCKDlBdcNEbJ/J8eRl7sH+bYIIm4cvHHY86gh2NEUnufFlFo0gGVTZR05Fj0cw3n/w7gR'
b'urNXz5JoeSIHVuNI3YTwzL9yEAaC0kuy8EbOlO2yx8yPGdfml9BRwOV7A6b8UFo9co4H7fz'
b'DdScMKU2yzvRYvp6N6Q2cJGBmPsemnXWWusb+1vZVWxcRAQmG3ogF6Z5rZSYAYH0N4rqJgH'
b'mQfzuyb+jrBvV/IOoV1EdO9jGSH9338aS47NjrmNEN/SpnS6eCWZUwwyHbPASuOvWiY4QH/'
b'0YZC6EGccwiUmt0ZOxIynk+tEyVPTkiS0V8RcZK6YKqMWHpKmPtLBzfuoA=='
)
@skipIf(NO_MOCK, NO_MOCK_REASON)
class EC2TestCase(TestCase):
class EC2TestCase(TestCase, LoaderModuleMockMixin):
'''
Unit TestCase for salt.cloud.clouds.ec2 module.
'''
def setUp(self):
super(EC2TestCase, self).setUp()
with tempfile.NamedTemporaryFile(dir=TMP, suffix='.pem', delete=True) as fp:
self.key_file = fp.name
def tearDown(self):
super(EC2TestCase, self).tearDown()
if os.path.exists(self.key_file):
os.remove(self.key_file)
def setup_loader_modules(self):
return {ec2: {'__opts__': {}}}
def test__validate_key_path_and_mode(self):
# Key file exists
@ -38,3 +66,24 @@ class EC2TestCase(TestCase):
with patch('os.path.exists', return_value=False):
self.assertRaises(
SaltCloudSystemExit, ec2._validate_key_path_and_mode, 'key_file')
@patch('salt.cloud.clouds.ec2._get_node')
@patch('salt.cloud.clouds.ec2.get_location')
@patch('salt.cloud.clouds.ec2.get_provider')
@patch('salt.utils.aws.query')
def test_get_password_data(self, query, get_provider, get_location, _get_node):
query.return_value = [
{
'passwordData': PASS_DATA
}
]
_get_node.return_value = {'instanceId': 'i-abcdef'}
get_location.return_value = 'us-west2'
get_provider.return_value = 'ec2'
with salt.utils.files.fopen(self.key_file, 'w') as fp:
fp.write(PRIVKEY_DATA)
ret = ec2.get_password_data(
name='i-abcddef', kwargs={'key_file': self.key_file}, call='action'
)
assert ret['passwordData'] == PASS_DATA
assert ret['password'] == b'testp4ss!'