From bba9d0d105b34c5108843f739f971617e18659aa Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 29 Sep 2016 16:19:25 +0300 Subject: [PATCH 01/28] `alternatives.install` state: detect `alternatives` command failed --- salt/modules/alternatives.py | 37 ++++++++++++++++++++++ salt/states/alternatives.py | 39 +++++++++++++++-------- tests/unit/modules/alternatives_test.py | 2 +- tests/unit/states/alternatives_test.py | 41 +++++++++++++++++-------- 4 files changed, 93 insertions(+), 26 deletions(-) diff --git a/salt/modules/alternatives.py b/salt/modules/alternatives.py index a11411589c..b6ed32a97d 100644 --- a/salt/modules/alternatives.py +++ b/salt/modules/alternatives.py @@ -10,6 +10,10 @@ from __future__ import absolute_import import os import logging +# Import Salt libs +import salt.utils + + __outputter__ = { 'display': 'txt', 'install': 'txt', @@ -59,6 +63,39 @@ def display(name): return out['stdout'] +def show_link(name): + ''' + Display master link for the alternative + + .. versionadded:: 2015.8.13,2016.3.4,Carbon + + CLI Example: + + .. code-block:: bash + + salt '*' alternatives.show_link editor + ''' + + path = '/var/lib/dpkg/alternatives/{0}'.format(name) + if _get_cmd() == 'alternatives': + path = '/var/lib/alternatives/{0}'.format(name) + + try: + with salt.utils.fopen(path, 'rb') as r_file: + return r_file.readlines()[1] + except OSError: + log.error( + 'alternatives: {0} does not exist'.format(name) + ) + except (IOError, IndexError) as exc: + log.error( + 'alternatives: unable to get master link for {0}. ' + 'Exception: {1}'.format(name, exc) + ) + + return False + + def show_current(name): ''' Display the current highest-priority alternative for a given alternatives diff --git a/salt/states/alternatives.py b/salt/states/alternatives.py index 33d00ef519..b3ddf8569c 100644 --- a/salt/states/alternatives.py +++ b/salt/states/alternatives.py @@ -33,6 +33,13 @@ __func_alias__ = { } +def __virtual__(): + ''' + Only load if alternatives execution module is available. + ''' + return True if 'alternatives.auto' in __salt__ else False + + def install(name, link, path, priority): ''' Install new alternative for defined @@ -63,24 +70,33 @@ def install(name, link, path, priority): 'comment': ''} isinstalled = __salt__['alternatives.check_installed'](name, path) - if not isinstalled: + if isinstalled: + ret['comment'] = 'Alternatives for {0} is already set to {1}'.format(name, path) + else: if __opts__['test']: ret['comment'] = ( 'Alternative will be set for {0} to {1} with priority {2}' ).format(name, path, priority) ret['result'] = None return ret - __salt__['alternatives.install'](name, link, path, priority) - ret['comment'] = ( - 'Setting alternative for {0} to {1} with priority {2}' - ).format(name, path, priority) - ret['changes'] = {'name': name, - 'link': link, - 'path': path, - 'priority': priority} - return ret - ret['comment'] = 'Alternatives for {0} is already set to {1}'.format(name, path) + out = __salt__['alternatives.install'](name, link, path, priority) + current = __salt__['alternatives.show_current'](name) + master_link = __salt__['alternatives.show_link'](name) + if current == path and master_link == link: + ret['comment'] = ( + 'Alternative for {0} set to path {1} with priority {2}' + ).format(name, current, priority) + ret['changes'] = {'name': name, + 'link': link, + 'path': path, + 'priority': priority} + else: + ret['result'] = False + ret['comment'] = ( + 'Alternative for {0} not installed: {1}' + ).format(name, out) + return ret @@ -214,7 +230,6 @@ def set_(name, path): __salt__['alternatives.set'](name, path) current = __salt__['alternatives.show_current'](name) if current == path: - ret['result'] = True ret['comment'] = ( 'Alternative for {0} set to path {1}' ).format(name, current) diff --git a/tests/unit/modules/alternatives_test.py b/tests/unit/modules/alternatives_test.py index d9eccd3df3..58209059c1 100644 --- a/tests/unit/modules/alternatives_test.py +++ b/tests/unit/modules/alternatives_test.py @@ -37,7 +37,7 @@ class AlternativesTestCase(TestCase): python_shell=False ) - with patch.dict(alternatives.__grains__, {'os_family': 'Ubuntu'}): + with patch.dict(alternatives.__grains__, {'os_family': 'Suse'}): mock = MagicMock( return_value={'retcode': 0, 'stdout': 'undoubtedly-salt'} ) diff --git a/tests/unit/states/alternatives_test.py b/tests/unit/states/alternatives_test.py index bb5dff40d0..5a4de2edbb 100644 --- a/tests/unit/states/alternatives_test.py +++ b/tests/unit/states/alternatives_test.py @@ -49,26 +49,33 @@ class AlternativesTestCase(TestCase): 'changes': {}, 'comment': ''} - mock = MagicMock(side_effect=[True, False, False]) - mock_bool = MagicMock(return_value=True) + bad_link = '/bin/pager' + err = 'the primary link for {0} must be {1}'.format(name, link) + + mock = MagicMock(side_effect=[True, False, False, False]) + mock_out = MagicMock(side_effect=['', err]) + mock_path = MagicMock(return_value=path) + mock_link = MagicMock(return_value=link) with patch.dict(alternatives.__salt__, {'alternatives.check_installed': mock, - 'alternatives.install': mock_bool}): + 'alternatives.install': mock_out, + 'alternatives.show_current': mock_path, + 'alternatives.show_link': mock_link}): comt = ('Alternatives for {0} is already set to {1}' - ).format(name, path) + ).format(name, path) ret.update({'comment': comt, 'result': True}) self.assertDictEqual(alternatives.install(name, link, path, priority), ret) comt = (('Alternative will be set for {0} to {1} with priority {2}' - ).format(name, path, priority)) + ).format(name, path, priority)) ret.update({'comment': comt, 'result': None}) with patch.dict(alternatives.__opts__, {'test': True}): self.assertDictEqual(alternatives.install(name, link, path, priority), ret) - comt = ('Setting alternative for {0} to {1} with priority {2}' - ).format(name, path, priority) + comt = ('Alternative for {0} set to path {1} with priority {2}' + ).format(name, path, priority) ret.update({'comment': comt, 'result': True, 'changes': {'name': name, 'link': link, 'path': path, 'priority': priority}}) @@ -76,6 +83,14 @@ class AlternativesTestCase(TestCase): self.assertDictEqual(alternatives.install(name, link, path, priority), ret) + comt = ('Alternative for {0} not installed: {1}' + ).format(name, err) + ret.update({'comment': comt, 'result': False, + 'changes': {}, 'link': bad_link}) + with patch.dict(alternatives.__opts__, {'test': False}): + self.assertDictEqual(alternatives.install(name, bad_link, path, + priority), ret) + # 'remove' function tests: 1 def test_remove(self): @@ -94,10 +109,10 @@ class AlternativesTestCase(TestCase): mock = MagicMock(side_effect=[True, True, True, False, False]) mock_bool = MagicMock(return_value=True) - mock_bol = MagicMock(side_effect=[False, True, True, False]) + mock_show = MagicMock(side_effect=[False, True, True, False]) with patch.dict(alternatives.__salt__, {'alternatives.check_exists': mock, - 'alternatives.show_current': mock_bol, + 'alternatives.show_current': mock_show, 'alternatives.remove': mock_bool}): comt = ('Alternative for {0} will be removed'.format(name)) ret.update({'comment': comt}) @@ -116,7 +131,7 @@ class AlternativesTestCase(TestCase): self.assertDictEqual(alternatives.remove(name, path), ret) comt = ('Alternative for {0} is set to it\'s default path True' - ).format(name) + ).format(name) ret.update({'comment': comt, 'result': True, 'changes': {}}) self.assertDictEqual(alternatives.remove(name, path), ret) @@ -175,17 +190,17 @@ class AlternativesTestCase(TestCase): mock = MagicMock(side_effect=[path, path, '']) mock_bool = MagicMock(return_value=True) - mock_bol = MagicMock(side_effect=[path, False, False, False, False]) + mock_show = MagicMock(side_effect=[path, False, False, False, False]) with patch.dict(alternatives.__salt__, {'alternatives.display': mock, - 'alternatives.show_current': mock_bol, + 'alternatives.show_current': mock_show, 'alternatives.set': mock_bool}): comt = ('Alternative for {0} already set to {1}'.format(name, path)) ret.update({'comment': comt}) self.assertDictEqual(alternatives.set_(name, path), ret) comt = ('Alternative for {0} will be set to path False' - ).format(name) + ).format(name) ret.update({'comment': comt, 'result': None}) with patch.dict(alternatives.__opts__, {'test': True}): self.assertDictEqual(alternatives.set_(name, path), ret) From e8e53d60bee55b24923eb8be0ddf803b00b4a87d Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Fri, 27 May 2016 08:29:16 -0600 Subject: [PATCH 02/28] pass __proxy__ in state.sls_id --- salt/modules/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index fc6ccd2431..ae6d2f534e 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -1122,7 +1122,7 @@ def sls_id( opts['test'] = _get_test_value(test, **kwargs) if 'pillarenv' in kwargs: opts['pillarenv'] = kwargs['pillarenv'] - st_ = salt.state.HighState(opts) + st_ = salt.state.HighState(opts, proxy=__proxy__) if isinstance(mods, six.string_types): split_mods = mods.split(',') st_.push_active() From 57ecbe6c53c979a3dead15af5118e127e8829599 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 29 Sep 2016 23:02:10 -0500 Subject: [PATCH 03/28] Update minion restart example to use onchanges instead of cmd.wait --- doc/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index a12b514042..954c071ca7 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -268,9 +268,9 @@ Linux/Unix - name: salt-minion - require: - pkg: salt-minion - cmd.wait: + cmd.run: - name: echo service salt-minion restart | at now + 1 minute - - watch: + - onchanges: - pkg: salt-minion To ensure that **at** is installed and **atd** is running, the following states From 3d15eedfe006046464c0cdcf16f4efd8cfaccefd Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 29 Sep 2016 23:02:39 -0500 Subject: [PATCH 04/28] Add additional information about onchanges/onchanges_in --- doc/ref/states/requisites.rst | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index e871322955..e811af062e 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -301,6 +301,51 @@ a useful way to execute a post hook after changing aspects of a system. If a state has multiple ``onchanges`` requisites then the state will trigger if any of the watched states changes. +.. note:: + One easy-to-make mistake is to use ``onchanges_in`` when ``onchanges`` is + supposed to be used. For example, the below configuration is not correct: + + .. code-block:: yaml + + myservice: + pkg.installed: + - name: myservice + file.managed: + - name: /etc/myservice/myservice.conf + - source: salt://myservice/files/myservice.conf + - mode: 600 + cmd.run: + - name: /usr/libexec/myservice/post-changes-hook.sh + - onchanges_in: + - file: /etc/myservice/myservice.conf + + This will set up a requisite relationship in which the ``cmd.run`` state + always executes, and the ``file.managed`` state only executes if the + ``cmd.run`` state has changes (which it always will, since the ``cmd.run`` + state includes the command results as changes). + + It may semantically seem like the the ``cmd.run`` state should only run + when there are changes in the file state, but remember that requisite + relationships involve one state watching another state, and a + :ref:`requisite_in ` does the opposite: it forces + the specified state to watch the state with the ``requisite_in``. + + The correct usage would be: + + .. code-block:: yaml + + myservice: + pkg.installed: + - name: myservice + file.managed: + - name: /etc/myservice/myservice.conf + - source: salt://myservice/files/myservice.conf + - mode: 600 + cmd.run: + - name: /usr/libexec/myservice/post-changes-hook.sh + - onchanges: + - file: /etc/myservice/myservice.conf + use ~~~ @@ -335,6 +380,7 @@ inherit inherited options. .. _requisites-require-in: .. _requisites-watch-in: +.. _requisites-onchanges-in: The _in versions of requisites ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 73eb773fb0b3939bf16f7f2c3fa1a1d9c688a557 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Fri, 30 Sep 2016 18:14:19 +0300 Subject: [PATCH 05/28] salt.modules.ini_manage: fix creating options in empty file --- salt/modules/ini_manage.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index 6648f6b13f..9c44b98f62 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -10,13 +10,16 @@ Edit ini files (for example /etc/sysctl.conf) ''' +from __future__ import absolute_import, print_function + # Import Python libs -from __future__ import print_function -from __future__ import absolute_import import re import json + +# Import Salt libs from salt.utils.odict import OrderedDict from salt.utils import fopen as _fopen +from salt.exceptions import CommandExecutionError __virtualname__ = 'ini' @@ -47,8 +50,8 @@ def set_option(file_name, sections=None): A dictionary representing the sections to be edited ini file The keys are the section names and the values are the dictionary containing the options - If the Ini does not contain sections the keys and values represent the - options + If the ini file does not contain sections the keys and values represent + the options API Example: @@ -66,11 +69,7 @@ def set_option(file_name, sections=None): salt '*' ini.set_option /path/to/ini '{section_foo: {key: value}}' ''' sections = sections or {} - changes = {} inifile = _Ini.get_ini_file(file_name) - if not inifile: - changes.update({'error': 'ini file not found'}) - return changes changes = inifile.update(sections) inifile.flush() return changes @@ -325,7 +324,13 @@ class _Ini(_Section): super(_Ini, self).__init__(name, inicontents, seperator, commenter) def refresh(self, inicontents=None): - inicontents = inicontents or _fopen(self.name).read() + try: + inicontents = inicontents or _fopen(self.name).read() + except (OSError, IOError) as exc: + raise CommandExecutionError( + "Unable to open file '{0}'. " + "Exception: {1}".format(self.name, exc) + ) if not inicontents: return for opt in self: @@ -339,10 +344,16 @@ class _Ini(_Section): self.update({sect_obj.name: sect_obj}) def flush(self): - with _fopen(self.name, 'w') as outfile: - ini_gen = self.gen_ini() - next(ini_gen) - outfile.writelines(ini_gen) + try: + with _fopen(self.name, 'w') as outfile: + ini_gen = self.gen_ini() + next(ini_gen) + outfile.writelines(ini_gen) + except (OSError, IOError) as exc: + raise CommandExecutionError( + "Unable to write file '{0}'. " + "Exception: {1}".format(self.name, exc) + ) @staticmethod def get_ini_file(file_name): From 891004f3beaa4543400057906facc0394987d44e Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Fri, 30 Sep 2016 14:13:34 -0600 Subject: [PATCH 06/28] try/except for when __proxy__ is not injected. --- salt/modules/state.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index ae6d2f534e..2441792657 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -1122,7 +1122,10 @@ def sls_id( opts['test'] = _get_test_value(test, **kwargs) if 'pillarenv' in kwargs: opts['pillarenv'] = kwargs['pillarenv'] - st_ = salt.state.HighState(opts, proxy=__proxy__) + try: + st_ = salt.state.HighState(opts, proxy=__proxy__) + except NameError: + st_ = salt.state.HighState(opts) if isinstance(mods, six.string_types): split_mods = mods.split(',') st_.push_active() From 3bb2cb6379652869621b03ac8cfcd2cd3663a54d Mon Sep 17 00:00:00 2001 From: Yoram Hekma Date: Wed, 28 Sep 2016 07:23:53 +0200 Subject: [PATCH 07/28] Update doc to reflect the version where 'none' was added as a pillar_source_merging_strategy --- doc/ref/configuration/master.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index aa9567b759..0137eb46e5 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -2648,6 +2648,7 @@ The pillar_source_merging_strategy option allows you to configure merging strategy between different sources. It accepts 5 values: * ``none``: +.. versionadded:: 2016.3.4 It will not do any merging at all and only parse the pillar data from the passed environment and 'base' if no environment was specified. * ``recurse``: From c5b8e442f9ca0c2dbc99b83f4c96c3a5853af5c8 Mon Sep 17 00:00:00 2001 From: James Loosli Date: Wed, 28 Sep 2016 08:49:43 -0700 Subject: [PATCH 08/28] a small, and unfortunate error --- salt/states/elasticsearch_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/elasticsearch_index.py b/salt/states/elasticsearch_index.py index 0ebde9249d..dd6bcfd0b8 100644 --- a/salt/states/elasticsearch_index.py +++ b/salt/states/elasticsearch_index.py @@ -50,7 +50,7 @@ def present(name, definition): ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''} - index_exists = __salt__['elasticsearch.index_exists'](name=name) + index_exists = __salt__['elasticsearch.index_exists'](index=name) if not index_exists: if __opts__['test']: ret['comment'] = 'Index {0} will be created'.format(name) From 5904cc04c67cd7fa2024c58549f04f830e2b67d8 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 30 Sep 2016 14:27:04 -0600 Subject: [PATCH 09/28] Skip cmd_unzip test if salt.utils.which('zip') isn't available This test relies on running archive.cmd_zip in order to have something to test running archive.cmd_unzip against. Some distros will have the unzip binary available, but not the zip binary. We need to protect this test against both so we don't have a false failure on trying to run archive.cmd_zip. --- tests/integration/modules/archive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/modules/archive.py b/tests/integration/modules/archive.py index f463d77467..e6b37f982c 100644 --- a/tests/integration/modules/archive.py +++ b/tests/integration/modules/archive.py @@ -189,6 +189,7 @@ class ArchiveTest(integration.ModuleCase): self._tear_down() + @skipIf(not salt.utils.which('zip'), 'Cannot find zip executable') @skipIf(not salt.utils.which('unzip'), 'Cannot find unzip executable') def test_cmd_unzip(self): ''' From cdf2a565644ab07963eb9ef8118be6cca533b902 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 3 Oct 2016 23:37:05 +0900 Subject: [PATCH 10/28] Fix issue where test suite could hang on shutdown This is most common in the develop branch but could occur here as well. --- tests/integration/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 96754822c8..85a523d084 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -704,7 +704,7 @@ class TestDaemon(object): self._exit_ssh() # Shutdown the multiprocessing logging queue listener salt_log_setup.shutdown_multiprocessing_logging() - salt_log_setup.shutdown_multiprocessing_logging_listener() + salt_log_setup.shutdown_multiprocessing_logging_listener(daemonizing=True) def pre_setup_minions(self): ''' From b5fcca9983c2722b1c122e5524ac695134f2639c Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Mon, 3 Oct 2016 11:14:02 -0600 Subject: [PATCH 11/28] modules.archive int tests: check for gzip, rar Similar to 8733586. --- tests/integration/modules/archive.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/modules/archive.py b/tests/integration/modules/archive.py index e6b37f982c..f96cd0bbaf 100644 --- a/tests/integration/modules/archive.py +++ b/tests/integration/modules/archive.py @@ -160,6 +160,7 @@ class ArchiveTest(integration.ModuleCase): self._tear_down() + @skipIf(not salt.utils.which('gzip'), 'Cannot find gzip executable') @skipIf(not salt.utils.which('gunzip'), 'Cannot find gunzip executable') def test_gunzip(self): ''' @@ -205,7 +206,7 @@ class ArchiveTest(integration.ModuleCase): self._tear_down() - @skipIf(not HAS_ZIPFILE, 'Cannot find zip python module') + @skipIf(not HAS_ZIPFILE, 'Cannot find zipfile python module') def test_zip(self): ''' Validate using the zip function @@ -219,7 +220,7 @@ class ArchiveTest(integration.ModuleCase): self._tear_down() - @skipIf(not HAS_ZIPFILE, 'Cannot find zip python module') + @skipIf(not HAS_ZIPFILE, 'Cannot find zipfile python module') def test_unzip(self): ''' Validate using the unzip function @@ -248,7 +249,8 @@ class ArchiveTest(integration.ModuleCase): self._tear_down() - @skipIf(not salt.utils.which_bin(('rar', 'unrar')), 'Cannot find rar or unrar executable') + @skipIf(not salt.utils.which('rar'), 'Cannot find rar executable') + @skipIf(not salt.utils.which('unrar'), 'Cannot find unrar executable') def test_unrar(self): ''' Validate using the unrar function From 928c99d2f74eb9600394977b2ff009a910b3f336 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 3 Oct 2016 20:23:59 -0500 Subject: [PATCH 12/28] Base rpmdev-vercmp comparison result on retcode This should be a more future-proof way of using this tool for version comparison. --- salt/modules/rpm.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/salt/modules/rpm.py b/salt/modules/rpm.py index f2063c2aaf..97246ec40d 100644 --- a/salt/modules/rpm.py +++ b/salt/modules/rpm.py @@ -633,21 +633,27 @@ def version_cmp(ver1, ver2, ignore_epoch=False): ver1 = _ensure_epoch(ver1) ver2 = _ensure_epoch(ver2) - result = __salt__['cmd.run'](['rpmdev-vercmp', ver1, ver2], - python_shell=False, - ignore_retcode=True).strip() - if result.endswith('equal'): + result = __salt__['cmd.run_all']( + ['rpmdev-vercmp', ver1, ver2], + python_shell=False, + redirect_stderr=True, + ignore_retcode=True) + # rpmdev-vercmp returns 0 on equal, 11 on greater-than, and + # 12 on less-than. + if result['retcode'] == 0: return 0 - elif 'is newer' in result: - newer_version = result.split()[0] - if newer_version == ver1: - return 1 - elif newer_version == ver2: - return -1 - log.warning( - 'Failed to interpret results of rpmdev-vercmp output: %s', - result - ) + elif result['retcode'] == 11: + return 1 + elif result['retcode'] == 12: + return -1 + else: + # We'll need to fall back to salt.utils.version_cmp() + log.warning( + 'Failed to interpret results of rpmdev-vercmp output. ' + 'This is probably a bug, and should be reported. ' + 'Return code was %s. Output: %s', + result['retcode'], result['stdout'] + ) else: # We'll need to fall back to salt.utils.version_cmp() log.warning( From b9f53434491d815b37b74d6122113ab05f4df204 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 4 Oct 2016 21:09:47 +0900 Subject: [PATCH 13/28] Another bit of detection for failed pip tests Don't bail out if the download failed for some reason on the remote end --- tests/integration/modules/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/pip.py b/tests/integration/modules/pip.py index 4bf6a272a8..e04ebd0992 100644 --- a/tests/integration/modules/pip.py +++ b/tests/integration/modules/pip.py @@ -45,7 +45,7 @@ class PipModuleTest(integration.ModuleCase): ''' Checks to see if a download error looks transitory ''' - return any(w in ret for w in ['URLError']) + return any(w in ret for w in ['URLError', 'Download error']) def pip_successful_install(self, target, expect=('flake8', 'pep8',)): ''' From 9df2fd11ddd759eb78bcdd8ebc5cc6c676fca8fa Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 4 Oct 2016 09:48:59 -0500 Subject: [PATCH 14/28] add __utils__ to vultr cloud provider --- salt/cloud/clouds/vultrpy.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/salt/cloud/clouds/vultrpy.py b/salt/cloud/clouds/vultrpy.py index 304e02b154..4470d5d1a1 100644 --- a/salt/cloud/clouds/vultrpy.py +++ b/salt/cloud/clouds/vultrpy.py @@ -29,7 +29,6 @@ import time # Import salt cloud libs import salt.config as config -import salt.utils.cloud from salt.exceptions import SaltCloudSystemExit # Get logging started @@ -146,7 +145,7 @@ def list_nodes_select(conn=None, call=None): ''' Return a list of the VMs that are on the provider, with select fields ''' - return salt.utils.cloud.list_nodes_select( + return __utils__['cloud.list_nodes_select']( list_nodes_full(), __opts__['query.selection'], call, ) @@ -190,7 +189,7 @@ def show_instance(name, call=None): # Find under which cloud service the name is listed, if any if name not in nodes: return {} - salt.utils.cloud.cache_node(nodes[name], __active_provider_name__, __opts__) + __utils__['cloud.cache_node'](nodes[name], __active_provider_name__, __opts__) return nodes[name] @@ -212,7 +211,7 @@ def create(vm_): if 'driver' not in vm_: vm_['driver'] = vm_['provider'] - salt.utils.cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'starting create', 'salt/cloud/{0}/creating'.format(vm_['name']), @@ -248,7 +247,7 @@ def create(vm_): log.info('Creating Cloud VM {0}'.format(vm_['name'])) - salt.utils.cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'requesting instance', 'salt/cloud/{0}/requesting'.format(vm_['name']), @@ -264,7 +263,7 @@ def create(vm_): log.error('Status 412 may mean that you are requesting an\n' 'invalid location, image, or size.') - salt.utils.cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'instance request failed', 'salt/cloud/{0}/requesting/failed'.format(vm_['name']), @@ -282,7 +281,7 @@ def create(vm_): # Show the traceback if the debug logging level is enabled exc_info_on_loglevel=logging.DEBUG ) - salt.utils.cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'instance request failed', 'salt/cloud/{0}/requesting/failed'.format(vm_['name']), @@ -316,12 +315,12 @@ def create(vm_): return False return data['default_password'] - vm_['ssh_host'] = salt.utils.cloud.wait_for_fun( + vm_['ssh_host'] = __utils__['cloud.wait_for_fun']( wait_for_hostname, timeout=config.get_cloud_config_value( 'wait_for_fun_timeout', vm_, __opts__, default=15 * 60), ) - vm_['password'] = salt.utils.cloud.wait_for_fun( + vm_['password'] = __utils__['cloud.wait_for_fun']( wait_for_default_password, timeout=config.get_cloud_config_value( 'wait_for_fun_timeout', vm_, __opts__, default=15 * 60), @@ -335,7 +334,7 @@ def create(vm_): ) # Bootstrap - ret = salt.utils.cloud.bootstrap(vm_, __opts__) + ret = __utils__['cloud.bootstrap'](vm_, __opts__) ret.update(show_instance(vm_['name'], call='action')) @@ -346,7 +345,7 @@ def create(vm_): ) ) - salt.utils.cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'created instance', 'salt/cloud/{0}/created'.format(vm_['name']), @@ -387,7 +386,7 @@ def _query(path, method='GET', data=None, params=None, header_dict=None, decode= if header_dict is None: header_dict = {} - result = salt.utils.http.query( + result = __utils__['http.query']( url, method=method, params=params, From 118ba8a772b1ea1702d4ea291c64cb8ece8a5d35 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 5 Oct 2016 18:24:09 +0900 Subject: [PATCH 15/28] Update alternatives module to strip newline chars Refs an error exposed by #36676 --- salt/modules/alternatives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/alternatives.py b/salt/modules/alternatives.py index bfaf9c69a3..fe2961c96a 100644 --- a/salt/modules/alternatives.py +++ b/salt/modules/alternatives.py @@ -82,7 +82,7 @@ def show_link(name): try: with salt.utils.fopen(path, 'rb') as r_file: - return r_file.readlines()[1] + return r_file.readlines()[1].rstrip('\n') except OSError: log.error( 'alternatives: {0} does not exist'.format(name) From 574e30e915b5447d15bc727d339c324910fb88f4 Mon Sep 17 00:00:00 2001 From: Maxime Guillet Date: Wed, 5 Oct 2016 12:19:54 +0200 Subject: [PATCH 16/28] Fix behavior of psql -c option with postgresql 9.6 psql -c option no longer implies --no-psqlrc with postgresql 9.6, it must be set explicitly. Fixes #36787. --- salt/modules/postgres.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 35d9058700..a9c5837127 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -306,6 +306,7 @@ def _psql_cmd(*args, **kwargs): cmd = [salt.utils.which('psql'), '--no-align', '--no-readline', + '--no-psqlrc', '--no-password'] # It is never acceptable to issue a password prompt. if user: cmd += ['--username', user] From 21f2a17a0706aa04389960d236aa9e991ba19200 Mon Sep 17 00:00:00 2001 From: Maxime Guillet Date: Wed, 5 Oct 2016 14:43:02 +0200 Subject: [PATCH 17/28] Fix postgresql tests by adding --no-psqlrc option introduced by #36787. --- tests/unit/modules/postgres_test.py | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/unit/modules/postgres_test.py b/tests/unit/modules/postgres_test.py index 61f3131b7b..5c0e8f606c 100644 --- a/tests/unit/modules/postgres_test.py +++ b/tests/unit/modules/postgres_test.py @@ -90,13 +90,13 @@ class PostgresTestCase(TestCase): owner='otheruser', runas='foo') postgres._run_psql.assert_has_calls([ - call(['/usr/bin/pgsql', '--no-align', '--no-readline', + call(['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'ALTER DATABASE "dbname" OWNER TO "otheruser"'], host='testhost', user='testuser', password='foo', runas='foo', port='testport'), - call(['/usr/bin/pgsql', '--no-align', '--no-readline', + call(['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'ALTER DATABASE "dbname" SET TABLESPACE "testspace"'], @@ -141,7 +141,7 @@ class PostgresTestCase(TestCase): ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'CREATE DATABASE "dbname" WITH TABLESPACE = testspace ' @@ -209,7 +209,7 @@ class PostgresTestCase(TestCase): runas='foo' ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'DROP DATABASE "test_db"'], @@ -258,7 +258,7 @@ class PostgresTestCase(TestCase): runas='foo' ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'DROP ROLE "testgroup"'], @@ -418,7 +418,7 @@ class PostgresTestCase(TestCase): runas='foo' ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'DROP ROLE "testuser"'], @@ -830,7 +830,7 @@ class PostgresTestCase(TestCase): db_password='testpassword' ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'CREATE SCHEMA "testschema"'], @@ -863,7 +863,7 @@ class PostgresTestCase(TestCase): db_password='testpassword' ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'maint_db', '-c', 'DROP SCHEMA "testschema"'], @@ -938,7 +938,7 @@ class PostgresTestCase(TestCase): password='testpassword' ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'testdb', '-c', 'CREATE LANGUAGE plpythonu'], @@ -978,7 +978,7 @@ class PostgresTestCase(TestCase): password='testpassword' ) postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'testdb', '-c', 'DROP LANGUAGE plpgsql'], @@ -1053,7 +1053,7 @@ class PostgresTestCase(TestCase): "ORDER BY relname) TO STDOUT WITH CSV HEADER") postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-v', 'datestyle=ISO,MDY', '-c', query], @@ -1092,7 +1092,7 @@ class PostgresTestCase(TestCase): "TO STDOUT WITH CSV HEADER") postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-v', 'datestyle=ISO,MDY', '-c', query], @@ -1243,7 +1243,7 @@ class PostgresTestCase(TestCase): query = 'GRANT ALL ON TABLE public."awl" TO "baruwa" WITH GRANT OPTION' postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-c', query], @@ -1270,7 +1270,7 @@ class PostgresTestCase(TestCase): query = 'GRANT ALL ON TABLE public."awl" TO "baruwa"' postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-c', query], @@ -1298,7 +1298,7 @@ class PostgresTestCase(TestCase): query = 'GRANT SELECT ON ALL TABLES IN SCHEMA public TO "baruwa"' postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-c', query], @@ -1329,7 +1329,7 @@ class PostgresTestCase(TestCase): query = 'GRANT admins TO "baruwa" WITH ADMIN OPTION' postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-c', query], @@ -1355,7 +1355,7 @@ class PostgresTestCase(TestCase): query = 'GRANT admins TO "baruwa"' postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-c', query], @@ -1386,7 +1386,7 @@ class PostgresTestCase(TestCase): query = 'REVOKE ALL ON TABLE public.awl FROM baruwa' postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-c', query], @@ -1416,7 +1416,7 @@ class PostgresTestCase(TestCase): query = 'REVOKE admins FROM baruwa' postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', 'testport', '--dbname', 'db_name', '-c', query], From 8b92ae2061997fda9b777ae52b3e1379b4486911 Mon Sep 17 00:00:00 2001 From: Maxime Guillet Date: Wed, 5 Oct 2016 16:08:42 +0200 Subject: [PATCH 18/28] Fix postgresql tests using position in the argument list of psql. Some tests use position number in the argument list of psql command to check their results. As #36787 add one element, next position in the list must be used. --- tests/unit/modules/postgres_test.py | 42 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/unit/modules/postgres_test.py b/tests/unit/modules/postgres_test.py index 5c0e8f606c..562d62f705 100644 --- a/tests/unit/modules/postgres_test.py +++ b/tests/unit/modules/postgres_test.py @@ -237,11 +237,11 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. + # query will be in the 15th argument. self.assertTrue( - postgres._run_psql.call_args[0][0][13].startswith('CREATE ROLE') + postgres._run_psql.call_args[0][0][14].startswith('CREATE ROLE') ) @patch('salt.modules.postgres._run_psql', @@ -286,13 +286,13 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. + # query will be in the 15th argument. self.assertTrue( re.match( 'ALTER.* "testgroup" .* UNENCRYPTED PASSWORD', - postgres._run_psql.call_args[0][0][13] + postgres._run_psql.call_args[0][0][14] ) ) @@ -320,10 +320,10 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. - call = postgres._run_psql.call_args[0][0][13] + # query will be in the 15th argument. + call = postgres._run_psql.call_args[0][0][14] self.assertTrue(re.match('CREATE ROLE "testuser"', call)) for i in ( 'INHERIT NOCREATEDB NOCREATEROLE ' @@ -449,16 +449,16 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. + # query will be in the 15th argument. self.assertTrue( re.match( 'ALTER ROLE "test_username" WITH INHERIT NOCREATEDB ' 'NOCREATEROLE NOREPLICATION LOGIN ' 'UNENCRYPTED PASSWORD [\'"]{0,5}test_role_pass[\'"]{0,5};' ' GRANT "test_groups" TO "test_username"', - postgres._run_psql.call_args[0][0][13] + postgres._run_psql.call_args[0][0][14] ) ) @@ -521,15 +521,15 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. + # query will be in the 15th argument. self.assertTrue( re.match( 'ALTER ROLE "test_username" WITH INHERIT NOCREATEDB ' 'CREATEROLE NOREPLICATION LOGIN NOPASSWORD;' ' GRANT "test_groups" TO "test_username"', - postgres._run_psql.call_args[0][0][13] + postgres._run_psql.call_args[0][0][14] ) ) @@ -557,9 +557,9 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. + # query will be in the 15th argument. self.assertTrue( re.match( 'ALTER ROLE "test_username" WITH INHERIT NOCREATEDB ' @@ -567,7 +567,7 @@ class PostgresTestCase(TestCase): 'ENCRYPTED PASSWORD ' '[\'"]{0,5}md531c27e68d3771c392b52102c01be1da1[\'"]{0,5}' '; GRANT "test_groups" TO "test_username"', - postgres._run_psql.call_args[0][0][13] + postgres._run_psql.call_args[0][0][14] ) ) @@ -583,13 +583,13 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. + # query will be in the 15th argument. self.assertTrue( re.match( 'SELECT setting FROM pg_catalog.pg_settings', - postgres._run_psql.call_args[0][0][13] + postgres._run_psql.call_args[0][0][14] ) ) From 6ce4653fa35dbed24f6283eb83964ffe008582d6 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 5 Oct 2016 23:50:42 +0900 Subject: [PATCH 19/28] Error on reaction with missing SLS file Closes #36579 --- salt/utils/reactor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/utils/reactor.py b/salt/utils/reactor.py index e8a88eb0b5..33adb1202e 100644 --- a/salt/utils/reactor.py +++ b/salt/utils/reactor.py @@ -59,8 +59,10 @@ class Reactor(salt.utils.process.SignalHandlingMultiprocessingProcess, salt.stat if glob_ref.startswith('salt://'): glob_ref = self.minion.functions['cp.cache_file'](glob_ref) - - for fn_ in glob.glob(glob_ref): + globbed_ref = glob.glob(glob_ref) + if not globbed_ref: + log.error('Can not render SLS {0} for tag {1}. File missing or not found.'.format(glob_ref, tag)) + for fn_ in globbed_ref: try: res = self.render_template( fn_, From b59c23bef1e515d785906f87a7eaa4d234353a6c Mon Sep 17 00:00:00 2001 From: Maxime Guillet Date: Wed, 5 Oct 2016 17:05:10 +0200 Subject: [PATCH 20/28] Fix one remaining postgresql tests linked to #36787. --- tests/unit/modules/postgres_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/modules/postgres_test.py b/tests/unit/modules/postgres_test.py index 562d62f705..fa63a42058 100644 --- a/tests/unit/modules/postgres_test.py +++ b/tests/unit/modules/postgres_test.py @@ -485,15 +485,15 @@ class PostgresTestCase(TestCase): runas='foo' ) # postgres._run_psql.call_args[0][0] will contain the list of CLI args. - # The first 13 elements of this list are initial args used in all (or + # The first 14 elements of this list are initial args used in all (or # virtually all) commands run through _run_psql(), so the actual SQL - # query will be in the 14th argument. + # query will be in the 15th argument. self.assertTrue( re.match( 'ALTER ROLE "test_username" WITH INHERIT NOCREATEDB ' 'CREATEROLE NOREPLICATION LOGIN;' ' GRANT "test_groups" TO "test_username"', - postgres._run_psql.call_args[0][0][13] + postgres._run_psql.call_args[0][0][14] ) ) From 21837370852b659fa95f0667bedcac85c95fd77e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 5 Oct 2016 12:28:19 -0500 Subject: [PATCH 21/28] do not load libvirt pillar if certtool is unavailable --- salt/pillar/libvirt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/pillar/libvirt.py b/salt/pillar/libvirt.py index 01fd42b96a..f71c7496c3 100644 --- a/salt/pillar/libvirt.py +++ b/salt/pillar/libvirt.py @@ -15,6 +15,10 @@ import subprocess import salt.utils +def __virtual__(): + return salt.utils.which('certtool') is not None + + def ext_pillar(minion_id, pillar, # pylint: disable=W0613 command): # pylint: disable=W0613 From 02b91ecf150218de1099589a2cce8a0dee86a95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Thu, 6 Oct 2016 09:00:27 +0200 Subject: [PATCH 22/28] states: glance: import keystone exceptions from new location keystoneclient.apiclient.exceptions has been deprecated since 0.7.1 in favor of keystoneclient.exceptions, and has been removed in 2.1.0, so the glance state fails to load with recent versions of keystoneclient. This commit tries to import from keystoneclient.exceptions first, and falls back to keystoneclient.apiclient.exceptions if that fails. --- salt/states/glance.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/salt/states/glance.py b/salt/states/glance.py index 952b18e543..a3b764ba1d 100644 --- a/salt/states/glance.py +++ b/salt/states/glance.py @@ -13,13 +13,23 @@ from salt.utils import warn_until # Import OpenStack libs try: - from keystoneclient.apiclient.exceptions import \ + from keystoneclient.exceptions import \ Unauthorized as kstone_Unauthorized + HAS_KEYSTONE = True +except ImportError: + try: + from keystoneclient.apiclient.exceptions import \ + Unauthorized as kstone_Unauthorized + HAS_KEYSTONE = True + except ImportError: + HAS_KEYSTONE = False + +try: from glanceclient.exc import \ HTTPUnauthorized as glance_Unauthorized - HAS_DEPENDENCIES = True + HAS_GLANCE = True except ImportError: - HAS_DEPENDENCIES = False + HAS_GLANCE = False log = logging.getLogger(__name__) @@ -28,7 +38,7 @@ def __virtual__(): ''' Only load if dependencies are loaded ''' - return HAS_DEPENDENCIES + return HAS_KEYSTONE and HAS_GLANCE def _find_image(name): From 342eee444d4c353bd1cd4fdc159cf193eb7fc214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Thu, 6 Oct 2016 09:25:56 +0200 Subject: [PATCH 23/28] states: glance: handle image list instead of dict glance.image_list now returns a list of images instead of a dict, but the glance state tried to use the values() method on it either way, leading to the following exception: An exception occurred in this state: Traceback (most recent call last): File "/usr/lib/python2.7/dist-packages/salt/state.py", line 1733, in call **cdata['kwargs']) File "/usr/lib/python2.7/dist-packages/salt/loader.py", line 1652, in wrapper return f(*args, **kwargs) File "/usr/lib/python2.7/dist-packages/salt/states/glance.py", line 155, in image_present image, msg = _find_image(name) File "/usr/lib/python2.7/dist-packages/salt/states/glance.py", line 60, in _find_image return images_dict.values()[0], 'Found image {0}'.format(name) AttributeError: 'list' object has no attribute 'values' This commit makes sure that we're always working with a list, even if glance.image_list returns a dict. --- salt/states/glance.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/salt/states/glance.py b/salt/states/glance.py index a3b764ba1d..8f44b8556e 100644 --- a/salt/states/glance.py +++ b/salt/states/glance.py @@ -49,26 +49,26 @@ def _find_image(name): - False, 'Found more than one image with given name' ''' try: - images_dict = __salt__['glance.image_list'](name=name) + images = __salt__['glance.image_list'](name=name) except kstone_Unauthorized: return False, 'keystoneclient: Unauthorized' except glance_Unauthorized: return False, 'glanceclient: Unauthorized' - log.debug('Got images_dict: {0}'.format(images_dict)) + log.debug('Got images: {0}'.format(images)) warn_until('Carbon', 'Starting with Carbon ' '\'glance.image_list\' is not supposed to return ' 'the images wrapped in a separate dict anymore.') - if len(images_dict) == 1 and 'images' in images_dict: - images_dict = images_dict['images'] + if type(images) is dict and len(images) == 1 and 'images' in images: + images = images['images'] - # I /think/ this will still work when glance.image_list - # starts returning a list instead of a dictionary... - if len(images_dict) == 0: + images_list = images.values() if type(images) is dict else images + + if len(images_list) == 0: return None, 'No image with name "{0}"'.format(name) - elif len(images_dict) == 1: - return images_dict.values()[0], 'Found image {0}'.format(name) - elif len(images_dict) > 1: + elif len(images_list) == 1: + return images_list[0], 'Found image {0}'.format(name) + elif len(images_list) > 1: return False, 'Found more than one image with given name' else: raise NotImplementedError From af044fd889ccc8ce49c7b8adb725ef02bf081512 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 7 Oct 2016 13:22:02 -0600 Subject: [PATCH 24/28] Pylint fix --- salt/cloud/clouds/vultrpy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/cloud/clouds/vultrpy.py b/salt/cloud/clouds/vultrpy.py index 5a5ef0344f..ff9ffa98f7 100644 --- a/salt/cloud/clouds/vultrpy.py +++ b/salt/cloud/clouds/vultrpy.py @@ -43,7 +43,6 @@ import urllib # Import salt cloud libs import salt.config as config import salt.ext.six as six -import salt.utils.cloud from salt.exceptions import ( SaltCloudConfigError, SaltCloudSystemExit From b7f5f82cf6d9fa5b3b878f45a8273569edd71f70 Mon Sep 17 00:00:00 2001 From: m03 Date: Sat, 1 Oct 2016 19:12:54 -0700 Subject: [PATCH 25/28] Fix issue 36679 win_servermanager failure --- salt/modules/win_servermanager.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_servermanager.py b/salt/modules/win_servermanager.py index 4d7f530622..682f68fe46 100644 --- a/salt/modules/win_servermanager.py +++ b/salt/modules/win_servermanager.py @@ -5,8 +5,9 @@ Manage Windows features via the ServerManager powershell module # Import Python libs from __future__ import absolute_import -import logging +import ast import json +import logging try: from shlex import quote as _cmd_quote # pylint: disable=E0611 @@ -25,6 +26,17 @@ def __virtual__(): ''' Load only on windows with servermanager module ''' + def _module_present(): + ''' + Check for the presence of the ServerManager module. + ''' + cmd = r"[Bool] (Get-Module -ListAvailable | Where-Object { $_.Name -eq 'ServerManager' })" + cmd_ret = __salt__['cmd.run_all'](cmd, shell='powershell', python_shell=True) + + if cmd_ret['retcode'] == 0: + return ast.literal_eval(cmd_ret['stdout']) + return False + if not salt.utils.is_windows(): return False @@ -33,7 +45,7 @@ def __virtual__(): 'Requires Remote Server Administration Tools which ' \ 'is only available on Windows 2008 R2 and later.' - if not _check_server_manager(): + if not _module_present(): return False, 'Failed to load win_servermanager module: ' \ 'ServerManager module not available. ' \ 'May need to install Remote Server Administration Tools.' From d179423a23b156b23c079b46a528350ca58dc9c7 Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Fri, 7 Oct 2016 13:50:03 -0600 Subject: [PATCH 26/28] Doc updates for Carbon release candidate (#36847) --- doc/topics/releases/carbon.rst | 15 +++++++++++++++ doc/topics/releases/index.rst | 8 +++++++- doc/topics/releases/releasecandidate.rst | 14 ++++++++------ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/doc/topics/releases/carbon.rst b/doc/topics/releases/carbon.rst index da6756c9b6..403ade5b15 100644 --- a/doc/topics/releases/carbon.rst +++ b/doc/topics/releases/carbon.rst @@ -4,6 +4,21 @@ Salt Release Notes - Codename Carbon ==================================== +Release Candidate +================= + +See :ref:`Installing/Testing a Salt Release Candidate ` for instructions to install the latest release candidate. + +Release Candidate Known Issues +------------------------------ + +- :issue:`36729`: Minion does not shutdown properly when attempting to start multiple minions on same host +- :issue:`36693`: /etc/salt/master.d/schedule.conf schedule jobs fail to run +- :issue:`36746`: When killing a job jid output missing +- :issue:`36748`: Proxy minion is not working +- :issue:`36134`: multi-master with failover does not failover when master goes down +- :issue:`36804`: error when using pkg.installed with url source + New Features ============ diff --git a/doc/topics/releases/index.rst b/doc/topics/releases/index.rst index 266cfc6084..98353877eb 100644 --- a/doc/topics/releases/index.rst +++ b/doc/topics/releases/index.rst @@ -8,15 +8,21 @@ information about the version numbering scheme. Latest Branch Release ===================== -|current_release_doc| +.. after carbon releases, replace :ref:`Release Candidate` with the following: + |current_release_doc| + +:ref:`Release Candidate ` Previous Releases ================= +.. after carbon releases, remove carbon from the list below: + .. releasestree:: :maxdepth: 1 :glob: + carbon 2016.3.* 2015.8.* 2015.5.* diff --git a/doc/topics/releases/releasecandidate.rst b/doc/topics/releases/releasecandidate.rst index db1c79247a..ee31b3ce15 100644 --- a/doc/topics/releases/releasecandidate.rst +++ b/doc/topics/releases/releasecandidate.rst @@ -1,12 +1,14 @@ :orphan: +.. _release-candidate: + =========================================== Installing/Testing a Salt Release Candidate =========================================== It's time for a new feature release of Salt! Follow the instructions below to install the latest release candidate of Salt, and try :doc:`all the shiny new -features `! Be sure to report any bugs you find on `Github +features `! Be sure to report any bugs you find on `Github `_. Installing Using Packages @@ -49,14 +51,14 @@ You can install a release candidate of Salt using `Salt Bootstrap .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P git v2016.3.0rc2 + sudo sh install_salt.sh -P git v2016.11.0rc1 If you want to also install a master using Salt Bootstrap, use the ``-M`` flag: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M git v2016.3.0rc2 + sudo sh install_salt.sh -P -M git v2016.11.0rc1 If you want to install only a master and not a minion using Salt Bootstrap, use the ``-M`` and ``-N`` flags: @@ -64,13 +66,13 @@ the ``-M`` and ``-N`` flags: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M -N git v2016.3.0rc2 + sudo sh install_salt.sh -P -M -N git v2016.11.0rc1 Installing Using PyPI ===================== Installing from the `source archive -`_ on +`_ on `PyPI `_ is fairly straightforward. .. note:: @@ -108,4 +110,4 @@ Then install salt using the following command: .. code-block:: bash - sudo pip install salt==2016.3.0rc2 + sudo pip install salt==v2016.11.0rc1 From 2e34277b65005a70ea6d9c4511018d645cd1c21f Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 7 Oct 2016 16:10:57 -0600 Subject: [PATCH 27/28] Add '--no-psqlrc' option to new-to-carbon postgres unit test --- tests/unit/modules/postgres_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/postgres_test.py b/tests/unit/modules/postgres_test.py index 527a682465..c5d562865c 100644 --- a/tests/unit/modules/postgres_test.py +++ b/tests/unit/modules/postgres_test.py @@ -158,7 +158,7 @@ class PostgresTestCase(TestCase): maintenance_db='maint_db', password='foo') postgres._run_psql.assert_called_once_with( - ['/usr/bin/pgsql', '--no-align', '--no-readline', + ['/usr/bin/pgsql', '--no-align', '--no-readline', '--no-psqlrc', '--no-password', '--username', 'testuser', '--host', 'testhost', '--port', '1234', '--dbname', 'maint_db', '-c', 'CREATE DATABASE "dbname" WITH ENCODING = \'utf8\' ' From 0b304e75a3965dc248885797902fca5e98e28160 Mon Sep 17 00:00:00 2001 From: Jorge Schrauwen Date: Sun, 9 Oct 2016 03:51:45 +0200 Subject: [PATCH 28/28] missing docs for salt.modules.apcups (#36863) --- doc/ref/modules/all/index.rst | 1 + doc/ref/modules/all/salt.modules.apcups.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 doc/ref/modules/all/salt.modules.apcups.rst diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 785c611e0c..f1374cee28 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -20,6 +20,7 @@ execution modules aliases alternatives apache + apcups apf aptpkg archive diff --git a/doc/ref/modules/all/salt.modules.apcups.rst b/doc/ref/modules/all/salt.modules.apcups.rst new file mode 100644 index 0000000000..60f4d9e2c8 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.apcups.rst @@ -0,0 +1,6 @@ +=================== +salt.modules.apcups +=================== + +.. automodule:: salt.modules.apcups + :members: