From ebb1f81a9b7585279dd6b70b1ab0a1af7042578a Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 8 Feb 2018 20:33:31 +0100 Subject: [PATCH 01/51] cmd run: when running in bg, force ignore_retcode=True --- salt/modules/cmdmod.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index ba9b421531..fa169abf5d 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -317,6 +317,9 @@ def _run(cmd, # yaml-ified into non-string types cwd = str(cwd) + if bg: + ignore_retcode = True + if not salt.utils.is_windows(): if not os.path.isfile(shell) or not os.access(shell, os.X_OK): msg = 'The shell {0} is not available'.format(shell) @@ -3118,7 +3121,6 @@ def run_bg(cmd, output_loglevel='debug', log_callback=None, reset_system_locale=True, - ignore_retcode=False, saltenv='base', password=None, **kwargs): @@ -3278,7 +3280,6 @@ def run_bg(cmd, log_callback=log_callback, timeout=timeout, reset_system_locale=reset_system_locale, - ignore_retcode=ignore_retcode, saltenv=saltenv, password=password, **kwargs From c54fcf7a2de6a0b0364475c38cbd58737adb4717 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 8 Feb 2018 21:43:02 +0100 Subject: [PATCH 02/51] cmd: move separate DRY logging blocks into _run, prevent logging on bg=True, don't use_vt on bg --- salt/modules/cmdmod.py | 118 +++++++++-------------------------------- 1 file changed, 25 insertions(+), 93 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index fa169abf5d..dc5ffb1e10 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -290,6 +290,7 @@ def _run(cmd, 'Check to ensure that the shell <{0}> is valid for this user.' .format(shell)) + output_loglevel = _check_loglevel(output_loglevel) log_callback = _check_cb(log_callback) if runas is None and '__context__' in globals(): @@ -319,6 +320,7 @@ def _run(cmd, if bg: ignore_retcode = True + use_vt = False if not salt.utils.is_windows(): if not os.path.isfile(shell) or not os.access(shell, os.X_OK): @@ -375,7 +377,7 @@ def _run(cmd, else: return cmd - if _check_loglevel(output_loglevel) is not None: + if output_loglevel is not None: # Always log the shell commands at INFO unless quiet logging is # requested. The command output is what will be controlled by the # 'loglevel' parameter. @@ -551,7 +553,7 @@ def _run(cmd, msg = ( 'Unable to run command \'{0}\' with the context \'{1}\', ' 'reason: '.format( - cmd if _check_loglevel(output_loglevel) is not None + cmd if output_loglevel is not None else 'REDACTED', kwargs ) @@ -598,7 +600,7 @@ def _run(cmd, to = '' if timeout: to = ' (timeout: {0}s)'.format(timeout) - if _check_loglevel(output_loglevel) is not None: + if output_loglevel is not None: msg = 'Running {0} in VT{1}'.format(cmd, to) log.debug(log_callback(msg)) stdout, stderr = '', '' @@ -672,6 +674,26 @@ def _run(cmd, except NameError: # Ignore the context error during grain generation pass + + # Log the output + if output_loglevel is not None: + if not ignore_retcode and ret['retcode'] != 0: + if output_loglevel < LOG_LEVELS['error']: + output_loglevel = LOG_LEVELS['error'] + msg = ( + 'Command \'{0}\' failed with return code: {1}'.format( + cmd, + ret['retcode'] + ) + ) + log.error(log_callback(msg)) + if ret['stdout']: + log.log(output_loglevel, 'stdout: {0}'.format(log_callback(ret['stdout']))) + if ret['stderr']: + log.log(output_loglevel, 'stderr: {0}'.format(log_callback(ret['stderr']))) + if ret['retcode']: + log.log(output_loglevel, 'retcode: {0}'.format(ret['retcode'])) + return ret @@ -953,21 +975,6 @@ def run(cmd, encoded_cmd=encoded_cmd, **kwargs) - log_callback = _check_cb(log_callback) - - lvl = _check_loglevel(output_loglevel) - if lvl is not None: - if not ignore_retcode and ret['retcode'] != 0: - if lvl < LOG_LEVELS['error']: - lvl = LOG_LEVELS['error'] - msg = ( - 'Command \'{0}\' failed with return code: {1}'.format( - cmd, - ret['retcode'] - ) - ) - log.error(log_callback(msg)) - log.log(lvl, 'output: {0}'.format(log_callback(ret['stdout']))) return ret['stdout'] @@ -1319,26 +1326,6 @@ def run_stdout(cmd, password=password, **kwargs) - log_callback = _check_cb(log_callback) - - lvl = _check_loglevel(output_loglevel) - if lvl is not None: - if not ignore_retcode and ret['retcode'] != 0: - if lvl < LOG_LEVELS['error']: - lvl = LOG_LEVELS['error'] - msg = ( - 'Command \'{0}\' failed with return code: {1}'.format( - cmd, - ret['retcode'] - ) - ) - log.error(log_callback(msg)) - if ret['stdout']: - log.log(lvl, 'stdout: {0}'.format(log_callback(ret['stdout']))) - if ret['stderr']: - log.log(lvl, 'stderr: {0}'.format(log_callback(ret['stderr']))) - if ret['retcode']: - log.log(lvl, 'retcode: {0}'.format(ret['retcode'])) return ret['stdout'] @@ -1501,26 +1488,6 @@ def run_stderr(cmd, password=password, **kwargs) - log_callback = _check_cb(log_callback) - - lvl = _check_loglevel(output_loglevel) - if lvl is not None: - if not ignore_retcode and ret['retcode'] != 0: - if lvl < LOG_LEVELS['error']: - lvl = LOG_LEVELS['error'] - msg = ( - 'Command \'{0}\' failed with return code: {1}'.format( - cmd, - ret['retcode'] - ) - ) - log.error(log_callback(msg)) - if ret['stdout']: - log.log(lvl, 'stdout: {0}'.format(log_callback(ret['stdout']))) - if ret['stderr']: - log.log(lvl, 'stderr: {0}'.format(log_callback(ret['stderr']))) - if ret['retcode']: - log.log(lvl, 'retcode: {0}'.format(ret['retcode'])) return ret['stderr'] @@ -1703,26 +1670,6 @@ def run_all(cmd, password=password, **kwargs) - log_callback = _check_cb(log_callback) - - lvl = _check_loglevel(output_loglevel) - if lvl is not None: - if not ignore_retcode and ret['retcode'] != 0: - if lvl < LOG_LEVELS['error']: - lvl = LOG_LEVELS['error'] - msg = ( - 'Command \'{0}\' failed with return code: {1}'.format( - cmd, - ret['retcode'] - ) - ) - log.error(log_callback(msg)) - if ret['stdout']: - log.log(lvl, 'stdout: {0}'.format(log_callback(ret['stdout']))) - if ret['stderr']: - log.log(lvl, 'stderr: {0}'.format(log_callback(ret['stderr']))) - if ret['retcode']: - log.log(lvl, 'retcode: {0}'.format(ret['retcode'])) return ret @@ -1886,21 +1833,6 @@ def retcode(cmd, password=password, **kwargs) - log_callback = _check_cb(log_callback) - - lvl = _check_loglevel(output_loglevel) - if lvl is not None: - if not ignore_retcode and ret['retcode'] != 0: - if lvl < LOG_LEVELS['error']: - lvl = LOG_LEVELS['error'] - msg = ( - 'Command \'{0}\' failed with return code: {1}'.format( - cmd, - ret['retcode'] - ) - ) - log.error(log_callback(msg)) - log.log(lvl, 'output: {0}'.format(log_callback(ret['stdout']))) return ret['retcode'] From 967b83940c076106aa26c6fd6a096b619a37cffe Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 29 Jan 2018 12:57:10 -0700 Subject: [PATCH 03/51] Fix rehash function Use SendMessageTimeout instead of SendMessage which was causing the system to hang in some scenarios Use ctypes to access the dll directly instead of using pywin32 as this seems to actually work --- salt/modules/win_path.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/salt/modules/win_path.py b/salt/modules/win_path.py index 3b446f7cea..e2f292df4e 100644 --- a/salt/modules/win_path.py +++ b/salt/modules/win_path.py @@ -12,12 +12,12 @@ from __future__ import absolute_import import logging import re import os +import ctypes from salt.ext.six.moves import map # Third party libs try: - from win32con import HWND_BROADCAST, WM_SETTINGCHANGE - from win32api import SendMessage + from win32con import HWND_BROADCAST, WM_SETTINGCHANGE, SMTO_ABORTIFHUNG HAS_WIN32 = True except ImportError: HAS_WIN32 = False @@ -47,7 +47,15 @@ def _normalize_dir(string): def rehash(): ''' - Send a WM_SETTINGCHANGE Broadcast to Windows to refresh the Environment variables + Send a WM_SETTINGCHANGE Broadcast to Windows to refresh the Environment + variables for new processes. + + .. note:: + This will only affect new processes that aren't launched by services. To + apply changes to the path to services, the host must be restarted. The + ``salt-minion``, if running as a service, will not see changes to the + environment until the system is restarted. See + `MSDN Documentation ` CLI Example: @@ -55,7 +63,12 @@ def rehash(): salt '*' win_path.rehash ''' - return bool(SendMessage(HWND_BROADCAST, WM_SETTINGCHANGE, 0, 'Environment')) + broadcast_message = ctypes.create_unicode_buffer('Environment') + user32 = ctypes.windll('user32', use_last_error=True) + result = user32.SendMessageTimeoutW(HWND_BROADCAST, WM_SETTINGCHANGE, 0, + broadcast_message, SMTO_ABORTIFHUNG, + 5000, 0) + return result == 1 def get_path(): From 79299361c3abd729a293fae823c8cd70bc0801ef Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 31 Jan 2018 14:29:37 -0700 Subject: [PATCH 04/51] Create refresh_environment salt util --- salt/modules/reg.py | 10 +++------- salt/modules/win_path.py | 10 ++-------- salt/utils/win_functions.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/salt/modules/reg.py b/salt/modules/reg.py index 740bd59fc6..a7df4effa5 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -38,7 +38,6 @@ from salt.ext.six.moves import range # pylint: disable=W0622,import-error # Import third party libs try: - import win32gui import win32api import win32con import pywintypes @@ -48,6 +47,7 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.win_functions from salt.exceptions import CommandExecutionError PY2 = sys.version_info[0] == 2 @@ -68,7 +68,7 @@ def __virtual__(): if not HAS_WINDOWS_MODULES: return (False, 'reg execution module failed to load: ' 'One of the following libraries did not load: ' - + 'win32gui, win32con, win32api') + 'win32con, win32api, pywintypes') return __virtualname__ @@ -193,11 +193,7 @@ def broadcast_change(): salt '*' reg.broadcast_change ''' - # https://msdn.microsoft.com/en-us/library/windows/desktop/ms644952(v=vs.85).aspx - _, res = win32gui.SendMessageTimeout( - win32con.HWND_BROADCAST, win32con.WM_SETTINGCHANGE, 0, 0, - win32con.SMTO_ABORTIFHUNG, 5000) - return not bool(res) + return salt.utils.win_functions.refresh_environment() def list_keys(hive, key=None, use_32bit_registry=False): diff --git a/salt/modules/win_path.py b/salt/modules/win_path.py index e2f292df4e..b70ab423fb 100644 --- a/salt/modules/win_path.py +++ b/salt/modules/win_path.py @@ -12,18 +12,17 @@ from __future__ import absolute_import import logging import re import os -import ctypes from salt.ext.six.moves import map # Third party libs try: - from win32con import HWND_BROADCAST, WM_SETTINGCHANGE, SMTO_ABORTIFHUNG HAS_WIN32 = True except ImportError: HAS_WIN32 = False # Import salt libs import salt.utils +import salt.utils.win_functions # Settings log = logging.getLogger(__name__) @@ -63,12 +62,7 @@ def rehash(): salt '*' win_path.rehash ''' - broadcast_message = ctypes.create_unicode_buffer('Environment') - user32 = ctypes.windll('user32', use_last_error=True) - result = user32.SendMessageTimeoutW(HWND_BROADCAST, WM_SETTINGCHANGE, 0, - broadcast_message, SMTO_ABORTIFHUNG, - 5000, 0) - return result == 1 + return salt.utils.win_functions.refresh_environment() def get_path(): diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 6c7ff4040b..a1f75a5a29 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -6,6 +6,7 @@ missing functions in other modules from __future__ import absolute_import import platform import re +import ctypes # Import Salt Libs from salt.exceptions import CommandExecutionError @@ -17,6 +18,7 @@ try: import win32api import win32net import win32security + from win32con import HWND_BROADCAST, WM_SETTINGCHANGE, SMTO_ABORTIFHUNG HAS_WIN32 = True except ImportError: HAS_WIN32 = False @@ -210,3 +212,30 @@ def escape_for_cmd_exe(arg): return meta_map[char] return meta_re.sub(escape_meta_chars, arg) + + +def refresh_environment(): + ''' + Send a WM_SETTINGCHANGE Broadcast to Windows to refresh the Environment + variables for new processes. + + .. note:: + This will only affect new processes that aren't launched by services. To + apply changes to the path or registry to services, the host must be + restarted. The ``salt-minion``, if running as a service, will not see + changes to the environment until the system is restarted. See + `MSDN Documentation ` + + CLI Example: + + ... code-block:: python + + import salt.utils.win_functions + salt.utils.win_functions.refresh_environment() + ''' + broadcast_message = ctypes.create_unicode_buffer('Environment') + user32 = ctypes.windll('user32', use_last_error=True) + result = user32.SendMessageTimeoutW(HWND_BROADCAST, WM_SETTINGCHANGE, 0, + broadcast_message, SMTO_ABORTIFHUNG, + 5000, 0) + return result == 1 From a3f9e99bc02c3887a38932511f4aa8302247cfaa Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 5 Feb 2018 15:47:16 -0700 Subject: [PATCH 05/51] Change to a generic function to broadcast change --- salt/modules/reg.py | 6 +++--- salt/modules/win_path.py | 2 +- salt/utils/win_functions.py | 26 ++++++++++++++++++++------ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/salt/modules/reg.py b/salt/modules/reg.py index a7df4effa5..3221b58d05 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -29,8 +29,8 @@ Values/Entries are name/data pairs. There can be many values in a key. The # When production windows installer is using Python 3, Python 2 code can be removed # Import _future_ python libs first & before any other code -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals + # Import python libs import sys import logging @@ -193,7 +193,7 @@ def broadcast_change(): salt '*' reg.broadcast_change ''' - return salt.utils.win_functions.refresh_environment() + return salt.utils.win_functions.broadcast_setting_change('Environment') def list_keys(hive, key=None, use_32bit_registry=False): diff --git a/salt/modules/win_path.py b/salt/modules/win_path.py index b70ab423fb..281e0211f2 100644 --- a/salt/modules/win_path.py +++ b/salt/modules/win_path.py @@ -62,7 +62,7 @@ def rehash(): salt '*' win_path.rehash ''' - return salt.utils.win_functions.refresh_environment() + return salt.utils.win_functions.broadcast_setting_change('Environment') def get_path(): diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index a1f75a5a29..f5ce159f6f 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -3,7 +3,7 @@ Various functions to be used by windows during start up and to monkey patch missing functions in other modules ''' -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import platform import re import ctypes @@ -214,10 +214,24 @@ def escape_for_cmd_exe(arg): return meta_re.sub(escape_meta_chars, arg) -def refresh_environment(): +def broadcast_setting_change(setting='Environment'): ''' - Send a WM_SETTINGCHANGE Broadcast to Windows to refresh the Environment - variables for new processes. + Send a WM_SETTINGCHANGE Broadcast to all Windows + + Args: + + setting (str): + A string value representing the portion of the system that has been + updated and needs to be refreshed. Default is ``Environment``. These + are some common values: + + - "Environment" : to effect a change in the environment variables + - "intl" : to effect a change in locale settings + - "Policy" : to effect a change in Group Policy Settings + - a leaf node in the registry + - the name of a section in the ``Win.ini`` file + + `See here ` .. note:: This will only affect new processes that aren't launched by services. To @@ -233,8 +247,8 @@ def refresh_environment(): import salt.utils.win_functions salt.utils.win_functions.refresh_environment() ''' - broadcast_message = ctypes.create_unicode_buffer('Environment') - user32 = ctypes.windll('user32', use_last_error=True) + broadcast_message = ctypes.create_unicode_buffer(setting) + user32 = ctypes.WinDLL('user32', use_last_error=True) result = user32.SendMessageTimeoutW(HWND_BROADCAST, WM_SETTINGCHANGE, 0, broadcast_message, SMTO_ABORTIFHUNG, 5000, 0) From 3a54e09cd9e22d7b5512b4f242ad4beac1852080 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 6 Feb 2018 16:12:11 -0700 Subject: [PATCH 06/51] Rename setting to message --- salt/utils/win_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index f5ce159f6f..2a659299de 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -214,13 +214,13 @@ def escape_for_cmd_exe(arg): return meta_re.sub(escape_meta_chars, arg) -def broadcast_setting_change(setting='Environment'): +def broadcast_setting_change(message='Environment'): ''' Send a WM_SETTINGCHANGE Broadcast to all Windows Args: - setting (str): + message (str): A string value representing the portion of the system that has been updated and needs to be refreshed. Default is ``Environment``. These are some common values: @@ -247,7 +247,7 @@ def broadcast_setting_change(setting='Environment'): import salt.utils.win_functions salt.utils.win_functions.refresh_environment() ''' - broadcast_message = ctypes.create_unicode_buffer(setting) + broadcast_message = ctypes.create_unicode_buffer(message) user32 = ctypes.WinDLL('user32', use_last_error=True) result = user32.SendMessageTimeoutW(HWND_BROADCAST, WM_SETTINGCHANGE, 0, broadcast_message, SMTO_ABORTIFHUNG, From d50d5f582fdca5356b637ffd069ae6946f2b29ef Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 7 Feb 2018 14:58:11 -0700 Subject: [PATCH 07/51] Add additional info to docs for `broadcast_setting_change` --- salt/utils/win_functions.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 2a659299de..26a8674fd2 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -233,12 +233,23 @@ def broadcast_setting_change(message='Environment'): `See here ` + See lParam within msdn docs for + `WM_SETTINGCHANGE ` + for more information on Broadcasting Messages. + + See GWL_WNDPROC within msdn docs for + `SetWindowLong ` + for information on how to retrieve those messages. + .. note:: This will only affect new processes that aren't launched by services. To apply changes to the path or registry to services, the host must be restarted. The ``salt-minion``, if running as a service, will not see - changes to the environment until the system is restarted. See + changes to the environment until the system is restarted. Services + inherit their environment from ``services.exe`` which does not respond + to messaging events. See `MSDN Documentation ` + for more information. CLI Example: From e84628c1ebaef5e06ad0c900b7c118d7aec9a051 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 7 Feb 2018 15:28:53 -0700 Subject: [PATCH 08/51] Add some comments to the code --- salt/utils/win_functions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 26a8674fd2..1853afd276 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -258,6 +258,25 @@ def broadcast_setting_change(message='Environment'): import salt.utils.win_functions salt.utils.win_functions.refresh_environment() ''' + # Listen for messages sent by this would involve working with the + # SetWindowLong function. This can be accessed via the win32gui or through + # ctypes. You can find examples on how to do this by searching for + # `Accessing WGL_WNDPROC` on the internet. Here are some examples of how + # this might work: + # + # # using win32gui + # import win32con + # import win32gui + # old_function = win32gui.SetWindowLong(window_handle, win32con.GWL_WNDPROC, new_function) + # + # # using ctypes + # import ctypes + # import win32con + # from ctypes import c_long, c_int + # user32 = ctypes.WinDLL('user32', use_last_error=True) + # WndProcType = ctypes.WINFUNCTYPE(c_int, c_long, c_int, c_int) + # new_function = WndProcType + # old_function = user32.SetWindowLongW(window_handle, win32con.GWL_WNDPROC, new_function) broadcast_message = ctypes.create_unicode_buffer(message) user32 = ctypes.WinDLL('user32', use_last_error=True) result = user32.SendMessageTimeoutW(HWND_BROADCAST, WM_SETTINGCHANGE, 0, From adc594c183c0ec28a67f70a6d41e8347bb720913 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 7 Feb 2018 15:30:51 -0700 Subject: [PATCH 09/51] Remove duplicate link --- salt/utils/win_functions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 1853afd276..145020bc84 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -231,8 +231,6 @@ def broadcast_setting_change(message='Environment'): - a leaf node in the registry - the name of a section in the ``Win.ini`` file - `See here ` - See lParam within msdn docs for `WM_SETTINGCHANGE ` for more information on Broadcasting Messages. From 29912adc15709993e93ab48d0da23e7d497f0698 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 8 Feb 2018 17:43:03 -0700 Subject: [PATCH 10/51] Move the test_rehash test to test_win_functions --- salt/utils/win_functions.py | 2 +- tests/unit/modules/test_win_path.py | 38 +------------------------- tests/unit/utils/test_win_functions.py | 8 ++++++ 3 files changed, 10 insertions(+), 38 deletions(-) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 145020bc84..662da797a9 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -254,7 +254,7 @@ def broadcast_setting_change(message='Environment'): ... code-block:: python import salt.utils.win_functions - salt.utils.win_functions.refresh_environment() + salt.utils.win_functions.broadcast_setting_change('Environment') ''' # Listen for messages sent by this would involve working with the # SetWindowLong function. This can be accessed via the win32gui or through diff --git a/tests/unit/modules/test_win_path.py b/tests/unit/modules/test_win_path.py index 8c80b1e1f1..c8d08b9418 100644 --- a/tests/unit/modules/test_win_path.py +++ b/tests/unit/modules/test_win_path.py @@ -20,49 +20,13 @@ from tests.support.mock import ( import salt.modules.win_path as win_path -class MockWin32API(object): - ''' - Mock class for win32api - ''' - def __init__(self): - pass - - @staticmethod - def SendMessage(*args): - ''' - Mock method for SendMessage - ''' - return [args[0]] - - -class MockWin32Con(object): - ''' - Mock class for win32con - ''' - HWND_BROADCAST = 1 - WM_SETTINGCHANGE = 1 - - def __init__(self): - pass - - @skipIf(NO_MOCK, NO_MOCK_REASON) class WinPathTestCase(TestCase, LoaderModuleMockMixin): ''' Test cases for salt.modules.win_path ''' def setup_loader_modules(self): - return {win_path: {'win32api': MockWin32API, - 'win32con': MockWin32Con, - 'SendMessage': MagicMock, - 'HWND_BROADCAST': MagicMock, - 'WM_SETTINGCHANGE': MagicMock}} - - def test_rehash(self): - ''' - Test to rehash the Environment variables - ''' - self.assertTrue(win_path.rehash()) + return {win_path: {}} def test_get_path(self): ''' diff --git a/tests/unit/utils/test_win_functions.py b/tests/unit/utils/test_win_functions.py index bbd0ef86fd..c35ab697da 100644 --- a/tests/unit/utils/test_win_functions.py +++ b/tests/unit/utils/test_win_functions.py @@ -12,6 +12,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.utils.win_functions as win_functions +import salt.utils @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -51,3 +52,10 @@ class WinFunctionsTestCase(TestCase): encoded = win_functions.escape_argument('C:\\Some Path\\With Spaces') self.assertEqual(encoded, '^"C:\\Some Path\\With Spaces^"') + + @skipIf(not salt.utils.is_windows(), 'WinDLL only available on Windows') + def test_broadcast_setting_change(self): + ''' + Test to rehash the Environment variables + ''' + self.assertTrue(win_functions.broadcast_setting_change()) From b9a2bc7b294f5bb7cf41232d7cfe883cebcd6f71 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 9 Feb 2018 09:53:14 -0700 Subject: [PATCH 11/51] Fix hyperlinks --- salt/modules/win_path.py | 2 +- salt/utils/win_functions.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/win_path.py b/salt/modules/win_path.py index 281e0211f2..833d605c01 100644 --- a/salt/modules/win_path.py +++ b/salt/modules/win_path.py @@ -54,7 +54,7 @@ def rehash(): apply changes to the path to services, the host must be restarted. The ``salt-minion``, if running as a service, will not see changes to the environment until the system is restarted. See - `MSDN Documentation ` + `MSDN Documentation `_ CLI Example: diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 662da797a9..db883566b8 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -232,11 +232,11 @@ def broadcast_setting_change(message='Environment'): - the name of a section in the ``Win.ini`` file See lParam within msdn docs for - `WM_SETTINGCHANGE ` + `WM_SETTINGCHANGE `_ for more information on Broadcasting Messages. See GWL_WNDPROC within msdn docs for - `SetWindowLong ` + `SetWindowLong `_ for information on how to retrieve those messages. .. note:: @@ -246,7 +246,7 @@ def broadcast_setting_change(message='Environment'): changes to the environment until the system is restarted. Services inherit their environment from ``services.exe`` which does not respond to messaging events. See - `MSDN Documentation ` + `MSDN Documentation `_ for more information. CLI Example: @@ -257,7 +257,7 @@ def broadcast_setting_change(message='Environment'): salt.utils.win_functions.broadcast_setting_change('Environment') ''' # Listen for messages sent by this would involve working with the - # SetWindowLong function. This can be accessed via the win32gui or through + # SetWindowLong function. This can be accessed via win32gui or through # ctypes. You can find examples on how to do this by searching for # `Accessing WGL_WNDPROC` on the internet. Here are some examples of how # this might work: From 2f712691cf79f9ec3fd249b2003d35426a5d7a64 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 17 Jan 2018 13:52:42 -0700 Subject: [PATCH 12/51] Exclude hidden directories in pkg.refresh_db --- salt/modules/win_pkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 4ba44d9b07..302dd714c7 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -650,7 +650,8 @@ def refresh_db(**kwargs): __salt__['cp.cache_dir']( repo_details.winrepo_source_dir, saltenv, - include_pat='*.sls' + include_pat='*.sls', + exclude_pat='E@\/\..*\/' # Exclude all hidden directories (.git) ) return genrepo(saltenv=saltenv, verbose=verbose, failhard=failhard) From 08b82e08752c4e452a66f96183241475e1b9726a Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 18 Jan 2018 10:52:46 -0700 Subject: [PATCH 13/51] Fix lint error, use raw --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 302dd714c7..3ab30cd353 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -651,7 +651,7 @@ def refresh_db(**kwargs): repo_details.winrepo_source_dir, saltenv, include_pat='*.sls', - exclude_pat='E@\/\..*\/' # Exclude all hidden directories (.git) + exclude_pat=r'E@\/\..*\/' # Exclude all hidden directories (.git) ) return genrepo(saltenv=saltenv, verbose=verbose, failhard=failhard) From 4803d92707d4325db8e655b744347d912d033de2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 22 Jan 2018 15:19:59 -0700 Subject: [PATCH 14/51] Add some documentation --- salt/modules/win_pkg.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 3ab30cd353..61fd4ef00e 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -581,9 +581,22 @@ def _refresh_db_conditional(saltenv, **kwargs): def refresh_db(**kwargs): ''' - Fetches metadata files and calls :py:func:`pkg.genrepo + Fetches metadata files from the winrepo_dir and calls :py:func:`pkg.genrepo ` to compile updated repository metadata. + The default winrepo directory on the master is `/srv/salt/win/repo-ng`. All + files that end with `.sls` in this and all subdirectories will be used to + generate the repository metadata (`winrepo.p`). + + .. note:: + Hidden directories (directories beginning with `.`, such as `.git`) will + be ignored. + + .. note:: + Directories under `/srv/salt/win/repo-ng` will be processed in + alphabetical order. If two software definition files contain the same + name, the last one processed wins. + Kwargs: saltenv (str): Salt environment. Default: ``base`` From 91c3da8dfd6445a3ade27ab209aa740b749e2d09 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 26 Jan 2018 12:56:08 -0700 Subject: [PATCH 15/51] Improve docs for pkg.refresh_db --- salt/modules/win_pkg.py | 63 ++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 61fd4ef00e..2dc7c29cce 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -580,41 +580,76 @@ def _refresh_db_conditional(saltenv, **kwargs): def refresh_db(**kwargs): - ''' - Fetches metadata files from the winrepo_dir and calls :py:func:`pkg.genrepo - ` to compile updated repository metadata. + r''' + Generates the local software metadata database (`winrepo.p`) on the minion. + The database is stored in a serialized format located by default at the + following location: + + `C:\salt\var\cache\salt\minion\files\base\win\repo-ng\winrepo.p` + + This module performs the following steps to generate the software metadata + database: + + - Fetch the package definition files (.sls) from `winrepo_source_dir` + (default `salt://win/repo-ng`) and cache them in + `\files\\` + (default: `C:\salt\var\cache\salt\minion\files\base\win\repo-ng`) + - Call :py:func:`pkg.genrepo ` to parse the + package definition files and generate the repository metadata database + file (`winrepo.p`) + - Return the report received from + :py:func:`pkg.genrepo ` The default winrepo directory on the master is `/srv/salt/win/repo-ng`. All files that end with `.sls` in this and all subdirectories will be used to - generate the repository metadata (`winrepo.p`). + generate the repository metadata database (`winrepo.p`). .. note:: - Hidden directories (directories beginning with `.`, such as `.git`) will - be ignored. + - Hidden directories (directories beginning with '`.`', such as + '`.git`') will be ignored. .. note:: - Directories under `/srv/salt/win/repo-ng` will be processed in - alphabetical order. If two software definition files contain the same - name, the last one processed wins. + There is no need to call `pkg.refresh_db` every time you work with the + pkg module. Automatic refresh will occur based on the following minion + configuration settings: + - `winrepo_cache_expire_min` + - `winrepo_cache_expire_max` + However, if the package definition files have changed, this function + should be called to ensure the minion has the latest information about + packages available to it. + + .. warning:: + Directories and files fetched from + (`/srv/salt/win/repo-ng`) will be processed in alphabetical order. If + two or more software definition files contain the same name, the last + one processed replaces all data from the files processed before it. + + For more information see + :ref:`Windows Software Repository ` Kwargs: saltenv (str): Salt environment. Default: ``base`` verbose (bool): - Return verbose data structure which includes 'success_list', a list - of all sls files and the package names contained within. Default - 'False' + Return a verbose data structure which includes 'success_list', a + list of all sls files and the package names contained within. + Default is 'False' failhard (bool): - If ``True``, an error will be raised if any repo SLS files failed to + If ``True``, an error will be raised if any repo SLS files fails to process. If ``False``, no error will be raised, and a dictionary containing the full results will be returned. Returns: dict: A dictionary containing the results of the database refresh. - .. Warning:: + .. note:: + A result with a `total: 0` generally means that the files are in the + wrong location on the master. Try running the following command on the + minion: `salt-call -l debug pkg.refresh saltenv=base` + + .. warning:: When calling this command from a state using `module.run` be sure to pass `failhard: False`. Otherwise the state will report failure if it encounters a bad software definition file. From 35c81faf5a88d67de76e24bb99bb34f4684b71ca Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 26 Jan 2018 13:04:47 -0700 Subject: [PATCH 16/51] Log the source_dir when caching the files --- salt/modules/win_pkg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 2dc7c29cce..476f8a39f5 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -695,6 +695,7 @@ def refresh_db(**kwargs): ) # Cache repo-ng locally + log.info('Fetching *.sls files from {0}'.format(repo_details.winrepo_source_dir)) __salt__['cp.cache_dir']( repo_details.winrepo_source_dir, saltenv, From 3646d5c8975ceee58056445a883c1f90c0b2bda3 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 26 Jan 2018 14:14:13 -0700 Subject: [PATCH 17/51] Fix some docs formatting, add some warnings --- .../windows/windows-package-manager.rst | 383 ++++++++++-------- salt/modules/win_pkg.py | 14 +- 2 files changed, 216 insertions(+), 181 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index cea071e888..597e389f45 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -301,25 +301,30 @@ can define multiple versions for the same piece of software. The lines following the version are indented two more spaces and contain all the information needed to install that package. -.. warning:: The package name and the ``full_name`` must be unique to all - other packages in the software repository. +.. warning:: + The package name and the ``full_name`` must be unique to all other packages + in the software repository. The version line is the version for the package to be installed. It is used when you need to install a specific version of a piece of software. -.. warning:: The version must be enclosed in quotes, otherwise the yaml parser - will remove trailing zeros. +.. warning:: + The version must be enclosed in quotes, otherwise the yaml parser will + remove trailing zeros. + +.. note:: + There are unique situations where previous versions are unavailable. Take + Google Chrome for example. There is only one url provided for a standalone + installation of Google Chrome. -.. note:: There are unique situations where previous versions are unavailable. - Take Google Chrome for example. There is only one url provided for a - standalone installation of Google Chrome. (https://dl.google.com/edgedl/chrome/install/GoogleChromeStandaloneEnterprise.msi) + When a new version is released, the url just points to the new version. To handle situations such as these, set the version to `latest`. Salt will install the version of Chrome at the URL and report that version. Here's an example: -.. code-block:: bash +.. code-block:: yaml chrome: latest: @@ -334,200 +339,230 @@ you need to install a specific version of a piece of software. Available parameters are as follows: -:param str full_name: The Full Name for the software as shown in "Programs and - Features" in the control panel. You can also get this information by - installing the package manually and then running ``pkg.list_pkgs``. Here's - an example of the output from ``pkg.list_pkgs``: +:param str full_name: + The Full Name for the software as shown in "Programs and Features" in the + control panel. You can also get this information by installing the package + manually and then running ``pkg.list_pkgs``. Here's an example of the output + from ``pkg.list_pkgs``: -.. code-block:: bash + .. code-block:: bash - salt 'test-2008' pkg.list_pkgs - test-2008 - ---------- - 7-Zip 9.20 (x64 edition): - 9.20.00.0 - Microsoft .NET Framework 4 Client Profile: - 4.0.30319,4.0.30319 - Microsoft .NET Framework 4 Extended: - 4.0.30319,4.0.30319 - Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022: - 9.0.21022 - Mozilla Firefox 17.0.1 (x86 en-US): - 17.0.1 - Mozilla Maintenance Service: - 17.0.1 - NSClient++ (x64): - 0.3.8.76 - Notepad++: - 6.4.2 - Salt Minion 0.16.0: - 0.16.0 + salt 'test-2008' pkg.list_pkgs + test-2008 + ---------- + 7-Zip 9.20 (x64 edition): + 9.20.00.0 + Microsoft .NET Framework 4 Client Profile: + 4.0.30319,4.0.30319 + Microsoft .NET Framework 4 Extended: + 4.0.30319,4.0.30319 + Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022: + 9.0.21022 + Mozilla Firefox 17.0.1 (x86 en-US): + 17.0.1 + Mozilla Maintenance Service: + 17.0.1 + NSClient++ (x64): + 0.3.8.76 + Notepad++: + 6.4.2 + Salt Minion 0.16.0: + 0.16.0 -Notice the Full Name for Firefox: Mozilla Firefox 17.0.0 (x86 en-US). That's -exactly what's in the ``full_name`` parameter in the software definition file. + Notice the Full Name for Firefox: ``Mozilla Firefox 17.0.0 (x86 en-US)``. + That's exactly what's in the ``full_name`` parameter in the software + definition file. -If any of the software insalled on the machine matches one of the software -definition files in the repository the full_name will be automatically renamed -to the package name. The example below shows the ``pkg.list_pkgs`` for a -machine that already has Mozilla Firefox 17.0.1 installed. + If any of the software installed on the machine matches one of the software + definition files in the repository, the full_name will be automatically + renamed to the package name. The example below shows the ``pkg.list_pkgs`` + for a machine that already has Mozilla Firefox 17.0.1 installed. -.. code-block:: bash + .. code-block:: bash + + test-2008: + ---------- + 7zip: + 9.20.00.0 + Microsoft .NET Framework 4 Client Profile: + 4.0.30319,4.0.30319 + Microsoft .NET Framework 4 Extended: + 4.0.30319,4.0.30319 + Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022: + 9.0.21022 + Mozilla Maintenance Service: + 17.0.1 + Notepad++: + 6.4.2 + Salt Minion 0.16.0: + 0.16.0 + firefox: + 17.0.1 + nsclient: + 0.3.9.328 + + .. important:: + The version number and ``full_name`` need to match the output from + ``pkg.list_pkgs`` so that the status can be verified when running a + highstate. + + .. note:: + It is still possible to successfully install packages using + ``pkg.install``, even if the ``full_name`` or the version number don't + match. However, this can make troubleshooting issues difficult, so be + careful. + +:param str installer: + The path to the ``.exe`` or ``.msi`` to use to install the package. This can + be a path or a URL. If it is a URL or a salt path (``salt://``), the package + will be cached locally and then executed. If it is a path to a file on disk + or a file share, it will be executed directly. + + .. note:: + If storing software in the same location as the winrepo it is best + practice to place each installer in its own directory rather than the + root of winrepo. Then you can place your package definition file in the + same directory with a name of ``init.sls``. This will be picked up + by ``pkg.refresh_db`` and processed properly. + +:param str install_flags: + Any flags that need to be passed to the installer to make it perform a + silent install. These can often be found by adding ``/?`` or ``/h`` when + running the installer from the command-line. A great resource for finding + these silent install flags can be found on the WPKG project's wiki_: + + .. warning:: + Salt will not return if the installer is waiting for user input so it is + imperative that the software package being installed has the ability to + install silently. + +:param str uninstaller: + The path to the program used to uninstall this software. This can be the + path to the same `exe` or `msi` used to install the software. It can also be + a GUID. You can find this value in the registry under the following keys: + + - Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall + - Software\\Wow6432None\\Microsoft\\Windows\\CurrentVersion\\Uninstall + +:param str uninstall_flags: + Any flags that need to be passed to the uninstaller to make it perform a + silent uninstall. These can often be found by adding ``/?`` or ``/h`` when + running the uninstaller from the command-line. A great resource for finding + these silent install flags can be found on the WPKG project's wiki_: + + .. warning:: + Salt will not return if the uninstaller is waiting for user input so it + is imperative that the software package being uninstalled has the + ability to uninstall silently. + + Here are some examples of installer and uninstaller settings: + + .. code-block:: yaml - test-2008: - ---------- 7zip: - 9.20.00.0 - Microsoft .NET Framework 4 Client Profile: - 4.0.30319,4.0.30319 - Microsoft .NET Framework 4 Extended: - 4.0.30319,4.0.30319 - Microsoft Visual C++ 2008 Redistributable - x64 9.0.21022: - 9.0.21022 - Mozilla Maintenance Service: - 17.0.1 - Notepad++: - 6.4.2 - Salt Minion 0.16.0: - 0.16.0 - firefox: - 17.0.1 - nsclient: - 0.3.9.328 + '9.20.00.0': + installer: salt://win/repo/7zip/7z920-x64.msi + full_name: 7-Zip 9.20 (x64 edition) + reboot: False + install_flags: '/qn /norestart' + msiexec: True + uninstaller: '{23170F69-40C1-2702-0920-000001000000}' + uninstall_flags: '/qn /norestart' -.. important:: The version number and ``full_name`` need to match the output - from ``pkg.list_pkgs`` so that the status can be verified when running - highstate. + Alternatively the ``uninstaller`` can also simply repeat the URL of an msi + file: -.. note:: It is still possible to successfully install packages using - ``pkg.install`` even if they don't match. This can make troubleshooting - difficult so be careful. + .. code-block:: yaml -:param str installer: The path to the ``.exe`` or ``.msi`` to use to install the - package. This can be a path or a URL. If it is a URL or a salt path - (salt://), the package will be cached locally and then executed. If it is a - path to a file on disk or a file share, it will be executed directly. + 7zip: + '9.20.00.0': + installer: salt://win/repo/7zip/7z920-x64.msi + full_name: 7-Zip 9.20 (x64 edition) + reboot: False + install_flags: '/qn /norestart' + msiexec: True + uninstaller: salt://win/repo/7zip/7z920-x64.msi + uninstall_flags: '/qn /norestart' -:param str install_flags: Any flags that need to be passed to the installer to - make it perform a silent install. These can often be found by adding ``/?`` - or ``/h`` when running the installer from the command-line. A great resource - for finding these silent install flags can be found on the WPKG project's wiki_: +:param msiexec: + This tells salt to use ``msiexec /i`` to install the package and + ``msiexec /x`` to uninstall. This is for ``.msi`` installations. Possible + options are: True, False or the path to ``msiexec.exe`` on your system -Salt will not return if the installer is waiting for user input so these are -important. + .. code-block:: yaml -:param str uninstaller: The path to the program used to uninstall this software. - This can be the path to the same `exe` or `msi` used to install the - software. It can also be a GUID. You can find this value in the registry - under the following keys: + 7zip: + '9.20.00.0': + installer: salt://win/repo/7zip/7z920-x64.msi + full_name: 7-Zip 9.20 (x64 edition) + reboot: False + install_flags: '/qn /norestart' + msiexec: 'C:\Windows\System32\msiexec.exe' + uninstaller: salt://win/repo/7zip/7z920-x64.msi + uninstall_flags: '/qn /norestart' - - Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall - - Software\\Wow6432None\\Microsoft\\Windows\\CurrentVersion\\Uninstall +:param bool allusers: + This parameter is specific to ``.msi`` installations. It tells ``msiexec`` + to install the software for all users. The default is ``True``. -:param str uninstall_flags: Any flags that need to be passed to the uninstaller - to make it perform a silent uninstall. These can often be found by adding - ``/?`` or ``/h`` when running the uninstaller from the command-line. A great - resource for finding these silent install flags can be found on the WPKG - project's wiki_: +:param bool cache_dir: + If ``True`` and the installer URL begins with ``salt://``, the entire + directory where the installer resides will be recursively cached. This is + useful for installers that depend on other files in the same directory for + installation. -Salt will not return if the uninstaller is waiting for user input so these are -important. - -Here are some examples of installer and uninstaller settings: - -.. code-block:: yaml - - 7zip: - '9.20.00.0': - installer: salt://win/repo/7zip/7z920-x64.msi - full_name: 7-Zip 9.20 (x64 edition) - reboot: False - install_flags: '/qn /norestart' - msiexec: True - uninstaller: '{23170F69-40C1-2702-0920-000001000000}' - uninstall_flags: '/qn /norestart' - -Alternatively the ``uninstaller`` can also simply repeat the URL of the msi file. - -.. code-block:: yaml - - 7zip: - '9.20.00.0': - installer: salt://win/repo/7zip/7z920-x64.msi - full_name: 7-Zip 9.20 (x64 edition) - reboot: False - install_flags: '/qn /norestart' - msiexec: True - uninstaller: salt://win/repo/7zip/7z920-x64.msi - uninstall_flags: '/qn /norestart' - -:param msiexec: This tells salt to use ``msiexec /i`` to install the - package and ``msiexec /x`` to uninstall. This is for `.msi` installations. - Possible options are: True, False or path to msiexec on your system - - 7zip: - '9.20.00.0': - installer: salt://win/repo/7zip/7z920-x64.msi - full_name: 7-Zip 9.20 (x64 edition) - reboot: False - install_flags: '/qn /norestart' - msiexec: 'C:\Windows\System32\msiexec.exe' - uninstaller: salt://win/repo/7zip/7z920-x64.msi - uninstall_flags: '/qn /norestart' - -:param str arch: This selects which ``msiexec.exe`` to use. Possible values: - ``x86``, ``x64`` - -:param bool allusers: This parameter is specific to `.msi` installations. It - tells `msiexec` to install the software for all users. The default is True. - -:param bool cache_dir: If true when installer URL begins with salt://, the - entire directory where the installer resides will be recursively cached. - This is useful for installers that depend on other files in the same - directory for installation. + .. warning:: + Do not place installer files in the root of winrepo + (``/srv/salt/win/repo-ng``). If the installer is in the root of winrepo + and the package definition for that installer has ``cache_dir: True`` + then the entire contents of winrepo will be cached to the minion. :param str cache_file: - When installer URL begins with salt://, this indicates single file to copy - down for use with the installer. Copied to the same location as the - installer. Use this over ``cache_dir`` if there are many files in the + When the installer URL begins with ``salt://``, this indicates a single file + to copy down for use with the installer. It is copied to the same location + as the installer. Use this over ``cache_dir`` if there are many files in the directory and you only need a specific file and don't want to cache additional files that may reside in the installer directory. -Here's an example for a software package that has dependent files: + Here's an example for a software package that has dependent files: -.. code-block:: yaml + .. code-block:: yaml - sqlexpress: - '12.0.2000.8': - installer: 'salt://win/repo/sqlexpress/setup.exe' - full_name: Microsoft SQL Server 2014 Setup (English) - reboot: False - install_flags: '/ACTION=install /IACCEPTSQLSERVERLICENSETERMS /Q' - cache_dir: True + sqlexpress: + '12.0.2000.8': + installer: 'salt://win/repo/sqlexpress/setup.exe' + full_name: Microsoft SQL Server 2014 Setup (English) + reboot: False + install_flags: '/ACTION=install /IACCEPTSQLSERVERLICENSETERMS /Q' + cache_dir: True -:param bool use_scheduler: If true, windows will use the task scheduler to run - the installation. This is useful for running the salt installation itself as - the installation process kills any currently running instances of salt. +:param bool use_scheduler: + If ``True``, Windows will use the task scheduler to run the installation. + This is useful for running the Salt installation itself as the installation + process kills any currently running instances of Salt. -:param str source_hash: This tells salt to compare a hash sum of the installer -to the provided hash sum before execution. The value can be formatted as -``hash_algorithm=hash_sum``, or it can be a URI to a file containing the hash -sum. -For a list of supported algorithms, see the `hashlib documentation -`_. +:param str source_hash: + This tells Salt to compare a hash sum of the installer to the provided hash + sum before execution. The value can be formatted as + ``=``, or it can be a URI to a file containing the + hash sum. -Here's an example of source_hash usage: + For a list of supported algorithms, see the `hashlib documentation + `_. -.. code-block:: yaml + Here's an example of source_hash usage: - messageanalyzer: - '4.0.7551.0': - full_name: 'Microsoft Message Analyzer' - installer: 'salt://win/repo/messageanalyzer/MessageAnalyzer64.msi' - install_flags: '/quiet /norestart' - uninstaller: '{1CC02C23-8FCD-487E-860C-311EC0A0C933}' - uninstall_flags: '/quiet /norestart' - msiexec: True - source_hash: 'sha1=62875ff451f13b10a8ff988f2943e76a4735d3d4' + .. code-block:: yaml + + messageanalyzer: + '4.0.7551.0': + full_name: 'Microsoft Message Analyzer' + installer: 'salt://win/repo/messageanalyzer/MessageAnalyzer64.msi' + install_flags: '/quiet /norestart' + uninstaller: '{1CC02C23-8FCD-487E-860C-311EC0A0C933}' + uninstall_flags: '/quiet /norestart' + msiexec: True + source_hash: 'sha1=62875ff451f13b10a8ff988f2943e76a4735d3d4' :param bool reboot: Not implemented diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 476f8a39f5..145697f1d1 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -697,8 +697,8 @@ def refresh_db(**kwargs): # Cache repo-ng locally log.info('Fetching *.sls files from {0}'.format(repo_details.winrepo_source_dir)) __salt__['cp.cache_dir']( - repo_details.winrepo_source_dir, - saltenv, + path=repo_details.winrepo_source_dir, + saltenv=saltenv, include_pat='*.sls', exclude_pat=r'E@\/\..*\/' # Exclude all hidden directories (.git) ) @@ -1293,11 +1293,11 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # single files if cache_dir and installer.startswith('salt:'): path, _ = os.path.split(installer) - __salt__['cp.cache_dir'](path, - saltenv, - False, - None, - 'E@init.sls$') + __salt__['cp.cache_dir'](path=path, + saltenv=saltenv, + include_empty=False, + include_pat=None, + exclude_pat='E@init.sls$') # Check to see if the cache_file is cached... if passed if cache_file and cache_file.startswith('salt:'): From 87dc554dc388e36b091b4e448acc9ffb2ea77810 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 31 Jan 2018 14:48:13 -0700 Subject: [PATCH 18/51] Add final updates to docs --- doc/topics/windows/windows-package-manager.rst | 14 ++++++++------ salt/modules/win_pkg.py | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 597e389f45..6d5186ff49 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -422,8 +422,8 @@ Available parameters are as follows: If storing software in the same location as the winrepo it is best practice to place each installer in its own directory rather than the root of winrepo. Then you can place your package definition file in the - same directory with a name of ``init.sls``. This will be picked up - by ``pkg.refresh_db`` and processed properly. + same directory. It is best practice to name the file ``init.sls``. This + will be picked up by ``pkg.refresh_db`` and processed properly. :param str install_flags: Any flags that need to be passed to the installer to make it perform a @@ -512,10 +512,12 @@ Available parameters are as follows: installation. .. warning:: - Do not place installer files in the root of winrepo - (``/srv/salt/win/repo-ng``). If the installer is in the root of winrepo - and the package definition for that installer has ``cache_dir: True`` - then the entire contents of winrepo will be cached to the minion. + Be aware that all files and directories in the same location as the + installer file will be copied down to the minion. If you place your + installer file in the root of winrepo (``/srv/salt/win/repo-ng``) and + ``cache_dir: True`` the entire contents of winrepo will be cached to + the minion. Therefore, it is best practice to place your installer files + in a subdirectory if they are to be stored in winrepo. :param str cache_file: When the installer URL begins with ``salt://``, this indicates a single file diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 145697f1d1..a8fa9bb183 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -614,7 +614,8 @@ def refresh_db(**kwargs): configuration settings: - `winrepo_cache_expire_min` - `winrepo_cache_expire_max` - However, if the package definition files have changed, this function + However, if the package definition files have changed, as would be the + case if you are developing a new package definition, this function should be called to ensure the minion has the latest information about packages available to it. From 437a457911e8dfd3d80db93a69859e62c83b5af3 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 5 Feb 2018 16:51:22 -0700 Subject: [PATCH 19/51] Skip hidden dirs in genrepo --- salt/modules/win_pkg.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index a8fa9bb183..56c18a1dde 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -838,6 +838,10 @@ def genrepo(**kwargs): to process. If ``False``, no error will be raised, and a dictionary containing the full results will be returned. + .. note:: + - Hidden directories (directories beginning with '`.`', such as + '`.git`') will be ignored. + Returns: dict: A dictionary of the results of the command @@ -864,6 +868,10 @@ def genrepo(**kwargs): short_path = os.path.relpath(root, repo_details.local_dest) if short_path == '.': short_path = '' + if re.search(r'[\\/]\..*', root): + log.debug('Skipping files in directory: {0}'.format(root)) + continue + for name in files: if name.endswith('.sls'): total_files_processed += 1 From 1282ae3a93df123e7196b8257367a7e5b8d6e651 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 5 Feb 2018 17:08:11 -0700 Subject: [PATCH 20/51] Skip hidden first --- salt/modules/win_pkg.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 56c18a1dde..2cfa7f8172 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -865,13 +865,16 @@ def genrepo(**kwargs): repo_details = _get_repo_details(saltenv) for root, _, files in os.walk(repo_details.local_dest, followlinks=False): - short_path = os.path.relpath(root, repo_details.local_dest) - if short_path == '.': - short_path = '' + + # Skip hidden directories (.git) if re.search(r'[\\/]\..*', root): log.debug('Skipping files in directory: {0}'.format(root)) continue + short_path = os.path.relpath(root, repo_details.local_dest) + if short_path == '.': + short_path = '' + for name in files: if name.endswith('.sls'): total_files_processed += 1 From 6d223cffa76b8a9a224490933dcca3edd0251f7b Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 6 Feb 2018 16:05:40 -0700 Subject: [PATCH 21/51] Add tip about passing bogus saltenv --- doc/topics/windows/windows-package-manager.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 6d5186ff49..1c5be46a9a 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -412,6 +412,11 @@ Available parameters are as follows: match. However, this can make troubleshooting issues difficult, so be careful. + .. tip:: + To force salt to display the full name when there's already an existing + package definition file on the system, you can pass a bogus ``saltenv`` + parameter to the command like so: ``pkg.list_pkgs saltenv=NotARealEnv`` + :param str installer: The path to the ``.exe`` or ``.msi`` to use to install the package. This can be a path or a URL. If it is a URL or a salt path (``salt://``), the package From ea41215646c765f026c179a8cbe034649ded2928 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 12 Feb 2018 14:03:04 -0700 Subject: [PATCH 22/51] Make the regex pattern less greedy --- salt/modules/win_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 2cfa7f8172..588e2ed38a 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -701,7 +701,7 @@ def refresh_db(**kwargs): path=repo_details.winrepo_source_dir, saltenv=saltenv, include_pat='*.sls', - exclude_pat=r'E@\/\..*\/' # Exclude all hidden directories (.git) + exclude_pat=r'E@\/\..*?\/' # Exclude all hidden directories (.git) ) return genrepo(saltenv=saltenv, verbose=verbose, failhard=failhard) From 679787699c0c972b909521a8b9ce21b43f6f39ec Mon Sep 17 00:00:00 2001 From: Ben Gridley Date: Mon, 12 Feb 2018 16:06:32 -0700 Subject: [PATCH 23/51] Add vpc_peering_connection_id to describe_route_tables route_keys This is required to fix state module function _routes_present when using a vpc_peering_connection. --- salt/modules/boto_vpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/boto_vpc.py b/salt/modules/boto_vpc.py index bebaafdd57..2fa6da3c2d 100644 --- a/salt/modules/boto_vpc.py +++ b/salt/modules/boto_vpc.py @@ -2576,6 +2576,7 @@ def describe_route_tables(route_table_id=None, route_table_name=None, 'instance_id': 'Instance', 'interface_id': 'NetworkInterfaceId', 'nat_gateway_id': 'NatGatewayId', + 'vpc_peering_connection_id': 'VpcPeeringConnectionId', } assoc_keys = {'id': 'RouteTableAssociationId', 'main': 'Main', From 13cdb526905e3cff1de8975e0c4099f1f89d4ca2 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 13 Feb 2018 11:41:41 -0600 Subject: [PATCH 24/51] cmdmod.py: runas workaround for platforms that don't set a USER env var Solaris doesn't set a USER env var in its default environment, that is if you run `su - someuser -c env` you get a HOME, PWD, LOGNAME, etc. env var, but not a USER. This commit makes sure that the USER env var is set to the runas user. --- salt/modules/cmdmod.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 233dca8647..976b56d488 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -465,6 +465,10 @@ def _run(cmd, else: run_env = os.environ.copy() run_env.update(env) + # Fix platforms like Solaris that don't set a USER env var in the + # user's default environment as obtained above. + if runas is not None and run_env.get('USER') != runas: + run_env['USER'] = runas if python_shell is None: python_shell = False From ebb244396b536fcd332e9c2838a1995a218ca4af Mon Sep 17 00:00:00 2001 From: Lee Webb Date: Fri, 19 Jan 2018 09:28:16 +1100 Subject: [PATCH 25/51] Patch around ResourceRecords needing to be present for AliasTarget entries to work --- salt/states/boto3_route53.py | 92 ++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/salt/states/boto3_route53.py b/salt/states/boto3_route53.py index 0a49dcc3fe..78ac92cdba 100644 --- a/salt/states/boto3_route53.py +++ b/salt/states/boto3_route53.py @@ -595,54 +595,55 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name # Convert any magic RR values to something AWS will understand, and otherwise clean them up. fixed_rrs = [] - for rr in ResourceRecords: - if rr.startswith('magic:'): - fields = rr.split(':') - if fields[1] == 'ec2_instance_tag': - if len(fields) != 5: - log.warning("Invalid magic RR value seen: '{}'. Passing as-is.".format(rr)) - fixed_rrs += [rr] - continue - tag_name = fields[2] - tag_value = fields[3] - instance_attr = fields[4] - good_states = ('pending', 'rebooting', 'running', 'stopping', 'stopped') - r = __salt__['boto_ec2.find_instances']( - tags={tag_name: tag_value}, return_objs=True, in_states=good_states, - region=region, key=key, keyid=keyid, profile=profile) - if len(r) < 1: - ret['comment'] = 'No EC2 instance with tag {} == {} found'.format(tag_name, - tag_value) - log.error(ret['comment']) - ret['result'] = False - return ret - if len(r) > 1: - ret['comment'] = 'Multiple EC2 instances with tag {} == {} found'.format( - tag_name, tag_value) - log.error(ret['comment']) - ret['result'] = False - return ret - instance = r[0] - res = getattr(instance, instance_attr, None) - if res: - log.debug('Found {} {} for instance {}'.format(instance_attr, res, instance.id)) - fixed_rrs += [_to_aws_encoding(res)] + if ResourceRecords: + for rr in ResourceRecords: + if rr.startswith('magic:'): + fields = rr.split(':') + if fields[1] == 'ec2_instance_tag': + if len(fields) != 5: + log.warning("Invalid magic RR value seen: '{}'. Passing as-is.".format(rr)) + fixed_rrs += [rr] + continue + tag_name = fields[2] + tag_value = fields[3] + instance_attr = fields[4] + good_states = ('pending', 'rebooting', 'running', 'stopping', 'stopped') + r = __salt__['boto_ec2.find_instances']( + tags={tag_name: tag_value}, return_objs=True, in_states=good_states, + region=region, key=key, keyid=keyid, profile=profile) + if len(r) < 1: + ret['comment'] = 'No EC2 instance with tag {} == {} found'.format(tag_name, + tag_value) + log.error(ret['comment']) + ret['result'] = False + return ret + if len(r) > 1: + ret['comment'] = 'Multiple EC2 instances with tag {} == {} found'.format( + tag_name, tag_value) + log.error(ret['comment']) + ret['result'] = False + return ret + instance = r[0] + res = getattr(instance, instance_attr, None) + if res: + log.debug('Found {} {} for instance {}'.format(instance_attr, res, instance.id)) + fixed_rrs += [_to_aws_encoding(res)] + else: + ret['comment'] = 'Attribute {} not found on instance {}'.format(instance_attr, + instance.id) + log.error(ret['comment']) + ret['result'] = False + return ret else: - ret['comment'] = 'Attribute {} not found on instance {}'.format(instance_attr, - instance.id) + ret['comment'] = ('Unknown RR magic value seen: {}. Please extend the ' + 'boto3_route53 state module to add support for your preferred ' + 'incantation.'.format(fields[1])) log.error(ret['comment']) ret['result'] = False return ret else: - ret['comment'] = ('Unknown RR magic value seen: {}. Please extend the ' - 'boto3_route53 state module to add support for your preferred ' - 'incantation.'.format(fields[1])) - log.error(ret['comment']) - ret['result'] = False - return ret - else: - fixed_rrs += [rr] - ResourceRecords = [{'Value': rr} for rr in sorted(fixed_rrs)] + fixed_rrs += [rr] + ResourceRecords = [{'Value': rr} for rr in sorted(fixed_rrs)] recordsets = __salt__['boto3_route53.get_resource_records'](HostedZoneId=HostedZoneId, StartRecordName=Name, StartRecordType=Type, region=region, key=key, keyid=keyid, @@ -691,9 +692,10 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name return ret ResourceRecordSet = { 'Name': Name, - 'Type': Type, - 'ResourceRecords': ResourceRecords + 'Type': Type } + if ResourceRecords: + ResourceRecordSet['ResourceRecords'] = ResourceRecords for u in updatable: ResourceRecordSet.update({u: locals().get(u)}) if locals().get(u) else None From f427b0febc84232a5f63cdc7fad450eb8538ce0b Mon Sep 17 00:00:00 2001 From: Lee Webb Date: Thu, 25 Jan 2018 10:35:18 +1100 Subject: [PATCH 26/51] Change formatting style of logging lines per review --- salt/states/boto3_route53.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/boto3_route53.py b/salt/states/boto3_route53.py index 78ac92cdba..28c9117ae6 100644 --- a/salt/states/boto3_route53.py +++ b/salt/states/boto3_route53.py @@ -601,7 +601,7 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name fields = rr.split(':') if fields[1] == 'ec2_instance_tag': if len(fields) != 5: - log.warning("Invalid magic RR value seen: '{}'. Passing as-is.".format(rr)) + log.warning("Invalid magic RR value seen: '%s'. Passing as-is." % (rr)) fixed_rrs += [rr] continue tag_name = fields[2] @@ -626,7 +626,7 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name instance = r[0] res = getattr(instance, instance_attr, None) if res: - log.debug('Found {} {} for instance {}'.format(instance_attr, res, instance.id)) + log.debug('Found %s %s for instance %s' % (instance_attr, res, instance.id)) fixed_rrs += [_to_aws_encoding(res)] else: ret['comment'] = 'Attribute {} not found on instance {}'.format(instance_attr, From 2ea3fef543395121a76a7cce06c3b11f7131863f Mon Sep 17 00:00:00 2001 From: Lee Webb Date: Mon, 29 Jan 2018 08:52:15 +1100 Subject: [PATCH 27/51] No lazy logging --- salt/states/boto3_route53.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/boto3_route53.py b/salt/states/boto3_route53.py index 28c9117ae6..54a66bdb07 100644 --- a/salt/states/boto3_route53.py +++ b/salt/states/boto3_route53.py @@ -601,7 +601,7 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name fields = rr.split(':') if fields[1] == 'ec2_instance_tag': if len(fields) != 5: - log.warning("Invalid magic RR value seen: '%s'. Passing as-is." % (rr)) + log.warning("Invalid magic RR value seen: '%s'. Passing as-is.", rr) fixed_rrs += [rr] continue tag_name = fields[2] @@ -626,7 +626,7 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name instance = r[0] res = getattr(instance, instance_attr, None) if res: - log.debug('Found %s %s for instance %s' % (instance_attr, res, instance.id)) + log.debug('Found %s %s for instance %s', instance_attr, res, instance.id) fixed_rrs += [_to_aws_encoding(res)] else: ret['comment'] = 'Attribute {} not found on instance {}'.format(instance_attr, From 8ee0a3a28bd5f7516cc897d48af4e88504f20b07 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 13 Feb 2018 13:42:20 -0600 Subject: [PATCH 28/51] Move Solaris USER workaround up a bit In its previous position, the run_env would not get the workaround when clean_env was set to True. --- salt/modules/cmdmod.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 976b56d488..794e924216 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -423,6 +423,10 @@ def _run(cmd, env_encoded = env_encoded.encode(__salt_system_encoding__) env_runas = dict(list(zip(*[iter(env_encoded.split(b'\0'))]*2))) env_runas.update(env) + # Fix platforms like Solaris that don't set a USER env var in the + # user's default environment as obtained above. + if env_runas.get('USER') != runas: + env_runas['USER'] = runas env = env_runas # Encode unicode kwargs to filesystem encoding to avoid a # UnicodeEncodeError when the subprocess is invoked. @@ -465,10 +469,6 @@ def _run(cmd, else: run_env = os.environ.copy() run_env.update(env) - # Fix platforms like Solaris that don't set a USER env var in the - # user's default environment as obtained above. - if runas is not None and run_env.get('USER') != runas: - run_env['USER'] = runas if python_shell is None: python_shell = False From 8cf13325eec08ba8e9418e2d26677f7900185b8c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 12 Feb 2018 22:47:59 -0600 Subject: [PATCH 29/51] salt.states.reg.present: Prevent traceback when reg data is binary This prevents a failed decode of undecodable data from resulting in a traceback by catching the exception and just using the original value in the changes dict. --- salt/states/reg.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/states/reg.py b/salt/states/reg.py index d9bc8a60e5..99cca3b364 100644 --- a/salt/states/reg.py +++ b/salt/states/reg.py @@ -192,9 +192,14 @@ def present(name, salt.utils.to_unicode(name, 'utf-8')) return ret + try: + vdata_decoded = salt.utils.to_unicode(vdata, 'utf-8') + except UnicodeDecodeError: + # vdata contains binary data that can't be decoded + vdata_decoded = vdata add_change = {'Key': r'{0}\{1}'.format(hive, key), 'Entry': u'{0}'.format(salt.utils.to_unicode(vname, 'utf-8') if vname else u'(Default)'), - 'Value': salt.utils.to_unicode(vdata, 'utf-8')} + 'Value': vdata_decoded} # Check for test option if __opts__['test']: From bb1cdc451e40634895c54345b3e828aac467367a Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Tue, 13 Feb 2018 23:48:50 -0800 Subject: [PATCH 30/51] salt.pillar.file_tree no stack trace when nodegroups undefined --- salt/pillar/file_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/pillar/file_tree.py b/salt/pillar/file_tree.py index 1a825f80a8..7ab69d1252 100644 --- a/salt/pillar/file_tree.py +++ b/salt/pillar/file_tree.py @@ -328,7 +328,7 @@ def ext_pillar(minion_id, ngroup_pillar = {} nodegroups_dir = os.path.join(root_dir, 'nodegroups') - if os.path.exists(nodegroups_dir) and len(__opts__['nodegroups']) > 0: + if os.path.exists(nodegroups_dir) and len(__opts__.get('nodegroups', ())) > 0: master_ngroups = __opts__['nodegroups'] ext_pillar_dirs = os.listdir(nodegroups_dir) if len(ext_pillar_dirs) > 0: From 3c6a5bf96760b69ec86a7b43e1039f9a4036bc0f Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Tue, 13 Feb 2018 23:49:32 -0800 Subject: [PATCH 31/51] salt.pillar.file_tree provide better debug info --- salt/pillar/file_tree.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/pillar/file_tree.py b/salt/pillar/file_tree.py index 7ab69d1252..ede20f798b 100644 --- a/salt/pillar/file_tree.py +++ b/salt/pillar/file_tree.py @@ -354,8 +354,8 @@ def ext_pillar(minion_id, else: if debug is True: log.debug( - 'file_tree: no nodegroups found in file tree directory ' - 'ext_pillar_dirs, skipping...' + 'file_tree: no nodegroups found in file tree directory %s, skipping...', + ext_pillar_dirs ) else: if debug is True: @@ -363,7 +363,12 @@ def ext_pillar(minion_id, host_dir = os.path.join(root_dir, 'hosts', minion_id) if not os.path.exists(host_dir): - # No data for host with this ID + if debug is True: + log.debug( + 'file_tree: no pillar data for minion %s found in file tree directory %s', + minion_id, + host_dir + ) return ngroup_pillar if not os.path.isdir(host_dir): From 0ba4954a4ba4ca91c188354d46cba16e7c27d23d Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Tue, 13 Feb 2018 23:46:55 -0800 Subject: [PATCH 32/51] salt.pillar.file_tree revise module documentation --- salt/pillar/file_tree.py | 338 ++++++++++++++++++++++----------------- 1 file changed, 188 insertions(+), 150 deletions(-) diff --git a/salt/pillar/file_tree.py b/salt/pillar/file_tree.py index ede20f798b..97187bedde 100644 --- a/salt/pillar/file_tree.py +++ b/salt/pillar/file_tree.py @@ -1,175 +1,97 @@ # -*- coding: utf-8 -*- ''' +The ``file_tree`` external pillar allows values from all files in a directory +tree to be imported as Pillar data. -``File_tree`` is an external pillar that allows -values from all files in a directory tree to be imported as Pillar data. +.. note:: -Note this is an external pillar, and is subject to the rules and constraints -governing external pillars detailed here: :ref:`external-pillars`. + This is an external pillar and is subject to the :ref:`rules and + constraints ` governing external pillars. .. versionadded:: 2015.5.0 -Example Configuration ---------------------- +In this pillar, data is organized by either Minion ID or Nodegroup name. To +setup pillar data for a specific Minion, place it in +``/hosts/``. To setup pillar data for an entire +Nodegroup, place it in ``/nodegroups/`` where +```` is the Nodegroup's name. + +Example ``file_tree`` Pillar +============================ + +Master Configuration +-------------------- .. code-block:: yaml ext_pillar: - file_tree: - root_dir: /path/to/root/directory + root_dir: /srv/ext_pillar follow_dir_links: False keep_newline: True -The ``root_dir`` parameter is required and points to the directory where files -for each host are stored. The ``follow_dir_links`` parameter is optional and -defaults to False. If ``follow_dir_links`` is set to True, this external pillar -will follow symbolic links to other directories. + node_groups: + internal_servers: 'L@bob,stuart,kevin' -.. warning:: - Be careful when using ``follow_dir_links``, as a recursive symlink chain - will result in unexpected results. +Pillar Configuration +-------------------- -If ``keep_newline`` is set to ``True``, then the pillar values for files ending -in newlines will keep that newline. The default behavior is to remove the -end-of-file newline. ``keep_newline`` should be turned on if the pillar data is -intended to be used to deploy a file using ``contents_pillar`` with a -:py:func:`file.managed ` state. +.. code-block:: bash -.. versionchanged:: 2015.8.4 - The ``raw_data`` parameter has been renamed to ``keep_newline``. In earlier - releases, ``raw_data`` must be used. Also, this parameter can now be a list - of globs, allowing for more granular control over which pillar values keep - their end-of-file newline. The globs match paths relative to the - directories named for minion IDs and nodegroups underneath the ``root_dir`` - (see the layout examples in the below sections). + (salt-master) # tree /srv/ext_pillar + /srv/ext_pillar/ + |-- hosts + | |-- bob + | | |-- apache + | | | `-- config.d + | | | |-- 00_important.conf + | | | `-- 20_bob_extra.conf + | | `-- corporate_app + | | `-- settings + | | `-- bob_settings.cfg + | `-- kevin + | |-- apache + | | `-- config.d + | | `-- 00_important.conf + | `-- corporate_app + | `-- settings + | `-- kevin_settings.cfg + `-- nodegroups + `-- internal_servers + `-- corporate_app + `-- settings + `-- common_settings.cfg - .. code-block:: yaml +Verify Pillar Data +------------------ - ext_pillar: - - file_tree: - root_dir: /path/to/root/directory - keep_newline: - - files/testdir/* +.. code-block:: bash + + (salt-master) # salt bob pillar.items + bob: + ---------- + apache: + ---------- + config.d: + ---------- + 00_important.conf: + + 20_bob_extra.conf: + + corporate_app: + ---------- + settings: + ---------- + common_settings: + // This is the main settings file for the corporate + // internal web app + main_setting: probably + bob_settings: + role: bob .. note:: - In earlier releases, this documentation incorrectly stated that binary - files would not affected by the ``keep_newline`` configuration. However, - this module does not actually distinguish between binary and text files. -.. versionchanged:: 2017.7.0 - Templating/rendering has been added. You can now specify a default render - pipeline and a black- and whitelist of (dis)allowed renderers. - - ``template`` must be set to ``True`` for templating to happen. - - .. code-block:: yaml - - ext_pillar: - - file_tree: - root_dir: /path/to/root/directory - render_default: jinja|yaml - renderer_blacklist: - - gpg - renderer_whitelist: - - jinja - - yaml - template: True - -Assigning Pillar Data to Individual Hosts ------------------------------------------ - -To configure pillar data for each host, this external pillar will recursively -iterate over ``root_dir``/hosts/``id`` (where ``id`` is a minion ID), and -compile pillar data with each subdirectory as a dictionary key and each file -as a value. - -For example, the following ``root_dir`` tree: - -.. code-block:: text - - ./hosts/ - ./hosts/test-host/ - ./hosts/test-host/files/ - ./hosts/test-host/files/testdir/ - ./hosts/test-host/files/testdir/file1.txt - ./hosts/test-host/files/testdir/file2.txt - ./hosts/test-host/files/another-testdir/ - ./hosts/test-host/files/another-testdir/symlink-to-file1.txt - -will result in the following pillar tree for minion with ID ``test-host``: - -.. code-block:: text - - test-host: - ---------- - files: - ---------- - another-testdir: - ---------- - symlink-to-file1.txt: - Contents of file #1. - - testdir: - ---------- - file1.txt: - Contents of file #1. - - file2.txt: - Contents of file #2. - -.. note:: - Subdirectories underneath ``root_dir``/hosts/``id`` become nested - dictionaries, as shown above. - - -Assigning Pillar Data to Entire Nodegroups ------------------------------------------- - -To assign Pillar data to all minions in a given nodegroup, this external pillar -recursively iterates over ``root_dir``/nodegroups/``nodegroup`` (where -``nodegroup`` is the name of a nodegroup), and like for individual hosts, -compiles pillar data with each subdirectory as a dictionary key and each file -as a value. - -.. important:: - If the same Pillar key is set for a minion both by nodegroup and by - individual host, then the value set for the individual host will take - precedence. - -For example, the following ``root_dir`` tree: - -.. code-block:: text - - ./nodegroups/ - ./nodegroups/test-group/ - ./nodegroups/test-group/files/ - ./nodegroups/test-group/files/testdir/ - ./nodegroups/test-group/files/testdir/file1.txt - ./nodegroups/test-group/files/testdir/file2.txt - ./nodegroups/test-group/files/another-testdir/ - ./nodegroups/test-group/files/another-testdir/symlink-to-file1.txt - -will result in the following pillar data for minions in the node group -``test-group``: - -.. code-block:: text - - test-host: - ---------- - files: - ---------- - another-testdir: - ---------- - symlink-to-file1.txt: - Contents of file #1. - - testdir: - ---------- - file1.txt: - Contents of file #1. - - file2.txt: - Contents of file #2. + The leaf data in the example shown is the contents of the pillar files. ''' from __future__ import absolute_import @@ -302,7 +224,123 @@ def ext_pillar(minion_id, renderer_whitelist=None, template=False): ''' - Compile pillar data for the specified minion ID + Compile pillar data from the given ``root_dir`` specific to Nodegroup names + and Minion IDs. + + If a Minion's ID is not found at ``/host/`` or if it + is not included in any Nodegroups named at + ``/nodegroups/``, no pillar data provided by this + pillar module will be available for that Minion. + + .. versionchanged:: 2017.7.0 + Templating/rendering has been added. You can now specify a default + render pipeline and a black- and whitelist of (dis)allowed renderers. + + :param:`template` must be set to ``True`` for templating to happen. + + .. code-block:: yaml + + ext_pillar: + - file_tree: + root_dir: /path/to/root/directory + render_default: jinja|yaml + renderer_blacklist: + - gpg + renderer_whitelist: + - jinja + - yaml + template: True + + :param minion_id: + The ID of the Minion whose pillar data is to be collected + + :param pillar: + Unused by the ``file_tree`` pillar module + + :param root_dir: + Filesystem directory used as the root for pillar data (e.g. + ``/srv/ext_pillar``) + + :param follow_dir_links: + Follow symbolic links to directories while collecting pillar files. + Defaults to ``False``. + + .. warning:: + + Care should be exercised when enabling this option as it will + follow links that point outside of :param:`root_dir`. + + .. warning:: + + Symbolic links that lead to infinite recursion are not filtered. + + :param debug: + Enable debug information at log level ``debug``. Defaults to + ``False``. This option may be useful to help debug errors when setting + up the ``file_tree`` pillar module. + + :param keep_newline: + Preserve the end-of-file newline in files. Defaults to ``False``. + This option may either be a boolean or a list of file globs (as defined + by the `Python fnmatch package + `_) for which end-of-file + newlines are to be kept. + + ``keep_newline`` should be turned on if the pillar data is intended to + be used to deploy a file using ``contents_pillar`` with a + :py:func:`file.managed ` state. + + .. versionchanged:: 2015.8.4 + The ``raw_data`` parameter has been renamed to ``keep_newline``. In + earlier releases, ``raw_data`` must be used. Also, this parameter + can now be a list of globs, allowing for more granular control over + which pillar values keep their end-of-file newline. The globs match + paths relative to the directories named for Minion IDs and + Nodegroup namess underneath the :param:`root_dir`. + + .. code-block:: yaml + + ext_pillar: + - file_tree: + root_dir: /srv/ext_pillar + keep_newline: + - apache/config.d/* + - corporate_app/settings/* + + .. note:: + In earlier releases, this documentation incorrectly stated that + binary files would not affected by the ``keep_newline``. However, + this module does not actually distinguish between binary and text + files. + + + :param render_default: + Override Salt's :conf_master:`default global renderer ` for + the ``file_tree`` pillar. + + .. code-block:: yaml + + render_default: jinja + + :param renderer_blacklist: + Disallow renderers for pillar files. + + .. code-block:: yaml + + renderer_blacklist: + - json + + :param renderer_whitelist: + Allow renderers for pillar files. + + .. code-block:: yaml + + renderer_whitelist: + - yaml + - jinja + + :param template: + Enable templating of pillar files. Defaults to ``False``. ''' # Not used del pillar From a5fbe4e95eda002813abdf7cfc7517d4ea8f4115 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Thu, 15 Feb 2018 16:15:20 +0200 Subject: [PATCH 33/51] Fix typo in postgres_user.present state function --- salt/states/postgres_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/postgres_user.py b/salt/states/postgres_user.py index f85264da20..d9f686871c 100644 --- a/salt/states/postgres_user.py +++ b/salt/states/postgres_user.py @@ -95,7 +95,7 @@ def present(name, encrypted to the previous format if it is not already done. - default_passwoord + default_password The password used only when creating the user, unless password is set. .. versionadded:: 2016.3.0 From e8678f633ddb0cfd5caece29b25c28bf94141565 Mon Sep 17 00:00:00 2001 From: Lee Webb Date: Tue, 13 Feb 2018 12:00:09 +1100 Subject: [PATCH 34/51] Fix Comment being None not '' and inject quotes into the TXT ChangeRecords --- salt/states/boto3_route53.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/states/boto3_route53.py b/salt/states/boto3_route53.py index 54a66bdb07..095afba829 100644 --- a/salt/states/boto3_route53.py +++ b/salt/states/boto3_route53.py @@ -137,7 +137,7 @@ def _from_aws_encoding(string): # XXX TODO def hosted_zone_present(name, Name=None, PrivateZone=False, - CallerReference=None, Comment='', VPCs=None, + CallerReference=None, Comment=None, VPCs=None, region=None, key=None, keyid=None, profile=None): ''' Ensure a hosted zone exists with the given attributes. @@ -642,6 +642,11 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name ret['result'] = False return ret else: + # for TXT records the entry must be encapsulated in quotes as required by the API + # this appears to be incredibly difficult with the jinja templating engine + # so inject the quotations here to make a viable ChangeBatch + if Type == 'TXT': + rr = '"%s"' % (rr) fixed_rrs += [rr] ResourceRecords = [{'Value': rr} for rr in sorted(fixed_rrs)] From a07bb487264b92d7d5078e77f70a5012eeb2939a Mon Sep 17 00:00:00 2001 From: Lee Webb Date: Wed, 14 Feb 2018 09:09:17 +1100 Subject: [PATCH 35/51] Correct formatting for lint --- salt/states/boto3_route53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/boto3_route53.py b/salt/states/boto3_route53.py index 095afba829..dbeb2e2c80 100644 --- a/salt/states/boto3_route53.py +++ b/salt/states/boto3_route53.py @@ -646,7 +646,7 @@ def rr_present(name, HostedZoneId=None, DomainName=None, PrivateZone=False, Name # this appears to be incredibly difficult with the jinja templating engine # so inject the quotations here to make a viable ChangeBatch if Type == 'TXT': - rr = '"%s"' % (rr) + rr = '"{}"'.format(rr) fixed_rrs += [rr] ResourceRecords = [{'Value': rr} for rr in sorted(fixed_rrs)] From 2a5d855d976acca23b5a034c53d9614581415bb7 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 16 Feb 2018 10:48:57 -0500 Subject: [PATCH 36/51] add required arg to dns_check jinja doc example --- doc/topics/jinja/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index a833507269..f60d9955d0 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -1348,7 +1348,7 @@ Example: .. code-block:: jinja - {{ 'www.google.com' | dns_check }} + {{ 'www.google.com' | dns_check(port=443) }} Returns: From b94d73c53e85100e27cdf476f421dcc4f22a23c5 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 16 Feb 2018 09:02:36 -0500 Subject: [PATCH 37/51] Pin tornado version in requirements file tornado needs to be >= 4.2.1, but less that 5.0. Tornado 5.0 is introducing backwards-incompatible changes. Therefore, we need to pin the version of tornado in base.txt until we can fix supporting Tornado 5.0 in Salt. Refs #45790 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4e735d3a21..1a16e368d0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,6 +3,6 @@ msgpack-python>0.3 PyYAML MarkupSafe requests>=1.0.0 -tornado>=4.2.1 +tornado>=4.2.1,<5.0 # Required by Tornado to handle threads stuff. futures>=2.0 From 5a0fe104f7b15b1023119bab1d88faf767c987c1 Mon Sep 17 00:00:00 2001 From: Wedge Jarrad Date: Sun, 18 Feb 2018 10:57:28 -0800 Subject: [PATCH 38/51] Fix contributing doc typo --- doc/topics/development/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/development/contributing.rst b/doc/topics/development/contributing.rst index ed4fd2aa12..3c21c9b9c0 100644 --- a/doc/topics/development/contributing.rst +++ b/doc/topics/development/contributing.rst @@ -223,7 +223,7 @@ branches, and dot release branches. .. note:: GitHub will open pull requests against Salt's main branch, ``develop``, - byndefault. Be sure to check which branch is selected when creating the + by default. Be sure to check which branch is selected when creating the pull request. The Develop Branch From aba00805f48cb1e9ad8a28d2d083b7a6fdb434cd Mon Sep 17 00:00:00 2001 From: gwiyeong Date: Mon, 19 Feb 2018 11:07:36 +0900 Subject: [PATCH 39/51] Adds set_close_callback function to removes stream instance after closed from a set streams. --- salt/transport/ipc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index d8d8235920..d93a62154e 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -559,6 +559,11 @@ class IPCMessagePublisher(object): io_loop=self.io_loop ) self.streams.add(stream) + + def discard_after_closed(): + self.streams.discard(stream) + + stream.set_close_callback(discard_after_closed) except Exception as exc: log.error('IPC streaming error: {0}'.format(exc)) From 48080a1bae77b43a6315c114a5c28daf747a5259 Mon Sep 17 00:00:00 2001 From: gwiyeong Date: Mon, 19 Feb 2018 11:07:36 +0900 Subject: [PATCH 40/51] Fixes memory leak, saltclients should be cleaned after used. --- salt/netapi/rest_tornado/saltnado.py | 36 ++++++++++------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 38e665b7df..b5fce465ed 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -236,28 +236,6 @@ logger = logging.getLogger() # - "wheel" (need async api...) -class SaltClientsMixIn(object): - ''' - MixIn class to container all of the salt clients that the API needs - ''' - # TODO: load this proactively, instead of waiting for a request - __saltclients = None - - @property - def saltclients(self): - if SaltClientsMixIn.__saltclients is None: - local_client = salt.client.get_local_client(mopts=self.application.opts) - # TODO: refreshing clients using cachedict - SaltClientsMixIn.__saltclients = { - 'local': local_client.run_job_async, - # not the actual client we'll use.. but its what we'll use to get args - 'local_async': local_client.run_job_async, - 'runner': salt.runner.RunnerClient(opts=self.application.opts).cmd_async, - 'runner_async': None, # empty, since we use the same client as `runner` - } - return SaltClientsMixIn.__saltclients - - AUTH_TOKEN_HEADER = 'X-Auth-Token' AUTH_COOKIE_NAME = 'session_id' @@ -388,7 +366,7 @@ class EventListener(object): del self.timeout_map[future] -class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylint: disable=W0223 +class BaseSaltAPIHandler(tornado.web.RequestHandler): # pylint: disable=W0223 ct_out_map = ( ('application/json', json.dumps), ('application/x-yaml', yaml.safe_dump), @@ -416,6 +394,16 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin self.application.opts, ) + if not hasattr(self, 'saltclients'): + local_client = salt.client.get_local_client(mopts=self.application.opts) + self.saltclients = { + 'local': local_client.run_job_async, + # not the actual client we'll use.. but its what we'll use to get args + 'local_async': local_client.run_job_async, + 'runner': salt.runner.RunnerClient(opts=self.application.opts).cmd_async, + 'runner_async': None, # empty, since we use the same client as `runner` + } + @property def token(self): ''' @@ -745,7 +733,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 self.write(self.serialize(ret)) -class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn): # pylint: disable=W0223 +class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 ''' Main API handler for base "/" ''' From 2062fd0e5cb809ecf3db9ce14ca7440eaaab6369 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Mon, 19 Feb 2018 12:59:00 +0200 Subject: [PATCH 41/51] [DOC] Put https link to the formulas doc page --- doc/topics/development/conventions/formulas.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/development/conventions/formulas.rst b/doc/topics/development/conventions/formulas.rst index 57a8d51075..de4e3e28bb 100644 --- a/doc/topics/development/conventions/formulas.rst +++ b/doc/topics/development/conventions/formulas.rst @@ -1245,7 +1245,7 @@ A sample skeleton for the ``README.rst`` file: .. note:: See the full `Salt Formulas installation and usage instructions - `_. + `_. Available states ================ From dd3f93655719a280567ece827c05c4da93c7e4fe Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Mon, 19 Feb 2018 12:32:14 +0100 Subject: [PATCH 42/51] Fix skipping Kubernetes tests if client is not installed When the Kubernetes client is not installed, the import of salt.modules.kubernetes will still succeed, but HAS_LIBS will be set to False (since the library import will be covered by a try-except clause). Therefore expect the salt.modules.kubernetes to always succeed and check kubernetes.HAS_LIBS instead for the presence of the kubernetes library. Signed-off-by: Benjamin Drung --- tests/unit/modules/test_kubernetes.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/unit/modules/test_kubernetes.py b/tests/unit/modules/test_kubernetes.py index 4e8f6cd4b5..92879e0359 100644 --- a/tests/unit/modules/test_kubernetes.py +++ b/tests/unit/modules/test_kubernetes.py @@ -16,15 +16,12 @@ from tests.support.mock import ( NO_MOCK_REASON ) -try: - from salt.modules import kubernetes -except ImportError: - kubernetes = False +from salt.modules import kubernetes @skipIf(NO_MOCK, NO_MOCK_REASON) -@skipIf(kubernetes is False, "Probably Kubernetes client lib is not installed. \ - Skipping test_kubernetes.py") +@skipIf(not kubernetes.HAS_LIBS, "Kubernetes client lib is not installed. " + "Skipping test_kubernetes.py") class KubernetesTestCase(TestCase, LoaderModuleMockMixin): ''' Test cases for salt.modules.kubernetes From 8d9a432fb2859e12337460a41e0175118d33d90b Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Mon, 19 Feb 2018 07:35:18 -0700 Subject: [PATCH 43/51] Add --assumeyes to yum/dnf commands in yumpkg.refresh_db Without --assumeyes, these commands can hang waiting for user confirmation if there is an unaccepted repository key. --- salt/modules/yumpkg.py | 4 ++-- tests/unit/modules/test_yumpkg.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 3db8aaa27f..0fc2034e04 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -1025,8 +1025,8 @@ def refresh_db(**kwargs): options = _get_options(**kwargs) - clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache'] - update_cmd = [_yum(), '--quiet', 'check-update'] + clean_cmd = [_yum(), '--quiet', '--assumeyes', 'clean', 'expire-cache'] + update_cmd = [_yum(), '--quiet', '--assumeyes', 'check-update'] if __grains__.get('os_family') == 'RedHat' \ and __grains__.get('osmajorrelease') == 7: diff --git a/tests/unit/modules/test_yumpkg.py b/tests/unit/modules/test_yumpkg.py index c7adcb28d0..882c1c50ab 100644 --- a/tests/unit/modules/test_yumpkg.py +++ b/tests/unit/modules/test_yumpkg.py @@ -311,11 +311,11 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): fromrepo='good', branch='foo') clean_cmd.assert_called_once_with( - ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=*', + ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=*', '--enablerepo=good', '--branch=foo'], python_shell=False) update_cmd.assert_called_once_with( - ['yum', '--quiet', 'check-update', + ['yum', '--quiet', '--assumeyes', 'check-update', '--setopt=autocheck_running_kernel=false', '--disablerepo=*', '--enablerepo=good', '--branch=foo'], output_loglevel='trace', @@ -333,11 +333,11 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): disablerepo='bad', branch='foo') clean_cmd.assert_called_once_with( - ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=bad', + ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=bad', '--enablerepo=good', '--branch=foo'], python_shell=False) update_cmd.assert_called_once_with( - ['yum', '--quiet', 'check-update', + ['yum', '--quiet', '--assumeyes', 'check-update', '--setopt=autocheck_running_kernel=false', '--disablerepo=bad', '--enablerepo=good', '--branch=foo'], output_loglevel='trace', @@ -354,7 +354,7 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): fromrepo='good', branch='foo') clean_cmd.assert_called_once_with( - ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=*', + ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=*', '--enablerepo=good', '--branch=foo'], python_shell=False) @@ -367,7 +367,7 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): disablerepo='bad', branch='foo') clean_cmd.assert_called_once_with( - ['yum', '--quiet', 'clean', 'expire-cache', '--disablerepo=bad', + ['yum', '--quiet', '--assumeyes', 'clean', 'expire-cache', '--disablerepo=bad', '--enablerepo=good', '--branch=foo'], python_shell=False) From 2a2c23c66b292c457ccdcbb7eadeb7fcade25a42 Mon Sep 17 00:00:00 2001 From: Ollie Armstrong Date: Mon, 20 Nov 2017 15:56:16 +0000 Subject: [PATCH 44/51] Fix acme state to correctly return on test Previously, the acme.cert state would return that a change was pending during a test run even in the case that the certificate would not have been touched. Changed the return value in this case so that it is not thought to require a change. This is reported in #40208 [0] and is also referenced in #42763 [1]. The issue #40208 looks to go on to recommend further changes beyond the scope of this 'quick fix'. [0] https://github.com/saltstack/salt/issues/40208#issuecomment-289637588 [1] https://github.com/saltstack/salt/issues/42763#issuecomment-345728031 --- salt/states/acme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/states/acme.py b/salt/states/acme.py index 43649a6426..ad1c9d0564 100644 --- a/salt/states/acme.py +++ b/salt/states/acme.py @@ -85,6 +85,7 @@ def cert(name, comment += 'would have been renewed' else: comment += 'would not have been touched' + ret['result'] = True ret['comment'] = comment return ret From d8fb051e44b83d5ea753b8f75a977d9dfdbfa39f Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Wed, 21 Feb 2018 15:37:57 +0200 Subject: [PATCH 45/51] [DOC] Fix code-blocks for reStructuredText --- doc/topics/development/conventions/formulas.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/topics/development/conventions/formulas.rst b/doc/topics/development/conventions/formulas.rst index de4e3e28bb..b11433f187 100644 --- a/doc/topics/development/conventions/formulas.rst +++ b/doc/topics/development/conventions/formulas.rst @@ -1234,7 +1234,7 @@ target platform, and any other installation or usage instructions or tips. A sample skeleton for the ``README.rst`` file: -.. code-block:: rest +.. code-block:: restructuredtext === foo @@ -1274,7 +1274,7 @@ A sample skeleton for the `CHANGELOG.rst` file: :file:`CHANGELOG.rst`: -.. code-block:: rest +.. code-block:: restructuredtext foo formula =========== From 57a60f62a313fe0ba8ed045aed8edf4acd1c18d8 Mon Sep 17 00:00:00 2001 From: zr Date: Sat, 17 Feb 2018 23:31:26 +0800 Subject: [PATCH 46/51] fix kernel subpackages install bug --- salt/modules/yumpkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 0fc2034e04..7b4e4e6d36 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -1396,8 +1396,8 @@ def install(name=None, break else: if pkgname is not None: - if re.match('kernel(-.+)?', pkgname): - # kernel and its subpackages support multiple + if re.match('^kernel(|-devel)$', pkgname): + # kernel and kernel-devel support multiple # installs as their paths do not conflict. # Performing a yum/dnf downgrade will be a # no-op so just do an install instead. It will From 0a481d707fa145307da21dd6fb7ac7f51492d6f2 Mon Sep 17 00:00:00 2001 From: Volodymyr Samodid Date: Tue, 20 Feb 2018 23:04:17 +0200 Subject: [PATCH 47/51] update digitalocean salt-cloud driver fix #45837 --- salt/cloud/clouds/digital_ocean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/clouds/digital_ocean.py b/salt/cloud/clouds/digital_ocean.py index daabcbddfe..358faaea4e 100644 --- a/salt/cloud/clouds/digital_ocean.py +++ b/salt/cloud/clouds/digital_ocean.py @@ -159,7 +159,7 @@ def avail_sizes(call=None): '-f or --function, or with the --list-sizes option' ) - items = query(method='sizes') + items = query(method='sizes', command='?per_page=100') ret = {} for size in items['sizes']: ret[size['slug']] = {} From 8d36730ef762a76323653b88724b12eaec82c654 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 20 Feb 2018 17:43:53 -0700 Subject: [PATCH 48/51] If no pubkey is passed in openmode fail If the pub entry in the load is empty, we should fail authentication in open mode. This is usually caught elsewhere for the other modes, because we would just write it to a file, but in this case, we only write it to a file if it actually exists, and if it is different from disk_key, so we would catch all other options when trying to load the public key. Fixes #46085 --- salt/transport/mixins/auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/transport/mixins/auth.py b/salt/transport/mixins/auth.py index 866493b854..f6a01a1d30 100644 --- a/salt/transport/mixins/auth.py +++ b/salt/transport/mixins/auth.py @@ -417,6 +417,10 @@ class AESReqServerMixin(object): log.debug('Host key change detected in open mode.') with salt.utils.fopen(pubfn, 'w+') as fp_: fp_.write(load['pub']) + elif not load['pub']: + log.error('Public key is empty: {0}'.format(load['id'])) + return {'enc': 'clear', + 'load': {'ret': False}} pub = None From c818d4b7911cad6bcd216d1a0b473ac08eab6cee Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 7 Feb 2018 16:13:43 -0700 Subject: [PATCH 49/51] Convert reg values to unicode for debug --- salt/modules/win_lgpo.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index ebaff9f040..3318693196 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2442,7 +2442,7 @@ class _policy_info(object): elif ord(val) == 1: return 'Enabled' else: - return 'Invalid Value' + return 'Invalid Value: %s' % repr(val) else: return 'Not Defined' except TypeError: @@ -5047,7 +5047,10 @@ def get(policy_class=None, return_full_policy_names=True, class_vals[policy_name] = __salt__['reg.read_value'](_pol['Registry']['Hive'], _pol['Registry']['Path'], _pol['Registry']['Value'])['vdata'] - log.debug('Value {0} found for reg policy {1}'.format(class_vals[policy_name], policy_name)) + log.debug( + 'Value {0} found for reg policy {1}'.format( + salt.utils.to_unicode(class_vals[policy_name]), + policy_name)) elif 'Secedit' in _pol: # get value from secedit _ret, _val = _findOptionValueInSeceditFile(_pol['Secedit']['Option']) From e9fa53d3b759970cf21993d4e95048814fd8ab3d Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 9 Feb 2018 13:45:24 -0700 Subject: [PATCH 50/51] Change the Invalid Data Message --- salt/modules/win_lgpo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 3318693196..2b467ca2ed 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2442,7 +2442,7 @@ class _policy_info(object): elif ord(val) == 1: return 'Enabled' else: - return 'Invalid Value: %s' % repr(val) + return 'Invalid Value: {0!r}'.format(val) # pylint: disable=repr-flag-used-in-string else: return 'Not Defined' except TypeError: From bcde5cc62550b09575f4f7fef0ab78f9b6d3eb72 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 21 Feb 2018 09:47:20 -0700 Subject: [PATCH 51/51] Update log statement --- salt/modules/win_lgpo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 2b467ca2ed..89b48b8b79 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -5048,9 +5048,9 @@ def get(policy_class=None, return_full_policy_names=True, _pol['Registry']['Path'], _pol['Registry']['Value'])['vdata'] log.debug( - 'Value {0} found for reg policy {1}'.format( - salt.utils.to_unicode(class_vals[policy_name]), - policy_name)) + 'Value %r found for reg policy %s', + class_vals[policy_name], policy_name + ) elif 'Secedit' in _pol: # get value from secedit _ret, _val = _findOptionValueInSeceditFile(_pol['Secedit']['Option'])