From ea0d74cd2738d176cbe65da5c63615193b1728e6 Mon Sep 17 00:00:00 2001 From: "xiaofei.sun" Date: Wed, 24 Aug 2016 23:20:01 +0800 Subject: [PATCH 01/27] fix salt-api opts --- salt/config/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index e774e11a69..394176e377 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1383,8 +1383,8 @@ CLOUD_CONFIG_DEFAULTS = { DEFAULT_API_OPTS = { # ----- Salt master settings overridden by Salt-API ---------------------> - 'pidfile': '/var/run/salt-api.pid', - 'logfile': '/var/log/salt/api', + 'pidfile': os.path.join(salt.syspaths.PIDFILE_DIR, 'salt-api.pid'), + 'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'api'), 'rest_timeout': 300, # <---- Salt master settings overridden by Salt-API ---------------------- } @@ -3273,10 +3273,11 @@ def api_config(path): ''' # Let's grab a copy of salt's master default opts defaults = DEFAULT_MASTER_OPTS + opts = client_config(path, defaults=defaults) # Let's override them with salt-api's required defaults - defaults.update(DEFAULT_API_OPTS) + opts.update(DEFAULT_API_OPTS) + return opts - return client_config(path, defaults=defaults) def spm_config(path): From b8214824fd81294a33458691af30dcf93f60ec08 Mon Sep 17 00:00:00 2001 From: "xiaofei.sun" Date: Sun, 4 Sep 2016 20:39:24 +0800 Subject: [PATCH 02/27] add simplify code --- salt/config/__init__.py | 1 - salt/master.py | 26 ++++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index b093d8d348..90b113fb70 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -3288,7 +3288,6 @@ def api_config(path): return opts - def spm_config(path): ''' Read in the salt master config file and add additional configs that diff --git a/salt/master.py b/salt/master.py index 8b26c4768c..91dea750e6 100644 --- a/salt/master.py +++ b/salt/master.py @@ -706,9 +706,9 @@ class ReqServer(SignalHandlingMultiprocessingProcess): name = 'MWorker-{0}'.format(ind) self.process_manager.add_process(MWorker, args=(self.opts, - self.master_key, self.key, req_channels, + self.master_key, name ), kwargs=kwargs, @@ -750,10 +750,10 @@ class MWorker(SignalHandlingMultiprocessingProcess): ''' def __init__(self, opts, - mkey, key, - req_channels, - name, + req_channels=None, + mkey=None, + name=None, **kwargs): ''' Create a salt master worker process @@ -765,8 +765,10 @@ class MWorker(SignalHandlingMultiprocessingProcess): :rtype: MWorker :return: Master worker ''' - kwargs['name'] = name - SignalHandlingMultiprocessingProcess.__init__(self, **kwargs) + if name: + kwargs['name'] = name + super(MWorker, self).__init__(**kwargs) + self.kwargs = kwargs self.opts = opts self.req_channels = req_channels @@ -781,22 +783,18 @@ class MWorker(SignalHandlingMultiprocessingProcess): # non-Windows platforms. def __setstate__(self, state): self._is_child = True - SignalHandlingMultiprocessingProcess.__init__(self, log_queue=state['log_queue']) - self.opts = state['opts'] - self.req_channels = state['req_channels'] - self.mkey = state['mkey'] - self.key = state['key'] + super(MWorker, self).__init__(**state) self.k_mtime = state['k_mtime'] SMaster.secrets = state['secrets'] def __getstate__(self): return {'opts': self.opts, + 'key': self.key, 'req_channels': self.req_channels, 'mkey': self.mkey, - 'key': self.key, 'k_mtime': self.k_mtime, - 'log_queue': self.log_queue, - 'secrets': SMaster.secrets} + 'secrets': SMaster.secrets, + 'kwargs': self.kwargs} def _handle_signals(self, signum, sigframe): for channel in getattr(self, 'req_channels', ()): From f2de71782be82f4e4fd91bdd0d37f3a6db1470ae Mon Sep 17 00:00:00 2001 From: "xiaofei.sun" Date: Sun, 4 Sep 2016 21:19:23 +0800 Subject: [PATCH 03/27] move back --- salt/config/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 90b113fb70..7aa47ea34f 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1384,8 +1384,8 @@ CLOUD_CONFIG_DEFAULTS = { DEFAULT_API_OPTS = { # ----- Salt master settings overridden by Salt-API ---------------------> - 'pidfile': os.path.join(salt.syspaths.PIDFILE_DIR, 'salt-api.pid'), - 'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'api'), + 'pidfile': '/var/run/salt-api.pid', + 'logfile': '/var/log/salt/api', 'rest_timeout': 300, # <---- Salt master settings overridden by Salt-API ---------------------- } @@ -3282,10 +3282,10 @@ def api_config(path): ''' # Let's grab a copy of salt's master default opts defaults = DEFAULT_MASTER_OPTS - opts = client_config(path, defaults=defaults) # Let's override them with salt-api's required defaults - opts.update(DEFAULT_API_OPTS) - return opts + defaults.update(DEFAULT_API_OPTS) + + return client_config(path, defaults=defaults) def spm_config(path): From 116d7ac3e5092dc2e4d49e07c0feae5b74ff7e4f Mon Sep 17 00:00:00 2001 From: David Boucha Date: Thu, 8 Sep 2016 10:57:57 -0600 Subject: [PATCH 04/27] If windows pkg db hasn't been created yet, refresh the db instead of stacktracing (#36008) * Don't raise an exception. Just return empty dict If the repo cache file doesn't exist, lets' just return an empty dict rather than raising an exception. * If repo cache file doesn't exist, refresh * add logging message * remove unused imports --- salt/modules/win_pkg.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index e804f095fc..e92c456b0f 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -16,7 +16,6 @@ A module to manage software on Windows # Import python libs from __future__ import absolute_import -import errno import os import locale import logging @@ -40,7 +39,7 @@ except ImportError: import shlex # Import salt libs -from salt.exceptions import CommandExecutionError, SaltRenderError +from salt.exceptions import SaltRenderError import salt.utils import salt.syspaths from salt.exceptions import MinionError @@ -1070,6 +1069,9 @@ def get_repo_data(saltenv='base'): # return __context__['winrepo.data'] repocache_dir = _get_local_repo_dir(saltenv=saltenv) winrepo = 'winrepo.p' + if not os.path.exists(os.path.join(repocache_dir, winrepo)): + log.debug('No winrepo.p cache file. Refresh pkg db now.') + refresh_db(saltenv=saltenv) try: with salt.utils.fopen( os.path.join(repocache_dir, winrepo), 'rb') as repofile: @@ -1080,12 +1082,6 @@ def get_repo_data(saltenv='base'): log.exception(exc) return {} except IOError as exc: - if exc.errno == errno.ENOENT: - # File doesn't exist - raise CommandExecutionError( - 'Windows repo cache doesn\'t exist, pkg.refresh_db likely ' - 'needed' - ) log.error('Not able to read repo file') log.exception(exc) return {} From 24b0387b925b8b54d2fd47a110814c8f1472a9cb Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Thu, 8 Sep 2016 15:48:44 -0600 Subject: [PATCH 05/27] Back-port #36070 to 2015.8 (#36169) --- salt/cloud/clouds/opennebula.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/cloud/clouds/opennebula.py b/salt/cloud/clouds/opennebula.py index f264b1fefd..2f8e76c272 100644 --- a/salt/cloud/clouds/opennebula.py +++ b/salt/cloud/clouds/opennebula.py @@ -31,7 +31,6 @@ import pprint import logging # Import salt cloud libs -import salt.utils.cloud import salt.config as config from salt.exceptions import ( SaltCloudConfigError, @@ -253,7 +252,7 @@ def list_nodes_select(call=None): ''' Return a list of the VMs that are on the provider, with select fields ''' - return salt.utils.cloud.list_nodes_select( + return __utils__['cloud.list_nodes_select']( list_nodes_full('function'), __opts__['query.selection'], call, ) @@ -381,7 +380,7 @@ def create(vm_): return node_data try: - data = salt.utils.cloud.wait_for_ip( + data = __utils__['cloud.wait_for_ip']( __query_node_data, update_args=(vm_['name'],), timeout=config.get_cloud_config_value( @@ -456,12 +455,12 @@ def script(vm_): ''' Return the script deployment object ''' - deploy_script = salt.utils.cloud.os_script( + deploy_script = __utils__['cloud.os_script']( config.get_cloud_config_value('script', vm_, __opts__), vm_, __opts__, - salt.utils.cloud.salt_config_to_yaml( - salt.utils.cloud.minion_config(__opts__, vm_) + __utils__['cloud.salt_config_to_yaml']( + __utils__['cloud.minion_config'](__opts__, vm_) ) ) return deploy_script From 345cd7f9c1b13b6fb108dcf0a3f60931475593d6 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 8 Sep 2016 16:30:35 -0600 Subject: [PATCH 06/27] Fix test_launchctl test --- tests/integration/modules/mac_service.py | 27 +++++++++--------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/integration/modules/mac_service.py b/tests/integration/modules/mac_service.py index 3cc0a7d52d..43e5b79256 100644 --- a/tests/integration/modules/mac_service.py +++ b/tests/integration/modules/mac_service.py @@ -7,6 +7,7 @@ integration tests for mac_service from __future__ import absolute_import, print_function # Import Salt Testing libs +from salttesting import skipIf from salttesting.helpers import ensure_in_syspath, destructiveTest ensure_in_syspath('../../') @@ -14,7 +15,11 @@ ensure_in_syspath('../../') import integration import salt.utils - +@skipIf(not salt.utils.is_darwin(), 'Test only available on Mac OS X') +@skipIf(not salt.utils.which('launchctl'), 'Test requires launchctl binary') +@skipIf(not salt.utils.which('plutil'), 'Test requires plutil binary') +@skipIf(salt.utils.get_uid(salt.utils.get_user()) != 0, + 'Test requires root') class MacServiceModuleTest(integration.ModuleCase): ''' Validate the mac_service module @@ -24,26 +29,14 @@ class MacServiceModuleTest(integration.ModuleCase): def setUp(self): ''' - Get current settings + Get current state of the test service ''' - if not salt.utils.is_darwin(): - self.skipTest('Test only available on Mac OS X') - - if not salt.utils.which('launchctl'): - self.skipTest('Test requires launchctl binary') - - if not salt.utils.which('plutil'): - self.skipTest('Test requires plutil binary') - - if salt.utils.get_uid(salt.utils.get_user()) != 0: - self.skipTest('Test requires root') - self.SERVICE_ENABLED = self.run_function('service.enabled', [self.SERVICE_NAME]) def tearDown(self): ''' - Reset to original settings + Reset the test service to the original state ''' if self.SERVICE_ENABLED: self.run_function('service.start', [self.SERVICE_NAME]) @@ -79,8 +72,8 @@ class MacServiceModuleTest(integration.ModuleCase): # Raise an error self.assertIn( - ' Failed to error service', - self.run_function('service.launchctl', ['error'])) + 'Failed to error service', + self.run_function('service.launchctl', ['error', 'bootstrap'])) def test_list(self): ''' From 1c83b37fe60f3f97dc08aa9946b3a749e9dd9cfb Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 8 Sep 2016 17:07:28 -0600 Subject: [PATCH 07/27] Fix unquoted integers --- tests/integration/modules/mac_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/modules/mac_service.py b/tests/integration/modules/mac_service.py index 43e5b79256..10e0c9a639 100644 --- a/tests/integration/modules/mac_service.py +++ b/tests/integration/modules/mac_service.py @@ -63,10 +63,11 @@ class MacServiceModuleTest(integration.ModuleCase): ''' # Expected Functionality self.assertTrue( - self.run_function('service.launchctl', ['error', 'bootstrap', 64])) + self.run_function('service.launchctl', + ['error', 'bootstrap', '64'])) self.assertEqual( self.run_function('service.launchctl', - ['error', 'bootstrap', 64], + ['error', 'bootstrap', '64'], return_stdout=True), '64: unknown error code') From ca8eb7e0760297c2bb8abc49fe58af57637de3fb Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Fri, 9 Sep 2016 15:27:20 +0300 Subject: [PATCH 08/27] Don't run the same signal handler twice. Catch os.kill errors. Else a Process Not Found error stops the killing loop and there are still alive master subprocesses. --- salt/utils/process.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/utils/process.py b/salt/utils/process.py index 20e74f635a..eefe8d377f 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -409,6 +409,10 @@ class ProcessManager(object): ''' Kill all of the children ''' + # first lets reset signal handlers to default one to prevent running this twice + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + # check that this is the correct process, children inherit this # handler, if we are in a child lets just run the original handler if os.getpid() != self._pid: @@ -433,7 +437,10 @@ class ProcessManager(object): log.trace('Terminating pid {0}: {1}'.format(pid, p_map['Process'])) if args: # escalate the signal to the process - os.kill(pid, args[0]) + try: + os.kill(pid, args[0]) + except OSError: + pass try: p_map['Process'].terminate() except OSError as exc: @@ -477,7 +484,7 @@ class ProcessManager(object): continue log.trace('Killing pid {0}: {1}'.format(pid, p_map['Process'])) try: - os.kill(signal.SIGKILL, pid) + os.kill(pid, signal.SIGKILL) except OSError: # in case the process has since decided to die, os.kill returns OSError if not p_map['Process'].is_alive(): @@ -648,6 +655,8 @@ class SignalHandlingMultiprocessingProcess(MultiprocessingProcess): signal.signal(signal.SIGTERM, self._handle_signals) def _handle_signals(self, signum, sigframe): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) msg = '{0} received a '.format(self.__class__.__name__) if signum == signal.SIGINT: msg += 'SIGINT' From 229504efef7e07453cf0fa1ff26f53008a343dae Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Fri, 9 Sep 2016 15:29:03 +0300 Subject: [PATCH 09/27] Removed unused import. --- salt/log/handlers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/log/handlers/__init__.py b/salt/log/handlers/__init__.py index 94bd3e2247..1532d22c56 100644 --- a/salt/log/handlers/__init__.py +++ b/salt/log/handlers/__init__.py @@ -11,7 +11,6 @@ from __future__ import absolute_import # Import python libs import sys -import atexit import logging import threading import logging.handlers From 28442b32f87ea8b874267d9a70713073611d3d7c Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 9 Sep 2016 08:47:51 -0600 Subject: [PATCH 10/27] Fix some lint --- tests/integration/modules/mac_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/modules/mac_service.py b/tests/integration/modules/mac_service.py index 10e0c9a639..83865940ff 100644 --- a/tests/integration/modules/mac_service.py +++ b/tests/integration/modules/mac_service.py @@ -15,6 +15,7 @@ ensure_in_syspath('../../') import integration import salt.utils + @skipIf(not salt.utils.is_darwin(), 'Test only available on Mac OS X') @skipIf(not salt.utils.which('launchctl'), 'Test requires launchctl binary') @skipIf(not salt.utils.which('plutil'), 'Test requires plutil binary') From d4628f3c6b707b1452a88f7af6992bc534973fe8 Mon Sep 17 00:00:00 2001 From: t2b Date: Fri, 9 Sep 2016 17:32:38 +0200 Subject: [PATCH 11/27] Allow additional kwargs in states.dockerng.image_present (#36156) Fixes #31513 --- salt/states/dockerng.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/states/dockerng.py b/salt/states/dockerng.py index 1a0d0f1f94..3826cce670 100644 --- a/salt/states/dockerng.py +++ b/salt/states/dockerng.py @@ -474,7 +474,8 @@ def image_present(name, load=None, force=False, insecure_registry=False, - client_timeout=CLIENT_TIMEOUT): + client_timeout=CLIENT_TIMEOUT, + **kwargs): ''' Ensure that an image is present. The image can either be pulled from a Docker registry, built from a Dockerfile, or loaded from a saved image. From b586ed75bd4d097bbcf443e4df2d964fd3ee8189 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 9 Sep 2016 13:14:36 -0600 Subject: [PATCH 12/27] if the backend stack traces when it should return an empty string (#36193) --- salt/client/ssh/state.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/client/ssh/state.py b/salt/client/ssh/state.py index dbe7854a7e..d643b1bd63 100644 --- a/salt/client/ssh/state.py +++ b/salt/client/ssh/state.py @@ -154,7 +154,10 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None, id_=None): for ref in file_refs[saltenv]: for name in ref: short = salt.utils.url.parse(name)[0] - path = file_client.cache_file(name, saltenv, cachedir=cachedir) + try: + path = file_client.cache_file(name, saltenv, cachedir=cachedir) + except IOError: + path = '' if path: tgt = os.path.join(env_root, short) tgt_dir = os.path.dirname(tgt) @@ -162,7 +165,10 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None, id_=None): os.makedirs(tgt_dir) shutil.copy(path, tgt) continue - files = file_client.cache_dir(name, saltenv, cachedir=cachedir) + try: + files = file_client.cache_dir(name, saltenv, cachedir=cachedir) + except IOError: + files = '' if files: for filename in files: fn = filename[filename.find(short) + len(short):] From bd61b88fc8dcb85b26c63533a84185f374c35f9e Mon Sep 17 00:00:00 2001 From: "xiaofei.sun" Date: Sat, 10 Sep 2016 09:37:53 +0800 Subject: [PATCH 13/27] fix log owner --- salt/log/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/log/setup.py b/salt/log/setup.py index 11355dd61a..85a03d9b6f 100644 --- a/salt/log/setup.py +++ b/salt/log/setup.py @@ -931,6 +931,8 @@ def patch_python_logging_handlers(): def __process_multiprocessing_logging_queue(opts, queue): import salt.utils salt.utils.appendproctitle('MultiprocessingLoggingQueue') + from salt.utils.verify import check_user + check_user(opts['user']) if salt.utils.is_windows(): # On Windows, creating a new process doesn't fork (copy the parent # process image). Due to this, we need to setup extended logging From ffd87b2f2f149d8db1b81fef3adc3693c9d5f5cd Mon Sep 17 00:00:00 2001 From: xiaoanyunfei Date: Sat, 10 Sep 2016 09:57:29 +0800 Subject: [PATCH 14/27] fix logqueue owner --- salt/log/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/log/setup.py b/salt/log/setup.py index 11355dd61a..85a03d9b6f 100644 --- a/salt/log/setup.py +++ b/salt/log/setup.py @@ -931,6 +931,8 @@ def patch_python_logging_handlers(): def __process_multiprocessing_logging_queue(opts, queue): import salt.utils salt.utils.appendproctitle('MultiprocessingLoggingQueue') + from salt.utils.verify import check_user + check_user(opts['user']) if salt.utils.is_windows(): # On Windows, creating a new process doesn't fork (copy the parent # process image). Due to this, we need to setup extended logging From 74dc90c7bbe3ea0b0f5ebe7678fead940e62374d Mon Sep 17 00:00:00 2001 From: xiaoanyunfei Date: Sat, 10 Sep 2016 10:04:22 +0800 Subject: [PATCH 15/27] cancle pr last --- salt/master.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/salt/master.py b/salt/master.py index 91dea750e6..8b26c4768c 100644 --- a/salt/master.py +++ b/salt/master.py @@ -706,9 +706,9 @@ class ReqServer(SignalHandlingMultiprocessingProcess): name = 'MWorker-{0}'.format(ind) self.process_manager.add_process(MWorker, args=(self.opts, + self.master_key, self.key, req_channels, - self.master_key, name ), kwargs=kwargs, @@ -750,10 +750,10 @@ class MWorker(SignalHandlingMultiprocessingProcess): ''' def __init__(self, opts, + mkey, key, - req_channels=None, - mkey=None, - name=None, + req_channels, + name, **kwargs): ''' Create a salt master worker process @@ -765,10 +765,8 @@ class MWorker(SignalHandlingMultiprocessingProcess): :rtype: MWorker :return: Master worker ''' - if name: - kwargs['name'] = name - super(MWorker, self).__init__(**kwargs) - self.kwargs = kwargs + kwargs['name'] = name + SignalHandlingMultiprocessingProcess.__init__(self, **kwargs) self.opts = opts self.req_channels = req_channels @@ -783,18 +781,22 @@ class MWorker(SignalHandlingMultiprocessingProcess): # non-Windows platforms. def __setstate__(self, state): self._is_child = True - super(MWorker, self).__init__(**state) + SignalHandlingMultiprocessingProcess.__init__(self, log_queue=state['log_queue']) + self.opts = state['opts'] + self.req_channels = state['req_channels'] + self.mkey = state['mkey'] + self.key = state['key'] self.k_mtime = state['k_mtime'] SMaster.secrets = state['secrets'] def __getstate__(self): return {'opts': self.opts, - 'key': self.key, 'req_channels': self.req_channels, 'mkey': self.mkey, + 'key': self.key, 'k_mtime': self.k_mtime, - 'secrets': SMaster.secrets, - 'kwargs': self.kwargs} + 'log_queue': self.log_queue, + 'secrets': SMaster.secrets} def _handle_signals(self, signum, sigframe): for channel in getattr(self, 'req_channels', ()): From 1b12940a1fc180331a118697e0fac5a96c41094c Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 12 Sep 2016 23:42:12 +0900 Subject: [PATCH 16/27] Docs clarification for module sync and state.apply (#36217) Closes #35340 --- doc/faq.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index e3e901d871..5a6a500da5 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -147,14 +147,21 @@ should be opened on our tracker_, with the following information: Why aren't my custom modules/states/etc. available on my Minions? ----------------------------------------------------------------- -Custom modules are only synced to Minions when :mod:`state.apply -`, :mod:`saltutil.sync_modules -`, or :mod:`saltutil.sync_all -` is run. Similarly, custom states are only -synced to Minions when :mod:`state.apply `, +Custom modules are synced to Minions when +:mod:`saltutil.sync_modules `, +or :mod:`saltutil.sync_all ` is run. +Custom modules are also synced by :mod:`state.apply` when run without +any arguments. + + +Similarly, custom states are synced to Minions +when :mod:`state.apply `, :mod:`saltutil.sync_states `, or :mod:`saltutil.sync_all ` is run. +Custom states are also synced by :mod:`state.apply` +when run without any arguments. + Other custom types (renderers, outputters, etc.) have similar behavior, see the documentation for the :mod:`saltutil ` module for more information. From 019ef1c47ce5a3af0aea9e47812b84e6fe556c0e Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Mon, 12 Sep 2016 11:42:43 -0600 Subject: [PATCH 17/27] Remove invalid JSON for Mac OS X (#36167) * Remove invalid JSON for Mac OS X * Check for fsevents line and remove --- salt/modules/npm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/modules/npm.py b/salt/modules/npm.py index 672d3073d3..09660a10c4 100644 --- a/salt/modules/npm.py +++ b/salt/modules/npm.py @@ -185,6 +185,11 @@ def _extract_json(npm_output): lines = lines[1:] while lines and not lines[-1].startswith('}') and not lines[-1].startswith(']'): lines = lines[:-1] + # Mac OSX with fsevents includes the following line in the return + # when a new module is installed which is invalid JSON: + # [fsevents] Success: "..." + while lines and lines[0].startswith('[fsevents]'): + lines = lines[1:] try: return json.loads(''.join(lines)) except ValueError: From d9bbba9b245b6adfcac46899ee3e715eb4a42705 Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Mon, 12 Sep 2016 11:57:41 -0600 Subject: [PATCH 18/27] Add missing documentation for the mac_package module (#36192) * Add missing documentation for the mac_package module * Standardize ID in docstrings --- salt/modules/mac_package.py | 117 ++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 38 deletions(-) diff --git a/salt/modules/mac_package.py b/salt/modules/mac_package.py index 10ae18eb8f..2af3528e84 100644 --- a/salt/modules/mac_package.py +++ b/salt/modules/mac_package.py @@ -44,22 +44,21 @@ def install(pkg, target='LocalSystem', store=False, allow_untrusted=False): ''' Install a pkg file + Args: + pkg (str): The package to install + target (str): The target in which to install the package to + store (bool): Should the package be installed as if it was from the + store? + allow_untrusted (bool): Allow the installation of untrusted packages? + + Returns: + dict: A dictionary containing the results of the installation + CLI Example: .. code-block:: bash salt '*' macpackage.install test.pkg - - target - The target in which to install the package to - - store - Should the package be installed as if it was from the store? - - allow_untrusted - Allow the installation of untrusted packages? - - ''' pkg = _quote(pkg) target = _quote(target) @@ -81,21 +80,21 @@ def install(pkg, target='LocalSystem', store=False, allow_untrusted=False): def install_app(app, target='/Applications/'): ''' - Install an app file + Install an app file by moving it into the specified Applications directory + + Args: + app (str): The location of the .app file + target (str): The target in which to install the package to + Default is ''/Applications/'' + + Returns: + str: The results of the rsync command CLI Example: .. code-block:: bash salt '*' macpackage.install_app /tmp/tmp.app /Applications/ - - app - The location of the .app file - - target - The target in which to install the package to - - ''' app = _quote(app) target = _quote(target) @@ -117,18 +116,19 @@ def install_app(app, target='/Applications/'): def uninstall_app(app): ''' - Uninstall an app file + Uninstall an app file by removing it from the Applications directory + + Args: + app (str): The location of the .app file + + Returns: + bool: True if successful, otherwise False CLI Example: .. code-block:: bash salt '*' macpackage.uninstall_app /Applications/app.app - - app - The location of the .app file - - ''' return __salt__['file.remove'](app) @@ -139,8 +139,18 @@ def mount(dmg): Attempt to mount a dmg file to a temporary location and return the location of the pkg file inside - dmg - The location of the dmg file to mount + Args: + dmg (str): The location of the dmg file to mount + + Returns: + tuple: Tuple containing the results of the command along with the mount + point + + CLI Example: + + .. code-block:: bash + + salt '*' macpackage.mount /tmp/software.dmg ''' temp_dir = __salt__['temp.dir'](prefix='dmg-') @@ -154,8 +164,17 @@ def unmount(mountpoint): ''' Attempt to unmount a dmg file from a temporary location - mountpoint - The location of the mount point + Args: + mountpoint (str): The location of the mount point + + Returns: + str: The results of the hdutil detach command + + CLI Example: + + .. code-block:: bash + + salt '*' macpackage.unmount /dev/disk2 ''' cmd = 'hdiutil detach "{0}"'.format(mountpoint) @@ -167,6 +186,14 @@ def installed_pkgs(): ''' Return the list of installed packages on the machine + Returns: + list: List of installed packages + + CLI Example: + + .. code-block:: bash + + salt '*' macpackage.installed_pkgs ''' cmd = 'pkgutil --pkgs' @@ -176,12 +203,19 @@ def installed_pkgs(): def get_pkg_id(pkg): ''' - Attempt to get the package id from a .pkg file + Attempt to get the package ID from a .pkg file - Returns all of the package ids if the pkg file contains multiple + Args: + pkg (str): The location of the pkg file - pkg - The location of the pkg file + Returns: + list: List of all of the package IDs + + CLI Example: + + .. code-block:: bash + + salt '*' macpackage.get_pkg_id /tmp/test.pkg ''' pkg = _quote(pkg) package_ids = [] @@ -217,12 +251,19 @@ def get_pkg_id(pkg): def get_mpkg_ids(mpkg): ''' - Attempt to get the package ids from a mounted .mpkg file + Attempt to get the package IDs from a mounted .mpkg file - Returns all of the package ids if the pkg file contains multiple + Args: + mpkg (str): The location of the mounted mpkg file - pkg - The location of the mounted mpkg file + Returns: + list: List of package IDs + + CLI Example: + + .. code-block:: bash + + salt '*' macpackage.get_mpkg_ids /dev/disk2 ''' mpkg = _quote(mpkg) package_infos = [] From 37c48472d639e6af5eb66f1bc9f69d5204299905 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 11 Sep 2016 21:12:57 -0500 Subject: [PATCH 19/27] Add documentation for fileserver_list_cache_time --- doc/ref/configuration/master.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index bcef7e17f4..9719b16cb8 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -1363,6 +1363,36 @@ is impacted. fileserver_limit_traversal: False +.. conf_master:: fileserver_list_cache_time + +``fileserver_list_cache_time`` +------------------------------ + +.. versionadded:: 2014.1.0 +.. versionchanged:: Carbon + The default was changed from ``30`` seconds to ``20``. + +Default: ``20`` + +Salt caches the list of files/symlinks/directories for each fileserver backend +and environment as they are requested, to guard against a performance +bottleneck at scale when many minions all ask the fileserver which files are +available simultaneously. This configuration parameter allows for the max age +of that cache to be altered. + +Set this value to ``0`` to disable use of this cache altogether, but keep in +mind that this may increase the CPU load on the master when running a highstate +on a large number of minions. + +.. note:: + Rather than altering this configuration parameter, it may be advisable to + use the :mod:`fileserver.clear_list_cache + ` runner to clear these caches. + +.. code-block:: yaml + + fileserver_list_cache_time: 5 + .. conf_master:: hash_type ``hash_type`` From 26c426f22a7ff3ccdcfac27a06bca003dd9ec31f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 11 Sep 2016 21:13:29 -0500 Subject: [PATCH 20/27] Add function to remove fileserver list caches to fileserver class --- salt/fileserver/__init__.py | 82 +++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index 3ad034c75e..c638f6847b 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -635,6 +635,88 @@ class Fileserver(object): except (IndexError, TypeError): return '', None + def clear_file_list_cache(self, load): + ''' + Deletes the file_lists cache files + ''' + if 'env' in load: + salt.utils.warn_until( + 'Oxygen', + 'Parameter \'env\' has been detected in the argument list. This ' + 'parameter is no longer used and has been replaced by \'saltenv\' ' + 'as of Salt Carbon. This warning will be removed in Salt Oxygen.' + ) + load.pop('env') + + saltenv = load.get('saltenv', []) + if saltenv is not None: + if not isinstance(saltenv, list): + try: + saltenv = [x.strip() for x in saltenv.split(',')] + except AttributeError: + saltenv = [x.strip() for x in str(saltenv).split(',')] + + for idx, val in enumerate(saltenv): + if not isinstance(val, six.string_types): + saltenv[idx] = six.text_type(val) + + ret = {} + fsb = self._gen_back(load.pop('fsbackend', None)) + list_cachedir = os.path.join(self.opts['cachedir'], 'file_lists') + try: + file_list_backends = os.listdir(list_cachedir) + except OSError as exc: + if exc.errno == errno.ENOENT: + log.debug('No file list caches found') + return {} + else: + log.error( + 'Failed to get list of saltenvs for which the master has ' + 'cached file lists: %s', exc + ) + + for back in file_list_backends: + # Account for the fact that the file_list cache directory for gitfs + # is 'git', hgfs is 'hg', etc. + back_virtualname = re.sub('fs$', '', back) + try: + cache_files = os.listdir(os.path.join(list_cachedir, back)) + except OSError as exc: + log.error( + 'Failed to find file list caches for saltenv \'%s\': %s', + back, exc + ) + continue + for cache_file in cache_files: + try: + cache_saltenv, extension = cache_file.rsplit('.', 1) + except ValueError: + # Filename has no dot in it. Not a cache file, ignore. + continue + if extension != 'p': + # Filename does not end in ".p". Not a cache file, ignore. + continue + elif back_virtualname not in fsb or \ + (saltenv is not None and cache_saltenv not in saltenv): + log.debug( + 'Skipping %s file list cache for saltenv \'%s\'', + back, cache_saltenv + ) + continue + try: + os.remove(os.path.join(list_cachedir, back, cache_file)) + except OSError as exc: + if exc.errno != errno.ENOENT: + log.error('Failed to remove %s: %s', + exc.filename, exc.strerror) + else: + ret.setdefault(back, []).append(cache_saltenv) + log.debug( + 'Removed %s file list cache for saltenv \'%s\'', + cache_saltenv, back + ) + return ret + def file_list(self, load): ''' Return a list of files from the dominant environment From 6722ad9ecf39cd2664663e4b9f8bb82ca83dfdfb Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 11 Sep 2016 21:14:25 -0500 Subject: [PATCH 21/27] Add runner function to remove file list caches --- salt/runners/fileserver.py | 108 +++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/salt/runners/fileserver.py b/salt/runners/fileserver.py index 82656970b5..2590077c1a 100644 --- a/salt/runners/fileserver.py +++ b/salt/runners/fileserver.py @@ -39,6 +39,114 @@ def envs(backend=None, sources=False): return fileserver.envs(back=backend, sources=sources) +def clear_file_list_cache(saltenv=None, backend=None): + ''' + .. versionadded:: Carbon + + The Salt fileserver caches the files/directories/symlinks for each + fileserver backend and environment as they are requested. This is done to + help the fileserver scale better. Without this caching, when + hundreds/thousands of minions simultaneously ask the master what files are + available, this would cause the master's CPU load to spike as it obtains + the same information separately for each minion. + + saltenv + By default, this runner will clear the file list caches for all + environments. This argument allows for a list of environments to be + passed, to clear more selectively. This list can be passed either as a + comma-separated string, or a Python list. + + backend + Similar to the ``saltenv`` parameter, this argument will restrict the + cache clearing to specific fileserver backends (the default behavior is + to clear from all enabled fileserver backends). This list can be passed + either as a comma-separated string, or a Python list. + + .. note: + The maximum age for the cached file lists (i.e. the age at which the + cache will be disregarded and rebuilt) is defined by the + :conf_master:`fileserver_list_cache_time` configuration parameter. + + Since the ability to clear these caches is often required by users writing + custom runners which add/remove files, this runner can easily be called + from within a custom runner using any of the following examples: + + .. code-block:: python + + # Clear all file list caches + __salt__['fileserver.clear_file_list_cache']() + # Clear just the 'base' saltenv file list caches + __salt__['fileserver.clear_file_list_cache'](saltenv='base') + # Clear just the 'base' saltenv file list caches from just the 'roots' + # fileserver backend + __salt__['fileserver.clear_file_list_cache'](saltenv='base', backend='roots') + # Clear all file list caches from the 'roots' fileserver backend + __salt__['fileserver.clear_file_list_cache'](backend='roots') + + .. note:: + In runners, the ``__salt__`` dictionary will likely be renamed to + ``__runner__`` in a future Salt release to distinguish runner functions + from remote execution functions. See `this GitHub issue`_ for + discussion/updates on this. + + .. _`this GitHub issue`: https://github.com/saltstack/salt/issues/34958 + + If using Salt's Python API (not a runner), the following examples are + equivalent to the ones above: + + .. code-block:: python + + import salt.config + import salt.runner + + opts = salt.config.master_config('/etc/salt/master') + opts['fun'] = 'fileserver.clear_file_list_cache' + + # Clear all file list_caches + opts['arg'] = [] # No arguments + runner = salt.runner.Runner(opts) + cleared = runner.run() + + # Clear just the 'base' saltenv file list caches + opts['arg'] = ['base', None] + runner = salt.runner.Runner(opts) + cleared = runner.run() + + # Clear just the 'base' saltenv file list caches from just the 'roots' + # fileserver backend + opts['arg'] = ['base', 'roots'] + runner = salt.runner.Runner(opts) + cleared = runner.run() + + # Clear all file list caches from the 'roots' fileserver backend + opts['arg'] = [None, 'roots'] + runner = salt.runner.Runner(opts) + cleared = runner.run() + + + This function will return a dictionary showing a list of environments which + were cleared for each backend. An empty return dictionary means that no + changes were made. + + CLI Examples: + + .. code-block:: bash + + # Clear all file list caches + salt-run fileserver.clear_file_list_cache + # Clear just the 'base' saltenv file list caches + salt-run fileserver.clear_file_list_cache saltenv=base + # Clear just the 'base' saltenv file list caches from just the 'roots' + # fileserver backend + salt-run fileserver.clear_file_list_cache saltenv=base backend=roots + # Clear all file list caches from the 'roots' fileserver backend + salt-run fileserver.clear_file_list_cache backend=roots + ''' + fileserver = salt.fileserver.Fileserver(__opts__) + load = {'saltenv': saltenv, 'fsbackend': backend} + return fileserver.clear_file_list_cache(load=load) + + def file_list(saltenv='base', backend=None): ''' Return a list of files from the salt fileserver From d3e9c4c5fbab3c6af0cad988a9af7dff9918b229 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 12 Sep 2016 01:13:16 -0500 Subject: [PATCH 22/27] Couple fixes to _gen_back 1. Wasn't properly handling the backend being passed in as a Python list 2. Didn't work properly in the test suite due to the backend being an ImmutableList instead of a list. This is because of how the opts are loaded in the test suite, all mutable types are immutable versions of themselves from salt.utils.immutabletypes. --- salt/fileserver/__init__.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index c638f6847b..820e9d66ba 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -5,6 +5,7 @@ File server pluggable modules and generic backend functions # Import python libs from __future__ import absolute_import +import collections import errno import fnmatch import logging @@ -325,10 +326,18 @@ class Fileserver(object): if not back: back = self.opts['fileserver_backend'] else: - try: - back = back.split(',') - except AttributeError: - back = six.text_type(back).split(',') + if not isinstance(back, list): + try: + back = back.split(',') + except AttributeError: + back = six.text_type(back).split(',') + + if isinstance(back, collections.Sequence): + # The test suite uses an ImmutableList type (based on + # collections.Sequence) for lists, which breaks this function in + # the test suite. This normalizes the value from the opts into a + # list if it is based on collections.Sequence. + back = list(back) ret = [] if not isinstance(back, list): From 322047a8449bc24bc65c67a125433e4e382e9a3b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 12 Sep 2016 01:16:09 -0500 Subject: [PATCH 23/27] Add integration tests for fileserver.clear_file_list_cache --- tests/integration/runners/fileserver.py | 69 ++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/tests/integration/runners/fileserver.py b/tests/integration/runners/fileserver.py index b0a79b7d22..4d3694ac67 100644 --- a/tests/integration/runners/fileserver.py +++ b/tests/integration/runners/fileserver.py @@ -4,6 +4,7 @@ Tests for the fileserver runner ''' # Import Python libs from __future__ import absolute_import +import contextlib # Import Salt Testing libs from salttesting.helpers import ensure_in_syspath @@ -13,7 +14,7 @@ ensure_in_syspath('../../') import integration -class ManageTest(integration.ShellCase): +class FileserverTest(integration.ShellCase): ''' Test the fileserver runner ''' @@ -70,6 +71,70 @@ class ManageTest(integration.ShellCase): ret = self.run_run_plus(fun='fileserver.envs', backend=['roots']) self.assertIsInstance(ret['return'], list) + def test_clear_file_list_cache(self): + ''' + fileserver.clear_file_list_cache + + If this test fails, then something may have changed in the test suite + and we may have more than just "roots" configured in the fileserver + backends. This assert will need to be updated accordingly. + ''' + @contextlib.contextmanager + def gen_cache(): + ''' + Create file_list cache so we have something to clear + ''' + self.run_run_plus(fun='fileserver.file_list') + yield + + # Test with no arguments + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache') + self.assertEqual(ret['return'], {'roots': ['base']}) + + # Test with backend passed as string + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache', + backend='roots') + self.assertEqual(ret['return'], {'roots': ['base']}) + + # Test with backend passed as list + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache', + backend=['roots']) + self.assertEqual(ret['return'], {'roots': ['base']}) + + # Test with backend passed as string, but with a nonsense backend + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache', + backend='notarealbackend') + self.assertEqual(ret['return'], {}) + + # Test with saltenv passed as string + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache', + saltenv='base') + self.assertEqual(ret['return'], {'roots': ['base']}) + + # Test with saltenv passed as list + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache', + saltenv=['base']) + self.assertEqual(ret['return'], {'roots': ['base']}) + + # Test with saltenv passed as string, but with a nonsense saltenv + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache', + saltenv='notarealsaltenv') + self.assertEqual(ret['return'], {}) + + # Test with both backend and saltenv passed + with gen_cache(): + ret = self.run_run_plus(fun='fileserver.clear_file_list_cache', + backend='roots', + saltenv='base') + self.assertEqual(ret['return'], {'roots': ['base']}) + def test_file_list(self): ''' fileserver.file_list @@ -117,4 +182,4 @@ class ManageTest(integration.ShellCase): if __name__ == '__main__': from integration import run_tests - run_tests(ManageTest) + run_tests(FileserverTest) From ae973a14fbb6845cb1e8eecc6d6ad4be1bea5314 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 12 Sep 2016 16:13:52 -0500 Subject: [PATCH 24/27] roots backend: Don't include '.' or '..' in empty_dirs --- salt/fileserver/roots.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index e137298c71..349437dd76 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -335,7 +335,8 @@ def _file_lists(load, form): dir_rel_fn = dir_rel_fn.replace('\\', '/') ret['dirs'].append(dir_rel_fn) if len(dirs) == 0 and len(files) == 0: - if not salt.fileserver.is_file_ignored(__opts__, dir_rel_fn): + if dir_rel_fn not in ('.', '..') \ + and not salt.fileserver.is_file_ignored(__opts__, dir_rel_fn): ret['empty_dirs'].append(dir_rel_fn) for fname in files: is_link = os.path.islink(os.path.join(root, fname)) From b7a1a678286a6f087ad2c83bb7ff760a82264516 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 12 Sep 2016 16:14:36 -0500 Subject: [PATCH 25/27] Improve accuracy of fileserver runner tests The tests that return files, symlinks, directories, and empty dirs were all only testing that the type of the return data was the same as what was expected. By not testing the content, we overlooked a corner case in which backends passed into the fileserver as a Python list would not be handled properly. This has since been fixed in PR #36244, but these tests will help keep this sort of issue from regressing. --- tests/integration/runners/fileserver.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/integration/runners/fileserver.py b/tests/integration/runners/fileserver.py index 4d3694ac67..3ddff5e1cf 100644 --- a/tests/integration/runners/fileserver.py +++ b/tests/integration/runners/fileserver.py @@ -24,18 +24,17 @@ class FileserverTest(integration.ShellCase): ''' ret = self.run_run_plus(fun='fileserver.dir_list') self.assertIsInstance(ret['return'], list) + self.assertTrue('_modules' in ret['return']) # Backend submitted as a string - ret = self.run_run_plus( - fun='fileserver.dir_list', - backend='roots') + ret = self.run_run_plus(fun='fileserver.dir_list', backend='roots') self.assertIsInstance(ret['return'], list) + self.assertTrue('_modules' in ret['return']) # Backend submitted as a list - ret = self.run_run_plus( - fun='fileserver.dir_list', - backend=['roots']) + ret = self.run_run_plus(fun='fileserver.dir_list', backend=['roots']) self.assertIsInstance(ret['return'], list) + self.assertTrue('_modules' in ret['return']) def test_empty_dir_list(self): ''' @@ -43,18 +42,21 @@ class FileserverTest(integration.ShellCase): ''' ret = self.run_run_plus(fun='fileserver.empty_dir_list') self.assertIsInstance(ret['return'], list) + self.assertEqual(ret['return'], []) # Backend submitted as a string ret = self.run_run_plus( fun='fileserver.empty_dir_list', backend='roots') self.assertIsInstance(ret['return'], list) + self.assertEqual(ret['return'], []) # Backend submitted as a list ret = self.run_run_plus( fun='fileserver.empty_dir_list', backend=['roots']) self.assertIsInstance(ret['return'], list) + self.assertEqual(ret['return'], []) def test_envs(self): ''' @@ -141,14 +143,17 @@ class FileserverTest(integration.ShellCase): ''' ret = self.run_run_plus(fun='fileserver.file_list') self.assertIsInstance(ret['return'], list) + self.assertTrue('grail/scene33' in ret['return']) # Backend submitted as a string ret = self.run_run_plus(fun='fileserver.file_list', backend='roots') self.assertIsInstance(ret['return'], list) + self.assertTrue('grail/scene33' in ret['return']) # Backend submitted as a list ret = self.run_run_plus(fun='fileserver.file_list', backend=['roots']) self.assertIsInstance(ret['return'], list) + self.assertTrue('grail/scene33' in ret['return']) def test_symlink_list(self): ''' @@ -156,14 +161,17 @@ class FileserverTest(integration.ShellCase): ''' ret = self.run_run_plus(fun='fileserver.symlink_list') self.assertIsInstance(ret['return'], dict) + self.assertTrue('dest_sym' in ret['return']) # Backend submitted as a string ret = self.run_run_plus(fun='fileserver.symlink_list', backend='roots') self.assertIsInstance(ret['return'], dict) + self.assertTrue('dest_sym' in ret['return']) # Backend submitted as a list ret = self.run_run_plus(fun='fileserver.symlink_list', backend=['roots']) self.assertIsInstance(ret['return'], dict) + self.assertTrue('dest_sym' in ret['return']) def test_update(self): ''' From 38e700c9779073c609d912669d0a13f3f26461da Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 12 Sep 2016 16:27:59 -0500 Subject: [PATCH 26/27] Reduce fileserver_list_cache_time default to 20 sec --- salt/fileserver/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index 820e9d66ba..0234b5735a 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -122,8 +122,8 @@ def check_file_list_cache(opts, form, list_cache, w_lock): age = time.time() - cache_stat.st_mtime else: # if filelist does not exists yet, mark it as expired - age = opts.get('fileserver_list_cache_time', 30) + 1 - if age < opts.get('fileserver_list_cache_time', 30): + age = opts.get('fileserver_list_cache_time', 20) + 1 + if age < opts.get('fileserver_list_cache_time', 20): # Young enough! Load this sucker up! with salt.utils.fopen(list_cache, 'rb') as fp_: log.trace('Returning file_lists cache data from ' From 9ede50b4517680c30bbff5a50d316ec18299e9e7 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 12 Sep 2016 15:43:06 -0600 Subject: [PATCH 27/27] fix archive test to only run on redhat --- tests/integration/states/archive.py | 34 +++++++++++++++++++---------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/integration/states/archive.py b/tests/integration/states/archive.py index d5f16288d3..d786700346 100644 --- a/tests/integration/states/archive.py +++ b/tests/integration/states/archive.py @@ -5,13 +5,15 @@ Tests for the archive state # Import python libs from __future__ import absolute_import import os +import platform +import socket import shutil import threading import tornado.ioloop import tornado.web # Import Salt Testing libs -from salttesting import TestCase +from salttesting import skipIf from salttesting.helpers import ensure_in_syspath ensure_in_syspath('../../') @@ -30,13 +32,17 @@ ARCHIVE_TAR_SOURCE = 'http://localhost:{0}/custom.tar.gz'.format(PORT) UNTAR_FILE = ARCHIVE_DIR + 'custom/README' ARCHIVE_TAR_HASH = 'md5=7643861ac07c30fe7d2310e9f25ca514' STATE_DIR = os.path.join(integration.FILES, 'file', 'base') +if '7' in platform.dist()[1]: + REDHAT7 = True +else: + REDHAT7 = False -class SetupWebServer(TestCase): +@skipIf(not REDHAT7, 'Only run on redhat7 for now due to hanging issues on other OS') +class ArchiveTest(integration.ModuleCase, + integration.SaltReturnAssertsMixIn): ''' - Setup and Teardown of Web Server - Only need to set this up once not - before all tests + Validate the archive state ''' @classmethod def webserver(cls): @@ -51,22 +57,26 @@ class SetupWebServer(TestCase): @classmethod def setUpClass(cls): + ''' + start tornado app on thread + and wait till its running + ''' cls.server_thread = threading.Thread(target=cls.webserver) cls.server_thread.daemon = True cls.server_thread.start() + # check if tornado app is up + port_closed = True + while port_closed: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(('127.0.0.1', PORT)) + if result == 0: + port_closed = False @classmethod def tearDownClass(cls): tornado.ioloop.IOLoop.instance().stop() cls.server_thread.join() - -class ArchiveTest(SetupWebServer, - integration.ModuleCase, - integration.SaltReturnAssertsMixIn): - ''' - Validate the archive state - ''' def _check_ext_remove(self, dir, file): ''' function to check if file was extracted