From 562bed8fec35faab862885cba55244d1572f9116 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 31 Oct 2013 15:53:11 -0600 Subject: [PATCH 01/12] Modify list_states to accept an option to perform a top-down search and to cease traversal after encountering a directory without a SLS file present. --- salt/fileclient.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 330ee7c6f4..045f83cbad 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -243,19 +243,36 @@ class Client(object): return '' - def list_states(self, env): + def list_states(self, env, limit_traversal=False): ''' Return a list of all available sls modules on the master for a given environment ''' states = [] - for path in self.file_list(env): - if path.endswith('.sls'): - # is an sls module! - if path.endswith('{0}init.sls'.format('/')): - states.append(path.replace('/', '.')[:-9]) - else: - states.append(path.replace('/', '.')[:-4]) + + if limit_traversal: + if env not in self.opts['file_roots']: + log.warning("During an attempt to list states for env {0}, the environment could not be found in the \ + configured file roots".format(env)) + return states + for path in self.opts['file_roots'][env]: + for root, dirs, files in os.walk(path, topdown=True): + log.debug("Searching for states in dirs {0} and files {1}".format(dirs, files)) + if not [file.endswith('.sls') for file in files]: + # Use shallow copy so we don't disturb the memory used by os.walk. Otherwise this breaks! + del dirs[:] + else: + states.extend([sls_file[:-4] for sls_file in files if sls_file.endswith('.sls')]) + + else: + for path in self.file_list(env): + if path.endswith('.sls'): + # is an sls module! + if path.endswith('{0}init.sls'.format('/')): + states.append(path.replace('/', '.')[:-9]) + else: + states.append(path.replace('/', '.')[:-4]) + return states def get_state(self, sls, env): From 459feb70f305ea167842b7cb84189b50558ad273 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 31 Oct 2013 17:07:02 -0600 Subject: [PATCH 02/12] Provide a configuration switch to enable limited directory traversal. --- conf/master | 8 ++++++++ salt/config.py | 2 ++ salt/fileclient.py | 5 ++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/conf/master b/conf/master index 34413b0698..8f2c0a17c7 100644 --- a/conf/master +++ b/conf/master @@ -327,6 +327,14 @@ # #fileserver_ignoresymlinks: True # +# By default, the Salt fileserver recurses fully into all defined environments +# to attempt to find files. To limit this behavior so that the fileserver only +# traverses directories with SLS files and special Salt directories like _modules, +# enable the option below. This might be useful for installations where a file root +# has a very large number of files and performance is impacted. Default is False. +# +# fileserver_limit_traversal: False +# # Git fileserver backend configuration # When using the git fileserver backend at least one git remote needs to be # defined. The user running the salt master will need read access to the repo. diff --git a/salt/config.py b/salt/config.py index 045c9213f2..97a21182da 100644 --- a/salt/config.py +++ b/salt/config.py @@ -146,6 +146,7 @@ VALID_OPTS = { 'fileserver_backend': list, 'fileserver_followsymlinks': bool, 'fileserver_ignoresymlinks': bool, + 'fileserver_limit_traversal': bool, 'max_open_files': int, 'auto_accept': bool, 'master_tops': bool, @@ -307,6 +308,7 @@ DEFAULT_MASTER_OPTS = { 'fileserver_backend': ['roots'], 'fileserver_followsymlinks': True, 'fileserver_ignoresymlinks': False, + 'fileserver_limit_traversal': False, 'max_open_files': 100000, 'hash_type': 'md5', 'conf_file': os.path.join(syspaths.CONFIG_DIR, 'master'), diff --git a/salt/fileclient.py b/salt/fileclient.py index 045f83cbad..98bba956db 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -243,11 +243,14 @@ class Client(object): return '' - def list_states(self, env, limit_traversal=False): + def list_states(self, env): ''' Return a list of all available sls modules on the master for a given environment ''' + + limit_traversal= self.__opts__.get(['fileserver_limit_traversal'], False) + states = [] if limit_traversal: From 2807b9412e68450cf0d0379d78b3b9bbae2ff387 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 31 Oct 2013 17:19:06 -0600 Subject: [PATCH 03/12] PEP8 and correct opts dict. --- salt/fileclient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 98bba956db..d54e5a6395 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -249,8 +249,7 @@ class Client(object): environment ''' - limit_traversal= self.__opts__.get(['fileserver_limit_traversal'], False) - + limit_traversal = self.opts.get('fileserver_limit_traversal', False) states = [] if limit_traversal: From ad98ed11ad05d4fffea22191b0330f99d1982f00 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 09:19:30 -0600 Subject: [PATCH 04/12] A minion with locally defined fileroots may also deep-recurse. Allow the minion to limit this behavior through a configuration option. --- conf/minion | 10 ++++++++++ salt/config.py | 1 + 2 files changed, 11 insertions(+) diff --git a/conf/minion b/conf/minion index ee14d256ea..c2c561c13c 100644 --- a/conf/minion +++ b/conf/minion @@ -344,6 +344,16 @@ # base: # - /srv/salt +# By default, the Salt fileserver recurses fully into all defined environments +# to attempt to find files. To limit this behavior so that the fileserver only +# traverses directories with SLS files and special Salt directories like _modules, +# enable the option below. This might be useful for installations where a file root +# has a very large number of files and performance is negatively impacted. +# +# Default is False. +# +# fileserver_limit_traversal: False + # The hash_type is the hash to use when discovering the hash of a file in # the local fileserver. The default is md5, but sha1, sha224, sha256, sha384 # and sha512 are also supported. diff --git a/salt/config.py b/salt/config.py index 97a21182da..dd82517adc 100644 --- a/salt/config.py +++ b/salt/config.py @@ -196,6 +196,7 @@ DEFAULT_MINION_OPTS = { 'file_roots': { 'base': [syspaths.BASE_FILE_ROOTS_DIR], }, + 'fileserver_limit_traversal': False, 'pillar_roots': { 'base': [syspaths.BASE_PILLAR_ROOTS_DIR], }, From 243eaebd4efcaa4ca0de8e209fd810e2c2beb4b9 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 10:20:18 -0600 Subject: [PATCH 05/12] Log the periodic mtime map refreshes. --- salt/fileserver/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index 5c2df10650..b795464864 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -135,6 +135,7 @@ class Fileserver(object): for fsb in back: fstr = '{0}.update'.format(fsb) if fstr in self.servers: + log.debug('Updating fileserver cache for the {0} backend'.format(fsb)) self.servers[fstr]() def envs(self, back=None, sources=False): From 7bbb3233103626d5d5fd4ab62478cf4ef289147e Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 11:53:47 -0600 Subject: [PATCH 06/12] Wasn't properly converting to the Salt-style dot-notation for directory separation. Also correctly strips the init file name. --- salt/fileclient.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index d54e5a6395..f04f36cebd 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -252,7 +252,9 @@ class Client(object): limit_traversal = self.opts.get('fileserver_limit_traversal', False) states = [] + limit_traversal=True if limit_traversal: + print "Limiting traversal" if env not in self.opts['file_roots']: log.warning("During an attempt to list states for env {0}, the environment could not be found in the \ configured file roots".format(env)) @@ -264,8 +266,20 @@ class Client(object): # Use shallow copy so we don't disturb the memory used by os.walk. Otherwise this breaks! del dirs[:] else: - states.extend([sls_file[:-4] for sls_file in files if sls_file.endswith('.sls')]) - + stripped_root = os.path.relpath(root, path) + for found_file in files: + stripped_root = os.path.relpath(root, path).replace('/', '.') + if found_file.endswith(('.sls')): + if found_file.endswith('init.sls'): + if stripped_root.endswith('.'): + stripped_root = stripped_root.rstrip('.') + states.append(stripped_root) + else: + if not stripped_root.endswith('.'): + stripped_root = stripped_root + '.' + if stripped_root.startswith('.'): + stripped_root = stripped_root.lstrip('.') + states.append(stripped_root + found_file[:-4]) else: for path in self.file_list(env): if path.endswith('.sls'): @@ -274,7 +288,7 @@ class Client(object): states.append(path.replace('/', '.')[:-9]) else: states.append(path.replace('/', '.')[:-4]) - + print states return states def get_state(self, sls, env): From ecbe75593c301f9ccd44a1b395bf71be29160045 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 11:57:52 -0600 Subject: [PATCH 07/12] Don't depend on string variable when logging a debug message. (This may be an indicator of a bug elsewhere.) --- salt/fileserver/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index b795464864..2e06babac0 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -135,7 +135,7 @@ class Fileserver(object): for fsb in back: fstr = '{0}.update'.format(fsb) if fstr in self.servers: - log.debug('Updating fileserver cache for the {0} backend'.format(fsb)) + log.debug('Updating fileserver cache') self.servers[fstr]() def envs(self, back=None, sources=False): From 14e69bb8f13ef764b72066463a3e8f4e357df966 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 11:58:54 -0600 Subject: [PATCH 08/12] Remove debugging. --- salt/fileclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index f04f36cebd..d63529126f 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -252,7 +252,6 @@ class Client(object): limit_traversal = self.opts.get('fileserver_limit_traversal', False) states = [] - limit_traversal=True if limit_traversal: print "Limiting traversal" if env not in self.opts['file_roots']: From de4deec97c73ed0387d3c46daf44166b866ef406 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 11:59:18 -0600 Subject: [PATCH 09/12] Remove duplicate. --- salt/fileclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index d63529126f..c2e7ceb1a7 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -265,7 +265,6 @@ class Client(object): # Use shallow copy so we don't disturb the memory used by os.walk. Otherwise this breaks! del dirs[:] else: - stripped_root = os.path.relpath(root, path) for found_file in files: stripped_root = os.path.relpath(root, path).replace('/', '.') if found_file.endswith(('.sls')): From c649946707b14eca673cba5aaa0a39a15a1050bb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 12:04:33 -0600 Subject: [PATCH 10/12] Use augmented assignment. --- salt/fileclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index c2e7ceb1a7..f6e372f3a9 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -274,7 +274,7 @@ class Client(object): states.append(stripped_root) else: if not stripped_root.endswith('.'): - stripped_root = stripped_root + '.' + stripped_root += '.' if stripped_root.startswith('.'): stripped_root = stripped_root.lstrip('.') states.append(stripped_root + found_file[:-4]) From 3c1f194aa9c260ac62debc92dde2a4cc2e54d91e Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 12:05:56 -0600 Subject: [PATCH 11/12] Remove final bit of debugging. --- salt/fileclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index f6e372f3a9..10a437da54 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -286,7 +286,6 @@ class Client(object): states.append(path.replace('/', '.')[:-9]) else: states.append(path.replace('/', '.')[:-4]) - print states return states def get_state(self, sls, env): From e81a2e20036b196b6bbc9eaa61e6039c7990ea03 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 1 Nov 2013 12:06:40 -0600 Subject: [PATCH 12/12] Kill more debugging. --- salt/fileclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 10a437da54..66c7b29226 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -253,7 +253,6 @@ class Client(object): states = [] if limit_traversal: - print "Limiting traversal" if env not in self.opts['file_roots']: log.warning("During an attempt to list states for env {0}, the environment could not be found in the \ configured file roots".format(env))