Merge pull request #8217 from cachedout/file_traversal_fixing

File traversal fixing #6928

conf/master

@@ -327,6 +327,14 @@
 #
 #fileserver_ignoresymlinks: True
 #
+# By default, the Salt fileserver recurses fully into all defined environments
+# to attempt to find files. To limit this behavior so that the fileserver only
+# traverses directories with SLS files and special Salt directories like _modules,
+# enable the option below. This might be useful for installations where a file root
+# has a very large number of files and performance is impacted. Default is False.
+#
+# fileserver_limit_traversal: False
+#
 # Git fileserver backend configuration
 # When using the git fileserver backend at least one git remote needs to be
 # defined. The user running the salt master will need read access to the repo.

conf/minion

@@ -344,6 +344,16 @@
 #  base:
 #    - /srv/salt
 
+# By default, the Salt fileserver recurses fully into all defined environments
+# to attempt to find files. To limit this behavior so that the fileserver only
+# traverses directories with SLS files and special Salt directories like _modules,
+# enable the option below. This might be useful for installations where a file root
+# has a very large number of files and performance is negatively impacted.
+#
+# Default is False.
+#
+# fileserver_limit_traversal: False
+
 # The hash_type is the hash to use when discovering the hash of a file in
 # the local fileserver. The default is md5, but sha1, sha224, sha256, sha384
 # and sha512 are also supported.

salt/config.py

@@ -146,6 +146,7 @@ VALID_OPTS = {
     'fileserver_backend': list,
     'fileserver_followsymlinks': bool,
     'fileserver_ignoresymlinks': bool,
+    'fileserver_limit_traversal': bool,
     'max_open_files': int,
     'auto_accept': bool,
     'master_tops': bool,
@@ -195,6 +196,7 @@ DEFAULT_MINION_OPTS = {
     'file_roots': {
         'base': [syspaths.BASE_FILE_ROOTS_DIR],
     },
+    'fileserver_limit_traversal': False,
     'pillar_roots': {
         'base': [syspaths.BASE_PILLAR_ROOTS_DIR],
     },
@@ -307,6 +309,7 @@ DEFAULT_MASTER_OPTS = {
     'fileserver_backend': ['roots'],
     'fileserver_followsymlinks': True,
     'fileserver_ignoresymlinks': False,
+    'fileserver_limit_traversal': False,
     'max_open_files': 100000,
     'hash_type': 'md5',
     'conf_file': os.path.join(syspaths.CONFIG_DIR, 'master'),

salt/fileclient.py

@@ -248,14 +248,43 @@ class Client(object):
         Return a list of all available sls modules on the master for a given
         environment
         '''
+
+        limit_traversal = self.opts.get('fileserver_limit_traversal', False)
         states = []
-        for path in self.file_list(env):
-            if path.endswith('.sls'):
-                # is an sls module!
-                if path.endswith('{0}init.sls'.format('/')):
-                    states.append(path.replace('/', '.')[:-9])
-                else:
-                    states.append(path.replace('/', '.')[:-4])
+
+        if limit_traversal:
+            if env not in self.opts['file_roots']:
+                log.warning("During an attempt to list states for env {0}, the environment could not be found in the \
+                            configured file roots".format(env))
+                return states
+            for path in self.opts['file_roots'][env]:
+                for root, dirs, files in os.walk(path, topdown=True):
+                    log.debug("Searching for states in dirs {0} and files {1}".format(dirs, files))
+                    if not [file.endswith('.sls') for file in files]:
+                        #  Use shallow copy so we don't disturb the memory used by os.walk. Otherwise this breaks!
+                        del dirs[:]
+                    else:
+                        for found_file in files:
+                            stripped_root = os.path.relpath(root, path).replace('/', '.')
+                            if found_file.endswith(('.sls')):
+                                if found_file.endswith('init.sls'):
+                                    if stripped_root.endswith('.'):
+                                        stripped_root = stripped_root.rstrip('.')
+                                    states.append(stripped_root)
+                                else:
+                                    if not stripped_root.endswith('.'):
+                                        stripped_root += '.'
+                                    if stripped_root.startswith('.'):
+                                        stripped_root = stripped_root.lstrip('.')
+                                    states.append(stripped_root + found_file[:-4])
+        else:
+            for path in self.file_list(env):
+                if path.endswith('.sls'):
+                    # is an sls module!
+                    if path.endswith('{0}init.sls'.format('/')):
+                        states.append(path.replace('/', '.')[:-9])
+                    else:
+                        states.append(path.replace('/', '.')[:-4])
         return states
 
     def get_state(self, sls, env):

salt/fileserver/__init__.py

@@ -139,6 +139,7 @@ class Fileserver(object):
         for fsb in back:
             fstr = '{0}.update'.format(fsb)
             if fstr in self.servers:
+                log.debug('Updating fileserver cache')
                 self.servers[fstr]()
 
     def envs(self, back=None, sources=False):