Merge pull request #34118 from ctrlrsf/i-33907-dont-walk-file-ignore-dirs

Don't traverse directories that match file_ignore_regex and file_ignore_glob

salt/fileclient.py

@@ -783,6 +783,8 @@ class LocalClient(Client):
             for root, dirs, files in os.walk(
                 os.path.join(path, prefix), followlinks=True
             ):
+                # Don't walk any directories that match file_ignore_regex or glob
+                dirs[:] = [d for d in dirs if not salt.fileserver.is_file_ignored(self.opts, d)]
                 for fname in files:
                     relpath = os.path.relpath(os.path.join(root, fname), path)
                     ret.append(sdecode(relpath))
@@ -801,6 +803,8 @@ class LocalClient(Client):
             for root, dirs, files in os.walk(
                 os.path.join(path, prefix), followlinks=True
             ):
+                # Don't walk any directories that match file_ignore_regex or glob
+                dirs[:] = [d for d in dirs if not salt.fileserver.is_file_ignored(self.opts, d)]
                 if len(dirs) == 0 and len(files) == 0:
                     ret.append(sdecode(os.path.relpath(root, path)))
         return ret

salt/fileserver/__init__.py

@@ -171,7 +171,7 @@ def check_env_cache(opts, env_cache):
     return None
 
 
-def generate_mtime_map(path_map):
+def generate_mtime_map(opts, path_map):
     '''
     Generate a dict of filename -> mtime
     '''
@@ -179,6 +179,8 @@ def generate_mtime_map(path_map):
     for saltenv, path_list in six.iteritems(path_map):
         for path in path_list:
             for directory, dirnames, filenames in os.walk(path):
+                # Don't walk any directories that match file_ignore_regex or glob
+                dirnames[:] = [d for d in dirnames if not is_file_ignored(opts, d)]
                 for item in filenames:
                     try:
                         file_path = os.path.join(directory, item)

salt/fileserver/roots.py

@@ -159,7 +159,7 @@ def update():
             'backend': 'roots'}
 
     # generate the new map
-    new_mtime_map = salt.fileserver.generate_mtime_map(__opts__['file_roots'])
+    new_mtime_map = salt.fileserver.generate_mtime_map(__opts__, __opts__['file_roots'])
 
     old_mtime_map = {}
     # if you have an old map, load that
@@ -327,6 +327,9 @@ def _file_lists(load, form):
             for root, dirs, files in os.walk(
                     path,
                     followlinks=__opts__['fileserver_followsymlinks']):
+                # Don't walk any directories that match file_ignore_regex or glob
+                dirs[:] = [d for d in dirs if not salt.fileserver.is_file_ignored(__opts__, d)]
+
                 dir_rel_fn = os.path.relpath(root, path)
                 if __opts__.get('file_client', 'remote') == 'local' and os.path.sep == "\\":
                     dir_rel_fn = dir_rel_fn.replace('\\', '/')
@@ -406,6 +409,8 @@ def symlink_list(load):
             prefix = ''
         # Adopting rsync functionality here and stopping at any encounter of a symlink
         for root, dirs, files in os.walk(os.path.join(path, prefix), followlinks=False):
+            # Don't walk any directories that match file_ignore_regex or glob
+            dirs[:] = [d for d in dirs if not salt.fileserver.is_file_ignored(__opts__, d)]
             for fname in files:
                 if not os.path.islink(os.path.join(root, fname)):
                     continue