Merge pull request #16659 from etuttle/s3fs-dirs

Properly discover directories in s3fs file server.  Fixes #16638.

salt/fileserver/s3fs.py

@@ -297,7 +297,7 @@ def dir_list(load):
         return ret
 
     # grab all the dirs from the buckets cache file
-    for dirs in _find_files(metadata[saltenv], dirs_only=True).values():
+    for dirs in _find_dirs(metadata[saltenv]).values():
         # trim env and trailing slash
         dirs = _trim_env_off_path(dirs, saltenv, trim_slash=True)
         # remove empty string left by the base env dir in single bucket mode
@@ -465,7 +465,7 @@ def _read_buckets_cache_file(cache_file):
     return data
 
 
-def _find_files(metadata, dirs_only=False):
+def _find_files(metadata):
     '''
     Looks for all the files in the S3 bucket cache metadata
     '''
@@ -476,10 +476,33 @@ def _find_files(metadata, dirs_only=False):
         if bucket_name not in ret:
             ret[bucket_name] = []
 
-        # grab the paths from the metadata
         filePaths = map(lambda k: k['Key'], data)
-        # filter out the files or the dirs depending on flag
-        ret[bucket_name] += filter(lambda k: k.endswith('/') == dirs_only, filePaths)
+        # filter out the dirs
+        ret[bucket_name] += filter(lambda k: not k.endswith('/'), filePaths)
+
+    return ret
+
+
+def _find_dirs(metadata):
+    '''
+    Looks for all the directories in the S3 bucket cache metadata.
+
+    Supports trailing '/' keys (as created by S3 console) as well as
+    directories discovered in the path of file keys.
+    '''
+
+    ret = {}
+
+    for bucket_name, data in metadata.iteritems():
+        if bucket_name not in ret:
+            ret[bucket_name] = set()
+
+        for path in [k['Key'] for k in data]:
+            prefix = ''
+            for part in path.split('/')[:-1]:
+                dir = prefix + part + '/'
+                ret[bucket_name].add(dir)
+                prefix = dir
 
     return ret
 

salt/pillar/s3.py

@@ -289,7 +289,7 @@ def _read_buckets_cache_file(cache_file):
     return data
 
 
-def _find_files(metadata, dirs_only=False):
+def _find_files(metadata):
     '''
     Looks for all the files in the S3 bucket cache metadata
     '''
@@ -302,8 +302,8 @@ def _find_files(metadata, dirs_only=False):
 
         # grab the paths from the metadata
         filePaths = map(lambda k: k['Key'], data)
-        # filter out the files or the dirs depending on flag
-        ret[bucket] += filter(lambda k: k.endswith('/') == dirs_only,
+        # filter out the dirs
+        ret[bucket] += filter(lambda k: not k.endswith('/'),
                               filePaths)
 
     return ret