Merge pull request #15388 from Setsuna666/feature-ext_pillar-file_tree-nodegroups

Feature - Add support for node groups in the file tree external pillar module

salt/pillar/file_tree.py

@@ -52,13 +52,54 @@ will result in the following pillar tree for minion with ID "test-host"::
 
                 file2.txt:
                     Contents of file #2.
+
+To fill pillar data for minion in a node group, file_tree recursively
+iterates over ``root_dir``/nodegroups/``nodegroup`` (where ``nodegroup`` is a
+minion node group), and constructs the same directory tree with contents of all
+the files inside the pillar tree.
+**IMPORTANT**: The host data take precedence over the node group data
+
+For example, the following ``root_dir`` tree::
+
+    ./nodegroups/
+    ./nodegroups/test-group/
+    ./nodegroups/test-group/files/
+    ./nodegroups/test-group/files/testdir/
+    ./nodegroups/test-group/files/testdir/file1.txt
+    ./nodegroups/test-group/files/testdir/file2.txt
+    ./nodegroups/test-group/files/another-testdir/
+    ./nodegroups/test-group/files/another-testdir/symlink-to-file1.txt
+
+will result in the following pillar tree for minion in the node group
+"test-group"::
+
+    test-host:
+        ----------
+        files:
+            ----------
+            another-testdir:
+                ----------
+                symlink-to-file1.txt:
+                    Contents of file #1.
+
+            testdir:
+                ----------
+                file1.txt:
+                    Contents of file #1.
+
+                file2.txt:
+                    Contents of file #2.
 '''
-# TODO: Add git support.
 
 # Import python libs
 import logging
 import os
 import os.path
+from copy import deepcopy
+
+# Import salt libs
+import salt.utils
+import salt.utils.minions
 
 # Set up logging
 log = logging.getLogger(__name__)
@@ -71,6 +112,25 @@ def _on_walk_error(err):
     log.error('"%s": %s', err.filename, err.strerror)
 
 
+# Thanks to Ross McFarland for the dict_merge function
+# (Source: https://www.xormedia.com/recursively-merge-dictionaries-in-python/)
+def _dict_merge(dict_a, dict_b):
+    '''
+    recursively merges dict's. not just simple dict_a['key'] = dict_b['key'], if
+    both dict_a and dict_b have a key who's value is a dict then _dict_merge
+    is called on both values and the result stored in the returned dictionary.
+    '''
+    if not isinstance(dict_b, dict):
+        return dict_b
+    result = deepcopy(dict_a)
+    for key, value in dict_b.iteritems():
+        if key in result and isinstance(result[key], dict):
+            result[key] = _dict_merge(result[key], value)
+        else:
+            result[key] = deepcopy(value)
+    return result
+
+
 def _construct_pillar(top_dir, follow_dir_links):
     '''
     Construct pillar from file tree.
@@ -108,14 +168,13 @@ def _construct_pillar(top_dir, follow_dir_links):
                 pillar_node[file_name] = open(file_path, 'rb').read()
             except IOError as err:
                 log.error('%s', str(err))
-            except:
-                log.error('Unknown exception while reading "%s"', file_path,
-                          exc_info=True)
 
     return pillar
 
 
-def ext_pillar(minion_id, pillar, root_dir=None, follow_dir_links=False):
+def ext_pillar(
+        minion_id, pillar, root_dir=None,
+        follow_dir_links=False, debug=False):
     '''
     Find pillar data for specified ID.
     '''
@@ -129,12 +188,41 @@ def ext_pillar(minion_id, pillar, root_dir=None, follow_dir_links=False):
         log.error('"%s" does not exist or not a directory', root_dir)
         return {}
 
+    ngroup_pillar = {}
+    nodegroups_dir = os.path.join(root_dir, 'nodegroups')
+    if os.path.exists(nodegroups_dir) and len(__opts__['nodegroups']) > 0:
+        master_ngroups = __opts__['nodegroups']
+        ext_pillar_dirs = os.listdir(nodegroups_dir)
+        if len(ext_pillar_dirs) > 0:
+            for nodegroup in ext_pillar_dirs:
+                if (os.path.isdir(nodegroups_dir) and
+                        nodegroup in master_ngroups):
+                    ckminions = salt.utils.minions.CkMinions(__opts__)
+                    match = ckminions.check_minions(
+                        master_ngroups[nodegroup],
+                        'compound')
+                    if minion_id in match:
+                        ngroup_dir = os.path.join(
+                            nodegroups_dir, str(nodegroup))
+                        ngroup_pillar.update(
+                            _construct_pillar(ngroup_dir, follow_dir_links))
+        else:
+            if debug is True:
+                log.debug('File tree - No nodegroups found in file tree \
+                            directory ext_pillar_dirs, skipping...')
+    else:
+        if debug is True:
+            log.debug('File tree - No nodegroups found in master configuration,\
+                        skipping nodegroups pillar function...')
+
     host_dir = os.path.join(root_dir, 'hosts', minion_id)
     if not os.path.exists(host_dir):
         # No data for host with this ID.
-        return {}
+        return ngroup_pillar
+
     if not os.path.isdir(host_dir):
         log.error('"%s" exists, but not a directory', host_dir)
-        return {}
+        return ngroup_pillar
 
-    return _construct_pillar(host_dir, follow_dir_links)
+    host_pillar = _construct_pillar(host_dir, follow_dir_links)
+    return _dict_merge(ngroup_pillar, host_pillar)