introduce recurse_list pillar_source_merging_strategy

doc/ref/configuration/master.rst

@@ -2252,6 +2252,34 @@ strategy between different sources. It accepts 4 values:
           element2: True
       baz: quux
 
+* recurse_list:
+
+  it will merge recursively mapping of data similar to ``recurse`` but merge
+  lists by aggregating them instead of replacing them.
+
+  .. code-block:: yaml
+
+      foo: 43
+      bar:
+         - 1
+         - 2
+
+  .. code-block:: yaml
+
+      bar:
+         - 3
+      baz: quux
+
+  will be merged as:
+
+  .. code-block:: yaml
+
+      foo: 42
+      bar:
+         - 1
+         - 2
+         - 3
+      baz: quux
 
 * aggregate:
 

salt/utils/dictupdate.py

@@ -17,7 +17,7 @@ from salt.serializers.yamlex import merge_recursive as _yamlex_merge_recursive
 log = logging.getLogger(__name__)
 
 
-def update(dest, upd, recursive_update=True):
+def update(dest, upd, recursive_update=True, merge_lists=False):
     '''
     Recursive version of the default dict.update
 
@@ -25,6 +25,10 @@ def update(dest, upd, recursive_update=True):
 
     If recursive_update=False, will use the classic dict.update, or fall back
     on a manual merge (helpful for non-dict types like FunctionWrapper)
+
+    If merge_lists=True, will aggregate list object types instead of replace.
+    This behavior is only activated when recursive_update=True. By default
+    merge_lists=False.
     '''
     if (not isinstance(dest, collections.Mapping)) \
             or (not isinstance(upd, collections.Mapping)):
@@ -41,11 +45,14 @@ def update(dest, upd, recursive_update=True):
                 dest_subkey = None
             if isinstance(dest_subkey, collections.Mapping) \
                     and isinstance(val, collections.Mapping):
-                ret = update(dest_subkey, val)
+                ret = update(dest_subkey, val, merge_lists=merge_lists)
                 dest[key] = ret
             elif isinstance(dest_subkey, list) \
                      and isinstance(val, list):
-                dest[key] = dest.get(key, []) + val
+                if merge_lists:
+                    dest[key] = dest.get(key, []) + val
+                else:
+                    dest[key] = upd[key]
             else:
                 dest[key] = upd[key]
         return dest
@@ -69,9 +76,9 @@ def merge_list(obj_a, obj_b):
     return ret
 
 
-def merge_recurse(obj_a, obj_b):
+def merge_recurse(obj_a, obj_b, merge_lists=False):
     copied = copy.deepcopy(obj_a)
-    return update(copied, obj_b)
+    return update(copied, obj_b, merge_lists=merge_lists)
 
 
 def merge_aggregate(obj_a, obj_b):
@@ -96,6 +103,8 @@ def merge(obj_a, obj_b, strategy='smart', renderer='yaml'):
         merged = merge_list(obj_a, obj_b)
     elif strategy == 'recurse':
         merged = merge_recurse(obj_a, obj_b)
+    elif strategy == 'recurse_list':
+        merged = merge_recurse(obj_a, obj_b, merge_lists=True)
     elif strategy == 'aggregate':
         #: level = 1 merge at least root data
         merged = merge_aggregate(obj_a, obj_b)

tests/unit/utils/dictupdate_test.py

@@ -26,12 +26,42 @@ class UtilDictupdateTestCase(TestCase):
         res = dictupdate.update(copy.deepcopy(self.dict1), {'A': 'Z'})
         self.assertEqual(res, mdict)
 
+        # level 1 value changes (list replacement)
+        mdict = copy.deepcopy(self.dict1)
+        mdict['A'] = [1, 2]
+        res = dictupdate.update(copy.deepcopy(mdict), {'A': [2,3]})
+        mdict['A'] = [2, 3]
+        self.assertEqual(res, mdict)
+
+        # level 1 value changes (list merge)
+        mdict = copy.deepcopy(self.dict1)
+        mdict['A'] = [1, 2]
+        res = dictupdate.update(copy.deepcopy(mdict), {'A': [3,4]},
+                                merge_lists=True)
+        mdict['A'] = [1, 2, 3, 4]
+        self.assertEqual(res, mdict)
+
         # level 2 value changes
         mdict = copy.deepcopy(self.dict1)
         mdict['C']['D'] = 'Z'
         res = dictupdate.update(copy.deepcopy(self.dict1), {'C': {'D': 'Z'}})
         self.assertEqual(res, mdict)
 
+        # level 2 value changes (list replacement)
+        mdict = copy.deepcopy(self.dict1)
+        mdict['C']['D'] = ['a', 'b']
+        res = dictupdate.update(copy.deepcopy(mdict), {'C': {'D': ['c', 'd']}})
+        mdict['C']['D'] = ['c', 'd']
+        self.assertEqual(res, mdict)
+
+        # level 2 value changes (list merge)
+        mdict = copy.deepcopy(self.dict1)
+        mdict['C']['D'] = ['a', 'b']
+        res = dictupdate.update(copy.deepcopy(mdict), {'C': {'D': ['c', 'd']}},
+                                merge_lists=True)
+        mdict['C']['D'] = ['a', 'b', 'c', 'd']
+        self.assertEqual(res, mdict)
+
         # level 3 value changes
         mdict = copy.deepcopy(self.dict1)
         mdict['C']['F']['G'] = 'Z'
@@ -41,6 +71,22 @@ class UtilDictupdateTestCase(TestCase):
         )
         self.assertEqual(res, mdict)
 
+        # level 3 value changes (list replacement)
+        mdict = copy.deepcopy(self.dict1)
+        mdict['C']['F']['G'] = ['a', 'b']
+        res = dictupdate.update(copy.deepcopy(mdict),
+            {'C': {'F': {'G': ['c', 'd']}}})
+        mdict['C']['F']['G'] = ['c', 'd']
+        self.assertEqual(res, mdict)
+
+        # level 3 value changes (list merge)
+        mdict = copy.deepcopy(self.dict1)
+        mdict['C']['F']['G'] = ['a', 'b']
+        res = dictupdate.update(copy.deepcopy(mdict),
+            {'C': {'F': {'G': ['c', 'd']}}}, merge_lists=True)
+        mdict['C']['F']['G'] = ['a', 'b', 'c', 'd']
+        self.assertEqual(res, mdict)
+
         # replace a sub-dictionary
         mdict = copy.deepcopy(self.dict1)
         mdict['C'] = 'Z'