diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 2fc845b5df..3980f17568 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -2252,6 +2252,34 @@ strategy between different sources. It accepts 4 values: element2: True baz: quux +* recurse_list: + + it will merge recursively mapping of data similar to ``recurse`` but merge + lists by aggregating them instead of replacing them. + + .. code-block:: yaml + + foo: 43 + bar: + - 1 + - 2 + + .. code-block:: yaml + + bar: + - 3 + baz: quux + + will be merged as: + + .. code-block:: yaml + + foo: 42 + bar: + - 1 + - 2 + - 3 + baz: quux * aggregate: diff --git a/salt/utils/dictupdate.py b/salt/utils/dictupdate.py index d7feed1495..033b22766c 100644 --- a/salt/utils/dictupdate.py +++ b/salt/utils/dictupdate.py @@ -17,7 +17,7 @@ from salt.serializers.yamlex import merge_recursive as _yamlex_merge_recursive log = logging.getLogger(__name__) -def update(dest, upd, recursive_update=True): +def update(dest, upd, recursive_update=True, merge_lists=False): ''' Recursive version of the default dict.update @@ -25,6 +25,10 @@ def update(dest, upd, recursive_update=True): If recursive_update=False, will use the classic dict.update, or fall back on a manual merge (helpful for non-dict types like FunctionWrapper) + + If merge_lists=True, will aggregate list object types instead of replace. + This behavior is only activated when recursive_update=True. By default + merge_lists=False. ''' if (not isinstance(dest, collections.Mapping)) \ or (not isinstance(upd, collections.Mapping)): @@ -41,11 +45,14 @@ def update(dest, upd, recursive_update=True): dest_subkey = None if isinstance(dest_subkey, collections.Mapping) \ and isinstance(val, collections.Mapping): - ret = update(dest_subkey, val) + ret = update(dest_subkey, val, merge_lists=merge_lists) dest[key] = ret elif isinstance(dest_subkey, list) \ and isinstance(val, list): - dest[key] = dest.get(key, []) + val + if merge_lists: + dest[key] = dest.get(key, []) + val + else: + dest[key] = upd[key] else: dest[key] = upd[key] return dest @@ -69,9 +76,9 @@ def merge_list(obj_a, obj_b): return ret -def merge_recurse(obj_a, obj_b): +def merge_recurse(obj_a, obj_b, merge_lists=False): copied = copy.deepcopy(obj_a) - return update(copied, obj_b) + return update(copied, obj_b, merge_lists=merge_lists) def merge_aggregate(obj_a, obj_b): @@ -96,6 +103,8 @@ def merge(obj_a, obj_b, strategy='smart', renderer='yaml'): merged = merge_list(obj_a, obj_b) elif strategy == 'recurse': merged = merge_recurse(obj_a, obj_b) + elif strategy == 'recurse_list': + merged = merge_recurse(obj_a, obj_b, merge_lists=True) elif strategy == 'aggregate': #: level = 1 merge at least root data merged = merge_aggregate(obj_a, obj_b) diff --git a/tests/unit/utils/dictupdate_test.py b/tests/unit/utils/dictupdate_test.py index d8f77af569..55668f5779 100644 --- a/tests/unit/utils/dictupdate_test.py +++ b/tests/unit/utils/dictupdate_test.py @@ -26,12 +26,42 @@ class UtilDictupdateTestCase(TestCase): res = dictupdate.update(copy.deepcopy(self.dict1), {'A': 'Z'}) self.assertEqual(res, mdict) + # level 1 value changes (list replacement) + mdict = copy.deepcopy(self.dict1) + mdict['A'] = [1, 2] + res = dictupdate.update(copy.deepcopy(mdict), {'A': [2,3]}) + mdict['A'] = [2, 3] + self.assertEqual(res, mdict) + + # level 1 value changes (list merge) + mdict = copy.deepcopy(self.dict1) + mdict['A'] = [1, 2] + res = dictupdate.update(copy.deepcopy(mdict), {'A': [3,4]}, + merge_lists=True) + mdict['A'] = [1, 2, 3, 4] + self.assertEqual(res, mdict) + # level 2 value changes mdict = copy.deepcopy(self.dict1) mdict['C']['D'] = 'Z' res = dictupdate.update(copy.deepcopy(self.dict1), {'C': {'D': 'Z'}}) self.assertEqual(res, mdict) + # level 2 value changes (list replacement) + mdict = copy.deepcopy(self.dict1) + mdict['C']['D'] = ['a', 'b'] + res = dictupdate.update(copy.deepcopy(mdict), {'C': {'D': ['c', 'd']}}) + mdict['C']['D'] = ['c', 'd'] + self.assertEqual(res, mdict) + + # level 2 value changes (list merge) + mdict = copy.deepcopy(self.dict1) + mdict['C']['D'] = ['a', 'b'] + res = dictupdate.update(copy.deepcopy(mdict), {'C': {'D': ['c', 'd']}}, + merge_lists=True) + mdict['C']['D'] = ['a', 'b', 'c', 'd'] + self.assertEqual(res, mdict) + # level 3 value changes mdict = copy.deepcopy(self.dict1) mdict['C']['F']['G'] = 'Z' @@ -41,6 +71,22 @@ class UtilDictupdateTestCase(TestCase): ) self.assertEqual(res, mdict) + # level 3 value changes (list replacement) + mdict = copy.deepcopy(self.dict1) + mdict['C']['F']['G'] = ['a', 'b'] + res = dictupdate.update(copy.deepcopy(mdict), + {'C': {'F': {'G': ['c', 'd']}}}) + mdict['C']['F']['G'] = ['c', 'd'] + self.assertEqual(res, mdict) + + # level 3 value changes (list merge) + mdict = copy.deepcopy(self.dict1) + mdict['C']['F']['G'] = ['a', 'b'] + res = dictupdate.update(copy.deepcopy(mdict), + {'C': {'F': {'G': ['c', 'd']}}}, merge_lists=True) + mdict['C']['F']['G'] = ['a', 'b', 'c', 'd'] + self.assertEqual(res, mdict) + # replace a sub-dictionary mdict = copy.deepcopy(self.dict1) mdict['C'] = 'Z'