From 9b55ac723095016b29f0783b9ba006ee911a372d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 5 Nov 2018 08:53:50 -0800 Subject: [PATCH 1/3] Adding _trim_dict_in_dict to utils/dictrim.py to be called from trim_dict when data contains a nested dictionary. This will ensure that values will still be trimmed correctly. --- salt/utils/dicttrim.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/salt/utils/dicttrim.py b/salt/utils/dicttrim.py index 58ddc83f0e..01e8bf13d9 100644 --- a/salt/utils/dicttrim.py +++ b/salt/utils/dicttrim.py @@ -6,6 +6,22 @@ import sys import salt.payload +def _trim_dict_in_dict(data, max_val_size, replace_with): + ''' + Takes a dictionary, max_val_size and replace_with + and recursively loops through and replaces any values + that are greater than max_val_size. + ''' + for key in data: + if isinstance(data[key], dict): + _trim_dict_in_dict(data[key], + max_val_size, + replace_with) + else: + if sys.getsizeof(data[key]) > max_val_size: + data[key] = replace_with + + def trim_dict( data, max_dict_bytes, @@ -63,8 +79,13 @@ def trim_dict( max_val_size = float(max_dict_bytes * (percent / 100)) try: for key in data: - if sys.getsizeof(data[key]) > max_val_size: - data[key] = replace_with + if isinstance(data[key], dict): + _trim_dict_in_dict(data[key], + max_val_size, + replace_with) + else: + if sys.getsizeof(data[key]) > max_val_size: + data[key] = replace_with percent = percent - stepper_size max_val_size = float(max_dict_bytes * (percent / 100)) if use_bin_type: From 7f77bfef99d009724457aa296bb0a140acc4cb94 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 5 Nov 2018 10:58:36 -0800 Subject: [PATCH 2/3] Adding test for utils/dicttrim.py --- tests/unit/utils/test_dicttrim.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/unit/utils/test_dicttrim.py diff --git a/tests/unit/utils/test_dicttrim.py b/tests/unit/utils/test_dicttrim.py new file mode 100644 index 0000000000..0ed34384f4 --- /dev/null +++ b/tests/unit/utils/test_dicttrim.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# Import python libs +from __future__ import absolute_import, print_function, unicode_literals + +# Import Salt Testing libs +from tests.support.unit import TestCase + +# Import Salt libs +import salt.utils.dicttrim as dicttrimmer + +import logging +log = logging.getLogger(__name__) + + +class DictTrimTestCase(TestCase): + + def setUp(self): + self.old_dict = {'a': 'b', 'c': 'x' * 10000} + self.new_dict = {'a': 'b', 'c': 'VALUE_TRIMMED'} + + def test_trim_dict(self): + ret = dicttrimmer.trim_dict(self.old_dict, 100) + self.assertEqual(ret, self.new_dict) + + +class RecursiveDictTrimTestCase(TestCase): + + def setUp(self): + self.old_dict = {'a': {'b': 1, 'c': 2, 'e': 'x' * 10000, 'f': '3'}} + self.new_dict = {'a': {'b': 1, 'c': 2, 'e': 'VALUE_TRIMMED', 'f': '3'}} + + def test_trim_dict(self): + ret = dicttrimmer.trim_dict(self.old_dict, 100) + self.assertEqual(ret, self.new_dict) From 039f3d1622c0f41313f5ed53179d6e0874afe0f0 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 5 Nov 2018 11:22:20 -0800 Subject: [PATCH 3/3] Increase limit for trimming. --- tests/unit/utils/test_dicttrim.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils/test_dicttrim.py b/tests/unit/utils/test_dicttrim.py index 0ed34384f4..3c21d0da4b 100644 --- a/tests/unit/utils/test_dicttrim.py +++ b/tests/unit/utils/test_dicttrim.py @@ -20,7 +20,7 @@ class DictTrimTestCase(TestCase): self.new_dict = {'a': 'b', 'c': 'VALUE_TRIMMED'} def test_trim_dict(self): - ret = dicttrimmer.trim_dict(self.old_dict, 100) + ret = dicttrimmer.trim_dict(self.old_dict, 1000) self.assertEqual(ret, self.new_dict) @@ -31,5 +31,5 @@ class RecursiveDictTrimTestCase(TestCase): self.new_dict = {'a': {'b': 1, 'c': 2, 'e': 'VALUE_TRIMMED', 'f': '3'}} def test_trim_dict(self): - ret = dicttrimmer.trim_dict(self.old_dict, 100) + ret = dicttrimmer.trim_dict(self.old_dict, 1000) self.assertEqual(ret, self.new_dict)