diff --git a/doc/topics/development/conventions/formulas.rst b/doc/topics/development/conventions/formulas.rst index 1b5e210a05..5eb3eb36b8 100644 --- a/doc/topics/development/conventions/formulas.rst +++ b/doc/topics/development/conventions/formulas.rst @@ -339,23 +339,36 @@ Pillar would replace the ``config`` value from the call above. .. note:: Protecting Expansion of Content with Special Characters - When templating keep in mind that YAML does have special characters - for quoting, flows and other special structure and content. When a - Jinja substitution may have special characters that will be - incorrectly parsed by YAML the expansion must be protected by quoting. - It is a good policy to quote all Jinja expansions especially when - values may originate from Pillar. Salt provides a Jinja filter for - doing just this: ``yaml_dquote`` + When templating keep in mind that YAML does have special characters for + quoting, flows and other special structure and content. When a Jinja + substitution may have special characters that will be incorrectly parsed by + YAML care must be taken. It is a good policy to use the ``yaml_encode`` or + the ``yaml_dquote`` Jinja filters: .. code-block:: jinja - {%- set baz = '"The quick brown fox . . ."' %} + {%- set foo = 7.7 %} + {%- set bar = none %} + {%- set baz = true %} {%- set zap = 'The word of the day is "salty".' %} + {%- set zip = '"The quick brown fox . . ."' %} + + foo: {{ foo|yaml_encode }} + bar: {{ bar|yaml_encode }} + baz: {{ baz|yaml_encode }} + zap: {{ zap|yaml_encode }} + zip: {{ zip|yaml_dquote }} + + The above will be rendered as below: + + .. code-block:: yaml + + foo: 7.7 + bar: null + baz: true + zap: "The word of the day is \"salty\"." + zip: "\"The quick brown fox . . .\"" - {%- load_yaml as foo %} - bar: {{ baz|yaml_dquote }} - zip: {{ zap|yaml_dquote }} - {%- endload %} Single-purpose SLS files ------------------------ diff --git a/salt/renderers/jinja.py b/salt/renderers/jinja.py index bf21cbfaa4..20f401c07c 100644 --- a/salt/renderers/jinja.py +++ b/salt/renderers/jinja.py @@ -143,6 +143,31 @@ strftime sequence Ensure that parsed data is a sequence. +yaml_encode + Serializes a single object into a YAML scalar with any necessary + handling for escaping special characters. This will work for any + scalar YAML data type: ints, floats, timestamps, booleans, strings, + unicode. It will *not* work for multi-objects such as sequences or + maps. + + .. code-block:: yaml + + {%- set bar = 7 %} + {%- set baz = none %} + {%- set zip = true %} + {%- set zap = 'The word of the day is "salty"' %} + + {%- load_yaml as foo %} + bar: {{ bar|yaml_encode }} + baz: {{ baz|yaml_encode }} + baz: {{ zip|yaml_encode }} + baz: {{ zap|yaml_encode }} + {%- endload %} + + In the above case ``{{ bar }}`` and ``{{ foo.bar }}`` should be + identical and ``{{ baz }}`` and ``{{ foo.baz }}`` should be + identical. + yaml_dquote Serializes a string into a properly-escaped YAML double-quoted string. This is useful when the contents of a string are unknown @@ -152,19 +177,25 @@ yaml_dquote .. code-block:: yaml - {%- set baz = '"The quick brown fox . . ."' %} - {%- set zap = 'The word of the day is "salty".' %} + {%- set bar = '"The quick brown fox . . ."' %} + {%- set baz = 'The word of the day is "salty".' %} {%- load_yaml as foo %} - bar: {{ baz|yaml_dquote }} - zip: {{ zap|yaml_dquote }} + bar: {{ bar|yaml_dquote }} + baz: {{ baz|yaml_dquote }} {%- endload %} + In the above case ``{{ bar }}`` and ``{{ foo.bar }}`` should be + identical and ``{{ baz }}`` and ``{{ foo.baz }}`` should be + identical. If variable contents are not guaranteed to be a string + then it is better to use ``yaml_encode`` which handles all YAML + scalar types. + yaml_squote Similar to the ``yaml_dquote`` filter but with single quotes. Note that YAML only allows special escapes inside double quotes so ``yaml_squote`` is not nearly as useful (viz. you likely want to - use ``yaml_dquote``). + use ``yaml_encode`` or ``yaml_dquote``). .. _`builtin filters`: http://jinja.pocoo.org/docs/templates/#builtin-filters .. _`timelib`: https://github.com/pediapress/timelib/ diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 05d6cda74d..362ed94b6c 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1813,6 +1813,27 @@ def yaml_squote(text): return ostream.getvalue() +def yaml_encode(data): + """A simple YAML encode that can take a single-element datatype and return + a string representation. + """ + yrepr = yaml.representer.SafeRepresenter() + ynode = yrepr.represent_data(data) + if not isinstance(ynode, yaml.ScalarNode): + raise TypeError( + "yaml_encode() only works with YAML scalar data;" + " failed for {0}".format(type(data)) + ) + + tag = ynode.tag.rsplit(':', 1)[-1] + ret = ynode.value + + if tag == "str": + ret = yaml_dquote(ynode.value) + + return ret + + def warn_until(version, message, category=DeprecationWarning, diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 4c14951323..6571037c02 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -364,8 +364,8 @@ class SerializerExtension(Extension, object): return data return explore(data) - def format_json(self, value): - return Markup(json.dumps(value, sort_keys=True).strip()) + def format_json(self, value, sort_keys=True): + return Markup(json.dumps(value, sort_keys=sort_keys).strip()) def format_yaml(self, value, flow_style=True): return Markup(yaml.dump(value, default_flow_style=flow_style, diff --git a/salt/utils/templates.py b/salt/utils/templates.py index fc9374a05c..06dd5e3761 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -265,6 +265,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None): jinja_env.filters['sequence'] = ensure_sequence_filter jinja_env.filters['yaml_dquote'] = salt.utils.yaml_dquote jinja_env.filters['yaml_squote'] = salt.utils.yaml_squote + jinja_env.filters['yaml_encode'] = salt.utils.yaml_encode jinja_env.globals['odict'] = OrderedDict jinja_env.globals['show_full_context'] = show_full_context diff --git a/tests/unit/utils/utils_test.py b/tests/unit/utils/utils_test.py index 5a0368e847..1d0ec25164 100644 --- a/tests/unit/utils/utils_test.py +++ b/tests/unit/utils/utils_test.py @@ -23,6 +23,7 @@ from salt.exceptions import (SaltInvocationError, SaltSystemExit, CommandNotFoun # Import Python libraries import os import datetime +import yaml import zmq from collections import namedtuple @@ -521,13 +522,20 @@ class UtilsTestCase(TestCase): self.assertEqual(ret, expected_ret) def test_yaml_dquote(self): - ret = utils.yaml_dquote(r'"\ "') - self.assertEqual(ret, r'"\"\\ \""') + for teststr in (r'"\ []{}"',): + self.assertEqual(teststr, yaml.safe_load(utils.yaml_dquote(teststr))) def test_yaml_squote(self): ret = utils.yaml_squote(r'"') self.assertEqual(ret, r"""'"'""") + def test_yaml_encode(self): + for testobj in (None, True, False, '[7, 5]', '"monkey"', 5, 7.5, "2014-06-02 15:30:29.7"): + self.assertEqual(testobj, yaml.safe_load(utils.yaml_encode(testobj))) + + for testobj in ({}, [], set()): + self.assertRaises(TypeError, utils.yaml_encode, testobj) + def test_compare_dicts(self): ret = utils.compare_dicts(old={'foo': 'bar'}, new={'foo': 'bar'}) self.assertEqual(ret, {})