diff --git a/doc/ref/renderers/all/salt.renderers.jinja.rst b/doc/ref/renderers/all/salt.renderers.jinja.rst index 673d6c4792..7bec28564e 100644 --- a/doc/ref/renderers/all/salt.renderers.jinja.rst +++ b/doc/ref/renderers/all/salt.renderers.jinja.rst @@ -102,8 +102,8 @@ the context into the included file is required: Variable Serializers ==================== -Salt allows to serialize any variable into **json** or **yaml**. For example this -variable:: +Salt allows to serialize any variable into **json** or **yaml**. For example +this variable:: data: foo: True @@ -126,6 +126,61 @@ will be rendered has:: json -> {"baz": [1, 2, 3], "foo": true, "bar": 42, "qux": 2.0} + +Strings and variables can be deserialized with **load_yaml** and **load_json** +filters. It allows to manipulate data directly in templates, easily: + +.. code-block:: yaml + + {%- set json_src = '{"foo": "bar", "baz": "qux"}'|load_json %} + My json foo is {{ json_src.foo }} + + {%- set yaml_src = "{bar: baz: qux}"|load_yaml %} + My yaml bar.baz is {{ yaml_src.bar.baz }} + +will be rendered has:: + + My json foo is bar + + My yaml bar.baz is qux + + +Template Serializers +==================== + +Salt implements **import_yaml** and **import_json** tags. They work like the +`import tag`_, except that the document is also deserialized. + +Imagine you have a generic state file in which you have the complete data of +your infrastucture: + +.. code-block:: yaml + + # everything.sls + users: + foo: + - john + bar: + - bob + baz: + - smith + +But you don't want to expose everything to a minion. This state file: + +.. code-block:: yaml + + # specialized.sls + {% load "everything.sls" as all %} + my_admins: + my_foo: {{ all.users.foo|yaml }} + +will be rendered has:: + + my_admins: + my_foo: [john] + +.. _`import tag`: http://jinja.pocoo.org/docs/templates/#import + Template Inheritance ==================== diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 674c4553b9..efc459bb09 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -5,17 +5,19 @@ Jinja loading utils to enable a more powerful backend for jinja templates # Import python libs from os import path import logging -from functools import partial import json # Import third party libs -from jinja2 import BaseLoader, Markup, TemplateNotFound +from jinja2 import BaseLoader, Markup, TemplateNotFound, nodes +from jinja2.environment import TemplateModule from jinja2.ext import Extension +from jinja2.exceptions import TemplateRuntimeError import yaml # Import salt libs import salt import salt.fileclient +from salt._compat import string_types log = logging.getLogger(__name__) @@ -100,9 +102,12 @@ class SaltCacheLoader(BaseLoader): class SerializerExtension(Extension, object): ''' - Serializes variables. + Yaml and Json manipulation. - For example, this dataset: + Format filters + ~~~~~~~~~~~~~~ + + Allows to jsonify or yamlify any datastructure. For example, this dataset: .. code-block:: python @@ -123,25 +128,119 @@ class SerializerExtension(Extension, object): yaml = {bar: 42, baz: [1, 2, 3], foo: true, qux: 2.0} json = {"baz": [1, 2, 3], "foo": true, "bar": 42, "qux": 2.0} + Load filters + ~~~~~~~~~~~~ + + Parse strings variable with the selected serializer: + + .. code-block:: jinja + + {%- set yaml_src = "{foo: it works}"|load_yaml %} + {%- set json_src = "{'bar': 'for real'}"|load_yaml %} + Dude, {{ yaml_src.foo }} {{ json_src.bar }}! + + will be rendered has:: + + Dude, it works for real! + + Template tags + ~~~~~~~~~~~~~ + + .. code-block:: jinja + + {% import_yaml "state2.sls" as state2 %} + {% import_json "state3.sls" as state3 %} + ''' + tags = set(['import_yaml', 'import_json']) + def __init__(self, environment): super(SerializerExtension, self).__init__(environment) self.environment.filters.update({ - 'yaml': partial(self.format, formatter='yaml'), - 'json': partial(self.format, formatter='json') + 'yaml': self.format_yaml, + 'json': self.format_json, + 'load_yaml': self.load_yaml, + 'load_json': self.load_json }) - def format(self, value, formatter, *args, **kwargs): - if formatter == 'json': - return Markup(json.dumps(value, sort_keys=True)) - elif formatter == 'yaml': - return Markup(yaml.dump(value, default_flow_style=True)) - raise ValueError('Serializer {0} is not implemented'.format(formatter)) + def format_json(self, value): + return Markup(json.dumps(value, sort_keys=True).strip()) + + def format_yaml(self, value): + return Markup(yaml.dump(value, default_flow_style=True).strip()) + + def load_yaml(self, value): + if isinstance(value, TemplateModule): + value = str(value) + try: + return yaml.load(value) + except AttributeError: + raise TemplateRuntimeError("Unable to load yaml from {0}".format(value)) + + def load_json(self, value): + if isinstance(value, TemplateModule): + value = str(value) + try: + return json.loads(value) + except (ValueError, TypeError): + raise TemplateRuntimeError("Unable to load json from {0}".format(value)) def parse(self, parser): - ''' - If called this method would throw ``NotImplementedError``. - While we don't need to implement this method, we override it so pylint - does not complain about an abstract method not implemented - ''' + if parser.stream.current.value == "import_yaml": + return self.parse_yaml(parser) + elif parser.stream.current.value == "import_json": + return self.parse_json(parser) + + parser.fail('Unknown format ' + parser.stream.current.value, + parser.stream.current.lineno) + + def parse_yaml(self, parser): + # import the document + node_import = parser.parse_import() + target = node_import.target + + # cleanup the remaining nodes + while parser.stream.current.type != 'block_end': + parser.stream.next() + + node_filter = nodes.Assign( + nodes.Name(target, 'load'), + self.call_method( + 'load_yaml', + [nodes.Name(target, 'load')] + ) + ).set_lineno( + parser.stream.current.lineno + ) + + return [ + node_import, + node_filter + ] + + + def parse_json(self, parser): + # import the document + node_import = parser.parse_import() + target = node_import.target + + + node_filter = nodes.Assign( + nodes.Name(target, 'load'), + self.call_method( + 'load_yaml', + [nodes.Name(target, 'load')] + ) + ).set_lineno( + parser.stream.current.lineno + ) + + # cleanup the remaining nodes + while parser.stream.current.type != 'block_end': + parser.stream.next() + + return [ + node_import, + node_filter, + ] diff --git a/tests/unit/templates/jinja_test.py b/tests/unit/templates/jinja_test.py index 602342f607..b347473932 100644 --- a/tests/unit/templates/jinja_test.py +++ b/tests/unit/templates/jinja_test.py @@ -18,7 +18,7 @@ from salt.utils.templates import render_jinja_tmpl # Import 3rd party libs import yaml -from jinja2 import Environment +from jinja2 import Environment, DictLoader, exceptions try: import timelib HAS_TIMELIB = True @@ -209,7 +209,18 @@ class TestGetTemplate(TestCase): class TestCustomExtensions(TestCase): - def test_serialize(self): + def test_serialize_json(self): + dataset = { + "foo": True, + "bar": 42, + "baz": [1, 2, 3], + "qux": 2.0 + } + env = Environment(extensions=[SerializerExtension]) + rendered = env.from_string('{{ dataset|json }}').render(dataset=dataset) + self.assertEquals(dataset, json.loads(rendered)) + + def test_serialize_yaml(self): dataset = { "foo": True, "bar": 42, @@ -220,8 +231,54 @@ class TestCustomExtensions(TestCase): rendered = env.from_string('{{ dataset|yaml }}').render(dataset=dataset) self.assertEquals(dataset, yaml.load(rendered)) - rendered = env.from_string('{{ dataset|json }}').render(dataset=dataset) - self.assertEquals(dataset, json.loads(rendered)) + def test_load_yaml(self): + env = Environment(extensions=[SerializerExtension]) + rendered = env.from_string('{% set document = "{foo: it works}"|load_yaml %}{{ document.foo }}').render() + self.assertEquals(rendered, u"it works") + + rendered = env.from_string('{% set document = document|load_yaml %}' + '{{ document.foo }}').render(document="{foo: it works}") + self.assertEquals(rendered, u"it works") + + with self.assertRaises(exceptions.TemplateRuntimeError): + env.from_string('{% set document = document|load_yaml %}' + '{{ document.foo }}').render(document={"foo": "it works"}) + + def test_load_json(self): + env = Environment(extensions=[SerializerExtension]) + rendered = env.from_string('{% set document = \'{"foo": "it works"}\'|load_json %}' + '{{ document.foo }}').render() + self.assertEquals(rendered, u"it works") + + rendered = env.from_string('{% set document = document|load_json %}' + '{{ document.foo }}').render(document='{"foo": "it works"}') + self.assertEquals(rendered, u"it works") + + # bad quotes + with self.assertRaises(exceptions.TemplateRuntimeError): + env.from_string("{{ document|load_json }}").render(document="{'foo': 'it works'}") + + # not a string + with self.assertRaises(exceptions.TemplateRuntimeError): + env.from_string('{{ document|load_json }}').render(document={"foo": "it works"}) + + def test_load_yaml_template(self): + loader = DictLoader({'foo': '{bar: "my god is blue", foo: [1, 2, 3]}'}) + env = Environment(extensions=[SerializerExtension], loader=loader) + rendered = env.from_string('{% import_yaml "foo" as doc %}{{ doc.bar }}').render() + self.assertEquals(rendered, u"my god is blue") + + with self.assertRaises(exceptions.TemplateNotFound): + env.from_string('{% import_yaml "does not exists" as doc %}').render() + + def test_load_json_template(self): + loader = DictLoader({'foo': '{"bar": "my god is blue", "foo": [1, 2, 3]}'}) + env = Environment(extensions=[SerializerExtension], loader=loader) + rendered = env.from_string('{% import_json "foo" as doc %}{{ doc.bar }}').render() + self.assertEquals(rendered, u"my god is blue") + + with self.assertRaises(exceptions.TemplateNotFound): + env.from_string('{% import_json "does not exists" as doc %}').render() if __name__ == '__main__':