Merge pull request #6233 from johnnoone/jinja-import-yaml

Jinja import yaml

doc/ref/renderers/all/salt.renderers.jinja.rst

@@ -102,8 +102,8 @@ the context into the included file is required:
 Variable Serializers
 ====================
 
-Salt allows to serialize any variable into **json** or **yaml**. For example this
-variable::
+Salt allows to serialize any variable into **json** or **yaml**. For example
+this variable::
 
     data:
       foo: True
@@ -126,6 +126,61 @@ will be rendered has::
 
     json -> {"baz": [1, 2, 3], "foo": true, "bar": 42, "qux": 2.0}
 
+
+Strings and variables can be deserialized with **load_yaml** and **load_json**
+filters. It allows to manipulate data directly in templates, easily:
+
+.. code-block:: yaml
+
+    {%- set json_src = '{"foo": "bar", "baz": "qux"}'|load_json %}
+    My json foo is {{ json_src.foo }}
+
+    {%- set yaml_src = "{bar: baz: qux}"|load_yaml %}
+    My yaml bar.baz is {{ yaml_src.bar.baz }}
+
+will be rendered has::
+
+    My json foo is bar
+
+    My yaml bar.baz is qux
+
+
+Template Serializers
+====================
+
+Salt implements **import_yaml** and **import_json** tags. They work like the
+`import tag`_, except that the document is also deserialized.
+
+Imagine you have a generic state file in which you have the complete data of
+your infrastucture:
+
+.. code-block:: yaml
+
+    # everything.sls
+    users:
+      foo:
+        - john
+      bar:
+        - bob
+      baz:
+        - smith
+
+But you don't want to expose everything to a minion. This state file:
+
+.. code-block:: yaml
+
+    # specialized.sls
+    {% load "everything.sls" as all %}
+    my_admins:
+      my_foo: {{ all.users.foo|yaml }}
+
+will be rendered has::
+
+    my_admins:
+      my_foo: [john]
+
+.. _`import tag`: http://jinja.pocoo.org/docs/templates/#import
+
 Template Inheritance
 ====================
 

salt/utils/jinja.py

@@ -5,17 +5,19 @@ Jinja loading utils to enable a more powerful backend for jinja templates
 # Import python libs
 from os import path
 import logging
-from functools import partial
 import json
 
 # Import third party libs
-from jinja2 import BaseLoader, Markup, TemplateNotFound
+from jinja2 import BaseLoader, Markup, TemplateNotFound, nodes
+from jinja2.environment import TemplateModule
 from jinja2.ext import Extension
+from jinja2.exceptions import TemplateRuntimeError
 import yaml
 
 # Import salt libs
 import salt
 import salt.fileclient
+from salt._compat import string_types
 
 log = logging.getLogger(__name__)
 
@@ -100,9 +102,12 @@ class SaltCacheLoader(BaseLoader):
 
 class SerializerExtension(Extension, object):
     '''
-    Serializes variables.
+    Yaml and Json manipulation.
 
-    For example, this dataset:
+    Format filters
+    ~~~~~~~~~~~~~~
+
+    Allows to jsonify or yamlify any datastructure. For example, this dataset:
 
     .. code-block:: python
 
@@ -123,25 +128,119 @@ class SerializerExtension(Extension, object):
         yaml = {bar: 42, baz: [1, 2, 3], foo: true, qux: 2.0}
         json = {"baz": [1, 2, 3], "foo": true, "bar": 42, "qux": 2.0}
 
+    Load filters
+    ~~~~~~~~~~~~
+
+    Parse strings variable with the selected serializer:
+
+    .. code-block:: jinja
+
+        {%- set yaml_src = "{foo: it works}"|load_yaml %}
+        {%- set json_src = "{'bar': 'for real'}"|load_yaml %}
+        Dude, {{ yaml_src.foo }} {{ json_src.bar }}!
+
+    will be rendered has::
+
+        Dude, it works for real!
+
+    Template tags
+    ~~~~~~~~~~~~~
+
+    .. code-block:: jinja
+
+        {% import_yaml "state2.sls" as state2 %}
+        {% import_json "state3.sls" as state3 %}
+
     '''
 
+    tags = set(['import_yaml', 'import_json'])
+
     def __init__(self, environment):
         super(SerializerExtension, self).__init__(environment)
         self.environment.filters.update({
-            'yaml': partial(self.format, formatter='yaml'),
-            'json': partial(self.format, formatter='json')
+            'yaml': self.format_yaml,
+            'json': self.format_json,
+            'load_yaml': self.load_yaml,
+            'load_json': self.load_json
         })
 
-    def format(self, value, formatter, *args, **kwargs):
-        if formatter == 'json':
-            return Markup(json.dumps(value, sort_keys=True))
-        elif formatter == 'yaml':
-            return Markup(yaml.dump(value, default_flow_style=True))
-        raise ValueError('Serializer {0} is not implemented'.format(formatter))
+    def format_json(self, value):
+        return Markup(json.dumps(value, sort_keys=True).strip())
+
+    def format_yaml(self, value):
+        return Markup(yaml.dump(value, default_flow_style=True).strip())
+
+    def load_yaml(self, value):
+        if isinstance(value, TemplateModule):
+            value = str(value)
+        try:
+            return yaml.load(value)
+        except AttributeError:
+            raise TemplateRuntimeError("Unable to load yaml from {0}".format(value))
+
+    def load_json(self, value):
+        if isinstance(value, TemplateModule):
+            value = str(value)
+        try:
+            return json.loads(value)
+        except (ValueError, TypeError):
+            raise TemplateRuntimeError("Unable to load json from {0}".format(value))
 
     def parse(self, parser):
-        '''
-        If called this method would throw ``NotImplementedError``.
-        While we don't need to implement this method, we override it so pylint
-        does not complain about an abstract method not implemented
-        '''
+        if parser.stream.current.value == "import_yaml":
+            return self.parse_yaml(parser)
+        elif parser.stream.current.value == "import_json":
+            return self.parse_json(parser)
+
+        parser.fail('Unknown format ' + parser.stream.current.value,
+                    parser.stream.current.lineno)
+
+    def parse_yaml(self, parser):
+        # import the document
+        node_import = parser.parse_import()
+        target = node_import.target
+
+        # cleanup the remaining nodes
+        while parser.stream.current.type != 'block_end':
+            parser.stream.next()
+
+        node_filter = nodes.Assign(
+                            nodes.Name(target, 'load'),
+                            self.call_method(
+                                'load_yaml',
+                                [nodes.Name(target, 'load')]
+                            )
+                        ).set_lineno(
+                            parser.stream.current.lineno
+                        )
+
+        return [
+            node_import,
+            node_filter
+        ]
+
+
+    def parse_json(self, parser):
+        # import the document
+        node_import = parser.parse_import()
+        target = node_import.target
+
+
+        node_filter = nodes.Assign(
+                            nodes.Name(target, 'load'),
+                            self.call_method(
+                                'load_yaml',
+                                [nodes.Name(target, 'load')]
+                            )
+                        ).set_lineno(
+                            parser.stream.current.lineno
+                        )
+
+        # cleanup the remaining nodes
+        while parser.stream.current.type != 'block_end':
+            parser.stream.next()
+
+        return [
+            node_import,
+            node_filter,
+        ]

tests/unit/templates/jinja_test.py

@@ -18,7 +18,7 @@ from salt.utils.templates import render_jinja_tmpl
 
 # Import 3rd party libs
 import yaml
-from jinja2 import Environment
+from jinja2 import Environment, DictLoader, exceptions
 try:
     import timelib
     HAS_TIMELIB = True
@@ -209,7 +209,18 @@ class TestGetTemplate(TestCase):
 
 
 class TestCustomExtensions(TestCase):
-    def test_serialize(self):
+    def test_serialize_json(self):
+        dataset = {
+            "foo": True,
+            "bar": 42,
+            "baz": [1, 2, 3],
+            "qux": 2.0
+        }
+        env = Environment(extensions=[SerializerExtension])
+        rendered = env.from_string('{{ dataset|json }}').render(dataset=dataset)
+        self.assertEquals(dataset, json.loads(rendered))
+
+    def test_serialize_yaml(self):
         dataset = {
             "foo": True,
             "bar": 42,
@@ -220,8 +231,54 @@ class TestCustomExtensions(TestCase):
         rendered = env.from_string('{{ dataset|yaml }}').render(dataset=dataset)
         self.assertEquals(dataset, yaml.load(rendered))
 
-        rendered = env.from_string('{{ dataset|json }}').render(dataset=dataset)
-        self.assertEquals(dataset, json.loads(rendered))
+    def test_load_yaml(self):
+        env = Environment(extensions=[SerializerExtension])
+        rendered = env.from_string('{% set document = "{foo: it works}"|load_yaml %}{{ document.foo }}').render()
+        self.assertEquals(rendered, u"it works")
+
+        rendered = env.from_string('{% set document = document|load_yaml %}'
+                                   '{{ document.foo }}').render(document="{foo: it works}")
+        self.assertEquals(rendered, u"it works")
+
+        with self.assertRaises(exceptions.TemplateRuntimeError):
+            env.from_string('{% set document = document|load_yaml %}'
+                                       '{{ document.foo }}').render(document={"foo": "it works"})
+
+    def test_load_json(self):
+        env = Environment(extensions=[SerializerExtension])
+        rendered = env.from_string('{% set document = \'{"foo": "it works"}\'|load_json %}'
+                                   '{{ document.foo }}').render()
+        self.assertEquals(rendered, u"it works")
+
+        rendered = env.from_string('{% set document = document|load_json %}'
+                                   '{{ document.foo }}').render(document='{"foo": "it works"}')
+        self.assertEquals(rendered, u"it works")
+
+        # bad quotes
+        with self.assertRaises(exceptions.TemplateRuntimeError):
+            env.from_string("{{ document|load_json }}").render(document="{'foo': 'it works'}")
+
+        # not a string
+        with self.assertRaises(exceptions.TemplateRuntimeError):
+            env.from_string('{{ document|load_json }}').render(document={"foo": "it works"})
+
+    def test_load_yaml_template(self):
+        loader = DictLoader({'foo': '{bar: "my god is blue", foo: [1, 2, 3]}'})
+        env = Environment(extensions=[SerializerExtension], loader=loader)
+        rendered = env.from_string('{% import_yaml "foo" as doc %}{{ doc.bar }}').render()
+        self.assertEquals(rendered, u"my god is blue")
+
+        with self.assertRaises(exceptions.TemplateNotFound):
+            env.from_string('{% import_yaml "does not exists" as doc %}').render()
+
+    def test_load_json_template(self):
+        loader = DictLoader({'foo': '{"bar": "my god is blue", "foo": [1, 2, 3]}'})
+        env = Environment(extensions=[SerializerExtension], loader=loader)
+        rendered = env.from_string('{% import_json "foo" as doc %}{{ doc.bar }}').render()
+        self.assertEquals(rendered, u"my god is blue")
+
+        with self.assertRaises(exceptions.TemplateNotFound):
+            env.from_string('{% import_json "does not exists" as doc %}').render()
 
 
 if __name__ == '__main__':