add custom jinja loader that caches files with salt FileClient.

this removes the recently added cachefile template filter.
salt states and file state templates using jinja now automatically
support includes when they are loaded from the salt state tree.

salt/renderers/json_jinja.py

@@ -7,11 +7,7 @@ high data format for salt states.
 
 import json
 import os
-
-# Import Third Party libs
-from jinja2 import Template, FileSystemLoader
-from jinja2.environment import Environment
-
+from salt.utils.jinja import get_template
 
 def render(template_file, env='', sls=''):
     '''
@@ -26,18 +22,7 @@ def render(template_file, env='', sls=''):
     passthrough['env'] = env
     passthrough['sls'] = sls
 
-    file_cache = '/files/%s/' % env
-    if file_cache in template_file:
-        def cachefile_filter(value):
-            __salt__['cp.cache_file']('salt://%s' % value)
-            return value
-        cache_dir, file_rel = template_file.split(file_cache, 1)
-        loader = FileSystemLoader(cache_dir + file_cache)
-        jinja_env = Environment(loader=loader)
-        jinja_env.filters['cachefile'] = cachefile_filter
-        template = jinja_env.get_template(file_rel)
-    else:
-        template = Template(open(template_file, 'r').read())
+    template = get_template(template_file, __opts__, env)
 
     json_data = template.render(**passthrough)
 

salt/renderers/yaml_jinja.py

@@ -6,12 +6,8 @@ high data format for salt states.
 '''
 
 import os
-
-# Import Third Party libs
-from jinja2 import Template, FileSystemLoader
 import yaml
-from jinja2.environment import Environment
-
+from salt.utils.jinja import get_template
 
 def render(template_file, env='', sls=''):
     '''
@@ -26,18 +22,7 @@ def render(template_file, env='', sls=''):
     passthrough['env'] = env
     passthrough['sls'] = sls
 
-    file_cache = '/files/%s/' % env
-    if file_cache in template_file:
-        def cachefile_filter(value):
-            __salt__['cp.cache_file']('salt://%s' % value)
-            return value
-        cache_dir, file_rel = template_file.split(file_cache, 1)
-        loader = FileSystemLoader(cache_dir + file_cache)
-        jinja_env = Environment(loader=loader)
-        jinja_env.filters['cachefile'] = cachefile_filter
-        template = jinja_env.get_template(file_rel)
-    else:
-        template = Template(open(template_file, 'r').read())
+    template = get_template(template_file, __opts__, env)
 
     yaml_data = template.render(**passthrough)
 

salt/states/file.py

@@ -177,7 +177,7 @@ def _jinja(sfn, name, source, user, group, mode, env):
          'data': <Error data or rendered file path>}
     '''
     try:
-        from jinja2 import Template
+        from salt.utils.jinja import get_template
     except ImportError:
         return {'result': False,
                 'data': 'Failed to import jinja'}
@@ -192,7 +192,7 @@ def _jinja(sfn, name, source, user, group, mode, env):
         passthrough['group'] = group
         passthrough['mode'] = mode
         passthrough['env'] = env
-        template = Template(open(sfn, 'r').read())
+        template = get_template(sfn, __opts__, env)
         open(tgt, 'w+').write(template.render(**passthrough))
         return {'result': True,
                     'data': tgt}

salt/utils/jinja.py

@@ -0,0 +1,76 @@
+from os import path
+from jinja2 import Template, BaseLoader, Environment
+from jinja2.loaders import split_template_path
+from jinja2.exceptions import TemplateNotFound
+import salt
+
+def get_template(filename, opts, env):
+    loader = SaltCacheLoader(opts, env)
+    if filename.startswith(loader.searchpath):
+        jinja = Environment(loader=loader)
+        relpath = path.relpath(filename, loader.searchpath)
+        # the template was already fetched
+        loader.cached.append(relpath)
+        return jinja.get_template(relpath)
+    else:
+        # fallback for templates outside the state tree
+        return Template(open(filename, 'r').read())
+
+class SaltCacheLoader(BaseLoader):
+    '''
+    A special jinja Template Loader for salt.
+    Requested templates are always fetched from the server
+    to guarantee that the file is up to date.
+    Templates are cached like regular salt states
+    and only loaded once per loader instance.
+    '''
+    def __init__(self, opts, env='base', encoding='utf-8'):
+        self.opts = opts
+        self.env = env
+        self.encoding = encoding
+        self.searchpath = path.join(opts['cachedir'], 'files', env)
+        self._file_client = None
+        self.cached = []
+        
+    def file_client(self):
+        '''
+        Return a file client. Instantiates on first call.
+        '''
+        if not self._file_client:
+            self._file_client = salt.minion.FileClient(self.opts)
+        return self._file_client
+
+    def cache_file(self, template):
+        '''
+        Cache a file from the salt master
+        '''
+        saltpath = path.join('salt://', template)
+        self.file_client().get_file(saltpath, '', True, self.env)
+
+    def check_cache(self, template):
+        '''
+        Cache a file only once
+        '''
+        if template not in self.cached:
+            self.cache_file(template)
+            self.cached.append(template)
+
+    def get_source(self, environment, template):
+        # checks for relative '..' paths
+        template = path.join(*split_template_path(template))
+        self.check_cache(template)
+        filepath = path.join(self.searchpath, template)
+        try:
+            f = open(filepath, 'rb')
+            contents = f.read().decode(self.encoding)
+        except IOError:
+            raise TemplateNotFound(template) 
+        finally:
+            f.close()
+        mtime = path.getmtime(filepath)
+        def uptodate():
+            try:
+                return path.getmtime(filepath) == mtime
+            except OSError:
+                return False
+        return contents, filepath, uptodate