Merge branch 'develop' of github.com:saltstack/salt into develop

MANIFEST.in

@@ -4,7 +4,7 @@ include LICENSE
 include README.rst
 include requirements.txt
 include tests/*.py
-recursive-include tests *.py
+recursive-include tests *
 include tests/integration/modules/files/*
 include tests/integration/files/*
 include tests/unit/templates/files/*

salt/cli/key.py

@@ -168,9 +168,12 @@ class Key(object):
         List keys
         '''
         selected_output = self.opts.get('selected_output_option', None)
-        printout = salt.output.get_printout(
+        if selected_output is None:
-            selected_output, self.opts, indent=2
+            printout = None
-        )
+        else:
+            printout = salt.output.get_printout(
+                selected_output, self.opts, indent=2
+            )
 
         if name in ('pre', 'un', 'unaccept', 'unaccepted'):
             self._list_pre(header=False, printer=printout)

salt/output/__init__.py

@@ -7,6 +7,13 @@ for managing outputters.
 import salt.loader
 
 
+STATIC = (
+          'yaml_out',
+          'text_out',
+          'raw_out',
+          'json_out',
+          )
+
 def display_output(data, out, opts=None):
     '''
     Print the passed data using the desired output
@@ -18,6 +25,15 @@ def get_printout(out, opts=None, **kwargs):
     '''
     Return a printer function
     '''
+    for outputter in STATIC:
+        if outputter in opts:
+            if opts[outputter]:
+                if outputter == 'text_out':
+                    out = 'txt'
+                else:
+                    out = outputter
+    if out is None:
+        out = 'pprint'
     if out.endswith('_out'):
         out = out[:-4]
     if opts is None:
@@ -27,5 +43,5 @@ def get_printout(out, opts=None, **kwargs):
         opts['color'] = not bool(opts.get('no_color', False))
     outputters = salt.loader.outputters(opts)
     if not out in outputters:
-        return None
+        return outputters['pprint']
     return outputters[out]

salt/states/file.py

@@ -186,66 +186,6 @@ def _error(ret, err_msg):
     return ret
 
 
-def _check_recurse(
-        name,
-        source,
-        clean,
-        require,
-        user,
-        group,
-        dir_mode,
-        file_mode,
-        env,
-        include_empty):
-    '''
-    Check what files will be changed by a recurse call
-    '''
-    vdir = set()
-    keep = set()
-    changes = {}
-    for fn_ in __salt__['cp.cache_dir'](source, env, include_empty):
-        if not fn_.strip():
-            continue
-        dest = _get_recurse_dest(name, fn_, source, env)
-        dirname = os.path.dirname(dest)
-        if not dirname in vdir:
-            # verify the directory perms if they are set
-            vdir.add(dirname)
-        if os.path.isfile(dest):
-            with open(fn_, 'r') as source_:
-                hsum = hashlib.md5(source_.read()).hexdigest()
-            source_sum = {'hash_type': 'md5',
-                          'hsum': hsum}
-            tchange = __salt__['file.check_file_meta'](
-                    dest,
-                    fn_,
-                    None,
-                    source_sum,
-                    user,
-                    group,
-                    file_mode,
-                    env)
-            if tchange:
-                changes[name] = tchange
-            keep.add(dest)
-        else:
-            keep.add(dest)
-            # The destination file is not present, make it
-            changes[name] = {'diff': 'New File'}
-    keep = list(keep)
-    if clean:
-        keep += _gen_keep_files(name, require)
-        for fn_ in _clean_dir(name, list(keep)):
-            changes[fn_] = {'diff': 'Remove'}
-    if changes:
-        comment = 'The following files are set to change:\n'
-        for fn_ in changes:
-            for key, val in changes[fn_].items():
-                comment += '{0}: {1} - {2}\n'.format(fn_, key, val)
-        return None, comment
-    return True, 'The directory {0} in in the correct state'.format(name)
-
-
 def _get_recurse_dest(prefix, fn_, source, env):
     '''
     Return the destination path to copy the file path, fn_(as returned by
@@ -881,6 +821,9 @@ def recurse(name,
         group=None,
         dir_mode=None,
         file_mode=None,
+        template=None,
+        context=None,
+        defaults=None,
         env=None,
         include_empty=False,
         backup='',
@@ -920,6 +863,17 @@ def recurse(name,
     file_mode
         The permissions mode to set any files created
 
+    template
+        If this setting is applied then the named templating engine will be
+        used to render the downloaded file, currently jinja, mako, and wempy
+        are supported
+
+    context
+        Overrides default context variables passed to the template.
+
+    defaults
+        Default context passed to the template.
+
     include_empty
         Set this to True if empty directories should also be created
         (default is False)
@@ -939,103 +893,139 @@ def recurse(name,
     if env is None:
         env = kwargs.get('__env__', 'base')
 
-    keep = set()
     # Verify the target directory
     if not os.path.isdir(name):
         if os.path.exists(name):
             # it is not a dir, but it exists - fail out
             return _error(
                 ret, 'The path {0} exists and is not a directory'.format(name))
-        __salt__['file.makedirs_perms'](
+        if not __opts__['test']:
+            __salt__['file.makedirs_perms'](
                 name, user, group, int(str(dir_mode), 8) if dir_mode else None)
 
-    if __opts__['test']:
+    def add_comment(path, comment):
-        ret['result'], ret['comment'] = _check_recurse(
+        comments = ret['comment'].setdefault(path, [])
-                name,
+        if isinstance(comment, basestring):
-                source,
+            comments.append(comment)
-                clean,
+        else:
-                require,
+            comments.extend(comment)
-                user,
+
-                group,
+    def merge_ret(path, _ret):
-                dir_mode,
+        # Use the most "negative" result code (out of True, None, False)
-                file_mode,
+        if _ret['result'] == False or ret['result'] == True:
-                env,
+            ret['result'] = _ret['result']
-                include_empty)
+
-        return ret
+        # Only include comments about files that changed
+        if _ret['result'] != True and _ret['comment']:
+            add_comment(path, _ret['comment'])
 
-    def update_changes_by_perms(path, mode, changetype='updated'):
-        _ret = {'name': name,
-                'changes': {},
-                'result': True,
-                'comment': []
-               }
-        __salt__['file.check_perms'](path, _ret, user, group, mode)
-        ret['result'] &= _ret['result'] # ie, once false, stay false.
-        if _ret['comment']:
-            comments = ret['comment'].setdefault(path, [])
-            comments.extend(_ret['comment'])
         if _ret['changes']:
-            ret['changes'][path] = changetype
+            ret['changes'][path] = _ret['changes']
+
+    def manage_file(path, source):
+        if clean and os.path.exists(path) and os.path.isdir(path):
+            _ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''}
+            if __opts__['test']:
+                _ret['comment'] = 'Replacing directory {0} with a file'.format(path)
+                _ret['result'] = None
+                merge_ret(path, _ret)
+                return
+            else:
+                shutil.rmtree(path)
+                _ret['changes'] = { 'diff': 'Replaced directory with a new file' }
+                merge_ret(path, _ret)
+
+        _ret = managed(
+            path,
+            source=source,
+            user=user,
+            group=group,
+            mode=file_mode,
+            template=template,
+            makedirs=True,
+            context=context,
+            replace=True,
+            defaults=defaults,
+            env=env,
+            backup=backup,
+            **kwargs)
+        merge_ret(path, _ret)
+
+    def manage_directory(path):
+        if clean and os.path.exists(path) and not os.path.isdir(path):
+            _ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''}
+            if __opts__['test']:
+                _ret['comment'] = 'Replacing {0} with a directory'.format(path)
+                _ret['result'] = None
+                merge_ret(path, _ret)
+                return
+            else:
+                os.remove(path)
+                _ret['changes'] = { 'diff': 'Replaced file with a directory' }
+                merge_ret(path, _ret)
+ 
+        _ret = directory(
+            path,
+            user=user,
+            group=group,
+            recurse=[],
+            mode=dir_mode,
+            makedirs=True,
+            clean=False,
+            require=None)
+        merge_ret(path, _ret)
 
     # If source is a list, find which in the list actually exists
     source, source_hash = __salt__['file.source_list'](source, '', env)
 
+    keep = set()
     vdir = set()
     for fn_ in __salt__['cp.cache_dir'](source, env, include_empty):
         if not fn_.strip():
             continue
         # fn_ here is the absolute source path of the file to copy from;
-        # it is either a normal file or an empty dir(if include_empthy==true).
+        # it is either a normal file or an empty dir(if include_empty==true).
 
         dest = _get_recurse_dest(name, fn_, source, env)
         dirname = os.path.dirname(dest)
         keep.add(dest)
-        if not os.path.isdir(dirname):
+
-            __salt__['file.makedirs'](dest, user=user, group=group)
         if not dirname in vdir:
             # verify the directory perms if they are set
-            update_changes_by_perms(dirname, dir_mode)
+            manage_directory(dirname)
             vdir.add(dirname)
-        if os.path.isfile(dest):
+
-            update_changes_by_perms(dest, file_mode)
+        if os.path.isdir(fn_) and include_empty:
-            srch = ''
+            #create empty dir
-            dsth = ''
+            manage_directory(dest)
-            # The file is present, if the sum differes replace it
-            with nested(open(fn_, 'r'), open(dest, 'r')) as (src_, dst_):
-                srch = hashlib.md5(src_.read()).hexdigest()
-                dsth = hashlib.md5(dst_.read()).hexdigest()
-            if srch != dsth:
-                # The downloaded file differes, replace!
-                salt.utils.copyfile(
-                        fn_,
-                        dest,
-                        __salt__['config.backup_mode'](backup),
-                        __opts__['cachedir'])
-                update_changes_by_perms(dest, file_mode)
-        elif os.path.isdir(dest) and include_empty:
-            #check perms
-            update_changes_by_perms(dest, dir_mode)
         else:
-            if os.path.isdir(fn_) and include_empty:
+            src = source + _get_recurse_dest('/', fn_, source, env)
-                #create empty dir
+            manage_file(dest, src)
-                os.mkdir(dest)
+
-                update_changes_by_perms(dest, dir_mode)
-            else:
-                # The destination file is not present, make it
-                salt.utils.copyfile(
-                        fn_,
-                        dest,
-                        __salt__['config.backup_mode'](backup),
-                        __opts__['cachedir'])
-                update_changes_by_perms(dest, file_mode)
-            ret['changes'][dest] = 'new'
     keep = list(keep)
     if clean:
+        # TODO: Use directory(clean=True) instead
         keep += _gen_keep_files(name, require)
         removed = _clean_dir(name, list(keep))
         if removed:
-            ret['changes']['removed'] = removed
+            if __opts__['test']:
-            ret['comment'] = 'Files cleaned from directory {0}'.format(name)
+                if ret['result']: ret['result'] = None
+                add_comment('removed', removed)
+            else:
+              ret['changes']['removed'] = removed
+
+    # Flatten comments until salt command line client learns
+    # to display structured comments in a readable fashion
+    ret['comment'] = '\n'.join("\n#### %s ####\n%s" % (k,
+            v if isinstance(v, basestring) else '\n'.join(v))
+        for (k, v) in ret['comment'].iteritems()).strip()
+
+    if not ret['comment']:
+        ret['comment'] = 'Recursively updated {0}'.format(name)
+
+    if not ret['changes'] and ret['result']:
+        ret['comment'] = 'The directory {0} is in the correct state'.format(name)
+
     return ret
 
 

tests/integration/shell/key.py

@@ -35,7 +35,14 @@ class KeyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
         '''
         data = self.run_key('-L --json-out')
         expect = [
-            '{"unaccepted": [], "accepted": ["minion", "sub_minion"], "rejected": []}',
+            '{',
+            '    "unaccepted": [], ',
+            '    "accepted": [',
+            '        "minion", ',
+            '        "sub_minion"',
+            '    ], ',
+            '    "rejected": []',
+            '}',
             ''
             ]
         self.assertEqual(data, expect)
@@ -60,10 +67,9 @@ class KeyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
         '''
         data = self.run_key('-L --raw-out')
         expect = [
-            "{'unaccepted': [], 'accepted': ['minion', "
+            "{'accepted': ['minion', 'sub_minion'], 'rejected': [], 'unaccepted': []}",
-            "'sub_minion'], 'rejected': []}",
             ''
-        ]
+            ]
         self.assertEqual(data, expect)
 
     def test_list_acc(self):

tests/integration/states/file.py

@@ -156,7 +156,42 @@ class FileTest(integration.ModuleCase):
         self.assertFalse(os.path.isfile(os.path.join(name, '36', 'scene')))
         result = self.state_result(ret)
         self.assertIsNone(result)
-        os.removedirs(name)
+        self.assertFalse(os.path.exists(name))
+
+    def test_recurse_template(self):
+        '''
+        file.recurse with jinja template enabled
+        '''
+        _ts = 'TEMPLATE TEST STRING'
+        name = os.path.join(integration.TMP, 'recurse_template_dir')
+        ret = self.run_state(
+            'file.recurse', name=name, source='salt://grail',
+            # For some strange reason passing defaults as a map does not work
+            template='jinja', defaults={'spam': _ts}, spam=_ts)
+        result = self.state_result(ret)
+        self.assertTrue(result)
+        self.assertIn(_ts, open(os.path.join(name, 'scene33'), 'r').read())
+        shutil.rmtree(name, ignore_errors=True)
+
+    def test_recurse_clean(self):
+        '''
+        file.recurse with clean=True
+        '''
+        name = os.path.join(integration.TMP, 'recurse_clean_dir')
+        os.makedirs(name)
+        strayfile = os.path.join(name, 'strayfile')
+        open(strayfile, 'w').close()
+        # Corner cases: replacing file with a directory and vice versa
+        open(os.path.join(name, '36'), 'w').close()
+        os.makedirs(os.path.join(name, 'scene33'))
+        ret = self.run_state(
+            'file.recurse', name=name, source='salt://grail', clean=True)
+        result = self.state_result(ret)
+        self.assertTrue(result)
+        self.assertFalse(os.path.exists(strayfile))
+        self.assertTrue(os.path.isfile(os.path.join(name, '36', 'scene')))
+        self.assertTrue(os.path.isfile(os.path.join(name, 'scene33')))
+        shutil.rmtree(name, ignore_errors=True)
 
     def test_sed(self):
         '''