Merge branch 'develop' of github.com:saltstack/salt into cherry-pick/CLOEXEC

.travis.yml

@@ -5,7 +5,7 @@ python:
   - '2.7'
 
 before_install:
-  - sudo apt-get update && sudo apt-get install swig supervisor rabbitmq-server
+  - sudo apt-get update && sudo apt-get install swig supervisor rabbitmq-server ruby
   - pip install http://dl.dropbox.com/u/174789/m2crypto-0.20.1.tar.gz
   - "if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi"
 

debian/changelog

@@ -1,3 +1,9 @@
+salt (0.10.5) unstable; urgency=low
+
+  * new release
+
+ -- Sean Channel <pentabular@gmail.com>  Thu, 15 Nov 2012 23:51:50 -0700
+
 salt (0.10.4-2) unstable; urgency=low
 
   * zmq3 support: b84f593770 86af5f03c3 c933c9e4ff a7e1d58f87 a5cdd8e313 c10bf6702f 2ea8b7e061 75f2a58b4e

doc/man/salt-call.1

@@ -1,4 +1,4 @@
-.TH "SALT-CALL" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT-CALL" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt-call \- salt-call Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .SH SYNOPSIS
 .sp
@@ -109,5 +109,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/man/salt-cp.1

@@ -1,4 +1,4 @@
-.TH "SALT-CP" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT-CP" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt-cp \- salt-cp Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .sp
 Copy a file to a set of systems
@@ -120,5 +120,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/man/salt-key.1

@@ -1,4 +1,4 @@
-.TH "SALT-KEY" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT-KEY" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt-key \- salt-key Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .SH SYNOPSIS
 .sp
@@ -141,5 +141,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/man/salt-master.1

@@ -1,4 +1,4 @@
-.TH "SALT-MASTER" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT-MASTER" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt-master \- salt-master Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .sp
 The Salt master daemon, used to control the Salt minions
@@ -81,5 +81,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/man/salt-minion.1

@@ -1,4 +1,4 @@
-.TH "SALT-MINION" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT-MINION" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt-minion \- salt-minion Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .sp
 The Salt minion daemon, receives commands from a remote Salt master.
@@ -82,5 +82,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/man/salt-run.1

@@ -1,4 +1,4 @@
-.TH "SALT-RUN" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT-RUN" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt-run \- salt-run Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .sp
 Execute a Salt runner
@@ -67,5 +67,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/man/salt-syndic.1

@@ -1,4 +1,4 @@
-.TH "SALT-SYNDIC" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT-SYNDIC" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt-syndic \- salt-syndic Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .sp
 The Salt syndic daemon, a special minion that passes through commands from a
@@ -76,5 +76,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/man/salt.1

@@ -1,4 +1,4 @@
-.TH "SALT" "1" "October 23, 2012" "0.10.4" "Salt"
+.TH "SALT" "1" "November 15, 2012" "0.10.5" "Salt"
 .SH NAME
 salt \- salt
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructeredText.
+.\" Man page generated from reStructuredText.
 .
 .SH SYNOPSIS
 .INDENT 0.0
@@ -227,5 +227,4 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
-.\" 
 .

doc/ref/file_server/index.rst

@@ -49,18 +49,43 @@ This example would instruct all Salt minions to download the vimrc from a
 directory with the same name as their os grain and copy it to /etc/vimrc
 
 For larger files, the cp.get_file module also supports gzip compression.
-Because gzip compression is CPU-intensive, this should only be used in
+Because gzip is CPU-intensive, this should only be used in
 scenarios where the compression ratio is very high (e.g. pretty-printed JSON
 or YAML files).
 
-Use the *gzip_compression* named argument to enable it.  Valid values are 1..9,
+Use the *gzip* named argument to enable it.  Valid values are 1..9,
 where 1 is the lightest compression and 9 the heaviest.  1 uses the least CPU
 on the master (and minion), 9 uses the most.
 
 .. code-block:: bash
 
-    # salt '*' cp.get_file salt://vimrc /etc/vimrc gzip_compression=5
+    # salt '*' cp.get_file salt://vimrc /etc/vimrc gzip=5
 
+Finally, note that by default cp.get_file does *not* create new destination
+directories if they do not exist.  To change this, use the *makedirs* argument:
+
+.. code-block:: bash
+
+    # salt '*' cp.get_file salt://vimrc /etc/vim/vimrc makedirs=True
+
+In this example, /etc/vim/ would be created if it didn't already exist.
+
+get_dir
+```````
+
+The cp.get_dir function can be used on the minion to download an entire
+directory from the master.  The syntax is very similar to get_file:
+
+.. code-block:: bash
+
+    # salt '*' cp.get_dir salt://etc/apache2 /etc
+
+cp.get_dir supports *template* rendering and *gzip* compression arguments just
+like get_file:
+
+.. code-block:: bash
+
+    # salt '*' cp.get_dir salt://etc/{{pillar.webserver}} /etc gzip=5 template=jinja
 
 
 File Server Client API

doc/ref/renderers/index.rst

@@ -47,7 +47,7 @@ The first line is a shebang that references the ``py`` renderer.
 
 Composing Renderers
 -------------------
-A render can be composed from other renderers by connecting them in a series
+A renderer can be composed from other renderers by connecting them in a series
 of pipes(``|``). In fact, the default ``Jinja + YAML`` renderer is implemented
 by combining a yaml renderer and a jinja renderer. Such renderer configuration
 is specified as: ``jinja | yaml``.
@@ -102,8 +102,9 @@ Here is a simple YAML renderer example:
 .. code-block:: python
 
     import yaml
-    def render(yaml_data, env='', sls='', argline='', **kws):
+    def render(yaml_data, env='', sls='', **kws):
         if not isinstance(yaml_data, basestring):
             yaml_data = yaml_data.read()
         data = yaml.load(yaml_data)
         return data if data else {}
+

doc/topics/releases/0.10.5.rst

@@ -0,0 +1,157 @@
+=========================
+Salt 0.10.5 Release Notes
+=========================
+
+Salt 0.10.5 is ready, and comes with some great new features. A few more
+interfaces have been modularized, like the outputter system. The job cache
+system has been made more powerful and can now store and retrieve jobs archived
+in external databases. The returner system has been extended to allow minions
+to easily retrieve data from a returner interface.
+
+As usual, this is an exciting release, with many noteworthy additions!
+
+Major Features
+==============
+
+External Job Cache
+------------------
+
+The external job cache is a system which allows for a returner interface to
+also act as a job cache. This system is intended to allow users to store
+job information in a central location for longer periods of time and to make
+the act of looking up information from jobs executed on other minions easier.
+
+Currently the external job cache is supported via the mongo and redis
+returners:
+
+.. code-block:: yaml
+
+    ext_job_cache: redis
+    redis.host: salt
+
+Once the external job cache is turned on the new `ret` module can be used on
+the minions to retrieve return information from the job cache. This can be a
+great way for minions to respond and react to other minions.
+
+OpenStack Additions
+--------------------
+
+OpenStack integration with Salt has been moving forward at a blistering pace.
+The new `nova`, `glance` and `keystone` modules represent the beginning of
+ongoing OpenStack integration.
+
+The Salt team has had many conversations with core OpenStack developers and
+is working on linking to OpenStack in powerful new ways.
+
+Wheel System
+------------
+
+A new API was added to the Salt Master which allows the master to be managed
+via an external API. This new system allows Salt API to easily hook into the
+Salt Master and manage configs, modify the state tree, manage the pillar and
+more. The main motivation for the wheel system is to enable features needed
+in the upcoming web ui so users can manage the master just as easily as they
+manage minions.
+
+The wheel system has also been hooked into the external auth system. This
+allows specific users to have granular access to manage components of the
+Salt Master.
+
+Render Pipes
+------------
+
+Jack Kuan has added a substantial new feature. The render pipes system allows
+Salt to treat the render system like unix pipes. This new system enables sls
+files to be passed through specific render engines. While the default renderer
+is still recommended, different engines can now be more easily merged. So to
+pipe the output of mako used in YAML use this shebang line:
+
+#!mako|yaml
+
+Salt Key Overhaul
+-----------------
+
+The Salt Key system was originally developed as only a cli interface, but as
+time went on it was pressed into becoming a clumsy API. This release marks a
+complete overhaul of Salt Key. Salt Key has been rewritten to function purely
+from an api and to use the outputter system. The benefit here is that the
+outputter system works much more cleanly with Salt Key now, and the internals
+of Salt Key can be used much more cleanly.
+
+Modular Outputters
+------------------
+
+The outputter system is now loaded in a modular way. This means that output
+systems can be more easily added by dropping a python file down on the master
+that contains the function `output`.
+
+Gzip from Fileserver
+--------------------
+
+Gzip compression has been added as an option to the cp.get_file and cp.get_dir
+commands. This will make file transfers more efficient and faster, especially
+over slower network links.
+
+Unified Module Configuration
+----------------------------
+
+In past releases of Salt, the minions needed to be configured for certain
+modules to function. This was difficult because it required pre-configuring the
+minions. 0.10.5 changes this by making all module configs on minions search the
+master config file for values.
+
+Now if a single database server is needed, then it can be defined in the master
+config and all minions will become aware of the configuration value.
+
+Salt Call Enhancements
+----------------------
+
+The ``salt-call`` command has been updated in a few ways. Now, ``salt-call``
+can take the --return option to send the data to a returner. Also,
+``salt-call`` now reports executions in the minion proc system, this allows the
+master to be aware of the operation salt-call is running.
+
+Death to pub_refresh and sub_timeout
+------------------------------------
+
+The old configuration values `pub_refresh` and `sub_timeout` have been removed.
+These options were in place to alleviate problems found in earlier versions of
+ZeroMQ which have since been fixed. The continued use of these options has
+proven to cause problems with message passing and have been completely removed.
+
+Git Revision Versions
+---------------------
+
+When running Salt directly from git (for testing or development, of course)
+it has been difficult to know exactly what code is being executed. The new
+versioning system will detect the git revision when building and how many
+commits have been made since the last release. A release from git will look
+like this:
+
+0.10.4-736-gec74d69
+
+Svn Module Addition
+-------------------
+
+Anthony Cornehl (twinshadow) contributed a module that adds Subversion support
+to Salt.  This great addition helps round out Salt's VCS support.
+
+Noteworthy Changes
+==================
+
+Arch Linux Defaults to Systemd
+------------------------------
+
+Arch Linux recently changed to use systemd by default and discontinued support
+for init scripts. Salt has followed suit and defaults to systemd now for
+managing services in Arch.
+
+Salt, Salt Cloud and Openstack
+------------------------------
+
+With the releases of Salt 0.10.5 and Salt Cloud 0.8.2, OpenStack becomes the
+first (non-OS) piece of software to include support both on the user level
+(with Salt Cloud) and the admin level (with Salt). We are excited to continue
+to extend support of other platforms at this level.
+
+

salt/config.py

@@ -147,7 +147,7 @@ def prepend_root_dir(opts, path_options):
                     opts[path_option])
 
 
-def minion_config(path):
+def minion_config(path, check_dns=True):
     '''
     Reads in the minion configuration file and sets up special options
     '''
@@ -230,24 +230,33 @@ def minion_config(path):
     if 'append_domain' in opts:
         opts['id'] = _append_domain(opts)
 
-    try:
+    if check_dns:
-        opts['master_ip'] = salt.utils.dns_check(opts['master'], True)
+        # Because I import salt.log bellow I need to re-import salt.utils here
-    except SaltClientError:
+        import salt.utils
-        if opts['retry_dns']:
+        try:
-            while True:
+            opts['master_ip'] = salt.utils.dns_check(opts['master'], True)
-                msg = ('Master hostname: {0} not found. '
+        except SaltClientError:
-                        'Retrying in {1} seconds').format(opts['master'],
+            if opts['retry_dns']:
-                                                            opts['retry_dns'])
+                while True:
-                log.warn(msg)
+                    import salt.log
-                print msg
+                    msg = ('Master hostname: {0} not found. Retrying in {1} '
-                time.sleep(opts['retry_dns'])
+                           'seconds').format(opts['master'], opts['retry_dns'])
-                try:
+                    if salt.log.is_console_configured():
-                    opts['master_ip'] = salt.utils.dns_check(opts['master'], True)
+                        log.warn(msg)
-                    break
+                    else:
-                except SaltClientError:
+                        print('WARNING: {0}'.format(msg))
-                    pass
+                    time.sleep(opts['retry_dns'])
-        else:
+                    try:
-            opts['master_ip'] = '127.0.0.1'
+                        opts['master_ip'] = salt.utils.dns_check(
+                            opts['master'], True
+                        )
+                        break
+                    except SaltClientError:
+                        pass
+            else:
+                opts['master_ip'] = '127.0.0.1'
+    else:
+        opts['master_ip'] = '127.0.0.1'
 
     opts['master_uri'] = 'tcp://{ip}:{port}'.format(ip=opts['master_ip'],
                                                     port=opts['master_port'])

salt/fileclient.py

@@ -96,7 +96,7 @@ class Client(object):
         yield dest
         os.umask(cumask)
 
-    def get_file(self, path, dest='', makedirs=False, env='base'):
+    def get_file(self, path, dest='', makedirs=False, env='base', gzip=None):
         '''
         Copies a file from the local files or master depending on
         implementation
@@ -257,7 +257,7 @@ class Client(object):
                 return dest
         return False
 
-    def get_dir(self, path, dest='', env='base'):
+    def get_dir(self, path, dest='', env='base', gzip=None):
         '''
         Get a directory recursively from the salt-master
         '''
@@ -284,7 +284,7 @@ class Client(object):
                     self.get_file(
                         'salt://{0}'.format(fn_),
                         '{0}/{1}'.format(dest, minion_relpath),
-                        True, env
+                        True, env, gzip
                     )
                 )
         # Replicate empty dirs from master
@@ -413,10 +413,10 @@ class LocalClient(Client):
                 return fnd
         return fnd
 
-    def get_file(self, path, dest='', makedirs=False, env='base', gzip_compression=None):
+    def get_file(self, path, dest='', makedirs=False, env='base', gzip=None):
         '''
         Copies a file from the local files directory into :param:`dest`
-        gzip_compression settings are ignored for local files
+        gzip compression settings are ignored for local files
         '''
         path = self._check_proto(path)
         fnd = self._find_file(path, env)
@@ -555,7 +555,7 @@ class RemoteClient(Client):
         self.auth = salt.crypt.SAuth(opts)
         self.sreq = salt.payload.SREQ(self.opts['master_uri'])
 
-    def get_file(self, path, dest='', makedirs=False, env='base', gzip_compression=None):
+    def get_file(self, path, dest='', makedirs=False, env='base', gzip=None):
         '''
         Get a single file from the salt-master
         path must be a salt server location, aka, salt://path/to/file, if
@@ -567,9 +567,9 @@ class RemoteClient(Client):
         load = {'path': path,
                 'env': env,
                 'cmd': '_serve_file'}
-        if gzip_compression:
+        if gzip:
-            gzip_compression = int(gzip_compression)
+            gzip = int(gzip)
-            load['gzip_compression'] = gzip_compression
+            load['gzip'] = gzip
 
         fn_ = None
         if dest:
@@ -608,9 +608,8 @@ class RemoteClient(Client):
             if not fn_:
                 with self._cache_loc(data['dest'], env) as cache_dest:
                     dest = cache_dest
-                    fn_ = salt.utils.fopen(dest, 'wb+')
+                    fn_ = open(dest, 'wb+')
-            gzip_compression = data.get('gzip_compression', None)
+            if data.get('gzip', None):
-            if gzip_compression:
                 data = salt.utils.gzip_util.uncompress(data['data'])
             else:
                 data = data['data']

salt/master.py

@@ -630,14 +630,14 @@ class AESFuncs(object):
         if not fnd['path']:
             return ret
         ret['dest'] = fnd['rel']
-        gzip_compression = load.get('gzip_compression', None)
+        gzip = load.get('gzip', None)
 
         with salt.utils.fopen(fnd['path'], 'rb') as fp_:
             fp_.seek(load['loc'])
             data = fp_.read(self.opts['file_buffer_size'])
-            if gzip_compression and data:
+            if gzip and data:
-                data = salt.utils.gzip_util.compress(data, gzip_compression)
+                data = salt.utils.gzip_util.compress(data, gzip)
-                ret['gzip_compression'] = gzip_compression
+                ret['gzip'] = gzip
             ret['data'] = data
         return ret
 

salt/modules/apt.py

@@ -70,14 +70,12 @@ def version(name):
 
         salt '*' pkg.version <package name>
     '''
-    # check for :i386 appended to 32bit name installed on 64bit machine
-    name32bit = '{0}:i386'.format(name)
-
     pkgs = list_pkgs(name)
+    # check for ':arch' appended to pkg name (i.e. 32 bit installed on 64 bit machine is ':i386')
+    if name.find(':') >= 0:
+        name = name.split(':')[0]
     if name in pkgs:
         return pkgs[name]
-    if name32bit in pkgs:
-        return pkgs[name32bit]
     else:
         return ''
 

salt/modules/augeas_cfg.py

@@ -1,5 +1,7 @@
 '''
 Manages configuration files via augeas
+
+:depends:   - Augeas Python adapter
 '''
 # Load Augeas libs
 try:

salt/modules/cassandra.py

@@ -1,19 +1,14 @@
 '''
 Cassandra NoSQL Database Module
 
-REQUIREMENT 1:
+:depends:   - pycassa Cassandra Python adapter
+:configuration:
+    The location of the 'nodetool' command, host, and thrift port needs to be
+    specified via pillar::
 
-The location of the 'nodetool' command, host, and thrift port
+        cassandra.nodetool: /usr/local/bin/nodetool
-needs to be specified via pillar.
+        cassandra.host: localhost
-
+        cassandra.thrift_port: 9160
-    cassandra.nodetool: /usr/local/bin/nodetool
-    cassandra.host: localhost
-    cassandra.thrift_port: 9160
-
-REQUIREMENT 2:
-
-The python module, 'pycassa', also needs to be installed on the
-minion.
 '''
 # Import Python libs
 import logging

salt/modules/cp.py

@@ -41,8 +41,47 @@ def recv(files, dest):
 
     return ret
 
+def _render_filenames(path, dest, env, template):
+    if not template:
+        return (path, dest)
 
-def get_file(path, dest, env='base', template=None, gzip_compression=None):
+    # render the path as a template using path_template_engine as the engine
+    if template not in salt.utils.templates.template_registry:
+        raise CommandExecutionError('Attempted to render file paths with unavailable engine '
+                  '{0}'.format(template))
+
+    kwargs = {}
+    kwargs['salt'] = __salt__
+    kwargs['pillar'] = __pillar__
+    kwargs['grains'] = __grains__
+    kwargs['opts'] = __opts__
+    kwargs['env'] = env
+
+    def _render(contents):
+        # write out path to temp file
+        tmp_path_fn = salt.utils.mkstemp()
+        with open(tmp_path_fn, 'w+') as fp_:
+            fp_.write(contents)
+        data = salt.utils.templates.template_registry[template](
+            tmp_path_fn,
+            to_str=True,
+            **kwargs
+        )
+        salt.utils.safe_rm(tmp_path_fn)
+        if not data['result']:
+            # Failed to render the template
+            raise CommandExecutionError('Failed to render file path with error: {0}'.format(
+                data['data']
+            ))
+        else:
+            return data['data']
+
+    path = _render(path)
+    dest = _render(dest)
+    return (path, dest)
+
+
+def get_file(path, dest, env='base', makedirs=False, template=None, gzip=None):
     '''
     Used to get a single file from the salt master
 
@@ -50,56 +89,13 @@ def get_file(path, dest, env='base', template=None, gzip_compression=None):
 
         salt '*' cp.get_file salt://path/to/file /minion/dest
     '''
-    if template is not None:
+    (path, dest) = _render_filenames(path, dest, env, template)
-        # render the path as a template using path_template_engine as the engine
-        if template not in salt.utils.templates.template_registry:
-            log.error('Attempted to render file paths with unavailable engine '
-                      '{0}'.format(template))
-            return ''
-
-        kwargs = {}
-        kwargs['salt'] = __salt__
-        kwargs['pillar'] = __pillar__
-        kwargs['grains'] = __grains__
-        kwargs['opts'] = __opts__
-        kwargs['env'] = env
-
-        def _render(contents):
-            # write out path to temp file
-            tmp_path_fn = salt.utils.mkstemp()
-            with open(tmp_path_fn, 'w+') as fp_:
-                fp_.write(contents)
-            data = salt.utils.templates.template_registry[template](
-                tmp_path_fn,
-                to_str=True,
-                **kwargs
-            )
-            salt.utils.safe_rm(tmp_path_fn)
-            if not data['result']:
-                # Failed to render the template
-                raise CommandExecutionError('Failed to render file path with error: {0}'.format(
-                    data['data']
-                ))
-            else:
-                return data['data']
-
-
-        try:
-            path = _render(path)
-        except CommandExecutionError as exc:
-            log.error(str(exc))
-            return ''
-        try:
-            dest = _render(dest)
-        except CommandExecutionError as exc:
-            log.error(str(exc))
-            return ''
 
     if not hash_file(path, env):
         return ''
     else:
         client = salt.fileclient.get_file_client(__opts__)
-        return client.get_file(path, dest, False, env, gzip_compression)
+        return client.get_file(path, dest, makedirs, env, gzip)
 
 
 def get_template(path, dest, template='jinja', env='base', **kwargs):
@@ -122,7 +118,7 @@ def get_template(path, dest, template='jinja', env='base', **kwargs):
     return client.get_template(path, dest, template, False, env, **kwargs)
 
 
-def get_dir(path, dest, env='base'):
+def get_dir(path, dest, env='base', template=None, gzip=None):
     '''
     Used to recursively copy a directory from the salt master
 
@@ -130,8 +126,10 @@ def get_dir(path, dest, env='base'):
 
         salt '*' cp.get_dir salt://path/to/dir/ /minion/dest
     '''
+    (path, dest) = _render_filenames(path, dest, env, template)
+
     client = salt.fileclient.get_file_client(__opts__)
-    return client.get_dir(path, dest, env)
+    return client.get_dir(path, dest, env, gzip)
 
 
 def get_url(path, dest, env='base'):

salt/modules/ebuild.py

@@ -1,5 +1,7 @@
 '''
 Support for Portage
+
+:optdepends:    - portage Python adapter
 '''
 
 try:

salt/modules/gem.py

@@ -22,7 +22,7 @@ def _gem(command, ruby=None, runas=None):
         return False
 
 
-def install(gems, ruby=None, runas=None):
+def install(gems, ruby=None, runas=None, version=None, rdoc=False, ri=False):
     '''
     Installs one or several gems.
 
@@ -32,8 +32,23 @@ def install(gems, ruby=None, runas=None):
         If RVM is installed, the ruby version and gemset to use.
     runas : None
         The user to run gem as.
+    version : None
+        Specify the version to install for the gem.
+        Doesn't play nice with multiple gems at once
+    rdoc : False
+        Generate RDoc documentation for the gem(s).
+    ri : False
+        Generate RI documentation for the gem(s).
     '''
-    return _gem('install {gems}'.format(gems=gems), ruby, runas=runas)
+    options = ''
+    if version:
+        options += ' --version {0}'.format(version)
+    if not rdoc:
+        options += ' --no-rdoc'
+    if not ri:
+        options += ' --no-ri'
+
+    return _gem('install {gems} {options}'.format(gems=gems, options=options), ruby, runas=runas)
 
 
 def uninstall(gems, ruby=None, runas=None):

salt/modules/glance.py

@@ -1,15 +1,16 @@
 '''
 Module for handling openstack glance calls.
 
-This module is not usable until the following are specified either in a pillar
+:optdepends:    - glanceclient Python adapter
-or in the minion's config file:
+:configuration: This module is not usable until the following are specified
+    either in a pillar or in the minion's config file::
 
-keystone.user: admin
+        keystone.user: admin
-keystone.password: verybadpass
+        keystone.password: verybadpass
-keystone.tenant: admin
+        keystone.tenant: admin
-keystone.tenant_id: f80919baedab48ec8931f200c65a50df
+        keystone.tenant_id: f80919baedab48ec8931f200c65a50df
-keystone.insecure: False   #(optional)
+        keystone.insecure: False   #(optional)
-keystone.auth_url: 'http://127.0.0.1:5000/v2.0/'
+        keystone.auth_url: 'http://127.0.0.1:5000/v2.0/'
 '''
 has_glance = False
 try:

salt/modules/groupadd.py

@@ -11,7 +11,7 @@ def __virtual__():
     '''
     Set the user module if the kernel is Linux
     '''
-    return 'group' if __grains__['kernel'] == 'Linux' else False
+    return 'group' if __grains__.get('kernel', '') == 'Linux' else False
 
 
 def add(name, gid=None, system=False):

salt/modules/keystone.py

@@ -1,15 +1,16 @@
 '''
 Module for handling openstack keystone calls.
 
-This module is not usable until the following are specified either in a pillar
+:optdepends:    - keystoneclient Python adapter
-or in the minion's config file:
+:configuration: This module is not usable until the following are specified
+    either in a pillar or in the minion's config file::
 
-keystone.user: admin
+        keystone.user: admin
-keystone.password: verybadpass
+        keystone.password: verybadpass
-keystone.tenant: admin
+        keystone.tenant: admin
-keystone.tenant_id: f80919baedab48ec8931f200c65a50df
+        keystone.tenant_id: f80919baedab48ec8931f200c65a50df
-keystone.insecure: False   #(optional)
+        keystone.insecure: False   #(optional)
-keystone.auth_url: 'http://127.0.0.1:5000/v2.0/'
+        keystone.auth_url: 'http://127.0.0.1:5000/v2.0/'
 '''
 has_keystone = False
 try:

salt/modules/kvm_hyper.py

@@ -2,7 +2,7 @@
 Provide the hyper module for kvm hypervisors. This is the interface used to
 interact with kvm on behalf of the salt-virt interface
 
-Required python modules: libvirt
+:depends:   - libvirt Python module
 '''
 
 # This is a test interface for the salt-virt system. The api in this file is

salt/modules/launchctl.py

@@ -1,5 +1,7 @@
 '''
 Module for the management of MacOS systems that use launchd/launchctl
+
+:depends:   - plistlib Python module
 '''
 
 import plistlib

salt/modules/ldapmod.py

@@ -1,37 +1,33 @@
 '''
 Module to provide LDAP commands via salt.
 
-REQUIREMENT 1:
+:depends:   - ldap Python module
+:configuration: In order to connect to LDAP, certain configuration is required
+    in the minion config on the LDAP server. The minimum configuration items
+    that must be set are::
 
-In order to connect to LDAP, certain configuration is required
+        ldap.basedn: dc=acme,dc=com (example values, adjust to suit)
-in the minion config on the LDAP server.
-The minimum configuration items that must be set are::
 
-    ldap.basedn: dc=acme,dc=com (example values, adjust to suit)
+    If your LDAP server requires authentication then you must also set::
 
-If your LDAP server requires authentication then you must also set::
+        ldap.binddn: admin
+        ldap.bindpw: password
 
-    ldap.binddn: admin
+    In addition, the following optional values may be set::
-    ldap.bindpw: password
 
-In addition, the following optional values may be set::
+        ldap.server: localhost (default=localhost, see warning below)
+        ldap.port: 389 (default=389, standard port)
+        ldap.tls: False (default=False, no TLS)
+        ldap.scope: 2 (default=2, ldap.SCOPE_SUBTREE)
+        ldap.attrs: [saltAttr] (default=None, return all attributes)
 
-    ldap.server: localhost (default=localhost, see warning below)
+.. warning::
-    ldap.port: 389 (default=389, standard port)
-    ldap.tls: False (default=False, no TLS)
-    ldap.scope: 2 (default=2, ldap.SCOPE_SUBTREE)
-    ldap.attrs: [saltAttr] (default=None, return all attributes)
 
-WARNING:
+    At the moment this module only recommends connection to LDAP services
-At the moment this module only recommends connection to LDAP services
+    listening on 'localhost'.  This is deliberate to avoid the potentially
-listening on 'localhost'.  This is deliberate to avoid the potentially
+    dangerous situation of multiple minions sending identical update commands
-dangerous situation of multiple minions sending identical update commands to
+    to the same LDAP server.  It's easy enough to override this behaviour, but
-the same LDAP server.  It's easy enough to override this behaviour,
+    badness may ensue - you have been warned.
-but badness may ensue - you have been warned.
-
-REQUIREMENT 2:
-
-Required python modules: ldap
 '''
 # Import Python libs
 import time

salt/modules/linux_sysctl.py

@@ -113,7 +113,7 @@ def persist(name, value, config='/etc/sysctl.conf'):
             # and it seems unnecessary to indent the below for
             # loop since it is a fairly large block of code.
             config_data = _fh.readlines()
-    except (IOError, OSError) as exc:
+    except (IOError, OSError):
         msg = 'Could not read from file: {0}'
         raise CommandExecutionError(msg.format(config))
 

salt/modules/mdadm.py

@@ -62,7 +62,8 @@ def detail(device='/dev/md0'):
 
     # Lets make sure the device exists before running mdadm
     if not os.path.exists(device):
-        raise CommandExecutionError("Device {0} doesn't exist!")
+        msg = "Device {0} doesn't exist!"
+        raise CommandExecutionError(msg.format(device))
 
     cmd = 'mdadm --detail {0}'.format(device)
     for line in __salt__['cmd.run_stdout'](cmd).splitlines():

salt/modules/mongodb.py

@@ -1,16 +1,16 @@
 '''
 Module to provide MongoDB functionality to Salt
 
-This module uses PyMongo, and accepts configuration details as parameters
+:configuration: This module uses PyMongo, and accepts configuration details as
-as well as configuration settings:
+    parameters as well as configuration settings::
 
-    mongodb.host: 'localhost'
+        mongodb.host: 'localhost'
-    mongodb.port: '27017'
+        mongodb.port: '27017'
-    mongodb.user: ''
+        mongodb.user: ''
-    mongodb.password: ''
+        mongodb.password: ''
 
-This data can also be passed into pillar. Options passed into opts will
+    This data can also be passed into pillar. Options passed into opts will
-overwrite options passed into pillar.
+    overwrite options passed into pillar.
 '''
 
 # Import python libs

salt/modules/mysql.py

@@ -1,25 +1,20 @@
 '''
 Module to provide MySQL compatibility to salt.
 
-REQUIREMENT 1:
+:depends:   - MySQLdb Python module
+:configuration: In order to connect to MySQL, certain configuration is required
+    in /etc/salt/minion on the relevant minions. Some sample configs might look
+    like::
 
-In order to connect to MySQL, certain configuration is required
+        mysql.host: 'localhost'
-in /etc/salt/minion on the relevant minions. Some sample configs
+        mysql.port: 3306
-might look like::
+        mysql.user: 'root'
+        mysql.pass: ''
+        mysql.db: 'mysql'
 
-    mysql.host: 'localhost'
+    You can also use a defaults file::
-    mysql.port: 3306
-    mysql.user: 'root'
-    mysql.pass: ''
-    mysql.db: 'mysql'
 
-You can also use a defaults file::
+        mysql.default_file: '/etc/mysql/debian.cnf'
-
-    mysql.default_file: '/etc/mysql/debian.cnf'
-
-REQUIREMENT 2:
-
-Required python modules: MySQLdb
 '''
 # Import Python libs
 import time

salt/modules/nova.py

@@ -1,13 +1,15 @@
 '''
 Module for handling openstack nova calls.
 
-This module is not usable until the user, password, tenant and auth url are
+:depends:   - novaclient Python module
-specified either in a pillar or in the minion's config file. For example:
+:configuration: This module is not usable until the user, password, tenant and
+    auth url are specified either in a pillar or in the minion's config file.
+    For example::
 
-keystone.user: admin
+        keystone.user: admin
-keystone.password: verybadpass
+        keystone.password: verybadpass
-keystone.tenant: admin
+        keystone.tenant: admin
-keystone.auth_url: 'http://127.0.0.1:5000/v2.0/'
+        keystone.auth_url: 'http://127.0.0.1:5000/v2.0/'
 '''
 has_nova = False
 try:

salt/modules/postgres.py

@@ -1,18 +1,18 @@
 '''
 Module to provide Postgres compatibility to salt.
 
-In order to connect to Postgres, certain configuration is required
+:configuration: In order to connect to Postgres, certain configuration is
-in /etc/salt/minion on the relevant minions. Some sample configs
+    required in /etc/salt/minion on the relevant minions. Some sample configs
-might look like::
+    might look like::
 
-    postgres.host: 'localhost'
+        postgres.host: 'localhost'
-    postgres.port: '5432'
+        postgres.port: '5432'
-    postgres.user: 'postgres'
+        postgres.user: 'postgres'
-    postgres.pass: ''
+        postgres.pass: ''
-    postgres.db: 'postgres'
+        postgres.db: 'postgres'
 
-This data can also be passed into pillar. Options passed into opts will
+    This data can also be passed into pillar. Options passed into opts will
-overwrite options passed into pillar
+    overwrite options passed into pillar
 '''
 
 # Import Python libs

salt/modules/ps.py

@@ -2,7 +2,7 @@
 A salt interface to psutil, a system and process library.
 See http://code.google.com/p/psutil.
 
-Required python modules: psutil
+:depends:   - psutil Python module
 '''
 
 import sys

salt/modules/reg.py

@@ -1,7 +1,7 @@
 '''
 Manage the registry on Windows
 
-Required python modules: _winreg
+:depends:   - winreg Python module
 '''
 
 # TODO: Figure out the exceptions _winreg can raise and properly  catch

salt/modules/saltutil.py

@@ -1,6 +1,8 @@
 '''
-The Saltutil module is used to manage the state of the salt minion itself. It is
+The Saltutil module is used to manage the state of the salt minion itself. It
-used to manage minion modules as well as automate updates to the salt minion
+is used to manage minion modules as well as automate updates to the salt minion
+
+:depends:   - esky Python module
 '''
 
 # Import Python libs

salt/modules/ssh.py

@@ -3,8 +3,23 @@ Manage client ssh components
 '''
 import os
 import re
-import binascii
 import hashlib
+import binascii
+import logging
+
+import salt.utils
+from salt.exceptions import (
+    SaltInvocationError,
+    CommandExecutionError,
+)
+
+log = logging.getLogger(__name__)
+
+def __virtual__():
+    # TODO: This could work on windows with some love
+    if __grains__['os'] == 'Windows':
+        return False
+    return 'ssh'
 
 
 def _refine_enc(enc):
@@ -66,25 +81,34 @@ def _replace_auth_key(
     lines = []
     uinfo = __salt__['user.info'](user)
     full = os.path.join(uinfo['home'], config)
-    with open(full, 'r') as f:
+    try:
-        for line in f:
+        # open the file for both reading AND writing
-            if line.startswith('#'):
+        with open(full, 'r') as _fh:
-                # Commented Line
+            for line in _fh:
-                lines.append(line)
+                if line.startswith('#'):
-                continue
+                    # Commented Line
-            comps = line.split()
+                    lines.append(line)
-            if len(comps) < 2:
+                    continue
-                # Not a valid line
+                comps = line.split()
-                lines.append(line)
+                if len(comps) < 2:
-                continue
+                    # Not a valid line
-            key_ind = 1
+                    lines.append(line)
-            if comps[0][:4:] not in ['ssh-', 'ecds']:
+                    continue
-                key_ind = 2
+                key_ind = 1
-            if comps[key_ind] == key:
+                if comps[0][:4:] not in ['ssh-', 'ecds']:
-                lines.append(auth_line)
+                    key_ind = 2
-            else:
+                if comps[key_ind] == key:
-                lines.append(line)
+                    lines.append(auth_line)
-    with open(full, 'w+') as f: f.writelines(lines)
+                else:
+                    lines.append(line)
+            _fh.close()
+            # Re-open the file writable after properly closing it
+            with open(full, 'w') as _fh:
+                # Write out any changes
+                _fh.writelines(lines)
+    except (IOError, OSError) as exc:
+        msg = 'Problem reading or writing to key file: {0}'
+        raise CommandExecutionError(msg.format(str(exc)))
 
 
 def _validate_keys(key_file):
@@ -93,9 +117,10 @@ def _validate_keys(key_file):
     '''
     ret = {}
     linere = re.compile(r'^(.*?)\s?((?:ssh\-|ecds).+)$')
+
     try:
-        with open(key_file, 'r') as f:
+        with open(key_file, 'r') as _fh:
-            for line in f:
+            for line in _fh:
                 if line.startswith('#'):
                     # Commented Line
                     continue
@@ -130,8 +155,9 @@ def _validate_keys(key_file):
                             'comment': comment,
                             'options': options,
                             'fingerprint': fingerprint}
-    except IOError:
+    except (IOError, OSError) as exc:
-        return {}
+        msg = 'Problem reading ssh key file {0}'
+        raise CommandExecutionError(msg.format(key_file))
 
     return ret
 
@@ -162,11 +188,14 @@ def host_keys(keydir=None):
 
         salt '*' ssh.host_keys
     '''
-    # Set up the default keydir - needs to support sshd_config parsing in the
+    # TODO: support parsing sshd_config for the key directory
-    # future
     if not keydir:
         if __grains__['kernel'] == 'Linux':
             keydir = '/etc/ssh'
+        else:
+            # If keydir is None, os.listdir() will blow up
+            msg = 'ssh.host_keys: Please specify a keydir'
+            raise SaltInvocationError(msg)
     keys = {}
     for fn_ in os.listdir(keydir):
         if fn_.startswith('ssh_host_'):
@@ -176,7 +205,8 @@ def host_keys(keydir=None):
             if len(top) > 1:
                 kname += '.{0}'.format(top[1])
             try:
-                keys[kname] = open(os.path.join(keydir, fn_), 'r').read()
+                with open(os.path.join(keydir, fn_), 'r') as _fh:
+                    keys[kname] = _fh.readline().strip()
             except (IOError, OSError):
                 keys[kname] = ''
     return keys
@@ -225,7 +255,7 @@ def check_key(user, key, enc, comment, options, config='.ssh/authorized_keys'):
 
     CLI Example::
 
-        salt '*' ssh.check_key <user> <key>
+        salt '*' ssh.check_key <user> <key> <enc> <comment> <options>
     '''
     current = auth_keys(user, config)
     nline = _format_auth_line(key, enc, comment, options)
@@ -256,37 +286,53 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'):
         # Remove the key
         uinfo = __salt__['user.info'](user)
         full = os.path.join(uinfo['home'], config)
+
+        # Return something sensible if the file doesn't exist
         if not os.path.isfile(full):
-            return 'User authorized keys file not present'
+            return 'Authorized keys file {1} not present'.format(full)
+
         lines = []
-        with open(full, 'r') as f:
+        try:
-            for line in f:
+            # Read every line in the file to find the right ssh key
-                if line.startswith('#'):
+            # and then write out the correct one. Open the file once
-                    # Commented Line
+            with open(full, 'r') as _fh:
+                for line in _fh:
+                    if line.startswith('#'):
+                        # Commented Line
+                        lines.append(line)
+                        continue
+
+                    # get "{options} key"
+                    ln = re.search(linere, line)
+                    if not ln:
+                        # not an auth ssh key, perhaps a blank line
+                        continue
+
+                    comps = ln.group(2).split()
+
+                    if len(comps) < 2:
+                        # Not a valid line
+                        lines.append(line)
+                        continue
+
+                    pkey = comps[1]
+
+                    # This is the key we are "deleting", so don't put
+                    # it in the list of keys to be re-added back
+                    if pkey == key:
+                        continue
+
                     lines.append(line)
-                    continue
 
-                # get "{options} key"
+            # Let the context manager do the right thing here and then
-                ln = re.search(linere, line)
+            # re-open the file in write mode to save the changes out.
-                if not ln:
+            with open(full, 'w') as _fh:
-                    # not an auth ssh key, perhaps a blank line
+                _fh.writelines(lines)
-                    continue
+        except (IOError, OSError) as exc:
-
+            log.warn('Could not read/write key file: {0}'.format(str(exc)))
-                comps = ln.group(2).split()
+            return 'Key not removed'
-
-                if len(comps) < 2:
-                    # Not a valid line
-                    lines.append(line)
-                    continue
-
-                pkey = comps[1]
-
-                if pkey == key:
-                    continue
-                else:
-                    lines.append(line)
-        with open(full, 'w+') as f: f.writelines(lines)
         return 'Key removed'
+    # TODO: Should this function return a simple boolean?
     return 'Key not present'
 
 def set_auth_key_from_file(
@@ -305,7 +351,8 @@ def set_auth_key_from_file(
     # TODO: add support for pulling keys from other file sources as well
     lfile = __salt__['cp.cache_file'](source, env)
     if not os.path.isfile(lfile):
-        return 'fail'
+        msg = 'Failed to pull key file from salt file server'
+        raise CommandExecutionError(msg)
 
     newkey = {}
     rval = ''
@@ -383,12 +430,21 @@ def set_auth_key(
             os.chmod(dpath, 448)
 
         if not os.path.isfile(fconfig):
-            open(fconfig, 'a+').write('{0}'.format(auth_line))
+            new_file = True
+        else:
+            new_file = False
+
+        try:
+            with open(fconfig, 'a+') as _fh:
+                _fh.write('{0}'.format(auth_line))
+        except (IOError, OSError) as exc:
+            msg = 'Could not write to key file: {0}'
+            raise CommandExecutionError(msg.format(str(exc)))
+
+        if new_file:
             if os.geteuid() == 0:
                 os.chown(fconfig, uinfo['uid'], uinfo['gid'])
             os.chmod(fconfig, 384)
-        else:
-            open(fconfig, 'a+').write('{0}'.format(auth_line))
         return 'new'
 
 
@@ -432,7 +488,7 @@ def get_known_host(user, hostname, config='.ssh/known_hosts'):
 
 def recv_known_host(user, hostname, enc=None, port=None, hash_hostname=False):
     '''
-    Retreive information about host public key from remote server
+    Retrieve information about host public key from remote server
 
     CLI Example::
 
@@ -546,12 +602,18 @@ def set_known_host(user, hostname,
     uinfo = __salt__['user.info'](user)
     full = os.path.join(uinfo['home'], config)
     line = '{hostname} {enc} {key}\n'.format(**remote_host)
-    with open(full, 'a') as fd:
+
-        fd.write(line)
+    try:
+        with open(full, 'a') as fd:
+            fd.write(line)
+    except (IOError, OSError) as exc:
+        raise CommandExecutionError("Couldn't append to known hosts file")
+
     if os.geteuid() == 0:
         os.chown(full, uinfo['uid'], uinfo['gid'])
     return {'status': 'updated', 'old': stored_host, 'new': remote_host}
 
+    # TODO: The lines below this are dead code, fix the above return and make these work
     status = check_known_host(user, hostname, fingerprint=fingerprint,
                                                config=config)
     if status == 'exists':

salt/modules/tls.py

@@ -3,16 +3,11 @@ A salt module for SSL/TLS.
 Can create a Certificate Authority (CA)
 or use Self-Signed certificates.
 
-REQUIREMENT 1:
+:depends:   - PyOpenSSL Python module
+:configuration: Add the following values in /etc/salt/minion for the CA module
+    to function properly::
 
-Required python modules: PyOpenSSL
+        ca.cert_base_path: '/etc/pki'
-
-REQUIREMENT 2:
-
-Add the following values in /etc/salt/minion for the
-CA module to function properly::
-
-    ca.cert_base_path: '/etc/pki'
 '''
 
 # Import Python libs

salt/modules/virt.py

@@ -1,7 +1,7 @@
 '''
 Work with virtual machines managed by libvirt
 
-Required python modules: libvirt
+:depends: libvirt Python module
 '''
 # Special Thanks to Michael Dehann, many of the concepts, and a few structures
 # of his in the virt func module have been used

salt/modules/win_disk.py

@@ -1,5 +1,7 @@
 '''
 Module for gathering disk information on Windows
+
+:depends:   - win32api Python module
 '''
 try:
     import ctypes

salt/modules/win_file.py

@@ -2,7 +2,10 @@
 Manage information about files on the minion, set/read user, group
 data
 
-Required python modules: win32api, win32con, win32security, ntsecuritycon
+:depends:   - win32api
+            - win32con
+            - win32security
+            - ntsecuritycon
 '''
 
 import os

salt/modules/win_pkg.py

@@ -1,5 +1,10 @@
 '''
 A module to manage software on Windows
+
+:depends:   - pythoncom
+            - win32com
+            - win32con
+            - win32api
 '''
 try:
     import pythoncom

salt/modules/yumpkg.py

@@ -1,7 +1,9 @@
 '''
 Support for YUM
 
-Required python modules: yum, rpm, rpmUtils
+:depends:   - yum Python module
+            - rpm Python module
+            - rpmUtils Python module
 '''
 try:
     import yum

salt/modules/zpool.py

@@ -3,7 +3,7 @@ zfs support.
 
 Assumes FreeBSD
 
-requires: mkfile
+:depends:   - mkfile
 '''
 import os
 

salt/output/__init__.py

@@ -28,19 +28,27 @@ def get_printout(out, opts=None, **kwargs):
     '''
     if opts is None:
         opts = {}
-    for outputter in STATIC:
+
-        if outputter in opts:
+    if 'output' in opts:
-            if opts[outputter]:
+        # new --out option
-                if outputter == 'text_out':
+        out = opts['output']
-                    out = 'txt'
+        if out == 'text':
-                else:
+            out = 'txt'
-                    out = outputter
+    else:
+        # XXX: This should be removed before 0.10.8 comes out
+        for outputter in STATIC:
+            if outputter in opts:
+                if opts[outputter]:
+                    if outputter == 'text_out':
+                        out = 'txt'
+                    else:
+                        out = outputter
+        if out and out.endswith('_out'):
+            out = out[:-4]
+
     if out is None:
         out = 'pprint'
-    if out.endswith('_out'):
+
-        out = out[:-4]
-    if opts is None:
-        opts = {}
     opts.update(kwargs)
     if not 'color' in opts:
         opts['color'] = not bool(opts.get('no_color', False))
@@ -55,4 +63,3 @@ def out_format(data, out, opts=None):
     Return the formatted outputter string for the passed data
     '''
     return get_printout(out, opts)(data).rstrip()
-

salt/pillar/__init__.py

@@ -70,6 +70,7 @@ class Pillar(object):
         self.opts = self.__gen_opts(opts, grains, id_, env)
         self.client = salt.fileclient.get_file_client(self.opts)
         if opts.get('file_client', '') == 'local':
+            opts['grains'] = grains
             self.functions = salt.loader.minion_mods(opts)
         else:
             self.functions = salt.loader.minion_mods(self.opts)

salt/renderers/jinja.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from StringIO import StringIO
 
 # Import Salt libs
 from salt.exceptions import SaltRenderError
@@ -6,13 +7,18 @@ import salt.utils.templates
 
 
 
-def render(template_file, env='', sls='', context=None, **kws):
+def render(template_file, env='', sls='', argline='', context=None, **kws):
     '''
     Render the template_file, passing the functions and grains into the
     Jinja rendering system.
 
     :rtype: string
     '''
+    from_str = argline=='-s'
+    if not from_str and argline:
+        raise SaltRenderError(
+                  'Unknown renderer option: {opt}'.format(opt=argline)
+              )
     tmp_data = salt.utils.templates.jinja(template_file, to_str=True,
                     salt=__salt__,
                     grains=__grains__,
@@ -24,5 +30,5 @@ def render(template_file, env='', sls='', context=None, **kws):
     if not tmp_data.get('result', False):
         raise SaltRenderError(tmp_data.get('data',
             'Unknown render error in jinja renderer'))
-    return tmp_data['data']
+    return StringIO(tmp_data['data'])
 

salt/renderers/mako.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from StringIO import StringIO
 
 import salt.utils.templates
 from salt.exceptions import SaltRenderError
@@ -21,4 +22,4 @@ def render(template_file, env='', sls='', context=None, **kws):
     if not tmp_data.get('result', False):
         raise SaltRenderError(tmp_data.get('data',
             'Unknown render error in mako renderer'))
-    return tmp_data['data']
+    return StringIO(tmp_data['data'])

salt/renderers/stateconf.py

@@ -6,14 +6,6 @@ arguments (including salt specific args, such as 'require', etc) as template
 context. The goal is to make writing reusable/configurable/ parameterized
 salt files easier and cleaner, therefore, additionally, it also:
 
-  - Adds support of absolute(eg, ``salt://path/to/salt/file``) and relative(eg,
-    ``path/to/salt/file``) template inclusion or import(ie, with ``<%include/>``
-    or ``<%namespace.../>``) for Mako. Example::
-
-
-       <%include file="templates/sls-parts.mako"/>
-       <%namespace file="salt://lib/templates/utils.mako" import="helper"/>
-
   - Recognizes the special state function, ``stateconf.set``, that configures a
     default list of named arguments useable within the template context of
     the salt file. Example::

salt/renderers/wempy.py

@@ -1,17 +1,17 @@
+from StringIO import StringIO
+
 # Import Salt libs
 from salt.exceptions import SaltRenderError
 import salt.utils.templates
 
 
-def render(template_file, env='', sls='', context=None, **kws):
+def render(template_file, env='', sls='', argline='', context=None, **kws):
     '''
     Render the data passing the functions and grains into the rendering system
 
     :rtype: string
     '''
-    tmp_data = salt.utils.templates.wempy(
+    tmp_data = salt.utils.templates.wempy(template_file, to_str=True,
-            template_file,
-            True,
             salt=__salt__,
             grains=__grains__,
             opts=__opts__,
@@ -21,5 +21,5 @@ def render(template_file, env='', sls='', context=None, **kws):
             context=context)
     if not tmp_data.get('result', False):
         raise SaltRenderError(tmp_data.get('data',
-            'Unknown render error in yaml_wempy renderer'))
+            'Unknown render error in the wempy renderer'))
-    return tmp_data['data']
+    return StringIO(tmp_data['data'])

salt/state.py

@@ -599,9 +599,9 @@ class State(object):
                             else:
                                 chunk.update(arg)
                 if names:
-                    for name in names:
+                    for low_name in names:
                         live = copy.deepcopy(chunk)
-                        live['name'] = name
+                        live['name'] = low_name
                         for fun in funcs:
                             live['fun'] = fun
                             chunks.append(live)

salt/states/cmd.py

@@ -29,6 +29,71 @@ syslog if there is no disk space:
     > /var/log/messages:
       cmd.run:
         - unless: echo 'foo' > /tmp/.test
+
+Note that when executing a command or script, the state(ie, changed or not) of
+the command is unknown to Salt's state system. Therefore, by default, the
+``cmd`` state assumes that any command execution results in a changed state. 
+
+This means that if a ``cmd`` state is watched by another state then the
+state that's watching will always be executed due to the `changed` state in
+the ``cmd`` state.
+
+Many state functions in this module now also accept a ``stateful`` argument.
+If ``stateful`` is specified to be true then it is assumed that the command
+or script will determine its own state and communicate it back by following
+a simple protocol described below:
+
+    If there's nothing in the stdout of the command, then assume no changes.
+    Otherwise, the stdout must be either in JSON or its `last` non-empty line
+    must be a string of key=value pairs delimited by spaces(no spaces on the
+    sides of ``=``).
+
+    If it's JSON then it must be a JSON object(ie, {}). 
+    If it's key=value pairs then quoting may be used to include spaces.
+    (Python's shlex module is used to parse the key=value string)
+
+    Two special keys or attributes are recognized in the output::
+
+      changed: bool (ie, 'yes', 'no', 'true', 'false', case-insensitive)
+      comment: str  (ie, any string)
+
+    So, only if 'changed' is true then assume the command execution has changed
+    the state, and any other key values or attributes in the output will be set
+    as part of the changes.
+
+    If there's a comment then it will be used as the comment of the state.
+
+    Here's an example of how one might write a shell script for use with a
+    stateful command::
+
+      #!/bin/bash
+      #
+      echo "Working hard..."
+
+      # writing the state line
+      echo  # an empty line here so the next line will be the last.
+      echo "changed=yes comment=\"something's changed!\" whatever=123"
+
+
+    And an example salt file using this module::
+
+        Run myscript:
+          cmd.run:
+            - name: /path/to/myscript
+            - cwd: /
+            - stateful: true
+        
+        Run only if myscript changed something:
+          cmd.wait:
+            - name: echo hello
+            - cwd: /
+            - watch:
+              - cmd: Run myscript
+
+    Note that if the ``cmd.wait`` state also specfies ``stateful: true``
+    it can then be watched by some other states as well.
+
+
 '''
 
 # Import python libs
@@ -524,65 +589,3 @@ def mod_watch(name, **kwargs):
                            ),
             'result': False}
 
-
-'''
-This module is similar to the built-in cmd state module except that rather
-than always resulting in a changed state, the state of a command or script
-execution is determined by the command or the script itself.
-
-This allows a state to watch for changes in another state that executes
-a command or script.
-
-This is done using a simple protocol similar to the one used by Ansible's
-modules. Here's how it works:
-
-If there's nothing in the stdout of the command, then assume no changes.
-Otherwise, the stdout must be either in JSON or its `last` non-empty line
-must be a string of key=value pairs delimited by spaces(no spaces on the
-sides of '=').
-
-If it's JSON then it must be a JSON object(ie, {}). 
-If it's key=value pairs then quoting may be used to include spaces.
-(Python's shlex module is used to parse the key=value string)
-
-Two special keys or attributes are recognized in the output::
-
-  changed: bool (ie, 'yes', 'no', 'true', 'false', case-insensitive)
-  comment: str  (ie, any string)
-
-So, only if 'changed' is true then assume the command execution has changed
-the state, and any other key values or attributes in the output will be set
-as part of the changes.
-
-If there's a comment then it will be used as the comment of the state.
-
-Here's an example of how one might write a shell script for use by this state
-function::
-
-  #!/bin/bash
-  #
-  echo "Working hard..."
-
-  # writing the state line
-  echo  # an empty line here so the next line will be the last.
-  echo "changed=yes comment=\"something's changed!\" whatever=123"
-
-
-And an example salt files using this module::
-
-    Run myscript:
-      state.run:
-        - name: /path/to/myscript
-        - cwd: /
-    
-    Run only if myscript changed something:
-      cmd.wait:
-        - name: echo hello
-        - cwd: /
-        - watch:
-          - state: Run myscript
-
-
-Note that instead of using `cmd.wait` in the example, `state.wait` can be
-used, and in which case it can then be watched by some other states.
-'''

salt/states/file.py

@@ -1652,7 +1652,7 @@ def rename(name, source, force=False, makedirs=False):
 
 def accumulated(name, filename, text, **kwargs):
     '''
-    Prepare accumulator which can me used in template in file.managed state.
+    Prepare accumulator which can be used in template in file.managed state.
     accumulator dictionary becomes available in template.
 
     name

salt/states/gem.py

@@ -22,7 +22,7 @@ def __virtual__():
     return 'gem' if 'gem.list' in __salt__ else False
 
 
-def installed(name, ruby=None, runas=None):
+def installed(name, ruby=None, runas=None, version=None, rdoc=False, ri=False):
     '''
     Make sure that a gem is installed.
 
@@ -32,9 +32,22 @@ def installed(name, ruby=None, runas=None):
         For RVM installations: the ruby version and gemset to target.
     runas : None
         The user to run gem as.
+    version : None
+        Specify the version to install for the gem.
+        Doesn't play nice with multiple gems at once
+    rdoc : False
+        Generate RDoc documentation for the gem(s).
+    ri : False
+        Generate RI documentation for the gem(s).
     '''
     ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}
-    if name in __salt__['gem.list'](name, ruby, runas=runas):
+
+    gems = __salt__['gem.list'](name, ruby, runas=runas)
+    if name in gems and version and version in gems[name]:
+        ret['result'] = True
+        ret['comment'] = 'Gem is already installed.'
+        return ret
+    elif name in gems:
         ret['result'] = True
         ret['comment'] = 'Gem is already installed.'
         return ret
@@ -42,7 +55,12 @@ def installed(name, ruby=None, runas=None):
     if __opts__['test']:
         ret['comment'] = 'The gem {0} would have been installed'.format(name)
         return ret
-    if __salt__['gem.install'](name, ruby, runas=runas):
+    if __salt__['gem.install'](name,
+                               ruby=ruby,
+                               runas=runas,
+                               version=version,
+                               rdoc=rdoc,
+                               ri=ri):
         ret['result'] = True
         ret['changes'][name] = 'Installed'
         ret['comment'] = 'Gem was successfully installed'

salt/states/pip.py

@@ -23,6 +23,7 @@ def installed(name,
               log=None,
               proxy=None,
               timeout=None,
+              repo=None,
               editable=None,
               find_links=None,
               index_url=None,
@@ -69,7 +70,7 @@ def installed(name,
         ret['comment'] = 'Error installing \'{0}\': {1}'.format(name, err)
         return ret
 
-    if ignore_installed == False and name in pip_list:
+    if ignore_installed == False and name.lower() in (p.lower() for p in pip_list):
         if force_reinstall == False and upgrade == False:
             ret['result'] = True
             ret['comment'] = 'Package already installed'
@@ -81,6 +82,9 @@ def installed(name,
                 name)
         return ret
 
+    if repo:
+        name = repo
+
     pip_install_call = __salt__['pip.install'](
         pkgs=name,
         requirements=requirements,

salt/states/rabbitmq_user.py

@@ -110,7 +110,7 @@ def absent(name,
             ret['comment'] = 'User {0} is not present'.format(name)
     else:
         if user_exists:
-            result = __salt__['rabbitmq.delete_user']
+            result = __salt__['rabbitmq.delete_user'](name, runas=runas)
             if 'Error' in result:
                 ret['result'] = False
                 ret['comment'] = result['Error']

salt/states/svn.py

@@ -100,10 +100,12 @@ def latest(name,
     ret['comment'] = out
     return ret
 
-def dirty(target,
+def dirty(name,
+          target,
           user=None,
           ignore_unversioned=False):
     '''
     Determine if the working directory has been changed.
     '''
+    ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}
     return _fail(ret, 'This function is not implemented yet.')

salt/utils/__init__.py

@@ -747,8 +747,8 @@ def mkstemp(*args, **kwargs):
     '''
     close_fd = kwargs.pop('close_fd', True)
     fd_, fpath = tempfile.mkstemp(*args, **kwargs)
-    if close_fd is True:
+    if close_fd is False:
-        os.close(fd_)
+        return (fd_, fpath)
-        del(fd_)
+    os.close(fd_)
-        return fpath
+    del(fd_)
-    return (fd_, fpath)
+    return fpath

salt/utils/jinja.py

@@ -6,8 +6,7 @@ from os import path
 import logging
 
 # Import third-party libs
-from jinja2 import (BaseLoader, Environment, StrictUndefined,
+from jinja2 import BaseLoader
-                    FileSystemLoader)
 from jinja2.exceptions import TemplateNotFound
 
 # Import Salt libs

salt/utils/mako.py

@@ -16,6 +16,11 @@ class SaltMakoTemplateLookup(TemplateCollection):
     If URL is an absolute path then it's treated as if it has been prefixed
     with salt://.
 
+    Examples::
+
+       <%include file="templates/sls-parts.mako"/>
+       <%namespace file="salt://lib/templates/utils.mako" import="helper"/>
+
     """
 
     def __init__(self, opts, env='base'):

salt/utils/parsers.py

@@ -12,7 +12,7 @@ import sys
 import logging
 import optparse
 from functools import partial
-from salt import config, log, version
+from salt import config, loader, log, version
 
 
 def _sorted(mixins_or_funcs):
@@ -297,12 +297,24 @@ class LogLevelMixIn(object):
 
     def setup_logfile_logger(self):
         lfkey = 'key_logfile' if 'key' in self.get_prog_name() else 'log_file'
+        if self.config.get('log_level_logfile', None) is None:
+            # Remove it from config so it inherits from log_level
+            self.config.pop('log_level_logfile', None)
         loglevel = self.config.get(
             'log_level_logfile', self.config['log_level']
         )
+
+        if self.config.get('log_fmt_logfile', None) is None:
+            # Remove it from config so it inherits from log_fmt_console
+            self.config.pop('log_fmt_logfile', None)
         logfmt = self.config.get(
             'log_fmt_logfile', self.config['log_fmt_console']
         )
+
+        if self.config.get('log_datefmt', None) is None:
+            # Remove it from config so it get's the default value bellow
+            self.config.pop('log_datefmt', None)
+
         datefmt = self.config.get('log_datefmt', '%Y-%m-%d %H:%M:%S')
         log.setup_logfile_logger(
             self.config[lfkey],
@@ -536,30 +548,55 @@ class OutputOptionsMixIn(object):
             '--raw-out',
             default=False,
             action='store_true',
-            help=('Print the output from the salt-key command in raw python '
+            help=('Print the output from the \'{0}\' command in raw python '
                   'form, this is suitable for re-reading the output into an '
-                  'executing python script with eval.')
+                  'executing python script with eval.'.format(
+                      self.get_prog_name()
+                  ))
         )
         group.add_option(
             '--yaml-out',
             default=False,
             action='store_true',
-            help='Print the output from the salt-key command in yaml.'
+            help='Print the output from the \'{0}\' command in yaml.'.format(
+                self.get_prog_name()
+            )
         )
         group.add_option(
             '--json-out',
             default=False,
             action='store_true',
-            help='Print the output from the salt-key command in json.'
+            help='Print the output from the \'{0}\' command in json.'.format(
+                self.get_prog_name()
+            )
         )
         if self._include_text_out_:
             group.add_option(
                 '--text-out',
                 default=False,
                 action='store_true',
-                help=('Print the output from the salt command in the same '
+                help=('Print the output from the \'{0}\' command in the same '
-                      'form the shell would.')
+                      'form the shell would.'.format(self.get_prog_name()))
             )
+
+        outputters = loader.outputters(
+            config.minion_config(
+                '/etc/salt/minion', check_dns=False
+            )
+        )
+
+        group.add_option(
+            '--out', '--output',
+            dest='output',
+            choices=outputters.keys(),
+            help=(
+                'Print the output from the \'{0}\' command using the '
+                'specified outputter. One of {1}.'.format(
+                    self.get_prog_name(),
+                    ', '.join([repr(k) for k in outputters])
+                )
+            )
+        )
         group.add_option(
             '--no-color',
             default=False,
@@ -567,19 +604,50 @@ class OutputOptionsMixIn(object):
             help='Disable all colored output'
         )
 
-        for option in group.option_list:
+        for option in self.output_options_group.option_list:
             def process(opt):
-                if getattr(self.options, opt.dest):
+                default = self.defaults.get(opt.dest)
-                    self.selected_output_option = opt.dest
+                if getattr(self.options, opt.dest, default) is False:
+                    return
+
+                # XXX: CLEAN THIS CODE WHEN 0.10.8 is about to come out
+                if version.__version_info__ >= (0, 10, 7):
+                    self.error(
+                        'The option {0} is deprecated. You must now use '
+                        '\'--out {1}\' instead.'.format(
+                            opt.get_opt_string(),
+                            opt.dest.split('_', 1)[0]
+                        )
+                    )
+
+                if opt.dest != 'out':
+                    msg = (
+                        'The option {0} is deprecated. Please consider using '
+                        '\'--out {1}\' instead.'.format(
+                            opt.get_opt_string(),
+                            opt.dest.split('_', 1)[0]
+                        )
+                    )
+                    if log.is_console_configured():
+                        logging.getLogger(__name__).warning(msg)
+                    else:
+                        sys.stdout.write('WARNING: {0}\n'.format(msg))
+
+                self.selected_output_option = opt.dest
 
             funcname = 'process_{0}'.format(option.dest)
             if not hasattr(self, funcname):
                 setattr(self, funcname, partial(process, option))
 
+    def process_output(self):
+        self.selected_output_option = self.options.output
+
     def _mixin_after_parsed(self):
         group_options_selected = filter(
-            lambda option: getattr(self.options, option.dest) and
+            lambda option: (
-                           option.dest.endswith('_out'),
+                getattr(self.options, option.dest) and
+                (option.dest.endswith('_out') or option.dest=='output')
+            ),
             self.output_options_group.option_list
         )
         if len(group_options_selected) > 1:

salt/utils/process.py

@@ -25,7 +25,7 @@ def set_pidfile(pidfile, user):
         if os.environ['os'].startswith('Windows'):
             return True
     import pwd  # after confirming not running Windows
-    import grp
+    #import grp
     try:
         pwnam = pwd.getpwnam(user)
         uid = pwnam[2]

salt/utils/templates.py

@@ -45,6 +45,7 @@ def wrap_tmpl_func(render_str):
                     tmplstr = tmplsrc.read()
         else:  # assume tmplsrc is file-like.
             tmplstr = tmplsrc.read()
+            tmplsrc.close()
         try:
             output = render_str(tmplstr, context)
         except SaltTemplateRenderError, exc:

salt/version.py

@@ -1,24 +1,39 @@
+'''
+Set up the version of Salt
+'''
+
+# Import Python libs
 import sys
 
-__version_info__ = (0, 10, 4)
+__version_info__ = (0, 10, 5)
 __version__ = '.'.join(map(str, __version_info__))
 
+
 # If we can get a version from Git use that instead, otherwise carry on
 try:
+    import os
     import subprocess
     from salt.utils import which
 
     git = which('git')
     if git:
-        p = subprocess.Popen([git, 'describe'],
+        p = subprocess.Popen(
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+            [git, 'describe'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            close_fds=True,
+            cwd=os.path.abspath(os.path.dirname(__file__))
+        )
         out, err = p.communicate()
         if out:
             __version__ = '{0}'.format(out.strip().lstrip('v'))
-            __version_info__ = tuple(__version__.split('-', 1)[0].split('.'))
+            __version_info__ = tuple(
+                [int(i) for i in __version__.split('-', 1)[0].split('.')]
+            )
 except Exception:
     pass
 
+
 def versions_report():
     libs = (
         ("Jinja2", "jinja2", "__version__"),

tests/integration/files/conf/minion

@@ -19,3 +19,5 @@ integration.test: True
 # Grains addons
 grains:
   test_grain: cheese
+  script: grail
+  alot: many

tests/integration/modules/cp.py

@@ -57,7 +57,7 @@ class CPModuleTest(integration.ModuleCase):
                 'salt://file.big',
                 tgt,
             ],
-            gzip_compression=5
+            gzip=5
         )
         with open(tgt, 'r') as scene:
             data = scene.read()
@@ -65,6 +65,24 @@ class CPModuleTest(integration.ModuleCase):
             self.assertNotIn('bacon', data)
             self.assertEqual(hash, hashlib.md5(data).hexdigest())
 
+    def test_get_file_makedirs(self):
+        '''
+        cp.get_file
+        '''
+        tgt = os.path.join(integration.TMP, 'make/dirs/scene33')
+        self.run_function(
+            'cp.get_file',
+            [
+                'salt://grail/scene33',
+                tgt,
+            ],
+            makedirs=True
+        )
+        with open(tgt, 'r') as scene:
+            data = scene.read()
+            self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
+            self.assertNotIn('bacon', data)
+
     def test_get_template(self):
         '''
         cp.get_template
@@ -98,6 +116,23 @@ class CPModuleTest(integration.ModuleCase):
         self.assertIn('empty', os.listdir(os.path.join(tgt, 'grail')))
         self.assertIn('scene', os.listdir(os.path.join(tgt, 'grail', '36')))
 
+    def test_get_dir_templated_paths(self):
+        '''
+        cp.get_dir
+        '''
+        tgt = os.path.join(integration.TMP, 'many')
+        self.run_function(
+            'cp.get_dir',
+            [
+                'salt://{{grains.script}}',
+                tgt.replace('many', '{{grains.alot}}')
+            ]
+        )
+        self.assertIn('grail', os.listdir(tgt))
+        self.assertIn('36', os.listdir(os.path.join(tgt, 'grail')))
+        self.assertIn('empty', os.listdir(os.path.join(tgt, 'grail')))
+        self.assertIn('scene', os.listdir(os.path.join(tgt, 'grail', '36')))
+
     def test_get_url(self):
         '''
         cp.get_url

tests/integration/shell/call.py

@@ -10,6 +10,7 @@
 import sys
 
 # Import salt libs
+from salt import version
 from saltunittest import TestLoader, TextTestRunner, skipIf
 import integration
 from integration import TestDaemon
@@ -30,9 +31,19 @@ class CallTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
 
     def test_text_output(self):
         out = self.run_call('--text-out test.fib 3')
-        self.assertEqual(
+        if version.__version_info__ < (0, 10, 8):
-            'local: ([0, 1, 1, 2]', ''.join(out).rsplit(",", 1)[0]
+            expect = [
-        )
+                "WARNING: The option --text-out is deprecated. Please "
+                "consider using '--out text' instead."
+            ]
+        else:
+            expect = []
+
+        expect += [
+            'local: ([0, 1, 1, 2]'
+        ]
+
+        self.assertEqual(''.join(expect), ''.join(out).rsplit(",", 1)[0])
 
     @skipIf(sys.platform.startswith('win'), 'This test does not apply on Win')
     def test_user_delete_kw_output(self):

tests/integration/shell/key.py

@@ -5,6 +5,7 @@ import shutil
 import tempfile
 
 # Import salt libs
+from salt import version
 from saltunittest import TestLoader, TextTestRunner
 import integration
 from integration import TestDaemon
@@ -37,8 +38,15 @@ class KeyTest(integration.ShellCase,
         test salt-key -L --json-out
         '''
         data = self.run_key('-L --json-out')
+        if version.__version_info__ < (0, 10, 8):
+            expect = [
+                "WARNING: The option --json-out is deprecated. Please "
+                "consider using '--out json' instead."
+            ]
+        else:
+            expect = []
 
-        expect = [
+        expect += [
             '{',
             '    "minions_rejected": [], ',
             '    "minions_pre": [], ',
@@ -55,7 +63,15 @@ class KeyTest(integration.ShellCase,
         test salt-key -L --yaml-out
         '''
         data = self.run_key('-L --yaml-out')
-        expect = [
+        if version.__version_info__ < (0, 10, 8):
+            expect = [
+                "WARNING: The option --yaml-out is deprecated. Please "
+                "consider using '--out yaml' instead."
+            ]
+        else:
+            expect = []
+
+        expect += [
             'minions:',
             '- minion',
             '- sub_minion',
@@ -69,7 +85,15 @@ class KeyTest(integration.ShellCase,
         test salt-key -L --raw-out
         '''
         data = self.run_key('-L --raw-out')
-        expect = [
+        if version.__version_info__ < (0, 10, 8):
+            expect = [
+                "WARNING: The option --raw-out is deprecated. Please "
+                "consider using '--out raw' instead."
+            ]
+        else:
+            expect = []
+
+        expect += [
             "{'minions_rejected': [], 'minions_pre': [], "
             "'minions': ['minion', 'sub_minion']}"
         ]

tests/unit/states/gem_test.py

@@ -19,7 +19,7 @@ gem.__opts__ = {'test': False}
 class TestGemState(TestCase):
 
     def test_installed(self):
-        gems = ['foo', 'bar']
+        gems = {'foo' : ['1.0'], 'bar' : ['2.0']}
         gem_list = MagicMock(return_value=gems)
         gem_install_succeeds = MagicMock(return_value=True)
         gem_install_fails = MagicMock(return_value=False)
@@ -32,14 +32,14 @@ class TestGemState(TestCase):
                 ret = gem.installed('quux')
                 self.assertEqual(True, ret['result'])
                 gem_install_succeeds.assert_called_once_with(
-                    'quux', None, runas=None)
+                    'quux', ruby=None, runas=None, version=None, rdoc=False, ri=False)
 
             with patch.dict(gem.__salt__,
                             {'gem.install': gem_install_fails}):
                 ret = gem.installed('quux')
                 self.assertEqual(False, ret['result'])
                 gem_install_fails.assert_called_once_with(
-                    'quux', None, runas=None)
+                    'quux', ruby=None, runas=None, version=None, rdoc=False, ri=False)
 
     def test_removed(self):
         gems = ['foo', 'bar']