Merge branch 'master' of https://github.com/saltstack/salt into develop

Conflicts:
	doc/ref/renderers/all/index.rst

.travis.yml

@@ -9,15 +9,15 @@ branches:
     - develop
 
 before_install:
-  - sudo apt-get update && sudo apt-get install swig
+  - sudo apt-get update && sudo apt-get install swig supervisor
   - pip install http://dl.dropbox.com/u/174789/m2crypto-0.20.1.tar.gz
   - "if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi"
 
 install: pip install -r requirements.txt --use-mirrors
-script: python setup.py test
+script: sudo -E python setup.py test --runtests-opts='--run-destructive'
 
 notifications:
   irc:
-    channels: "irc.freenode.org#salt"
+    channels: "irc.freenode.org#salt-devel"
     on_success: change
     on_failure: change

AUTHORS

@@ -40,10 +40,12 @@ Eric Poelke <epoelke@gmail.com>
 Erik Nolte <enolte@beyondoblivion.com>
 Evan Borgstrom <evan@fatbox.ca>
 Jed Glazner <jglazner@coldcrow.com>
+Jeff Bauer <jbauer@rubic.com>
 Jeffrey C. Ollie <jeff@ocjtech.us>
 Jeff Schroeder <jeffschroeder@computer.org>
 Jonas Buckner <buckner.jonas@gmail.com>
 Joseph Hall <perlhoser@gmail.com>
+Josmar Dias <josmarnet@gmail.com>
 Kent Tenney <ktenney@gmail.com>
 Marc Abramowitz <marc+github@marc-abramowitz.com>
 Markus Gattol <markus.gattol@sunoano.org>
@@ -56,6 +58,7 @@ Nathaniel Whiteinge <seth@eseth.com>
 Nigel Owen <nigelowen2.gmail.com>
 Pedro Algarvio <pedro@algarvio.me>
 Pierre Carrier <pierre@spotify.com>
+Rhys Elsmore <me@rhys.io>
 Seth House <seth@eseth.com>
 Seth Vidal <skvidal@fedoraproject.org>
 Thomas Schreiber <tom@rizumu.us>

HACKING.rst

@@ -63,6 +63,11 @@ Create a new `virtualenv`_::
 
     virtualenv /path/to/your/virtualenv
 
+.. note:: site packages
+
+    If you wish to use installed packages rather than have pip download and
+    compile new ones into this environment, add "--system-site-packages".
+
 .. _`virtualenv`: http://pypi.python.org/pypi/virtualenv
 
 Activate the virtualenv::
@@ -75,11 +80,18 @@ Install Salt (and dependencies) into the virtualenv::
 
 .. note:: Installing M2Crypto
 
-    If you and encounter the error ``command 'swig' failed with exit status 1``
+    You may need ``swig`` and ``libssl-dev`` to build M2Crypto. If you 
+    encounter the error ``command 'swig' failed with exit status 1``
     while installing M2Crypto, try installing it with the following command::
 
         env SWIG_FEATURES="-cpperraswarn -includeall -D__`uname -m`__ -I/usr/include/openssl" pip install M2Crypto
 
+    Debian and Ubuntu systems have modified openssl libraries and mandate that
+    a patched version of M2Crypto be installed. This means that M2Crypto
+    needs to be installed via apt:
+
+        apt-get install python-m2crypto
+
 Running a self-contained development version
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -110,14 +122,21 @@ Edit the minion config file:
     "saltdev". This isn't strictly necessary but it will serve as a reminder of
     which Salt installation you are working with.
 
+.. note:: Using `salt-call` with a :doc:`Standalone Minion </topics/tutorials/standalone_minion>`
+
+    If you plan to run `salt-call` with this self-contained development
+    environment in a masterless setup, you should invoke `salt-call` with
+    ``-c /path/to/your/virtualenv/etc/salt`` so that salt can find the minion
+    config file. Without the ``-c`` option, Salt finds its config files in `/etc/salt`.
+
 Start the master and minion, accept the minon's key, and verify your local Salt
 installation is working::
 
-    salt-master -c ./etc/salt/master -d
+    salt-master -c ./etc/salt -d
-    salt-minion -c ./etc/salt/minion -d
+    salt-minion -c ./etc/salt -d
-    salt-key -c ./etc/salt/master -L
+    salt-key -c ./etc/salt -L
-    salt-key -c ./etc/salt/master -A
+    salt-key -c ./etc/salt -A
-    salt -c ./etc/salt/master '*' test.ping
+    salt -c ./etc/salt '*' test.ping
 
 File descriptor limit
 ~~~~~~~~~~~~~~~~~~~~~
@@ -126,9 +145,11 @@ Check your file descriptor limit with::
 
     ulimit -n
 
-If it is less than 1024, you should increase it with::
+If it is less than 2047, you should increase it with::
+
+    ulimit -n 2047
+    (or "limit descriptors 2047" for c-shell)
 
-    ulimit -n 1024
 
 Running the tests
 ~~~~~~~~~~~~~~~~~
@@ -144,3 +165,7 @@ If you are on Python < 2.7 then you will also need unittest2::
 Finally you use setup.py to run the tests with the following command::
 
     ./setup.py test
+
+For greater control while running the tests, please try::
+
+	./tests/runtests.py -h

MANIFEST.in

@@ -1,4 +1,5 @@
 include AUTHORS
+include HACKING.rst
 include LICENSE
 include README.rst
 include requirements.txt

conf/master.template

@@ -9,22 +9,50 @@
 # The address of the interface to bind to
 #interface: 0.0.0.0
 
-# The port used by the publisher
+# The tcp port used by the publisher
 #publish_port: 4505
 
-# The user to run salt
+# Refresh the publisher connections when sending out commands, this is a fix
+# for zeromq losing some minion connections. Default: True
+#pub_refresh: True
+
+# The user to run the salt-master as. Salt will update all permissions to
+# allow the specified user to run the master. If the modified files cause
+# conflicts set verify_env to False.
 #user: root
 
+# Max open files
+# Each minion connecting to the master uses AT LEAST one file descriptor, the
+# master subscription connection. If enough minions connect you might start
+# seeing on the console(and then salt-master crashes):
+#   Too many open files (tcp_listener.cpp:335)
+#   Aborted (core dumped)
+#
+# By default this value will be the one of `ulimit -Hn`, ie, the hard limit for
+# max open files.
+#
+# If you wish to set a different value than the default one, uncomment and
+# configure this setting. Remember that this value CANNOT be higher than the
+# hard limit. Raising the hard limit depends on your OS and/or distribution,
+# a good way to find the limit is to search the internet for(for example):
+#   raise max open files hard limit debian
+#
+#max_open_files: 100000
+
 # The number of worker threads to start, these threads are used to manage
 # return calls made from minions to the master, if the master seems to be
 # running slowly, increase the number of threads
 #worker_threads: 5
 
-# The port used by the communication interface
+# The port used by the communication interface. The ret (return) port is the
+# interface used for the file server, authentication, job returnes, etc.
 #ret_port: 4506
 
+# Specify the location of the daemon process ID file
+#pidfile: /var/run/salt-master.pid
+
 # The root directory prepended to these options: pki_dir, cachedir,
-# sock_dir, log_file, autosign_file.
+# sock_dir, log_file, autosign_file, extension_modules
 #root_dir: /
 
 # Directory used to store public key data
@@ -33,7 +61,10 @@
 # Directory to store job and cache data
 #cachedir: /var/cache/salt
 
-# Set the number of hours to keep old job information
+# Verify and set permissions on configuration directories at startup
+#verify_env: True
+
+# Set the number of hours to keep old job information in the job cache
 #keep_jobs: 24
 
 # Set the default timeout for the salt command and api, the default is 5
@@ -41,7 +72,7 @@
 #timeout: 5
 
 # Set the directory used to hold unix sockets
-#sock_dir: /tmp/salt-unix
+#sock_dir: /var/run/salt
 
 # The master maintains a job cache, while this is a great addition it can be
 # a burden on the master for larger deployments (over 5000 minions).
@@ -50,6 +81,9 @@
 #
 #job_cache: True
 
+# Cache minion grains and pillar data in the cachedir.
+#minion_data_cache: True
+
 # Set the acceptance level for serialization of messages. This should only be
 # set if the master is newer than 0.9.5 and the minion are older. This option
 # allows a 0.9.5 and newer master to communicate with minions 0.9.4 and
@@ -100,6 +134,16 @@
 # If an autosign_file is specified permissive access will allow group access
 # to that specific file.
 #permissive_pki_access: False
+#
+# Allow users on the master access to execute specific commands on minions.
+# This setting should be treated with care since it opens up execution
+# capabilities to non root users. By default this capability is completely
+# disabled.
+#
+# client_acl:
+#   larry:
+#     - test.ping
+#     - network.*
 
 
 #####    Master Module Management    #####
@@ -109,8 +153,9 @@
 # Add any additional locations to look for master runners
 #runner_dirs: []
 #
-#Enable Cython for master side modules
+# Enable Cython for master side modules
 #cython_enable: False
+#
 
 #####      State System settings     #####
 ##########################################
@@ -131,6 +176,17 @@
 # The failhard option tells the minions to stop immediately after the first
 # failure detected in the state execution, defaults to False
 #failhard: False
+#
+# The state_verbose and state_output settings can be used to change the way
+# state system data is printed to the display. By default all data is printed.
+# The state_verbose setting can be set to True or False, when set to False
+# all data that has a result of True and no changes will be suppressed.
+#state_verbose: True
+# 
+# The state_output setting changes if the output is the full multi line
+# output for each changed state if set to 'full', but if set to 'terse'
+# the output will be shortened to a single line.
+#state_output: full
 
 #####      File Server settings      #####
 ##########################################
@@ -180,7 +236,7 @@
 #
 #ext_pillar:
 #  - hiera: /etc/hiera.yaml
-#  - cmd: cat /etc/salt/yaml
+#  - cmd_yaml: cat /etc/salt/yaml
 #
 
 #####          Syndic settings       #####
@@ -242,29 +298,11 @@
 #     - manage.up
 #
 
-#####         Cluster settings       #####
-##########################################
-# Salt supports automatic clustering, salt creates a single ip address which
-# is shared among the individual salt components using ucarp. The private key
-# and all of the minion keys are maintained across the defined cluster masters.
-# The failover service is automatically managed via these settings
-
-# List the identifiers for the other cluster masters in this manner:
-# [saltmaster-01.foo.com,saltmaster-02.foo.com,saltmaster-03.foo.com]
-# The members of this master array must be running as salt minions to
-# facilitate the distribution of cluster information
-#cluster_masters: []
-
-# The cluster modes are "paranoid" and "full"
-# paranoid will only distribute the accepted minion public keys.
-# full will also distribute the master private key.
-#cluster_mode: paranoid
-
-
 #####         Logging settings       #####
 ##########################################
 # The location of the master log file
 #log_file: /var/log/salt/master
+#key_logfile: /var/log/salt/key
 #
 # The level of messages to send to the log file.
 # One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'.
@@ -275,7 +313,7 @@
 # The date and time format used in log messages. Allowed date/time formating
 # can be seen here:
 #	http://docs.python.org/library/time.html#time.strftime
-#log_datefmt: '%H:%M:%S'
+#log_datefmt: '%Y-%m-%d %H:%M:%S'
 #
 # The format of the console logging messages. Allowed formatting options can
 # be seen here:

conf/minion.template

@@ -16,6 +16,9 @@
 # The user to run salt
 #user: root
 
+# Specify the location of the daemon process ID file
+#pidfile: /var/run/salt-minion.pid
+
 # The root directory prepended to these options: pki_dir, cachedir, log_file.
 #root_dir: /
 
@@ -34,6 +37,17 @@
 # FQDN (for instance, Solaris).
 #append_domain:
 
+# Custom static grains for this minion can be specified here and used in SLS
+# files just like all other grains. This example sets 4 custom grains, with
+# the 'roles' grain having two values that can be matched against:
+#grains:
+#  roles:
+#    - webserver
+#    - memcache
+#  deployment: datacenter4
+#  cabinet: 13
+#  cab_u: 14-15
+
 # If the connection to the server is interrupted, the minion will
 # attempt to reconnect. sub_timeout allows you to control the rate
 # of reconnection attempts (in seconds). To disable reconnects, set
@@ -43,22 +57,48 @@
 # Where cache data goes
 #cachedir: /var/cache/salt
 
+# Verify and set permissions on configuration directories at startup
+#verify_env: True
+
 # The minion can locally cache the return data from jobs sent to it, this
 # can be a good way to keep track of jobs the minion has executed
 # (on the minion side). By default this feature is disabled, to enable
 # set cache_jobs to True
 #cache_jobs: False
 
+# set the directory used to hold unix sockets
+#sock_dir: /var/run/salt
+
+# Backup files that are replaced by file.managed and file.recurse under
+# 'cachedir'/file_backups relative to their original location and appended
+# with a timestamp. The only valid setting is "minion". Disabled by default.
+#
+# Alternatively this can be specified for each file in state files:
+#
+# /etc/ssh/sshd_config:
+#   file.managed:
+#     - source: salt://ssh/sshd_config
+#       - backup: minion
+#
+#backup_mode: minion
+
 # When waiting for a master to accept the minion's public key, salt will
 # continuously attempt to reconnect until successful. This is the time, in
 # seconds, between those reconnection attempts.
-#acceptance_wait_time = 10
+#acceptance_wait_time: 10
 
 # When healing a dns_check is run, this is to make sure that the originally
 # resolved dns has not changed, if this is something that does not happen in
 # your environment then set this value to False.
 #dns_check: True
 
+# Windows platforms lack posix IPC and must rely on slower TCP based inter-
+# process communications. Set ipc_mode to 'tcp' on such systems
+#ipc_mode: ipc
+#
+# Overwrite the default tcp ports used by the minion when in tcp mode
+#tcp_pub_port: 4510
+#tcp_pull_port: 4511
 
 # The minion can include configuration from other files. To enable this,
 # pass a list of paths to this option. The paths can be either relative or
@@ -69,12 +109,12 @@
 #
 #
 # Include a config file from some other path:
-#include: /etc/salt/extra_config
+# include: /etc/salt/extra_config
 #
 # Include config from several files and directories:
-#include:
+# include:
-# - /etc/salt/extra_config
+#  - /etc/salt/extra_config
-# - /etc/roles/webserver
+#  - /etc/roles/webserver
 
 #####   Minion module management     #####
 ##########################################
@@ -102,6 +142,7 @@
 #
 # Enable Cython modules searching and loading. (Default: False)
 #cython_enable: False
+#
 
 #####    State Management Settings    #####
 ###########################################
@@ -119,6 +160,10 @@
 #
 #renderer: yaml_jinja
 #
+# The failhard option tells the minions to stop immediately after the first
+# failure detected in the state execution, defaults to False
+#failhard: False
+#
 # state_verbose allows for the data returned from the minion to be more
 # verbose. Normally only states that fail or states that have changes are
 # returned, but setting state_verbose to True will return all states that
@@ -145,6 +190,20 @@
 # If using the local file directory, then the state top file name needs to be
 # defined, by default this is top.sls.
 #state_top: top.sls
+#
+# Run states when the minion daemon starts. To enable, set startup_states to:
+# 'highstate' -- Execute state.highstate
+# 'sls' -- Read in the sls_list option and execute the named sls files
+# 'top' -- Read top_file option and execute based on that file on the Master 
+#startup_states: ''
+#
+# list of states to run when the minion starts up if startup_states is 'sls'
+#sls_list: 
+#  - edit.vim
+#  - hyper
+#
+# top file to execute if startup_states is 'top'
+#top_file: ''
 
 #####     File Directory Settings    #####
 ##########################################
@@ -204,6 +263,21 @@
 # you've given access to. This is potentially quite insecure.
 #permissive_pki_access: False
 
+# The state_verbose and state_output settings can be used to change the way
+# state system data is printed to the display. By default all data is printed.
+# The state_verbose setting can be set to True or False, when set to False
+# all data that has a result of True and no changes will be suppressed.
+#state_verbose: True
+#
+# The state_output setting changes if the output is the full multi line
+# output for each changed state if set to 'full', but if set to 'terse'
+# the output will be shortened to a single line.
+#state_output: full
+#
+# Fingerprint of the master public key to double verify the master is valid,
+# the master fingerprint can be found by running "salt-key -F master" on the
+# salt master.
+#master_finger: ''
 
 ######         Thread settings        #####
 ###########################################
@@ -224,7 +298,7 @@
 #
 # The date and time format used in log messages. Allowed date/time formating
 # can be seen on http://docs.python.org/library/time.html#time.strftime
-#log_datefmt: '%H:%M:%S'
+#log_datefmt: '%Y-%m-%d %H:%M:%S'
 #
 # The format of the console logging messages. Allowed formatting options can
 # be seen on http://docs.python.org/library/logging.html#logrecord-attributes
@@ -249,6 +323,9 @@
 # the module name is followed by a . and then the value. Also, all top level
 # data must be applied via the yaml dict construct, some examples:
 #
+# You can specify that all modules should run in test mode:
+#test: True
+#
 # A simple value for the test module:
 #test.foo: foo
 #
@@ -257,3 +334,16 @@
 #
 # A dict for the test module:
 #test.baz: {spam: sausage, cheese: bread}
+
+
+######      Update settings          ######
+###########################################
+# Using the features in Esky, a salt minion can both run as a frozen app and
+# be updated on the fly. These options control how the update process
+# (saltutil.update()) behaves.
+#
+# The url for finding and downloading updates. Disabled by default.
+#update_url: False
+#
+# The list of services to restart after a successful update. Empty by default.
+#update_restart_services: []

debian/changelog

@@ -1,3 +1,16 @@
+salt (0.10.3) precise; urgency=low
+
+  * New upstream version
+
+ -- Thomas S Hatch <thatch@saltstack.com>  Sun, 30 Aug 2012 13:34:10 -0700
+
+salt (0.10.2) precise; urgency=low
+
+  * Non-maintainer upload.
+  * New upstream version
+
+ -- Dave Rawks <drawks@mint>  Wed, 01 Aug 2012 13:34:10 -0700
+
 salt (0.9.9) precise; urgency=low
 
   * New upstream version

debian/control

@@ -10,7 +10,8 @@ Build-Depends: debhelper (>= 8),
                python-m2crypto,
                python-zmq (>= 2.1.9),
                libzmq-dev (>= 2.1.9),
-               python-jinja2
+               python-jinja2,
+               msgpack-python
 Standards-Version: 3.9.3
 Homepage: http://saltstack.org
 #Vcs-Git: git://git.debian.org/collab-maint/salt.git

debian/salt-master.upstart

@@ -5,7 +5,7 @@ start on (net-device-up
           and runlevel [2345])
 stop on runlevel [!2345]
 
-expect daemon
+respawn limit 10 5
 respawn
 
-exec /usr/bin/salt-master -d
+exec /usr/bin/salt-master >/dev/null 2>&1

debian/salt-minion.upstart

@@ -6,7 +6,6 @@ start on (net-device-up
 stop on runlevel [!2345]
 
 respawn limit 10 5
+respawn
 
-script
+exec /usr/bin/salt-minion >/dev/null 2>&1
-  exec /usr/bin/python /usr/bin/salt-minion > /dev/null 2>&1
-end script

debian/salt-syndic.upstart

@@ -6,7 +6,6 @@ start on (net-device-up
 stop on runlevel [!2345]
 
 respawn limit 10 5
+respawn
 
-script
+exec /usr/bin/salt-syndic >/dev/null 2>&1
-  exec /usr/bin/python /usr/bin/salt-syndic > /dev/null 2>&1
-end script

doc/_templates/page.html

@@ -4,3 +4,24 @@
     <li><a href="http://saltstack.org">&laquo; SaltStack.org</a>&nbsp;|&nbsp;</li>
     <li><a href="{{ pathto('index') }}">Documentation home</a></li>
 {%- endblock %}
+
+{% block body %}
+{{ super() }}
+<h2>Comments</h2>
+<div id="disqus_thread"></div>
+<noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
+<a href="http://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>
+{% endblock %}
+
+{% block footer %}
+{{ super() }}
+<script type="text/javascript">
+    var disqus_shortname = 'saltstack';
+
+    (function() {
+        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+        dsq.src = 'http://' + disqus_shortname + '.disqus.com/embed.js';
+        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+    })();
+</script>
+{% endblock %}

doc/conf.py

@@ -107,7 +107,7 @@ rst_prolog = """\
 
 # A shortcut for linking to tickets on the GitHub issue tracker
 extlinks = {
-    'blob': ('https://github.com/saltstack/salt/blob/v%s/%%s' % __version__, None),
+    'blob': ('https://github.com/saltstack/salt/blob/%s/%%s' % 'develop', None),
     'download': ('https://github.com/downloads/saltstack/salt/%s', None),
     'issue': ('https://github.com/saltstack/salt/issues/%s', 'issue '),
 }

doc/contents.rst

@@ -24,7 +24,6 @@ Full Table of Contents
     topics/troubleshooting/index
     topics/troubleshooting/yaml_idiosyncrasies
     topics/community
-    topics/tutorials/standalone_minion
     topics/projects/index
     topics/event/index
 
@@ -37,8 +36,11 @@ Full Table of Contents
     ref/states/all/index
     ref/renderers/*
     ref/renderers/all/index
+    ref/pillar/*
+    ref/pillar/all/index
     ref/runners
     ref/peer
+    ref/clientacl
     ref/syndic
     ref/python-api
     ref/file_server/index

doc/index.rst

@@ -201,6 +201,11 @@ Salt is many splendid things.
     Use Salt programmatically from your own scripts and programs easily and
     simply via ``import salt``.
 
+:doc:`Automatic Updates and Frozen Deployments <ref/esky>`
+    Use a frozen install to make deployments easier (Even on Windows!). Or
+    take advantage of automatic updates to keep your minions running your
+    latest builds.
+
 Reference
 ---------
 

doc/man/salt-call.1

@@ -1,4 +1,4 @@
-.TH "SALT-CALL" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT-CALL" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt-call \- salt-call Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .SH SYNOPSIS
 .sp
@@ -37,6 +37,10 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 salt\-call [options]
 .ft P
 .fi
+.SH DESCRIPTION
+.sp
+The salt\-call command is used to run module functions locally on a minion
+instead of executing them from the master.
 .SH OPTIONS
 .INDENT 0.0
 .TP
@@ -105,4 +109,5 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/man/salt-cp.1

@@ -1,4 +1,4 @@
-.TH "SALT-CP" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT-CP" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt-cp \- salt-cp Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .sp
 Copy a file to a set of systems
@@ -120,4 +120,5 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/man/salt-key.1

@@ -1,4 +1,4 @@
-.TH "SALT-KEY" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT-KEY" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt-key \- salt-key Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .SH SYNOPSIS
 .sp
@@ -81,8 +81,8 @@ Delete the named minion key for command execution.
 .UNINDENT
 .INDENT 0.0
 .TP
-.B \-D DELETE_ALL, \-\-delete\-all=DELETE_ALL
+.B \-D, \-\-delete\-all
-Deleta all keys
+Delete all keys
 .UNINDENT
 .INDENT 0.0
 .TP
@@ -91,9 +91,54 @@ The master configuration file needs to be read to determine where the Salt
 keys are stored via the pki_dir configuration value;
 default=/etc/salt/master
 .UNINDENT
+.INDENT 0.0
+.TP
+.B \-p PRINT, \-\-print=PRINT
+Print the specified public key
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-P, \-\-print\-all
+Print all public keys
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-q, \-\-quiet
+Supress output
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-y, \-\-yes
+Answer \(aqYes\(aq to all questions presented, defaults to False
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-key\-logfile=KEY_LOGFILE
+Send all output to a file. Default is /var/log/salt/key
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-gen\-keys=GEN_KEYS
+Set a name to generate a keypair for use with salt
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-gen\-keys\-dir=GEN_KEYS_DIR
+Set the directory to save the generated keypair.  Only works
+with \(aqgen_keys_dir\(aq option; default is the current directory.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-keysize=KEYSIZE
+Set the keysize for the generated key, only works with
+the \(aq\-\-gen\-keys\(aq option, the key size must be 2048 or
+higher, otherwise it will be rounded up to 2048. The
+default is 2048.
+.UNINDENT
 .SH AUTHOR
 Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors file
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/man/salt-master.1

@@ -1,4 +1,4 @@
-.TH "SALT-MASTER" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT-MASTER" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt-master \- salt-master Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .sp
 The Salt master daemon, used to control the Salt minions
@@ -81,4 +81,5 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/man/salt-minion.1

@@ -1,4 +1,4 @@
-.TH "SALT-MINION" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT-MINION" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt-minion \- salt-minion Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .sp
 The Salt minion daemon, receives commands from a remote Salt master.
@@ -82,4 +82,5 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/man/salt-run.1

@@ -1,4 +1,4 @@
-.TH "SALT-RUN" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT-RUN" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt-run \- salt-run Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .sp
 Execute a Salt runner
@@ -67,4 +67,5 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/man/salt-syndic.1

@@ -1,4 +1,4 @@
-.TH "SALT-SYNDIC" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT-SYNDIC" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt-syndic \- salt-syndic Documentation
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .sp
 The Salt syndic daemon, a special minion that passes through commands from a
@@ -76,4 +76,5 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/man/salt.1

@@ -1,4 +1,4 @@
-.TH "SALT" "1" "July 27, 2012" "0.10.2" "Salt"
+.TH "SALT" "1" "September 30, 2012" "0.10.3" "Salt"
 .SH NAME
 salt \- salt
 .
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.\" Man page generated from reStructuredText.
+.\" Man page generated from reStructeredText.
 .
 .SH SYNOPSIS
 .INDENT 0.0
@@ -145,7 +145,7 @@ file.
 .TP
 .B \-\-return
 Chose an alternative returner to call on the minion, if an alternative
-returner is used then the return will not come back tot he command line
+returner is used then the return will not come back to the command line
 but will be sent to the specified return system.
 .UNINDENT
 .INDENT 0.0
@@ -209,4 +209,5 @@ Thomas S. Hatch <thatch45@gmail.com> and many others, please see the Authors fil
 .SH COPYRIGHT
 2012, Thomas S. Hatch
 .\" Generated by docutils manpage writer.
+.\" 
 .

doc/ref/cli/salt-call.rst

@@ -9,6 +9,12 @@ Synopsis
 
     salt-call [options]
 
+Description
+===========
+
+The salt-call command is used to run module functions locally on a minion 
+instead of executing them from the master.
+
 Options
 =======
 

doc/ref/cli/salt-key.rst

@@ -51,12 +51,48 @@ Options
 
     Delete the named minion key for command execution.
 
-.. option:: -D DELETE_ALL, --delete-all=DELETE_ALL
+.. option:: -D, --delete-all
 
-    Deleta all keys
+    Delete all keys
 
 .. option:: -c CONFIG, --config=CONFIG
 
     The master configuration file needs to be read to determine where the Salt
     keys are stored via the pki_dir configuration value;
     default=/etc/salt/master
+
+.. option:: -p PRINT, --print=PRINT
+
+   Print the specified public key
+
+.. option:: -P, --print-all
+
+   Print all public keys
+
+.. option:: -q, --quiet
+
+   Supress output
+
+.. option:: -y, --yes
+
+   Answer 'Yes' to all questions presented, defaults to False
+
+.. option:: --key-logfile=KEY_LOGFILE
+
+   Send all output to a file. Default is /var/log/salt/key
+
+.. option:: --gen-keys=GEN_KEYS
+
+   Set a name to generate a keypair for use with salt
+
+.. option:: --gen-keys-dir=GEN_KEYS_DIR
+
+   Set the directory to save the generated keypair.  Only works
+   with 'gen_keys_dir' option; default is the current directory.
+
+.. option:: --keysize=KEYSIZE
+
+   Set the keysize for the generated key, only works with
+   the '--gen-keys' option, the key size must be 2048 or
+   higher, otherwise it will be rounded up to 2048. The
+   default is 2048.

doc/ref/cli/salt.rst

@@ -108,7 +108,7 @@ Options
 .. option:: --return
 
     Chose an alternative returner to call on the minion, if an alternative
-    returner is used then the return will not come back tot he command line
+    returner is used then the return will not come back to the command line
     but will be sent to the specified return system.
 
 .. option:: -Q, --query

doc/ref/clientacl.rst

@@ -0,0 +1,40 @@
+=================
+Client ACL system
+=================
+
+The salt client acl system is a means to allow system users other than root to
+have access to execute select salt commands on minions from the master.
+
+The client acl system is configured in the master configuration file via the
+``client_acl`` configuration option. Under the ``client_acl`` configuration
+option the users open to send commands are specified and then a list of regular
+expressions which specify the minion functions which will be made available to
+specified user. This configuration is much like the ``peer`` configuration:
+
+.. code-block:: yaml
+
+    # Allow thatch to execute anything and allow fred to use ping and pkg
+    client_acl:
+      thatch:
+        - .*
+      fred:
+        - ping.*
+        - pkg.*
+
+Permission Issues
+=================
+
+Directories required for ``client_acl`` must be modified to be readable by the
+users specified:
+
+.. code-block:: bash
+
+    chmod 755 /var/cache/salt /var/cache/salt/jobs /var/run/salt
+
+If you are upgrading from earlier versions of salt you must also remove any
+existing user keys and re-start the Salt master:
+
+.. code-block:: bash
+
+    rm /var/cache/salt/.*keys
+    service salt-master restart

doc/ref/configuration/master.rst

@@ -194,6 +194,34 @@ public keys from the minions
 
     auto_accept: False
 
+.. conf_master:: autosign_file
+
+``autosign_file``
+-----------------
+
+Default ``not defined``
+
+If the autosign_file is specified incoming keys specified in
+the autosign_file will be automatically accepted. Regular expressions as
+well as globbing can be used. This is insecure!
+
+.. conf_master:: client_acl
+
+``client_acl``
+--------------
+
+Default: {}
+
+Enable user accounts on the master to execute specific modules. These modules
+can be expressed as regular expressions
+
+.. code-block:: yaml
+
+    client_acl:
+      fred:
+        - test.ping
+        - pkg.*
+
 
 Master Module Management
 ------------------------
@@ -362,6 +390,8 @@ The buffer size in the file server in bytes
 
     file_buffer_size: 1048576
 
+.. _pillar-configuration:
+
 Pillar Configuration
 --------------------
 
@@ -406,8 +436,9 @@ Default:: ``None``
 
     ext_pillar:
       - hiera: /etc/hiera.yaml
-      - cmd: cat /etc/salt/yaml
+      - cmd_yaml: cat /etc/salt/yaml
 
+There are additional details at :ref:`salt-pillars`
 
 Syndic Server Settings
 ----------------------
@@ -568,3 +599,13 @@ still wish to have 'salt.modules' at the 'debug' level:
   log_granular_levels:
     'salt': 'warning',
     'salt.modules': 'debug'
+
+``default_include``
+-------------------
+
+Default: ``master.d/*.conf``
+
+The minion can include configuration from other files. Per default the
+minion will automatically include all config files from `master.d/*.conf`
+where minion.d is relative to the directory of the minion configuration
+file.

doc/ref/configuration/minion.rst

@@ -112,6 +112,19 @@ The location for minion cache data.
 
     cachedir: /var/cache/salt
 
+.. conf_minion:: cachedir
+
+``backup_mode``
+---------------
+
+Default: ``[]``
+
+Backup files replaced by file.managed and file.recurse under cachedir.
+
+.. code-block:: yaml
+
+    backup_mode: minion
+
 .. conf_minion:: cache_jobs
 
 ``cache_jobs``
@@ -407,6 +420,16 @@ still wish to have 'salt.modules' at the 'debug' level:
 
 .. conf_minion:: include
 
+``default_include``
+-------------------
+
+Default: ``minion.d/*.conf``
+
+The minion can include configuration from other files. Per default the
+minion will automatically include all config files from `minion.d/*.conf`
+where minion.d is relative to the directory of the minion configuration
+file.
+
 ``include``
 -----------
 
@@ -415,12 +438,12 @@ Default: ``not defined``
 The minion can include configuration from other files. To enable this,
 pass a list of paths to this option. The paths can be either relative or
 absolute; if relative, they are considered to be relative to the directory
-the main minion configuration file lives in. Paths can make use of 
+the main minion configuration file lives in. Paths can make use of
 shell-style globbing. If no files are matched by a path passed to this
 option then the minion will log a warning message.
 
 .. code-block:: yaml
-    
+
     # Include files from a minion.d directory in the same
     # directory as the minion config file
     include: minion.d/*
@@ -433,3 +456,40 @@ option then the minion will log a warning message.
       - extra_config
       - minion.d/*
       - /etc/roles/webserver
+
+
+Frozen Build Update Settings
+----------------------------
+
+These options control how :py:func:`salt.modules.saltutil.update` works with esky
+frozen apps. For more information look at `<https://github.com/cloudmatrix/esky/>`_.
+
+.. conf_minion:: update_url
+
+``update_url``
+--------------
+
+Default: ``False`` (Update feature is disabled)
+
+The url to use when looking for application updates. Esky depends on directory
+listings to search for new versions. A webserver running on your Master is a
+good starting point for most setups.
+
+.. code-block:: yaml
+
+    update_url: 'http://salt.example.com/minion-updates'
+
+.. conf_minion:: update_restart_services
+
+``update_restart_services``
+---------------------------
+
+Default: ``[]`` (service restarting on update is disabled)
+
+A list of services to restart when the minion software is updated. This would
+typically just be a list containing the minion's service name, but you may
+have other services that need to go with it.
+
+.. code-block:: yaml
+
+    update_restart_services: ['salt-minion']

doc/ref/esky.rst

@@ -0,0 +1,85 @@
+======================================
+Automatic Updates / Frozen Deployments
+======================================
+
+.. versionadded:: 0.10.3.d
+
+Salt has support for the
+`Esky <https://github.com/cloudmatrix/esky>`_ application freezing and update
+tool. This tool allows one to build a complete zipfile out of the salt scripts
+and all their dependencies - including shared objects / DLLs.
+
+Getting Started
+===============
+
+To build frozen applications, you'll need a suitable build environment for each
+of your platforms. You should probably set up a virtualenv in order to limit
+the scope of Q/A.
+
+This process does work on Windows. Follow the directions at
+`<https://github.com/saltstack/salt-windows-install>`_ for details on
+installing Salt in Windows. Only the 32-bit Python and dependencies have been
+tested, but they have been tested on 64-bit Windows.
+
+You will need to install ``esky`` and ``bbfreeze`` from Pypi in order to enable
+the ``bdist_esky`` command in ``setup.py``.
+
+Building and Freezing
+=====================
+
+Once you have your tools installed and the environment configured, you can then
+``python setup.py sdist`` to get the eggs prepared. After that is done, run
+``python setup.py bdist_esky`` to have Esky traverse the module tree and pack
+all the scripts up into a redistributable. There will be an appropriately
+versioned ``salt-VERSION.zip`` in ``dist/`` if everything went smoothly.
+
+Windows
+-------
+You will need to add ``C:\Python27\lib\site-packages\zmq`` to your PATH
+variable. This helps bbfreeze find the zmq dll so it can pack it up.
+
+Using the Frozen Build
+======================
+
+Unpack the zip file in your desired install location. Scripts like
+``salt-minion`` and ``salt-call`` will be in the root of the zip file. The
+associated libraries and bootstrapping will be in the directories at the same
+level. (Check the `Esky <https://github.com/cloudmatrix/esky>`_ documentation
+for more information)
+
+To support updating your minions in the wild, put your builds on a web server
+that your minions can reach. :py:func:`salt.modules.saltutil.update` will
+trigger an update and (optionally) a restart of the minion service under the
+new version.
+
+Gotchas
+=======
+
+My Windows minion isn't responding
+----------------------------------
+The process dispatch on Windows is slower than it is on *nix. You may need to
+add '-t 15' to your salt calls to give them plenty of time to return.
+
+Windows and the Visual Studio Redist
+------------------------------------
+You will need to install the Visual C++ 2008 32-bit redistributable on all
+Windows minions. Esky has an option to pack the library into the zipfile,
+but OpenSSL does not seem to acknowledge the new location. If you get a
+``no OPENSSL_Applink`` error on the console when trying to start your
+frozen minion, you have forgotten to install the redistributable.
+
+Mixed Linux environments and Yum
+--------------------------------
+The Yum Python module doesn't appear to be available on any of the standard
+Python package mirrors. If you need to support RHEL/CentOS systems, you
+should build on that platform to support all your Linux nodes. Also remember
+to build your virtualenv with ``--system-site-packages`` so that the
+``yum`` module is included.
+
+Automatic (Python) module discovery
+-----------------------------------
+Automatic (Python) module discovery does not work with the late-loaded scheme that
+Salt uses for (Salt) modules. You will need to explicitly add any
+misbehaving modules to the ``freezer_includes`` in Salt's ``setup.py``.
+Always check the zipped application to make sure that the necessary modules
+were included.

doc/ref/modules/all/index.rst

@@ -22,8 +22,11 @@ Full list of builtin modules
     apt
     archive
     augeas_cfg
+    bluez
     brew
     butterkvm
+    ca
+    cassandra
     cluster
     cmdmod
     cp
@@ -34,6 +37,7 @@ Full list of builtin modules
     disk
     django
     ebuild
+    event
     file
     freebsdjail
     freebsdkmod
@@ -52,16 +56,22 @@ Full list of builtin modules
     launchctl
     linux_sysctl
     mdadm
+    monit
     moosefs
     mount
     mysql
     network
     nginx
+    nzbget
+    openbsdpkg
+    openbsdservice
     osxdesktop
     pacman
     pillar
     pip
+    pkgng
     postgres
+    poudriere
     ps
     publish
     puppet
@@ -96,6 +106,7 @@ Full list of builtin modules
     win_service
     win_shadow
     win_useradd
-    yumpkg
     yumpkg5
+    yumpkg
+    zfs
     zypper

doc/ref/modules/all/salt.modules.bluez.rst

@@ -0,0 +1,6 @@
+==================
+salt.modules.bluez
+==================
+
+.. automodule:: salt.modules.bluez
+    :members:

doc/ref/modules/all/salt.modules.ca.rst

@@ -0,0 +1,6 @@
+===============
+salt.modules.ca
+===============
+
+.. automodule:: salt.modules.ca
+    :members:

doc/ref/modules/all/salt.modules.cassandra.rst

@@ -0,0 +1,6 @@
+======================
+salt.modules.cassandra
+======================
+
+.. automodule:: salt.modules.cassandra
+    :members:

doc/ref/modules/all/salt.modules.event.rst

@@ -0,0 +1,6 @@
+==================
+salt.modules.event
+==================
+
+.. automodule:: salt.modules.event
+    :members:

doc/ref/modules/all/salt.modules.monit.rst

@@ -0,0 +1,6 @@
+==================
+salt.modules.monit
+==================
+
+.. automodule:: salt.modules.monit
+    :members:

doc/ref/modules/all/salt.modules.nzbget.rst

@@ -0,0 +1,6 @@
+===================
+salt.modules.nzbget
+===================
+
+.. automodule:: salt.modules.nzbget
+    :members:

doc/ref/modules/all/salt.modules.openbsdpkg.rst

@@ -0,0 +1,6 @@
+=======================
+salt.modules.openbsdpkg
+=======================
+
+.. automodule:: salt.modules.openbsdpkg
+    :members:

doc/ref/modules/all/salt.modules.openbsdservice.rst

@@ -0,0 +1,6 @@
+===========================
+salt.modules.openbsdservice
+===========================
+
+.. automodule:: salt.modules.openbsdservice
+    :members:

doc/ref/modules/all/salt.modules.pkgng.rst

@@ -0,0 +1,6 @@
+==================
+salt.modules.pkgng
+==================
+
+.. automodule:: salt.modules.pkgng
+    :members:

doc/ref/modules/all/salt.modules.poudriere.rst

@@ -0,0 +1,6 @@
+======================
+salt.modules.poudriere
+======================
+
+.. automodule:: salt.modules.poudriere
+    :members:

doc/ref/modules/all/salt.modules.zfs.rst

@@ -0,0 +1,6 @@
+================
+salt.modules.zfs
+================
+
+.. automodule:: salt.modules.zfs
+    :members:

doc/ref/modules/index.rst

@@ -159,7 +159,7 @@ Documentation
 =============
 
 Salt modules are self documenting, the :func:`sys.doc` function will return the
-documentation for all available Facter modules:
+documentation for all available modules:
 
 .. code-block:: bash
 

doc/ref/peer.rst

@@ -6,7 +6,7 @@ Salt 0.9.0 introduced the capability for Salt minions to publish commands. The
 intent of this feature is not for Salt minions to act as independent brokers
 one with another, but to allow Salt minions to pass commands to each other.
 
-In Salt 1.0 the ability to execute runners from the master was added. This
+In Salt 0.10.0 the ability to execute runners from the master was added. This
 allows for the master to return collective data from runners back to the
 minions via the peer interface.
 

doc/ref/pillar/all/index.rst

@@ -0,0 +1,15 @@
+.. _all-salt.pillars:
+
+============================
+Full list of builtin pillars
+============================
+
+.. currentmodule:: salt.pillar
+
+.. autosummary::
+    :toctree:
+    :template: autosummary.rst.tmpl
+
+    cmd_yaml
+    hiera
+    mongo

doc/ref/pillar/all/salt.pillar.cmd_yaml.rst

@@ -0,0 +1,6 @@
+====================
+salt.pillar.cmd_yaml
+====================
+
+.. automodule:: salt.pillar.cmd_yaml
+    :members:

doc/ref/pillar/all/salt.pillar.hiera.rst

@@ -0,0 +1,6 @@
+=================
+salt.pillar.hiera
+=================
+
+.. automodule:: salt.pillar.hiera
+    :members:

doc/ref/pillar/all/salt.pillar.mongo.rst

@@ -0,0 +1,6 @@
+=================
+salt.pillar.mongo
+=================
+
+.. automodule:: salt.pillar.mongo
+    :members:

doc/ref/pillar/index.rst

@@ -0,0 +1,15 @@
+
+.. _salt-pillars:
+
+=======
+Pillars
+=======
+
+Salt includes a number of built-in external pillars, listed at
+:ref:`all-salt.pillars`.
+
+You may also wish to look at the standard pillar documentation, at
+:ref:`pillar-configuration`
+
+The source for the built-in Salt returners can be found here:
+:blob:`salt/pillar`

doc/ref/renderers/all/index.rst

@@ -13,7 +13,14 @@ Full list of builtin renderers
     json_jinja
     json_mako
     json_wempy
+<<<<<<< HEAD
     yaml_jinja
     yaml_mako
     yaml_wempy
     py
+=======
+    py
+    yaml_jinja
+    yaml_mako
+    yaml_wempy
+>>>>>>> 267d900f19a3f62c323a65af414a3b7f65233484

doc/ref/renderers/all/salt.renderers.json_wempy.rst

@@ -0,0 +1,6 @@
+=========================
+salt.renderers.json_wempy
+=========================
+
+.. automodule:: salt.renderers.json_wempy
+    :members:

doc/ref/renderers/all/salt.renderers.yaml_wempy.rst

@@ -0,0 +1,6 @@
+=========================
+salt.renderers.yaml_wempy
+=========================
+
+.. automodule:: salt.renderers.yaml_wempy
+    :members:

doc/ref/renderers/index.rst

@@ -53,7 +53,8 @@ Writing a renderer is easy, all that is required is that a Python module
 is placed in the rendered directory and that the module implements the
 render function. The render function will be passed the path of the SLS file.
 In the render function, parse the passed file and return the data structure
-derived from the file.
+derived from the file. You can place your custom renderers in a ``_renderers``
+directory in your file root (``/srv/salt/``).
 
 Examples
 --------

doc/ref/returners/all/salt.returners.mongo_return.rst

@@ -3,4 +3,4 @@ salt.returners.mongo_return
 ===========================
 
 .. automodule:: salt.returners.mongo_return
-    :members:
+    :members:

doc/ref/states/all/index.rst

@@ -19,14 +19,17 @@ Full list of builtin states
     group
     host
     kmod
+    module
     mount
     mysql_database
     mysql_grants
     mysql_user
     network
     pip
+    pkgng
     pkg
     postgres_database
+    postgres_user
     rvm
     selinux
     service

doc/ref/states/all/salt.states.module.rst

@@ -0,0 +1,6 @@
+==================
+salt.states.module
+==================
+
+.. automodule:: salt.states.module
+    :members:

doc/ref/states/all/salt.states.pkgng.rst

@@ -0,0 +1,6 @@
+=================
+salt.states.pkgng
+=================
+
+.. automodule:: salt.states.pkgng
+    :members:

doc/ref/states/all/salt.states.postgres_user.rst

@@ -0,0 +1,6 @@
+=========================
+salt.states.postgres_user
+=========================
+
+.. automodule:: salt.states.postgres_user
+    :members:

doc/ref/states/backup_mode.rst

@@ -3,7 +3,7 @@ State File Backups
 ==================
 
 In 0.10.2 a new feature was added for backing up files that are replaced by
-the file.managed and file.recuse states. The new feature is called the backup
+the file.managed and file.recurse states. The new feature is called the backup
 mode. Setting the backup mode is easy, but is can be set in a number of
 places.
 

doc/ref/states/include.rst

@@ -0,0 +1,26 @@
+===================
+Include and Exclude
+===================
+
+Salt sls files can include other sls files and exclude sls files that have been
+otherwise included. This allows for an sls file to easily extend or manipulate
+other sls files.
+
+Exclude
+=======
+
+The exclude statement, added in Salt 0.10.3 allows an sls to hard exclude
+another sls file or a specific id. The component is excluded after the
+high data has been compiled, so nothing should be able to override an
+exclude.
+
+Since the exclude can remove an id or an sls the type of component to
+exclude needs to be defined. an exclude statement that verifies that the
+running highstate does not contain the `http` sls and the `/etc/vimrc` id
+would look like this:
+
+.. code-block:: yaml
+
+    exclude:
+      - sls: http
+      - id: /etc/vimrc

doc/ref/states/ordering.rst

@@ -210,3 +210,18 @@ set the order to ``last``:
     vim:
       pkg.installed:
         - order: last
+
+Remember that requisite statements override the order option. So the order
+option should be applied to the highest component of the requisite chain:
+
+.. code-block:: yaml
+
+    vim:
+      pkg.installed:
+        - order: last
+        - require:
+          - file: /etc/vimrc
+
+    /etc/vimrc:
+      file.managed:
+        - source: salt://edit/vimrc

doc/ref/states/requisites.rst

@@ -78,4 +78,94 @@ the service is reloaded or restarted.
 Use
 ---
 
-# This needs to be filled in
+The ``use`` requisite is used to inherit the arguments passed in another
+id declaration. This is useful when many files need to have the same defaults.
+
+The ``use`` statement was developed primarily for the networking states but
+can be used on any states in Salt. This made sense for the networking state
+because it can define a long list of options that need to be applied to
+multiple network interfaces.
+
+Require In
+----------
+
+The ``require_in`` requisite is the literal reverse of ``require``. If
+a state declaration needs to be required by another state declaration then
+require_in can accommodate it, so these two sls files would be the same in
+the end:
+
+Using ``require``
+
+.. code-block:: yaml
+
+    httpd:
+      pkg:
+        - installed
+      service:
+        - running
+        - require:
+          - pkg: httpd
+
+Using ``require_in``
+
+.. code-block:: yaml
+
+    httpd:
+      pkg:
+        - installed
+        - require_in:
+          - service: httpd
+      service:
+        - running
+
+The ``require_in`` statement is particularly useful when assigning a require
+in a sperate sls file. For instance it may be common for httpd to require
+components used to set up php or mod_python, but the http state does not need
+to be aware of the additional components that require it when it is set up:
+
+http.sls
+
+.. code-block:: yaml
+
+    httpd:
+      pkg:
+        - installed
+      service:
+        - running
+        - require:
+          - pkg: httpd
+
+php.sls
+
+.. code-block:: yaml
+
+    include:
+      - http
+
+    php:
+      pkg:
+        - installed
+        - require_in:
+          - service: httpd
+
+mod_python.sls
+
+.. code-block:: yaml
+
+    include:
+      - http
+
+    mod_python:
+      pkg:
+        - installed
+        - require_in:
+          - service: httpd
+
+Now the httpd server will only start if php or mod_python are first verified to
+be installed. Thus allowing for a requisite to be defined "after the fact".
+
+Watch In
+--------
+
+Watch in functions the same was as require in, but applies a watch statement
+rather than a require statement to the external state declaration.

doc/ref/states/startup.rst

@@ -0,0 +1,44 @@
+==============
+Startup States
+==============
+
+Sometimes it may be desired that the salt minion execute a state run when it is
+started. This alleviates the need for the master to initiate a state run on a
+new minion and can make provisioning much easier.
+
+As of Salt 0.10.3 the minion config reads options that allow for states to be
+executed at startup. The options are `startup_states`, `sls_list` and
+`top_file`.
+
+The `startup_states` option can be passed one of a number of arguments to
+define how to execute states. The available options are:
+
+highstate
+  Execute ``state.highstate``
+
+sls
+  Read in the ``sls_list`` option and execute the named sls files
+
+top
+  Read in the ``top_file`` option and execute states based on that top file
+  on the Salt Master
+  
+Examples:
+---------
+
+Execute ``state.highstate`` when starting the minion:
+
+
+.. code-block:: yaml
+
+   startup_states: highstate
+
+Execute the sls files `edit.vim` and `hyper`:
+
+.. code-block:: yaml
+
+    startup_states: sls
+
+    sls_list:
+      - edit.vim
+      - hyper

doc/ref/states/testing.rst

@@ -0,0 +1,33 @@
+=============
+State Testing
+=============
+
+Executing a Salt state run can potentially change many aspects of a system and
+it may be desirable to first see what a state run is going to change before
+applying the run.
+
+Salt has a test interface to report on exactly what will be changed, this
+interface can be invoked on any of the major state run functions:
+
+.. code-block:: bash
+
+    # salt \* state.highstate test=True
+    # salt \* state.sls test=True
+    # salt \* state.single test=True
+
+The test run is mandated by adding the ``test=True`` option to the states. The
+return information will show states that will be applied in yellow and the
+result is reported as `None`.
+
+Default Test
+============
+
+If the value `test` is set to True in the minion configuration file then states
+will default to being executed in test mode. If this value is set then states
+can still be run by calling test=False:
+
+.. code-block:: bash
+
+    # salt \* state.highstate test=False
+    # salt \* state.sls test=False
+    # salt \* state.single test=False

doc/topics/community.rst

@@ -35,6 +35,12 @@ can use the `Freenode webchat client`__ right from your browser.
 .. __: http://webchat.freenode.net/?channels=salt&uio=Mj10cnVlJjk9dHJ1ZSYxMD10cnVl83
 .. __: http://irclog.perlgeek.de/salt/
 
+Salt development
+----------------
+
+If you wish to discuss the development of Salt itself join us in
+``#salt-devel``.
+
 .. _community-github:
 
 Follow on Github
@@ -69,6 +75,8 @@ A few examples of salt states from the community:
 * https://github.com/kevingranade/kevingranade-salt-state
 * https://github.com/uggedal/states
 * https://github.com/mattmcclean/salt-openstack/tree/master/salt
+* https://github.com/rentalita/ubuntu-setup/
+* https://github.com/brutasse/states
 
 Follow on ohloh
 ===============

doc/topics/event/index.rst

@@ -18,7 +18,7 @@ by the same system user that Salt is running as. To listen to events a
 SaltEvent object needs to be created and then the get_event function needs to
 be run. The SaltEvent object needs to know the location that the Salt unix
 sockets are kept. In the configuration this is the ``sock_dir`` option. The
-``sock_dir`` option defaults to "/tmp/.salt-unix" on most systems.
+``sock_dir`` option defaults to "/var/run/salt" on most systems.
 
 The following code will check for a single event:
 
@@ -26,7 +26,7 @@ The following code will check for a single event:
 
     import salt.utils.event
 
-    event = salt.utils.event.MasterEvent('/tmp/.salt-unix')
+    event = salt.utils.event.MasterEvent('/var/run/salt')
 
     data = event.get_event()
 
@@ -41,19 +41,19 @@ instead of the default 5.
 
     import salt.utils.event
 
-    event = salt.utils.event.MasterEvent('/tmp/.salt-unix')
+    event = salt.utils.event.MasterEvent('/var/run/salt')
 
     data = event.get_event(wait=10, tag='auth')
 
-Instead of looking for a single event, the iter_event method can be used to
+Instead of looking for a single event, the iter_events method can be used to
-make a generator which will continually yield salt events. The iter_event
+make a generator which will continually yield salt events. The iter_events
 method also accepts a tag, but not a wait time:
 
 .. code-block:: python
 
     import salt.utils.event
 
-    event = salt.utils.event.MasterEvent('/tmp/.salt-unix')
+    event = salt.utils.event.MasterEvent('/var/run/salt')
 
-    for data in event.iter_event(tag='auth'):
+    for data in event.iter_events(tag='auth'):
         print(data)

doc/topics/installation/index.rst

@@ -50,3 +50,4 @@ Platform-specific installation instructions
     freebsd
     gentoo
     windows
+    solaris

doc/topics/installation/solaris.rst

@@ -0,0 +1,106 @@
+=======
+Solaris
+=======
+
+Salt was added to the OpenCSW package repository in September of 2012 by Romeo Theriault <romeot@hawaii.edu> at version 0.10.2 of Salt. It has mainly been tested on Solaris 10 (sparc), though it is built for, and should work fine on Solaris 10 (x86), Solaris 9 (sparc/x86) and 11 (sparc/x86) also. Most of the testing has also just focused on the minion, though it has verified that the master starts up successfully on Solaris 10.
+
+Comments and patches for better support on these platforms is very welcome. Currently at version 0.10.2 of salt, grain detection is weak but patches that very much improve the grain detection will be released in 0.10.3. Work is also underway to include support for services and packages in Solaris.
+
+Salt is dependent on the following additional packages. These will automatically be installed as
+dependencies of the ``py_salt`` package. ::
+
+   py_yaml
+   py_pyzmq
+   py_jinja2
+   py_msgpack_python
+   py_m2crypto
+   py_crypto
+   python
+
+Installation
+============
+
+To install Salt from the OpenCSW package repository you first need to install `pkgutil`_ assuming you don't already have it installed:
+
+On Solaris 10:
+
+.. code-block:: bash
+
+   pkgadd -d http://get.opencsw.org/now
+
+On Solaris 9:
+
+.. code-block:: bash
+
+   wget http://mirror.opencsw.org/opencsw/pkgutil.pkg
+   pkgadd -d pkgutil.pkg all
+
+Once pkgutil is installed you'll need to edit it's config file ``/etc/opt/csw/pkgutil.conf`` to point it at the unstable catalog:
+
+.. code-block:: diff
+
+   - #mirror=http://mirror.opencsw.org/opencsw/testing
+   + mirror=http://mirror.opencsw.org/opencsw/unstable
+
+Ok, time to install salt.
+
+.. code-block:: bash
+
+   # Update the catalog
+   root> /opt/csw/bin/pkgutil -U
+   # Install salt
+   root> /opt/csw/bin/pkgutil -i -y py_salt
+
+Minion Configuration
+=============
+
+Now that salt is installed you can find it's configuration files in:
+
+``/etc/opt/csw/salt/``
+
+You'll want to edit the minion config file to set the name of your salt master server:
+
+.. code-block:: diff
+
+    - #master: salt
+    + master: your-salt-server
+
+You can now start the salt minion like so:
+
+On Solaris 10:
+
+.. code-block:: bash
+
+    svcadm enable salt-minion
+
+
+On Solaris 9:
+
+.. code-block:: bash
+
+    /etc/init.d/salt-minion start
+
+You should now be able to log onto the salt master and check to see if the salt-minion key is awaiting acceptance:
+
+.. code-block:: bash
+
+   salt-key -l un
+ 
+Accept the key:
+
+.. code-block:: bash
+
+    salt-key -a <your-salt-minion>
+
+Run a simple test against the minion:
+
+.. code-block:: bash
+
+    salt '<your-salt-minion>' test.ping
+
+Troubleshooting
+=============
+
+Logs are in ``/var/log/salt``
+
+.. _pkgutil: http://www.opencsw.org/manual/for-administrators/getting-started.html

doc/topics/pillar/index.rst

@@ -17,8 +17,7 @@ Salt file server the ``pillar_roots`` option in the master config is based
 on environments mapping to directories. The pillar data is then mapped to
 minions based on matchers in a top file which is laid out in the same way
 as the state top file. Salt pillars can use the same matcher types as the
-standard top file, and if you are having difficulty matching a specific minion
+standard top file.
-in your pillar top file, you may want to specify PCRE matching.  
 
 The configuration for the ``pillar_roots`` in the master config is identical in
 behavior and function as the ``file_roots`` configuration:
@@ -40,23 +39,6 @@ used for States, and has the same structure:
     base:
       '*':
         - packages
-      'someminion':
-        - someminion-specials
-
-This simple pillar top file declares that information for all minions can be
-found in the ``packages.sls`` file [#nokeyvalueintop]_, while
-``someminion-specials.sls`` contains overriding or additional information just
-for one special minion.
-
-.. code-block:: yaml
-
-    base:
-      '.*':
-        - match: pcre
-        - packages
-      '(someminion|anotherminion)':
-        - match: pcre
-        - someminion-specials
 
 This further example shows how to enable pcre matching in the salt pillar file. 
 The flexibility enabled by pcre matching is particularly useful in salt pillar
@@ -75,12 +57,6 @@ files.
     {% endif %}
     somekey: globalvalue
 
-``/srv/pillar/someminion-specials.sls``
-
-.. code-block:: yaml
-
-    somekey: specialvalue
-
 Now this data can be used from within modules, renderers, State SLS files, and
 more via the shared pillar `dict`_:
 
@@ -129,6 +105,16 @@ this:
 
 .. _`dict`: http://docs.python.org/library/stdtypes.html#mapping-types-dict
 
+Viewing Minion Pillar
+=====================
+
+Once the pillar is set up the data can be viewed on the minion via the
+``pillar.data`` module:
+
+.. code-block:: bash
+
+    # salt '*' pillar.data
+
 Footnotes
 ---------
 

doc/topics/releases/0.10.2.rst

@@ -49,7 +49,7 @@ more powerful, since the minions that will match can be pre determined.
 Backup Files
 ------------
 
-By default all files replaced by the file.managed and file.recuse states we
+By default all files replaced by the file.managed and file.recurse states we
 simply deleted. 0.10.2 adds a new option. By setting the backup option to
 ``minion`` the files are backed up before they are replaced.
 
@@ -148,7 +148,7 @@ Bluetooth
 ^^^^^^^^^
 
 Baisc ``bluez`` support for managing and controlling Bluetooth devices.
-Supports scanning as well as pairing/unpairing.
+Supports scanning as well as pairing/unpairing by Joseph Hall.
 
 Test Updates
 ============

doc/topics/releases/0.10.3.rst

@@ -0,0 +1,180 @@
+=========================
+Salt 0.10.3 Release Notes
+=========================
+
+The latest taste of Salt has come, this release has many fixes and feature
+additions. Modifications have been made to make ZeroMQ connections more
+reliable, the begining of the ACL system is in place, a new command line
+parsing system has been added, dynamic module distribution has become more
+environment aware, the new `master_finger` option and many more!
+
+Major Features
+==============
+
+ACL System
+----------
+
+The new ACL system has been introduced. The ACL system allows for system users
+other than root to execute salt commands. Users can be allowed to execute
+specific commands in the same way that minions are opened up to the peer
+system.
+
+The configuration value to open up the ACL system is called ``client_acl``
+and is configured like so:
+
+.. code-block:: yaml
+
+    client_acl:
+      fred:
+        - test..*
+        - pkg.list_pkgs
+
+Where `fred` is allowed access to functions in the test module and to the
+``pkg.list_pkgs`` function.
+
+Master Finger Option
+--------------------
+
+The `master_finger` option has been added to improve the security of minion
+provisioning. The `master_finger` option allows for the fingerprint of the
+master public key to be set in the configuration file to double verify that the
+master is valid. This option was added in response to a motivation to pre
+authenticate the master when provisioning new minions to help prevent
+man in the middle attacks in some situations.
+
+Salt Key Fingerprint Generation
+-------------------------------
+
+The ability to generate fingerprints of keys used by Salt has been added to
+``salt-key``. The new option `finger` accepts the name of the key to generate
+and display a fingerprint for.
+
+.. code-block:: base
+
+    salt-key -F master
+
+Will display the fingerprints for the master public and private keys.
+
+Parsing System
+--------------
+
+Pedro Algavio, aka s0undt3ch, has added a substantial update to the command
+line parsing system that makes the help message output much cleaner and easier
+to search through. Salt parsers now have `--versions-report` besides usual 
+`--version` info which you can provide when reporting any issues found.
+
+Key Generation
+--------------
+
+We have reduced the requirements needed for `salt-key` to generate minion keys.  
+You're no longer required to have salt configured and it's common directories 
+created just to generate keys. This might prove useful if you're batch creating 
+keys to pre-load on minions.
+
+Startup States
+--------------
+
+A few configuration options have been added which allow for states to be run
+when the minion daemon starts. This can be a great advantage when deploying
+with Salt because the minion can apply states right when it first runs. To
+use startup states set the ``startup_states`` configuration option on the
+minion to `highstate`.
+
+New Exclude Declaration
+-----------------------
+
+Some users have asked about adding the ability to ensure that other sls files
+or ids are excluded from a state run. The exclude statement will delete all of
+the data loaded from the specified sls file or will delete the specified id:
+
+.. code-block:: yaml
+
+    exclude:
+      - sls: http
+      - id: /etc/vimrc
+
+Max Open Files
+--------------
+
+While we're currently unable to properly handle ZeroMQ's abort signals when the 
+max open files is reached, due to the way that's handled on ZeroMQ's, we have 
+minimized the chances of this happening without at least warning the user.
+
+More State Output Options
+-------------------------
+
+Some major changes have been made to the state output system. In the past state
+return data was printed in a very verbose fashion and only states that failed
+or made changes were printed by default. Now two options can be passed to the
+master and minion configuration files to change the behavior of the state
+output. State output can be set to verbose (default) or non-verbose with the
+``state_verbose`` option:
+
+.. code-block:: yaml
+
+    state_verbose: False
+
+It is noteworthy that the state_verbose option used to be set to `False` by
+default but has been changed to `True` by default in 0.10.3 due to many
+requests for the change.
+
+Te next option to be aware of new and called ``state_output``. This option
+allows for the state output to be set to `full` (default) or `terse`.
+
+The `full` output is the standard state output, but the new `terse` output
+will print only one line per state making the output much easier to follow when
+executing a large state system.
+
+.. code-block:: yaml
+
+    state_output: terse
+
+
+`state.file.append` Improvements
+--------------------------------
+
+The salt state `file.append()` tries *not* to append existing text. Previously 
+the matching check was being made line by line. While this kind of check might 
+be enough for most cases, if the text being appended was multi-line, the check 
+would not work properly. This issue is now properly handled, the match is done 
+as a whole ignoring any white space addition or removal except inside commas.  
+For those thinking that, in order to properly match over multiple lines, salt 
+will load the whole file into memory, that's not true. For most cases this is 
+not important but an erroneous order to read a 4GB file, if not properly 
+handled, like salt does, could make salt chew that amount of memory.  Salt has 
+a buffered file reader which will keep in memory a maximum of 256KB and 
+iterates over the file in chunks of 32KB to test for the match, more than 
+enough, if not, explain your usage on a ticket. With this change, also 
+`salt.modules.file.contains()`, `salt.modules.file.contains_regex()`, 
+`salt.modules.file.contains_glob()` and `salt.utils.find` now do the searching 
+and/or matching using the buffered chunks approach explained above.
+
+Two new keyword arguments were also added, `makedirs` and `source`.
+The first, `makedirs` will create the necessary directories in order to append 
+to the specified file, of course, it only applies if we're trying to append to 
+a non-existing file on a non-existing directory:
+
+.. code-block:: yaml
+
+    /tmp/salttest/file-append-makedirs:
+        file.append:
+            text: foo
+            makedirs: True
+
+
+The second, `source`, allows to append the contents of a file instead of 
+specifying the text.
+
+.. code-block:: yaml
+
+    /tmp/salttest/file-append-source:
+
+    file.append:
+        - source: salt://testfile
+
+Security Fix
+============
+
+A timing vulnerability was uncovered in the code which decrypts the AES
+messages sent over the network. This has been fixed and upgrading is
+strongly recommended.

doc/topics/releases/0.9.3.rst

@@ -196,7 +196,7 @@ file.symlink
 file.recurse
     The recurse state function will recursively download a directory on the
     master file server and place it on the minion. Any change in the files on
-    the master will be pushed to the minion. The recuse function is very
+    the master will be pushed to the minion. The recurse function is very
     powerful and has been tested by pushing out the full Linux kernel source.
 
     .. code-block:: yaml

doc/topics/targeting/nodegroups.rst

@@ -8,6 +8,11 @@ Node groups
         A predefined group of minions declared in the master configuration file
         :conf_master:`nodegroups` setting as a compound target.
 
+Nodegroups are declared using a compound target specification. The compount
+target documentation can be found here:
+
+:doc:`Compound Matchers <topics/targeting/compound>`
+
 For example, in the master config file :conf_master:`nodegroups` setting::
 
     nodegroups:

doc/topics/tests/index.rst

@@ -0,0 +1,30 @@
+=============
+Writing Tests
+=============
+
+Salt uses a test platform to verify functionality of components in a simple
+way. Two testing systems exist to enable testing salt functions in somewhat
+real environments. The two subsystems available are integration tests and
+unit tests.
+
+Salt uses the python standard library unittest2 system for testing.
+
+Integration Tests
+=================
+
+The integration tests start up a number of salt daemons to test functionality
+in a live environment. These daemons include 2 salt masters, 1 syndic and 2
+minions. This allows for the syndic interface to be tested and master/minion
+communication to be verified. All of the integration tests are executed as
+live salt commands sent through the started daemons.
+
+* :doc:`Writing integration tests <integration>`
+
+Integration tests are particularly good at testing modules, states and shell
+commands.
+
+Unit Tests
+==========
+
+Direct unit tests are also available, these tests are good for internal
+functions.

doc/topics/tests/integration.rst

@@ -0,0 +1,214 @@
+=================
+Integration Tests
+=================
+
+The Salt integration tests come with a number of classes and methods which
+allow for components to be easily tested. These classes are generally inherited
+from and provide specific methods for hooking into the running integration test
+environment created by the integration tests.
+
+It is noteworthy that since integration tests validate against a running
+environment that they are generally the preferred means to write tests.
+
+The integration system is all located under tests/integration in the Salt
+source tree.
+
+Integration Classes
+===================
+
+The integration classes are located in tests/integration/__init__.py and
+can be extended therein. There are three classes available to extend:
+
+ModuleCase
+----------
+
+Used to define executions run via the master to minions and to call
+single modules and states.
+
+The available methods are as follows:
+
+run_function:
+    Run a single salt function and condition the return down to match the
+    behavior of the raw function call. This will run the command and only
+    return the results from a single minion to verify.
+
+state_result:
+    Return the result data from a single state return
+
+run_state:
+    Run the state.single command and return the state return structure
+
+
+
+
+SyndicCase
+----------
+
+Used to execute remote commands via a syndic, only used to verify the
+capabilities of the Syndic.
+
+The available methods are as follows:
+
+run_function:
+    Run a single salt function and condition the return down to match the
+    behavior of the raw function call. This will run the command and only
+    return the results from a single minion to verify.
+
+ShellCase
+---------
+
+Shell out to the scripts which ship with Salt.
+
+The available methods are as follows:
+
+run_script:
+    Execute a salt script with the given argument string
+
+run_salt:
+    Execute the salt command, pass in the argument string as it would be
+    passed on the command line.
+
+run_run:
+    Execute the salt-run command, pass in the argument string as it would be
+    passed on the command line.
+
+run_run_plus:
+    Execute Salt run and the salt run function and return the data from
+    each in a dict
+
+run_key:
+    Execute the salt-key command, pass in the argument string as it would be
+    passed on the command line.
+
+run_cp:
+    Execute salt-cp, pass in the argument string as it would be
+    passed on the command line.
+
+run_call:
+    Execute salt-call, pass in the argument string as it would be
+    passed on the command line.
+
+
+Examples
+========
+
+Module Example via ModuleCase Class
+-----------------------------------
+
+Import the integration module, this module is already added to the python path
+by the test execution. Inherit from the ``integration.ModuleCase`` class. The
+tests that execute against salt modules should be placed in the
+`tests/integration/modules` directory so that they will be detected by the test
+system.
+
+Now the workhorse method ``run_function`` can be used to test a module:
+
+.. code-block:: python
+
+    
+    import os
+    import integration
+
+
+    class TestModuleTest(integration.ModuleCase):
+        '''
+        Validate the test module
+        '''
+        def test_ping(self):
+            '''
+            test.ping
+            '''
+            self.assertTrue(self.run_function('test.ping'))
+
+        def test_echo(self):
+            '''
+            test.echo
+            '''
+            self.assertEqual(self.run_function('test.echo', ['text']), 'text')
+
+ModuleCase can also be used to test states, when testing states place the test
+module in the `tests/integration/states` directory. The ``state_result`` and
+the ``run_state`` methods are the workhorse here:
+
+.. code-block:: python
+
+    import os
+    import shutil
+    import integration
+
+    HFILE = os.path.join(integration.TMP, 'hosts')
+
+    class HostTest(integration.ModuleCase):
+        '''
+        Validate the host state
+        '''
+
+        def setUp(self):
+            shutil.copyfile(os.path.join(integration.FILES, 'hosts'), HFILE)
+            super(HostTest, self).setUp()
+
+        def tearDown(self):
+            if os.path.exists(HFILE):
+                os.remove(HFILE)
+            super(HostTest, self).tearDown()
+
+        def test_present(self):
+            '''
+            host.present
+            '''
+            name = 'spam.bacon'
+            ip = '10.10.10.10'
+            ret = self.run_state('host.present', name=name, ip=ip)
+            result = self.state_result(ret)
+            self.assertTrue(result)
+            with open(HFILE) as fp_:
+                output = fp_.read()
+                self.assertIn('{0}\t\t{1}'.format(ip, name), output)
+
+The above example also demonstrates using the integration files and the
+integration state tree. The variable `integration.FILES` will point to the
+directory used to store files that can be used or added to to help enable tests
+that require files. The location `integration.TMP` can also be used to store
+temporary files that the test system will clean up when the execution finishes.
+
+The integration state tree can be found at `tests/integration/files/file/base`.
+This is where the referenced `host.present` sls file resides.
+
+Shell Example via ShellCase
+---------------------------
+
+Validating the shell commands can be done via shell tests. Here are some
+examples:
+
+.. code-block:: python
+
+    import sys
+    import shutil
+    import tempfile
+
+    import integration
+
+    class KeyTest(integration.ShellCase):
+        '''
+        Test salt-key script
+        '''
+
+        _call_binary_ = 'salt-key'
+
+        def test_list(self):
+            '''
+            test salt-key -L
+            '''
+            data = self.run_key('-L')
+            expect = [
+                    'Unaccepted Keys:',
+                    'Accepted Keys:',
+                    'minion',
+                    'sub_minion',
+                    'Rejected:', '']
+            self.assertEqual(data, expect)
+
+This example verifies that the ``salt-key`` command executes and returns as
+expected by making use of the ``run_key`` method.
+
+All shell tests should be placed in the `tests/integraion/shell` directory.

doc/topics/troubleshooting/index.rst

@@ -182,3 +182,26 @@ Common YAML Gotchas
 An extensive list of
 :doc:`YAML idiosyncrasies</topics/troubleshooting/yaml_idiosyncrasies>`
 has been compiled.
+
+Live Python Debug Output
+========================
+
+If the minion or master seems to be unresponsive, a SIGUSR1 can be passed to
+the processes to display where in the code they are running. If encountering a
+situation like this, this debug information can be invaluable. First make
+sure the master of minion are running in the foreground:
+
+.. code-block:: bash
+
+    # salt-master -l debug
+    # salt-minion -l debug
+
+The pass the signal to the master or minion when it seems to be unresponsive:
+
+.. code-block:: bash
+
+    killall -SIGUSR1 salt-master
+    killall -SIGUSR1 salt-minion
+
+When filing an issue or sending questions to the mailing list for a problem
+with an unresponsive daemon this information can be invaluable.

doc/topics/troubleshooting/yaml_idiosyncrasies.rst

@@ -78,7 +78,7 @@ is not desirable, then a deeply nested dict can be declared with curly braces:
 Integers are Parsed as Integers
 ===============================
 
-NOTE: This has been fixed in salt 0.9.10, as of this release passing an
+NOTE: This has been fixed in salt 0.10.0, as of this release passing an
 integer that is preceded by a 0 will be correctly parsed
 
 When passing `integers`_ into an SLS file, they are passed as integers. This means
@@ -99,7 +99,7 @@ This is best explained when setting the mode for a file:
 
 Salt manages this well, since the mode is passed as 644, but if the mode is
 zero padded as 0644, then it is read by YAML as an integer and evaluated as
-a hexadecimal value, 0644 becomes 420. Therefore, if the file mode is
+an octal value, 0644 becomes 420. Therefore, if the file mode is
 preceded by a 0 then it needs to be passed as a string:
 
 .. code-block:: yaml
@@ -173,3 +173,39 @@ WORKS:
         - name: AAAAB3NzaC...
         - enc: dsa
 
+YAML support only plain ASCII
+=============================
+
+According to YAML specification, only ASCII characters can be used.
+
+Within double-quotes, special characters may be represented with C-style
+escape sequences starting with a backslash ( \\ ).
+
+Examples:
+
+.. code-block:: yaml
+
+    - micro: "\u00b5"
+    - copyright: "\u00A9"
+    - A: "\x41"
+    - alpha: "\u0251"
+    - Alef: "\u05d0"
+
+
+    
+List of useable `Unicode characters`_  will help you to identify correct numbers.
+
+.. _`Unicode characters`: http://en.wikipedia.org/wiki/List_of_Unicode_characters
+
+
+Python can also be used to discover the Unicode number for a character:
+
+.. code-block:: python
+
+    repr(u"Text with wrong characters i need to figure out")
+
+This shell command can find wrong characters in your SLS files:
+
+.. code-block: shell
+    find . -name '*.sls'  -exec  grep --color='auto' -P -n '[^\x00-\x7F]' \{} \;
+

doc/topics/tutorials/extend_with_require_watch.rst

@@ -0,0 +1,4 @@
+.. admonition:: Using extend with require or watch
+
+    The ``extend`` statement works differently for ``require`` or ``watch``.
+    It appends to, rather than replacing the requisite component.

doc/topics/tutorials/requisite_incl.rst

@@ -4,6 +4,7 @@ following the :doc:`installation </topics/installation/index>` and the :doc:`con
 
 .. admonition:: Stuck?
 
-    If you get stuck at any point, there are many ways to :doc:`get help from
+    There are many ways to :doc:`get help from the Salt community
-    the Salt community </topics/community>` including our mailing list and our
+    </topics/community>` including our
-    IRC channel.
+    `mailing list <http://groups.google.com/group/salt-users/>`_
+    and our `IRC channel <http://webchat.freenode.net/>`_ #salt.

doc/topics/tutorials/starting_states.rst

@@ -145,8 +145,8 @@ out as just files. A SLS is just a file and files to download are just files.
 The Apache example would be laid out in the root of the Salt file server like
 this: ::
 
-    /apache/init.sls
+    apache/init.sls
-    /apache/httpd.conf
+    apache/httpd.conf
 
 So the httpd.conf is just a file in the apache directory, and is referenced
 directly.
@@ -154,7 +154,7 @@ directly.
 But with more than a single SLS file, more components can be added to the
 toolkit, consider this SSH example:
 
-``/ssh/init.sls:``
+``ssh/init.sls:``
 
 .. code-block:: yaml
    :linenos:
@@ -211,13 +211,13 @@ toolkit, consider this SSH example:
 
 Now our State Tree looks like this: ::
 
-    /apache/init.sls
+    apache/init.sls
-    /apache/httpd.conf
+    apache/httpd.conf
-    /ssh/init.sls
+    ssh/init.sls
-    /ssh/server.sls
+    ssh/server.sls
-    /ssh/banner
+    ssh/banner
-    /ssh/ssh_config
+    ssh/ssh_config
-    /ssh/sshd_config
+    ssh/sshd_config
 
 This example now introduces the ``include`` statement. The include statement
 includes another SLS file so that components found in it can be required,
@@ -236,7 +236,7 @@ needs to be placed.
 
 These examples will add more watchers to apache and change the ssh banner.
 
-``/ssh/custom-server.sls:``
+``ssh/custom-server.sls:``
 
 .. code-block:: yaml
    :linenos:
@@ -249,7 +249,7 @@ These examples will add more watchers to apache and change the ssh banner.
         file:
           - source: salt://ssh/custom-banner
 
-``/python/mod_python.sls:``
+``python/mod_python.sls:``
 
 .. code-block:: yaml
    :linenos:
@@ -273,9 +273,7 @@ to configure the banner.
 In the new mod_python SLS the mod_python package is added, but more importantly
 the apache service was extended to also watch the mod_python package.
 
-There is a bit of a trick here, in the extend statement Requisite Statements
+.. include:: extend_with_require_watch.rst
-are extended, so the ``- pkg: mod_python`` is appended to the watch list. But
-all other statements are overwritten.
 
 Understanding the Render System
 ===============================
@@ -313,7 +311,7 @@ available, ``salt``, ``grains``, and ``pillar``. The ``salt`` object allows for
 any Salt function to be called from within the template, and ``grains`` allows for
 the Grains to be accessed from within the template. A few examples:
 
-``/apache/init.sls:``
+``apache/init.sls:``
 
 .. code-block:: yaml
    :linenos:
@@ -356,7 +354,7 @@ Red Hat, then the name of the Apache package and service needs to be httpd.
 A more aggressive way to use Jinja can be found here, in a module to set up
 a MooseFS distributed filesystem chunkserver:
 
-``/moosefs/chunk.sls:``
+``moosefs/chunk.sls:``
 
 .. code-block:: yaml
    :linenos:
@@ -428,7 +426,7 @@ but a SLS file set to use another renderer can be easily added to the tree.
 
 This example shows a very basic Python SLS file:
 
-``/python/django.sls:``
+``python/django.sls:``
 
 .. code-block:: python
    :linenos:
@@ -467,14 +465,14 @@ needed by using a pure Python SLS.
 Running and debugging salt states.
 ----------------------------------
 
-after writing out your top.sls file, to run it you call
+Once the rules in an SLS are ready, they need to be tested to ensure they
-``salt '*' state.highstate``. If you get back just the hostnames with 
+work properly. To invoke the rules, simply execute ``salt '*' state.highstate``
-a : after, but no return, then chances are there is a problem with the sls
+on the command line. If you get back just the hostnames with a `:` after,
-files.  To debug these, to see what's going on, and see the errors, use the
+but no return, chances are there is a problem with the one or more of the sls
-``salt-call`` command like so: ``salt-call state.highstate -l debug``. This
+files. Use the ``salt-call`` command: ``salt-call state.highstate -l debug``
-should help you figure out what's going wrong.  You can also start the minions
+and examine the output for errors. This should help troubleshoot the issue.
-in the foreground in debug mode, as a possible way to help debug as well.
+The minions can also be started in the foreground in debug mode. Start the
-To start the minion in debug mode call it like this: ``salt-minion -l debug``.
+minion in debug mode with: ``salt-minion -l debug``.
 
 
 Now onto the :doc:`States tutorial, part 1</topics/tutorials/states_pt1>`.

doc/topics/tutorials/states_pt2.rst

@@ -2,12 +2,14 @@
 States tutorial, part 2
 =======================
 
-This tutorial builds on the topic covered in :doc:`part 1 <states_pt1>`. It is
+.. note:: 
-recommended that you begin there.
 
-In the last Salt States tutorial we covered the basics of installing a package.
+  This tutorial builds on the topic covered in :doc:`part 1 <states_pt1>`.
-In this tutorial we will modify our ``webserver.sls`` file to be more
+  It is recommended that you begin there.
-complicated, have requirements, and use even more Salt States.
+
+In the :doc:`last part <states_pt1>` of the Salt States tutorial we covered
+the basics of installing a package. We will now modify our ``webserver.sls``
+file to have requirements, and use even more Salt States.
 
 Call multiple States
 ====================
@@ -158,4 +160,4 @@ Next steps
 ==========
 
 In :doc:`part 3 <states_pt3>` we will discuss how to use includes, extends and
-templating to make hugely complicated State Tree configurations dead-simple.
+templating to make a more complete State Tree configuration.

doc/topics/tutorials/states_pt3.rst

@@ -2,26 +2,27 @@
 States tutorial, part 3
 =======================
 
-This tutorial builds on the topic covered in :doc:`part 2 <states_pt2>`. It is
+.. note::
-recommended that you begin there.
 
-This tutorial will cover more advanced templating and configuration techniques
+  This tutorial builds on the topic covered in :doc:`part1 <states_pt1>` and
-for ``sls`` files.
+  :doc:`part 2 <states_pt2>`. It is recommended that you begin there.
+
+This part of the tutorial will cover more advanced templating and
+configuration techniques for ``sls`` files.
 
 Templating SLS modules
 ======================
 
-SLS modules may require programming logic or inline executions. This is
+SLS modules may require programming logic or inline execution. This is
 accomplished with module templating. The default module templating system used
 is `Jinja2`_  and may be configured by changing the :conf_master:`renderer`
 value in the master config.
 
 .. _`Jinja2`: http://jinja.pocoo.org/
 
-All states are passed through a templating system when they are initially read,
+All states are passed through a templating system when they are initially read.
-so all that is required to make use of the templating system is to add some
+To make use of the templating system, simple add some templating markup.
-templating code. An example of an sls module with templating may look like
+An example of an sls module with templating markup may look like this:
-this:
 
 .. code-block:: yaml
 
@@ -45,8 +46,8 @@ Using Grains in SLS modules
 ===========================
 
 Often times a state will need to behave differently on different systems.
-:doc:`Salt grains </topics/targeting/grains>` can be used from within sls modules. An object
+:doc:`Salt grains </topics/targeting/grains>` objects are made available
-called ``grains`` is made available in the template context:
+in the template context. The `grains` can be used from within sls modules:
 
 .. code-block:: yaml
 
@@ -83,31 +84,31 @@ The Salt module functions are also made available in the template context as
     {% endfor %}
 
 Below is an example that uses the ``network.hwaddr`` function to retrieve the
-MAC address for eth0:
+MAC address for eth0::
 
     salt['network.hwaddr']('eth0')
 
 Advanced SLS module syntax
 ==========================
 
-Last we will cover some incredibly useful techniques for more complex State
+Lastly, we will cover some incredibly useful techniques for more complex State
 trees.
 
 :term:`Include declaration`
 ---------------------------
 
-You have seen an example of how to spread a Salt tree across several files but
+A previous example showed how to spread a Salt tree across several files.
-in order to be able to have :term:`requisite references <requisite reference>`
+Similarly, :term:`requisite references <requisite references>` span multiple
-span multiple files you must use an :term:`include declaration`. For example:
+files by using an :term:`include declaration`. For example:
 
-``python-libs.sls``:
+``python/python-libs.sls``:
 
 .. code-block:: yaml
 
     python-dateutil:
       pkg.installed
 
-``django.sls``:
+``python/django.sls``:
 
 .. code-block:: yaml
 
@@ -126,14 +127,14 @@ You can modify previous declarations by using an :term:`extend declaration`. For
 example the following modifies the Apache tree to also restart Apache when the
 vhosts file is changed:
 
-``apache.sls``:
+``apache/apache.sls``:
 
 .. code-block:: yaml
 
     apache:
       pkg.installed
 
-``mywebsite.sls``:
+``apache/mywebsite.sls``:
 
 .. code-block:: yaml
 
@@ -148,8 +149,9 @@ vhosts file is changed:
 
     /etc/httpd/extra/httpd-vhosts.conf:
       file.managed:
-        - source: salt://httpd-vhosts.conf
+        - source: salt://apache/httpd-vhosts.conf
 
+.. include:: extend_with_require_watch.rst
 
 :term:`Name declaration`
 ------------------------
@@ -158,10 +160,10 @@ You can override the :term:`ID declaration` by using a :term:`name
 declaration`. For example, the previous example is a bit more maintainable if
 rewritten as follows:
 
-``mywebsite.sls``:
+``apache/mywebsite.sls``:
 
 .. code-block:: yaml
-    :emphasize-lines: 8,10,13
+    :emphasize-lines: 8,10,12
 
     include:
       - apache
@@ -175,7 +177,7 @@ rewritten as follows:
     mywebsite:
       file.managed:
         - name: /etc/httpd/extra/httpd-vhosts.conf
-        - source: salt://httpd-vhosts.conf
+        - source: salt://apache/httpd-vhosts.conf
 
 :term:`Names declaration`
 -------------------------

pkg/rpm/0001-Only-expect-args-if-they-are-actually-passed-in.patch

@@ -0,0 +1,26 @@
+From 4d65fbe3ef36e74c41d96f3c33aa3cf35ab4b09b Mon Sep 17 00:00:00 2001
+From: Joseph Hall <perlhoser@gmail.com>
+Date: Tue, 31 Jul 2012 05:34:27 -0600
+Subject: [PATCH 12/13] Only expect args if they are actually passed in
+
+---
+ salt/modules/disk.py | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/salt/modules/disk.py b/salt/modules/disk.py
+index 0fca708..0964962 100644
+--- a/salt/modules/disk.py
++++ b/salt/modules/disk.py
+@@ -33,7 +33,8 @@ def usage(args=None):
+         cmd = 'df -kP'
+     else:
+         cmd = 'df'
+-    cmd = cmd + ' -' + args
++    if args:
++        cmd = cmd + ' -' + args
+     ret = {}
+     out = __salt__['cmd.run'](cmd).split('\n')
+     for line in out:
+-- 
+1.7.11.2
+

pkg/rpm/salt.spec

@@ -9,8 +9,8 @@
 %{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")}
 
 Name: salt
-Version: 0.10.1
+Version: 0.10.2
-Release: 1%{?dist}
+Release: 2%{?dist}
 Summary: A parallel remote execution system
 
 Group:   System Environment/Daemons
@@ -24,6 +24,7 @@ Source4: %{name}-master.service
 Source5: %{name}-syndic.service
 Source6: %{name}-minion.service
 Source7: README.fedora
+Patch0: 0001-Only-expect-args-if-they-are-actually-passed-in.patch
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 
 BuildArch: noarch
@@ -45,7 +46,6 @@ Requires: python26-zmq
 Requires: python26-jinja2
 Requires: python26-PyYAML
 Requires: python26-m2crypto
-Requires: python26-PyXML
 Requires: python26-msgpack
 
 %else
@@ -62,7 +62,6 @@ Requires: python-zmq
 Requires: python-jinja2
 Requires: PyYAML
 Requires: m2crypto
-Requires: PyXML
 Requires: python-msgpack
 
 %endif
@@ -108,6 +107,7 @@ Salt minion is queried and controlled from the master.
 
 %prep
 %setup -q
+%patch0 -p1
 
 %build
 
@@ -261,8 +261,15 @@ fi
 %endif
 
 %changelog
-* Sat Jun 16 2012 Clint Savage <herlo1@gmail.com> - 0.10.0-1
+* Thu Aug 2 2012 Clint Savage <herlo1@gmail.com> - 0.10.2-2
-- Moved to upstream release 0.10.0
+- Fix upstream bug #1730 per RHBZ#845295
+
+* Sat Jul 31 2012 Clint Savage <herlo1@gmail.com> - 0.10.2-1
+- Moved to upstream release 0.10.2
+- Removed PyXML as a dependency
+
+* Sat Jun 16 2012 Clint Savage <herlo1@gmail.com> - 0.10.1-1
+- Moved to upstream release 0.10.1
 
 * Sat Apr 28 2012 Clint Savage <herlo1@gmail.com> - 0.9.9.1-1
 - Moved to upstream release 0.9.9.1

pkg/salt_bash_completion

@@ -0,0 +1,323 @@
+#!/bin/bash
+
+# written by David Pravec
+#   - feel free to /msg alekibango on IRC if you want to talk about this file
+
+# TODO: check if --config|-c was used and use configured config file for queries
+# TODO: solve somehow completion for  salt -G pythonversion:[tab]
+#       (not sure what to do with lists)
+# TODO: --range[tab] --   how?
+# TODO: -E --exsel[tab] -- how?
+# TODO: --compound[tab] -- how?
+# TODO: use history to extract some words, esp. if ${cur} is empty
+# TODO: TEST EVERYTING a lot
+# TODO: cache results of some functions?  where? how long?
+# TODO: is it ok to use '--timeout 2' ?
+
+
+_salt_get_grains(){
+    if [ "$1" = 'local' ] ; then 
+        salt-call --text-out -- grains.ls | sed  's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g'
+    else
+      salt '*' --timeout 2 --text-out -- grains.ls | sed  's/^.*\[//' | tr -d ",']" |sed 's:\([a-z0-9]\) :\1\: :g'
+    fi
+}
+
+_salt_get_grain_values(){
+    if [ "$1" = 'local' ] ; then
+        salt-call --text-out -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' 
+    else
+        salt '*' --timeout 2 --text-out -- grains.item $1 |sed 's/^\S*:\s//' |grep -v '^\s*$' 
+    fi
+}
+
+
+_salt(){
+    local cur prev opts _salt_grains _salt_coms pprev ppprev 
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    if [ ${COMP_CWORD} -gt 2 ]; then
+	pprev="${COMP_WORDS[COMP_CWORD-2]}"
+    fi
+    if [ ${COMP_CWORD} -gt 3 ]; then
+	ppprev="${COMP_WORDS[COMP_CWORD-3]}"
+    fi
+
+    opts="--help -h --version -c --compound --raw-out --text-out --json-out --no-color \
+          --timeout -t --static -s --batch-size -b -E --pcre -L --list \
+          -G --grain --grain-pcre -X --exsel -N --nodegroup -R --range --return \
+          -Q --query -c --config -s --static -t --timeout \
+          -b --batch-size  -X --exsel" 
+          
+    if [[ "${cur}" == -* ]] ; then
+        COMPREPLY=($(compgen -W "${opts}" -- ${cur}))
+        return 0
+    fi
+
+    # 2 special cases for filling up grain values
+    case "${pprev}" in
+    -G|--grain|--grain-pcre)
+    if [ "${cur}" = ":" ]; then
+        COMPREPLY=($(compgen -W "`_salt_get_grain_values ${prev}`"  ))	
+        return 0
+    fi
+    ;;
+    esac 
+    case "${ppprev}" in
+    -G|--grain|--grain-pcre)
+        if [ "${prev}" = ":" ]; then
+        COMPREPLY=( $(compgen -W "`_salt_get_grain_values ${pprev}`" -- ${cur}) )
+        return 0
+        fi
+    ;;
+    esac  
+ 
+    if [ "${cur}" = "=" ] && [[ "${prev}" == --* ]]; then
+       cur="" 
+    fi
+    if [ "${prev}" = "=" ] && [[ "${pprev}" == --* ]]; then
+       prev="${pprev}"
+    fi
+ 
+   case "${prev}" in
+ 
+     -c|--config)
+        COMPREPLY=($(compgen -f -- ${cur}))
+        return 0
+        ;;
+     salt)
+        COMPREPLY=($(compgen -W "\'*\' ${opts} `salt-key --no-color -l acc`" -- ${cur}))
+        return 0
+        ;;
+     -E|--pcre) 
+        COMPREPLY=($(compgen -W "`salt-key --no-color -l acc`" -- ${cur}))
+        return 0
+        ;;
+     -G|--grain|--grain-pcre)
+        COMPREPLY=($(compgen -W "$(_salt_get_grains)" -- ${cur})) 
+        return 0
+	;;
+     -C|--compound)
+        COMPREPLY=() # TODO: finish this one? how?
+        return 0
+        ;;
+     -t|--timeout)
+        COMPREPLY=($( compgen -W "1 2 3 4 5 6 7 8 9 10 15 20 30 40 60 90 120 180" -- ${cur}))
+        return 0
+        ;;
+     -b|--batch|--batch-size)
+        COMPREPLY=($(compgen -W "1 2 3 4 5 6 7 8 9 10 15 20 30 40 50 60 70 80 90 100 120 150 200"))
+        return 0
+        ;;
+     -X|--exsel) # TODO: finish this one? how?
+        return 0
+        ;;
+     -N|--nodegroup)  
+	    MASTER_CONFIG='/etc/salt/master'
+        COMPREPLY=($(compgen -W "`awk -F ':'  'BEGIN {print_line = 0};  /^nodegroups/ {print_line = 1;getline } print_line && /^  */ {print $1} /^[^ ]/ {print_line = 0}' <${MASTER_CONFIG}`" -- ${cur})) 
+        return 0  
+     ;;
+    esac
+
+    _salt_coms="$(salt '*' --timeout 2 --text-out -- sys.list_functions | sed 's/^.*\[//' | tr -d ",']" )"
+    all="${opts} ${_salt_coms}"
+    COMPREPLY=( $(compgen -W "${all}" -- ${cur}) )
+
+  return 0
+}
+
+complete -F _salt salt
+
+
+_saltkey(){
+    local cur prev opts prev pprev
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    opts="--version -h --help -l --list -L --list-all -a --accept \
+         -R --reject-all -p --print -P --print-all -r --reject \
+         -d --delete -q --quiet -D --delete-all --key-logfile -c \
+         --config -q --quiet --gen-keys --gen-keys-dir \
+         --keysize accept-all -A "
+    if [ ${COMP_CWORD} -gt 2 ]; then
+        pprev="${COMP_WORDS[COMP_CWORD-2]}"
+    fi
+    if [ ${COMP_CWORD} -gt 3 ]; then
+        ppprev="${COMP_WORDS[COMP_CWORD-3]}"
+    fi
+    if [[ "${cur}" == -* ]] ; then
+        COMPREPLY=($(compgen -W "${opts}" -- ${cur}))
+        return 0
+    fi
+
+    if [ "${cur}" = "=" ] && [[ "${prev}" == --* ]]; then
+       cur="" 
+    fi
+    if [ "${prev}" = "=" ] && [[ "${pprev}" == --* ]]; then
+       prev="${pprev}"
+    fi
+
+    case "${prev}" in 
+     -a|--accept)
+        COMPREPLY=($(compgen -W "$(salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur}))
+        return 0
+      ;;
+     -r|--reject)
+        COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color)" -- ${cur}))
+        return 0
+        ;;
+     -d|--delete)
+        COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color; salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur}))
+        return 0
+        ;;
+     -c|--config)
+        COMPREPLY=($(compgen -f -- ${cur}))
+        return 0
+        ;;
+     --keysize)
+        COMPREPLY=($(compgen -W "2048 3072 4096 5120 6144" -- ${cur}))
+        return 0
+        ;;
+     --gen-keys) 
+        return 0
+        ;;
+     --gen-keys-dir)
+        COMPREPLY=($(compgen -d -- ${cur}))
+        return 0
+        ;;
+     -p|--print)
+        COMPREPLY=($(compgen -W "$(salt-key -l acc --no-color; salt-key -l un --no-color; salt-key -l rej --no-color)" -- ${cur}))
+        return 0
+     ;;
+     -l|--list)
+        COMPREPLY=($(compgen -W "pre un acc accepted unaccepted rej rejected all" -- ${cur}))
+        return 0
+     ;;
+     --accept-all)
+	return 0
+     ;;
+    esac
+    COMPREPLY=($(compgen -W "${opts} " -- ${cur}))
+    return 0
+}
+
+complete -F _saltkey salt-key
+
+_saltcall(){
+    local cur prev opts _salt_coms pprev ppprev
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    opts="-h --help -l --log-level  -d --doc -m --module-dirs --raw-out --text-out --yaml-out --json-out --no-color"
+    if [ ${COMP_CWORD} -gt 2 ]; then
+        pprev="${COMP_WORDS[COMP_CWORD-2]}"
+    fi
+    if [ ${COMP_CWORD} -gt 3 ]; then
+        ppprev="${COMP_WORDS[COMP_CWORD-3]}"
+    fi
+    if [[ "${cur}" == -* ]] ; then
+        COMPREPLY=($(compgen -W "${opts}" -- ${cur}))
+        return 0
+    fi
+    
+    if [ "${cur}" = "=" ] && [[ ${prev} == --* ]]; then
+       cur=""
+    fi
+    if [ "${prev}" = "=" ] && [[ ${pprev} == --* ]]; then
+       prev="${pprev}"
+    fi
+    
+    case ${prev} in
+        -m|--module-dirs)
+                COMPREPLY=( $(compgen -d ${cur} ))
+		return 0
+ 	 	;;
+	-l|--log-level)
+		COMPREPLY=( $(compgen -W "info none garbage trace warning error debug" -- ${cur}))
+		return 0
+		;;
+	-g|grains)
+                return 0
+		;;
+	salt-call)
+                COMPREPLY=($(compgen -W "${opts}" -- ${cur}))
+	        return 0
+		;;
+    esac
+
+    _salt_coms="$(salt-call --text-out -- sys.list_functions|sed 's/^.*\[//' | tr -d ",']"  )"
+    COMPREPLY=( $(compgen -W "${opts} ${_salt_coms}" -- ${cur} ))
+    return 0
+}
+
+complete -F _saltcall salt-call
+
+
+_saltcp(){
+    local cur prev opts target prefpart postpart helper filt pprev ppprev
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    opts="-h --help -L --list -E --pcre -G --grain  --grain-pcre -R --range -C --compound -c --config= -t --timeout= " 
+    if [[ "${cur}" == -* ]] ; then
+        COMPREPLY=($(compgen -W "${opts}" -- ${cur}))
+        return 0
+    fi
+    
+    if [ "${cur}" = "=" ] && [[ "${prev}" == --* ]]; then
+       cur="" 
+    fi
+    if [ "${prev}" = "=" ] && [[ "${pprev}" == --* ]]; then
+       prev=${pprev}
+    fi
+    
+    case ${prev} in
+ 	salt-cp)
+	    COMPREPLY=($(compgen -W "${opts} `salt-key -l acc --no-color`" -- ${cur}))
+	    return 0
+	;;       
+        -t|--timeout)
+	    # those numbers are just a hint
+            COMPREPLY=($(compgen -W "2 3 4 8 10 15 20 25 30 40 60 90 120 180 240 300" -- ${cur} ))
+	    return 0
+        ;;
+	-E|--pcre)
+            COMPREPLY=($(compgen -W "`salt-key -l acc --no-color`" -- ${cur}))
+            return 0
+	;;
+	-L|--list)
+	    # IMPROVEMENTS ARE WELCOME
+	    prefpart="${cur%,*},"
+	    postpart=${cur##*,}
+	    filt="^\($(echo ${cur}| sed 's:,:\\|:g')\)$"
+            helper=($(salt-key -l acc --no-color | grep -v "${filt}" | sed "s/^/${prefpart}/"))
+	    COMPREPLY=($(compgen -W "${helper[*]}" -- ${cur}))
+
+	    return 0
+	;;
+	-G|--grain|--grain-pcre)
+            COMPREPLY=($(compgen -W "$(_salt_get_grains)" -- ${cur})) 
+            return 0
+	    ;;
+	    # FIXME
+	-R|--range)
+	    # FIXME ??
+	    return 0
+	;;
+	-C|--compound)
+	    # FIXME ??
+	    return 0
+	;;
+	-c|--config)
+	    COMPREPLY=($(compgen -f -- ${cur}))
+	    return 0
+	;;
+    esac
+   
+   # default is using opts:
+   COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+}
+
+complete -F _saltcp salt-cp
+

salt/__init__.py

@@ -1,378 +1,165 @@
 '''
 Make me some salt!
 '''
-from salt.version import __version__
 
 # Import python libs
 import os
 import sys
+import logging
+import getpass
 
-# Import salt libs, the try block bypasses an issue at build time so that c
+# Import salt libs, the try block bypasses an issue at build time so that
 # modules don't cause the build to fail
+from salt.version import __version__
+
 try:
     import salt.config
-    from salt.utils import parser as optparse
+    from salt.utils import parsers
-    from salt.utils.process import set_pidfile
     from salt.utils.verify import check_user, verify_env, verify_socket
 except ImportError as e:
     if e.args[0] != 'No module named _msgpack':
         raise
 
-class Master(object):
+
+class Master(parsers.MasterOptionParser):
     '''
     Creates a master server
     '''
-    def __init__(self):
-        self.cli = self.__parse_cli()
-        # command line overrides config
-        if self.cli['user']:
-            self.opts['user'] = self.cli['user']
-
-        # Send the pidfile location to the opts
-        if self.cli['pidfile']:
-            self.opts['pidfile'] = self.cli['pidfile']
-
-    def __parse_cli(self):
-        '''
-        Parse the cli for options passed to a master daemon
-        '''
-        import salt.log
-        parser = optparse.OptionParser(version="%%prog %s" % __version__)
-        parser.add_option('-d',
-                '--daemon',
-                dest='daemon',
-                default=False,
-                action='store_true',
-                help='Run the master as a daemon')
-        parser.add_option('-c',
-                '--config',
-                dest='config',
-                default='/etc/salt/master',
-                help='Pass in an alternative configuration file')
-        parser.add_option('-u',
-                '--user',
-                dest='user',
-                help='Specify user to run master')
-        parser.add_option('--pid-file',
-                dest='pidfile',
-                help=('Specify the location of the pidfile.'))
-        parser.add_option('-l',
-                '--log-level',
-                dest='log_level',
-                choices=list(salt.log.LOG_LEVELS),
-                help='Console log level. One of %s. For the logfile settings '
-                     'see the config file. Default: \'warning\'.' %
-                     ', '.join([repr(l) for l in salt.log.SORTED_LEVEL_NAMES]))
-
-        options, args = parser.parse_args()
-
-        self.opts = salt.config.master_config(options.config)
-
-        if not options.log_level:
-            options.log_level = self.opts['log_level']
-
-        salt.log.setup_console_logger(
-            options.log_level,
-            log_format=self.opts['log_fmt_console'],
-            date_format=self.opts['log_datefmt']
-        )
-
-        cli = {'daemon': options.daemon,
-               'config': options.config,
-               'user': options.user,
-               'pidfile': options.pidfile}
-
-        return cli
 
     def start(self):
         '''
         Run the sequence to start a salt master server
         '''
+        self.parse_args()
+
         try:
-            verify_env([
+            if self.config['verify_env']:
-                self.opts['pki_dir'],
+                verify_env([
-                os.path.join(self.opts['pki_dir'], 'minions'),
+                    self.config['pki_dir'],
-                os.path.join(self.opts['pki_dir'], 'minions_pre'),
+                    os.path.join(self.config['pki_dir'], 'minions'),
-                os.path.join(self.opts['pki_dir'], 'minions_rejected'),
+                    os.path.join(self.config['pki_dir'], 'minions_pre'),
-                self.opts['cachedir'],
+                    os.path.join(self.config['pki_dir'], 'minions_rejected'),
-                os.path.join(self.opts['cachedir'], 'jobs'),
+                    self.config['cachedir'],
-                os.path.dirname(self.opts['log_file']),
+                    os.path.join(self.config['cachedir'], 'jobs'),
-                self.opts['sock_dir'],
+                    os.path.dirname(self.config['log_file']),
-            ], self.opts['user'],
+                    self.config['sock_dir'],
-            permissive=self.opts['permissive_pki_access'])
+                ],
+                self.config['user'],
+                permissive=self.config['permissive_pki_access'],
+                pki_dir=self.config['pki_dir'],
+                )
         except OSError, err:
             sys.exit(err.errno)
 
-        import salt.log
+        self.setup_logfile_logger()
-        salt.log.setup_logfile_logger(
+        logging.getLogger(__name__).warn('Setting up the Salt Master')
-            self.opts['log_file'],
-            self.opts['log_level_logfile'] or self.opts['log_level'],
-            log_format=self.opts['log_fmt_logfile'],
-            date_format=self.opts['log_datefmt']
-        )
-        for name, level in self.opts['log_granular_levels'].items():
-            salt.log.set_logger_level(name, level)
 
-        import logging
-        log = logging.getLogger(__name__)
         # Late import so logging works correctly
-        if not verify_socket(
+        if not verify_socket(self.config['interface'],
-                self.opts['interface'],
+                             self.config['publish_port'],
-                self.opts['publish_port'],
+                             self.config['ret_port']):
-                self.opts['ret_port']
+            self.exit(4, 'The ports are not available to bind\n')
-                ):
-            log.critical('The ports are not available to bind')
-            sys.exit(4)
 
         import salt.master
-        master = salt.master.Master(self.opts)
+        master = salt.master.Master(self.config)
-        if self.cli['daemon']:
+        self.daemonize_if_required()
-            # Late import so logging works correctly
+        self.set_pidfile()
-            import salt.utils
+        if check_user(self.config['user']):
-            salt.utils.daemonize()
-        set_pidfile(self.opts['pidfile'])
-        if check_user(self.opts['user'], log):
             try:
                 master.start()
             except salt.master.MasterExit:
                 sys.exit()
 
 
-class Minion(object):
+class Minion(parsers.MinionOptionParser):
     '''
     Create a minion server
     '''
-    def __init__(self):
-        self.cli = self.__parse_cli()
-        # command line overrides config
-        if self.cli['user']:
-            self.opts['user'] = self.cli['user']
-
-    def __parse_cli(self):
-        '''
-        Parse the cli input
-        '''
-        import salt.log
-        parser = optparse.OptionParser(version="%%prog %s" % __version__)
-        parser.add_option('-d',
-                '--daemon',
-                dest='daemon',
-                default=False,
-                action='store_true',
-                help='Run the minion as a daemon')
-        parser.add_option('-c',
-                '--config',
-                dest='config',
-                default='/etc/salt/minion',
-                help='Pass in an alternative configuration file')
-        parser.add_option('-u',
-                '--user',
-                dest='user',
-                help='Specify user to run minion')
-        parser.add_option('--pid-file',
-                dest='pidfile',
-                default='/var/run/salt-minion.pid',
-                help=('Specify the location of the pidfile. Default'
-                      ' %default'))
-        parser.add_option('-l',
-                '--log-level',
-                dest='log_level',
-                choices=list(salt.log.LOG_LEVELS),
-                help='Console log level. One of %s. For the logfile settings '
-                     'see the config file. Default: \'warning\'.' %
-                     ', '.join([repr(l) for l in salt.log.SORTED_LEVEL_NAMES]))
-
-        options, args = parser.parse_args()
-
-        self.opts = salt.config.minion_config(options.config)
-
-        if not options.log_level:
-            options.log_level = self.opts['log_level']
-
-        salt.log.setup_console_logger(
-            options.log_level,
-            log_format=self.opts['log_fmt_console'],
-            date_format=self.opts['log_datefmt']
-        )
-
-        cli = {'daemon': options.daemon,
-               'config': options.config,
-               'user': options.user,
-               'pidfile': options.pidfile}
-
-        return cli
 
     def start(self):
         '''
         Execute this method to start up a minion.
         '''
+        self.parse_args()
+
         try:
-            verify_env([
+            if self.config['verify_env']:
-                self.opts['pki_dir'],
+                verify_env([
-                self.opts['cachedir'],
+                    self.config['pki_dir'],
-                self.opts['sock_dir'],
+                    self.config['cachedir'],
-                self.opts['extension_modules'],
+                    self.config['sock_dir'],
-                os.path.dirname(self.opts['log_file']),
+                    self.config['extension_modules'],
-            ], self.opts['user'],
+                    os.path.dirname(self.config['log_file']),
-            permissive=self.opts['permissive_pki_access'])
+                ],
+                self.config['user'],
+                permissive=self.config['permissive_pki_access'],
+                pki_dir=self.config['pki_dir'],
+                )
         except OSError, err:
             sys.exit(err.errno)
 
-        import salt.log
+        self.setup_logfile_logger()
-        salt.log.setup_logfile_logger(
+        logging.getLogger(__name__).warn(
-            self.opts['log_file'],
+            'Setting up the Salt Minion "{0}"'.format(
-            self.opts['log_level_logfile'] or self.opts['log_level'],
+                self.config['id']
-            log_format=self.opts['log_fmt_logfile'],
+            )
-            date_format=self.opts['log_datefmt']
         )
-        for name, level in self.opts['log_granular_levels'].items():
-            salt.log.set_logger_level(name, level)
 
-        import logging
         # Late import so logging works correctly
         import salt.minion
-        log = logging.getLogger(__name__)
+        # If the minion key has not been accepted, then Salt enters a loop
-        if self.cli['daemon']:
+        # waiting for it, if we daemonize later then the minion could halt
-            # Late import so logging works correctly
+        # the boot process waiting for a key to be accepted on the master.
-            import salt.utils
+        # This is the latest safe place to daemonize
-            # If the minion key has not been accepted, then Salt enters a loop
+        self.daemonize_if_required()
-            # waiting for it, if we daemonize later then the minion could halt
-            # the boot process waiting for a key to be accepted on the master.
-            # This is the latest safe place to daemonize
-            salt.utils.daemonize()
         try:
-            minion = salt.minion.Minion(self.opts)
+            minion = salt.minion.Minion(self.config)
-            set_pidfile(self.cli['pidfile'])
+            self.set_pidfile()
-            if check_user(self.opts['user'], log):
+            if check_user(self.config['user']):
                 minion.tune_in()
         except KeyboardInterrupt:
             log.warn('Stopping the Salt Minion')
             raise SystemExit('\nExiting on Ctrl-c')
 
 
-class Syndic(object):
+class Syndic(parsers.SyndicOptionParser):
     '''
     Create a syndic server
     '''
-    def __init__(self):
-        self.cli = self.__parse_cli()
-        # command line overrides config
-        if self.cli['user']:
-            self.opts['user'] = self.cli['user']
-
-    def __prep_opts(self, cli):
-        '''
-        Generate the opts used by the syndic
-        '''
-        opts = salt.config.master_config(cli['master_config'])
-        opts['_minion_conf_file'] = opts['conf_file']
-        opts.update(salt.config.minion_config(cli['minion_config']))
-        if 'syndic_master' in opts:
-            # Some of the opts need to be changed to match the needed opts
-            # in the minion class.
-            opts['master'] = opts['syndic_master']
-            opts['master_ip'] = salt.utils.dns_check(opts['master'])
-
-            opts['master_uri'] = ('tcp://' + opts['master_ip'] +
-                                  ':' + str(opts['master_port']))
-            opts['_master_conf_file'] = opts['conf_file']
-            opts.pop('conf_file')
-            return opts
-        err = ('The syndic_master needs to be configured in the salt master '
-               'config, EXITING!\n')
-        sys.stderr.write(err)
-        sys.exit(2)
-
-    def __parse_cli(self):
-        '''
-        Parse the cli for options passed to a syndic daemon
-        '''
-        import salt.log
-        parser = optparse.OptionParser(version="%%prog %s" % __version__)
-        parser.add_option('-d',
-                '--daemon',
-                dest='daemon',
-                default=False,
-                action='store_true',
-                help='Run the syndic as a daemon')
-        parser.add_option('--master-config',
-                dest='master_config',
-                default='/etc/salt/master',
-                help='Pass in an alternative master configuration file')
-        parser.add_option('--minion-config',
-                dest='minion_config',
-                default='/etc/salt/minion',
-                help='Pass in an alternative minion configuration file')
-        parser.add_option('-u',
-                '--user',
-                dest='user',
-                help='Specify user to run syndic')
-        parser.add_option('--pid-file',
-                dest='pidfile',
-                default='/var/run/salt-syndic.pid',
-                help=('Specify the location of the pidfile. Default'
-                      ' %default'))
-        parser.add_option('-l',
-                '--log-level',
-                dest='log_level',
-                choices=list(salt.log.LOG_LEVELS),
-                help='Console log level. One of %s. For the logfile settings '
-                     'see the config file. Default: \'warning\'.' %
-                     ', '.join([repr(l) for l in salt.log.LOG_LEVELS]))
-
-        options, args = parser.parse_args()
-
-        cli = {'daemon': options.daemon,
-               'minion_config': options.minion_config,
-               'master_config': options.master_config,
-               'pidfile': options.pidfile,
-               'user': options.user}
-
-        self.opts = self.__prep_opts(cli)
-
-        if not options.log_level:
-            options.log_level = self.opts['log_level']
-
-        salt.log.setup_console_logger(
-            options.log_level,
-            log_format=self.opts['log_fmt_console'],
-            date_format=self.opts['log_datefmt']
-        )
-
-        return cli
 
     def start(self):
         '''
         Execute this method to start up a syndic.
         '''
+        self.parse_args()
         try:
-            verify_env([
+            if self.config['verify_env']:
-                self.opts['pki_dir'], self.opts['cachedir'],
+                verify_env([
-                os.path.dirname(self.opts['log_file']),
+                        self.config['pki_dir'], self.config['cachedir'],
-            ], self.opts['user'],
+                        os.path.dirname(self.config['log_file']),
-            permissive=self.opts['permissive_pki_access'])
+                    ],
+                    self.config['user'],
+                    permissive=self.config['permissive_pki_access'],
+                    pki_dir=self.config['pki_dir'],
+                )
         except OSError, err:
             sys.exit(err.errno)
-        import salt.log
-        salt.log.setup_logfile_logger(
-            self.opts['log_file'], self.opts['log_level']
-        )
-        for name, level in self.opts['log_granular_levels'].items():
-            salt.log.set_logger_level(name, level)
 
-        import logging
+        self.setup_logfile_logger()
+        logging.getLogger(__name__).warn(
+            'Setting up the Salt Syndic Minion "{0}"'.format(
+                self.config['id']
+            )
+        )
 
         # Late import so logging works correctly
         import salt.minion
-        log = logging.getLogger(__name__)
+        self.daemonize_if_required()
-        if self.cli['daemon']:
+        self.set_pidfile()
-            # Late import so logging works correctly
+
-            import salt.utils
+        if check_user(self.config['user']):
-            salt.utils.daemonize()
-        set_pidfile(self.cli['pidfile'])
-        if check_user(self.opts['user'], log):
             try:
-                syndic = salt.minion.Syndic(self.opts)
+                syndic = salt.minion.Syndic(self.config)
                 syndic.tune_in()
             except KeyboardInterrupt:
                 log.warn('Stopping the Salt Syndic Minion')

salt/_compat.py

@@ -38,16 +38,20 @@ else:
 
 
 def text_(s, encoding='latin-1', errors='strict'):
-    """ If ``s`` is an instance of ``binary_type``, return
+    '''
-    ``s.decode(encoding, errors)``, otherwise return ``s``"""
+    If ``s`` is an instance of ``binary_type``, return
+    ``s.decode(encoding, errors)``, otherwise return ``s``
+    '''
     if isinstance(s, binary_type):
         return s.decode(encoding, errors)
     return s
 
 
 def bytes_(s, encoding='latin-1', errors='strict'):
-    """ If ``s`` is an instance of ``text_type``, return
+    '''
-    ``s.encode(encoding, errors)``, otherwise return ``s``"""
+    If ``s`` is an instance of ``text_type``, return
+    ``s.encode(encoding, errors)``, otherwise return ``s``
+    '''
     if isinstance(s, text_type):
         return s.encode(encoding, errors)
     return s
@@ -64,37 +68,41 @@ else:
             s = s.encode('ascii')
         return str(s)
 
-ascii_native_.__doc__ = """
+ascii_native_.__doc__ = '''
 Python 3: If ``s`` is an instance of ``text_type``, return
 ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')``
 
 Python 2: If ``s`` is an instance of ``text_type``, return
 ``s.encode('ascii')``, otherwise return ``str(s)``
-"""
+'''
 
 
 if PY3:
     def native_(s, encoding='latin-1', errors='strict'):
-        """ If ``s`` is an instance of ``text_type``, return
+        '''
-        ``s``, otherwise return ``str(s, encoding, errors)``"""
+        If ``s`` is an instance of ``text_type``, return
+        ``s``, otherwise return ``str(s, encoding, errors)``
+        '''
         if isinstance(s, text_type):
             return s
         return str(s, encoding, errors)
 else:
     def native_(s, encoding='latin-1', errors='strict'):
-        """ If ``s`` is an instance of ``text_type``, return
+        '''
-        ``s.encode(encoding, errors)``, otherwise return ``str(s)``"""
+        If ``s`` is an instance of ``text_type``, return
+        ``s.encode(encoding, errors)``, otherwise return ``str(s)``
+        '''
         if isinstance(s, text_type):
             return s.encode(encoding, errors)
         return str(s)
 
-native_.__doc__ = """
+native_.__doc__ = '''
 Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise
 return ``str(s, encoding, errors)``
 
 Python 2: If ``s`` is an instance of ``text_type``, return
 ``s.encode(encoding, errors)``, otherwise return ``str(s)``
-"""
+'''
 
 if PY3:
     from urllib import parse

salt/cli/__init__.py

@@ -4,7 +4,6 @@ The management of salt command line utilities are stored in here
 
 # Import python libs
 import os
-import sys
 
 # Import salt components
 import salt.cli.caller
@@ -15,309 +14,68 @@ import salt.client
 import salt.output
 import salt.runner
 
-from salt.utils import parser as optparse
+from salt.utils import parsers
 from salt.utils.verify import verify_env
-from salt import __version__ as VERSION
+from salt.exceptions import SaltInvocationError, SaltClientError
-from salt.exceptions import SaltInvocationError, SaltClientError, \
-    SaltException
 
 
-class SaltCMD(object):
+class SaltCMD(parsers.SaltCMDOptionParser):
     '''
     The execution of a salt command happens here
     '''
-    def __init__(self):
-        '''
-        Create a SaltCMD object
-        '''
-        self.opts = self.__parse()
-
-    def __parse(self):
-        '''
-        Parse the command line
-        '''
-        usage = "%prog [options] '<target>' <function> [arguments]"
-        parser = optparse.OptionParser(version="%%prog %s" % VERSION, usage=usage)
-
-        parser.add_option('-t',
-                '--timeout',
-                default=None,
-                dest='timeout',
-                help=('Set the return timeout for batch jobs; '
-                      'default=5 seconds'))
-        parser.add_option('-s',
-                '--static',
-                default=False,
-                dest='static',
-                action='store_true',
-                help=('Return the data from minions as a group after they '
-                      'all return.'))
-        parser.add_option('-v',
-                '--verbose',
-                default=False,
-                dest='verbose',
-                action='store_true',
-                help=('Turn on command verbosity, display jid and active job '
-                     'queries'))
-        parser.add_option('-b',
-                '--batch',
-                '--batch-size',
-                default='',
-                dest='batch',
-                help=('Execute the salt job in batch mode, pass either the '
-                      'number of minions to batch at a time, or the '
-                      'percentage of minions to have running'))
-        parser.add_option('-E',
-                '--pcre',
-                default=False,
-                dest='pcre',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'servers, use pcre regular expressions'))
-        parser.add_option('-L',
-                '--list',
-                default=False,
-                dest='list',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'servers, take a comma delimited list of servers.'))
-        parser.add_option('-G',
-                '--grain',
-                default=False,
-                dest='grain',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use a grain value to identify targets, the syntax '
-                      'for the target is the grain key followed by a glob'
-                      'expression:\n"os:Arch*"'))
-        parser.add_option('--grain-pcre',
-                default=False,
-                dest='grain_pcre',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use a grain value to identify targets, the syntax '
-                      'for the target is the grain key followed by a pcre '
-                      'regular expression:\n"os:Arch.*"'))
-        parser.add_option('-X',
-                '--exsel',
-                default=False,
-                dest='exsel',
-                action='store_true',
-                help=('Instead of using shell globs use the return code '
-                      'of a function.'))
-        parser.add_option('-I',
-                '--pillar',
-                default=False,
-                dest='pillar',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use a pillar value to identify targets, the syntax '
-                      'for the target is the pillar key followed by a glob'
-                      'expression:\n"role:production*"'))
-        parser.add_option('-N',
-                '--nodegroup',
-                default=False,
-                dest='nodegroup',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use one of the predefined nodegroups to identify a '
-                      'list of targets.'))
-        parser.add_option('-R',
-                '--range',
-                default=False,
-                dest='range',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use a range expression to identify targets. '
-                      'Range expressions look like %cluster'))
-        parser.add_option('-C',
-                '--compound',
-                default=False,
-                dest='compound',
-                action='store_true',
-                help=('The compound target option allows for multiple '
-                       'target types to be evaluated, allowing for greater '
-                       'granularity in target matching. The compound target '
-                       'is space delimited, targets other than globs are '
-                       'preceted with an identifyer matching the specific '
-                       'targets argument type: salt \'G@os:RedHat and '
-                       'webser* or E@database.*\''))
-        parser.add_option('--return',
-                default='',
-                dest='return',
-                metavar='RETURNER',
-                help=('Set an alternative return method. By default salt will '
-                      'send the return data from the command back to the '
-                      'master, but the return data can be redirected into '
-                      'any number of systems, databases or applications.'))
-        parser.add_option('-Q',
-                '--query',
-                dest='query',
-                action='store_true',
-                help=('This option is deprecated and will be removed in a '
-                      'future release, please use salt-run jobs instead\n'
-                      'Execute a salt command query, this can be used to find '
-                      'the results of a previous function call: -Q test.echo'))
-        parser.add_option('-c',
-                '--config',
-                default='/etc/salt/master',
-                dest='conf_file',
-                help=('The location of the salt master configuration file, '
-                      'the salt master settings are required to know where '
-                      'the connections are; default=/etc/salt/master'))
-        parser.add_option('--raw-out',
-                default=False,
-                action='store_true',
-                dest='raw_out',
-                help=('Print the output from the salt command in raw python '
-                      'form, this is suitable for re-reading the output into '
-                      'an executing python script with eval.'))
-        parser.add_option('--text-out',
-                default=False,
-                action='store_true',
-                dest='txt_out',
-                help=('Print the output from the salt command in the same '
-                      'form the shell would.'))
-        parser.add_option('--yaml-out',
-                default=False,
-                action='store_true',
-                dest='yaml_out',
-                help='Print the output from the salt command in yaml.')
-        parser.add_option('--json-out',
-                default=False,
-                action='store_true',
-                dest='json_out',
-                help='Print the output from the salt command in json.')
-        parser.add_option('--no-color',
-                default=False,
-                action='store_true',
-                dest='no_color',
-                help='Disable all colored output')
-
-        options, args = parser.parse_args()
-
-        opts = {}
-
-        for k, v in options.__dict__.items():
-            if v is not None:
-                opts[k] = v
-
-        if not options.timeout is None:
-            opts['timeout'] = int(options.timeout)
-
-        if options.query:
-            opts['query'] = options.query
-            if len(args) < 1:
-                err = ('Please pass in a command to query the old salt '
-                       'calls for.')
-                sys.stderr.write(err + '\n')
-                sys.exit('2')
-            opts['cmd'] = args[0]
-        else:
-            # Catch invalid invocations of salt such as: salt run
-            if len(args) <= 1:
-                parser.print_help()
-                parser.exit(1)
-
-            if opts['list']:
-                opts['tgt'] = args[0].split(',')
-            else:
-                opts['tgt'] = args[0]
-
-            # Detect compound command and set up the data for it
-            if ',' in args[1]:
-                opts['fun'] = args[1].split(',')
-                opts['arg'] = []
-                for comp in ' '.join(args[2:]).split(','):
-                    opts['arg'].append(comp.split())
-                if len(opts['fun']) != len(opts['arg']):
-                    err = ('Cannot execute compound command without defining '
-                           'all arguments.')
-                    sys.stderr.write(err + '\n')
-                    sys.exit(42)
-            else:
-                opts['fun'] = args[1]
-                opts['arg'] = args[2:]
-
-        return opts
 
     def run(self):
         '''
         Execute the salt command line
         '''
+        self.parse_args()
+
         try:
-            local = salt.client.LocalClient(self.opts['conf_file'])
+            local = salt.client.LocalClient(self.get_config_file_path('master'))
         except SaltClientError as exc:
-            sys.stderr.write('{0}\n'.format(exc))
+            self.exit(2, '{0}\n'.format(exc))
-            sys.exit(2)
             return
 
-        if 'query' in self.opts:
+        if self.options.query:
-            ret = local.find_cmd(self.opts['cmd'])
+            ret = local.find_cmd(self.config['cmd'])
             for jid in ret:
                 if isinstance(ret, list) or isinstance(ret, dict):
-                    # Determine the proper output method and run it
-                    get_outputter = salt.output.get_outputter
-                    if self.opts['raw_out']:
-                        printout = get_outputter('raw')
-                    elif self.opts['json_out']:
-                        printout = get_outputter('json')
-                    elif self.opts['txt_out']:
-                        printout = get_outputter('txt')
-                    elif self.opts['yaml_out']:
-                        printout = get_outputter('yaml')
-                    else:
-                        printout = get_outputter(None)
-
                     print('Return data for job {0}:'.format(jid))
-                    printout(ret[jid])
+                    salt.output.display_output(ret[jid], None, self.config)
                     print('')
-        elif self.opts['batch']:
+        elif self.options.batch:
-            batch = salt.cli.batch.Batch(self.opts)
+            batch = salt.cli.batch.Batch(self.config)
             batch.run()
         else:
-            if not 'timeout' in self.opts:
+            if self.options.timeout <= 0:
-                self.opts['timeout'] = local.opts['timeout']
+                self.options.timeout = local.opts['timeout']
-            args = [self.opts['tgt'],
+
-                    self.opts['fun'],
+            args = [
-                    self.opts['arg'],
+                self.config['tgt'],
-                    self.opts['timeout'],
+                self.config['fun'],
-                    ]
+                self.config['arg'],
-            if self.opts['pcre']:
+                self.options.timeout,
-                args.append('pcre')
+            ]
-            elif self.opts['list']:
+
-                args.append('list')
+            if self.selected_target_option:
-            elif self.opts['grain']:
+                args.append(self.selected_target_option)
-                args.append('grain')
-            elif self.opts['grain_pcre']:
-                args.append('grain_pcre')
-            elif self.opts['exsel']:
-                args.append('exsel')
-            elif self.opts['pillar']:
-                args.append('pillar')
-            elif self.opts['nodegroup']:
-                args.append('nodegroup')
-            elif self.opts['range']:
-                args.append('range')
-            elif self.opts['compound']:
-                args.append('compound')
             else:
                 args.append('glob')
 
-            if self.opts['return']:
+            if getattr(self.options, 'return'):
-                args.append(self.opts['return'])
+                args.append(getattr(self.options, 'return'))
             else:
                 args.append('')
             try:
                 # local will be None when there was an error
                 if local:
-                    if self.opts['static']:
+                    if self.options.static:
-                        if self.opts['verbose']:
+                        if self.options.verbose:
                             args.append(True)
                         full_ret = local.cmd_full_return(*args)
                         ret, out = self._format_ret(full_ret)
                         self._output_ret(ret, out)
-                    elif self.opts['fun'] == 'sys.doc':
+                    elif self.config['fun'] == 'sys.doc':
                         ret = {}
                         out = ''
                         for full_ret in local.cmd_cli(*args):
@@ -325,7 +83,7 @@ class SaltCMD(object):
                             ret.update(ret_)
                         self._output_ret(ret, out)
                     else:
-                        if self.opts['verbose']:
+                        if self.options.verbose:
                             args.append(True)
                         for full_ret in local.cmd_cli(*args):
                             ret, out = self._format_ret(full_ret)
@@ -339,29 +97,11 @@ class SaltCMD(object):
         Print the output from a single return to the terminal
         '''
         # Handle special case commands
-        if self.opts['fun'] == 'sys.doc':
+        if self.config['fun'] == 'sys.doc':
             self._print_docs(ret)
         else:
             # Determine the proper output method and run it
-            get_outputter = salt.output.get_outputter
+            salt.output.display_output(ret, out, self.config)
-            if isinstance(ret, list) or isinstance(ret, dict):
-                if self.opts['raw_out']:
-                    printout = get_outputter('raw')
-                elif self.opts['json_out']:
-                    printout = get_outputter('json')
-                elif self.opts['txt_out']:
-                    printout = get_outputter('txt')
-                elif self.opts['yaml_out']:
-                    printout = get_outputter('yaml')
-                elif out:
-                    printout = get_outputter(out)
-                else:
-                    printout = get_outputter(None)
-            # Pretty print any salt exceptions
-            elif isinstance(ret, SaltException):
-                printout = get_outputter("txt")
-            color = not bool(self.opts['no_color'])
-            printout(ret, color=color)
 
     def _format_ret(self, full_ret):
         '''
@@ -381,7 +121,8 @@ class SaltCMD(object):
         '''
         docs = {}
         if not ret:
-            sys.stderr.write('No minions found to gather docs from\n')
+            self.exit(2, 'No minions found to gather docs from\n')
+
         for host in ret:
             for fun in ret[host]:
                 if fun not in docs:
@@ -393,485 +134,104 @@ class SaltCMD(object):
             print('')
 
 
-class SaltCP(object):
+class SaltCP(parsers.SaltCPOptionParser):
     '''
     Run the salt-cp command line client
     '''
-    def __init__(self):
-        self.opts = self.__parse()
-
-    def __parse(self):
-        '''
-        Parse the command line
-        '''
-        usage = "%prog [options] '<target>' SOURCE DEST"
-        parser = optparse.OptionParser(version="%%prog %s" % VERSION, usage=usage)
-
-        parser.add_option('-t',
-                '--timeout',
-                default=5,
-                type=int,
-                dest='timeout',
-                help=('Set the return timeout for batch jobs; '
-                      'default=5 seconds'))
-        parser.add_option('-E',
-                '--pcre',
-                default=False,
-                dest='pcre',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'servers, use pcre regular expressions'))
-        parser.add_option('-L',
-                '--list',
-                default=False,
-                dest='list',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'servers, take a comma delimited list of servers.'))
-        parser.add_option('-G',
-                '--grain',
-                default=False,
-                dest='grain',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use a grain value to identify targets, the syntax '
-                      'for the target is the grain key followed by a glob'
-                      'expression:\n"os:Arch*"'))
-        parser.add_option('--grain-pcre',
-                default=False,
-                dest='grain_pcre',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use a grain value to identify targets, the syntax '
-                      'for the target is the grain key followed by a pcre '
-                      'regular expression:\n"os:Arch.*"'))
-        parser.add_option('-N',
-                '--nodegroup',
-                default=False,
-                dest='nodegroup',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use one of the predefined nodegroups to identify a '
-                      'list of targets.'))
-        parser.add_option('-R',
-                '--range',
-                default=False,
-                dest='range',
-                action='store_true',
-                help=('Instead of using shell globs to evaluate the target '
-                      'use a range expressions to identify targets. '
-                      'Range expressions look like %cluster'))
-        parser.add_option('-c',
-                '--config',
-                default='/etc/salt/master',
-                dest='conf_file',
-                help=('The location of the salt master configuration file, '
-                      'the salt master settings are required to know where '
-                      'the connections are; default=/etc/salt/master'))
-
-        options, args = parser.parse_args()
-
-        opts = {}
-
-        for k, v in options.__dict__.items():
-            if v is not None:
-                opts[k] = v
-
-        # salt-cp needs arguments
-        if len(args) <= 1:
-            parser.print_help()
-            parser.exit(1)
-
-        if opts['list']:
-            opts['tgt'] = args[0].split(',')
-        else:
-            opts['tgt'] = args[0]
-        opts['src'] = args[1:-1]
-        opts['dest'] = args[-1]
-
-        return opts
 
     def run(self):
         '''
         Execute salt-cp
         '''
-        cp_ = salt.cli.cp.SaltCP(self.opts)
+        self.parse_args()
+        cp_ = salt.cli.cp.SaltCP(self.config)
         cp_.run()
 
 
-class SaltKey(object):
+class SaltKey(parsers.SaltKeyOptionParser):
     '''
     Initialize the Salt key manager
     '''
-    def __init__(self):
-        self.opts = self.__parse()
-
-    def __parse(self):
-        '''
-        Parse the command line options for the salt key
-        '''
-        parser = optparse.OptionParser(version="%%prog %s" % VERSION)
-
-        parser.add_option('-l',
-                '--list',
-                dest='list',
-                default='',
-                help=('List the public keys. Takes the args: '
-                      '"pre", "un", "unaccepted": Unaccepted/unsigned keys '
-                      '"acc", "accepted": Accepted/signed keys '
-                      '"rej", "rejected": Rejected keys '
-                      '"all": all keys'))
-
-        parser.add_option('-L',
-                '--list-all',
-                dest='list_all',
-                default=False,
-                action='store_true',
-                help='List all public keys.  Deprecated: use "--list all"')
-
-        parser.add_option('-a',
-                '--accept',
-                dest='accept',
-                default='',
-                help='Accept the following key')
-
-        parser.add_option('-A',
-                '--accept-all',
-                dest='accept_all',
-                default=False,
-                action='store_true',
-                help='Accept all pending keys')
-
-        parser.add_option('-r',
-                '--reject',
-                dest='reject',
-                default='',
-                help='Reject the specified public key')
-
-        parser.add_option('-R',
-                '--reject-all',
-                dest='reject_all',
-                default=False,
-                action='store_true',
-                help='Reject all pending keys')
-
-        parser.add_option('-p',
-                '--print',
-                dest='print',
-                default='',
-                help='Print the specified public key')
-
-        parser.add_option('-P',
-                '--print-all',
-                dest='print_all',
-                default=False,
-                action='store_true',
-                help='Print all public keys')
-
-        parser.add_option('-d',
-                '--delete',
-                dest='delete',
-                default='',
-                help='Delete the named key')
-
-        parser.add_option('-D',
-                '--delete-all',
-                dest='delete_all',
-                default=False,
-                action='store_true',
-                help='Delete all keys')
-
-        parser.add_option('-q',
-                '--quiet',
-                dest='quiet',
-                default=False,
-                action='store_true',
-                help='Supress output')
-
-        parser.add_option('-y',
-                '--yes',
-                dest='yes',
-                default=False,
-                action='store_true',
-                help='Answer Yes to all questions presented, defaults to False'
-                )
-
-        parser.add_option('--key-logfile',
-                dest='key_logfile',
-                help=('Send all output to a file. '
-                      'Default is /var/log/salt/key'))
-
-        parser.add_option('--gen-keys',
-                dest='gen_keys',
-                default='',
-                help='Set a name to generate a keypair for use with salt')
-
-        parser.add_option('--gen-keys-dir',
-                dest='gen_keys_dir',
-                default='.',
-                help=('Set the direcotry to save the generated keypair, '
-                      'only works with "gen_keys_dir" option; default=.'))
-
-        parser.add_option('--keysize',
-                dest='keysize',
-                default=2048,
-                type=int,
-                help=('Set the keysize for the generated key, only works with '
-                      'the "--gen-keys" option, the key size must be 2048 or '
-                      'higher, otherwise it will be rounded up to 2048'
-                      '; default=2048'))
-
-        parser.add_option('-c',
-                '--config',
-                dest='conf_file',
-                default='/etc/salt/master',
-                help='Pass in an alternative configuration file')
-
-        parser.add_option('--raw-out',
-                default=False,
-                action='store_true',
-                dest='raw_out',
-                help=('Print the output from the salt-key command in raw python '
-                      'form, this is suitable for re-reading the output into '
-                      'an executing python script with eval.'))
-
-        parser.add_option('--yaml-out',
-                default=False,
-                action='store_true',
-                dest='yaml_out',
-                help='Print the output from the salt-key command in yaml.')
-
-        parser.add_option('--json-out',
-                default=False,
-                action='store_true',
-                dest='json_out',
-                help='Print the output from the salt-key command in json.')
-
-        parser.add_option('--no-color',
-                default=False,
-                action='store_true',
-                dest='no_color',
-                help='Disable all colored output')
-
-        options, args = parser.parse_args()
-
-        opts = {}
-        opts.update(salt.config.master_config(options.conf_file))
-
-        for k, v in options.__dict__.items():
-            if k == 'keysize':
-                if v < 2048:
-                    opts[k] = 2048
-                else:
-                    opts[k] = v
-            elif v is not None:
-                opts[k] = v
-        # I decided to always set this to info, since it really all is info or
-        # error.
-        opts['loglevel'] = 'info'
-        return opts
 
     def run(self):
         '''
-        Execute saltkey
+        Execute salt-key
         '''
-        verify_env([
+        self.parse_args()
-            os.path.join(self.opts['pki_dir'], 'minions'),
+
-            os.path.join(self.opts['pki_dir'], 'minions_pre'),
+        if self.config['verify_env']:
-            os.path.join(self.opts['pki_dir'], 'minions_rejected'),
+            verify_env_dirs = []
-            os.path.dirname(self.opts['log_file']),
+            if not self.config['gen_keys']:
-            ],
+                verify_env_dirs.extend([
-            self.opts['user'],
+                    os.path.join(self.config['pki_dir'], 'minions'),
-            permissive=self.opts['permissive_pki_access'])
+                    os.path.join(self.config['pki_dir'], 'minions_pre'),
-        import salt.log
+                    os.path.join(self.config['pki_dir'], 'minions_rejected'),
-        salt.log.setup_logfile_logger(self.opts['key_logfile'],
+                    os.path.dirname(self.config['key_logfile'])
-                                      self.opts['loglevel'])
+                ])
-        key = salt.cli.key.Key(self.opts)
+
+            verify_env(
+                verify_env_dirs,
+                self.config['user'],
+                permissive=self.config['permissive_pki_access'],
+                pki_dir=self.config['pki_dir'],
+            )
+
+        self.setup_logfile_logger()
+
+        key = salt.cli.key.Key(self.config)
         key.run()
 
 
-class SaltCall(object):
+class SaltCall(parsers.SaltCallOptionParser):
     '''
     Used to locally execute a salt command
     '''
-    def __init__(self):
-        self.opts = self.__parse()
-
-    def __parse(self):
-        '''
-        Parse the command line arguments
-        '''
-        usage = "%prog [options] <function> [arguments]"
-        parser = optparse.OptionParser(
-                version='salt-call {0}'.format(VERSION),
-                usage=usage
-                )
-
-        parser.add_option('-g',
-                '--grains',
-                dest='grains_run',
-                default=False,
-                action='store_true',
-                help='Return the information generated by the salt grains')
-        parser.add_option('-m',
-                '--module-dirs',
-                dest='module_dirs',
-                default='',
-                help=('Specify an additional directories to pull modules '
-                      'from, multiple directories can be delimited by commas'))
-        parser.add_option('-c',
-                '--config',
-                dest='conf_file',
-                default='/etc/salt/minion',
-                help='Pass in an alternative configuration file')
-        parser.add_option('-d',
-                '--doc',
-                dest='doc',
-                default=False,
-                action='store_true',
-                help=('Return the documentation for the specified module of '
-                      'for all modules if none are specified'))
-        parser.add_option('-l',
-                '--log-level',
-                default='info',
-                dest='log_level',
-                help='Set the output level for salt-call')
-        parser.add_option('--raw-out',
-                default=False,
-                action='store_true',
-                dest='raw_out',
-                help=('Print the output from the salt command in raw python '
-                      'form, this is suitable for re-reading the output into '
-                      'an executing python script with eval.'))
-        parser.add_option('--text-out',
-                default=False,
-                action='store_true',
-                dest='txt_out',
-                help=('Print the output from the salt command in the same '
-                      'form the shell would.'))
-        parser.add_option('--yaml-out',
-                default=False,
-                action='store_true',
-                dest='yaml_out',
-                help='Print the output from the salt command in yaml.')
-        parser.add_option('--json-out',
-                default=False,
-                action='store_true',
-                dest='json_out',
-                help='Print the output from the salt command in json.')
-        parser.add_option('--no-color',
-                default=False,
-                dest='no_color',
-                action='store_true',
-                help='Disable all colored output')
-
-        options, args = parser.parse_args()
-
-        opts = {}
-        opts.update(salt.config.minion_config(options.conf_file))
-
-        for k, v in options.__dict__.items():
-            if k == 'module_dirs':
-                opts[k] = v.split(',')
-            else:
-                opts[k] = v
-
-        if len(args) >= 1:
-            opts['fun'] = args[0]
-            opts['arg'] = args[1:]
-        elif opts['grains_run']:
-            pass
-        else:
-            # salt-call should not ever be called without arguments
-            parser.print_help()
-            parser.exit(1)
-
-        verify_env([opts['pki_dir'],
-                    opts['cachedir'],
-                    os.path.dirname(opts['log_file']),
-                    ],
-                    opts['user'],
-                    permissive=opts['permissive_pki_access'])
-
-        return opts
 
     def run(self):
         '''
         Execute the salt call!
         '''
-        import salt.log
+        self.parse_args()
-        salt.log.setup_console_logger(
+
-            self.opts['log_level'],
+        if self.config['verify_env']:
-            log_format=self.opts['log_fmt_console'],
+            verify_env([
-            date_format=self.opts['log_datefmt'],
+                    self.config['pki_dir'],
-        )
+                    self.config['cachedir'],
-        caller = salt.cli.caller.Caller(self.opts)
+                    os.path.dirname(self.config['log_file'])
+                ],
+                self.config['user'],
+                permissive=self.config['permissive_pki_access'],
+                pki_dir=self.config['pki_dir'],
+            )
+
+        caller = salt.cli.caller.Caller(self.config)
+
+        if self.options.doc:
+            caller.print_docs()
+            self.exit(0)
+
+        if self.options.grains_run:
+            caller.print_grains()
+            self.exit(0)
+
         caller.run()
 
 
-class SaltRun(object):
+class SaltRun(parsers.SaltRunOptionParser):
-    '''
-    Used to execute salt convenience functions
-    '''
-    def __init__(self):
-        self.opts = self.__parse()
-
-    def __parse(self):
-        '''
-        Parse the command line arguments
-        '''
-        parser = optparse.OptionParser(version="%%prog %s" % VERSION)
-
-        parser.add_option('-c',
-                '--config',
-                dest='conf_file',
-                default='/etc/salt/master',
-                help=('Change the location of the master configuration; '
-                      'default=/etc/salt/master'))
-
-        parser.add_option('-t',
-                '--timeout',
-                dest='timeout',
-                default='1',
-                help=('Change the timeout, if applicable, for the salt runner; '
-                      'default=1'))
-
-        parser.add_option('-d',
-                '--doc',
-                '--documentation',
-                dest='doc',
-                default=False,
-                action='store_true',
-                help=('Display documentation for runners, pass a module or '
-                      'a runner to see documentation on only that '
-                      'module/runner.'))
-
-        options, args = parser.parse_args()
-
-        opts = {}
-        opts.update(salt.config.master_config(options.conf_file))
-        opts['conf_file'] = options.conf_file
-        opts['doc'] = options.doc
-        if len(args) > 0:
-            opts['fun'] = args[0]
-        else:
-            opts['fun'] = ''
-        if len(args) > 1:
-            opts['arg'] = args[1:]
-        else:
-            opts['arg'] = []
-
-        return opts
 
     def run(self):
         '''
         Execute salt-run
         '''
-        runner = salt.runner.Runner(self.opts)
+        self.parse_args()
-        # Run this here so SystemExit isn't raised
+
-        # anywhere else when someone tries to  use
+        runner = salt.runner.Runner(self.config)
-        # the runners via the python api
+        if self.options.doc:
-        try:
+            runner._print_docs()
-            runner.run()
+        else:
-        except SaltClientError as exc:
+            # Run this here so SystemExit isn't raised anywhere else when
-            raise SystemExit(str(exc))
+            # someone tries to use the runners via the python api
+            try:
+                runner.run()
+            except SaltClientError as exc:
+                raise SystemExit(str(exc))

salt/cli/batch.py

@@ -29,22 +29,10 @@ class Batch(object):
                 [],
                 1,
                 ]
-        if self.opts['pcre']:
+
-            args.append('pcre')
+        selected_target_option = self.opts.get('selected_target_option', None)
-        elif self.opts['list']:
+        if selected_target_option is not None:
-            args.append('list')
+            args.append(selected_target_option)
-        elif self.opts['grain']:
-            args.append('grain')
-        elif self.opts['grain_pcre']:
-            args.append('grain_pcre')
-        elif self.opts['exsel']:
-            args.append('exsel')
-        elif self.opts['pillar']:
-            args.append('pillar')
-        elif self.opts['nodegroup']:
-            args.append('nodegroup')
-        elif self.opts['compound']:
-            args.append('compound')
         else:
             args.append('glob')
 

salt/cli/caller.py

@@ -9,10 +9,9 @@ import logging
 import traceback
 
 # Import salt libs
-import salt
-import salt.utils
 import salt.loader
 import salt.minion
+import salt.output
 from salt._compat import string_types
 from salt.log import LOG_LEVELS
 
@@ -83,7 +82,7 @@ class Caller(object):
                 if func.__doc__:
                     docs[name] = func.__doc__
         for name in sorted(docs):
-            if name.startswith(self.opts['fun']):
+            if name.startswith(self.opts.get('fun', '')):
                 print('{0}:\n{1}\n'.format(name, docs[name]))
 
     def print_grains(self):
@@ -91,44 +90,20 @@ class Caller(object):
         Print out the grains
         '''
         grains = salt.loader.grains(self.opts)
-        printout = self._get_outputter(out='yaml')
+        printout = salt.output.get_printout(grains, 'yaml', self.opts, indent=2)
-        # If --json-out is specified, pretty print it
+        printout(grains, color=not bool(self.opts['no_color']))
-        if 'json_out' in self.opts and self.opts['json_out']:
-            printout.indent = 2
-        printout(grains)
-
-    def _get_outputter(self, out=None):
-        get_outputter = salt.output.get_outputter
-        if self.opts['raw_out']:
-            printout = get_outputter('raw')
-        elif self.opts['json_out']:
-            printout = get_outputter('json')
-        elif self.opts['txt_out']:
-            printout = get_outputter('txt')
-        elif self.opts['yaml_out']:
-            printout = get_outputter('yaml')
-        elif out:
-            printout = get_outputter(out)
-        else:
-            printout = get_outputter(None)
-        return printout
 
     def run(self):
         '''
         Execute the salt call logic
         '''
-        if self.opts['doc']:
+        ret = self.call()
-            self.print_docs()
+        printout = salt.output.get_printout(
-        elif self.opts['grains_run']:
+            ret, ret.get('out', None), self.opts, indent=2
-            self.print_grains()
+        )
-        else:
+        if printout is None:
-            ret = self.call()
+            printout = salt.output.get_outputter(None)
-            # Determine the proper output method and run it
+        printout(
-            if 'out' in ret:
+                {'local': ret['return']},
-                printout = self._get_outputter(ret['out'])
+                color=not bool(self.opts['no_color']),
-            else:
+                **self.opts)
-                printout = self._get_outputter()
-            if 'json_out' in self.opts and self.opts['json_out']:
-                printout.indent = 2
-            color = not bool(self.opts['no_color'])
-            printout({'local': ret['return']}, color=color)

salt/cli/cp.py

@@ -71,18 +71,10 @@ class SaltCP(object):
                 arg,
                 self.opts['timeout'],
                 ]
-        if self.opts['pcre']:
+
-            args.append('pcre')
+        selected_target_option = self.opts.get('selected_target_option', None)
-        elif self.opts['list']:
+        if selected_target_option is not None:
-            args.append('list')
+            args.append(selected_target_option)
-        elif self.opts['grain']:
-            args.append('grain')
-        elif self.opts['grain_pcre']:
-            args.append('grain_pcre')
-        elif self.opts['nodegroup']:
-            args.append('nodegroup')
-        elif self.opts['range']:
-            args.append('range')
 
         ret = local.cmd(*args)
 

salt/cli/key.py

@@ -21,10 +21,27 @@ class Key(object):
     '''
     def __init__(self, opts):
         self.opts = opts
-        self.event = salt.utils.event.SaltEvent(opts['sock_dir'], 'master')
+        self.event = salt.utils.event.MasterEvent(opts['sock_dir'])
         self.colors = salt.utils.get_colors(
                 not bool(self.opts.get('no_color', False))
                 )
+        if not opts.get('gen_keys', None):
+            # Only check for a master running IF we need it.
+            # While generating keys we don't
+            self._check_master()
+
+    def _check_master(self):
+        '''
+        Log if the master is not running
+        '''
+        if not os.path.exists(
+                os.path.join(
+                    self.opts['sock_dir'],
+                    'publish_pull.ipc'
+                    )
+                ):
+            self._log('Master is not running', level='error')
+
 
     def _cli_opts(self, **kwargs):
         '''
@@ -42,7 +59,8 @@ class Key(object):
                 'print_all': False,
                 'delete': '',
                 'delete_all': False,
-                'quiet': Fasle,
+                'finger': '',
+                'quiet': False,
                 'yes': True,
                 'gen_keys': '',
                 'gen_keys_dir': '.',
@@ -87,7 +105,7 @@ class Key(object):
         if hasattr(log, level):
             log_msg = getattr(log, level)
             log_msg(message)
-        if not self.opts['quiet']:
+        if not self.opts.get('quiet', False):
             print(message)
 
     def _list_pre(self, header=True, printer=None):
@@ -148,9 +166,11 @@ class Key(object):
         '''
         List keys
         '''
-        printout = self._get_outputter()
+        selected_output = self.opts.get('selected_output_option', None)
-        if 'json_out' in self.opts and self.opts['json_out']:
+        printout = salt.output.get_printout(
-            printout.indent = 2
+            {}, selected_output, self.opts, indent=2
+        )
+
         if name in ('pre', 'un', 'unaccept', 'unaccepted'):
             self._list_pre(header=False, printer=printout)
         elif name in ('acc', 'accept', 'accepted'):
@@ -174,18 +194,6 @@ class Key(object):
                     ).format(name)
             self._log(err, level='error')
 
-    def _get_outputter(self):
-        get_outputter = salt.output.get_outputter
-        if self.opts['raw_out']:
-            printout = get_outputter('raw')
-        elif self.opts['json_out']:
-            printout = get_outputter('json')
-        elif self.opts['yaml_out']:
-            printout = get_outputter('yaml')
-        else:
-            printout = None # use default color output
-        return printout
-
     def _print_key(self, name):
         '''
         Print out the specified public key
@@ -330,7 +338,6 @@ class Key(object):
         Delete all keys
         '''
         # Don't ask for verification if yes is not set
-        yes = self.opts.get('yes', True)
         del_ = set()
         for dir in ("acc", "rej", "pre"):
             for key in self._keys(dir):
@@ -372,6 +379,9 @@ class Key(object):
             self._reject(key)
 
     def _check_minions_directories(self):
+        '''
+        Return the minion keys directory paths
+        '''
         minions_accepted = os.path.join(self.opts['pki_dir'], 'minions')
         minions_pre = os.path.join(self.opts['pki_dir'], 'minions_pre')
         minions_rejected = os.path.join(self.opts['pki_dir'],
@@ -384,6 +394,32 @@ class Key(object):
                 sys.exit(42)
         return minions_accepted, minions_pre, minions_rejected
 
+    def finger(self):
+        '''
+        Return the fingerprint for a specified key
+        '''
+        fkey = self.opts.get('finger', 'master')
+        dirs = list(self._check_minions_directories())
+        dirs.append(self.opts['pki_dir'])
+        sigs = {}
+        for dir_ in dirs:
+            pub = os.path.join(dir_, '{0}.pub'.format(fkey))
+            fin = salt.utils.pem_finger(pub)
+            if fin:
+                self._log('Signature for {0} public key: {1}'.format(fkey, fin))
+                sigs['{0}.pub'.format(fkey)] = fin
+            pub = os.path.join(dir_, '{0}'.format(fkey))
+            fin = salt.utils.pem_finger(pub)
+            if fin:
+                self._log('Signature for {0} public key: {1}'.format(fkey, fin))
+                sigs['{0}.pub'.format(fkey)] = fin
+            pri = os.path.join(dir_, '{0}.pem'.format(fkey))
+            fin = salt.utils.pem_finger(pri)
+            if fin:
+                self._log('Signature for {0} private key: {1}'.format(fkey, fin))
+                sigs['{0}.pem'.format(fkey)] = fin
+        return sigs
+
     def run(self):
         '''
         Run the logic for saltkey
@@ -393,6 +429,7 @@ class Key(object):
                     self.opts['gen_keys_dir'],
                     self.opts['gen_keys'],
                     self.opts['keysize'])
+            self._log('Keys generation complete', level='info')
             return
         if self.opts['list']:
             self._list(self.opts['list'])
@@ -414,5 +451,7 @@ class Key(object):
             self._delete_key()
         elif self.opts['delete_all']:
             self._delete_all()
+        elif self.opts['finger']:
+            self.finger()
         else:
             self._list('all')

salt/client.py

@@ -36,9 +36,6 @@ import time
 import getpass
 import fnmatch
 
-# Import zmq modules
-import zmq
-
 # Import salt modules
 import salt.config
 import salt.payload
@@ -75,18 +72,22 @@ class LocalClient(object):
     def __init__(self, c_path='/etc/salt/master'):
         self.opts = salt.config.master_config(c_path)
         self.serial = salt.payload.Serial(self.opts)
-        self.key = self.__read_master_key()
         self.salt_user = self.__get_user()
+        self.key = self.__read_master_key()
         self.event = salt.utils.event.MasterEvent(self.opts['sock_dir'])
 
     def __read_master_key(self):
         '''
         Read in the rotating master authentication key
         '''
-        keyfile = os.path.join(self.opts['cachedir'], '.root_key')
+        key_user = self.salt_user
+        if key_user.startswith('sudo_'):
+            key_user = 'root'
+        keyfile = os.path.join(
+                self.opts['cachedir'], '.{0}_key'.format(key_user)
+                )
         # Make sure all key parent directories are accessible
-        user = self.opts.get('user', 'root')
+        salt.utils.verify.check_parent_dirs(keyfile, key_user)
-        salt.utils.verify.check_parent_dirs(keyfile, user)
 
         try:
             with open(keyfile, 'r') as KEY:
@@ -107,12 +108,12 @@ class LocalClient(object):
             env_vars = ['SUDO_USER', 'USER', 'USERNAME']
             for evar in env_vars:
                 if evar in os.environ:
-                    return os.environ[evar]
+                    return 'sudo_{0}'.format(os.environ[evar])
-            return None
+            return user
         # If the running user is just the specified user in the
         # conf file, don't pass the user as it's implied.
         elif user == self.opts['user']:
-            return None
+            return user
         return user
 
     def _check_glob_minions(self, expr):
@@ -120,13 +121,7 @@ class LocalClient(object):
         Return the minions found by looking via globs
         '''
         cwd = os.getcwd()
-        try:
+        os.chdir(os.path.join(self.opts['pki_dir'], 'minions'))
-            os.chdir(os.path.join(self.opts['pki_dir'], 'minions'))
-        except OSError:
-            err = ('The Salt Master has not been set up on this system, '
-                   'a salt-master needs to be running to use the salt command')
-            sys.stderr.write(err)
-            sys.exit(2)
         ret = set(glob.glob(expr))
         os.chdir(cwd)
         return ret
@@ -177,18 +172,20 @@ class LocalClient(object):
                     continue
                 if comps[0] not in grains:
                     minions.remove(id_)
-                if isinstance(grains[comps[0]], list):
+                    continue
+                if isinstance(grains.get(comps[0]), list):
                     # We are matching a single component to a single list member
                     found = False
                     for member in grains[comps[0]]:
                         if fnmatch.fnmatch(str(member).lower(), comps[1].lower()):
                             found = True
+                            break
                     if found:
                         continue
                     minions.remove(id_)
                     continue
                 if fnmatch.fnmatch(
-                    str(grains[comps[0]]).lower(),
+                    str(grains.get(comps[0], '').lower()),
                     comps[1].lower(),
                     ):
                     continue
@@ -276,10 +273,13 @@ class LocalClient(object):
         arg = condition_kwarg(arg, kwarg)
         if timeout is None:
             timeout = self.opts['timeout']
-        jid = salt.utils.prep_jid(
+        try:
-                self.opts['cachedir'],
+            jid = salt.utils.prep_jid(
-                self.opts['hash_type']
+                    self.opts['cachedir'],
-                )
+                    self.opts['hash_type']
+                    )
+        except Exception:
+            jid = ''
         pub_data = self.pub(
             tgt,
             fun,
@@ -288,6 +288,11 @@ class LocalClient(object):
             ret,
             jid=jid,
             timeout=timeout)
+        if not pub_data:
+            err = ('Failed to authenticate, is this user permitted to execute '
+                   'commands?\n')
+            sys.stderr.write(err)
+            sys.exit(4)
         if pub_data['jid'] == '0':
             # Failed to connect to the master and send the pub
             return {}
@@ -312,10 +317,13 @@ class LocalClient(object):
         arg = condition_kwarg(arg, kwarg)
         if timeout is None:
             timeout = self.opts['timeout']
-        jid = salt.utils.prep_jid(
+        try:
-                self.opts['cachedir'],
+            jid = salt.utils.prep_jid(
-                self.opts['hash_type']
+                    self.opts['cachedir'],
-                )
+                    self.opts['hash_type']
+                    )
+        except Exception:
+            jid = ''
         pub_data = self.pub(
             tgt,
             fun,
@@ -324,6 +332,11 @@ class LocalClient(object):
             ret,
             jid=jid,
             timeout=timeout)
+        if not pub_data:
+            err = ('Failed to authenticate, is this user permitted to execute '
+                   'commands?\n')
+            sys.stderr.write(err)
+            sys.exit(4)
         if pub_data['jid'] == '0':
             print('Failed to connect to the Master, is the Salt Master running?')
             yield {}
@@ -369,6 +382,11 @@ class LocalClient(object):
             ret,
             jid=jid,
             timeout=timeout)
+        if not pub_data:
+            err = ('Failed to authenticate, is this user permitted to execute '
+                   'commands?\n')
+            sys.stderr.write(err)
+            sys.exit(4)
         if pub_data['jid'] == '0':
             # Failed to connect to the master and send the pub
             yield {}
@@ -409,6 +427,11 @@ class LocalClient(object):
             ret,
             jid=jid,
             timeout=timeout)
+        if not pub_data:
+            err = ('Failed to authenticate, is this user permitted to execute '
+                   'commands?\n')
+            sys.stderr.write(err)
+            sys.exit(4)
         if pub_data['jid'] == '0':
             # Failed to connect to the master and send the pub
             yield {}
@@ -448,6 +471,11 @@ class LocalClient(object):
             ret,
             jid=jid,
             timeout=timeout)
+        if not pub_data:
+            err = ('Failed to authenticate, is this user permitted to execute '
+                   'commands?\n')
+            sys.stderr.write(err)
+            sys.exit(4)
         if pub_data['jid'] == '0':
             # Failed to connect to the master and send the pub
             return {}
@@ -524,7 +552,7 @@ class LocalClient(object):
                 # The timeout +1 has not been reached and there is still a
                 # write tag for the syndic
                 continue
-            if len(fret) >= len(minions):
+            if len(found.intersection(minions)) >= len(minions):
                 # All minions have returned, break out of the loop
                 break
             if int(time.time()) > start + timeout:
@@ -542,7 +570,7 @@ class LocalClient(object):
                     continue
                 if verbose:
                     if tgt_type == 'glob' or tgt_type == 'pcre':
-                        if not len(fret) >= len(minions):
+                        if len(found.intersection(minions)) >= len(minions):
                             print('\nThe following minions did not return:')
                             fail = sorted(list(minions.difference(found)))
                             for minion in fail:
@@ -596,7 +624,7 @@ class LocalClient(object):
                 # The timeout +1 has not been reached and there is still a
                 # write tag for the syndic
                 continue
-            if len(ret) >= len(minions):
+            if len(found.intersection(minions)) >= len(minions):
                 break
             if int(time.time()) > start + timeout:
                 break
@@ -649,7 +677,7 @@ class LocalClient(object):
                 # The timeout +1 has not been reached and there is still a
                 # write tag for the syndic
                 continue
-            if len(ret) >= len(minions):
+            if len(set(ret.keys()).intersection(minions)) >= len(minions):
                 # All Minions have returned
                 return ret
             if int(time.time()) > start + timeout:
@@ -704,7 +732,7 @@ class LocalClient(object):
                 # The timeout +1 has not been reached and there is still a
                 # write tag for the syndic
                 continue
-            if len(ret) >= len(minions):
+            if len(set(ret.keys()).intersection(minions)) >= len(minions):
                 return ret
             if int(time.time()) > start + timeout:
                 return ret
@@ -731,7 +759,6 @@ class LocalClient(object):
             print('-' * len(msg) + '\n')
         if timeout is None:
             timeout = self.opts['timeout']
-        inc_timeout = timeout
         jid_dir = salt.utils.jid_dir(
                 jid,
                 self.opts['cachedir'],
@@ -743,7 +770,7 @@ class LocalClient(object):
         wtag = os.path.join(jid_dir, 'wtag*')
         # Check to see if the jid is real, if not return the empty dict
         if not os.path.isdir(jid_dir):
-            return ret_
+            return ret
         # Wait for the hosts to check in
         while True:
             raw = self.event.get_event(timeout, jid)
@@ -752,12 +779,12 @@ class LocalClient(object):
                 ret[raw['id']] = {'ret': raw['return']}
                 if 'out' in raw:
                     ret[raw['id']]['out'] = raw['out']
-                if len(found) >= len(minions):
+                if len(found.intersection(minions)) >= len(minions):
                     # All minions have returned, break out of the loop
                     break
                 continue
             # Then event system timeout was reached and nothing was returned
-            if len(found) >= len(minions):
+            if len(found.intersection(minions)) >= len(minions):
                 # All minions have returned, break out of the loop
                 break
             if glob.glob(wtag) and not int(time.time()) > start + timeout + 1:
@@ -815,17 +842,20 @@ class LocalClient(object):
         while True:
             raw = self.event.get_event(timeout, jid)
             if not raw is None:
+                if 'syndic' in raw:
+                    minions.update(raw['syndic'])
+                    continue
                 found.add(raw['id'])
                 ret = {raw['id']: {'ret': raw['return']}}
                 if 'out' in raw:
                     ret[raw['id']]['out'] = raw['out']
                 yield ret
-                if len(found) >= len(minions):
+                if len(found.intersection(minions)) >= len(minions):
                     # All minions have returned, break out of the loop
                     break
                 continue
             # Then event system timeout was reached and nothing was returned
-            if len(found) >= len(minions):
+            if len(found.intersection(minions)) >= len(minions):
                 # All minions have returned, break out of the loop
                 break
             if glob.glob(wtag) and not int(time.time()) > start + timeout + 1:
@@ -867,8 +897,6 @@ class LocalClient(object):
                 self.opts['cachedir'],
                 self.opts['hash_type']
                 )
-        start = 999999999999
-        gstart = int(time.time())
         found = set()
         # Check to see if the jid is real, if not return the empty dict
         if not os.path.isdir(jid_dir):
@@ -926,15 +954,19 @@ class LocalClient(object):
         match the regex, this will then be used to parse the returns to
         make sure everyone has checked back in.
         '''
-        return {'glob': self._check_glob_minions,
+        try:
-                'pcre': self._check_pcre_minions,
+            minions = {'glob': self._check_glob_minions,
-                'list': self._check_list_minions,
+                    'pcre': self._check_pcre_minions,
-                'grain': self._check_grain_minions,
+                    'list': self._check_list_minions,
-                'grain_pcre': self._check_grain_pcre_minions,
+                    'grain': self._check_grain_minions,
-                'exsel': self._all_minions,
+                    'grain_pcre': self._check_grain_pcre_minions,
-                'pillar': self._all_minions,
+                    'exsel': self._all_minions,
-                'compound': self._all_minions,
+                    'pillar': self._all_minions,
-                }[expr_form](expr)
+                    'compound': self._all_minions,
+                    }[expr_form](expr)
+        except Exception:
+            minions = expr
+        return minions
 
     def pub(self, tgt, fun, arg=(), expr_form='glob',
             ret='', jid='', timeout=5):
@@ -991,13 +1023,7 @@ class LocalClient(object):
         # return what we get back
         minions = self.check_minions(tgt, expr_form)
 
-        if self.opts['order_masters']:
+        if not minions:
-            # If we're a master of masters, ignore the check_minion and
-            # set the minions to the target.  This speeds up wait time
-            # for lists and ranges and makes regex and other expression
-            # forms possible
-            minions = tgt
-        elif not minions:
             return {'jid': None,
                     'minions': minions}
 
@@ -1023,6 +1049,8 @@ class LocalClient(object):
                 'tcp://{0[interface]}:{0[ret_port]}'.format(self.opts),
                 )
         payload = sreq.send('clear', payload_kwargs)
+        if not payload:
+            return payload
         return {'jid': payload['load']['jid'],
                 'minions': minions}
 

salt/config.py

@@ -7,7 +7,6 @@ import glob
 import os
 import socket
 import logging
-import tempfile
 
 # import third party libs
 import yaml
@@ -26,10 +25,11 @@ from salt.exceptions import SaltClientError
 
 log = logging.getLogger(__name__)
 
-__dflt_log_datefmt = '%H:%M:%S'
+__dflt_log_datefmt = '%Y-%m-%d %H:%M:%S'
 __dflt_log_fmt_console = '[%(levelname)-8s] %(message)s'
 __dflt_log_fmt_logfile = '%(asctime)s,%(msecs)03.0f [%(name)-17s][%(levelname)-8s] %(message)s'
 
+
 def _validate_file_roots(file_roots):
     '''
     If the file_roots option has a key that is None then we will error out,
@@ -139,8 +139,9 @@ def prepend_root_dir(opts, path_options):
     root_dir = os.path.abspath(opts['root_dir'])
     for path_option in path_options:
         if path_option in opts:
-            opts[path_option] = os.path.normpath(
+            if opts[path_option].startswith(opts['root_dir']):
-                    os.sep.join([root_dir, opts[path_option]]))
+                opts[path_option] = opts[path_option][len(opts['root_dir']):]
+            opts[path_option] = salt.utils.path_join(root_dir, opts[path_option])
 
 
 def minion_config(path):
@@ -149,6 +150,7 @@ def minion_config(path):
     '''
     opts = {'master': 'salt',
             'master_port': '4506',
+            'master_finger': '',
             'user': 'root',
             'root_dir': '/',
             'pki_dir': '/etc/salt/pki',
@@ -156,12 +158,16 @@ def minion_config(path):
             'cachedir': '/var/cache/salt',
             'cache_jobs': False,
             'conf_file': path,
-            'sock_dir': os.path.join(tempfile.gettempdir(), '.salt-unix'),
+            'sock_dir': '/var/run/salt',
+            'backup_mode': '',
             'renderer': 'yaml_jinja',
             'failhard': False,
             'autoload_dynamic_modules': True,
             'environment': None,
             'state_top': 'top.sls',
+            'startup_states': '',
+            'sls_list': [],
+            'top_file': '',
             'file_client': 'remote',
             'file_roots': {
                 'base': ['/srv/salt'],
@@ -182,8 +188,11 @@ def minion_config(path):
             'open_mode': False,
             'multiprocessing': True,
             'sub_timeout': 60,
+            'ipc_mode': 'ipc',
+            'tcp_pub_port': 4510,
+            'tcp_pull_port': 4511,
             'log_file': '/var/log/salt/minion',
-            'log_level': 'warning',
+            'log_level': None,
             'log_level_logfile': None,
             'log_datefmt': __dflt_log_datefmt,
             'log_fmt_console': __dflt_log_fmt_console,
@@ -191,14 +200,21 @@ def minion_config(path):
             'log_granular_levels': {},
             'test': False,
             'cython_enable': False,
-            'state_verbose': False,
+            'state_verbose': True,
+            'state_output': 'full',
             'acceptance_wait_time': 10,
             'dns_check': True,
+            'verify_env': True,
             'grains': {},
             'permissive_pki_access': False,
             'default_include': 'minion.d/*.conf',
+            'update_url': False,
+            'update_restart_services': [],
             }
 
+    if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
+        opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')
+
     load_config(opts, path, 'SALT_MINION_CONFIG')
 
     default_include = opts.get('default_include', [])
@@ -242,7 +258,7 @@ def master_config(path):
             'publish_port': '4505',
             'user': 'root',
             'worker_threads': 5,
-            'sock_dir': os.path.join(tempfile.gettempdir(), '.salt-unix'),
+            'sock_dir': '/var/run/salt',
             'ret_port': '4506',
             'timeout': 5,
             'keep_jobs': 24,
@@ -258,9 +274,15 @@ def master_config(path):
             'pillar_roots': {
                 'base': ['/srv/pillar'],
                 },
+            'ext_pillar': {},
+            'syndic_master': '',
+            'runner_dirs': [],
+            'client_acl': {},
             'file_buffer_size': 1048576,
+            'max_open_files': 100000,
             'hash_type': 'md5',
             'conf_file': path,
+            'pub_refresh': True,
             'open_mode': False,
             'auto_accept': False,
             'renderer': 'yaml_jinja',
@@ -271,7 +293,7 @@ def master_config(path):
             'job_cache': True,
             'minion_data_cache': True,
             'log_file': '/var/log/salt/master',
-            'log_level': 'warning',
+            'log_level': None,
             'log_level_logfile': None,
             'log_datefmt': __dflt_log_datefmt,
             'log_fmt_console': __dflt_log_fmt_console,
@@ -282,13 +304,19 @@ def master_config(path):
             'cluster_mode': 'paranoid',
             'range_server': 'range:80',
             'serial': 'msgpack',
+            'state_verbose': True,
+            'state_output': 'full',
             'nodegroups': {},
             'cython_enable': False,
             'key_logfile': '/var/log/salt/key',
+            'verify_env': True,
             'permissive_pki_access': False,
             'default_include': 'master.d/*.conf',
     }
 
+    if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
+        opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')
+
     load_config(opts, path, 'SALT_MASTER_CONFIG')
 
     default_include = opts.get('default_include', [])

salt/crypt.py

@@ -8,7 +8,6 @@ authenticating peers
 import os
 import sys
 import hmac
-import getpass
 import hashlib
 import logging
 import tempfile
@@ -17,9 +16,6 @@ import tempfile
 from M2Crypto import RSA
 from Crypto.Cipher import AES
 
-# Import zeromq libs
-import zmq
-
 # Import salt utils
 import salt.utils
 import salt.payload
@@ -69,7 +65,7 @@ def gen_keys(keydir, keyname, keysize):
     priv = '{0}.pem'.format(base)
     pub = '{0}.pub'.format(base)
 
-    gen = RSA.gen_key(keysize, 1)
+    gen = RSA.gen_key(keysize, 1, callback=lambda x,y,z:None)
     cumask = os.umask(191)
     gen.save_key(priv, None)
     os.umask(cumask)
@@ -240,6 +236,7 @@ class Auth(object):
         and the decrypted aes key for transport decryption.
         '''
         auth = {}
+        m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub)
         try:
             self.opts['master_ip'] = salt.utils.dns_check(
                     self.opts['master'],
@@ -247,7 +244,10 @@ class Auth(object):
                     )
         except SaltClientError:
             return 'retry'
-        sreq = salt.payload.SREQ(self.opts['master_uri'])
+        sreq = salt.payload.SREQ(
+                self.opts['master_uri'],
+                self.opts.get('id', '')
+                )
         try:
             payload = sreq.send_auto(self.minion_sign_in_payload())
         except SaltReqTimeoutError:
@@ -266,21 +266,35 @@ class Auth(object):
                 else:
                     log.error(
                         'The Salt Master has cached the public key for this '
-                        'node, this salt minion will wait for %s seconds '
+                        'node, this salt minion will wait for {0} seconds '
-                        'before attempting to re-authenticate',
+                        'before attempting to re-authenticate'.format(
-                        self.opts['acceptance_wait_time']
+                            self.opts['acceptance_wait_time']
+                        )
                     )
                     return 'retry'
         if not self.verify_master(payload['pub_key'], payload['token']):
-            m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub)
             log.critical(
                 'The Salt Master server\'s public key did not authenticate!\n'
                 'If you are confident that you are connecting to a valid Salt '
                 'Master, then remove the master public key and restart the '
-                'Salt Minion.\nThe master public key can be found at:\n%s',
+                'Salt Minion.\nThe master public key can be found '
-                m_pub_fn
+                'at:\n{0}'.format(m_pub_fn)
             )
             sys.exit(42)
+        if self.opts.get('master_finger', False):
+            if salt.utils.pem_finger(m_pub_fn) != self.opts['master_finger']:
+                log.critical((
+                    'The specified fingerprint in the master configuration '
+                    'file:\n{0}\nDoes not match the authenticating master\'s '
+                    'key:\n{1}\nVerify that the configured fingerprint '
+                    'matches the fingerprint of the correct master and that '
+                    'this minion is not subject to a man in the middle attack'
+                    ).format(
+                        self.opts['master_finger'],
+                        salt.utils.pem_finger(m_pub_fn)
+                        )
+                    )
+                sys.exit(42)
         auth['aes'] = self.decrypt_aes(payload['aes'])
         auth['publish_port'] = payload['publish_port']
         return auth
@@ -334,7 +348,14 @@ class Crypticle(object):
         aes_key, hmac_key = self.keys
         sig = data[-self.SIG_SIZE:]
         data = data[:-self.SIG_SIZE]
-        if hmac.new(hmac_key, data, hashlib.sha256).digest() != sig:
+        mac_bytes = hmac.new(hmac_key, data, hashlib.sha256).digest()
+        if len(mac_bytes) != len(sig):
+            log.warning('Failed to authenticate message')
+            raise AuthenticationError('message authentication failed')
+        result = 0
+        for x, y in zip(mac_bytes, sig):
+            result |= ord(x) ^ ord(y)
+        if result != 0:
             log.warning('Failed to authenticate message')
             raise AuthenticationError('message authentication failed')
         iv_bytes = data[:self.AES_BLOCK_SIZE]

salt/fileclient.py

@@ -12,7 +12,6 @@ import subprocess
 
 # Import third-party libs
 import yaml
-import zmq
 
 # Import salt libs
 from salt.exceptions import MinionError, SaltReqTimeoutError
@@ -21,6 +20,7 @@ import salt.crypt
 import salt.loader
 import salt.utils
 import salt.payload
+import salt.utils
 import salt.utils.templates
 from salt._compat import (
     URLError, HTTPError, BaseHTTPServer, urlparse, url_open)
@@ -69,7 +69,7 @@ class Client(object):
 
         filelist = []
 
-        for root, dirs, files in os.walk(destdir):
+        for root, dirs, files in os.walk(destdir, followlinks=True):
             for name in files:
                 path = os.path.join(root, name)
                 filelist.append(path)
@@ -96,7 +96,8 @@ class Client(object):
 
     def get_file(self, path, dest='', makedirs=False, env='base'):
         '''
-        Copies a file from the local files or master depending on implementation
+        Copies a file from the local files or master depending on
+        implementation
         '''
         raise NotImplementedError
 
@@ -138,6 +139,11 @@ class Client(object):
         '''
         ret = []
         path = self._check_proto(path)
+        log.info(
+            'Caching directory \'{0}\' for environment \'{1}\''.format(
+                path, env
+            )
+        )
         for fn_ in self.file_list(env):
             if fn_.startswith(path):
                 local = self.cache_file('salt://{0}'.format(fn_), env)
@@ -148,20 +154,20 @@ class Client(object):
         if include_empty:
             # Break up the path into a list containing the bottom-level directory
             # (the one being recursively copied) and the directories preceding it
-            separated = string.rsplit(path,'/',1)
+            #separated = string.rsplit(path, '/', 1)
-            if len(separated) != 2:
+            #if len(separated) != 2:
-                # No slashes in path. (This means all files in env will be copied)
+            #    # No slashes in path. (This means all files in env will be copied)
-                prefix = ''
+            #    prefix = ''
-            else:
+            #else:
-                prefix = separated[0]
+            #    prefix = separated[0]
             for fn_ in self.file_list_emptydirs(env):
                 if fn_.startswith(path):
-                    dest = os.path.normpath(
+                    dest = salt.utils.path_join(
-                      os.sep.join([
+                        self.opts['cachedir'],
-                      self.opts['cachedir'],
+                        'files',
-                      'files',
+                        env
-                      env])) 
+                    )
-                    minion_dir = '%s/%s' % (dest,fn_)
+                    minion_dir = '{0}/{1}'.format(dest, fn_)
                     if not os.path.isdir(minion_dir):
                         os.makedirs(minion_dir)
                     ret.append(minion_dir)
@@ -197,6 +203,12 @@ class Client(object):
         '''
         return []
 
+    def dir_list(self, env='base'):
+        '''
+        This function must be overwritten
+        '''
+        return []
+
     def is_cached(self, path, env='base'):
         '''
         Returns the full path to a file if it is cached locally on the minion
@@ -224,9 +236,9 @@ class Client(object):
             if path.endswith('.sls'):
                 # is an sls module!
                 if path.endswith('{0}init.sls'.format(os.sep)):
-                    states.append(path.replace(os.sep, '.')[:-9])
+                    states.append(path.replace('/', '.')[:-9])
                 else:
-                    states.append(path.replace(os.sep, '.')[:-4])
+                    states.append(path.replace('/', '.')[:-4])
         return states
 
     def get_state(self, sls, env):
@@ -266,16 +278,20 @@ class Client(object):
                 # Remove the leading directories from path to derive
                 # the relative path on the minion.
                 minion_relpath = string.lstrip(fn_[len(prefix):], '/')
-                ret.append(self.get_file('salt://{0}'.format(fn_),
+                ret.append(
-                                         '%s/%s' % (dest, minion_relpath),
+                    self.get_file(
-                                         True, env))
+                        'salt://{0}'.format(fn_),
+                        '{0}/{1}'.format(dest, minion_relpath),
+                        True, env
+                    )
+                )
         # Replicate empty dirs from master
         for fn_ in self.file_list_emptydirs(env):
             if fn_.startswith(path):
                 # Remove the leading directories from path to derive
                 # the relative path on the minion.
                 minion_relpath = string.lstrip(fn_[len(prefix):], '/')
-                minion_mkdir = '%s/%s' % (dest, minion_relpath)
+                minion_mkdir = '{0}/{1}'.format(dest, minion_relpath)
                 os.makedirs(minion_mkdir)
                 ret.append(minion_mkdir)
         ret.sort()
@@ -296,13 +312,13 @@ class Client(object):
                 else:
                     return ''
         else:
-            dest = os.path.normpath(
+            dest = salt.utils.path_join(
-                os.sep.join([
+                self.opts['cachedir'],
-                    self.opts['cachedir'],
+                'extrn_files',
-                    'extrn_files',
+                env,
-                    env,
+                url_data.netloc,
-                    url_data.netloc,
+                url_data.path
-                    url_data.path]))
+            )
             destdir = os.path.dirname(dest)
             if not os.path.isdir(destdir):
                 os.makedirs(destdir)
@@ -354,13 +370,13 @@ class Client(object):
             return ''
         if not dest:
             # No destination passed, set the dest as an extrn_files cache
-            dest = os.path.normpath(
+            dest = salt.utils.path_join(
-                os.sep.join([
+                self.opts['cachedir'],
-                    self.opts['cachedir'],
+                'extrn_files',
-                    'extrn_files',
+                env,
-                    env,
+                url_data.netloc,
-                    url_data.netloc,
+                url_data.path
-                    url_data.path]))
+            )
         destdir = os.path.dirname(dest)
         if not os.path.isdir(destdir):
             if makedirs:
@@ -413,7 +429,7 @@ class LocalClient(Client):
         if env not in self.opts['file_roots']:
             return ret
         for path in self.opts['file_roots'][env]:
-            for root, dirs, files in os.walk(path):
+            for root, dirs, files in os.walk(path, followlinks=True):
                 for fn in files:
                     ret.append(
                         os.path.relpath(
@@ -434,11 +450,23 @@ class LocalClient(Client):
         if env not in self.opts['file_roots']:
             return ret
         for path in self.opts['file_roots'][env]:
-            for root, dirs, files in os.walk(path):
+            for root, dirs, files in os.walk(path, followlinks=True):
                 if len(dirs) == 0 and len(files) == 0:
                     ret.append(os.path.relpath(root, path))
         return ret
 
+    def dir_list(self, env='base'):
+        '''
+        List the dirs in the file_roots
+        '''
+        ret = []
+        if env not in self.opts['file_roots']:
+            return ret
+        for path in self.opts['file_roots'][env]:
+            for root, dirs, files in os.walk(path, followlinks=True):
+                ret.append(os.path.relpath(root, path))
+        return ret
+
     def hash_file(self, path, env='base'):
         '''
         Return the hash of a file, to get the hash of a file in the file_roots
@@ -450,9 +478,8 @@ class LocalClient(Client):
             path = self._check_proto(path)
         except MinionError:
             if not os.path.isfile(path):
-                err = ('Specified file {0} is not present to generate '
+                err = 'Specified file {0} is not present to generate hash'
-                        'hash').format(path)
+                log.warning(err.format(path))
-                log.warning(err)
                 return ret
             else:
                 with open(path, 'rb') as f:
@@ -532,6 +559,7 @@ class RemoteClient(Client):
         dest is ommited, then the downloaded file will be placed in the minion
         cache
         '''
+        log.info('Fetching file \'{0}\''.format(path))
         path = self._check_proto(path)
         load = {'path': path,
                 'env': env,
@@ -612,6 +640,23 @@ class RemoteClient(Client):
         except SaltReqTimeoutError:
             return ''
 
+    def dir_list(self, env='base'):
+        '''
+        List the dirs on the master
+        '''
+        load = {'env': env,
+                'cmd': '_dir_list'}
+        try:
+            return self.auth.crypticle.loads(
+                    self.sreq.send(
+                        'aes',
+                        self.auth.crypticle.dumps(load),
+                        3,
+                        60)
+                    )
+        except SaltReqTimeoutError:
+            return ''
+
     def hash_file(self, path, env='base'):
         '''
         Return the hash of a file, to get the hash of a file on the salt
@@ -622,9 +667,8 @@ class RemoteClient(Client):
             path = self._check_proto(path)
         except MinionError:
             if not os.path.isfile(path):
-                err = ('Specified file {0} is not present to generate '
+                err = 'Specified file {0} is not present to generate hash'
-                        'hash').format(path)
+                log.warning(err.format(path))
-                log.warning(err)
                 return {}
             else:
                 ret = {}

salt/grains/core.py

@@ -18,6 +18,13 @@ import socket
 import sys
 import re
 import platform
+
+# Extend the default list of supported distros. This will be used for the
+# /etc/DISTRO-release checking that is part of platform.linux_distribution()
+from platform import _supported_dists
+_supported_dists += ('arch', 'mageia', 'meego', 'vmware', 'bluewhite64',
+                     'slamd64', 'enterprise', 'ovs', 'system')
+
 import salt.utils
 
 # Solve the Chicken and egg problem where grains need to run before any
@@ -26,38 +33,21 @@ import salt.modules.cmdmod
 __salt__ = {'cmd.run': salt.modules.cmdmod._run_quiet}
 
 
-def _kernel():
-    '''
-    Return the kernel type
-    '''
-    # Provides:
-    # kernel
-    grains = {}
-    grains['kernel'] = __salt__['cmd.run']('uname -s').strip()
-
-    if grains['kernel'] == 'aix':
-        grains['kernelrelease'] = __salt__['cmd.run']('oslevel -s').strip()
-    else:
-        grains['kernelrelease'] = __salt__['cmd.run']('uname -r').strip()
-    if 'kernel' not in grains:
-        grains['kernel'] = 'Unknown'
-    if not grains['kernel']:
-        grains['kernel'] = 'Unknown'
-    return grains
-
-
 def _windows_cpudata():
     '''
     Return the cpu information for Windows systems architecture
     '''
     # Provides:
     #   cpuarch
-    #   num_cpus
     #   cpu_model
     grains = {}
-    grains['cpuarch'] = platform.machine()
     if 'NUMBER_OF_PROCESSORS' in os.environ:
-        grains['num_cpus'] = os.environ['NUMBER_OF_PROCESSORS']
+        # Cast to int so that the logic isn't broken when used as a
+        # conditional in templating. Also follows _linux_cpudata()
+        try:
+            grains['num_cpus'] = int(os.environ['NUMBER_OF_PROCESSORS'])
+        except ValueError:
+            grains['num_cpus'] = 1
     grains['cpu_model'] = platform.processor()
     return grains
 
@@ -67,25 +57,11 @@ def _linux_cpudata():
     Return the cpu information for Linux systems architecture
     '''
     # Provides:
-    #   cpuarch
     #   num_cpus
     #   cpu_model
     #   cpu_flags
     grains = {}
     cpuinfo = '/proc/cpuinfo'
-    # Grab the Arch
-    arch = __salt__['cmd.run']('uname -m').strip()
-    grains['cpuarch'] = arch
-    # Some systems such as Debian don't like uname -m
-    # so fallback gracefully to the processor type
-    if not grains['cpuarch'] or grains['cpuarch'] == 'unknown':
-        arch = __salt__['cmd.run']('uname -p')
-        grains['cpuarch'] = arch
-    if not grains['cpuarch'] or grains['cpuarch'] == 'unknown':
-        arch = __salt__['cmd.run']('uname -i')
-        grains['cpuarch'] = arch
-    if not grains['cpuarch'] or grains['cpuarch'] == 'unknown':
-        grains['cpuarch'] = 'Unknown'
     # Parse over the cpuinfo file
     if os.path.isfile(cpuinfo):
         for line in open(cpuinfo, 'r').readlines():
@@ -126,12 +102,30 @@ def _bsd_cpudata(osdata):
     grains['cpu_flags'] = []
     try:
         grains['num_cpus'] = int(grains['num_cpus'])
-    except Exception:
+    except ValueError:
-        grains['num_cpus'] = 0
+        grains['num_cpus'] = 1
 
     return grains
 
 
+def _sunos_cpudata(osdata):
+    '''
+    Return the cpu information for Solaris-like systems
+    '''
+    # Provides:
+    #   cpuarch
+    #   num_cpus
+    #   cpu_model
+    grains = {'num_cpus': 0}
+
+    grains['cpuarch'] = __salt__['cmd.run']('uname -p').strip()
+    for line in __salt__['cmd.run']('/usr/sbin/psrinfo 2>/dev/null').split('\n'):
+        grains['num_cpus'] += 1
+    grains['cpu_model'] = __salt__['cmd.run']('kstat -p cpu_info:0:cpu_info0:implementation').split()[1].strip()
+    
+    return grains
+
+
 def _memdata(osdata):
     '''
     Gather information about the system memory
@@ -154,6 +148,11 @@ def _memdata(osdata):
         if sysctl:
             mem = __salt__['cmd.run']('{0} -n hw.physmem'.format(sysctl)).strip()
             grains['mem_total'] = str(int(mem) / 1024 / 1024)
+    elif osdata['kernel'] == 'SunOS':
+        for line in __salt__['cmd.run']('/usr/sbin/prtconf 2>/dev/null').split('\n'):
+            comps = line.split(' ')	
+            if comps[0].strip() == 'Memory' and comps[1].strip() == 'size:':
+                grains['mem_total'] = int(comps[2].strip()) 
     elif osdata['kernel'] == 'Windows':
         for line in __salt__['cmd.run']('SYSTEMINFO /FO LIST').split('\n'):
             comps = line.split(':')
@@ -195,6 +194,9 @@ def _virtual(osdata):
         # Product Name: Virtual Machine
         elif 'Manufacturer: Microsoft' in output and 'Virtual Machine' in output:
             grains['virtual'] = 'VirtualPC'
+        # Manufacturer: Parallels Software International Inc.
+        elif 'Parallels Software' in output:
+            grains['virtual'] = 'Parallels'
     # Fall back to lspci if dmidecode isn't available
     elif lspci:
         model = __salt__['cmd.run']('lspci').lower()
@@ -207,7 +209,7 @@ def _virtual(osdata):
             grains['virtual'] = 'kvm'
         elif 'virtio' in model:
             grains['virtual'] = 'kvm'
-    choices = ('Linux', 'OpenBSD', 'SunOS', 'HP-UX')
+    choices = ('Linux', 'OpenBSD', 'HP-UX')
     isdir = os.path.isdir
     if osdata['kernel'] in choices:
         if isdir('/proc/vz'):
@@ -245,9 +247,7 @@ def _virtual(osdata):
             # If a Dom0 or DomU was detected, obviously this is xen
             if 'dom' in grains.get('virtual_subtype', '').lower():
                 grains['virtual'] = 'xen'
-        elif isdir('/.SUNWnative'):
+        if os.path.isfile('/proc/cpuinfo'):
-            grains['virtual'] = 'zone'
-        elif os.path.isfile('/proc/cpuinfo'):
             if 'QEMU Virtual CPU' in open('/proc/cpuinfo', 'r').read():
                 grains['virtual'] = 'kvm'
     elif osdata['kernel'] == 'FreeBSD':
@@ -264,6 +264,16 @@ def _virtual(osdata):
                 grains['virtual_subtype'] = 'jail'
             if 'QEMU Virtual CPU' in model:
                 grains['virtual'] = 'kvm'
+    elif osdata['kernel'] == 'SunOS':
+        # Check if it's a "regular" zone. (i.e. Solaris 10/11 zone)
+        zonename = salt.utils.which('zonename')
+        if zonename:
+            zone = __salt__['cmd.run']('{0}'.format(zonename)).strip() 
+            if zone != "global":
+                grains['virtual'] = 'zone'
+        # Check if it's a branded zone (i.e. Solaris 8/9 zone)
+        if isdir('/.SUNWnative'):
+            grains['virtual'] = 'zone'
     return grains
 
 
@@ -275,6 +285,8 @@ def _ps(osdata):
     bsd_choices = ('FreeBSD', 'NetBSD', 'OpenBSD', 'MacOS')
     if osdata['os'] in bsd_choices:
         grains['ps'] = 'ps auxwww'
+    if osdata['os'] == 'Solaris':
+        grains['ps'] = '/usr/ucb/ps auxwww'
     elif osdata['os'] == 'Windows':
         grains['ps'] = 'tasklist.exe'
     elif osdata.get('virtual', '') == 'openvzhn':
@@ -283,33 +295,11 @@ def _ps(osdata):
         grains['ps'] = 'ps -efH'
     return grains
 
-
-def _linux_platform_data(osdata):
-    '''
-    The platform module is very smart about figuring out linux distro
-    information. Instead of re-inventing the wheel, lets use it!
-    '''
-    # Provides:
-    #    osrelease
-    #    oscodename
-    grains = {}
-    (osname, osrelease, oscodename) = platform.dist()
-    if 'os' not in osdata and osname:
-        grains['os'] = osname
-    if osrelease:
-        grains['osrelease'] = osrelease
-    if oscodename:
-        grains['oscodename'] = oscodename
-    return grains
-
-
 def _windows_platform_data(osdata):
     '''
     Use the platform module for as much as we can.
     '''
     # Provides:
-    #    osrelease
-    #    osversion
     #    osmanufacturer
     #    manufacturer
     #    productname
@@ -320,13 +310,6 @@ def _windows_platform_data(osdata):
     #    windowsdomain
 
     grains = {}
-    (osname, hostname, osrelease, osversion, machine, processor) = platform.uname()
-    if 'os' not in osdata and osname:
-        grains['os'] = osname
-    if osrelease:
-        grains['osrelease'] = osrelease
-    if osversion:
-        grains['osversion'] = osversion
     get_these_grains = {
         'OS Manufacturer': 'osmanufacturer',
         'System Manufacturer': 'manufacturer',
@@ -355,143 +338,118 @@ def id_():
     '''
     return {'id': __opts__['id']}
 
+# This maps (at most) the first ten characters (no spaces, lowercased) of 
+# 'osfullname' to the 'os' grain that Salt traditionally uses.
+_os_name_map = {
+    'redhatente': 'RedHat',
+    'debian': 'Debian',
+    'arch': 'Arch',
+}
+
+# Map the 'os' grain to the 'os_family' grain
+_os_family_map = {
+    'Ubuntu': 'Debian',
+    'Fedora': 'RedHat',
+    'CentOS': 'RedHat',
+    'GoOSe': 'RedHat',
+    'Scientific': 'RedHat',
+    'Amazon': 'RedHat',
+    'CloudLinux': 'RedHat',
+    'Mandrake': 'Mandriva',
+    'ESXi': 'VMWare',
+    'VMWareESX': 'VMWare',
+    'Bluewhite64': 'Bluewhite',
+    'Slamd64': 'Slackware',
+    'OVS': 'Oracle',
+    'OEL': 'Oracle',
+    'SLES': 'Suse',
+    'SLED': 'Suse',
+    'openSUSE': 'Suse',
+    'SUSE': 'Suse'
+}
+
 
 def os_data():
     '''
     Return grains pertaining to the operating system
     '''
     grains = {}
-    if 'os' in os.environ:
+    # Windows Server 2008 64-bit
-        if os.environ['os'].startswith('Windows'):
+    # ('Windows', 'MINIONNAME', '2008ServerR2', '6.1.7601', 'AMD64', 'Intel64 Fam ily 6 Model 23 Stepping 6, GenuineIntel')
-            grains['os'] = 'Windows'
+    # Ubuntu 10.04
-            grains['os_family'] = 'Windows'
+    # ('Linux', 'FIRE66VMA01', '2.6.32-38-server', '#83-Ubuntu SMP Wed Jan 4 11:26:59 UTC 2012', 'x86_64', '')
-            grains['kernel'] = 'Windows'
+    (grains['kernel'], grains['host'],
-            grains.update(_memdata(grains))
+     grains['kernelrelease'], version, grains['cpuarch'], _) = platform.uname()
-            grains.update(_windows_platform_data(grains))
+    if grains['kernel'] == 'Windows':
-            grains.update(_windows_cpudata())
+        grains['osrelease'] = grains['kernelrelease']
-            grains.update(_ps(grains))
+        grains['osversion'] = grains['kernelrelease'] = version
-            return grains
+        grains['os'] = 'Windows'
-    grains.update(_kernel())
+        grains['os_family'] = 'Windows'
-
+        grains.update(_memdata(grains))
-    if grains['kernel'] == 'Linux':
+        grains.update(_windows_platform_data(grains))
+        grains.update(_windows_cpudata())
+        grains.update(_ps(grains))
+        return grains
+    elif grains['kernel'] == 'Linux':
         # Add lsb grains on any distro with lsb-release
-        if os.path.isfile('/etc/lsb-release'):
-            for line in open('/etc/lsb-release').readlines():
-                # Matches any possible format:
-                #     DISTRIB_ID="Ubuntu"
-                #     DISTRIB_ID='Mageia'
-                #     DISTRIB_ID=Fedora
-                #     DISTRIB_RELEASE='10.10'
-                #     DISTRIB_CODENAME='squeeze'
-                #     DISTRIB_DESCRIPTION='Ubuntu 10.10'
-                regex = re.compile('^(DISTRIB_(?:ID|RELEASE|CODENAME|DESCRIPTION))=(?:\'|")?([\w\s\.-_]+)(?:\'|")?')
-                match = regex.match(line)
-                if match:
-                    # Adds: lsb_distrib_{id,release,codename,description}
-                    grains['lsb_{0}'.format(match.groups()[0].lower())] = match.groups()[1].rstrip()
         try:
             import lsb_release
             release = lsb_release.get_distro_information()
             for key, value in release.iteritems():
                 grains['lsb_{0}'.format(key.lower())] = value  # override /etc/lsb-release
         except ImportError:
-            pass
+            # if the python library isn't available, default to regex
-        if os.path.isfile('/etc/arch-release'):
+            if os.path.isfile('/etc/lsb-release'):
-            grains['os'] = 'Arch'
+                for line in open('/etc/lsb-release').readlines():
-            grains['os_family'] = 'Arch'
+                    # Matches any possible format:
-        elif os.path.isfile('/etc/debian_version'):
+                    #     DISTRIB_ID="Ubuntu"
-            grains['os'] = 'Debian'
+                    #     DISTRIB_ID='Mageia'
-            grains['os_family'] = 'Debian'
+                    #     DISTRIB_ID=Fedora
-            if 'lsb_distrib_id' in grains:
+                    #     DISTRIB_RELEASE='10.10'
-                if 'Ubuntu' in grains['lsb_distrib_id']:
+                    #     DISTRIB_CODENAME='squeeze'
-                    grains['os'] = 'Ubuntu'
+                    #     DISTRIB_DESCRIPTION='Ubuntu 10.10'
-                elif os.path.isfile('/etc/issue.net') and \
+                    regex = re.compile('^(DISTRIB_(?:ID|RELEASE|CODENAME|DESCRIPTION))=(?:\'|")?([\w\s\.-_]+)(?:\'|")?')
-                  'Ubuntu' in open('/etc/issue.net').readline():
+                    match = regex.match(line)
-                    grains['os'] = 'Ubuntu'
+                    if match:
-        elif os.path.isfile('/etc/gentoo-release'):
+                        # Adds: lsb_distrib_{id,release,codename,description}
-            grains['os'] = 'Gentoo'
+                        grains['lsb_{0}'.format(match.groups()[0].lower())] = match.groups()[1].rstrip()
-            grains['os_family'] = 'Gentoo'
-        elif os.path.isfile('/etc/fedora-release'):
-            grains['os'] = 'Fedora'
-            grains['os_family'] = 'RedHat'
-        elif os.path.isfile('/etc/mandriva-version'):
-            grains['os'] = 'Mandriva'
-            grains['os_family'] = 'Mandriva'
-        elif os.path.isfile('/etc/mandrake-version'):
-            grains['os'] = 'Mandrake'
-            grains['os_family'] = 'Mandriva'
-        elif os.path.isfile('/etc/mageia-version'):
-            grains['os'] = 'Mageia'
-            grains['os_family'] = 'Mageia'
-        elif os.path.isfile('/etc/meego-version'):
-            grains['os'] = 'MeeGo'
-            grains['os_family'] = 'MeeGo'
-        elif os.path.isfile('/etc/vmware-version'):
-            grains['os'] = 'VMWareESX'
-            grains['os_family'] = 'VMWare'
-        elif os.path.isfile('/etc/bluewhite64-version'):
-            grains['os'] = 'Bluewhite64'
-            grains['os_family'] = 'Bluewhite'
-        elif os.path.isfile('/etc/slamd64-version'):
-            grains['os'] = 'Slamd64'
-            grains['os_family'] = 'Slackware'
-        elif os.path.isfile('/etc/slackware-version'):
-            grains['os'] = 'Slackware'
-            grains['os_family'] = 'Slackware'
-        elif os.path.isfile('/etc/enterprise-release'):
-            grains['os_family'] = 'Oracle'
-            if os.path.isfile('/etc/ovs-release'):
-                grains['os'] = 'OVS'
-            else:
-                grains['os'] = 'OEL'
-        elif os.path.isfile('/etc/redhat-release'):
-            grains['os_family'] = 'RedHat'
-            data = open('/etc/redhat-release', 'r').read()
-            if 'centos' in data.lower():
-                grains['os'] = 'CentOS'
-            elif 'scientific' in data.lower():
-                grains['os'] = 'Scientific'
-            else:
-                grains['os'] = 'RedHat'
-        elif os.path.isfile('/etc/system-release'):
-            grains['os_family'] = 'RedHat'
-            data = open('/etc/system-release', 'r').read()
-            if 'amazon' in data.lower():
-                grains['os'] = 'Amazon'
-        elif os.path.isfile('/etc/SuSE-release'):
-            grains['os_family'] = 'Suse'
-            data = open('/etc/SuSE-release', 'r').read()
-            if 'SUSE LINUX Enterprise Server' in data:
-                grains['os'] = 'SLES'
-            elif 'SUSE LINUX Enterprise Desktop' in data:
-                grains['os'] = 'SLED'
-            elif 'openSUSE' in data:
-                grains['os'] = 'openSUSE'
-            else:
-                grains['os'] = 'SUSE'
         # Use the already intelligent platform module to get distro info
-        grains.update(_linux_platform_data(grains))
+        (osname, osrelease, oscodename) = platform.linux_distribution(
-        # If the Linux version can not be determined
+                                              supported_dists=_supported_dists)
-        if not 'os' in grains:
+        # Try to assign these three names based on the lsb info, they tend to
-            grains['os'] = 'Unknown {0}'.format(grains['kernel'])
+        # be more accurate than what python gets from /etc/DISTRO-release.
-            grains['os_family'] = 'Unknown'
+        # It's worth noting that Ubuntu has patched their Python distribution
-    elif grains['kernel'] == 'sunos':
+        # so that platform.linux_distribution() does the /etc/lsb-release
+        # parsing, but we do it anyway here for the sake for full portability.
+        grains['osfullname'] = grains.get('lsb_distrib_id', osname)
+        grains['osrelease'] = grains.get('lsb_distrib_release', osrelease)
+        grains['oscodename'] = grains.get('lsb_distrib_codename', oscodename)
+        # return the first ten characters with no spaces, lowercased
+        shortname = grains['osfullname'].replace(' ', '').lower()[:10]
+        # this maps the long names from the /etc/DISTRO-release files to the
+        # traditional short names that Salt has used.
+        grains['os'] = _os_name_map.get(shortname, grains['osfullname'])
+        grains.update(_linux_cpudata())
+    elif grains['kernel'] == 'SunOS':
         grains['os'] = 'Solaris'
-        grains['os_family'] = 'Solaris'
+        grains.update(_sunos_cpudata(grains))
     elif grains['kernel'] == 'VMkernel':
         grains['os'] = 'ESXi'
-        grains['os_family'] = 'VMWare'
     elif grains['kernel'] == 'Darwin':
         grains['os'] = 'MacOS'
-        grains['os_family'] = 'MacOS'
         grains.update(_bsd_cpudata(grains))
     else:
         grains['os'] = grains['kernel']
-        grains['os_family'] = grains['kernel']
+    if grains['kernel'] in ('FreeBSD', 'OpenBSD'):
-    if grains['kernel'] == 'Linux':
-        grains.update(_linux_cpudata())
-    elif grains['kernel'] in ('FreeBSD', 'OpenBSD'):
         grains.update(_bsd_cpudata(grains))
+    if not grains['os']:
+        grains['os'] = 'Unknown {0}'.format(grains['kernel'])
+        grains['os_family'] = 'Unknown'
+    else:
+        # this assigns family names based on the os name
+        # family defaults to the os name if not found
+        grains['os_family'] = _os_family_map.get(grains['os'],
+                                                 grains['os'])
 
     grains.update(_memdata(grains))
 
@@ -516,14 +474,9 @@ def hostname():
     #   localhost
     #   domain
     grains = {}
-    grains['fqdn'] = socket.getfqdn()
-    comps = grains['fqdn'].split('.')
-    grains['host'] = comps[0]
     grains['localhost'] = socket.gethostname()
-    if len(comps) > 1:
+    grains['fqdn'] = socket.getfqdn()
-        grains['domain'] = '.'.join(comps[1:])
+    (grains['host'], grains['domain']) = grains['fqdn'].partition('.')[::2]
-    else:
-        grains['domain'] = ''
     return grains
 
 

salt/grains/opts.py

@@ -0,0 +1,12 @@
+'''
+Simple grain to merge the opts into the grains directly if the grain_opts
+configuration value is set
+'''
+
+def opts():
+    '''
+    Return the minion configuration settings
+    '''
+    if __opts__.get('grain_opts', False) or __pillar__.get('grain_opts', False):
+        return __opts__
+    return {}

salt/loader.py

@@ -5,19 +5,21 @@ Routines to set up a minion
 # This module still needs package support, so that the functions dict
 # returned can send back functions like: foo.bar.baz
 
-
 # Import python libs
 import os
 import imp
+import sys
 import salt
 import logging
 import tempfile
+import traceback
 
 # Import Salt libs
 from salt.exceptions import LoaderError
 
 log = logging.getLogger(__name__)
 salt_base_path = os.path.dirname(salt.__file__)
+loaded_base_name = 'salt.loaded'
 
 
 def _create_loader(opts, ext_type, tag, ext_dirs=True, ext_type_dirs=None):
@@ -40,6 +42,10 @@ def _create_loader(opts, ext_type, tag, ext_dirs=True, ext_type_dirs=None):
             ext_type_types.extend(opts[ext_type_dirs])
 
     module_dirs = ext_type_types + [ext_types, sys_types]
+    _generate_module('{0}.int'.format(loaded_base_name))
+    _generate_module('{0}.int.{1}'.format(loaded_base_name, tag))
+    _generate_module('{0}.ext'.format(loaded_base_name))
+    _generate_module('{0}.ext.{1}'.format(loaded_base_name, tag))
     return Loader(module_dirs, opts, tag)
 
 
@@ -52,15 +58,10 @@ def minion_mods(opts):
     if opts.get('providers', False):
         if isinstance(opts['providers'], dict):
             for mod, provider in opts['providers'].items():
-                funcs = raw_mod(opts,
+                funcs = raw_mod(opts, provider, functions)
-                        provider,
-                        functions)
                 if funcs:
                     for func in funcs:
-                        f_key = '{0}{1}'.format(
+                        f_key = '{0}{1}'.format(mod, func[func.rindex('.'):])
-                                mod,
-                                func[func.rindex('.'):]
-                                )
                         functions[f_key] = funcs[func]
     return functions
 
@@ -73,12 +74,14 @@ def raw_mod(opts, name, functions):
     return load.gen_module(name, functions)
 
 
-def returners(opts):
+def returners(opts, functions):
     '''
     Returns the returner modules
     '''
     load = _create_loader(opts, 'returners', 'returner')
-    return load.filter_func('returner')
+    pack = {'name': '__salt__',
+            'value': functions}
+    return load.filter_func('returner', pack)
 
 
 def pillars(opts, functions):
@@ -105,7 +108,9 @@ def render(opts, functions):
     '''
     Returns the render modules
     '''
-    load = _create_loader(opts, 'renderers', 'render', ext_type_dirs='render_dirs')
+    load = _create_loader(
+        opts, 'renderers', 'render', ext_type_dirs='render_dirs'
+    )
     pack = {'name': '__salt__',
             'value': functions}
     rend = load.filter_func('render', pack)
@@ -125,20 +130,21 @@ def grains(opts):
     if not 'grains' in opts:
         pre_opts = {}
         salt.config.load_config(
-                pre_opts,
+            pre_opts, opts['conf_file'], 'SALT_MINION_CONFIG'
-                opts['conf_file'],
+        )
-                'SALT_MINION_CONFIG'
-                )
         default_include = pre_opts.get('default_include', [])
         include = pre_opts.get('include', [])
-        pre_opts = salt.config.include_config(default_include, pre_opts,
+        pre_opts = salt.config.include_config(
-                                              opts['conf_file'], verbose=False)
+            default_include, pre_opts, opts['conf_file'], verbose=False
-        pre_opts = salt.config.include_config(include, pre_opts,
+        )
-                                              opts['conf_file'], verbose=True)
+        pre_opts = salt.config.include_config(
+            include, pre_opts, opts['conf_file'], verbose=True
+        )
         if 'grains' in pre_opts:
             opts['grains'] = pre_opts['grains']
         else:
             opts['grains'] = {}
+
     load = _create_loader(opts, 'grains', 'grain', ext_dirs=False)
     grains = load.gen_grains()
     grains.update(opts['grains'])
@@ -151,9 +157,7 @@ def call(fun, **kwargs):
     '''
     args = kwargs.get('args', [])
     dirs = kwargs.get('dirs', [])
-    module_dirs = [
+    module_dirs = [os.path.join(salt_base_path, 'modules')] + dirs
-        os.path.join(salt_base_path, 'modules'),
-        ] + dirs
     load = Loader(module_dirs)
     return load.call(fun, args)
 
@@ -163,14 +167,27 @@ def runner(opts):
     Directly call a function inside a loader directory
     '''
     load = _create_loader(
-            opts,
+        opts, 'runners', 'runner', ext_type_dirs='runner_dirs'
-            'runners',
+    )
-            'runner',
-            ext_type_dirs='runner_dirs'
-            )
     return load.gen_functions()
 
 
+def _generate_module(name):
+    if name in sys.modules:
+        return
+
+    code = "'''Salt loaded {0} parent module'''".format(name.split('.')[-1])
+    module = imp.new_module(name)
+    exec code in module.__dict__
+    sys.modules[name] = module
+
+
+def _mod_type(module_path):
+    if module_path.startswith(salt_base_path):
+        return 'int'
+    return 'ext'
+
+
 class Loader(object):
     '''
     Used to load in arbitrary modules from a directory, the Loader can
@@ -238,9 +255,9 @@ class Loader(object):
                                 return getattr(
                                     mod, fun[fun.rindex('.') + 1:])(*arg)
                 except ImportError:
-                    log.info("Cython is enabled in options though it's not "
+                    log.info('Cython is enabled in options though it\'s not '
-                             "present in the system path. Skipping Cython "
+                             'present in the system path. Skipping Cython '
-                             "modules.")
+                             'modules.')
         return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)
 
     def gen_module(self, name, functions, pack=None):
@@ -264,6 +281,7 @@ class Loader(object):
                         full = full_test
         if not full:
             return None
+
         cython_enabled = False
         if self.opts.get('cython_enable', True) is True:
             try:
@@ -274,24 +292,24 @@ class Loader(object):
                 log.info('Cython is enabled in the options but not present '
                          'in the system path. Skipping Cython modules.')
         try:
-            if full.endswith('.pyx'):
+            if full.endswith('.pyx') and cython_enabled:
                 # If there's a name which ends in .pyx it means the above
                 # cython_enabled is True. Continue...
                 mod = pyximport.load_module(name, full, tempfile.gettempdir())
             else:
                 fn_, path, desc = imp.find_module(name, self.module_dirs)
                 mod = imp.load_module(
-                        '{0}_{1}'.format(name, self.tag),
+                    '{0}.{1}.{2}.{3}'.format(
-                        fn_,
+                        loaded_base_name, _mod_type(path), self.tag, name
-                        path,
+                    ), fn_, path, desc
-                        desc
+                )
-                        )
         except ImportError as exc:
-            log.debug(('Failed to import module {0}: {1}').format(name, exc))
+            log.debug('Failed to import module {0}: {1}'.format(name, exc))
             return mod
         except Exception as exc:
-            log.warning(('Failed to import module {0}, this is due most'
+            trb = traceback.format_exc()
-                ' likely to a syntax error: {1}').format(name, exc))
+            log.warning('Failed to import module {0}, this is due most likely '
+                        'to a syntax error: {1}'.format(name, trb))
             return mod
         if hasattr(mod, '__opts__'):
             mod.__opts__.update(self.opts)
@@ -311,7 +329,7 @@ class Loader(object):
         if hasattr(mod, '__init__'):
             if callable(mod.__init__):
                 try:
-                    mod.__init__()
+                    mod.__init__(self.opts)
                 except TypeError:
                     pass
         funcs = {}
@@ -324,11 +342,12 @@ class Loader(object):
                     if 'BaseException' in func.__bases__:
                         # the callable object is an exception, don't load it
                         continue
+
                 funcs[
-                        '{0}.{1}'.format(
+                    '{0}.{1}'.format(
-                            mod.__name__[:mod.__name__.rindex('_')],
+                        mod.__name__[mod.__name__.rindex('.')+1:], attr
-                            attr)
+                    )
-                        ] = func
+                ] = func
                 self._apply_outputter(func, mod)
         if not hasattr(mod, '__salt__'):
             mod.__salt__ = functions
@@ -364,7 +383,8 @@ class Loader(object):
                     continue
                 if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
                     or (cython_enabled and fn_.endswith('.pyx'))
-                    or os.path.isdir(fn_)):
+                    or os.path.isdir(os.path.join(mod_dir, fn_))):
+
                     extpos = fn_.rfind('.')
                     if extpos > 0:
                         _name = fn_[:extpos]
@@ -377,24 +397,44 @@ class Loader(object):
                     # If there's a name which ends in .pyx it means the above
                     # cython_enabled is True. Continue...
                     mod = pyximport.load_module(
-                            '{0}_{1}'.format(name, self.tag),
+                        '{0}.{1}.{2}.{3}'.format(
-                            names[name],
+                            loaded_base_name,
-                            tempfile.gettempdir())
+                            _mod_type(names[name]),
+                            self.tag,
+                            name
+                        ), names[name], tempfile.gettempdir()
+                    )
                 else:
                     fn_, path, desc = imp.find_module(name, self.module_dirs)
                     mod = imp.load_module(
-                            '{0}_{1}'.format(name, self.tag),
+                        '{0}.{1}.{2}.{3}'.format(
-                            fn_,
+                            loaded_base_name, _mod_type(path), self.tag, name
-                            path,
+                        ), fn_, path, desc
-                            desc
+                    )
-                            )
+                    # reload all submodules if necessary
+                    submodules = [
+                        getattr(mod, sname) for sname in dir(mod) if
+                        type(getattr(mod, sname))==type(mod)
+                    ]
+                    # reload only custom "sub"modules i.e is a submodule in
+                    # parent module that are still available on disk (i.e. not
+                    # removed during sync_modules)
+                    for submodule in submodules:
+                        try:
+                            smname = '{0}.{1}.{2}'.format(loaded_base_name, self.tag, name)
+                            smfile = os.path.splitext(submodule.__file__)[0] + ".py"
+                            if submodule.__name__.startswith(smname) and os.path.isfile(smfile):
+                                reload(submodule)
+                        except AttributeError:
+                            continue
             except ImportError as exc:
-                log.debug(('Failed to import module {0}, this is most likely'
+                log.debug('Failed to import module {0}, this is most likely '
-                           ' NOT a problem: {1}').format(name, exc))
+                          'NOT a problem: {1}'.format(name, exc))
                 continue
             except Exception as exc:
-                log.warning(('Failed to import module {0}, this is due most'
+                trb = traceback.format_exc()
-                    ' likely to a syntax error: {1}').format(name, exc))
+                log.warning('Failed to import module {0}, this is due most '
+                            'likely to a syntax error: {1}'.format(name, trb))
                 continue
             modules.append(mod)
         for mod in modules:
@@ -418,7 +458,7 @@ class Loader(object):
             if hasattr(mod, '__init__'):
                 if callable(mod.__init__):
                     try:
-                        mod.__init__()
+                        mod.__init__(self.opts)
                     except TypeError:
                         pass
 
@@ -444,10 +484,11 @@ class Loader(object):
                         pass
                     else:
                         funcs[
-                                '{0}.{1}'.format(
+                            '{0}.{1}'.format(
-                                    mod.__name__[:mod.__name__.rindex('_')],
+                                mod.__name__[mod.__name__.rindex('.')+1:],
-                                    attr)
+                                attr
-                                ] = func
+                            )
+                        ] = func
                         self._apply_outputter(func, mod)
         for mod in modules:
             if not hasattr(mod, '__salt__'):
@@ -534,10 +575,11 @@ class Loader(object):
                 continue
             try:
                 ret = fun()
-            except Exception as exc:
+            except Exception:
+                trb = traceback.format_exc()
                 log.critical(('Failed to load grains defined in grain file '
-                              '{0} in function {1}, error: {2}').format(
+                              '{0} in function {1}, error:\n{2}').format(
-                                  key, fun, exc))
+                                  key, fun, trb))
                 continue
             if not isinstance(ret, dict):
                 continue