Merge pull request #21999 from basepi/merge-forward-develop

Merge forward from 2014.7 to 2015.2

doc/topics/releases/2015.2.0.rst

@@ -14,6 +14,15 @@ changes, creation, deletion etc.
 This allows for the changes to be sent up to the master where the
 reactor can respond to changes.
 
+Lazy Loader
+========
+
+The Lazy Loader is a significant overhaul of Salt's module loader system. The
+Lazy Loader will lazily load modules on access, instead of all on start. In
+addition to major performance improvement this "sandboxes" modules-- meaning a
+bad/broken import of a single module will only effect jobs that require accessing
+the broken module. (:pull: `20274`)
+
 Salt SSH
 ========
 
@@ -39,6 +48,9 @@ Misc Fixes/Additions
 - LocalClient may now optionally raise SaltClientError exceptions. If using
   this class directly, checking for and handling this exception is recommended.
   (:issue: `21501`)
+- The SAuth object is now a singleton, meaning authentication state is
+  global (per master) on each minion. This reduces sign-ins of minions from 3->1
+  per startup.
 
 Deprecations
 ============
@@ -65,3 +77,11 @@ Deprecations
   This alteration can be accomplished as follows:
 
     ``ALTER TABLE salt_events ADD master_id VARCHAR(255) NOT NULL;``
+
+Known Issues
+===========
+
+- In multimaster mode, a minion may become temporarily unresponsive
+  if modules or pillars are refreshed at the same time that one
+  or more masters are down. This can be worked around by setting
+  'auth_timeout' and 'auth_tries' down to shorter periods.  

salt/cloud/clouds/ec2.py

@@ -1929,10 +1929,10 @@ def wait_for_instance(
             'win_password', vm_, __opts__, default=''
         )
         win_deploy_auth_retries = config.get_cloud_config_value(
-            'win_deploy_auth_retries', vm_, __opts__, default='10'
+            'win_deploy_auth_retries', vm_, __opts__, default=10
         )
         win_deploy_auth_retry_delay = config.get_cloud_config_value(
-            'win_deploy_auth_retry_delay', vm_, __opts__, default='1'
+            'win_deploy_auth_retry_delay', vm_, __opts__, default=1
         )
         if win_passwd and win_passwd == 'auto':
             log.debug('Waiting for auto-generated Windows EC2 password')

salt/fileclient.py

@@ -595,6 +595,7 @@ class Client(object):
                 stream=True,
                 username=url_data.username,
                 password=url_data.password
+                **get_kwargs
             )
             response = query['handle']
             chunk_size = 32 * 1024

salt/minion.py

@@ -550,7 +550,7 @@ class MultiMinion(MinionBase):
             sys.exit(salt.defaults.exitcodes.EX_GENERIC)
         ret = {}
         for master in set(self.opts['master']):
-            s_opts = copy.copy(self.opts)
+            s_opts = copy.deepcopy(self.opts)
             s_opts['master'] = master
             s_opts['multimaster'] = True
             ret[master] = {'opts': s_opts,
@@ -931,7 +931,7 @@ class Minion(MinionBase):
 
         self.opts['grains'] = salt.loader.grains(self.opts, force_refresh)
         if self.opts.get('multimaster', False):
-            s_opts = copy.copy(self.opts)
+            s_opts = copy.deepcopy(self.opts)
             functions = salt.loader.minion_mods(s_opts, notify=notify)
         else:
             functions = salt.loader.minion_mods(self.opts, notify=notify)

salt/modules/dockerio.py

@@ -1723,7 +1723,7 @@ def push(repo, tag=None, quiet=False, insecure_registry=False):
                                        oper='>=',
                                        ver2='0.5.0'):
             kwargs['insecure_registry'] = insecure_registry
-        ret = client.pull(repo, **kwargs)
+        ret = client.push(repo, **kwargs)
         if ret:
             image_logs, infos = _parse_image_multilogs_string(ret)
             if image_logs:

salt/modules/http.py

@@ -2,6 +2,8 @@
 '''
 Module for making various web calls. Primarily designed for webhooks and the
 like, but also useful for basic http testing.
+
+.. versionadded:: 2015.2
 '''
 from __future__ import absolute_import
 
@@ -13,6 +15,8 @@ def query(url, **kwargs):
     '''
     Query a resource, and decode the return data
 
+    .. versionaddedd:: 2015.2
+
     CLI Example:
 
     .. code-block:: bash
@@ -30,6 +34,8 @@ def update_ca_bundle(target=None, source=None, merge_files=None):
     '''
     Update the local CA bundle file from a URL
 
+    .. versionaddedd:: 2015.2
+
     CLI Example:
 
     .. code-block:: bash

salt/netapi/rest_tornado/saltnado.py

@@ -805,8 +805,8 @@ class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn):
                 chunk_ret = yield getattr(self, '_disbatch_{0}'.format(low['client']))(low)
                 ret.append(chunk_ret)
             except Exception as ex:
-                # TODO: log?
                 ret.append('Unexpected exception while handling request: {0}'.format(ex))
+                logger.error('Unexpected exception while handling request:', exc_info=True)
 
         self.write(self.serialize({'return': ret}))
         self.finish()
@@ -841,16 +841,10 @@ class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn):
             # if you have more to go, lets disbatch jobs
             while len(inflight_futures) < maxflight and len(minions) > 0:
                 minion_id = minions.pop(0)
-                f_call['args'][0] = [minion_id]  # set the tgt to the minion
+                batch_chunk = dict(chunk)
-                pub_data = self.saltclients['local'](*f_call.get('args', ()),
+                batch_chunk['tgt'] = [minion_id]
-                                                     **f_call.get('kwargs', {}))
+                batch_chunk['expr_form'] = 'list'
-                # if the job didn't publish, lets not wait around for nothing
+                future = self._disbatch_local(batch_chunk)
-                # we'll just skip
-                # TODO: set header??, some special return?, Or just ignore it (like we do in CLI)
-                if 'jid' not in pub_data:
-                    continue
-                tag = tagify([pub_data['jid'], 'ret', minion_id], 'job')
-                future = self.application.event_listener.get_event(self, tag=tag)
                 inflight_futures.append(future)
 
             # if we have nothing to wait for, don't wait
@@ -860,10 +854,10 @@ class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn):
             # wait until someone is done
             finished_future = yield Any(inflight_futures)
             try:
-                event = finished_future.result()
+                b_ret = finished_future.result()
             except TimeoutException:
                 break
-            chunk_ret[event['data']['id']] = event['data']['return']
+            chunk_ret.update(b_ret)
             inflight_futures.remove(finished_future)
 
         raise tornado.gen.Return(chunk_ret)
@@ -878,10 +872,6 @@ class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn):
         f_call = salt.utils.format_call(self.saltclients['local'], chunk)
         # fire a job off
         try:
-            ping_pub_data = self.saltclients['local'](chunk['tgt'],
-                                                      'test.ping',
-                                                      [],
-                                                      expr_form=f_call['kwargs']['expr_form'])
             pub_data = self.saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {}))
         except EauthAuthenticationError:
             raise tornado.gen.Return('Not authorized to run this job')
@@ -891,42 +881,110 @@ class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn):
         if 'jid' not in pub_data:
             raise tornado.gen.Return('No minions matched the target. No command was sent, no jid was assigned.')
 
-        # get the tag that we are looking for
-        ping_tag = tagify([ping_pub_data['jid'], 'ret'], 'job')
-        ret_tag = tagify([pub_data['jid'], 'ret'], 'job')
-
         # seed minions_remaining with the pub_data
         minions_remaining = pub_data['minions']
 
-        ret_event = self.application.event_listener.get_event(self, tag=ret_tag)
+        syndic_min_wait = None
-        ping_event = self.application.event_listener.get_event(self, tag=ping_tag, timeout=self.application.opts['gather_job_timeout'])
+        if self.application.opts['order_masters']:
+            syndic_min_wait = tornado.gen.sleep(self.application.opts['syndic_wait'])
 
-        # while we are waiting on all the mininons
+        job_not_running = self.job_not_running(pub_data['jid'],
-        while len(minions_remaining) > 0 or not self.min_syndic_wait_done():
+                                               chunk['tgt'],
-            event_future = yield Any([ret_event, ping_event])
+                                               f_call['kwargs']['expr_form'],
-            try:
+                                               minions_remaining=minions_remaining
-                event = event_future.result()
+                                               )
-            # if you hit a timeout, just stop waiting ;)
+
-            except TimeoutException:
+        # if we have a min_wait, do that
-                break
+        if syndic_min_wait is not None:
-            # If someone returned from the ping, and they are new-- add to minions_remaining
+            yield syndic_min_wait
-            if event_future == ping_event:
+        # we are completed when either all minions return or the job isn't running anywhere
-                ping_id = event['data']['id']
+        chunk_ret = yield self.all_returns(pub_data['jid'],
-                if ping_id not in chunk_ret and ping_id not in minions_remaining:
+                                           finish_futures=[job_not_running],
-                    minions_remaining.append(ping_id)
+                                           minions_remaining=minions_remaining,
-                ping_event = self.application.event_listener.get_event(self, tag=ping_tag, timeout=self.application.opts['gather_job_timeout'])
+                                           )
-            # if it is a ret future, its just a regular return
-            else:
-                chunk_ret[event['data']['id']] = event['data']['return']
-                # its possible to get a return that wasn't in the minion_remaining list
-                try:
-                    minions_remaining.remove(event['data']['id'])
-                except ValueError:
-                    pass
-                ret_event = self.application.event_listener.get_event(self, tag=ret_tag)
 
         raise tornado.gen.Return(chunk_ret)
 
+    @tornado.gen.coroutine
+    def all_returns(self,
+                    jid,
+                    finish_futures=None,
+                    minions_remaining=None,
+                    ):
+        '''
+        Return a future which will complete once all returns are completed
+        (according to minions_remaining), or one of the passed in "finish_futures" completes
+        '''
+        if finish_futures is None:
+            finish_futures = []
+        if minions_remaining is None:
+            minions_remaining = []
+
+        ret_tag = tagify([jid, 'ret'], 'job')
+        chunk_ret = {}
+        while True:
+            ret_event = self.application.event_listener.get_event(self,
+                                                      tag=ret_tag,
+                                                      )
+            f = yield Any([ret_event] + finish_futures)
+            if f in finish_futures:
+                raise tornado.gen.Return(chunk_ret)
+            event = f.result()
+            chunk_ret[event['data']['id']] = event['data']['return']
+            # its possible to get a return that wasn't in the minion_remaining list
+            try:
+                minions_remaining.remove(event['data']['id'])
+            except ValueError:
+                pass
+            if len(minions_remaining) == 0:
+                raise tornado.gen.Return(chunk_ret)
+
+    @tornado.gen.coroutine
+    def job_not_running(self,
+                  jid,
+                  tgt,
+                  tgt_type,
+                  minions_remaining=None,
+                  ):
+        '''
+        Return a future which will complete once jid (passed in) is no longer
+        running on tgt
+        '''
+        if minions_remaining is None:
+            minions_remaining = []
+
+        ping_pub_data = self.saltclients['local'](tgt,
+                                                  'saltutil.find_job',
+                                                  [jid],
+                                                  expr_form=tgt_type)
+        ping_tag = tagify([ping_pub_data['jid'], 'ret'], 'job')
+
+        minion_running = False
+        while True:
+            try:
+                event = yield self.application.event_listener.get_event(self,
+                                                                        tag=ping_tag,
+                                                                        timeout=self.application.opts['gather_job_timeout'],
+                                                                        )
+            except TimeoutException:
+                if not minion_running:
+                    raise tornado.gen.Return(True)
+                else:
+                    ping_pub_data = self.saltclients['local'](tgt,
+                                                              'saltutil.find_job',
+                                                              [jid],
+                                                              expr_form=tgt_type)
+                    ping_tag = tagify([ping_pub_data['jid'], 'ret'], 'job')
+                    minion_running = False
+                    continue
+            # Minions can return, we want to see if the job is running...
+            if event['data'].get('return', {}) == {}:
+                continue
+            minion_running = True
+            id_ = event['data']['id']
+            if id_ not in minions_remaining:
+                minions_remaining.append(event['data']['id'])
+
     @tornado.gen.coroutine
     def _disbatch_local_async(self, chunk):
         '''

salt/output/__init__.py

@@ -149,7 +149,10 @@ def get_printout(out, opts=None, **kwargs):
 
     outputters = salt.loader.outputters(opts)
     if out not in outputters:
-        log.error('Invalid outputter {0} specified, fall back to nested'.format(out))
+        # Since the grains outputter was removed we don't need to fire this
+        # error when old minions are asking for it
+        if out != 'grains':
+            log.error('Invalid outputter {0} specified, fall back to nested'.format(out))
         return outputters['nested']
     return outputters[out]
 

salt/renderers/pyobjects.py

@@ -319,7 +319,6 @@ def render(template, saltenv='base', sls='', salt_data=True, **kwargs):
         load_states()
 
     # these hold the scope that our sls file will be executed with
-    _locals = {}
     _globals = {}
 
     # create our StateFactory objects
@@ -441,6 +440,6 @@ def render(template, saltenv='base', sls='', salt_data=True, **kwargs):
     Registry.enabled = True
 
     # now exec our template using our created scopes
-    exec_(final_template, _globals, _locals)
+    exec_(final_template, _globals)
 
     return Registry.salt_data()

salt/returners/carbon_return.py

@@ -2,44 +2,60 @@
 '''
 Take data from salt and "return" it into a carbon receiver
 
-Add the following configuration to the minion configuration files::
+Add the following configuration to the minion configuration file:
+
+.. code-block:: yaml
 
     carbon.host: <server ip address>
     carbon.port: 2003
 
 Errors when trying to convert data to numbers may be ignored by setting
-``carbon.skip_on_error`` to `True`::
+``carbon.skip_on_error`` to `True`:
+
+.. code-block:: yaml
 
     carbon.skip_on_error: True
 
 By default, data will be sent to carbon using the plaintext protocol. To use
-the pickle protocol, set ``carbon.mode`` to ``pickle``::
+the pickle protocol, set ``carbon.mode`` to ``pickle``:
+
+.. code-block:: yaml
 
     carbon.mode: pickle
 
-Carbon settings may also be configured as::
+Carbon settings may also be configured as:
+
+.. code-block:: yaml
 
     carbon:
-        host: <server IP or hostname>
+      host: <server IP or hostname>
-        port: <carbon port>
+      port: <carbon port>
-        skip_on_error: True
+      skip_on_error: True
-        mode: (pickle|text)
+      mode: (pickle|text)
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location:
+
+.. code-block:: yaml
 
     alternative.carbon:
-        host: <server IP or hostname>
+      host: <server IP or hostname>
-        port: <carbon port>
+      port: <carbon port>
-        skip_on_error: True
+      skip_on_error: True
-        mode: (pickle|text)
+      mode: (pickle|text)
 
-  To use the carbon returner, append '--return carbon' to the salt command. ex:
+To use the carbon returner, append '--return carbon' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return carbon
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return carbon --return_config alternative
 '''

salt/returners/couchdb_return.py

@@ -23,7 +23,9 @@ To use the couchdb returner, append ``--return couchdb`` to the salt command. Ex
 
     salt '*' test.ping --return couchdb
 
-To use the alternative configuration, append ``--return_config alternative`` to the salt command. Example:
+To use the alternative configuration, append ``--return_config alternative`` to the salt command.
+
+.. versionadded:: 2015.2.0
 
 .. code-block:: bash
 

salt/returners/hipchat_return.py

@@ -21,40 +21,46 @@ the default location::
     hipchat.api_version
     hipchat.from_name
 
-Hipchat settings may also be configured as::
+Hipchat settings may also be configured as:
+
+.. code-block:: yaml
 
     hipchat:
-        room_id: RoomName
+      room_id: RoomName
-        api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-        api_version: v1
+      api_version: v1
-        from_name: user@email.com
+      from_name: user@email.com
 
     alternative.hipchat:
-        room_id: RoomName
+      room_id: RoomName
-        api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-        api_version: v1
+      api_version: v1
-        from_name: user@email.com
+      from_name: user@email.com
 
     hipchat_profile:
-        api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+      api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-        api_version: v1
+      api_version: v1
-        from_name: user@email.com
+      from_name: user@email.com
 
     hipchat:
-        profile: hipchat_profile
+      profile: hipchat_profile
-        room_id: RoomName
+      room_id: RoomName
 
     alternative.hipchat:
-        profile: hipchat_profile
+      profile: hipchat_profile
-        room_id: RoomName
+      room_id: RoomName
 
-  To use the HipChat returner, append '--return hipchat' to the salt command. ex:
+To use the HipChat returner, append '--return hipchat' to the salt command.
 
-  .. code-block:: bash
+.. code-block:: bash
 
     salt '*' test.ping --return hipchat
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return hipchat --return_config alternative
 '''

salt/returners/memcache_return.py

@@ -4,25 +4,35 @@ Return data to a memcache server
 
 To enable this returner the minion will need the python client for memcache
 installed and the following values configured in the minion or master
-config, these are the defaults:
+config, these are the defaults.
+
+.. code-block:: yaml
 
     memcache.host: 'localhost'
     memcache.port: '11211'
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location.
+
+.. code-block:: yaml
 
     alternative.memcache.host: 'localhost'
     alternative.memcache.port: '11211'
 
 python2-memcache uses 'localhost' and '11211' as syntax on connection.
 
-  To use the memcache returner, append '--return memcache' to the salt command. ex:
+To use the memcache returner, append '--return memcache' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return memcache
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return memcache --return_config alternative
 '''

salt/returners/mongo_future_return.py

@@ -7,7 +7,9 @@ Required python modules: pymongo
 
 This returner will send data from the minions to a MongoDB server. To
 configure the settings for your MongoDB server, add the following lines
-to the minion config files::
+to the minion config files:
+
+.. cod-block:: yaml
 
     mongo.db: <database name>
     mongo.host: <server ip address>
@@ -22,7 +24,9 @@ should greatly improve performance. Indexes are not created by default.
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location:
+
+.. code-block:: yaml
 
     alternative.mongo.db: <database name>
     alternative.mongo.host: <server ip address>
@@ -34,11 +38,17 @@ the default location::
 This mongo returner is being developed to replace the default mongodb returner
 in the future and should not be considered API stable yet.
 
-  To use the mongo returner, append '--return mongo' to the salt command. ex:
+To use the mongo returner, append '--return mongo' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return mongo
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return mongo --return_config alternative
 '''

salt/returners/mongo_return.py

@@ -7,7 +7,9 @@ Required python modules: pymongo
 
 This returner will send data from the minions to a MongoDB server. To
 configure the settings for your MongoDB server, add the following lines
-to the minion config files::
+to the minion config files.
+
+.. code-block:: yaml
 
     mongo.db: <database name>
     mongo.host: <server ip address>
@@ -17,7 +19,9 @@ to the minion config files::
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location.
+
+.. code-block:: yaml
 
     alternative.mongo.db: <database name>
     alternative.mongo.host: <server ip address>
@@ -25,11 +29,17 @@ the default location::
     alternative.mongo.password: <MongoDB user password>
     alternative.mongo.port: 27017
 
-  To use the mongo returner, append '--return mongo' to the salt command. ex:
+To use the mongo returner, append '--return mongo' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return mongo_return
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return mongo_return --return_config alternative
 '''

salt/returners/mysql.py

@@ -9,7 +9,9 @@ Return data to a mysql server
 
 To enable this returner the minion will need the python client for mysql
 installed and the following values configured in the minion or master
-config, these are the defaults::
+config, these are the defaults:
+
+.. code-block:: yaml
 
     mysql.host: 'salt'
     mysql.user: 'salt'
@@ -19,7 +21,9 @@ config, these are the defaults::
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location:
+
+.. code-block:: yaml
 
     alternative.mysql.host: 'salt'
     alternative.mysql.user: 'salt'
@@ -27,7 +31,9 @@ the default location::
     alternative.mysql.db: 'salt'
     alternative.mysql.port: 3306
 
-Use the following mysql database schema::
+Use the following mysql database schema:
+
+.. code-block:: sql
 
     CREATE DATABASE  `salt`
       DEFAULT CHARACTER SET utf8
@@ -81,11 +87,17 @@ Use the following mysql database schema::
 
 Required python modules: MySQLdb
 
-  To use the mysql returner, append '--return mysql' to the salt command. ex:
+To use the mysql returner, append '--return mysql' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return mysql
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return mysql --return_config alternative
 '''

salt/returners/odbc.py

@@ -63,7 +63,9 @@ the default location::
 
 Running the following commands against Microsoft SQL Server in the desired
 database as the appropriate user should create the database tables
-correctly.  Replace with equivalent SQL for other ODBC-compliant servers::
+correctly.  Replace with equivalent SQL for other ODBC-compliant servers
+
+.. code-block:: sql
 
     --
     -- Table structure for table 'jids'
@@ -98,11 +100,17 @@ correctly.  Replace with equivalent SQL for other ODBC-compliant servers::
     CREATE INDEX salt_returns_jid on dbo.salt_returns(jid);
     CREATE INDEX salt_returns_fun on dbo.salt_returns(fun);
 
-  To use this returner, append '--return odbc' to the salt command. ex:
+  To use this returner, append '--return odbc' to the salt command.
+
+  .. code-block:: bash
 
     salt '*' status.diskusage --return odbc
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+  To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+  .. versionadded:: 2015.2.0
+
+  .. code-block:: bash
 
     salt '*' test.ping --return odbc --return_config alternative
 '''

salt/returners/postgres.py

@@ -8,7 +8,9 @@ Return data to a postgresql server
 :platform:      all
 
 To enable this returner the minion will need the psycopg2 installed and
-the following values configured in the minion or master config::
+the following values configured in the minion or master config:
+
+.. code-block:: yaml
 
     returner.postgres.host: 'salt'
     returner.postgres.user: 'salt'
@@ -18,7 +20,9 @@ the following values configured in the minion or master config::
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location:
+
+.. code-block:: yaml
 
     alternative.returner.postgres.host: 'salt'
     alternative.returner.postgres.user: 'salt'
@@ -27,7 +31,9 @@ the default location::
     alternative.returner.postgres.port: 5432
 
 Running the following commands as the postgres user should create the database
-correctly::
+correctly:
+
+.. code-block:: sql
 
     psql << EOF
     CREATE ROLE salt WITH PASSWORD 'salt';
@@ -66,11 +72,17 @@ correctly::
 
 Required python modules: psycopg2
 
-  To use the postgres returner, append '--return postgres' to the salt command. ex:
+To use the postgres returner, append '--return postgres' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return postgres
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return postgres --return_config alternative
 '''

salt/returners/postgres_local_cache.py

@@ -9,7 +9,9 @@ cope with scale.
 :platform:      all
 
 To enable this returner the minion will need the psycopg2 installed and
-the following values configured in the master config::
+the following values configured in the master config:
+
+.. code-block:: yaml
 
     master_job_cache: postgres_local_cache
     master_job_cache.postgres.host: 'salt'
@@ -19,7 +21,9 @@ the following values configured in the master config::
     master_job_cache.postgres.port: 5432
 
 Running the following command as the postgres user should create the database
-correctly::
+correctly:
+
+.. code-block:: sql
 
     psql << EOF
     CREATE ROLE salt WITH PASSWORD 'salt';
@@ -28,6 +32,8 @@ correctly::
 
 and then:
 
+.. code-block:: sql
+
     psql -h localhost -U salt << EOF
     --
     -- Table structure for table 'jids'

salt/returners/redis_return.py

@@ -14,17 +14,25 @@ config, these are the defaults:
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location:
+
+.. code-block:: yaml
 
     alternative.redis.db: '0'
     alternative.redis.host: 'salt'
     alternative.redis.port: 6379
 
-  To use the redis returner, append '--return redis' to the salt command. ex:
+To use the redis returner, append '--return redis' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return redis
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return redis --return_config alternative
 '''

salt/returners/sentry_return.py

@@ -3,7 +3,9 @@
 Salt returner that report execution results back to sentry. The returner will
 inspect the payload to identify errors and flag them as such.
 
-Pillar need something like::
+Pillar needs something like:
+
+.. code-block:: yaml
 
     raven:
       servers:

salt/returners/slack_returner.py

@@ -43,13 +43,15 @@ Hipchat settings may also be configured as::
         profile: slack_profile
         channel: RoomName
 
-  To use the HipChat returner, append '--return slack' to the salt command. ex:
+To use the HipChat returner, append '--return slack' to the salt command.
 
-  .. code-block:: bash
+.. code-block:: bash
 
     salt '*' test.ping --return slack
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return slack --return_config alternative
 '''

salt/returners/sms_return.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 
 '''
-Return data by SMS
+Return data by SMS.
+
+.. versionadded:: 2015.2.0
 
 :maintainer:    Damian Myerscough
 :maturity:      new
@@ -10,14 +12,18 @@ Return data by SMS
 
 To enable this returner the minion will need the python twilio library
 installed and the following values configured in the minion or master
-config::
+config:
+
+.. code-block:: yaml
 
     twilio.sid: 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
     twilio.token: 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
     twilio.to: '+1415XXXXXXX'
     twilio.from: '+1650XXXXXXX'
 
-To use the sms returner, append '--return sms' to the salt command. ex:
+To use the sms returner, append '--return sms' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return sms
 

salt/returners/smtp_return.py

@@ -51,13 +51,17 @@ There are a few things to keep in mind:
   structure (which can be very large). Also note that the subject is always
   unencrypted.
 
-  To use the SMTP returner, append '--return smtp' to the salt command. ex:
+To use the SMTP returner, append '--return smtp' to the salt command.
 
-  .. code-block:: bash
+.. code-block:: bash
 
     salt '*' test.ping --return smtp
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return smtp --return_config alternative
 

salt/returners/sqlite3_return.py

@@ -12,19 +12,25 @@ In order to use this returner the database file must exist,
 have the appropriate schema defined, and be accessible to the
 user whom the minion process is running as. This returner
 requires the following values configured in the master or
-minion config::
+minion config:
+
+.. code-block:: yaml
 
     returner.sqlite3.database: /usr/lib/salt/salt.db
     returner.sqlite3.timeout: 5.0
 
 Alternative configuration values can be used by prefacing the configuration.
 Any values not found in the alternative configuration will be pulled from
-the default location::
+the default location:
+
+.. code-block:: yaml
 
     alternative.returner.sqlite3.database: /usr/lib/salt/salt.db
     alternative.returner.sqlite3.timeout: 5.0
 
-Use the commands to create the sqlite3 database and tables::
+Use the commands to create the sqlite3 database and tables:
+
+.. code-block:: sql
 
     sqlite3 /usr/lib/salt/salt.db << EOF
     --
@@ -51,11 +57,17 @@ Use the commands to create the sqlite3 database and tables::
       );
     EOF
 
-  To use the sqlite returner, append '--return sqlite3' to the salt command. ex:
+To use the sqlite returner, append '--return sqlite3' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return sqlite3
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return sqlite3 --return_config alternative
 

salt/returners/syslog_return.py

@@ -7,7 +7,9 @@ Required python modules: syslog, json
 The syslog returner simply reuses the operating system's syslog
 facility to log return data
 
-To use the syslog returner, append '--return syslog' to the salt command. ex:
+To use the syslog returner, append '--return syslog' to the salt command.
+
+.. code-block:: bash
 
     salt '*' test.ping --return syslog
 

salt/returners/xmpp_return.py

@@ -42,13 +42,17 @@ XMPP settings may also be configured as::
         profile: xmpp_profile
         recipient: someone-else@xmpp.example.com
 
-  To use the XMPP returner, append '--return xmpp' to the salt command. ex:
+To use the XMPP returner, append '--return xmpp' to the salt command.
 
-  .. code-block:: bash
+.. code-block:: bash
 
     salt '*' test.ping --return xmpp
 
-  To use the alternative configuration, append '--return_config alternative' to the salt command. ex:
+To use the alternative configuration, append '--return_config alternative' to the salt command.
+
+.. versionadded:: 2015.2.0
+
+.. code-block:: bash
 
     salt '*' test.ping --return xmpp --return_config alternative
 '''

salt/runners/http.py

@@ -2,6 +2,8 @@
 '''
 Module for making various web calls. Primarily designed for webhooks and the
 like, but also useful for basic http testing.
+
+.. versionadded:: 2015.2
 '''
 from __future__ import absolute_import
 # Import Python libs
@@ -17,6 +19,8 @@ def query(url, output=True, **kwargs):
     '''
     Query a resource, and decode the return data
 
+    .. versionaddedd:: 2015.2
+
     CLI Example:
 
     .. code-block:: bash
@@ -40,6 +44,8 @@ def update_ca_bundle(target=None, source=None, merge_files=None):
     '''
     Update the local CA bundle file from a URL
 
+    .. versionaddedd:: 2015.2
+
     CLI Example:
 
     .. code-block:: bash

salt/states/http.py

@@ -3,6 +3,8 @@
 HTTP monitoring states
 
 Perform an HTTP query and statefully return the result
+
+.. versionaddedd:: 2015.2
 '''
 from __future__ import absolute_import
 
@@ -18,6 +20,8 @@ __monitor__ = [
 def query(name, match=None, match_type='string', status=None, **kwargs):
     '''
     Perform an HTTP query and statefully return the result
+
+    .. versionaddedd:: 2015.2
     '''
     # Monitoring state, but changes may be made over HTTP
     ret = {'name': name,

salt/states/pkg.py

@@ -1557,7 +1557,7 @@ def purged(name, version=None, pkgs=None, **kwargs):
                 'comment': str(exc)}
 
 
-def uptodate(name, refresh=False):
+def uptodate(name, refresh=False, **kwargs):
     '''
     .. versionadded:: 2014.7.0
 
@@ -1569,6 +1569,11 @@ def uptodate(name, refresh=False):
 
     refresh
         refresh the package database before checking for new upgrades
+
+    kwargs
+        Any keyword arguments to pass through to ``pkg.upgrade``.
+
+        .. versionadded:: 2015.2
     '''
     ret = {'name': name,
            'changes': {},
@@ -1598,7 +1603,7 @@ def uptodate(name, refresh=False):
         ret['result'] = None
         return ret
 
-    updated = __salt__['pkg.upgrade'](refresh=refresh)
+    updated = __salt__['pkg.upgrade'](refresh=refresh, **kwargs)
 
     if updated.get('result') is False:
         ret.update(updated)

salt/templates/debian_ip/debian_eth.jinja

@@ -30,11 +30,11 @@
 {%endif%}{% if interface.provider %}    provider {{interface.provider}}
 {%endif%}{% if interface.unit %}    unit {{interface.unit}}
 {%endif%}{% if interface.options %}    options {{interface.options}}
-{%endif%}{% if interface.master %}    bond_master {{interface.master}}
+{%endif%}{% if interface.master %}    bond-master {{interface.master}}
 {%endif%}{% if interface.dns_nameservers %}    dns-nameservers {%for item in interface.dns_nameservers %}{{item}} {%endfor%}
 {%endif%}{% if interface.dns_search %}    dns-search {% for item in interface.dns_search %}{{item}} {%endfor%}
 {%endif%}{% if interface.ethtool %}{%for item in interface.ethtool_keys %}    {{item}} {{interface.ethtool[item]}}
-{%endfor%}{%endif%}{% if interface.bonding %}{%for item in interface.bonding_keys %}    bond_{{item}} {{interface.bonding[item]}}
+{%endfor%}{%endif%}{% if interface.bonding %}{%for item in interface.bonding_keys %}    bond-{{item}} {{interface.bonding[item]}}
 {%endfor%}{%endif%}{% if interface.bridging %}{%for item in interface.bridging_keys %}    bridge_{{item}} {{interface.bridging[item]}}
 {%endfor%}{%endif%}{% if interface.wireless_essid %}    wireless-essid {{interface.wireless_essid}}
 {%endif%}{% if interface.wireless_mode %}    wireless-mode {{interface.wireless_mode}}

salt/utils/http.py

@@ -2,6 +2,8 @@
 '''
 Utils for making various web calls. Primarily designed for REST, SOAP, webhooks
 and the like, but also useful for basic HTTP testing.
+
+.. versionaddedd:: 2015.2
 '''
 
 # Import python libs
@@ -59,6 +61,12 @@ try:
 except ImportError:
     HAS_MSGPACK = False
 
+try:
+    import certifi
+    HAS_CERTIFI = True
+except ImportError:
+    HAS_CERTIFI = False
+
 log = logging.getLogger(__name__)
 JARFILE = os.path.join(syspaths.CACHE_DIR, 'cookies.txt')
 SESSIONJARFILE = os.path.join(syspaths.CACHE_DIR, 'cookies.session.p')
@@ -75,6 +83,7 @@ def query(url,
           header_file=None,
           username=None,
           password=None,
+          auth=None,
           decode=False,
           decode_type='auto',
           status=False,
@@ -455,6 +464,9 @@ def get_ca_bundle(opts=None):
         if os.path.exists(path):
             return path
 
+    if salt.utils.is_windows() and HAS_CERTIFI:
+        return certifi.where()
+
     return None
 
 

salt/utils/job.py

@@ -71,6 +71,7 @@ def store_job(opts, load, event=None, mminion=None):
         return
 
     # otherwise, write to the master cache
+    savefstr = '{0}.save_load'.format(job_cache)
     fstr = '{0}.returner'.format(job_cache)
     if 'fun' not in load and load.get('return', {}):
         ret_ = load.get('return', {})
@@ -79,6 +80,8 @@ def store_job(opts, load, event=None, mminion=None):
         if 'user' in ret_:
             load.update({'user': ret_['user']})
     try:
+        if 'jid' in load:
+            mminion.returners[savefstr](load['jid'], load)
         mminion.returners[fstr](load)
     except KeyError:
         emsg = "Returner '{0}' does not support function returner".format(job_cache)

tests/unit/pyobjects_test.py

@@ -99,6 +99,16 @@ from salt://map.sls import Samba as Other
 Pkg.removed("samba-imported", names=[Other.server, Other.client])
 '''
 
+random_password_template = '''#!pyobjects
+import random, string
+password = ''.join(random.SystemRandom().choice(
+        string.ascii_letters + string.digits) for _ in range(20))
+'''
+
+random_password_import_template = '''#!pyobjecs
+from salt://password.sls import password
+'''
+
 
 class StateTests(TestCase):
     def setUp(self):
@@ -336,6 +346,15 @@ class MapTests(RendererMixin, TestCase):
         ret = samba_with_grains({'os_family': 'RedHat', 'os': 'CentOS'})
         assert_ret(ret, 'samba', 'samba', 'smb')
 
+    def test_random_password(self):
+        '''Test for https://github.com/saltstack/salt/issues/21796'''
+        ret = self.render(random_password_template)
+
+    def test_import_random_password(self):
+        '''Import test for https://github.com/saltstack/salt/issues/21796'''
+        self.write_template_file("password.sls", random_password_template)
+        ret = self.render(random_password_import_template)
+
 
 class SaltObjectTests(TestCase):
     def test_salt_object(self):