Merge branch '2016.3' into 'develop'

Conflicts:
  - salt/client/mixins.py
  - salt/engines/sqs_events.py
  - salt/minion.py

conf/master

@@ -64,7 +64,7 @@
 
 # Directory for custom modules. This directory can contain subdirectories for
 # each of Salt's module types such as "runners", "output", "wheel", "modules",
-# "states", "returners", etc.
+# "states", "returners", "engines", etc.
 # Like 'extension_modules' but can take an array of paths
 #module_dirs: <no default>
 #   - /var/cache/salt/minion/extmods

doc/ref/configuration/master.rst

@@ -238,7 +238,7 @@ The directory to store the pki authentication keys.
 
 Directory for custom modules. This directory can contain subdirectories for
 each of Salt's module types such as ``runners``, ``output``, ``wheel``,
-``modules``, ``states``, ``returners``, etc. This path is appended to
+``modules``, ``states``, ``returners``, ``engines``, etc. This path is appended to
 :conf_master:`root_dir`.
 
 .. code-block:: yaml

doc/ref/configuration/minion.rst

@@ -1198,8 +1198,6 @@ enabled and can be disabled by changing this value to ``False``.
 ``environment``
 ---------------
 
-Default: ``None``
-
 Normally the minion is not isolated to any single environment on the master
 when running states, but the environment can be isolated on the minion side
 by statically setting it. Remember that the recommended way to manage
@@ -1214,14 +1212,22 @@ environments is to isolate via the top file.
 ``state_top_saltenv``
 ---------------------
 
-Default: not set
+This option has no default value. Set it to an environment name to ensure that
+*only* the top file from that environment is considered during a
+:ref:`highstate <running-highstate>`.
 
-Set this option to an environment name, to ensure that *only* the top file from
-that environment is considered during a :ref:`highstate <running-highstate>`.
+.. note::
+    Using this value does not change the merging strategy. For instance, if
+    :conf_minion:`top_file_merging_strategy` is left at its default, and
+    :conf_minion:`state_top_saltenv` is set to ``foo``, then any sections for
+    environments other than ``foo`` in the top file for the ``foo`` environment
+    will be ignored. With :conf_minion:`top_file_merging_strategy` set to
+    ``base``, all states from all environments in the ``base`` top file will
+    be applied, while all other top files are ignored.
 
 .. code-block:: yaml
 
-    state_top_saltenv: base
+    state_top_saltenv: dev
 
 .. conf_minion:: top_file_merging_strategy
 

doc/ref/states/top.rst

@@ -437,9 +437,8 @@ Scenario 2 - No Environment Specified, :conf_minion:`top_file_merging_strategy`
 In this scenario, assuming that the ``base`` environment's top file was
 evaluated first, the ``base1``, ``dev1``, and ``qa1`` states would be applied
 to all minions. If, for instance, the ``qa`` environment is not defined in
-**/srv/salt/base/top.sls**, then the ``qa`` section in
-**/srv/salt/dev/top.sls** would be used and the ``qa2`` states would be applied
-to all minions.
+**/srv/salt/base/top.sls**, then because there is no top file for the ``qa``
+environment, no states from the ``qa`` environment would be applied.
 
 Scenario 3 - No Environment Specified, :conf_minion:`top_file_merging_strategy` is "same"
 -----------------------------------------------------------------------------------------

salt/client/mixins.py

@@ -368,6 +368,10 @@ class SyncClientMixin(object):
             data['fun_args'] = args + ([kwargs] if kwargs else [])
             func_globals['__jid_event__'].fire_event(data, 'new')
 
+            # Update the event data with loaded args and kwargs
+            data['fun_args'] = args + ([kwargs] if kwargs else [])
+            func_globals['__jid_event__'].fire_event(data, 'new')
+
             # Initialize a context for executing the method.
             with tornado.stack_context.StackContext(self.functions.context_dict.clone):
                 data['return'] = self.functions[fun](*args, **kwargs)

salt/client/ssh/__init__.py

@@ -1129,11 +1129,12 @@ ARGS = {10}\n'''.format(self.minion_config,
                     if not self.tty:
                         # If RSTR is not seen in both stdout and stderr then there
                         # was a thin deployment problem.
-                        return 'ERROR: Failure deploying thin: {0}\n{1}'.format(stdout, stderr), stderr, retcode
+                        log.error('ERROR: Failure deploying thin, retrying: {0}\n{1}'.format(stdout, stderr), stderr, retcode)
+                        return self.cmd_block()
                     elif not re.search(RSTR_RE, stdout):
                         # If RSTR is not seen in stdout with tty, then there
                         # was a thin deployment problem.
-                        return 'ERROR: Failure deploying thin: {0}\n{1}'.format(stdout, stderr), stderr, retcode
+                        log.error('ERROR: Failure deploying thin, retrying: {0}\n{1}'.format(stdout, stderr), stderr, retcode)
                 while re.search(RSTR_RE, stdout):
                     stdout = re.split(RSTR_RE, stdout, 1)[1].strip()
                 if self.tty:

salt/cloud/clouds/vmware.py

@@ -1425,7 +1425,7 @@ def list_nodes_min(kwargs=None, call=None):
     vm_list = salt.utils.vmware.get_mors_with_properties(_get_si(), vim.VirtualMachine, vm_properties)
 
     for vm in vm_list:
-        ret[vm["name"]] = True
+        ret[vm['name']] = {'state': 'Running', 'id': vm['name']}
 
     return ret
 

salt/engines/sqs_events.py

@@ -28,8 +28,9 @@ configuration is necessary. More Information available at::
 
    http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
 
-If IAM roles are not used you need to specify them either in a pillar or
-in the config file of the master or minion, as appropriate:
+If IAM roles are not (or for ``boto`` version < 2.5.1) used you need to
+specify them either in a pillar or in the config file of the master or
+minion, as appropriate:
 
 To deserialize the message from json:
 

salt/minion.py

@@ -2485,6 +2485,14 @@ class SyndicManager(MinionBase):
                 break
             master_id = masters.pop(0)
 
+    def _reset_event_aggregation(self):
+        self.job_rets = {}
+        self.raw_events = []
+
+    def reconnect_event_bus(self, something):
+        future = self.local.event.set_event_handler(self._process_event)
+        self.io_loop.add_future(future, self.reconnect_event_bus)
+
     # Syndic Tune In
     def tune_in(self):
         '''
@@ -2501,7 +2509,9 @@ class SyndicManager(MinionBase):
         # register the event sub to the poller
         self.job_rets = {}
         self.raw_events = []
-        self.local.event.set_event_handler(self._process_event)
+        self._reset_event_aggregation()
+        future = self.local.event.set_event_handler(self._process_event)
+        self.io_loop.add_future(future, self.reconnect_event_bus)
 
         # forward events every syndic_event_forward_timeout
         self.forward_events = tornado.ioloop.PeriodicCallback(self._forward_events,

salt/modules/file.py

@@ -4608,8 +4608,15 @@ def makedirs_(path,
             break
 
         directories_to_create.append(dirname)
+        current_dirname = dirname
         dirname = os.path.dirname(dirname)
 
+        if current_dirname == dirname:
+            raise SaltInvocationError(
+                'Recursive creation for path \'{0}\' would result in an '
+                'infinite loop. Please use an absolute path.'.format(dirname)
+            )
+
     # create parent directories from the topmost to the most deeply nested one
     directories_to_create.reverse()
     for directory_to_create in directories_to_create:

salt/transport/ipc.py

@@ -279,6 +279,9 @@ class IPCClient(object):
         if hasattr(self, '_connecting_future') and not self._connecting_future.done():  # pylint: disable=E0203
             future = self._connecting_future  # pylint: disable=E0203
         else:
+            if hasattr(self, '_connecting_future'):
+                # read previous future result to prevent the "unhandled future exception" error
+                self._connecting_future.exc_info()  # pylint: disable=E0203
             future = tornado.concurrent.Future()
             self._connecting_future = future
             self._connect(timeout=timeout)
@@ -693,6 +696,7 @@ class IPCMessageSubscriber(IPCClient):
             except Exception as exc:
                 log.error('Exception occurred while Subscriber handling stream: {0}'.format(exc))
 
+    @tornado.gen.coroutine
     def read_async(self, callback):
         '''
         Asynchronously read messages and invoke a callback when they are ready.
@@ -701,13 +705,14 @@ class IPCMessageSubscriber(IPCClient):
         '''
         while not self.connected():
             try:
-                self.connect()
+                yield self.connect(timeout=5)
             except tornado.iostream.StreamClosedError:
                 log.trace('Subscriber closed stream on IPC {0} before connect'.format(self.socket_path))
+                yield tornado.gen.sleep(1)
             except Exception as exc:
                 log.error('Exception occurred while Subscriber connecting: {0}'.format(exc))
-
-        self.io_loop.spawn_callback(self._read_async, callback)
+                yield tornado.gen.sleep(1)
+        yield self._read_async(callback)
 
     def close(self):
         '''

salt/utils/event.py

@@ -773,7 +773,7 @@ class SaltEvent(object):
         if not self.cpub:
             self.connect_pub()
         # This will handle reconnects
-        self.subscriber.read_async(event_handler)
+        return self.subscriber.read_async(event_handler)
 
     def __del__(self):
         # skip exceptions in destroy-- since destroy() doesn't cover interpreter