Merge pull request #41021 from MTecknology/develop

New feature: automatic job batching

conf/master

@@ -282,6 +282,15 @@
 # a value for you. Default is disabled.
 # ipc_write_buffer: 'dynamic'
 
+# These two batch settings, batch_safe_limit and batch_safe_size, are used to
+# automatically switch to a batch mode execution. If a command would have been
+# sent to more than <batch_safe_limit> minions, then run the command in
+# batches of <batch_safe_size>. If no batch_safe_size is specified, a default
+# of 8 will be used. If no batch_safe_limit is specified, then no automatic
+# batching will occur.
+#batch_safe_limit: 100
+#batch_safe_size: 8
+
 
 #####        Security settings       #####
 ##########################################

salt/cli/salt.py

@@ -39,8 +39,7 @@ class SaltCMD(parsers.SaltCMDOptionParser):
 
         try:
             # We don't need to bail on config file permission errors
-            # if the CLI
-            # process is run with the -a flag
+            # if the CLI process is run with the -a flag
             skip_perm_errors = self.options.eauth != ''
 
             self.local_client = salt.client.get_local_client(
@@ -60,7 +59,8 @@ class SaltCMD(parsers.SaltCMDOptionParser):
             return
 
         if self.options.preview_target:
-            self._preview_target()
+            minion_list = self._preview_target()
+            self._output_ret(minion_list, self.config.get('output', 'nested'))
             return
 
         if self.options.timeout <= 0:
@@ -88,6 +88,15 @@ class SaltCMD(parsers.SaltCMDOptionParser):
         else:
             kwargs['tgt_type'] = 'glob'
 
+        # If batch_safe_limit is set, check minions matching target and
+        # potentially switch to batch execution
+        if self.options.batch_safe_limit > 1:
+            if len(self._preview_target()) >= self.options.batch_safe_limit:
+                print_cli('\nNOTICE: Too many minions targeted, switching to batch execution.')
+                self.options.batch = self.options.batch_safe_size
+                self._run_batch()
+                return
+
         if getattr(self.options, 'return'):
             kwargs['ret'] = getattr(self.options, 'return')
 
@@ -202,8 +211,7 @@ class SaltCMD(parsers.SaltCMDOptionParser):
         '''
         Return a list of minions from a given target
         '''
-        minion_list = self.local_client.gather_minions(self.config['tgt'], self.selected_target_option or 'glob')
-        self._output_ret(minion_list, self.config.get('output', 'nested'))
+        return self.local_client.gather_minions(self.config['tgt'], self.selected_target_option or 'glob')
 
     def _run_batch(self):
         import salt.cli.batch
@@ -250,6 +258,7 @@ class SaltCMD(parsers.SaltCMDOptionParser):
 
         else:
             try:
+                self.config['batch'] = self.options.batch
                 batch = salt.cli.batch.Batch(self.config, eauth=eauth, parser=self.options)
             except salt.exceptions.SaltClientError as exc:
                 # We will print errors to the console further down the stack

salt/utils/parsers.py

@@ -1913,6 +1913,20 @@ class SaltCMDOptionParser(six.with_metaclass(OptionParserMeta,
             help=('Wait the specified time in seconds after each job is done '
                   'before freeing the slot in the batch for the next one.')
         )
+        self.add_option(
+            '--batch-safe-limit',
+            default=0,
+            dest='batch_safe_limit',
+            type=int,
+            help=('Execute the salt job in batch mode if the job would have '
+                  'executed on more than this many minions.')
+        )
+        self.add_option(
+            '--batch-safe-size',
+            default=8,
+            dest='batch_safe_size',
+            help=('Batch size to use for batch jobs created by batch-safe-limit.')
+        )
         self.add_option(
             '--return',
             default='',