Merge pull request #29964 from The-Loeki/6691-cmd-no-wait

Implement #6691 cmd in background

salt/modules/cmdmod.py

@@ -237,6 +237,7 @@ def _run(cmd,
          pillar_override=None,
          use_vt=False,
          password=None,
+         bg=False,
          **kwargs):
     '''
     Do the DRY thing and only call subprocess.Popen() once
@@ -418,7 +419,10 @@ def _run(cmd,
               'stdin': str(stdin) if stdin is not None else stdin,
               'stdout': stdout,
               'stderr': stderr,
-              'with_communicate': with_communicate}
+              'with_communicate': with_communicate,
+              'timeout': timeout,
+              'bg': bg,
+              }
 
     if umask is not None:
         _umask = str(umask).lstrip('0')
@@ -470,7 +474,7 @@ def _run(cmd,
             )
 
         try:
-            proc.wait(timeout)
+            proc.run()
         except TimedProcTimeoutError as exc:
             ret['stdout'] = str(exc)
             ret['stderr'] = ''
@@ -662,6 +666,7 @@ def run(cmd,
         ignore_retcode=False,
         saltenv='base',
         use_vt=False,
+        bg=False,
         **kwargs):
     r'''
     Execute the passed command and return the output as a string
@@ -692,6 +697,8 @@ def run(cmd,
     :param bool python_shell: If False, let python handle the positional
     arguments. Set to True to use shell features, such as pipes or redirection
 
+    :param bool bg: If True, run command in background and do not await or deliver it's results
+
     :param list env: A list of environment variables to be set prior to
     execution.
 
@@ -829,7 +836,8 @@ def run(cmd,
                pillarenv=kwargs.get('pillarenv'),
                pillar_override=kwargs.get('pillar'),
                use_vt=use_vt,
-               password=kwargs.get('password', None))
+               password=kwargs.get('password', None),
+               bg=bg)
 
     log_callback = _check_cb(log_callback)
 
@@ -888,6 +896,7 @@ def shell(cmd,
         ignore_retcode=False,
         saltenv='base',
         use_vt=False,
+        bg=False,
         **kwargs):
     '''
     Execute the passed command and return the output as a string.
@@ -914,6 +923,8 @@ def shell(cmd,
     :param int shell: Shell to execute under. Defaults to the system default
     shell.
 
+    :param bool bg: If True, run command in background and do not await or deliver it's results
+
     :param list env: A list of environment variables to be set prior to
     execution.
 
@@ -1048,6 +1059,7 @@ def shell(cmd,
                saltenv=saltenv,
                use_vt=use_vt,
                python_shell=python_shell,
+               bg=bg,
                **kwargs)
 
 
@@ -1841,6 +1853,7 @@ def script(source,
            __env__=None,
            saltenv='base',
            use_vt=False,
+           bg=False,
            **kwargs):
     '''
     Download a script from a remote location and execute the script locally.
@@ -1880,6 +1893,8 @@ def script(source,
     :param bool python_shell: If False, let python handle the positional
     arguments. Set to True to use shell features, such as pipes or redirection
 
+    :param bool bg: If True, run script in background and do not await or deliver it's results
+
     :param list env: A list of environment variables to be set prior to
     execution.
 
@@ -2026,7 +2041,8 @@ def script(source,
                pillarenv=kwargs.get('pillarenv'),
                pillar_override=kwargs.get('pillar'),
                use_vt=use_vt,
-               password=kwargs.get('password', None))
+               password=kwargs.get('password', None),
+               bg=bg)
     _cleanup_tempfile(path)
     return ret
 
@@ -2316,6 +2332,7 @@ def run_chroot(root,
                ignore_retcode=False,
                saltenv='base',
                use_vt=False,
+               bg=False,
                **kwargs):
     '''
     .. versionadded:: 2014.7.0
@@ -2464,7 +2481,8 @@ def run_chroot(root,
                    saltenv=saltenv,
                    pillarenv=kwargs.get('pillarenv'),
                    pillar=kwargs.get('pillar'),
-                   use_vt=use_vt)
+                   use_vt=use_vt,
+                   bg=bg)
 
     # Kill processes running in the chroot
     for i in range(6):
@@ -2708,3 +2726,171 @@ def powershell(cmd,
     except Exception:
         log.error("Error converting PowerShell JSON return", exc_info=True)
         return {}
+
+
+def run_bg(cmd,
+        cwd=None,
+        runas=None,
+        shell=DEFAULT_SHELL,
+        python_shell=None,
+        env=None,
+        clean_env=False,
+        template=None,
+        umask=None,
+        log_callback=None,
+        timeout=None,
+        reset_system_locale=True,
+        saltenv='base',
+        **kwargs):
+    r'''
+    .. versionadded: Boron
+
+    Execute the passed command in the background and return it's PID
+
+    Note that ``env`` represents the environment variables for the command, and
+    should be formatted as a dict, or a YAML string which resolves to a dict.
+
+    :param str cmd: The command to run. ex: 'ls -lart /home'
+
+    :param str cwd: The current working directory to execute the command in,
+    defaults to `/root` (`C:\` in windows)
+
+    :param str runas: User to run script as. If running on a Windows minion you
+    must also pass a password
+
+    :param str shell: Shell to execute under. Defaults to the system default
+    shell.
+
+    :param bool python_shell: If False, let python handle the positional
+    arguments. Set to True to use shell features, such as pipes or redirection
+
+    :param list env: A list of environment variables to be set prior to
+    execution.
+
+        Example:
+
+        .. code-block:: yaml
+
+            salt://scripts/foo.sh:
+              cmd.script:
+                - env:
+                  - BATCH: 'yes'
+
+        .. warning::
+
+            The above illustrates a common PyYAML pitfall, that **yes**,
+            **no**, **on**, **off**, **true**, and **false** are all loaded as
+            boolean ``True`` and ``False`` values, and must be enclosed in
+            quotes to be used as strings. More info on this (and other) PyYAML
+            idiosyncrasies can be found :doc:`here
+            </topics/troubleshooting/yaml_idiosyncrasies>`.
+
+        Variables as values are not evaluated. So $PATH in the following
+        example is a literal '$PATH':
+
+        .. code-block:: yaml
+
+            salt://scripts/bar.sh:
+              cmd.script:
+                - env: "PATH=/some/path:$PATH"
+
+        One can still use the existing $PATH by using a bit of Jinja:
+
+        .. code-block:: yaml
+
+            {% set current_path = salt['environ.get']('PATH', '/bin:/usr/bin') %}
+
+            mycommand:
+              cmd.run:
+                - name: ls -l /
+                - env:
+                  - PATH: {{ [current_path, '/my/special/bin']|join(':') }}
+
+    :param bool clean_env: Attempt to clean out all other shell environment
+    variables and set only those provided in the 'env' argument to this
+    function.
+
+    :param str template: If this setting is applied then the named templating
+    engine will be used to render the downloaded file. Currently jinja, mako,
+    and wempy are supported
+
+    :param str umask: The umask (in octal) to use when running the command.
+
+    :param int timeout: A timeout in seconds for the executed process to return.
+
+    .. warning::
+
+        This function does not process commands through a shell
+        unless the python_shell flag is set to True. This means that any
+        shell-specific functionality such as 'echo' or the use of pipes,
+        redirection or &&, should either be migrated to cmd.shell or
+        have the python_shell=True flag set here.
+
+        The use of python_shell=True means that the shell will accept _any_ input
+        including potentially malicious commands such as 'good_command;rm -rf /'.
+        Be absolutely certain that you have sanitized your input prior to using
+        python_shell=True
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt '*' cmd.run_bg "ls -l | awk '/foo/{print \\$2}'"
+
+    The template arg can be set to 'jinja' or another supported template
+    engine to render the command arguments before execution.
+    For example:
+
+    .. code-block:: bash
+
+        salt '*' cmd.run_bg template=jinja "ls -l /tmp/{{grains.id}} | awk '/foo/{print \\$2}'"
+
+    Specify an alternate shell with the shell parameter:
+
+    .. code-block:: bash
+
+        salt '*' cmd.run "Get-ChildItem C:\\ " shell='powershell'
+
+    If an equal sign (``=``) appears in an argument to a Salt command it is
+    interpreted as a keyword argument in the format ``key=val``. That
+    processing can be bypassed in order to pass an equal sign through to the
+    remote shell command by manually specifying the kwarg:
+
+    .. code-block:: bash
+
+        salt '*' cmd.run cmd='sed -e s/=/:/g'
+    '''
+
+    python_shell = _python_shell_default(python_shell,
+                                         kwargs.get('__pub_jid', ''))
+    res = _run(cmd,
+               stdin=None,
+               stderr=None,
+               stdout=None,
+               output_loglevel=None,
+               use_vt=None,
+               bg=True,
+               with_communicate=False,
+               rstrip=False,
+
+               runas=runas,
+               shell=shell,
+               python_shell=python_shell,
+               cwd=cwd,
+               env=env,
+               clean_env=clean_env,
+               template=template,
+               umask=umask,
+               log_callback=log_callback,
+               timeout=timeout,
+               reset_system_locale=reset_system_locale,
+               # ignore_retcode=ignore_retcode,
+               saltenv=saltenv,
+               pillarenv=kwargs.get('pillarenv'),
+               pillar_override=kwargs.get('pillar'),
+               # password=kwargs.get('password', None),
+               )
+
+    return {
+        'pid': res['pid']
+    }

salt/utils/timed_subprocess.py

@@ -14,13 +14,28 @@ class TimedProc(object):
     '''
     def __init__(self, args, **kwargs):
 
+        self.wait = not kwargs.pop('bg', False)
         self.stdin = kwargs.pop('stdin', None)
-        if self.stdin is not None:
+        self.with_communicate = kwargs.pop('with_communicate', self.wait)
+        self.timeout = kwargs.pop('timeout', None)
+
+        # If you're not willing to wait for the process
+        # you can't define any stdin, stdout or stderr
+        if not self.wait:
+            self.stdin = kwargs['stdin'] = None
+            self.with_communicate = False
+        elif self.stdin is not None:
             # Translate a newline submitted as '\n' on the CLI to an actual
             # newline character.
             self.stdin = self.stdin.replace('\\n', '\n')
             kwargs['stdin'] = subprocess.PIPE
-        self.with_communicate = kwargs.pop('with_communicate', True)
+
+        if not self.with_communicate:
+            self.stdout = kwargs['stdout'] = None
+            self.stderr = kwargs['stderr'] = None
+
+        if self.timeout and not isinstance(self.timeout, (int, float)):
+            raise salt.exceptions.TimedProcTimeoutError('Error: timeout {0} must be a number'.format(self.timeout))
 
         try:
             self.process = subprocess.Popen(args, **kwargs)
@@ -35,24 +50,23 @@ class TimedProc(object):
             self.process = subprocess.Popen(args, **kwargs)
         self.command = args
 
-    def wait(self, timeout=None):
+    def run(self):
         '''
         wait for subprocess to terminate and return subprocess' return code.
         If timeout is reached, throw TimedProcTimeoutError
         '''
         def receive():
             if self.with_communicate:
-                (self.stdout, self.stderr) = self.process.communicate(input=self.stdin)
-            else:
+                self.stdout, self.stderr = self.process.communicate(input=self.stdin)
+            elif self.wait:
                 self.process.wait()
-                (self.stdout, self.stderr) = (None, None)
 
-        if timeout:
-            if not isinstance(timeout, (int, float)):
-                raise salt.exceptions.TimedProcTimeoutError('Error: timeout must be a number')
+        if not self.timeout:
+            receive()
+        else:
             rt = threading.Thread(target=receive)
             rt.start()
-            rt.join(timeout)
+            rt.join(self.timeout)
             if rt.isAlive():
                 # Subprocess cleanup (best effort)
                 self.process.kill()
@@ -64,9 +78,7 @@ class TimedProc(object):
                 raise salt.exceptions.TimedProcTimeoutError(
                     '{0} : Timed out after {1} seconds'.format(
                         self.command,
-                        str(timeout),
+                        str(self.timeout),
                     )
                 )
-        else:
-            receive()
         return self.process.returncode