Merge pull request #19637 from s0undt3ch/develop

Merge branch '2015.2' into develop

salt/modules/brew.py

@@ -36,7 +36,7 @@ def _list_taps():
     return _call_brew(cmd)['stdout'].splitlines()
 
 
-def _tap(tap):
+def _tap(tap, runas=None):
     '''
     Add unofficial Github repos to the list of formulas that brew tracks,
     updates, and installs from.
@@ -45,7 +45,7 @@ def _tap(tap):
         return True
 
     cmd = 'brew tap {0}'.format(tap)
-    if _call_brew(cmd)['retcode']:
+    if __salt__['cmd.retcode'](cmd, python_shell=False, runas=runas):
         log.error('Failed to tap "{0}"'.format(tap))
         return False
 

salt/modules/disk.py

@@ -72,7 +72,7 @@ def usage(args=None):
     if flags:
         cmd += ' -{0}'.format(flags)
     ret = {}
-    out = __salt__['cmd.run'](cmd).splitlines()
+    out = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
     for line in out:
         if not line:
             continue
@@ -123,7 +123,7 @@ def inodeusage(args=None):
     if flags:
         cmd += ' -{0}'.format(flags)
     ret = {}
-    out = __salt__['cmd.run'](cmd).splitlines()
+    out = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
     for line in out:
         if line.startswith('Filesystem'):
             continue
@@ -172,7 +172,7 @@ def percent(args=None):
     else:
         cmd = 'df'
     ret = {}
-    out = __salt__['cmd.run'](cmd).splitlines()
+    out = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
     for line in out:
         if not line:
             continue

salt/modules/freebsdservice.py

@@ -49,7 +49,7 @@ def _get_rcscript(name):
     Return full path to service rc script
     '''
     cmd = '{0} -r'.format(_cmd())
-    for line in __salt__['cmd.run_stdout'](cmd).splitlines():
+    for line in __salt__['cmd.run_stdout'](cmd, python_shell=False).splitlines():
         if line.endswith('{0}{1}'.format(os.path.sep, name)):
             return line
     return None
@@ -65,7 +65,7 @@ def _get_rcvar(name):
 
     cmd = '{0} {1} rcvar'.format(_cmd(), name)
 
-    for line in __salt__['cmd.run_stdout'](cmd, python_path=False).splitlines():
+    for line in __salt__['cmd.run_stdout'](cmd, python_shell=False).splitlines():
         if '_enable="' not in line:
             continue
         rcvar, _ = line.split('=', 1)

salt/modules/qemu_img.py

@@ -45,6 +45,7 @@ def make_image(location, size, fmt):
             'qemu-img create -f {0} {1} {2}M'.format(
                 fmt,
                 location,
-                size)):
+                size),
+                python_shell=False):
         return location
     return ''

salt/modules/rdp.py

@@ -41,7 +41,7 @@ def _psrdp(cmd):
            '-Namespace root\\CIMV2\\TerminalServices -Computer . '
            '-Authentication 6 -ErrorAction Stop')
     return __salt__['cmd.run']('{0} ; {1}'.format(rdp, cmd),
-                               shell='powershell')
+                               shell='powershell', python_shell=True)
 
 
 def enable():

salt/modules/service.py

@@ -76,7 +76,7 @@ def start(name):
         _GRAINMAP.get(__grains__.get('os'), '/etc/init.d'),
         name
     ) + ' start'
-    return not __salt__['cmd.retcode'](cmd)
+    return not __salt__['cmd.retcode'](cmd, python_shell=False)
 
 
 def stop(name):
@@ -93,7 +93,7 @@ def stop(name):
         _GRAINMAP.get(__grains__.get('os'), '/etc/init.d'),
         name
     ) + ' stop'
-    return not __salt__['cmd.retcode'](cmd)
+    return not __salt__['cmd.retcode'](cmd, python_shell=False)
 
 
 def restart(name):
@@ -110,7 +110,7 @@ def restart(name):
         _GRAINMAP.get(__grains__.get('os'), '/etc/init.d'),
         name
     ) + ' restart'
-    return not __salt__['cmd.retcode'](cmd)
+    return not __salt__['cmd.retcode'](cmd, python_shell=False)
 
 
 def status(name, sig=None):
@@ -143,7 +143,7 @@ def reload_(name):
         _GRAINMAP.get(__grains__.get('os'), '/etc/init.d'),
         name
     ) + ' reload'
-    return not __salt__['cmd.retcode'](cmd)
+    return not __salt__['cmd.retcode'](cmd, python_shell=False)
 
 
 def available(name):

salt/modules/supervisord.py

@@ -19,11 +19,10 @@ from salt.exceptions import CommandExecutionError, CommandNotFoundError
 
 
 def __virtual__():
-    HAS_SUPER = salt.utils.which('supervisorctl')
+    # We can't decide at load time whether supervisorctl is present. The
-    if HAS_SUPER:
+    # function _get_supervisorctl_bin does a much more thorough job and can
-        return True
+    # only be accurate at call time.
-    else:
+    return True
-        return False
 
 
 def _get_supervisorctl_bin(bin_env):

salt/modules/win_servermanager.py

@@ -5,6 +5,12 @@ Manage Windows features via the ServerManager powershell module
 from __future__ import absolute_import
 
 
+# Import python libs
+try:
+    from shlex import quote as _cmd_quote  # pylint: disable=E0611
+except ImportError:
+    from pipes import quote as _cmd_quote
+
 # Import salt libs
 import salt.utils
 
@@ -97,7 +103,8 @@ def install(feature, recurse=False):
     sub = ''
     if recurse:
         sub = '-IncludeAllSubFeature'
-    out = _srvmgr('Add-WindowsFeature -Name {0} {1} -erroraction silentlycontinue | format-list'.format(feature, sub))
+    out = _srvmgr('Add-WindowsFeature -Name {0} {1} -erroraction silentlycontinue | format-list'.format(
+                  _cmd_quote(feature), sub))
     return _parse_powershell_list(out)
 
 
@@ -118,5 +125,6 @@ def remove(feature):
 
         salt -t 600 '*' win_servermanager.remove Telnet-Client
     '''
-    out = _srvmgr('Remove-WindowsFeature -Name {0} -erroraction silentlycontinue | format-list'.format(feature))
+    out = _srvmgr('Remove-WindowsFeature -Name {0} -erroraction silentlycontinue | format-list'.format(
+                  _cmd_quote(feature)))
     return _parse_powershell_list(out)

salt/syspaths.py

@@ -21,54 +21,107 @@
 from __future__ import absolute_import
 import sys
 import os.path
-import logging
-log = logging.getLogger(__name__)
-if 'SETUP_DIRNAME' in globals():
-    # This is from the exec() call in Salt's setup.py
-    THIS_FILE = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py')  # pylint: disable=E0602
-else:
-    THIS_FILE = __file__
 
-# These defaults won't changes and are not to be overridden
+__PLATFORM = sys.platform.lower()
-INSTALL_DIR = os.path.dirname(os.path.realpath(THIS_FILE))
-CLOUD_DIR = os.path.join(INSTALL_DIR, 'cloud')
-BOOTSTRAP = os.path.join(CLOUD_DIR, 'deploy', 'bootstrap-salt.sh')
 
 try:
     # Let's try loading the system paths from the generated module at
     # installation time.
-    from salt._syspaths import (  # pylint: disable=W0611,E0611,import-error
+    import salt._syspaths as __generated_syspaths  # pylint: disable=no-name-in-module
-        ROOT_DIR,                 # because pylint thinks that _syspaths is an
+except ImportError:
-        CONFIG_DIR,               # attribute of salt.__init__
+    class __generated_syspaths(object):
-        CACHE_DIR,
+        __slots__ = ('ROOT_DIR',
-        SOCK_DIR,
+                     'CONFIG_DIR',
-        SRV_ROOT_DIR,
+                     'CACHE_DIR',
-        BASE_FILE_ROOTS_DIR,
+                     'SOCK_DIR',
-        BASE_PILLAR_ROOTS_DIR,
+                     'SRV_ROOT_DIR',
-        BASE_MASTER_ROOTS_DIR,
+                     'BASE_FILE_ROOTS_DIR',
-        LOGS_DIR,
+                     'BASE_PILLAR_ROOTS_DIR',
-        PIDFILE_DIR,
+                     'BASE_MASTER_ROOTS_DIR',
-    )
+                     'LOGS_DIR',
-except ImportError as error:
+                     'PIDFILE_DIR')
-    log.debug('Error importing salt._syspaths with exception {0}'.format(error))
+        ROOT_DIR = CONFIG_DIR = CACHE_DIR = SOCK_DIR = None
-    # The installation time was not generated, let's define the default values
+        SRV_ROOT_DIR = BASE_FILE_ROOTS_DIR = BASE_PILLAR_ROOTS_DIR = None
-    __platform = sys.platform.lower()
+        BASE_MASTER_ROOTS_DIR = LOGS_DIR = PIDFILE_DIR = None
-    if __platform.startswith('win'):
+
+
+# Let's find out the path of this module
+if 'SETUP_DIRNAME' in globals():
+    # This is from the exec() call in Salt's setup.py
+    __THIS_FILE = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py')  # pylint: disable=E0602
+else:
+    __THIS_FILE = __file__
+
+
+# These values are always relative to salt's installation directory
+INSTALL_DIR = os.path.dirname(os.path.realpath(__THIS_FILE))
+CLOUD_DIR = os.path.join(INSTALL_DIR, 'cloud')
+BOOTSTRAP = os.path.join(CLOUD_DIR, 'deploy', 'bootstrap-salt.sh')
+
+ROOT_DIR = __generated_syspaths.ROOT_DIR
+if ROOT_DIR is None:
+    # The installation time value was not provided, let's define the default
+    if __PLATFORM.startswith('win'):
         ROOT_DIR = r'c:\salt' or '/'
-        CONFIG_DIR = os.path.join(ROOT_DIR, 'conf')
     else:
         ROOT_DIR = '/'
-        if 'freebsd' in __platform:
+
-            CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'local', 'etc', 'salt')
+CONFIG_DIR = __generated_syspaths.CONFIG_DIR
-        elif 'netbsd' in __platform:
+if CONFIG_DIR is None:
-            CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'pkg', 'etc', 'salt')
+    if __PLATFORM.startswith('win'):
-        else:
+        CONFIG_DIR = os.path.join(ROOT_DIR, 'conf')
-            CONFIG_DIR = os.path.join(ROOT_DIR, 'etc', 'salt')
+    elif 'freebsd' in __PLATFORM:
+        CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'local', 'etc', 'salt')
+    elif 'netbsd' in __PLATFORM:
+        CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'pkg', 'etc', 'salt')
+    else:
+        CONFIG_DIR = os.path.join(ROOT_DIR, 'etc', 'salt')
+
+CACHE_DIR = __generated_syspaths.CACHE_DIR
+if CACHE_DIR is None:
     CACHE_DIR = os.path.join(ROOT_DIR, 'var', 'cache', 'salt')
+
+SOCK_DIR = __generated_syspaths.SOCK_DIR
+if SOCK_DIR is None:
     SOCK_DIR = os.path.join(ROOT_DIR, 'var', 'run', 'salt')
+
+SRV_ROOT_DIR = __generated_syspaths.SRV_ROOT_DIR
+if SRV_ROOT_DIR is None:
     SRV_ROOT_DIR = os.path.join(ROOT_DIR, 'srv')
+
+BASE_FILE_ROOTS_DIR = __generated_syspaths.BASE_FILE_ROOTS_DIR
+if BASE_FILE_ROOTS_DIR is None:
     BASE_FILE_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt')
+
+BASE_PILLAR_ROOTS_DIR = __generated_syspaths.BASE_PILLAR_ROOTS_DIR
+if BASE_FILE_ROOTS_DIR is None:
     BASE_PILLAR_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'pillar')
+
+BASE_MASTER_ROOTS_DIR = __generated_syspaths.BASE_MASTER_ROOTS_DIR
+if BASE_MASTER_ROOTS_DIR is None:
     BASE_MASTER_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt-master')
+
+LOGS_DIR = __generated_syspaths.LOGS_DIR
+if LOGS_DIR is None:
     LOGS_DIR = os.path.join(ROOT_DIR, 'var', 'log', 'salt')
+
+PIDFILE_DIR = __generated_syspaths.PIDFILE_DIR
+if PIDFILE_DIR is None:
     PIDFILE_DIR = os.path.join(ROOT_DIR, 'var', 'run')
+
+
+__all__ = [
+    'ROOT_DIR',
+    'CONFIG_DIR',
+    'CACHE_DIR',
+    'SOCK_DIR',
+    'SRV_ROOT_DIR',
+    'BASE_FILE_ROOTS_DIR',
+    'BASE_PILLAR_ROOTS_DIR',
+    'BASE_MASTER_ROOTS_DIR',
+    'LOGS_DIR',
+    'PIDFILE_DIR',
+    'INSTALL_DIR',
+    'CLOUD_DIR',
+    'BOOTSTRAP'
+]

setup.py

@@ -14,6 +14,7 @@ from __future__ import print_function, with_statement
 import os
 import sys
 import glob
+import time
 try:
     from urllib2 import urlopen
 except ImportError:
@@ -102,11 +103,10 @@ except ImportError:
 
 SALT_VERSION = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', 'version.py')
 SALT_VERSION_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_version.py')
-SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt')
+SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), '_requirements.txt')
-SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt')
+SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'zeromq-requirements.txt')
-SALT_CLOUD_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'cloud.txt')
+SALT_CLOUD_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'cloud-requirements.txt')
-SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt')
+SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'raet-requirements.txt')
-SALT_SYSPATHS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', 'syspaths.py')
 
 # Salt SSH Packaging Detection
 PACKAGED_FOR_SALT_SSH_FILE = os.path.join(os.path.abspath(SETUP_DIRNAME), '.salt-ssh-package')
@@ -115,7 +115,6 @@ PACKAGED_FOR_SALT_SSH = os.path.isfile(PACKAGED_FOR_SALT_SSH_FILE)
 
 # pylint: disable=W0122
 exec(compile(open(SALT_VERSION).read(), SALT_VERSION, 'exec'))
-exec(compile(open(SALT_SYSPATHS).read(), SALT_SYSPATHS, 'exec'))
 # pylint: enable=W0122
 
 
@@ -436,33 +435,58 @@ class Install(install):
     def initialize_options(self):
         install.initialize_options(self)
         # pylint: disable=undefined-variable
-        self.salt_root_dir = ROOT_DIR
+        if __saltstack_version__.info >= SaltStackVersion.from_name('Boron'):
-        self.salt_config_dir = CONFIG_DIR
+            # XXX: Remove the Salt Specific Options In Salt Boron. They are now global options
-        self.salt_cache_dir = CACHE_DIR
+            raise DistutilsArgError(
-        self.salt_sock_dir = SOCK_DIR
+                'Developers, please remove the salt paths configuration '
-        self.salt_srv_root_dir = SRV_ROOT_DIR
+                'setting from the setup\'s install command'
-        self.salt_base_file_roots_dir = BASE_FILE_ROOTS_DIR
+            )
-        self.salt_base_pillar_roots_dir = BASE_PILLAR_ROOTS_DIR
-        self.salt_base_master_roots_dir = BASE_MASTER_ROOTS_DIR
-        self.salt_logs_dir = LOGS_DIR
-        self.salt_pidfile_dir = PIDFILE_DIR
         # pylint: enable=undefined-variable
+        self.salt_root_dir = None
+        self.salt_config_dir = None
+        self.salt_cache_dir = None
+        self.salt_sock_dir = None
+        self.salt_srv_root_dir = None
+        self.salt_base_file_roots_dir = None
+        self.salt_base_pillar_roots_dir = None
+        self.salt_base_master_roots_dir = None
+        self.salt_logs_dir = None
+        self.salt_pidfile_dir = None
 
     def finalize_options(self):
         install.finalize_options(self)
 
+        logged_warnings = False
         for optname in ('root_dir', 'config_dir', 'cache_dir', 'sock_dir',
                         'srv_root_dir', 'base_file_roots_dir',
                         'base_pillar_roots_dir', 'base_master_roots_dir',
                         'logs_dir', 'pidfile_dir'):
             optvalue = getattr(self, 'salt_{0}'.format(optname))
-            if not optvalue:
+            if optvalue is not None:
-                raise DistutilsArgError(
+                dist_opt_value = getattr(self.distribution, 'salt_{0}'.format(optname))
-                    'The value of --salt-{0} needs a proper path value'.format(
+                logged_warnings = True
+                log.warn(
+                    'The \'--salt-{0}\' setting is now a global option just pass it '
+                    'right after \'setup.py\'. This install setting will still work '
+                    'until Salt Boron but please migrate to the global setting as '
+                    'soon as possible.'.format(
                         optname.replace('_', '-')
                     )
+
                 )
-            setattr(self.distribution, 'salt_{0}'.format(optname), optvalue)
+                if dist_opt_value is not None:
+                    raise DistutilsArgError(
+                        'The \'--salt-{0}\' setting was passed as a global option '
+                        'and as an option to the install command. Please only pass '
+                        'one of them, preferrably the global option since the other '
+                        'is now deprecated and will be removed in Salt Boron.'.format(
+                            optname.replace('_', '-')
+                        )
+                    )
+                setattr(self.distribution, 'salt_{0}'.format(optname), optvalue)
+
+        if logged_warnings is True:
+            time.sleep(3)
 
     def run(self):
         # Let's set the running_salt_install attribute so we can add
@@ -527,7 +551,28 @@ class SaltDistribution(distutils.dist.Distribution):
     global_options = distutils.dist.Distribution.global_options + [
         ('ssh-packaging', None, 'Run in SSH packaging mode'),
         ('salt-transport=', None, 'The transport to prepare salt for. Choices are \'zeromq\' '
-                                  '\'raet\' or \'both\'. Defaults to \'zeromq\'', 'zeromq')
+                                  '\'raet\' or \'both\'. Defaults to \'zeromq\'', 'zeromq')] + [
+        # Salt's Paths Configuration Settings
+        ('salt-root-dir=', None,
+         'Salt\'s pre-configured root directory'),
+        ('salt-config-dir=', None,
+         'Salt\'s pre-configured configuration directory'),
+        ('salt-cache-dir=', None,
+         'Salt\'s pre-configured cache directory'),
+        ('salt-sock-dir=', None,
+         'Salt\'s pre-configured socket directory'),
+        ('salt-srv-root-dir=', None,
+         'Salt\'s pre-configured service directory'),
+        ('salt-base-file-roots-dir=', None,
+         'Salt\'s pre-configured file roots directory'),
+        ('salt-base-pillar-roots-dir=', None,
+         'Salt\'s pre-configured pillar roots directory'),
+        ('salt-base-master-roots-dir=', None,
+         'Salt\'s pre-configured master roots directory'),
+        ('salt-logs-dir=', None,
+         'Salt\'s pre-configured logs directory'),
+        ('salt-pidfile-dir=', None,
+         'Salt\'s pre-configured pidfiles directory'),
     ]
 
     def __init__(self, attrs=None):
@@ -536,6 +581,19 @@ class SaltDistribution(distutils.dist.Distribution):
         self.ssh_packaging = PACKAGED_FOR_SALT_SSH
         self.salt_transport = None
 
+        # Salt Paths Configuration Settings
+        self.salt_root_dir = None
+        self.salt_config_dir = None
+        self.salt_cache_dir = None
+        self.salt_sock_dir = None
+        self.salt_srv_root_dir = None
+        self.salt_base_file_roots_dir = None
+        self.salt_base_pillar_roots_dir = None
+        self.salt_base_master_roots_dir = None
+        self.salt_logs_dir = None
+        self.salt_pidfile_dir = None
+
+
         self.name = 'salt-ssh' if PACKAGED_FOR_SALT_SSH else 'salt'
         self.version = __version__  # pylint: disable=undefined-variable
         self.description = 'Portable, distributed, remote execution and configuration management system'

tests/integration/__init__.py

@@ -1177,8 +1177,8 @@ class ShellCase(AdaptedConfigurationTestCaseMixIn, ShellTestCase):
         Execute salt-run
         '''
         arg_str = '-c {0}{async_flag} {1}'.format(self.get_config_dir(),
-                                             arg_str,
+                                                  arg_str,
-                                             async_flag=' --async' if async else '')
+                                                  async_flag=' --async' if async else '')
         return self.run_script('salt-run', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr)
 
     def run_run_plus(self, fun, options='', *arg, **kwargs):

tests/unit/modules/brew_test.py

@@ -61,7 +61,8 @@ class BrewTestCase(TestCase):
         mock_cmd = MagicMock(return_value='')
         with patch.dict(brew.__salt__, {'cmd.run_all': mock_failure,
                                         'file.get_user': mock_user,
-                                        'cmd.run': mock_cmd}):
+                                        'cmd.run': mock_cmd,
+                                        'cmd.retcode': mock_failure}):
             self.assertFalse(brew._tap('homebrew/test'))
 
     @patch('salt.modules.brew._list_taps', MagicMock(return_value=TAPS_LIST))
@@ -69,12 +70,7 @@ class BrewTestCase(TestCase):
         '''
         Tests adding unofficial Github repos to the list of brew taps
         '''
-        mock_success = MagicMock(return_value={'retcode': 0})
+        with patch.dict(brew.__salt__, {'cmd.retcode': MagicMock(return_value=0)}):
-        mock_user = MagicMock(return_value='foo')
-        mock_cmd = MagicMock(return_value='')
-        with patch.dict(brew.__salt__, {'cmd.run_all': mock_success,
-                                        'file.get_user': mock_user,
-                                        'cmd.run': mock_cmd}):
             self.assertTrue(brew._tap('homebrew/test'))
 
     # '_homebrew_bin' function tests: 1