Merge remote-tracking branch 'upstream/develop' into feature_sqlite3_returner

doc/topics/tutorials/cloud_controller.rst

@@ -77,7 +77,7 @@ of libvirt certificates. These certificates allow for virtual machine
 migration. Salt comes with a system used to auto deploy these certificates.
 Salt manages the signing authority key and generates keys for libvirt clients
 on the master, signs them with the certificate authority and uses pillar to
-distrbute them. This is managed via the ``libvirt`` state. Simply execute this
+distribute them. This is managed via the ``libvirt`` state. Simply execute this
 formula on the minion to ensure that the certificate is in place and up to
 date:
 

salt/client/ssh/__init__.py

@@ -616,11 +616,11 @@ def prep_trans_tar(opts, chunks, file_refs):
                     break
                 files = file_client.cache_dir(name, env, True)
                 if files:
-                    for file in files:
+                    for filename in files:
                         tgt = os.path.join(
                                 env_root,
                                 short,
-                                file[file.find(short) + len(short):],
+                                filename[filename.find(short) + len(short):],
                                 )
                         tgt_dir = os.path.dirname(tgt)
                         if not os.path.isdir(tgt_dir):

salt/client/ssh/state.py

@@ -128,11 +128,11 @@ def prep_trans_tar(opts, chunks, file_refs):
                     break
                 files = file_client.cache_dir(name, env, True)
                 if files:
-                    for file in files:
+                    for filename in files:
                         tgt = os.path.join(
                                 env_root,
                                 short,
-                                file[file.find(short) + len(short):],
+                                filename[filename.find(short) + len(short):],
                                 )
                         tgt_dir = os.path.dirname(tgt)
                         if not os.path.isdir(tgt_dir):

salt/loader.py

@@ -190,7 +190,7 @@ def fileserver(opts, backends):
 
 def roster(opts, whitelist=None):
     '''
-    Returns the file server modules
+    Returns the roster modules
     '''
     load = _create_loader(opts, 'roster', 'roster')
     ret = load.gen_functions(whitelist=whitelist)

salt/modules/alternatives.py

@@ -23,6 +23,11 @@ __outputter__ = {
 
 log = logging.getLogger(__name__)
 
+# Don't shadow built-in's.
+__func_alias__ = {
+    'set_': 'set'
+}
+
 
 def __virtual__():
     '''
@@ -151,7 +156,7 @@ def auto(name):
     return out['stdout']
 
 
-def set(name, path):
+def set_(name, path):
     '''
     Manually set the alternative <path> for <name>.
 

salt/modules/cmdmod.py

@@ -6,15 +6,15 @@ access to the master root execution access to all salt minions
 '''
 
 # Import python libs
+import functools
+import json
 import logging
 import os
 import shutil
 import subprocess
-import functools
 import sys
-import json
-import yaml
 import traceback
+import yaml
 
 # Import salt libs
 import salt.utils
@@ -174,8 +174,9 @@ def _run(cmd,
     '''
     Do the DRY thing and only call subprocess.Popen() once
     '''
-    # Set the default working directory to the home directory
+    # Set the default working directory to the home directory of the user
-    # of the user salt-minion is running as.  Default:  /root
+    # salt-minion is running as. Defaults to home directory of user under which
+    # the minion is running.
     if not cwd:
         cwd = os.path.expanduser('~{0}'.format('' if not runas else runas))
 
@@ -187,6 +188,10 @@ def _run(cmd,
             cwd = '/'
             if salt.utils.is_windows():
                 cwd = os.tempnam()[:3]
+    else:
+        # Handle edge cases where numeric/other input is entered, and would be
+        # yaml-ified into non-string types
+        cwd = str(cwd)
 
     if not salt.utils.is_windows():
         if not os.path.isfile(shell) or not os.access(shell, os.X_OK):
@@ -296,7 +301,7 @@ def _run(cmd,
               'stdin': str(stdin) if stdin is not None else stdin,
               'stdout': stdout,
               'stderr': stderr,
-              'with_communicate' : with_communicate}
+              'with_communicate': with_communicate}
 
     if umask:
         try:
@@ -321,8 +326,18 @@ def _run(cmd,
         kwargs['executable'] = shell
         kwargs['close_fds'] = True
 
+    if not os.path.isabs(cwd) or not os.path.isdir(cwd):
+        raise CommandExecutionError(
+            'Specified cwd {0!r} either not absolute or does not exist'
+            .format(cwd)
+        )
+
     # This is where the magic happens
-    proc = salt.utils.timed_subprocess.TimedProc(cmd, **kwargs)
+    try:
+        proc = salt.utils.timed_subprocess.TimedProc(cmd, **kwargs)
+    except (OSError, IOError) as exc:
+        raise CommandExecutionError('Unable to run command: {0}'.format(exc))
+
     try:
         proc.wait(timeout)
     except salt.exceptions.TimedProcTimeoutError, e:

salt/modules/lxc.py

@@ -15,6 +15,11 @@ import salt.utils
 # Set up logging
 log = logging.getLogger(__name__)
 
+# Don't shadow built-in's.
+__func_alias__ = {
+    'list_': 'list'
+}
+
 
 def __virtual__():
     if not salt.utils.which('lxc'):
@@ -217,7 +222,7 @@ def create(name, config=None, profile=None, options=None, **kwargs):
         return {'created': False, 'error': 'container could not be created'}
 
 
-def list():
+def list_():
     '''
     List defined containers (running, stopped, and frozen).
 

salt/modules/modjk.py

@@ -203,7 +203,7 @@ def list_running_members(lbn, profile='default'):
         salt '*' modjk.list_running_members loadbalancer1 other-profile
     '''
 
-    config = get_running()
+    config = get_running(profile)
     try:
         return config['worker.{0}.balance_workers'.format(lbn)].split(',')
     except KeyError:

salt/modules/pip.py

@@ -727,7 +727,9 @@ def list_(prefix=None,
     '''
     packages = {}
 
-    cmd = [_get_pip_bin(bin_env), 'freeze']
+    pip_bin = _get_pip_bin(bin_env)
+    pip_version_cmd = [pip_bin, '--version']
+    cmd = [pip_bin, 'freeze']
 
     if runas is not None:
         # The user is using a deprecated argument, warn!
@@ -751,6 +753,14 @@ def list_(prefix=None,
     cmd_kwargs = dict(runas=user, cwd=cwd)
     if bin_env and os.path.isdir(bin_env):
         cmd_kwargs['env'] = {'VIRTUAL_ENV': bin_env}
+
+    if not prefix or prefix in ('p', 'pi', 'pip'):
+        pip_version_result = __salt__['cmd.run_all'](' '.join(pip_version_cmd),
+                                                     **cmd_kwargs)
+        if pip_version_result['retcode'] > 0:
+            raise CommandExecutionError(pip_version_result['stderr'])
+        packages['pip'] = pip_version_result['stdout'].split()[1]
+
     result = __salt__['cmd.run_all'](' '.join(cmd), **cmd_kwargs)
     if result['retcode'] > 0:
         raise CommandExecutionError(result['stderr'])

salt/modules/seed.py

@@ -19,6 +19,11 @@ import salt.config
 # Set up logging
 log = logging.getLogger(__name__)
 
+# Don't shadow built-in's.
+__func_alias__ = {
+    'apply_': 'apply'
+}
+
 
 def _mount(path, ftype):
     mpt = None
@@ -44,7 +49,7 @@ def _umount(mpt, ftype):
         __salt__['img.umount_image'](mpt)
 
 
-def apply(path, id_=None, config=None, approve_key=True, install=True):
+def apply_(path, id_=None, config=None, approve_key=True, install=True):
     '''
     Seed a location (disk image, directory, or block device) with the
     minion config, approve the minion's key, and/or install salt-minion.

salt/modules/upstart.py

@@ -293,7 +293,7 @@ def restart(name, **kwargs):
     return not __salt__['cmd.retcode'](cmd)
 
 
-def full_restart(name, **kwargs):
+def full_restart(name):
     '''
     Do a full restart (stop/start) of the named service
 

salt/pillar/git_pillar.py

@@ -12,7 +12,6 @@ ext_pillar:
 from copy import deepcopy
 import logging
 import os
-import time
 
 # Import third party libs
 HAS_GIT = False
@@ -23,7 +22,6 @@ except ImportError:
     pass
 
 # Import salt libs
-import salt.utils
 from salt.pillar import Pillar
 
 # Set up logging

salt/renderers/py.py

@@ -3,6 +3,51 @@ Pure python state renderer
 
 The sls file should contain a function called ``run`` which returns high state
 data
+
+In this module, a few objects are defined for you, including the usual
+(with``__`` added) ``__salt__`` dictionary, ``__grains__``,
+``__pillar__``, ``__opts__``, ``__env__``, and ``__sls__``.
+
+.. code-block:: python
+   :linenos:
+
+    #!py
+
+    def run():
+        config = {}
+
+        if __grains__['os'] == 'Ubuntu':
+            user = 'ubuntu'
+            group = 'ubuntu'
+            home = '/home/{0}'.format(user)
+        else:
+            user = 'root'
+            group = 'root'
+            home = '/root/'
+
+        config['s3cmd'] = {
+            'pkg': [
+                'installed',
+                {'name': 's3cmd'},
+                ],
+            }
+
+        config[home + '/.s3cfg'} = {
+            'file.managed': [{
+                'source': 'salt://s3cfg/templates/s3cfg',
+                'template': 'jinja',
+                'user': user,
+                'group': group,
+                'mode': 600,
+                'context': {
+                    'aws_key': __pillar__['AWS_ACCESS_KEY_ID'],
+                    'aws_secret_key': __pillar__['AWS_SECRET_ACCESS_KEY'],
+                    },
+                }],
+            }
+
+        return config
+
 '''
 
 # Import python libs
@@ -26,12 +71,12 @@ def render(template, env='', sls='', tmplpath=None, **kws):
     tmp_data = salt.utils.templates.py(
             template,
             True,
-            salt=__salt__,
+            __salt__=__salt__,
-            grains=__grains__,
+            __grains__=__grains__,
-            opts=__opts__,
+            __opts__=__opts__,
-            pillar=__pillar__,
+            __pillar__=__pillar__,
-            env=env,
+            __env__=env,
-            sls=sls,
+            __sls__=sls,
             **kws)
     if not tmp_data.get('result', False):
         raise SaltRenderError(tmp_data.get('data',

salt/states/keystone.py

@@ -14,12 +14,6 @@ Management of Keystone users.
 
 '''
 
-# Import python libs
-import sys
-
-# Import salt libs
-import salt.utils
-
 
 def __virtual__():
     '''

salt/states/svn.py

@@ -38,7 +38,8 @@ def latest(name,
            username=None,
            password=None,
            force=False,
-           externals=True):
+           externals=True,
+           trust=False):
     '''
     Checkout or update the working directory to the latest revision from the
     remote repository.
@@ -69,6 +70,9 @@ def latest(name,
 
     externals : True
         Change to False to not checkout or update externals
+
+    trust : False
+        Automatically trust the remote server. SVN's --trust-server-cert
     '''
     ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}
     if not target:
@@ -109,6 +113,9 @@ def latest(name,
     if externals is False:
         opts += ('--ignore-externals',)
 
+    if trust:
+        opts += ('--trust-server-cert',)
+
     if svn_cmd == 'svn.update':
         out = __salt__[svn_cmd](cwd, basename, user, username, password, *opts)
 
@@ -121,6 +128,7 @@ def latest(name,
                                        fmt='dict')[0]['Revision']
         if current_rev != new_rev:
             ret['changes']['revision'] = "{0} => {1}".format(current_rev, new_rev)
+    
     else:
         out = __salt__[svn_cmd](cwd, name, basename, user, username, password, *opts)
 

salt/utils/__init__.py

@@ -1341,7 +1341,7 @@ def warn_until(version_info,
     :param category: The warning class to be thrown, by default
                      ``DeprecationWarning``
     :param stacklevel: There should be no need to set the value of
-                       ``stacklevel`` salt should be able to do the right thing
+                       ``stacklevel``. Salt should be able to do the right thing.
     :param _version_info_: In order to reuse this function for other SaltStack
                            projects, they need to be able to provide the
                            version info to compare to.
@@ -1380,6 +1380,65 @@ def warn_until(version_info,
         warnings.warn(message, category, stacklevel=stacklevel)
 
 
+def kwargs_warn_until(kwargs,
+               version_info,
+               category=DeprecationWarning,
+               stacklevel=None,
+               _version_info_=None,
+               _dont_call_warnings=False):
+    '''
+    Helper function to raise a warning (by default, a ``DeprecationWarning``)
+    when unhandled keyword arguments are passed to function, until the
+    provided ``version_info``, after which, a ``RuntimeError`` will be raised
+    to remind the developers to remove the ``**kwargs`` because the target
+    version has been reached.
+    This function is used to help deprecate unused legacy ``**kwargs`` that
+    were added to function parameters lists to preserve backwards compatibility
+    when removing a parameter. See
+    :doc:`the deprecation development docs </topics/development/deprecations>`
+    for the modern strategy for deprecating a function parameter.
+
+    :param kwargs: The caller's ``**kwargs`` argument value (a ``dict``).
+    :param version_info: The version info after which the warning becomes a
+                         ``RuntimeError``. For example ``(0, 17)``.
+    :param category: The warning class to be thrown, by default
+                     ``DeprecationWarning``
+    :param stacklevel: There should be no need to set the value of
+                       ``stacklevel``. Salt should be able to do the right thing.
+    :param _version_info_: In order to reuse this function for other SaltStack
+                           projects, they need to be able to provide the
+                           version info to compare to.
+    :param _dont_call_warnings: This parameter is used just to get the
+                                functionality until the actual error is to be
+                                issued. When we're only after the salt version
+                                checks to raise a ``RuntimeError``.
+    '''
+    if not isinstance(version_info, tuple):
+        raise RuntimeError(
+            'The \'version_info\' argument should be passed as a tuple.'
+        )
+
+    if stacklevel is None:
+        # Attribute the warning to the calling function,
+        # not to kwargs_warn_until() or warn_until()
+        stacklevel = 3
+
+    if _version_info_ is None:
+        _version_info_ = salt.version.__version_info__
+
+    if kwargs or _version_info_ >= version_info:
+        removal_version = '.'.join(str(component) for component in version_info)
+        arg_names = ', '.join('\'{0}\''.format(key) for key in kwargs)
+        warn_until(version_info,
+           message='The following parameter(s) have been deprecated and '
+           'will be removed in {0}: {1}.'.format(removal_version, arg_names),
+           category=category,
+           stacklevel=stacklevel,
+           _version_info_=_version_info_,
+           _dont_call_warnings=_dont_call_warnings
+        )
+
+
 def version_cmp(pkg1, pkg2):
     '''
     Compares two version strings using distutils.version.LooseVersion. This is

salt/utils/network.py

@@ -162,7 +162,7 @@ def _interfaces_ip(out):
         for line in group.splitlines():
             if not ' ' in line:
                 continue
-            match = re.match(r'^\d*:\s+([\w.]+)(?:@)?(\w+)?:\s+<(.+)>', line)
+            match = re.match(r'^\d*:\s+([\w.]+)(?:@)?([\w.]+)?:\s+<(.+)>', line)
             if match:
                 iface, parent, attrs = match.groups()
                 if 'UP' in attrs.split(','):

salt/utils/parsers.py

@@ -1422,6 +1422,13 @@ class SaltCallOptionParser(OptionParser, ConfigDirMixIn, MergeConfigMixIn,
             help=('Exit with the salt call retcode and not the salt binary '
                   'retcode')
         )
+        self.add_option(
+            '--id',
+            default='',
+            dest='id',
+            help=('Specify the minion id to use. If this option is omitted, '
+                  'the id option from the minion config will be used.')
+        )
 
     def _mixin_after_parsed(self):
         if not self.args and not self.options.grains_run \

tests/integration/__init__.py

@@ -135,6 +135,7 @@ class TestDaemon(object):
         self.sub_minion_opts = salt.config.minion_config(
             os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf', 'sub_minion')
         )
+        self.sub_minion_opts['root_dir'] = os.path.join(TMP, 'subsalt')
         self.sub_minion_opts['user'] = running_tests_user
         #if sys.version_info < (2, 7):
         #    self.sub_minion_opts['multiprocessing'] = False

tests/integration/modules/cmdmod.py

@@ -200,6 +200,22 @@ sys.stdout.write('cheese')
             'hello' == self.run_function(
                 'cmd.run', ['sleep 1 && echo hello', 'timeout=2']))
 
+    def test_run_cwd_doesnt_exist_issue_7154(self):
+        '''
+        cmd.run should fail and raise
+        salt.exceptions.CommandExecutionError if the cwd dir does not
+        exist
+        '''
+        from salt.exceptions import CommandExecutionError
+        import salt.modules.cmdmod as cmdmod
+        cmd = 'echo OHAI'
+        cwd = '/path/to/nowhere'
+        try:
+            cmdmod.run_all(cmd, cwd=cwd)
+        except CommandExecutionError:
+            pass
+        else:
+            raise RuntimeError
 
 if __name__ == '__main__':
     from integration import run_tests

tests/integration/states/pip.py

@@ -318,6 +318,68 @@ class PipStateTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
             os.unlink(req_filename)
         # <---- Using user ---------------------------------------------------
 
+    def test_issue_6833_pip_upgrade_pip(self):
+        # Create the testing virtualenv
+        venv_dir = os.path.join(
+            integration.TMP, '6833-pip-upgrade-pip'
+        )
+        ret = self.run_function('virtualenv.create', [venv_dir])
+        try:
+            try:
+                self.assertEqual(ret['retcode'], 0)
+                self.assertIn(
+                    'New python executable',
+                    ret['stdout']
+                )
+            except AssertionError:
+                import pprint
+                pprint.pprint(ret)
+                raise
+
+            # Let's install a fixed version pip over whatever pip was
+            # previously installed
+            ret = self.run_function(
+                'pip.install', ['pip==1.3.1'], upgrade=True,
+                ignore_installed=True,
+                bin_env=venv_dir
+            )
+            try:
+                self.assertEqual(ret['retcode'], 0)
+                self.assertIn(
+                    'Successfully installed pip',
+                    ret['stdout']
+                )
+            except AssertionError:
+                import pprint
+                pprint.pprint(ret)
+                raise
+
+            # Le't make sure we have pip 1.3.1 installed
+            self.assertEqual(
+                self.run_function('pip.list', ['pip'], bin_env=venv_dir),
+                {'pip': '1.3.1'}
+            )
+
+            # Now the actual pip upgrade pip test
+            ret = self.run_state(
+                'pip.installed', name='pip==1.4.1', upgrade=True,
+                bin_env=venv_dir
+            )
+            try:
+                self.assertSaltTrueReturn(ret)
+                self.assertInSaltReturn(
+                    'Installed',
+                    ret,
+                    ['changes', 'pip==1.4.1']
+                )
+            except AssertionError:
+                import pprint
+                pprint.pprint(ret)
+                raise
+        finally:
+            if os.path.isdir(venv_dir):
+                shutil.rmtree(venv_dir)
+
 
 if __name__ == '__main__':
     from integration import run_tests

tests/unit/modules/pip_test.py

@@ -783,14 +783,14 @@ class PipTestCase(TestCase):
             'pycrypto==2.6'
         ]
         mock = MagicMock(
-            return_value={
+            side_effect=[
-                'retcode': 0,
+                {'retcode': 0, 'stdout': 'pip MOCKED_VERSION'},
-                'stdout': '\n'.join(eggs)
+                {'retcode': 0, 'stdout': '\n'.join(eggs)}
-            }
+            ]
         )
         with patch.dict(pip.__salt__, {'cmd.run_all': mock}):
             ret = pip.list_()
-            mock.assert_called_once_with(
+            mock.assert_called_with(
                 'pip freeze',
                 runas=None,
                 cwd=None
@@ -801,6 +801,7 @@ class PipTestCase(TestCase):
                     'M2Crypto': '0.21.1',
                     'bbfreeze-loader': '1.1.0',
                     'bbfreeze': '1.1.0',
+                    'pip': 'MOCKED_VERSION',
                     'pycrypto': '2.6'
                 }
             )
@@ -829,7 +830,7 @@ class PipTestCase(TestCase):
         )
         with patch.dict(pip.__salt__, {'cmd.run_all': mock}):
             ret = pip.list_(prefix='bb')
-            mock.assert_called_once_with(
+            mock.assert_called_with(
                 'pip freeze',
                 runas=None,
                 cwd=None
@@ -923,7 +924,7 @@ class PipTestCase(TestCase):
         warnings.resetwarnings()
         warnings.filterwarnings('always', '', DeprecationWarning, __name__)
 
-        mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
+        mock = MagicMock(return_value={'retcode': 0, 'stdout': 'pip VERSION'})
         with patch.dict(pip.__salt__, {'cmd.run_all': mock}):
             with warnings.catch_warnings(record=True) as w:
                 pip.list_('blah', runas='me!')

tests/unit/utils/warnings_test.py

@@ -3,7 +3,7 @@
     tests.unit.utils.warnings_test
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    Test ``salt.utils.warn_until``
+    Test ``salt.utils.warn_until`` and ``salt.utils.kwargs_warn_until``
 
     :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
     :copyright: © 2013 by the SaltStack Team, see AUTHORS for more details.
@@ -20,13 +20,13 @@ from salttesting.mock import NO_MOCK, NO_MOCK_REASON, patch
 ensure_in_syspath('../../')
 
 # Import salt libs
-from salt.utils import warn_until
+from salt.utils import warn_until, kwargs_warn_until
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
 class WarnUntilTestCase(TestCase):
 
     @patch('salt.version')
-    def test_warning_raised(self, salt_version_mock):
+    def test_warn_until_warning_raised(self, salt_version_mock):
         # We *always* want *all* warnings thrown on this module
         warnings.filterwarnings('always', '', DeprecationWarning, __name__)
 
@@ -75,6 +75,54 @@ class WarnUntilTestCase(TestCase):
                 (0, 17), 'Foo', _dont_call_warnings=True
             )
 
+    @patch('salt.version')
+    def test_kwargs_warn_until_warning_raised(self, salt_version_mock):
+        # We *always* want *all* warnings thrown on this module
+        warnings.filterwarnings('always', '', DeprecationWarning, __name__)
+
+        # Define a salt version info
+        salt_version_mock.__version_info__ = (0, 16)
+
+        def raise_warning(**kwargs):
+            kwargs_warn_until(
+                kwargs,
+                (0, 17),
+            )
+
+        # raise_warning({...}) should show warning until version info is >= (0, 17)
+        with warnings.catch_warnings(record=True) as recorded_warnings:
+            raise_warning(foo=42) # with a kwarg
+            self.assertEqual(
+                'The following parameter(s) have been deprecated and '
+                'will be removed in 0.17: \'foo\'.',
+                str(recorded_warnings[0].message)
+            )
+        # With no **kwargs, should not show warning until version info is >= (0, 17)
+        with warnings.catch_warnings(record=True) as recorded_warnings:
+            kwargs_warn_until(
+                {},  # no kwargs
+                (0, 17),
+            )
+            self.assertEqual(0, len(recorded_warnings))
+
+        # Let's set version info to (0, 17), a RuntimeError should be raised
+        # regardless of whether or not we pass any **kwargs.
+        salt_version_mock.__version_info__ = (0, 17)
+        with self.assertRaisesRegexp(
+                RuntimeError,
+                r'The warning triggered on filename \'(.*)warnings_test.py\', '
+                r'line number ([\d]+), is supposed to be shown until version '
+                r'\'0.17\' is released. Current version is now \'0.17\'. Please '
+                r'remove the warning.'):
+            raise_warning()  # no kwargs
+        with self.assertRaisesRegexp(
+                RuntimeError,
+                r'The warning triggered on filename \'(.*)warnings_test.py\', '
+                r'line number ([\d]+), is supposed to be shown until version '
+                r'\'0.17\' is released. Current version is now \'0.17\'. Please '
+                r'remove the warning.'):
+            raise_warning(bar='baz', qux='quux')  # some kwargs
+
 
 if __name__ == '__main__':
     from integration import run_tests