Merge branch 'develop' of github.com:saltstack/salt into develop

salt/cli/__init__.py

@@ -166,7 +166,7 @@ class SaltKey(parsers.SaltKeyOptionParser):
                 os.path.join(self.config['pki_dir'], 'minions'),
                 os.path.join(self.config['pki_dir'], 'minions_pre'),
                 os.path.join(self.config['pki_dir'], 'minions_rejected'),
-                os.path.dirname(self.config['log_file']),
+                os.path.dirname(self.config['key_logfile']),
             ],
             self.config['user'],
             permissive=self.config['permissive_pki_access']

salt/cli/key.py

@@ -87,7 +87,7 @@ class Key(object):
         if hasattr(log, level):
             log_msg = getattr(log, level)
             log_msg(message)
-        if not self.opts['quiet']:
+        if not self.opts.get('quiet', False):
             print(message)
 
     def _list_pre(self, header=True, printer=None):

salt/config.py

@@ -139,7 +139,7 @@ def prepend_root_dir(opts, path_options):
     root_dir = os.path.abspath(opts['root_dir'])
     for path_option in path_options:
         if path_option in opts:
-            opts[path_option] = os.path.join(root_dir, opts[path_option])
+            opts[path_option] = salt.utils.path_join(root_dir, opts[path_option])
 
 
 def minion_config(path):

salt/fileclient.py

@@ -21,6 +21,7 @@ import salt.crypt
 import salt.loader
 import salt.utils
 import salt.payload
+import salt.utils
 import salt.utils.templates
 from salt._compat import (
     URLError, HTTPError, BaseHTTPServer, urlparse, url_open)
@@ -156,7 +157,7 @@ class Client(object):
                 prefix = separated[0]
             for fn_ in self.file_list_emptydirs(env):
                 if fn_.startswith(path):
-                    dest = os.path.join(
+                    dest = salt.utils.path_join(
                         self.opts['cachedir'],
                         'files',
                         env
@@ -296,7 +297,7 @@ class Client(object):
                 else:
                     return ''
         else:
-            dest = os.path.join(
+            dest = salt.utils.path_join(
                 self.opts['cachedir'],
                 'extrn_files',
                 env,
@@ -354,7 +355,7 @@ class Client(object):
             return ''
         if not dest:
             # No destination passed, set the dest as an extrn_files cache
-            dest = os.path.join(
+            dest = salt.utils.path_join(
                 self.opts['cachedir'],
                 'extrn_files',
                 env,

salt/minion.py

@@ -275,10 +275,12 @@ class Minion(object):
 
         function_name = data['fun']
         if function_name in self.functions:
+            ret['success'] = False
             try:
                 func = self.functions[data['fun']]
                 args, kw = detect_kwargs(func, data['arg'], data)
                 ret['return'] = func(*args, **kw)
+                ret['success'] = True
             except CommandNotFoundError as exc:
                 msg = 'Command required for \'{0}\' not found: {1}'
                 log.debug(msg.format(function_name, str(exc)))
@@ -334,10 +336,12 @@ class Minion(object):
                 except Exception:
                     pass
 
+            ret['success'][data['fun'][ind]] = False
             try:
                 func = self.functions[data['fun'][ind]]
                 args, kw = detect_kwargs(func, data['arg'][ind], data)
                 ret['return'][data['fun'][ind]] = func(*args, **kw)
+                ret['success'][data['fun'][ind]] = True
             except Exception as exc:
                 trb = traceback.format_exc()
                 log.warning(

salt/modules/mongodb.py

@@ -0,0 +1,166 @@
+'''
+Module to provide MongoDB functionality to Salt
+
+This module uses PyMongo, and accepts configuration details as parameters
+as well as configuration settings:
+
+    mongodb.host: 'localhost'
+    mongodb.port: '27017'
+    mongodb.user: ''
+    mongodb.password: ''
+
+This data can also be passed into pillar. Options passed into opts will
+overwrite options passed into pillar.
+'''
+
+import logging
+
+# Import third party libs
+try:
+    import pymongo
+    has_mongodb = True
+except ImportError:
+    has_mongodb = False
+
+log = logging.getLogger(__name__)
+__opts__ = {}
+
+def __virtual__():
+    '''
+    Only load this module if pymongo is installed
+    '''
+    if has_mongodb:
+        return 'mongodb'
+    else:
+        return False
+
+def _connect(user=None, password=None, host=None, port=None, database="admin"):
+    '''
+    Returns a tuple of (user, host, port) with config, pillar, or default
+    values assigned to missing values.
+    '''
+    if not user:
+        user = __opts__.get('mongodb.user') or __pillar__.get('mongodb.user')
+    if not password:
+        password = __opts__.get('mongodb.password') or __pillar__.get('mongodb.password')
+    if not host:
+        host = __opts__.get('mongodb.host') or __pillar__.get('mongodb.host')
+    if not port:
+        port = __opts__.get('mongodb.port') or __pillar__.get('mongodb.port')
+
+    try:
+        conn = pymongo.connection.Connection(host=host, port=port)
+        db = pymongo.database.Database(conn, database)
+        if user and password:
+            db.authenticate(user, password)
+    except pymongo.errors.PyMongoError as e:
+        log.error("Error connecting to database {0}".format(database.message))
+        return False
+
+    return conn
+
+def db_list(user=None, password=None, host=None, port=None):
+    '''
+    List all Mongodb databases
+    '''
+    conn = _connect(user, password, host, port)
+
+    try:
+        log.info("Listing databases")
+        return conn.database_names()
+    except pymongo.errors.PyMongoError as e:
+        log.error(e)
+        return e.message
+
+def db_exists(name, user=None, password=None, host=None, port=None, database="admin"):
+    '''
+    Checks if a database exists in Mongodb
+    '''
+    dbs = db_list(user, password, host, port)
+    for db in dbs:
+        if name == db:
+            return True
+
+    return False
+
+def db_remove(name, user=None, password=None, host=None, port=None):
+    '''
+    Remove a Mongodb database
+    '''
+    conn = _connect(user, password, host, port)
+
+    try:
+        log.info("Removing database {0}".format(name))
+        conn.drop_database(name)
+    except pymongo.errors.PyMongoError as e:
+        log.error("Removing database {0} failed with error: {1}".format(
+            name, e.message))
+        return e.message
+
+    return True
+
+def user_list(user=None, password=None, host=None, port=None, database="admin"):
+    '''
+    List users of a Mongodb database
+    '''
+    conn = _connect(user, password, host, port)
+
+    try:
+        log.info("Listing users")
+        db = pymongo.database.Database(conn, database)
+
+        output = []
+
+        for user in db.system.users.find():
+            output.append([("user", user['user']), ("readOnly", user['readOnly'])])
+
+        return output
+
+    except pymongo.errors.PyMongoError as e:
+        log.error("Listing users failed with error: {0}".format(e.message))
+        return e.message
+
+def user_exists(name, user=None, password=None, host=None, port=None, database="admin"):
+    '''
+    Checks if a user exists in Mongodb
+    '''
+    users = user_list(user, password, host, port, database)
+    for user in users:
+        if name == dict(user).get('user'):
+            return True
+
+    return False
+
+def user_create(name, passwd, user=None, password=None, host=None, port=None, database="admin"):
+    '''
+    Create a Mongodb user
+    '''
+    conn = _connect(user, password, host, port)
+
+    try:
+        log.info("Creating user {0}".format(name))
+        db = pymongo.database.Database(conn, database)
+        db.add_user(name, passwd)
+    except pymongo.errors.PyMongoError as e:
+        log.error("Creating database {0} failed with error: {1}".format(
+            name, e.message))
+        return e.message
+
+    return True
+
+def user_remove(name, user=None, password=None, host=None, port=None, database="admin"):
+    '''
+    Remove a Mongodb user
+    '''
+    conn = _connect(user, password, host, port)
+
+    try:
+        log.info("Removing user {0}".format(name))
+        db = pymongo.database.Database(conn, database)
+        db.remove_user(name)
+    except pymongo.errors.PyMongoError as e:
+        log.error("Creating database {0} failed with error: {1}".format(
+            name, e.message))
+        return e.message
+
+    return True

salt/modules/mysql.py

@@ -673,7 +673,8 @@ def grant_exists(grant,
     # perhaps should be replaced/reworked with a better/cleaner solution.
     target = __grant_generate(grant, database, user, host, grant_option, escape)
 
-    if target in user_grants(user, host):
+    grants = user_grants(user, host)
+    if grants is not False and target in grants:
         log.debug("Grant exists.")
         return True
 

salt/runner.py

@@ -43,7 +43,7 @@ class Runner(object):
         '''
         Execute the runner sequence
         '''
-        if self.opts['doc']:
+        if self.opts.get('doc', False):
             self._print_docs()
         else:
             self._verify_fun()

salt/states/mongodb_database.py

@@ -0,0 +1,51 @@
+'''
+Management of Mongodb databases
+
+Only deletion is supported, creation doesnt make sense
+and can be done using mongodb_user.present
+'''
+def absent(name,
+           user=None,
+           password=None,
+           host=None,
+           port=None):
+    '''
+    Ensure that the named database is absent
+
+    name
+        The name of the database to remove
+
+    user
+        The user to connect as (must be able to create the user)
+
+    password
+        The password of the user
+
+    host
+        The host to connect to
+
+    port
+        The port to connect to
+
+    '''
+    ret = {'name': name,
+           'changes': {},
+           'result': True,
+           'comment': ''}
+
+    #check if database exists and remove it
+    if __salt__['mongodb.db_exists'](name, user, password, host, port):
+        if __opts__['test']:
+            ret['result'] = None
+            ret['comment'] = ('Database {0} is present and needs to be removed'
+                    ).format(name)
+            return ret
+        if __salt__['mongodb.db_remove'](name, user, password, host, port):
+            ret['comment'] = 'Database {0} has been removed'.format(name)
+            ret['changes'][name] = 'Absent'
+            return ret
+
+    # fallback
+    ret['comment'] = ('User {0} is not present, so it cannot be removed'
+            ).format(name)
+    return ret

salt/states/mongodb_user.py

@@ -0,0 +1,110 @@
+'''
+Management of Mongodb users
+===========================
+'''
+
+def present(name,
+            passwd,
+            database="admin",
+            user=None,
+            password=None,
+            host=None,
+            port=None):
+    '''
+    Ensure that the user is present with the specified properties
+
+    name
+        The name of the user to manage
+
+    passwd
+        The password of the user
+
+    user
+        The user to connect as (must be able to create the user)
+
+    password
+        The password of the user
+
+    host
+        The host to connect to
+
+    port
+        The port to connect to
+
+    database
+        The database to create the user in (if the db doesn't exist, it will be created)
+
+    '''
+    ret = {'name': name,
+           'changes': {},
+           'result': True,
+           'comment': 'User {0} is already present'.format(name)}
+    # check if user exists
+    if __salt__['mongodb.user_exists'](name, user, password, host, port):
+        return ret
+
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = ('User {0} is not present and needs to be created'
+                ).format(name)
+        return ret
+    # The user is not present, make it!
+    if __salt__['mongodb.user_create'](name, passwd, user, password, host, port, database=database):
+        ret['comment'] = 'User {0} has been created'.format(name)
+        ret['changes'][name] = 'Present'
+    else:
+        ret['comment'] = 'Failed to create database {0}'.format(name)
+        ret['result'] = False
+
+    return ret
+
+
+def absent(name,
+           user=None,
+           password=None,
+           host=None,
+           port=None,
+           database="admin"):
+    '''
+    Ensure that the named user is absent
+
+    name
+        The name of the user to remove
+
+    user
+        The user to connect as (must be able to create the user)
+
+    password
+        The password of the user
+
+    host
+        The host to connect to
+
+    port
+        The port to connect to
+
+    database
+        The database to create the user in (if the db doesn't exist, it will be created)
+
+    '''
+    ret = {'name': name,
+           'changes': {},
+           'result': True,
+           'comment': ''}
+
+    #check if user exists and remove it
+    if __salt__['mongodb.user_exists'](name, user, password, host, port, database=database):
+        if __opts__['test']:
+            ret['result'] = None
+            ret['comment'] = ('User {0} is present and needs to be removed'
+                    ).format(name)
+            return ret
+        if __salt__['mongodb.user_remove'](name, user, password, host, port, database=database):
+            ret['comment'] = 'User {0} has been removed'.format(name)
+            ret['changes'][name] = 'Absent'
+            return ret
+
+    # fallback
+    ret['comment'] = ('User {0} is not present, so it cannot be removed'
+            ).format(name)
+    return ret

salt/utils/__init__.py

@@ -15,6 +15,7 @@ import datetime
 import tempfile
 import shutil
 import time
+import platform
 from calendar import month_abbr as months
 
 # Import Salt libs
@@ -468,3 +469,28 @@ def copyfile(source, dest, backup_mode='', cachedir=''):
         # TODO, backup to master
         pass
     shutil.move(tgt, dest)
+
+
+def path_join(*parts):
+    """
+    This functions tries to solve some issues when joining multiple absolute
+    paths on both *nix and windows platforms.
+
+    See tests/unit/utils/path_join_test.py for some examples on what's being
+    talked about here.
+    """
+    # Normalize path converting any os.sep as needed
+    parts = [os.path.normpath(p) for p in parts]
+
+    root = parts.pop(0)
+    if not parts:
+        return root
+
+    if platform.system().lower() == 'windows':
+        if len(root) == 1:
+            root += ':'
+        root = root.rstrip(os.sep) + os.sep
+
+    return os.path.normpath(os.path.join(
+        root, *[p.lstrip(os.sep) for p in parts]
+    ))

salt/utils/parsers.py

@@ -171,7 +171,12 @@ class ConfigDirMixIn(DeprecatedConfigMessage):
                 continue
 
             value = getattr(self.options, option.dest, None)
-            if value is not None:
+            default = self.defaults.get(option.dest)
+            if value is not None and default is not None and (value != default):
+                # Only override if there's a cli value, and if the cli value
+                # is not the defined default(we need to be able to make changes
+                # on the config file be respected, IF there's no value, other
+                # than the default one from cli)
                 self.config[option.dest] = value
 
         # Merge parser group options if any
@@ -250,8 +255,9 @@ class LogLevelMixIn(object):
         )
 
     def setup_logfile_logger(self):
+        lfkey = 'key_logfile' if 'key' in self.get_prog_name() else 'log_file'
         log.setup_logfile_logger(
-            self.config['log_file'],
+            self.config[lfkey],
             self.config['log_level_logfile'] or self.config['log_level'],
             log_format=self.config['log_fmt_logfile'],
             date_format=self.config['log_datefmt']
@@ -768,12 +774,6 @@ class SaltKeyOptionParser(OptionParser, ConfigDirMixIn, LogLevelMixIn,
 
         self.add_option(
             '--key-logfile',
-            help=("DEPRECATED: Please use '--log-file' instead.\n"
-                  "Send all output to a file.")
-        )
-
-        self.add_option(
-            '--log-file',
             default='/var/log/salt/key',
             help=('Send all output to a file. Default is %default')
         )
@@ -826,13 +826,6 @@ class SaltKeyOptionParser(OptionParser, ConfigDirMixIn, LogLevelMixIn,
         elif self.options.keysize > 32768:
             self.error("The maximum value for keysize is 32768")
 
-    def process_key_logfile(self):
-        if self.options.key_logfile:
-            sys.stderr.write("The '--key-logfile' option is deprecated and "
-                             "will be removed in the future. Please use "
-                             "'--log-file' instead.")
-            self.config["log_file"] = self.options.key_logfile
-
     def _mixin_after_parsed(self):
         # It was decided to always set this to info, since it really all is
         # info or error.

salt/utils/verify.py

@@ -109,6 +109,8 @@ def verify_env(dirs, user, permissive=False):
         sys.stderr.write(err)
         sys.exit(2)
     for dir_ in dirs:
+        if not dir_:
+            continue
         if not os.path.isdir(dir_):
             try:
                 cumask = os.umask(63)  # 077
@@ -146,8 +148,8 @@ def verify_env(dirs, user, permissive=False):
                         if permissive and fmode.st_gid in groups:
                             pass
                         else:
-                          # chown the file for the new user
-                          os.chown(path, uid, gid)
+                            # chown the file for the new user
+                            os.chown(path, uid, gid)
                 for name in dirs:
                     path = os.path.join(root, name)
                     fmode = os.stat(path)
@@ -155,8 +157,8 @@ def verify_env(dirs, user, permissive=False):
                         if permissive and fmode.st_gid in groups:
                             pass
                         else:
-                           # chown the file for the new user
-                           os.chown(path, uid, gid)
+                            # chown the file for the new user
+                            os.chown(path, uid, gid)
         # Allow the pki dir to be 700 or 750, but nothing else.
         # This prevents other users from writing out keys, while
         # allowing the use-case of 3rd-party software (like django)

tests/integration/__init__.py

@@ -320,14 +320,37 @@ class ShellCase(TestCase):
         cmd = '{0} {1} {2} {3}'.format(ppath, PYEXEC, path, arg_str)
 
         if catch_stderr:
-            out, err = subprocess.Popen(
+            process = subprocess.Popen(
                 cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-            ).communicate()
+            )
+            if sys.version_info[0:2] < (2, 7):
+                # On python 2.6, the subprocess'es communicate() method uses
+                # select which, is limited by the OS to 1024 file descriptors
+                # We need more available descriptors to run the tests which
+                # need the stderr output.
+                # So instead of .communicate() we wait for the process to
+                # finish, but, as the python docs state "This will deadlock
+                # when using stdout=PIPE and/or stderr=PIPE and the child
+                # process generates enough output to a pipe such that it
+                # blocks waiting for the OS pipe buffer to accept more data.
+                # Use communicate() to avoid that." <- a catch, catch situation
+                #
+                # Use this work around were it's needed only, python 2.6
+                process.wait()
+                out = process.stdout.read()
+                err = process.stderr.read()
+            else:
+                out, err = process.communicate()
+            # Force closing stderr/stdout to release file descriptors
+            process.stdout.close()
+            process.stderr.close()
             return out.split('\n'), err.split('\n')
 
-        data = subprocess.Popen(
+        process = subprocess.Popen(
             cmd, shell=True, stdout=subprocess.PIPE
-        ).communicate()
+        )
+        data = process.communicate()
+        process.stdout.close()
         return data[0].split('\n')
 
     def run_salt(self, arg_str):

tests/integration/files/conf/master

@@ -13,3 +13,4 @@ peer:
   '.*':
     - 'test.*'
 log_file: master
+key_logfile: key

tests/integration/states/file.py

@@ -59,11 +59,14 @@ class FileTest(integration.ModuleCase):
         file.absent
         '''
         name = os.path.join(integration.TMP, 'link_to_kill')
-        os.symlink(name, '{0}.tgt'.format(name))
+        if not os.path.islink('{0}.tgt'.format(name)):
+            os.symlink(name, '{0}.tgt'.format(name))
         ret = self.run_state('file.absent', name=name)
         result = ret[next(iter(ret))]['result']
         self.assertTrue(result)
         self.assertFalse(os.path.islink(name))
+        if os.path.islink('{0}.tgt'.format(name)):
+            os.unlink('{0}.tgt'.format(name))
 
     def test_test_absent(self):
         '''

tests/unit/config_test.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""
+    tests.unit.config
+    ~~~~~~~~~~~~~~~~~
+
+    :copyright: © 2012 UfSoft.org - :email:`Pedro Algarvio (pedro@algarvio.me)`
+    :license: Apache 2.0, see LICENSE for more details.
+"""
+
+import os
+import tempfile
+from saltunittest import TestCase, TestLoader, TextTestRunner
+from salt import config as sconfig
+
+class ConfigTestCase(TestCase):
+    def test_proper_path_joining(self):
+        fpath = tempfile.mktemp()
+        open(fpath, 'w').write(
+            "root_dir: /\n"
+            "key_logfile: key\n"
+        )
+        config = sconfig.master_config(fpath)
+        # os.path.join behaviour
+        self.assertEqual(config['key_logfile'], os.path.join('/', 'key'))
+        # os.sep.join behaviour
+        self.assertNotEqual(config['key_logfile'], '//key')
+
+
+if __name__ == "__main__":
+    loader = TestLoader()
+    tests = loader.loadTestsFromTestCase(ConfigTestCase)
+    TextTestRunner(verbosity=1).run(tests)

tests/unit/utils/path_join_test.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+"""
+    tests.unit.utils.path_join_test
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: © 2012 UfSoft.org - :email:`Pedro Algarvio (pedro@algarvio.me)`
+    :license: Apache 2.0, see LICENSE for more details.
+"""
+
+import os
+import sys
+import posixpath
+import ntpath
+import platform
+import tempfile
+
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+
+from saltunittest import TestCase, TestLoader, TextTestRunner
+from salt.utils import path_join
+
+class PathJoinTestCase(TestCase):
+
+    PLATFORM_FUNC = platform.system
+    BUILTIN_MODULES = sys.builtin_module_names
+
+    NIX_PATHS = (
+        (('/', 'key'), '/key'),
+        (('/usr/local', '/etc/salt/pki'), '/usr/local/etc/salt/pki')
+    )
+
+    WIN_PATHS = (
+        (('c:', 'temp', 'foo'), 'c:\\temp\\foo'),
+        (('c:', r'\temp', r'\foo'), 'c:\\temp\\foo'),
+        (('c:\\', r'\temp', r'\foo'), 'c:\\temp\\foo'),
+        ((r'c:\\', r'\temp', r'\foo'), 'c:\\temp\\foo'),
+        (('c:', r'\temp', r'\foo', 'bar'), 'c:\\temp\\foo\\bar'),
+        (('c:', r'\temp', r'\foo\bar'), 'c:\\temp\\foo\\bar'),
+        (('c', r'\temp', r'\foo\bar'), 'c:\\temp\\foo\\bar')
+    )
+
+    def test_nix_paths(self):
+        if platform.system().lower() == "windows":
+            self.skipTest(
+                "Windows platform found. not running *nix path_join tests"
+            )
+        for idx, (parts, expected) in enumerate(self.NIX_PATHS):
+            path = path_join(*parts)
+            self.assertEqual(
+                '{0}: {1}'.format(idx, path),
+                '{0}: {1}'.format(idx, expected)
+            )
+
+    def test_windows_paths(self):
+        if platform.system().lower() != "windows":
+            self.skipTest(
+                "Non windows platform found. not running non patched os.path path_join tests"
+            )
+
+        for idx, (parts, expected) in enumerate(self.WIN_PATHS):
+            path = path_join(*parts)
+            self.assertEqual(
+                '{0}: {1}'.format(idx, path),
+                '{0}: {1}'.format(idx, expected)
+            )
+
+    def test_windows_paths_patched_path_module(self):
+        if platform.system().lower() == "windows":
+            self.skipTest(
+                "Windows platform found. not running patched os.path path_join tests"
+            )
+
+        self.__patch_path()
+
+        for idx, (parts, expected) in enumerate(self.WIN_PATHS):
+            path = path_join(*parts)
+            self.assertEqual(
+                '{0}: {1}'.format(idx, path),
+                '{0}: {1}'.format(idx, expected)
+            )
+
+        self.__unpatch_path()
+
+    def __patch_path(self):
+        import imp
+        modules = list(self.BUILTIN_MODULES[:])
+        modules.pop(modules.index('posix'))
+        modules.append('nt')
+
+        code = """'''Salt unittest loaded NT module'''"""
+        module = imp.new_module('nt')
+        exec code in module.__dict__
+        sys.modules['nt'] = module
+
+        sys.builtin_module_names = modules
+        platform.system = lambda: "windows"
+
+        for module in (ntpath, os, os.path, tempfile):
+            reload(module)
+
+    def __unpatch_path(self):
+        del sys.modules['nt']
+        sys.builtin_module_names = self.BUILTIN_MODULES[:]
+        platform.system = self.PLATFORM_FUNC
+
+        for module in (posixpath, os, os.path, tempfile, platform):
+            reload(module)
+
+if __name__ == "__main__":
+    loader = PathJoinTestCase()
+    tests = loader.loadTestsFromTestCase(PathJoinTestCase)
+    TextTestRunner(verbosity=1).run(tests)