Merge branch 'develop' into patch-1

doc/conf.py

@@ -504,6 +504,7 @@ epub_copyright = copyright
 epub_scheme = 'URL'
 epub_identifier = 'http://saltstack.com/'
 
+epub_tocdup = False
 #epub_tocdepth = 3
 
 

doc/ref/cache/all/salt.cache.etcd_cache.rst

@@ -0,0 +1,5 @@
+salt.cache.etcd_cache module
+=============================
+
+.. automodule:: salt.cache.etcd_cache
+    :members:

doc/ref/cache/all/salt.cache.mysql_cache.rst

@@ -0,0 +1,5 @@
+salt.cache.mysql_cache module
+=============================
+
+.. automodule:: salt.cache.mysql_cache
+    :members:

salt/cache/etcd_cache.py

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+'''
+Minion data cache plugin for Etcd key/value data store.
+
+.. versionadded:: develop
+
+It is up to the system administrator to set up and configure the Etcd
+infrastructure. All is needed for this plugin is a working Etcd agent
+with a read-write access to the key-value store.
+
+The related documentation can be found in the `Etcd documentation`_.
+
+To enable this cache plugin, the master will need the python client for
+Etcd installed. This can be easily installed with pip:
+
+.. code-block: bash
+
+    pip install python-etcd
+
+Optionally, depending on the Etcd agent configuration, the following values
+could be set in the master config. These are the defaults:
+
+.. code-block:: yaml
+
+    etcd.host: 127.0.0.1
+    etcd.port: 2379
+    etcd.protocol: http
+    etcd.allow_reconnect: True
+    etcd.allow_redirect: False
+    etcd.srv_domain: None
+    etcd.read_timeout: 60
+    etcd.username: None
+    etcd.password: None
+    etcd.cert: None
+    etcd.ca_cert: None
+
+Related docs could be found in the `python-etcd documentation`_.
+
+To use the etcd as a minion data cache backend, set the master ``cache`` config
+value to ``etcd``:
+
+.. code-block:: yaml
+
+    cache: etcd
+
+
+.. _`Etcd documentation`: https://github.com/coreos/etcd
+.. _`python-etcd documentation`: http://python-etcd.readthedocs.io/en/latest/
+
+'''
+from __future__ import absolute_import
+import logging
+try:
+    import etcd
+    HAS_ETCD = True
+except ImportError:
+    HAS_ETCD = False
+
+from salt.exceptions import SaltCacheError
+
+_DEFAULT_PATH_PREFIX = "/salt_cache"
+
+if HAS_ETCD:
+    # The client logging tries to decode('ascii') binary data
+    # and is too verbose
+    etcd._log.setLevel(logging.INFO)  # pylint: disable=W0212
+
+log = logging.getLogger(__name__)
+client = None
+path_prefix = None
+
+# Module properties
+
+__virtualname__ = 'etcd'
+__func_alias__ = {'ls': 'list'}
+
+
+def __virtual__():
+    '''
+    Confirm that python-etcd package is installed.
+    '''
+    if not HAS_ETCD:
+        return (False, "Please install python-etcd package to use etcd data "
+                "cache driver")
+
+    return __virtualname__
+
+
+def _init_client():
+    '''Setup client and init datastore.
+    '''
+    global client, path_prefix
+    if client is not None:
+        return
+
+    etcd_kwargs = {
+            'host': __opts__.get('etcd.host', '127.0.0.1'),
+            'port': __opts__.get('etcd.port', 2379),
+            'protocol': __opts__.get('etcd.protocol', 'http'),
+            'allow_reconnect': __opts__.get('etcd.allow_reconnect', True),
+            'allow_redirect': __opts__.get('etcd.allow_redirect', False),
+            'srv_domain': __opts__.get('etcd.srv_domain', None),
+            'read_timeout': __opts__.get('etcd.read_timeout', 60),
+            'username': __opts__.get('etcd.username', None),
+            'password': __opts__.get('etcd.password', None),
+            'cert': __opts__.get('etcd.cert', None),
+            'ca_cert': __opts__.get('etcd.ca_cert', None),
+    }
+    path_prefix = __opts__.get('etcd.path_prefix', _DEFAULT_PATH_PREFIX)
+    if path_prefix != "":
+        path_prefix = '/{0}'.format(path_prefix.strip('/'))
+    log.info("etcd: Setting up client with params: %r", etcd_kwargs)
+    client = etcd.Client(**etcd_kwargs)
+    try:
+        client.get(path_prefix)
+    except etcd.EtcdKeyNotFound:
+        log.info("etcd: Creating dir %r", path_prefix)
+        client.write(path_prefix, None, dir=True)
+
+
+def store(bank, key, data):
+    '''
+    Store a key value.
+    '''
+    _init_client()
+    etcd_key = '{0}/{1}/{2}'.format(path_prefix, bank, key)
+    try:
+        value = __context__['serial'].dumps(data)
+        client.set(etcd_key, value)
+    except Exception as exc:
+        raise SaltCacheError(
+            'There was an error writing the key, {0}: {1}'.format(etcd_key, exc)
+        )
+
+
+def fetch(bank, key):
+    '''
+    Fetch a key value.
+    '''
+    _init_client()
+    etcd_key = '{0}/{1}/{2}'.format(path_prefix, bank, key)
+    try:
+        value = client.get(etcd_key).value
+        if value is None:
+            return {}
+        return __context__['serial'].loads(value)
+    except Exception as exc:
+        raise SaltCacheError(
+            'There was an error reading the key, {0}: {1}'.format(
+                etcd_key, exc
+            )
+        )
+
+
+def flush(bank, key=None):
+    '''
+    Remove the key from the cache bank with all the key content.
+    '''
+    _init_client()
+    if key is None:
+        etcd_key = '{0}/{1}'.format(path_prefix, bank)
+    else:
+        etcd_key = '{0}/{1}/{2}'.format(path_prefix, bank, key)
+    try:
+        client.get(etcd_key)
+    except etcd.EtcdKeyNotFound:
+        return  # nothing to flush
+    try:
+        client.delete(etcd_key, recursive=True)
+    except Exception as exc:
+        raise SaltCacheError(
+            'There was an error removing the key, {0}: {1}'.format(
+                etcd_key, exc
+            )
+        )
+
+
+def _walk(r):
+    '''
+    Recursively walk dirs. Return flattened list of keys.
+    r: etcd.EtcdResult
+    '''
+    if not r.dir:
+        return [r.key.split('/', 3)[3]]
+
+    keys = []
+    for c in client.get(r.key).children:
+        keys.extend(_walk(c))
+    return keys
+
+
+def ls(bank):
+    '''
+    Return an iterable object containing all entries stored in the specified
+    bank.
+    '''
+    _init_client()
+    path = '{0}/{1}'.format(path_prefix, bank)
+    try:
+        return _walk(client.get(path))
+    except Exception as exc:
+        raise SaltCacheError(
+            'There was an error getting the key "{0}": {1}'.format(
+                bank, exc
+            )
+        )
+
+
+def contains(bank, key):
+    '''
+    Checks if the specified bank contains the specified key.
+    '''
+    _init_client()
+    etcd_key = '{0}/{1}/{2}'.format(path_prefix, bank, key)
+    try:
+        r = client.get(etcd_key)
+        # return True for keys, not dirs
+        return r.dir is False
+    except etcd.EtcdKeyNotFound:
+        return False
+    except Exception as exc:
+        raise SaltCacheError(
+            'There was an error getting the key, {0}: {1}'.format(
+                etcd_key, exc
+            )
+        )

salt/cache/mysql_cache.py

@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+'''
+Minion data cache plugin for MySQL database.
+
+.. versionadded:: develop
+
+It is up to the system administrator to set up and configure the MySQL
+infrastructure. All is needed for this plugin is a working MySQL server.
+
+The module requires the `salt_cache` database to exists but creates its own
+table if needed. The keys are indexed using the `bank` and `etcd_key` columns.
+
+To enable this cache plugin, the master will need the python client for
+MySQL installed. This can be easily installed with pip:
+
+.. code-block: bash
+
+    pip install python-mysql
+
+Optionally, depending on the MySQL agent configuration, the following values
+could be set in the master config. These are the defaults:
+
+.. code-block:: yaml
+
+    mysql.host: 127.0.0.1
+    mysql.port: 2379
+    mysql.user: None
+    mysql.password: None
+    mysql.database: salt_cache
+    mysql.table_name: cache
+
+Related docs could be found in the `python-mysql documentation`_.
+
+To use the mysql as a minion data cache backend, set the master ``cache`` config
+value to ``mysql``:
+
+.. code-block:: yaml
+
+    cache: mysql
+
+
+.. _`MySQL documentation`: https://github.com/coreos/mysql
+.. _`python-mysql documentation`: http://python-mysql.readthedocs.io/en/latest/
+
+'''
+from __future__ import absolute_import
+from time import sleep
+import logging
+try:
+    import MySQLdb
+    HAS_MYSQL = True
+except ImportError:
+    HAS_MYSQL = False
+
+from salt.exceptions import SaltCacheError
+
+_DEFAULT_DATABASE_NAME = "salt_cache"
+_DEFAULT_CACHE_TABLE_NAME = "cache"
+_RECONNECT_INTERVAL_SEC = 0.050
+
+log = logging.getLogger(__name__)
+client = None
+_mysql_kwargs = None
+_table_name = None
+
+# Module properties
+
+__virtualname__ = 'mysql'
+__func_alias__ = {'ls': 'list'}
+
+
+def __virtual__():
+    '''
+    Confirm that python-mysql package is installed.
+    '''
+    if not HAS_MYSQL:
+        return (False, "Please install python-mysql package to use mysql data "
+                "cache driver")
+    return __virtualname__
+
+
+def run_query(conn, query, retries=3):
+    '''
+    Get a cursor and run a query. Reconnect up to `retries` times if
+    needed.
+    Returns: cursor, affected rows counter
+    Raises: SaltCacheError, AttributeError, MySQLdb.OperationalError
+    '''
+    try:
+        cur = conn.cursor()
+        out = cur.execute(query)
+        return cur, out
+    except (AttributeError, MySQLdb.OperationalError) as e:
+        if retries == 0:
+            raise
+        # reconnect creating new client
+        sleep(_RECONNECT_INTERVAL_SEC)
+        if conn is None:
+            log.debug("mysql_cache: creating db connection")
+        else:
+            log.info("mysql_cache: recreating db connection due to: %r", e)
+        global client
+        client = MySQLdb.connect(**_mysql_kwargs)
+        return run_query(client, query, retries - 1)
+    except Exception as e:
+        if len(query) > 150:
+            query = query[:150] + "<...>"
+        raise SaltCacheError("Error running {0}: {1}".format(query, e))
+
+
+def _create_table():
+    '''
+    Create table if needed
+    '''
+    # Explicitely check if the table already exists as the library logs a
+    # warning on CREATE TABLE
+    query = """SELECT COUNT(TABLE_NAME) FROM information_schema.tables
+        WHERE table_schema = '{0}' AND table_name = '{1}'""".format(
+            _mysql_kwargs['db'],
+            _table_name,
+        )
+    cur, _ = run_query(client, query)
+    r = cur.fetchone()
+    cur.close()
+    if r[0] == 1:
+        return
+
+    query = """CREATE TABLE IF NOT EXISTS {0} (
+      bank CHAR(255),
+      etcd_key CHAR(255),
+      data MEDIUMBLOB,
+      PRIMARY KEY(bank, etcd_key)
+    );""".format(_table_name)
+    log.info("mysql_cache: creating table %s", _table_name)
+    cur, _ = run_query(client, query)
+    cur.close()
+
+
+def _init_client():
+    """Initialize connection and create table if needed
+    """
+    if client is not None:
+        return
+
+    global _mysql_kwargs, _table_name
+    _mysql_kwargs = {
+        'host': __opts__.get('mysql.host', '127.0.0.1'),
+        'user': __opts__.get('mysql.user', None),
+        'passwd': __opts__.get('mysql.password', None),
+        'db': __opts__.get('mysql.database', _DEFAULT_DATABASE_NAME),
+        'port': __opts__.get('mysql.port', 3306),
+        'unix_socket': __opts__.get('mysql.unix_socket', None),
+        'connect_timeout': __opts__.get('mysql.connect_timeout', None),
+        'autocommit': True,
+    }
+    _table_name = __opts__.get('mysql.table_name', _table_name)
+    # TODO: handle SSL connection parameters
+
+    for k, v in _mysql_kwargs.items():
+        if v is None:
+            _mysql_kwargs.pop(k)
+    kwargs_copy = _mysql_kwargs.copy()
+    kwargs_copy['passwd'] = "<hidden>"
+    log.info("mysql_cache: Setting up client with params: %r", kwargs_copy)
+    # The MySQL client is created later on by run_query
+    _create_table()
+
+
+def store(bank, key, data):
+    '''
+    Store a key value.
+    '''
+    _init_client()
+    data = __context__['serial'].dumps(data)
+    query = "REPLACE INTO {0} (bank, etcd_key, data) values('{1}', '{2}', " \
+        "'{3}')".format(_table_name, bank, key, data)
+    cur, cnt = run_query(client, query)
+    cur.close()
+    if cnt not in (1, 2):
+        raise SaltCacheError(
+            'Error storing {0} {1} returned {2}'.format(bank, key, cnt)
+        )
+
+
+def fetch(bank, key):
+    '''
+    Fetch a key value.
+    '''
+    _init_client()
+    query = "SELECT data FROM {0} WHERE bank='{1}' AND etcd_key='{2}'".format(
+        _table_name, bank, key)
+    cur, _ = run_query(client, query)
+    r = cur.fetchone()
+    cur.close()
+    if r is None:
+        return {}
+    return __context__['serial'].loads(r[0])
+
+
+def flush(bank, key=None):
+    '''
+    Remove the key from the cache bank with all the key content.
+    '''
+    _init_client()
+    query = "DELETE FROM {0} WHERE bank='{1}'".format(_table_name, bank)
+    if key is not None:
+        query += " AND etcd_key='{0}'".format(key)
+
+    cur, _ = run_query(client, query)
+    cur.close()
+
+
+def ls(bank):
+    '''
+    Return an iterable object containing all entries stored in the specified
+    bank.
+    '''
+    _init_client()
+    query = "SELECT etcd_key FROM {0} WHERE bank='{1}'".format(
+        _table_name, bank)
+    cur, _ = run_query(client, query)
+    out = [row[0] for row in cur.fetchall()]
+    cur.close()
+    return out
+
+
+def contains(bank, key):
+    '''
+    Checks if the specified bank contains the specified key.
+    '''
+    _init_client()
+    query = "SELECT COUNT(data) FROM {0} WHERE bank='{1}' " \
+        "AND etcd_key='{2}'".format(_table_name, bank, key)
+    cur, _ = run_query(client, query)
+    r = cur.fetchone()
+    cur.close()
+    return r[0] == 1

salt/modules/glassfish.py

@@ -646,3 +646,34 @@ def delete_jdbc_resource(name, target='server', server=None):
     Delete a JDBC resource
     '''
     return _delete_element(name, 'resources/jdbc-resource', {'target': target}, server)
+
+
+# System properties
+def get_system_properties(server=None):
+    '''
+    Get system properties
+    '''
+    properties = {}
+    data = _api_get('system-properties', server)
+
+    # Get properties into a dict
+    if any(data['extraProperties']['systemProperties']):
+        for element in data['extraProperties']['systemProperties']:
+            properties[element['name']] = element['value']
+        return properties
+    return {}
+
+
+def update_system_properties(data, server=None):
+    '''
+    Update system properties
+    '''
+    _api_post('system-properties', _clean_data(data), server)
+    return data
+
+
+def delete_system_properties(name, server=None):
+    '''
+    Delete a system property
+    '''
+    _api_delete('system-properties/{0}'.format(name), None, server)

salt/states/glassfish.py

@@ -561,3 +561,86 @@ def jdbc_datasource_absent(name, both=True, server=None):
         ret['result'] = False
         ret['comment'] = 'Error: {0}'.format(pool_ret['error'])
     return ret
+
+
+def system_properties_present(server=None, **kwargs):
+    '''
+    Ensures that the system properties are present
+
+    properties
+        The system properties
+    '''
+    ret = {'name': '', 'result': None, 'comment': None, 'changes': {}}
+
+    del kwargs['name']
+    try:
+        data = __salt__['glassfish.get_system_properties'](server=server)
+    except requests.ConnectionError as error:
+        if __opts__['test']:
+            ret['changes'] = kwargs
+            ret['result'] = None
+            return ret
+        else:
+            ret['error'] = "Can't connect to the server"
+            return ret
+
+    ret['changes'] = {'data': data, 'kwargs': kwargs}
+    if not data == kwargs:
+        data.update(kwargs)
+        if not __opts__['test']:
+            try:
+                __salt__['glassfish.update_system_properties'](data, server=server)
+                ret['changes'] = kwargs
+                ret['result'] = True
+                ret['comment'] = 'System properties updated'
+            except CommandExecutionError as error:
+                ret['comment'] = error
+                ret['result'] = False
+        else:
+            ret['result'] = None
+            ret['changes'] = kwargs
+            ret['coment'] = 'System properties would have been updated'
+    else:
+        ret['changes'] = None
+        ret['result'] = True
+        ret['comment'] = 'System properties are already up-to-date'
+    return ret
+
+
+def system_properties_absent(name, server=None):
+    '''
+    Ensures that the system property doesn't exists
+
+    name
+        Name of the system property
+    '''
+    ret = {'name': '', 'result': None, 'comment': None, 'changes': {}}
+
+    try:
+        data = __salt__['glassfish.get_system_properties'](server=server)
+    except requests.ConnectionError as error:
+        if __opts__['test']:
+            ret['changes'] = {'Name': name}
+            ret['result'] = None
+            return ret
+        else:
+            ret['error'] = "Can't connect to the server"
+            return ret
+
+    if name in data:
+        if not __opts__['test']:
+            try:
+                __salt__['glassfish.delete_system_properties'](name, server=server)
+                ret['result'] = True
+                ret['comment'] = 'System properties deleted'
+            except CommandExecutionError as error:
+                ret['comment'] = error
+                ret['result'] = False
+        else:
+            ret['result'] = None
+            ret['comment'] = 'System properties would have been deleted'
+        ret['changes'] = {'Name': name}
+    else:
+        ret['result'] = True
+        ret['comment'] = 'System properties are already absent'
+    return ret

salt/states/postgres_schema.py

@@ -77,6 +77,11 @@ def present(dbname, name,
 
     # The schema is not present, make it!
     if schema_attr is None:
+        if __opts__['test']:
+            ret['result'] = None
+            ret['comment'] = 'Schema {0} is set to be created' \
+                             ' in database {1}.'.format(name, dbname)
+            return ret
         cret = __salt__['postgres.schema_create'](dbname,
                                                   name,
                                                   owner=owner,
@@ -139,7 +144,12 @@ def absent(dbname, name,
 
     # check if schema exists and remove it
     if __salt__['postgres.schema_exists'](dbname, name, **db_args):
-        if __salt__['postgres.schema_remove'](dbname, name, **db_args):
+        if __opts__['test']:
+            ret['result'] = None
+            ret['comment'] = 'Schema {0} is set to be removed' \
+                             ' from database {1}'.format(name, dbname)
+            return ret
+        elif __salt__['postgres.schema_remove'](dbname, name, **db_args):
             ret['comment'] = 'Schema {0} has been removed' \
                              ' from database {1}'.format(name, dbname)
             ret['changes'][name] = 'Absent'

salt/states/postgres_user.py

@@ -95,7 +95,7 @@ def present(name,
         encrypted to the previous
         format if it is not already done.
 
-    default_passwoord
+    default_password
         The password used only when creating the user, unless password is set.
 
         .. versionadded:: 2016.3.0

tests/unit/states/test_postgres_schema.py

@@ -45,10 +45,11 @@ class PostgresSchemaTestCase(TestCase, LoaderModuleMockMixin):
         mock = MagicMock(return_value=name)
         with patch.dict(postgres_schema.__salt__,
                         {'postgres.schema_get': mock}):
-            comt = ('Schema {0} already exists in database {1}'.format(name,
-                                                                       dbname))
-            ret.update({'comment': comt})
-            self.assertDictEqual(postgres_schema.present(dbname, name), ret)
+            with patch.dict(postgres_schema.__opts__, {'test': False}):
+                comt = ('Schema {0} already exists in database {1}'.format(name,
+                                                                           dbname))
+                ret.update({'comment': comt})
+                self.assertDictEqual(postgres_schema.present(dbname, name), ret)
 
     # 'absent' function tests: 1
 
@@ -66,19 +67,26 @@ class PostgresSchemaTestCase(TestCase, LoaderModuleMockMixin):
                'comment': ''}
 
         mock_t = MagicMock(side_effect=[True, False])
-        mock = MagicMock(side_effect=[True, True, False])
+        mock = MagicMock(side_effect=[True, True, True, False])
         with patch.dict(postgres_schema.__salt__,
                         {'postgres.schema_exists': mock,
                          'postgres.schema_remove': mock_t}):
-            comt = ('Schema {0} has been removed from database {1}'.
-                    format(name, dbname))
-            ret.update({'comment': comt, 'result': True,
-                        'changes': {name: 'Absent'}})
-            self.assertDictEqual(postgres_schema.absent(dbname, name), ret)
+            with patch.dict(postgres_schema.__opts__, {'test': True}):
+                comt = ('Schema {0} is set to be removed from database {1}'.
+                        format(name, dbname))
+                ret.update({'comment': comt, 'result': None})
+                self.assertDictEqual(postgres_schema.absent(dbname, name), ret)
 
-            comt = ('Schema {0} failed to be removed'.format(name))
-            ret.update({'comment': comt, 'result': False, 'changes': {}})
-            self.assertDictEqual(postgres_schema.absent(dbname, name), ret)
+            with patch.dict(postgres_schema.__opts__, {'test': False}):
+                comt = ('Schema {0} has been removed from database {1}'.
+                        format(name, dbname))
+                ret.update({'comment': comt, 'result': True,
+                            'changes': {name: 'Absent'}})
+                self.assertDictEqual(postgres_schema.absent(dbname, name), ret)
+
+                comt = ('Schema {0} failed to be removed'.format(name))
+                ret.update({'comment': comt, 'result': False, 'changes': {}})
+                self.assertDictEqual(postgres_schema.absent(dbname, name), ret)
 
             comt = ('Schema {0} is not present in database {1},'
                     ' so it cannot be removed'.format(name, dbname))