Merge pull request #13468 from techhat/db

Add sdb loader type

doc/topics/sdb/index.rst

@@ -0,0 +1,95 @@
+.. _sdb:
+
+===============================
+Storing Data in Other Databases
+===============================
+The SDB interface is designed to store and retrieve data that, unlike pillars
+and grains, is not necessarily minion-specific. The initial design goal was to
+allow passwords to be stored in a secure database, such as one managed by the
+keyring package, rather than as plain-text files. However, as a generic database
+interface, it could conceptually be used for a number of other purposes.
+
+SDB was added to Salt in version Helium. SDB is currently experimental, and
+should probably not be used in production.
+
+
+SDB Configuration
+================
+In order to use the SDB interface, a configuration profile must be set up in
+either the master or minion configuration file. The configuration stanza
+includes the name/ID that the profile will be referred to as, a ``driver``
+setting, and any other arguments that are necessary for the SDB module that will
+be used. For instance, a profile called ``mykeyring``, which uses the
+``system`` service in the ``keyring`` module would look like:
+
+.. code-block:: yaml
+
+    mykeyring:
+      driver: keyring
+      service: system
+
+It is recommended to keep the name of the profile simple, as it is used in the
+SDB URI as well.
+
+
+SDB URIs
+========
+SDB is designed to make small database queries (hence the name, SDB) using a
+compact URL. This allows users to reference a database value quickly inside
+a number of Salt configuration areas, without a lot of overhead. The basic
+format of an SDB URI is:
+
+.. code-block:: yaml
+
+    sdb://<profile>/<args>
+
+The profile refers to the configuration profile defined in either the master or
+the minion configuration file. The args are specific to the module referred to
+in the profile, but will typically only need to refer to the key of a
+key/value pair inside the database. This is because the profile itself should
+define as many other parameters as possible.
+
+For example, a profile might be set up to reference credentials for a specific
+OpenStack account. The profile might look like:
+
+.. code-block:: yaml
+
+    kevinopenstack:
+      driver: keyring
+      service salt.cloud.openstack.kevin
+
+And the URI used to reference the password might look like:
+
+.. code-block:: yaml
+
+    sdb://kevinopenstack/password
+
+
+Writing SDB Modules
+===================
+There is currently one function that MUST exist in any SDB module (``get()``)
+and one that MAY exist (``set_()``). If using a (``set_()``) function, a
+``__func_alias__`` dictionary MUST be declared in the module as well:
+
+.. code-block:: python
+
+    __func_alias__ = {
+        'set_': 'set',
+    }
+
+This is because ``set`` is a Python built-in, and therefore functions should not
+be created which are called ``set()``. The ``__func_alias__`` functionality is
+provided via Salt's loader interfaces, and allows legally-named functions to be
+referred to using names that would otherwise be unwise to use.
+
+The ``get()`` function is required, as it will be called via functions in other
+areas of the code which make use of the ``sdb://`` URI. For example, the
+``config.get`` function in the ``config`` execution module uses this function.
+
+The ``set_()`` function may be provided, but is not required, as some sources
+may be read-only, or may be otherwise unwise to access via a URI (for instance,
+because of SQL injection attacks).
+
+A simple example of an SDB module is ``salt/sdb/keyring_db.py``, as it provides
+basic examples of most, if not all, of the types of functionality that are
+available not only for SDB modules, but for Salt modules in general.

salt/loader.py

@@ -408,6 +408,20 @@ def queues(opts):
     return load.gen_functions()
 
 
+def sdb(opts, functions=None, whitelist=None):
+    '''
+    Make a very small database call
+    '''
+    load = _create_loader(opts, 'sdb', 'sdb')
+    pack = {'name': '__sdb__',
+            'value': functions}
+    return LazyLoader(load,
+                      functions,
+                      pack,
+                      whitelist=whitelist,
+                      )
+
+
 def clouds(opts):
     '''
     Return the cloud functions

salt/modules/config.py

@@ -11,6 +11,7 @@ import os
 import salt.utils
 import salt._compat
 import salt.syspaths as syspaths
+import salt.utils.sdb as sdb
 
 __proxyenabled__ = ['*']
 
@@ -226,16 +227,20 @@ def get(key, default=''):
     '''
     ret = salt.utils.traverse_dict_and_list(__opts__, key, '_|-')
     if ret != '_|-':
-        return ret
+        return sdb.sdb_get(ret, __opts__)
+
     ret = salt.utils.traverse_dict_and_list(__grains__, key, '_|-')
     if ret != '_|-':
-        return ret
+        return sdb.sdb_get(ret, __opts__)
+
     ret = salt.utils.traverse_dict_and_list(__pillar__, key, '_|-')
     if ret != '_|-':
-        return ret
+        return sdb.sdb_get(ret, __opts__)
+
     ret = salt.utils.traverse_dict_and_list(__pillar__.get('master', {}), key, '_|-')
     if ret != '_|-':
-        return ret
+        return sdb.sdb_get(ret, __opts__)
+
     return default
 
 

salt/sdb/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+'''
+SDB Module Directory
+'''

salt/sdb/keyring_db.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+'''
+Keyring Database Module
+
+:maintainer:    SaltStack
+:maturity:      New
+:depends:       keyring
+:platform:      all
+
+This module allows access to the keyring package using an ``sdb://`` URI. This
+package is located at ``https://pypi.python.org/pypi/keyring``.
+
+Care must be taken when using keyring. Not all keyend backends are supported on
+all operating systems. Also, many backends require an agent to be running in
+order to work. For instance, the "Secret Service" backend requires a compatible
+agent such as ``gnome-keyring-daemon`` or ``kwallet`` to be running. The
+keyczar backend does not seem to enjoy the benefits of an agent, and so using
+it will require either that the password is typed in manually (which is
+unreasonable for the salt-minion and salt-master daemons, especially in
+production) or an agent is written for it.
+
+Like all sdb modules, the keyring module requires a configuration profile to
+be configured in either the minion or master configuration file. This profile
+requires very little. In the example:
+
+.. code-block:: yaml
+
+    mykeyring:
+      - driver: keyring
+      - service: system
+
+The ``driver`` refers to the keyring module, ``service`` refers to the service
+that will be used inside of keyring (which may be likened unto a database
+table) and ``mykeyring`` refers to the name that will appear in the URI:
+
+.. code-block:: yaml
+
+    password: sdb://mykeyring/mypassword
+
+The underlying backend configuration must be configured via keyring itself. For
+examples and documentation, see keyring:
+
+https://pypi.python.org/pypi/keyring
+
+.. versionadded:: 2014.1.4 (Hydrogen)
+'''
+
+# import python libs
+import logging
+
+try:
+    import keyring
+    HAS_LIBS = True
+except ImportError:
+    HAS_LIBS = False
+
+log = logging.getLogger(__name__)
+
+__func_alias__ = {
+    'set_': 'set'
+}
+
+__virtualname__ = 'keyring'
+
+
+def __virtual__():
+    '''
+    Only load the module if keyring is installed
+    '''
+    if HAS_LIBS:
+        return __virtualname__
+    return False
+
+
+def set_(key, value, service=None, profile=None):
+    '''
+    Set a key/value pair in a keyring service
+    '''
+    service = _get_service(service, profile)
+    keyring.set_password(service, key, value)
+
+
+def get(key, service=None, profile=None):
+    '''
+    Get a value from a keyring service
+    '''
+    service = _get_service(service, profile)
+    return keyring.get_password(service, key)
+
+
+def _get_service(service, profile):
+    '''
+    Get a service name
+    '''
+    if isinstance(profile, dict) and 'service' in profile:
+        return profile['service']
+
+    return service

salt/utils/sdb.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+'''
+Basic functions for accessing the SDB interface
+'''
+import salt.loader
+from salt._compat import string_types
+
+
+def sdb_get(uri, opts):
+    '''
+    Get a value from a db, using a uri in the form of sdb://<profile>/<key>. If
+    the uri provided does not start with sdb://, then it will be returned as-is.
+    '''
+    if not isinstance(uri, string_types):
+        return uri
+
+    if not uri.startswith('sdb://'):
+        return uri
+
+    comps = uri.replace('sdb://', '').split('/')
+
+    if len(comps) < 2:
+        return uri
+
+    profile = opts.get(comps[0], {})
+    if 'driver' not in profile:
+        return uri
+
+    fun = '{0}.get'.format(profile['driver'])
+    query = comps[1]
+
+    loaded_db = salt.loader.sdb(opts, fun)
+    return loaded_db[fun](query, profile=profile)
+
+
+def sdb_set(uri, value, opts):
+    '''
+    Get a value from a db, using a uri in the form of sdb://<profile>/<key>. If
+    the uri provided does not start with sdb://, then it will be returned as-is.
+    '''
+    if not isinstance(uri, string_types):
+        return uri
+
+    if not uri.startswith('sdb://'):
+        return False
+
+    comps = uri.replace('sdb://', '').split('/')
+
+    if len(comps) < 2:
+        return False
+
+    profile = opts.get(comps[0], {})
+    if 'driver' not in profile:
+        return False
+
+    fun = '{0}.set'.format(profile['driver'])
+    query = comps[1]
+
+    loaded_db = salt.loader.sdb(opts, fun)
+    return loaded_db[fun](query, value, profile=profile)