Use consul as an external pillar source. Requires python-consul

salt/pillar/consul_pillar.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+'''
+Use consul data as a Pillar source
+
+:depends:  - python-consul
+
+In order to use an consul server, a profile must be created in the master
+configuration file:
+
+.. code-block:: yaml
+
+    my_consul_config:
+      consul.host: 127.0.0.1
+      consul.port: 8500
+
+After the profile is created, configure the external pillar system to use it.
+Optionally, a root may be specified.
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - consul: my_consul_config
+
+    ext_pillar:
+      - consul: my_consul_config root=/salt
+
+Using these configuration profiles, multiple consul sources may also be used:
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - consul: my_consul_config
+      - consul: my_other_consul_config
+
+The ``minion_id`` may be used in the ``root`` path to expose minion-specific
+information stored in consul.
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - consul: my_consul_config root=/salt/%(minion_id)s
+
+Minion-specific values may override shared values when the minion-specific root
+appears after the shared root:
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - consul: my_consul_config root=/salt-shared
+      - consul: my_other_consul_config root=/salt-private/%(minion_id)s
+
+'''
+from __future__ import absolute_import
+
+# Import python libs
+import logging
+
+import re
+
+from salt.exceptions import CommandExecutionError
+
+# Import third party libs
+try:
+  import consul
+  HAS_CONSUL = True
+except ImportError:
+  HAS_CONSUL = False
+
+__virtualname__ = 'consul'
+
+# Set up logging
+log = logging.getLogger(__name__)
+
+
+def __virtual__():
+  '''
+  Only return if python-consul is installed
+  '''
+  return __virtualname__ if HAS_CONSUL else False
+
+
+def ext_pillar(minion_id,
+               pillar,  # pylint: disable=W0613
+               conf):
+  '''
+  Check consul for all data
+  '''
+  comps = conf.split()
+
+  profile = None
+  if comps[0]:
+    profile = comps[0]
+  client = get_conn(__opts__, profile)
+
+  path = ''
+  if len(comps) > 1 and comps[1].startswith('root='):
+    path = comps[1].replace('root=', '')
+
+  # put the minion's ID in the path if necessary
+  path %= {
+    'minion_id': minion_id
+  }
+
+  try:
+    pillar = fetch_tree(client, path)
+  except KeyError:
+    log.error('No such key in consul profile {0}: {1}'.format(profile, path))
+    pillar = {}
+
+  return pillar
+
+def consul_fetch(client, path):
+  '''
+  Query consul for all keys/values within base path
+  '''
+  return client.kv.get(path, recurse=True)
+
+def fetch_tree(client, path):
+  '''
+  Grab data from consul, trim base path and remove any keys which
+  are folders. Take the remaining data and send it to be formatted
+  in such a way as to be used as pillar data.
+  '''
+  index, items = consul_fetch(client, path)
+  ret = {}
+  has_children = re.compile(r'/$')
+
+  log.debug('Fetched items: %r', format(items))
+
+  for item in reversed(items):
+    key = re.sub(r'^' + path + '/?', '', item['Key'])
+    if key != "":
+      log.debug('key/path - {0}: {1}'.format(path, key))
+      log.debug('has_children? %r', format(has_children.search(key)))
+      if has_children.search(key) == None:
+        ret = pillar_format(ret, key.split('/'), item['Value'])
+        log.debug('Fetching subkeys for key: %r', format(item))
+
+  return ret
+
+def pillar_format(ret, keys, value):
+  '''
+  Perform data formatting to be used as pillar data and
+  merge it with the current pillar data
+  '''
+  Get t
+  if value == None:
+    return ret
+  array_data = value.split('\n')
+  pillar_value = array_data[0] if len(array_data) == 1 else array_data
+  keyvalue = keys.pop()
+  pil = {keyvalue: pillar_value}
+  keys.reverse()
+  for k in keys:
+    pil = {k: pil}
+
+  return dict_merge(ret, pil)
+
+def dict_merge(d1, d2):
+  '''
+  Take 2 dictionaries and deep merge them
+  '''
+  master = d1.copy()
+  for (k, v) in d2.iteritems():
+    if k in master and isinstance(master[k], dict):
+      master[k] = dict_merge(master[k], v)
+    else:
+      master[k] = v
+  return master
+
+def get_conn(opts, profile):
+
+  '''
+  Return a client object for accessing consul
+  '''
+  opts_pillar = opts.get('pillar', {})
+  opts_master = opts_pillar.get('master', {})
+
+  opts_merged = {}
+  opts_merged.update(opts_master)
+  opts_merged.update(opts_pillar)
+  opts_merged.update(opts)
+
+  if profile:
+    conf = opts_merged.get(profile, {})
+  else:
+    conf = opts_merged
+
+  consul_host = conf.get('consul.host', '127.0.0.1')
+  consul_port = conf.get('consul.port', 8500)
+
+  if HAS_CONSUL:
+    return consul.Consul(host=consul_host, port=consul_port)
+  else:
+    raise CommandExecutionError(
+            '(unable to import consul, '
+            'module most likely not installed. Download python-consul '
+            'module and be sure to import consul)'
+    )

tests/unit/pillar/consul_test.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+
+# Import python libs
+from __future__ import absolute_import
+
+# Import Salt Testing libs
+from salttesting import TestCase, skipIf
+from salttesting.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch
+from salttesting.helpers import ensure_in_syspath
+
+ensure_in_syspath('../../')
+
+# Import Salt Libs
+from salt.pillar import consul_pillar
+
+OPTS = {'consul_config': {'consul.port': 8500, 'consul.host': '172.17.0.15'}}
+
+consul_pillar.__opts__ = OPTS
+
+PILLAR_DATA = [
+  {'Value': '/path/to/certs/testsite1.crt', 'Key': u'test-shared/sites/testsite1/ssl/certs/SSLCertificateFile'},
+  {'Value': '/path/to/certs/testsite1.key', 'Key': u'test-shared/sites/testsite1/ssl/certs/SSLCertificateKeyFile'},
+  {'Value': None, 'Key': u'test-shared/sites/testsite1/ssl/certs/'},
+  {'Value': 'True', 'Key': u'test-shared/sites/testsite1/ssl/force'},
+  {'Value': None, 'Key': u'test-shared/sites/testsite1/ssl/'},
+  {'Value': 'salt://sites/testsite1.tmpl', 'Key': u'test-shared/sites/testsite1/template'},
+  {'Value': 'test.example.com', 'Key': u'test-shared/sites/testsite1/uri'},
+  {'Value': None, 'Key': u'test-shared/sites/testsite1/'},
+  {'Value': None, 'Key': u'test-shared/sites/'},
+  {'Value': 'Test User', 'Key': u'test-shared/user/full_name'},
+  {'Value': 'adm\nwww-data\nmlocate', 'Key': u'test-shared/user/groups'},
+  {'Value': None, 'Key': u'test-shared/user/blankvalue'},
+  {'Value': 'test', 'Key': u'test-shared/user/login'},
+  {'Value': None, 'Key': u'test-shared/user/'}
+]
+
+SIMPLE_DICT = { 'key1': { 'key2': 'val1'} }
+
+#OPTS =
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+@skipIf(not consul_pillar.HAS_CONSUL, 'no consul-python')
+class ConsulPillarTestCase(TestCase):
+  '''
+  Test cases for salt.pillar.consul_pillar
+  '''
+  #@patch(consul_pillar.ext_pillar, '__builtin__.__opts__', {'consul_config': {'consul.port': 8500, 'consul.host': '172.17.0.15'}})
+  def get_pillar(self):
+    consul_pillar.get_conn = MagicMock(return_value='consul_connection')
+    #consul_pillar.consul_fetch = MagicMock(return_value=('2232', PILLAR_DATA))
+    #with patch.dict(consul_pillar.__opts__, {'consul_config': {'consul.port': 8500, 'consul.host': '172.17.0.15'}}):
+    pillar = consul_pillar
+    return pillar
+
+  def test_connection(self):
+    pillar = self.get_pillar()
+    with patch.object(consul_pillar, 'consul_fetch', MagicMock(return_value=('2232', PILLAR_DATA))):
+      pillar_data = pillar.ext_pillar('testminion', {}, 'consul_config root=test-shared/')
+      pillar.get_conn.assert_called_once_with(OPTS, 'consul_config')
+
+  def test_pillar_data(self):
+    pillar = self.get_pillar()
+    with patch.object(consul_pillar, 'consul_fetch', MagicMock(return_value=('2232', PILLAR_DATA))):
+      pillar_data = pillar.ext_pillar('testminion', {}, 'consul_config root=test-shared/')
+      pillar.consul_fetch.assert_called_once_with('consul_connection', 'test-shared/')
+      assert pillar_data.keys() == [u'user', u'sites']
+      #print(type(pillar_data[u'user'][u'groups']))
+      assert isinstance(pillar_data[u'user'][u'groups'], list)
+      self.assertNotIn('blankvalue', pillar_data[u'user'])
+
+  def test_dict_merge(self):
+    pillar = self.get_pillar()
+    test_dict = {}
+    with patch.dict(test_dict, SIMPLE_DICT):
+      print(pillar.dict_merge(test_dict, SIMPLE_DICT))
+      self.assertDictEqual(pillar.dict_merge(test_dict, SIMPLE_DICT), SIMPLE_DICT)
+    with patch.dict(test_dict, { 'key1': { 'key3': {'key4': 'value'}}}):
+      self.assertDictEqual(pillar.dict_merge(test_dict, SIMPLE_DICT), { 'key1': { 'key2': 'val1', 'key3': {'key4': 'value'}}})