add hg_pillar and corresponding unit test

salt/pillar/hg_pillar.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014 Floris Bruynooghe <flub@devork.be>
+
+"""Use remote Mercurial repository as a Pillar source
+
+The module depends on the ``hglib`` python module being available.
+This is the same requirement as for hgfs_ so should not pose any extra
+hurdles.
+
+This external Pillar source can be configued in the master config file as such:
+
+.. code-block:: yaml
+
+   ext_pillar:
+     - hg: ssh://hg@example.co/user/repo
+"""
+
+import copy
+import hashlib
+import logging
+import os
+
+import salt.pillar
+
+try:
+    import hglib
+except ImportError:
+    hglib = None
+
+
+__virtualname__ = 'hg'
+log = logging.getLogger(__name__)
+
+
+# The default option values
+__opts__ = {}
+
+
+def __virtual__():
+    """Only load if hglib is available"""
+    ext_pillar_sources = [x for x in __opts__.get('ext_pillar', [])]
+    if not any(['hg' in x for x in ext_pillar_sources]):
+        return False
+    if not hglib:
+        log.error('hglib not present')
+        return False
+    return __virtualname__
+
+
+def __init__(__opts__):
+    """Initialise
+
+    This is called every time a minion calls this external pillar.
+    """
+
+
+def ext_pillar(minion_id, pillar, repo, branch='default', root=None):
+    '''
+    Extract pillar from an hg repository
+    '''
+    with Repo(repo) as repo:
+        repo.update(branch)
+    envname = 'base' if branch == 'default' else branch
+    if root:
+        path = os.path.normpath(os.path.join(repo.working_dir, root))
+    else:
+        path = repo.working_dir
+
+    # Do not recurse, Pillar will call ext_pillar again!
+    if __opts__['pillar_roots'].get(envname, []) == [path]:
+        return {}
+
+    opts = copy.deepcopy(__opts__)
+    opts['pillar_roots'][envname] = [path]
+    pil = salt.pillar.Pillar(opts, __grains__, minion_id, envname)
+    return pil.compile_pillar()
+
+
+def update(repo_uri):
+    """Execute an hg pull on all the repos"""
+    with Repo(repo_uri) as repo:
+        repo.pull()
+
+
+def envs():
+    """Return a list of branches that can be used as environments"""
+
+
+def purge_cache():
+    """Purge the hg_pillar cache"""
+
+
+class Repo(object):
+    '''
+    Deal with remote hg (mercurial) repository for Pillar
+    '''
+
+    def __init__(self, repo_uri):
+        ''' Initialize a hg repo (or open it if it already exists) '''
+        self.repo_uri = repo_uri
+        cachedir = os.path.join(__opts__['cachedir'], 'hg_pillar')
+        hash_type = getattr(hashlib, __opts__.get('hash_type', 'md5'))
+        repo_hash = hash_type(repo_uri).hexdigest()
+        self.working_dir = os.path.join(cachedir, repo_hash)
+        if not os.path.isdir(self.working_dir):
+            self.repo = hglib.clone(repo_uri, self.working_dir)
+            self.repo.open()
+        else:
+            self.repo = hglib.open(self.working_dir)
+
+    def pull(self):
+        log.debug('Updating hg repo from hg_pillar module (pull)')
+        self.repo.pull()
+
+    def update(self, branch='default'):
+        ''' ensure we are using the latest revision in the hg repository '''
+        log.debug('Updating hg repo from hg_pillar module (pull)')
+        self.repo.pull()
+        log.debug('Updating hg repo from hg_pillar module (update)')
+        self.repo.update(branch, clean=True)
+
+    def close(self):
+        """Cleanup mercurial command server"""
+        self.repo.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()

tests/unit/pillar/hg_test.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+'''test for pillar hg_pillar.py'''
+
+# Import python libs
+from __future__ import absolute_import
+
+import os
+import tempfile
+import shutil
+import subprocess
+import yaml
+
+# Import Salt Testing libs
+from salttesting import TestCase, skipIf
+from salttesting.mock import NO_MOCK, NO_MOCK_REASON
+
+import integration
+
+COMMIT_USER_NAME = 'test_user'
+# file contents
+PILLAR_CONTENT = {'gna': 'hello'}
+FILE_DATA = {
+             'top.sls': {'base': {'*': ['user']}},
+             'user.sls': PILLAR_CONTENT
+             }
+
+# Import Salt Libs
+from salt.pillar import hg_pillar
+HGLIB = hg_pillar.hglib
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+@skipIf(HGLIB == None, 'python-hglib no')
+class HgPillarTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn):
+    'test hg_pillar pillar'
+    maxDiff = None
+
+    def setUp(self):
+        super(HgPillarTestCase, self).setUp()
+        self.tmpdir = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR)
+        cachedir = os.path.join(self.tmpdir, 'cachedir')
+        os.makedirs(os.path.join(cachedir, 'hg_pillar'))
+        self.hg_repo_path = self._create_hg_repo()
+        hg_pillar.__opts__ = {
+                'cachedir': cachedir,
+                'pillar_roots': {},
+                'file_roots': {},
+                'state_top': 'top.sls',
+                'extension_modules': '',
+                'renderer': 'yaml_jinja',
+                'pillar_opts': False
+                }
+        hg_pillar.__grains__ = {}
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+        super(HgPillarTestCase, self).tearDown()
+
+    def _create_hg_repo(self):
+        'create repo in tempdir'
+        hg_repo = os.path.join(self.tmpdir, 'repo_pillar')
+        os.makedirs(hg_repo)
+        subprocess.check_call(["hg", "init", hg_repo])
+        for filename in FILE_DATA:
+            with open(os.path.join(hg_repo, filename), 'w') as data_file:
+                yaml.dump(FILE_DATA[filename], data_file)
+        subprocess.check_call(['hg', 'ci', '-A', '-R', hg_repo, '-m', 'first commit', '-u', COMMIT_USER_NAME])
+        return hg_repo
+
+    def test_base(self):
+        'check hg repo is imported correctly'
+        mypillar = hg_pillar.ext_pillar('*', None, 'file://{0}'.format(self.hg_repo_path))
+        self.assertEqual(PILLAR_CONTENT, mypillar)