Vault: Add support for expanding lists in policies

salt/modules/vault.py

@@ -38,8 +38,25 @@ Functions to interact with Hashicorp Vault.
         Grains are also available, for example like this:
         ``my-policies/{grains[os]}``
 
-        Optional. If policies is not configured, saltstack/minions and
-        saltstack/{minion} are used as defaults.
+        If a template contains a grain which evaluates to a list, it will be
+        expanded into multiple policies. For example, given the template
+        ``saltstack/by-role/{grains[roles]}``, and a minion having these grains:
+
+        .. code-block: yaml
+
+            grains:
+                roles:
+                    - web
+                    - database
+
+        The minion will have the policies ``saltstack/by-role/web`` and
+        ``saltstack/by-role/database``. Note however that list members which do
+        not have simple string representations, such as dictionaries or objects,
+        do not work and will throw an exception. Strings and numbers are
+        examples of types which work well.
+
+        Optional. If policies is not configured, ``saltstack/minions`` and
+        ``saltstack/{minion}`` are used as defaults.
 
 
     Add this segment to the master configuration file, or

salt/runners/vault.py

@@ -11,6 +11,7 @@ documented in the execution module docs.
 from __future__ import absolute_import
 import base64
 import logging
+import string
 import requests
 
 import salt.crypt
@@ -117,7 +118,58 @@ def _get_policies(minion_id, config):
 
     policies = []
     for pattern in policy_patterns:
-        policies.append(pattern.format(**mappings))
+        for expanded_pattern in _expand_pattern_lists(pattern, **mappings):
+            policies.append(expanded_pattern.format(**mappings))
 
     log.debug('{0} policies: {1}'.format(minion_id, policies))
     return policies
+
+
+def _expand_pattern_lists(pattern, **mappings):
+    '''
+    Expands the pattern for any list-valued mappings, such that for any list of
+    length N in the mappings present in the pattern, N copies of the pattern are
+    returned, each with an element of the list substituted.
+
+    pattern:
+        A pattern to expand, for example ``by-role/{grains[roles]}``
+
+    mappings:
+        A dictionary of variables that can be expanded into the pattern.
+
+    Example: Given the pattern `` by-role/{grains[roles]}`` and the below grains
+
+    .. code-block:: yaml
+
+        grains:
+            roles:
+                - web
+                - database
+
+    This function will expand into two patterns,
+    ``[by-role/web, by-role/database]``.
+
+    Note that this method does not expand any non-list patterns.
+    '''
+    expanded_patterns = []
+    f = string.Formatter()
+    '''
+    This function uses a string.Formatter to get all the formatting tokens from
+    the pattern, then recursively replaces tokens whose expanded value is a
+    list. For a list with N items, it will create N new pattern strings and
+    then continue with the next token. In practice this is expected to not be
+    very expensive, since patterns will typically involve a handful of lists at
+    most.
+    '''  # pylint: disable=W0105
+    for (_, field_name, _, _) in f.parse(pattern):
+        if field_name is None:
+            continue
+        (value, _) = f.get_field(field_name, None, mappings)
+        if isinstance(value, list):
+            token = '{{{0}}}'.format(field_name)
+            expanded = [pattern.replace(token, str(elem)) for elem in value]
+            for expanded_item in expanded:
+                result = _expand_pattern_lists(expanded_item, **mappings)
+                expanded_patterns += result
+            return expanded_patterns
+    return [pattern]

tests/unit/runners/test_vault.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+'''
+Unit tests for the Vault runner
+'''
+
+# Import Python Libs
+from __future__ import absolute_import
+import logging
+from salt.ext import six
+from salt.runners import vault
+
+# Import Salt Testing Libs
+from salttesting import skipIf, TestCase
+from salttesting.helpers import ensure_in_syspath
+from salttesting.mock import (
+    MagicMock,
+    patch,
+    NO_MOCK,
+    NO_MOCK_REASON
+)
+
+
+log = logging.getLogger(__name__)
+ensure_in_syspath('../../')
+
+vault.__opts__ = {}
+
+
+class VaultTest(TestCase):
+    '''
+    Tests for the runner module of the Vault integration
+    '''
+
+    def setUp(self):
+        self.grains = {
+                        'id': 'test-minion',
+                        'roles': ['web', 'database'],
+                        'aux': ['foo', 'bar'],
+                        'deep': {
+                            'foo': {
+                                'bar': {
+                                    'baz': [
+                                        'hello',
+                                        'world'
+                                    ]
+                                }
+                            }
+                        },
+                        'dictlist': [
+                            {'foo': 'bar'},
+                            {'baz': 'qux'}
+                        ]
+                      }
+
+    def test_pattern_list_expander(self):
+        '''
+        Ensure _expand_pattern_lists works as intended:
+        - Expand list-valued patterns
+        - Do not change non-list-valued tokens
+        '''
+        cases = {
+                    'no-tokens-to-replace': ['no-tokens-to-replace'],
+                    'single-dict:{minion}': ['single-dict:{minion}'],
+                    'single-list:{grains[roles]}': ['single-list:web', 'single-list:database'],
+                    'multiple-lists:{grains[roles]}+{grains[aux]}': [
+                       'multiple-lists:web+foo',
+                       'multiple-lists:web+bar',
+                       'multiple-lists:database+foo',
+                       'multiple-lists:database+bar',
+                    ],
+                    'single-list-with-dicts:{grains[id]}+{grains[roles]}+{grains[id]}': [
+                        'single-list-with-dicts:{grains[id]}+web+{grains[id]}',
+                        'single-list-with-dicts:{grains[id]}+database+{grains[id]}'
+                    ],
+                    'deeply-nested-list:{grains[deep][foo][bar][baz]}': [
+                        'deeply-nested-list:hello',
+                        'deeply-nested-list:world'
+                    ]
+                }
+
+        # The mappings dict is assembled in _get_policies, so emulate here
+        mappings = {'minion': self.grains['id'], 'grains': self.grains}
+        for case, correct_output in six.iteritems(cases):
+            output = vault._expand_pattern_lists(case, **mappings)  # pylint: disable=protected-access
+            diff = set(output).symmetric_difference(set(correct_output))
+            if len(diff) != 0:
+                log.debug('Test {0} failed'.format(case))
+                log.debug('Expected:\n\t{0}\nGot\n\t{1}'.format(output, correct_output))
+                log.debug('Difference:\n\t{0}'.format(diff))
+            self.assertEqual(output, correct_output)
+
+    @skipIf(NO_MOCK, NO_MOCK_REASON)
+    def test_get_policies(self):
+        '''
+        Ensure _get_policies works as intended, including expansion of lists
+        '''
+        cases = {
+                    'no-tokens-to-replace': ['no-tokens-to-replace'],
+                    'single-dict:{minion}': ['single-dict:test-minion'],
+                    'single-list:{grains[roles]}': ['single-list:web', 'single-list:database'],
+                    'multiple-lists:{grains[roles]}+{grains[aux]}': [
+                       'multiple-lists:web+foo',
+                       'multiple-lists:web+bar',
+                       'multiple-lists:database+foo',
+                       'multiple-lists:database+bar',
+                    ],
+                    'single-list-with-dicts:{grains[id]}+{grains[roles]}+{grains[id]}': [
+                        'single-list-with-dicts:test-minion+web+test-minion',
+                        'single-list-with-dicts:test-minion+database+test-minion'
+                    ],
+                    'deeply-nested-list:{grains[deep][foo][bar][baz]}': [
+                        'deeply-nested-list:hello',
+                        'deeply-nested-list:world'
+                    ]
+                }
+
+        with patch('salt.utils.minions.get_minion_data',
+                   MagicMock(return_value=(None, self.grains, None))):
+            for case, correct_output in six.iteritems(cases):
+                test_config = {'policies': [case]}
+                output = vault._get_policies('test-minion', test_config)  # pylint: disable=protected-access
+                diff = set(output).symmetric_difference(set(correct_output))
+                if len(diff) != 0:
+                    log.debug('Test {0} failed'.format(case))
+                    log.debug('Expected:\n\t{0}\nGot\n\t{1}'.format(output, correct_output))
+                    log.debug('Difference:\n\t{0}'.format(diff))
+                self.assertEqual(output, correct_output)
+
+
+if __name__ == '__main__':
+    from integration import run_tests  # pylint: disable=import-error
+    run_tests(VaultTest, needs_daemon=False)