Merge pull request #41576 from m03/issue-41062-ext-http

Pass grains in http_json and http_yaml external pillar URLs

salt/pillar/http_json.py

@@ -6,7 +6,7 @@ A module that adds data to the Pillar structure retrieved by an http request
 Configuring the HTTP_JSON ext_pillar
 ====================================
 
-Set the following Salt config to setup Foreman as external pillar source:
+Set the following Salt config to setup http json result as external pillar source:
 
 .. code-block:: yaml
 
@@ -17,6 +17,16 @@ Set the following Salt config to setup Foreman as external pillar source:
         username: username
         password: password
 
+If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped
+in <> brackets) in the url in order to populate pillar data based on the grain value.
+
+.. code-block:: yaml
+
+  ext_pillar:
+    - http_json:
+        url: http://example.com/api/<nodename>
+        with_grains: True
+
 Module Documentation
 ====================
 '''
@@ -24,32 +34,61 @@ Module Documentation
 # Import python libs
 from __future__ import absolute_import
 import logging
+import re
 
 # Import Salt libs
-import salt.ext.six as six
+try:
+    from salt.ext.six.moves.urllib.parse import quote as _quote
+    _HAS_DEPENDENCIES = True
+except ImportError:
+    _HAS_DEPENDENCIES = False
+
+# Set up logging
+_LOG = logging.getLogger(__name__)
+
+
+def __virtual__():
+    return _HAS_DEPENDENCIES
 
 
 def ext_pillar(minion_id,
                pillar,  # pylint: disable=W0613
-               url=None):
+               url,
+               with_grains=False):
     '''
     Read pillar data from HTTP response.
 
-    :param url String to make request
+    :param str url: Url to request.
-    :returns dict with pillar data to add
+    :param bool with_grains: Whether to substitute strings in the url with their grain values.
-    :returns empty if error
-    '''
-    # Set up logging
-    log = logging.getLogger(__name__)
 
+    :return: A dictionary of the pillar data to add.
+    :rtype: dict
+    '''
+    grain_pattern = r'<(?P<grain_name>.*?)>'
+
+    if with_grains:
+        # Get the value of the grain and substitute each grain
+        # name for the url-encoded version of its grain value.
+        for match in re.finditer(grain_pattern, url):
+            grain_name = match.group('grain_name')
+            grain_value = __salt__['grains.get'](grain_name, None)
+
+            if not grain_value:
+                _LOG.error("Unable to get minion '%s' grain: %s", minion_id, grain_name)
+                return {}
+
+            grain_value = _quote(str(grain_value))
+            url = re.sub('<{0}>'.format(grain_name), grain_value, url)
+
+    _LOG.debug('Getting url: %s', url)
     data = __salt__['http.query'](url=url, decode=True, decode_type='json')
 
     if 'dict' in data:
         return data['dict']
 
-    log.error('Error caught on query to' + url + '\nMore Info:\n')
+    _LOG.error("Error on minion '%s' http query: %s\nMore Info:\n", minion_id, url)
 
-    for k, v in six.iteritems(data):
+    for key in data:
-        log.error(k + ' : ' + v)
+        _LOG.error('%s: %s', key, data[key])
 
     return {}

salt/pillar/http_yaml.py

@@ -17,6 +17,16 @@ Set the following Salt config to setup an http endpoint as the external pillar s
         username: username
         password: password
 
+If the with_grains parameter is set, grain keys wrapped in can be provided (wrapped
+in <> brackets) in the url in order to populate pillar data based on the grain value.
+
+.. code-block:: yaml
+
+  ext_pillar:
+    - http_yaml:
+        url: http://example.com/api/<nodename>
+        with_grains: True
+
 Module Documentation
 ====================
 '''
@@ -24,32 +34,62 @@ Module Documentation
 # Import python libs
 from __future__ import absolute_import
 import logging
+import re
 
 # Import Salt libs
-import salt.ext.six as six
+try:
+    from salt.ext.six.moves.urllib.parse import quote as _quote
+    _HAS_DEPENDENCIES = True
+except ImportError:
+    _HAS_DEPENDENCIES = False
+
+
+# Set up logging
+_LOG = logging.getLogger(__name__)
+
+
+def __virtual__():
+    return _HAS_DEPENDENCIES
 
 
 def ext_pillar(minion_id,
                pillar,  # pylint: disable=W0613
-               url):
+               url,
-    """
+               with_grains=False):
+    '''
     Read pillar data from HTTP response.
 
-    :param url String to make request
+    :param str url: Url to request.
-    :returns dict with pillar data to add
+    :param bool with_grains: Whether to substitute strings in the url with their grain values.
-    :returns empty if error
-    """
-    # Set up logging
-    log = logging.getLogger(__name__)
 
+    :return: A dictionary of the pillar data to add.
+    :rtype: dict
+    '''
+    grain_pattern = r'<(?P<grain_name>.*?)>'
+
+    if with_grains:
+        # Get the value of the grain and substitute each grain
+        # name for the url-encoded version of its grain value.
+        for match in re.finditer(grain_pattern, url):
+            grain_name = match.group('grain_name')
+            grain_value = __salt__['grains.get'](grain_name, None)
+
+            if not grain_value:
+                _LOG.error("Unable to get minion '%s' grain: %s", minion_id, grain_name)
+                return {}
+
+            grain_value = _quote(str(grain_value))
+            url = re.sub('<{0}>'.format(grain_name), grain_value, url)
+
+    _LOG.debug('Getting url: %s', url)
     data = __salt__['http.query'](url=url, decode=True, decode_type='yaml')
 
     if 'dict' in data:
         return data['dict']
 
-    log.error('Error caught on query to' + url + '\nMore Info:\n')
+    _LOG.error("Error on minion '%s' http query: %s\nMore Info:\n", minion_id, url)
 
-    for k, v in six.iteritems(data):
+    for key in data:
-        log.error(k + ' : ' + v)
+        _LOG.error('%s: %s', key, data[key])
 
     return {}