Move pillar/pepa.py from develop to 2014.7

salt/pillar/pepa.py

@@ -1,15 +1,271 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 '''
-Configuration templating using Hierarchical substitution and Jinja.
+Pepa
+====
 
-Documentation: https://github.com/mickep76/pepa
+Configuration templating for SaltStack using Hierarchical substitution and Jinja.
+
+Configuring Pepa
+================
+
+.. code-block:: yaml
+
+    extension_modules: /srv/salt/ext
+
+    ext_pillar:
+      - pepa:
+          resource: host                # Name of resource directory and sub-key in pillars
+          sequence:                     # Sequence used for hierarchical substitution
+            - hostname:                 # Name of key
+                name: input             # Alias used for template directory
+                base_only: True         # Only use templates from Base environment, i.e. no staging
+            - default:
+            - environment:
+            - location..region:
+                name: region
+            - location..country:
+                name: country
+            - location..datacenter:
+                name: datacenter
+            - roles:
+            - osfinger:
+                name: os
+            - hostname:
+                name: override
+                base_only: True
+          subkey: True                  # Create a sub-key in pillars, named after the resource in this case [host]
+          subkey_only: True             # Only create a sub-key, and leave the top level untouched
+
+    pepa_roots:                         # Base directory for each environment
+      base: /srv/pepa/base              # Path for base environment
+      dev: /srv/pepa/base               # Associate dev with base
+      qa: /srv/pepa/qa
+      prod: /srv/pepa/prod
+
+    # Use a different delimiter for nested dictionaries, defaults to '..' since some keys may use '.' in the name
+    #pepa_delimiter: ..
+
+    # Supply Grains for Pepa, this should **ONLY** be used for testing or validation
+    #pepa_grains:
+    #  environment: dev
+
+    # Supply Pillar for Pepa, this should **ONLY** be used for testing or validation
+    #pepa_pillars:
+    #  saltversion: 0.17.4
+
+    # Enable debug for Pepa, and keep Salt on warning
+    #log_level: debug
+
+    #log_granular_levels:
+    #  salt: warning
+    #  salt.loaded.ext.pillar.pepa: debug
+
+Pepa can also be used in Master-less SaltStack setup.
+
+Command line
+============
+
+.. code-block:: bash
+
+    usage: pepa.py [-h] [-c CONFIG] [-d] [-g GRAINS] [-p PILLAR] [-n] [-v]
+                   hostname
+
+    positional arguments:
+      hostname              Hostname
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -c CONFIG, --config CONFIG
+                            Configuration file
+      -d, --debug           Print debug info
+      -g GRAINS, --grains GRAINS
+                            Input Grains as YAML
+      -p PILLAR, --pillar PILLAR
+                            Input Pillar as YAML
+      -n, --no-color        No color output
+      -v, --validate        Validate output
+
+Templates
+=========
+
+Templates is configuration for a host or software, that can use information from Grains or Pillars. These can then be used for hierarchically substitution.
+
+**Example File:** host/input/test_example_com.yaml
+
+.. code-block:: yaml
+
+    location..region: emea
+    location..country: nl
+    location..datacenter: foobar
+    environment: dev
+    roles:
+      - salt.master
+    network..gateway: 10.0.0.254
+    network..interfaces..eth0..hwaddr: 00:20:26:a1:12:12
+    network..interfaces..eth0..dhcp: False
+    network..interfaces..eth0..ipv4: 10.0.0.3
+    network..interfaces..eth0..netmask: 255.255.255.0
+    network..interfaces..eth0..fqdn: {{ hostname }}
+    cobbler..profile: fedora-19-x86_64
+
+As you see in this example you can use Jinja directly inside the template.
+
+**Example File:** host/region/amer.yaml
+
+.. code-block:: yaml
+
+    network..dns..servers:
+      - 10.0.0.1
+      - 10.0.0.2
+    time..ntp..servers:
+      - ntp1.amer.example.com
+      - ntp2.amer.example.com
+      - ntp3.amer.example.com
+    time..timezone: America/Chihuahua
+    yum..mirror: yum.amer.example.com
+
+Each template is named after the value of the key using lowercase and all extended characters are replaced with underscore.
+
+**Example:**
+
+osfinger: Fedora-19
+
+**Would become:**
+
+fedora_19.yaml
+
+Nested dictionaries
+===================
+
+In order to create nested dictionaries as output you can use double dot **".."** as a delimiter. You can change this using "pepa_delimiter" we choose double dot since single dot is already used by key names in some modules, and using ":" requires quoting in the YAML.
+
+**Example:**
+
+.. code-block:: yaml
+
+    network..dns..servers:
+      - 10.0.0.1
+      - 10.0.0.2
+    network..dns..options:
+      - timeout:2
+      - attempts:1
+      - ndots:1
+    network..dns..search:
+      - example.com
+
+**Would become:**
+
+.. code-block:: yaml
+
+    network:
+      dns:
+        servers:
+          - 10.0.0.1
+          - 10.0.0.2
+        options:
+          - timeout:2
+          - attempts:1
+          - ndots:1
+        search:
+          - example.com
+
+Operators
+=========
+
+Operators can be used to merge/unset a list/hash or set the key as immutable, so it can't be changed.
+
+=========== ================================================
+Operator    Description
+=========== ================================================
+merge()     Merge list or hash
+unset()     Unset key
+immutable() Set the key as immutable, so it can't be changed
+imerge()    Set immutable and merge
+iunset()    Set immutable and unset
+=========== ================================================
+
+**Example:**
+
+.. code-block:: yaml
+
+    network..dns..search..merge():
+      - foobar.com
+      - dummy.nl
+    owner..immutable(): Operations
+    host..printers..unset():
+
+Validation
+==========
+
+Since it's very hard to test Jinja as is, the best approach is to run all the permutations of input and validate the output, i.e. Unit Testing.
+
+To facilitate this in Pepa we use YAML, Jinja and Cerberus <https://github.com/nicolaiarocci/cerberus>.
+
+Schema
+======
+
+So this is a validation schema for network configuration, as you see it can be customized with Jinja just as Pepa templates.
+
+This was designed to be run as a build job in Jenkins or similar tool. You can provide Grains/Pillar input using either the config file or command line arguments.
+
+**File Example: host/validation/network.yaml**
+
+.. code-block:: yaml
+
+    network..dns..search:
+      type: list
+      allowed:
+        - example.com
+
+    network..dns..options:
+      type: list
+      allowed: ['timeout:2', 'attempts:1', 'ndots:1']
+
+    network..dns..servers:
+      type: list
+      schema:
+        regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$
+
+    network..gateway:
+      type: string
+      regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$
+
+    {% if network.interfaces is defined %}
+    {% for interface in network.interfaces %}
+
+    network..interfaces..{{ interface }}..dhcp:
+      type: boolean
+
+    network..interfaces..{{ interface }}..fqdn:
+      type: string
+      regex: ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-zA-Z]{2,6}$
+
+    network..interfaces..{{ interface }}..hwaddr:
+      type: string
+      regex: ^([0-9a-f]{1,2}\\:){5}[0-9a-f]{1,2}$
+
+    network..interfaces..{{ interface }}..ipv4:
+      type: string
+      regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$
+
+    network..interfaces..{{ interface }}..netmask:
+      type: string
+      regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$
+
+    {% endfor %}
+    {% endif %}
+
+Links
+=====
+
+For more examples and information see <https://github.com/mickep76/pepa>.
 '''
 
 __author__ = 'Michael Persson <michael.ake.persson@gmail.com>'
 __copyright__ = 'Copyright (c) 2013 Michael Persson'
 __license__ = 'Apache License, Version 2.0'
-__version__ = '0.6.4'
+__version__ = '0.6.6'
 
 # Import python libs
 import logging
@@ -18,7 +274,7 @@ import glob
 import yaml
 import jinja2
 import re
-
+from os.path import isfile, join
 
 # Only used when called from a terminal
 log = None
@@ -32,6 +288,11 @@ if __name__ == '__main__':
     parser.add_argument('-g', '--grains', help='Input Grains as YAML')
     parser.add_argument('-p', '--pillar', help='Input Pillar as YAML')
     parser.add_argument('-n', '--no-color', action='store_true', help='No color output')
+    parser.add_argument('-v', '--validate', action='store_true', help='Validate output')
+    parser.add_argument('-q', '--query-api', action='store_true', help='Query Saltstack REST API for Grains')
+    parser.add_argument('--url', default='https://salt:8000', help='URL for SaltStack REST API')
+    parser.add_argument('-u', '--username', help='Username for SaltStack REST API')
+    parser.add_argument('-P', '--password', help='Password for SaltStack REST API')
     args = parser.parse_args()
 
     LOG_LEVEL = logging.WARNING
@@ -64,23 +325,15 @@ __opts__ = {
     'pepa_roots': {
         'base': '/srv/salt'
     },
-    'pepa_delimiter': '..'
+    'pepa_delimiter': '..',
+    'pepa_validate': False
 }
 
-try:
-    from os.path import isfile, join
-    HAS_OS_PATH = True
-except ImportError:
-    HAS_OS_PATH = False
-
 
 def __virtual__():
     '''
     Only return if all the modules are available
     '''
-    if not HAS_OS_PATH:
-        return False
-
     return True
 
 
@@ -150,16 +403,24 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only=
             entries = [inp[categ]]
 
         for entry in entries:
+            results_jinja = None
             results = None
             fn = join(templdir, re.sub(r'\W', '_', entry.lower()) + '.yaml')
             if isfile(fn):
                 log.info("Loading template: {0}".format(fn))
                 template = jinja2.Template(open(fn).read())
                 output['pepa_templates'].append(fn)
-                data = key_value_to_tree(output)
+
-                data['grains'] = __grains__.copy()
+                try:
-                data['pillar'] = pillar.copy()
+                    data = key_value_to_tree(output)
-                results = yaml.load(template.render(data))
+                    data['grains'] = __grains__.copy()
+                    data['pillar'] = pillar.copy()
+                    results_jinja = template.render(data)
+                    results = yaml.load(results_jinja)
+                except jinja2.UndefinedError, err:
+                    log.error('Failed to parse JINJA template: {0}\n{1}'.format(fn, err))
+                except yaml.YAMLError, err:
+                    log.error('Failed to parse YAML in template: {0}\n{1}'.format(fn, err))
             else:
                 log.info("Template doesn't exist: {0}".format(fn))
                 continue
@@ -177,46 +438,36 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only=
                         log.warning('Key {0} is immutable, changes are not allowed'.format(key))
                     elif rkey in immutable:
                         log.warning("Key {0} is immutable, changes are not allowed".format(rkey))
-                    elif operator == 'merge()':
+                    elif operator == 'merge()' or operator == 'imerge()':
-                        log.debug("Merge key {0}: {1}".format(rkey, results[key]))
+                        if operator == 'merge()':
-                        if rkey in output and type(results[key]) != type(output[rkey]):
+                            log.debug("Merge key {0}: {1}".format(rkey, results[key]))
-                            log.warning('You can''t merge different types for key {0}'.format(rkey))
+                        else:
+                            log.debug("Set immutable and merge key {0}: {1}".format(rkey, results[key]))
+                            immutable[rkey] = True
+                        if rkey not in output:
+                            log.error('Cant\'t merge key {0} doesn\'t exist'.format(rkey))
+                        elif type(results[key]) != type(output[rkey]):
+                            log.error('Can\'t merge different types for key {0}'.format(rkey))
                         elif type(results[key]) is dict:
                             output[rkey].update(results[key])
                         elif type(results[key]) is list:
                             output[rkey].extend(results[key])
                         else:
-                            log.warning('Unsupported type need to be list or dict for key {0}'.format(rkey))
+                            log.error('Unsupported type need to be list or dict for key {0}'.format(rkey))
-                    elif operator == 'unset()':
+                    elif operator == 'unset()' or operator == 'iunset()':
-                        log.debug("Unset key {0}".format(rkey))
+                        if operator == 'unset()':
-                        try:
+                            log.debug("Unset key {0}".format(rkey))
+                        else:
+                            log.debug("Set immutable and unset key {0}".format(rkey))
+                            immutable[rkey] = True
+                        if rkey in output:
                             del output[rkey]
-                        except KeyError:
-                            pass
                     elif operator == 'immutable()':
                         log.debug("Set immutable and substitute key {0}: {1}".format(rkey, results[key]))
                         immutable[rkey] = True
                         output[rkey] = results[key]
-                    elif operator == 'imerge()':
-                        log.debug("Set immutable and merge key {0}: {1}".format(rkey, results[key]))
-                        immutable[rkey] = True
-                        if rkey in output and type(results[key]) != type(output[rkey]):
-                            log.warning('You can''t merge different types for key {0}'.format(rkey))
-                        elif type(results[key]) is dict:
-                            output[rkey].update(results[key])
-                        elif type(results[key]) is list:
-                            output[rkey].extend(results[key])
-                        else:
-                            log.warning('Unsupported type need to be list or dict for key {0}'.format(rkey))
-                    elif operator == 'iunset()':
-                        log.debug("Set immutable and unset key {0}".format(rkey))
-                        immutable[rkey] = True
-                        try:
-                            del output[rkey]
-                        except KeyError:
-                            pass
                     elif operator is not None:
-                        log.warning('Unsupported operator {0}, skipping key {1}'.format(operator, rkey))
+                        log.error('Unsupported operator {0}, skipping key {1}'.format(operator, rkey))
                     else:
                         log.debug("Substitute key {0}: {1}".format(key, results[key]))
                         output[key] = results[key]
@@ -230,6 +481,8 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only=
         pillar_data[resource] = tree.copy()
     else:
         pillar_data = tree
+    if __opts__['pepa_validate']:
+        pillar_data['pepa_keys'] = output.copy()
     return pillar_data
 
 
@@ -298,8 +551,55 @@ if __name__ == '__main__':
     if args.pillar:
         __pillar__.update(yaml.load(args.pillar))
 
+    # Validate or not
+    if args.validate:
+        __opts__['pepa_validate'] = True
+
+    if args.query_api:
+        import requests
+        import getpass
+
+        username = args.username
+        password = args.password
+        if username is None:
+            username = raw_input('Username: ')
+        if password is None:
+            password = getpass.getpass()
+
+        log.info('Authenticate REST API')
+        auth = {'username': username, 'password': password, 'eauth': 'pam'}
+        request = requests.post(args.url + '/login', auth)
+
+        if not request.ok:
+            raise RuntimeError('Failed to authenticate to SaltStack REST API: {0}'.format(request.text))
+
+        response = request.json()
+        token = response['return'][0]['token']
+
+        log.info('Request Grains from REST API')
+        headers = {'X-Auth-Token': token, 'Accept': 'application/json'}
+        request = requests.get(args.url + '/minions/' + args.hostname, headers=headers)
+
+        result = request.json().get('return', [{}])[0]
+        if args.hostname not in result:
+            raise RuntimeError('Failed to get Grains from SaltStack REST API')
+
+        __grains__ = result[args.hostname]
+#        print yaml.safe_dump(__grains__, indent=4, default_flow_style=False)
+
     # Print results
-    result = ext_pillar(args.hostname, __pillar__, __opts__['ext_pillar'][loc]['pepa']['resource'], __opts__['ext_pillar'][loc]['pepa']['sequence'])
+    ex_subkey = False
+    ex_subkey_only = False
+    if 'subkey' in __opts__['ext_pillar'][loc]['pepa']:
+        ex_subkey = __opts__['ext_pillar'][loc]['pepa']['subkey']
+    if 'subkey_only' in __opts__['ext_pillar'][loc]['pepa']:
+        ex_subkey_only = __opts__['ext_pillar'][loc]['pepa']['subkey_only']
+
+    result = ext_pillar(args.hostname, __pillar__, __opts__['ext_pillar'][loc]['pepa']['resource'],
+                        __opts__['ext_pillar'][loc]['pepa']['sequence'], ex_subkey, ex_subkey_only)
+
+    if __opts__['pepa_validate']:
+        validate(result, __opts__['ext_pillar'][loc]['pepa']['resource'])
 
     yaml.dumper.SafeDumper.ignore_aliases = lambda self, data: True
     if not args.no_color: