diff --git a/salt/pillar/pepa.py b/salt/pillar/pepa.py old mode 100755 new mode 100644 index 42aa29f9c5..d051078c6d --- a/salt/pillar/pepa.py +++ b/salt/pillar/pepa.py @@ -1,15 +1,271 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- ''' -Configuration templating using Hierarchical substitution and Jinja. +Pepa +==== -Documentation: https://github.com/mickep76/pepa +Configuration templating for SaltStack using Hierarchical substitution and Jinja. + +Configuring Pepa +================ + +.. code-block:: yaml + + extension_modules: /srv/salt/ext + + ext_pillar: + - pepa: + resource: host # Name of resource directory and sub-key in pillars + sequence: # Sequence used for hierarchical substitution + - hostname: # Name of key + name: input # Alias used for template directory + base_only: True # Only use templates from Base environment, i.e. no staging + - default: + - environment: + - location..region: + name: region + - location..country: + name: country + - location..datacenter: + name: datacenter + - roles: + - osfinger: + name: os + - hostname: + name: override + base_only: True + subkey: True # Create a sub-key in pillars, named after the resource in this case [host] + subkey_only: True # Only create a sub-key, and leave the top level untouched + + pepa_roots: # Base directory for each environment + base: /srv/pepa/base # Path for base environment + dev: /srv/pepa/base # Associate dev with base + qa: /srv/pepa/qa + prod: /srv/pepa/prod + + # Use a different delimiter for nested dictionaries, defaults to '..' since some keys may use '.' in the name + #pepa_delimiter: .. + + # Supply Grains for Pepa, this should **ONLY** be used for testing or validation + #pepa_grains: + # environment: dev + + # Supply Pillar for Pepa, this should **ONLY** be used for testing or validation + #pepa_pillars: + # saltversion: 0.17.4 + + # Enable debug for Pepa, and keep Salt on warning + #log_level: debug + + #log_granular_levels: + # salt: warning + # salt.loaded.ext.pillar.pepa: debug + +Pepa can also be used in Master-less SaltStack setup. + +Command line +============ + +.. code-block:: bash + + usage: pepa.py [-h] [-c CONFIG] [-d] [-g GRAINS] [-p PILLAR] [-n] [-v] + hostname + + positional arguments: + hostname Hostname + + optional arguments: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + Configuration file + -d, --debug Print debug info + -g GRAINS, --grains GRAINS + Input Grains as YAML + -p PILLAR, --pillar PILLAR + Input Pillar as YAML + -n, --no-color No color output + -v, --validate Validate output + +Templates +========= + +Templates is configuration for a host or software, that can use information from Grains or Pillars. These can then be used for hierarchically substitution. + +**Example File:** host/input/test_example_com.yaml + +.. code-block:: yaml + + location..region: emea + location..country: nl + location..datacenter: foobar + environment: dev + roles: + - salt.master + network..gateway: 10.0.0.254 + network..interfaces..eth0..hwaddr: 00:20:26:a1:12:12 + network..interfaces..eth0..dhcp: False + network..interfaces..eth0..ipv4: 10.0.0.3 + network..interfaces..eth0..netmask: 255.255.255.0 + network..interfaces..eth0..fqdn: {{ hostname }} + cobbler..profile: fedora-19-x86_64 + +As you see in this example you can use Jinja directly inside the template. + +**Example File:** host/region/amer.yaml + +.. code-block:: yaml + + network..dns..servers: + - 10.0.0.1 + - 10.0.0.2 + time..ntp..servers: + - ntp1.amer.example.com + - ntp2.amer.example.com + - ntp3.amer.example.com + time..timezone: America/Chihuahua + yum..mirror: yum.amer.example.com + +Each template is named after the value of the key using lowercase and all extended characters are replaced with underscore. + +**Example:** + +osfinger: Fedora-19 + +**Would become:** + +fedora_19.yaml + +Nested dictionaries +=================== + +In order to create nested dictionaries as output you can use double dot **".."** as a delimiter. You can change this using "pepa_delimiter" we choose double dot since single dot is already used by key names in some modules, and using ":" requires quoting in the YAML. + +**Example:** + +.. code-block:: yaml + + network..dns..servers: + - 10.0.0.1 + - 10.0.0.2 + network..dns..options: + - timeout:2 + - attempts:1 + - ndots:1 + network..dns..search: + - example.com + +**Would become:** + +.. code-block:: yaml + + network: + dns: + servers: + - 10.0.0.1 + - 10.0.0.2 + options: + - timeout:2 + - attempts:1 + - ndots:1 + search: + - example.com + +Operators +========= + +Operators can be used to merge/unset a list/hash or set the key as immutable, so it can't be changed. + +=========== ================================================ +Operator Description +=========== ================================================ +merge() Merge list or hash +unset() Unset key +immutable() Set the key as immutable, so it can't be changed +imerge() Set immutable and merge +iunset() Set immutable and unset +=========== ================================================ + +**Example:** + +.. code-block:: yaml + + network..dns..search..merge(): + - foobar.com + - dummy.nl + owner..immutable(): Operations + host..printers..unset(): + +Validation +========== + +Since it's very hard to test Jinja as is, the best approach is to run all the permutations of input and validate the output, i.e. Unit Testing. + +To facilitate this in Pepa we use YAML, Jinja and Cerberus . + +Schema +====== + +So this is a validation schema for network configuration, as you see it can be customized with Jinja just as Pepa templates. + +This was designed to be run as a build job in Jenkins or similar tool. You can provide Grains/Pillar input using either the config file or command line arguments. + +**File Example: host/validation/network.yaml** + +.. code-block:: yaml + + network..dns..search: + type: list + allowed: + - example.com + + network..dns..options: + type: list + allowed: ['timeout:2', 'attempts:1', 'ndots:1'] + + network..dns..servers: + type: list + schema: + regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$ + + network..gateway: + type: string + regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$ + + {% if network.interfaces is defined %} + {% for interface in network.interfaces %} + + network..interfaces..{{ interface }}..dhcp: + type: boolean + + network..interfaces..{{ interface }}..fqdn: + type: string + regex: ^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-zA-Z]{2,6}$ + + network..interfaces..{{ interface }}..hwaddr: + type: string + regex: ^([0-9a-f]{1,2}\\:){5}[0-9a-f]{1,2}$ + + network..interfaces..{{ interface }}..ipv4: + type: string + regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$ + + network..interfaces..{{ interface }}..netmask: + type: string + regex: ^([0-9]{1,3}\\.){3}[0-9]{1,3}$ + + {% endfor %} + {% endif %} + +Links +===== + +For more examples and information see . ''' __author__ = 'Michael Persson ' __copyright__ = 'Copyright (c) 2013 Michael Persson' __license__ = 'Apache License, Version 2.0' -__version__ = '0.6.4' +__version__ = '0.6.6' # Import python libs import logging @@ -18,7 +274,7 @@ import glob import yaml import jinja2 import re - +from os.path import isfile, join # Only used when called from a terminal log = None @@ -32,6 +288,11 @@ if __name__ == '__main__': parser.add_argument('-g', '--grains', help='Input Grains as YAML') parser.add_argument('-p', '--pillar', help='Input Pillar as YAML') parser.add_argument('-n', '--no-color', action='store_true', help='No color output') + parser.add_argument('-v', '--validate', action='store_true', help='Validate output') + parser.add_argument('-q', '--query-api', action='store_true', help='Query Saltstack REST API for Grains') + parser.add_argument('--url', default='https://salt:8000', help='URL for SaltStack REST API') + parser.add_argument('-u', '--username', help='Username for SaltStack REST API') + parser.add_argument('-P', '--password', help='Password for SaltStack REST API') args = parser.parse_args() LOG_LEVEL = logging.WARNING @@ -64,23 +325,15 @@ __opts__ = { 'pepa_roots': { 'base': '/srv/salt' }, - 'pepa_delimiter': '..' + 'pepa_delimiter': '..', + 'pepa_validate': False } -try: - from os.path import isfile, join - HAS_OS_PATH = True -except ImportError: - HAS_OS_PATH = False - def __virtual__(): ''' Only return if all the modules are available ''' - if not HAS_OS_PATH: - return False - return True @@ -150,16 +403,24 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only= entries = [inp[categ]] for entry in entries: + results_jinja = None results = None fn = join(templdir, re.sub(r'\W', '_', entry.lower()) + '.yaml') if isfile(fn): log.info("Loading template: {0}".format(fn)) template = jinja2.Template(open(fn).read()) output['pepa_templates'].append(fn) - data = key_value_to_tree(output) - data['grains'] = __grains__.copy() - data['pillar'] = pillar.copy() - results = yaml.load(template.render(data)) + + try: + data = key_value_to_tree(output) + data['grains'] = __grains__.copy() + data['pillar'] = pillar.copy() + results_jinja = template.render(data) + results = yaml.load(results_jinja) + except jinja2.UndefinedError, err: + log.error('Failed to parse JINJA template: {0}\n{1}'.format(fn, err)) + except yaml.YAMLError, err: + log.error('Failed to parse YAML in template: {0}\n{1}'.format(fn, err)) else: log.info("Template doesn't exist: {0}".format(fn)) continue @@ -177,46 +438,36 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only= log.warning('Key {0} is immutable, changes are not allowed'.format(key)) elif rkey in immutable: log.warning("Key {0} is immutable, changes are not allowed".format(rkey)) - elif operator == 'merge()': - log.debug("Merge key {0}: {1}".format(rkey, results[key])) - if rkey in output and type(results[key]) != type(output[rkey]): - log.warning('You can''t merge different types for key {0}'.format(rkey)) + elif operator == 'merge()' or operator == 'imerge()': + if operator == 'merge()': + log.debug("Merge key {0}: {1}".format(rkey, results[key])) + else: + log.debug("Set immutable and merge key {0}: {1}".format(rkey, results[key])) + immutable[rkey] = True + if rkey not in output: + log.error('Cant\'t merge key {0} doesn\'t exist'.format(rkey)) + elif type(results[key]) != type(output[rkey]): + log.error('Can\'t merge different types for key {0}'.format(rkey)) elif type(results[key]) is dict: output[rkey].update(results[key]) elif type(results[key]) is list: output[rkey].extend(results[key]) else: - log.warning('Unsupported type need to be list or dict for key {0}'.format(rkey)) - elif operator == 'unset()': - log.debug("Unset key {0}".format(rkey)) - try: + log.error('Unsupported type need to be list or dict for key {0}'.format(rkey)) + elif operator == 'unset()' or operator == 'iunset()': + if operator == 'unset()': + log.debug("Unset key {0}".format(rkey)) + else: + log.debug("Set immutable and unset key {0}".format(rkey)) + immutable[rkey] = True + if rkey in output: del output[rkey] - except KeyError: - pass elif operator == 'immutable()': log.debug("Set immutable and substitute key {0}: {1}".format(rkey, results[key])) immutable[rkey] = True output[rkey] = results[key] - elif operator == 'imerge()': - log.debug("Set immutable and merge key {0}: {1}".format(rkey, results[key])) - immutable[rkey] = True - if rkey in output and type(results[key]) != type(output[rkey]): - log.warning('You can''t merge different types for key {0}'.format(rkey)) - elif type(results[key]) is dict: - output[rkey].update(results[key]) - elif type(results[key]) is list: - output[rkey].extend(results[key]) - else: - log.warning('Unsupported type need to be list or dict for key {0}'.format(rkey)) - elif operator == 'iunset()': - log.debug("Set immutable and unset key {0}".format(rkey)) - immutable[rkey] = True - try: - del output[rkey] - except KeyError: - pass elif operator is not None: - log.warning('Unsupported operator {0}, skipping key {1}'.format(operator, rkey)) + log.error('Unsupported operator {0}, skipping key {1}'.format(operator, rkey)) else: log.debug("Substitute key {0}: {1}".format(key, results[key])) output[key] = results[key] @@ -230,6 +481,8 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only= pillar_data[resource] = tree.copy() else: pillar_data = tree + if __opts__['pepa_validate']: + pillar_data['pepa_keys'] = output.copy() return pillar_data @@ -298,8 +551,55 @@ if __name__ == '__main__': if args.pillar: __pillar__.update(yaml.load(args.pillar)) + # Validate or not + if args.validate: + __opts__['pepa_validate'] = True + + if args.query_api: + import requests + import getpass + + username = args.username + password = args.password + if username is None: + username = raw_input('Username: ') + if password is None: + password = getpass.getpass() + + log.info('Authenticate REST API') + auth = {'username': username, 'password': password, 'eauth': 'pam'} + request = requests.post(args.url + '/login', auth) + + if not request.ok: + raise RuntimeError('Failed to authenticate to SaltStack REST API: {0}'.format(request.text)) + + response = request.json() + token = response['return'][0]['token'] + + log.info('Request Grains from REST API') + headers = {'X-Auth-Token': token, 'Accept': 'application/json'} + request = requests.get(args.url + '/minions/' + args.hostname, headers=headers) + + result = request.json().get('return', [{}])[0] + if args.hostname not in result: + raise RuntimeError('Failed to get Grains from SaltStack REST API') + + __grains__ = result[args.hostname] +# print yaml.safe_dump(__grains__, indent=4, default_flow_style=False) + # Print results - result = ext_pillar(args.hostname, __pillar__, __opts__['ext_pillar'][loc]['pepa']['resource'], __opts__['ext_pillar'][loc]['pepa']['sequence']) + ex_subkey = False + ex_subkey_only = False + if 'subkey' in __opts__['ext_pillar'][loc]['pepa']: + ex_subkey = __opts__['ext_pillar'][loc]['pepa']['subkey'] + if 'subkey_only' in __opts__['ext_pillar'][loc]['pepa']: + ex_subkey_only = __opts__['ext_pillar'][loc]['pepa']['subkey_only'] + + result = ext_pillar(args.hostname, __pillar__, __opts__['ext_pillar'][loc]['pepa']['resource'], + __opts__['ext_pillar'][loc]['pepa']['sequence'], ex_subkey, ex_subkey_only) + + if __opts__['pepa_validate']: + validate(result, __opts__['ext_pillar'][loc]['pepa']['resource']) yaml.dumper.SafeDumper.ignore_aliases = lambda self, data: True if not args.no_color: