From 60378b4d3da706615ee0a20a8868712bbba95b08 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Thu, 23 Apr 2015 14:40:05 -0600 Subject: [PATCH 1/3] Add REST driver for SDB --- salt/sdb/rest.py | 123 +++++++++++++++++++++++++++++++++++++++++++++++ salt/template.py | 62 +++++++++++++----------- 2 files changed, 157 insertions(+), 28 deletions(-) create mode 100644 salt/sdb/rest.py diff --git a/salt/sdb/rest.py b/salt/sdb/rest.py new file mode 100644 index 0000000000..0af36be3c3 --- /dev/null +++ b/salt/sdb/rest.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +''' +Generic REST API SDB Module + +:maintainer: SaltStack +:maturity: New +:platform: all + +.. versionadded:: Beryllium + +This module allows access to a REST interface using an ``sdb://`` URI. + +Like all sdb modules, the etcd module requires a configuration profile to +be configured in either the minion or master configuration file. This profile +requires very little. In the example: + +.. code-block:: yaml + + my-rest-api: + driver: rest + urls: + url: https://api.github.com/ + keys: + url: https://api.github.com/users/{{user}}/keys + requests_lib: True + +The ``driver`` refers to the REST module, and must be set to ``rest`` in order +to use this driver. Each of the other items inside this block refers to a +separate set of HTTP items, including a URL and any options associated with it. +The options used here should match the options available in +``salt.utils.http.query()``. + +In order to call the ``urls`` item in the example, the following reference can +be made inside a configuration file: + +.. code-block:: yaml + + github_urls: sdb://my-rest-api/urls + +Key/Value pairs may also be used with this driver, and merged into the URL using +the configured renderer (``jinja``, by default). For instance, in order to use +the ``keys`` item in the example, the following reference can be made: + +.. code-block:: yaml + + github_urls: sdb://my-rest-api/keys?user=myuser + +This will cause the following URL to actually be called: + +.. code-block:: yaml + + https://api.github.com/users/myuser/keys + +Key/Value pairs will NOT be passed through as GET data. If GET data needs to be +sent to the URL, then it should be configured in the SDB configuration block. +For instance: + +.. code-block:: yaml + + another-rest-api: + driver: rest + user_data: + url: https://api.example.com/users/ + params: + user: myuser +''' + +# import python libs +from __future__ import absolute_import +import logging + +import salt.loader +import salt.utils.http as http +from salt.template import compile_template + +log = logging.getLogger(__name__) + +__func_alias__ = { + 'set_': 'set' +} + +def set_(key, value, service=None, profile=None): # pylint: disable=W0613 + ''' + Set a key/value pair in the REST interface + ''' + return query(key, value, service, profile) + + +def get(key, service=None, profile=None): # pylint: disable=W0613 + ''' + Get a value from the REST interface + ''' + return query(key, None, service, profile) + + +def query(key, value=None, service=None, profile=None): # pylint: disable=W0613 + ''' + Get a value from the REST interface + ''' + comps = key.split('?') + key = comps[0] + key_vars = {} + for pair in comps[1].split('&'): + pair_key, pair_val = pair.split('=') + key_vars[pair_key] = pair_val + + renderer = __opts__.get('renderer', 'yaml_jinja') + rend = salt.loader.render(__opts__, {}) + url = compile_template( + ':string:', + rend, + renderer, + input_data=profile[key]['url'], + **key_vars + ) + + result = http.query( + url, + decode=True, + **key_vars + ) + + return result['dict'] diff --git a/salt/template.py b/salt/template.py index c24d7262b7..6515a358f8 100644 --- a/salt/template.py +++ b/salt/template.py @@ -31,6 +31,7 @@ def compile_template(template, default, saltenv='base', sls='', + input_data='', **kwargs): ''' Take the path to a template and return the high data structure @@ -50,29 +51,30 @@ def compile_template(template, _dont_call_warnings=True ) - # Template was specified incorrectly - if not isinstance(template, string_types): - log.error('Template was specified incorrectly: {0}'.format(template)) - return ret - # Template does not exist - if not os.path.isfile(template): - log.error('Template does not exist: {0}'.format(template)) - return ret - # Template is an empty file - if salt.utils.is_empty(template): - log.warn('Template is an empty file: {0}'.format(template)) - return ret + if template != ':string:': + # Template was specified incorrectly + if not isinstance(template, string_types): + log.error('Template was specified incorrectly: {0}'.format(template)) + return ret + # Template does not exist + if not os.path.isfile(template): + log.error('Template does not exist: {0}'.format(template)) + return ret + # Template is an empty file + if salt.utils.is_empty(template): + log.warn('Template is an empty file: {0}'.format(template)) + return ret + + with codecs.open(template, encoding=SLS_ENCODING) as ifile: + # data input to the first render function in the pipe + input_data = ifile.read() + if not input_data.strip(): + # Template is nothing but whitespace + log.error('Template is nothing but whitespace: {0}'.format(template)) + return ret # Get the list of render funcs in the render pipe line. - render_pipe = template_shebang(template, renderers, default) - - with codecs.open(template, encoding=SLS_ENCODING) as ifile: - # data input to the first render function in the pipe - input_data = ifile.read() - if not input_data.strip(): - # Template is nothing but whitespace - log.error('Template is nothing but whitespace: {0}'.format(template)) - return ret + render_pipe = template_shebang(template, renderers, default, input_data) input_data = string_io(input_data) for render, argline in render_pipe: @@ -117,7 +119,7 @@ def compile_template_str(template, renderers, default): return compile_template(fn_, renderers, default) -def template_shebang(template, renderers, default): +def template_shebang(template, renderers, default, input_data): ''' Check the template shebang line and return the list of renderers specified in the pipe. @@ -137,15 +139,19 @@ def template_shebang(template, renderers, default): ''' render_pipe = [] + line = '' # Open up the first line of the sls template - with salt.utils.fopen(template, 'r') as ifile: - line = ifile.readline() + if template == ':string:': + line = input_data.split()[0] + else: + with salt.utils.fopen(template, 'r') as ifile: + line = ifile.readline() - # Check if it starts with a shebang and not a path - if line.startswith('#!') and not line.startswith('#!/'): + # Check if it starts with a shebang and not a path + if line.startswith('#!') and not line.startswith('#!/'): - # pull out the shebang data - render_pipe = check_render_pipe_str(line.strip()[2:], renderers) + # pull out the shebang data + render_pipe = check_render_pipe_str(line.strip()[2:], renderers) if not render_pipe: render_pipe = check_render_pipe_str(default, renderers) From e5a186e86335fc5334281f59710128a8af68e09a Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Thu, 23 Apr 2015 14:41:04 -0600 Subject: [PATCH 2/3] Fixing etcd reference --- salt/sdb/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sdb/rest.py b/salt/sdb/rest.py index 0af36be3c3..13ad498943 100644 --- a/salt/sdb/rest.py +++ b/salt/sdb/rest.py @@ -10,7 +10,7 @@ Generic REST API SDB Module This module allows access to a REST interface using an ``sdb://`` URI. -Like all sdb modules, the etcd module requires a configuration profile to +Like all REST modules, the REST module requires a configuration profile to be configured in either the minion or master configuration file. This profile requires very little. In the example: From a6d2b3d51a11c42284c4e8546a494b51a3dea73f Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Thu, 23 Apr 2015 15:27:16 -0600 Subject: [PATCH 3/3] Lint --- salt/sdb/rest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sdb/rest.py b/salt/sdb/rest.py index 13ad498943..2fc55c044b 100644 --- a/salt/sdb/rest.py +++ b/salt/sdb/rest.py @@ -79,6 +79,7 @@ __func_alias__ = { 'set_': 'set' } + def set_(key, value, service=None, profile=None): # pylint: disable=W0613 ''' Set a key/value pair in the REST interface