valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-08 01:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #3320 from kjkuan/pydsl-updates

Browse Source 
Pydsl updates
This commit is contained in:
Thomas S Hatch 2013-01-18 11:27:16 -08:00
commit 08471effee
4 changed files with 268 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
salt/modules/pydsl.py
View File

@ -56,9 +56,8 @@ Example of a ``cmd`` state calling a python function::
#TODOs:
#
# - modify the stateconf renderer so that we can pipe pydsl to
# it to make use of its features, particularly the implicit
# ordering of states using ordered dict.
# - support exclude declarations
# - support include declarations with env
#
# - allow this:
# state('X').cmd.run.cwd = '/'
@ -84,6 +83,11 @@ REQUISITES = set("require watch use require_in watch_in use_in".split())
class PyDslError(Exception):
pass
class Options(dict):
def __getattr__(self, name):
return self.get(name)
def sls(sls):
return Sls(sls)
@ -97,15 +101,31 @@ class Sls(object):
self.includes = []
self.extends = []
self.decls = []
self.options = Options()
self.funcs = [] # track the ordering of state func declarations
def set(self, **options):
self.options.update(options)
def include(self, *sls_names):
self.includes.extend(sls_names)
def extend(self, *state_funcs):
if self.options.ordered and self.last_func():
raise PyDslError("Can't extend while the ordered option is turned on!")
for f in state_funcs:
self.extends.append(self.all_decls[f.mod._state_id])
for f in state_funcs:
self.decls.pop()
id = f.mod._state_id
self.extends.append(self.all_decls[id])
i = len(self.decls)
for decl in reversed(self.decls):
i -= 1
if decl._id == id:
del self.decls[i]
break
try:
self.funcs.remove(f) # untrack it
except ValueError:
pass
def state(self, id=None):
if not id:
@ -114,10 +134,16 @@ class Sls(object):
try:
return self.all_decls[id]
except KeyError:
self.all_decls[id] = s = StateDeclaration(id)
self.all_decls[id] = s = StateDeclaration(id, self)
self.decls.append(s)
return s
def last_func(self):
return self.funcs[-1] if self.funcs else None
def track_func(self, statefunc):
self.funcs.append(statefunc)
def to_highstate(self, slsmod=None):
# generate a state that uses the stateconf.set state, which
# is a no-op state, to hold a reference to the sls module
@ -159,7 +185,8 @@ class Sls(object):
class StateDeclaration(object):
def __init__(self, id=None):
def __init__(self, id, sls):
self._sls = sls
self._id = id
self._mods = []
@ -235,6 +262,13 @@ class StateFunction(object):
self.name = name
self.args = []
sls = Sls.all_decls[parent_mod._state_id]._sls
if sls.options.ordered:
last_f = sls.last_func()
if last_f:
self.require(last_f.mod)
sls.track_func(self)
def __call__(self, *args, **kws):
self.configure(args, kws)
return self

168
salt/renderers/stateconf.py
View File

@ -2,6 +2,10 @@
# for a guide to using this module.
#
# TODO:
#
# - support exclude declarations
# - support include declarations with env
#
# - sls meta/info state: Eg,
#
# sls_info:
@ -37,6 +41,9 @@ __opts__ = {
'stateconf_end_marker': r'#\s*-+\s*end of state config\s*-+',
# eg, something like "# --- end of state config --" works by default.
'stateconf_start_state': '.start',
# name of the state id for the generated start state.
'stateconf_goal_state': '.goal',
# name of the state id for the generated goal state.
@ -61,7 +68,7 @@ def __init__(opts):
MOD_BASENAME = ospath.basename(__file__)
INVALID_USAGE_ERROR = SaltRenderError(
'Invalid use of {0} renderer!\n'
'''Usage: #!{1} [-Go] <data_renderer> [options] . <template_renderer> [options]
'''Usage: #!{1} [-GoSp] [<data_renderer> [options] . <template_renderer> [options]]
where an example <data_renderer> would be yaml and a <template_renderer> might
be jinja. Each renderer can be passed its renderer specific options.
@ -72,46 +79,24 @@ Options(for this renderer):
-o Indirectly order the states by adding requires such that they will be
executed in the order they are defined in the sls. Implies using yaml -o.
-s Generate the start state that gets inserted as the first state in
the sls. This only makes sense if your high state data dict is ordered.
-p Assume high state input. This option allows you to pipe high state data
through this renderer. With this option, the use of stateconf.set state
in the sls will have no effect, but other features of the renderer still
apply.
'''.format(MOD_BASENAME, MOD_BASENAME)
)
def render(template_file, env='', sls='', argline='', **kws):
def render(input, env='', sls='', argline='', **kws):
gen_start_state = False
no_goal_state = False
implicit_require = False
renderers = kws['renderers']
opts, args = getopt.getopt(argline.split(), 'Go')
argline = ' '.join(args) if args else 'yaml . jinja'
if ('-G', '') in opts:
no_goal_state = True
# Split on the first dot surrounded by spaces but not preceded by a
# backslash. A backslash preceded dot will be replaced with just dot.
args = [
arg.strip().replace('\\.', '.')
for arg in re.split(r'\s+(?<!\\)\.\s+', argline, 1)
]
try:
name, rd_argline = (args[0] + ' ').split(' ', 1)
render_data = renderers[name] # eg, the yaml renderer
if ('-o', '') in opts:
if name == 'yaml':
implicit_require = True
rd_argline = '-o ' + rd_argline
else:
raise SaltRenderError(
'Implicit ordering is only supported if the yaml renderer '
'is used!'
)
name, rt_argline = (args[1] + ' ').split(' ', 1)
render_template = renderers[name] # eg, the mako renderer
except KeyError, err:
raise SaltRenderError('Renderer: {0} is not available!'.format(err))
except IndexError:
raise INVALID_USAGE_ERROR
def process_sls_data(data, context=None, extract=False):
sls_dir = ospath.dirname(sls.replace('.', ospath.sep))
ctx = dict(sls_dir=sls_dir if sls_dir else '.')
@ -124,7 +109,9 @@ def render(template_file, env='', sls='', argline='', **kws):
argline=rt_argline.strip(), **kws
)
high = render_data(tmplout, env, sls, argline=rd_argline.strip())
return process_high_data(high, extract)
def process_high_data(high, extract):
# make a copy so that the original, un-preprocessed highstate data
# structure can be used later for error checking if anything goes
# wrong during the preprocessing.
@ -144,6 +131,9 @@ def render(template_file, env='', sls='', argline='', **kws):
)
add_implicit_requires(data)
if gen_start_state:
add_start_state(data)
if not extract and not no_goal_state:
add_goal_state(data)
@ -153,6 +143,7 @@ def render(template_file, env='', sls='', argline='', **kws):
extract_state_confs(data)
except Exception, err:
raise
if isinstance(err, SaltRenderError):
raise
log.exception(
@ -166,34 +157,72 @@ def render(template_file, env='', sls='', argline='', **kws):
raise SaltRenderError('\n'.join(errors))
raise SaltRenderError('sls preprocessing/rendering failed!')
return data
#----------------------
renderers = kws['renderers']
opts, args = getopt.getopt(argline.split(), 'Gosp')
argline = ' '.join(args) if args else 'yaml . jinja'
if isinstance(template_file, basestring):
with salt.utils.fopen(template_file, 'r') as ifile:
sls_templ = ifile.read()
else: # assume file-like
sls_templ = template_file.read()
if ('-G', '') in opts:
no_goal_state = True
if ('-o', '') in opts:
implicit_require = True
if ('-s', '') in opts:
gen_start_state = True
# first pass to extract the state configuration
match = re.search(__opts__['stateconf_end_marker'], sls_templ)
if match:
process_sls_data(sls_templ[:match.start()], extract=True)
# if some config has been extracted then remove the sls-name prefix
# of the keys in the extracted stateconf.set context to make them easier
# to use in the salt file.
if STATE_CONF:
tmplctx = STATE_CONF.copy()
if tmplctx:
prefix = sls + '::'
for k in tmplctx.keys():
if k.startswith(prefix):
tmplctx[k[len(prefix):]] = tmplctx[k]
del tmplctx[k]
if ('-p', '') in opts:
data = process_high_data(input, extract=False)
else:
tmplctx = {}
# Split on the first dot surrounded by spaces but not preceded by a
# backslash. A backslash preceded dot will be replaced with just dot.
args = [
arg.strip().replace('\\.', '.')
for arg in re.split(r'\s+(?<!\\)\.\s+', argline, 1)
]
try:
name, rd_argline = (args[0] + ' ').split(' ', 1)
render_data = renderers[name] # eg, the yaml renderer
if implicit_require:
if name == 'yaml':
rd_argline = '-o ' + rd_argline
else:
raise SaltRenderError(
'Implicit ordering is only supported if the yaml renderer '
'is used!'
)
name, rt_argline = (args[1] + ' ').split(' ', 1)
render_template = renderers[name] # eg, the mako renderer
except KeyError, err:
raise SaltRenderError('Renderer: {0} is not available!'.format(err))
except IndexError:
raise INVALID_USAGE_ERROR
# do a second pass that provides the extracted conf as template context
data = process_sls_data(sls_templ, tmplctx)
if isinstance(input, basestring):
with salt.utils.fopen(input, 'r') as ifile:
sls_templ = ifile.read()
else: # assume file-like
sls_templ = input.read()
# first pass to extract the state configuration
match = re.search(__opts__['stateconf_end_marker'], sls_templ)
if match:
process_sls_data(sls_templ[:match.start()], extract=True)
# if some config has been extracted then remove the sls-name prefix
# of the keys in the extracted stateconf.set context to make them easier
# to use in the salt file.
if STATE_CONF:
tmplctx = STATE_CONF.copy()
if tmplctx:
prefix = sls + '::'
for k in tmplctx.keys():
if k.startswith(prefix):
tmplctx[k[len(prefix):]] = tmplctx[k]
del tmplctx[k]
else:
tmplctx = {}
# do a second pass that provides the extracted conf as template context
data = process_sls_data(sls_templ, tmplctx)
if log.isEnabledFor(logging.DEBUG):
import pprint # FIXME: pprint OrderedDict
@ -315,10 +344,10 @@ def rename_state_ids(data, sls, is_extend=False):
# update "local" references to the renamed states.
for sid, states, _, args in statelist(data):
if sid == 'extend' and not is_extend:
rename_state_ids(states, sls, True)
continue
if 'extend' in data and not is_extend:
rename_state_ids(data['extend'], sls, True)
for sid, _, _, args in statelist(data):
for req, sname, sid in nvlist2(args, REQUISITES):
if sid.startswith('.'):
req[sname] = _local_to_abs_sid(sid, sls)
@ -343,6 +372,7 @@ def rename_state_ids(data, sls, is_extend=False):
del data[sid]
REQUIRE = set(['require', 'watch'])
REQUIRE_IN = set(['require_in', 'watch_in'])
EXTENDED_REQUIRE = {}
@ -413,6 +443,20 @@ def add_implicit_requires(data):
prev_state = (state_name(sname), sid)
def add_start_state(data):
start_sid = __opts__['stateconf_start_state']
if start_sid in data:
raise SaltRenderError(
'Can\'t generate start state({0})! The same state id already '
'exists!'.format(start_sid)
)
if not data:
return
first = statelist(data, set(['include', 'exclude', 'extend'])).next()[0]
reqin = {state_name(data[first].iterkeys().next()): first}
data[start_sid] = { STATE_FUNC: [ {'require_in': [reqin]} ] }
def add_goal_state(data):
goal_sid = __opts__['stateconf_goal_state']
if goal_sid in data:

104
tests/unit/pydsl_test.py
View File

@ -1,12 +1,15 @@
# Import Python libs
import sys
import sys, os
import shutil
import tempfile
from cStringIO import StringIO
# Import Salt libs
from saltunittest import TestCase
import salt.loader
import salt.config
from salt.state import State
from salt.state import State, HighState
from salt.renderers.yaml import HAS_ORDERED_DICT
REQUISITES = ['require', 'require_in', 'use', 'use_in', 'watch', 'watch_in']
@ -16,11 +19,12 @@ OPTS = salt.config.master_config('whatever, just load the defaults!')
# master conf or minion conf, it doesn't matter.
OPTS['id'] = 'whatever'
OPTS['file_client'] = 'local'
OPTS['file_roots'] = dict(base=['/'])
OPTS['file_roots'] = dict(base=['/tmp'])
OPTS['test'] = False
OPTS['grains'] = salt.loader.grains(OPTS)
STATE = State(OPTS)
def render_sls(content, sls='', env='base', **kws):
return STATE.rend['pydsl'](
StringIO(content), env=env, sls=sls,
@ -107,13 +111,16 @@ include(
'another.sls.file',
'more.sls.file'
)
A = state('A').cmd.run('echo hoho', cwd='/')
state('B').cmd.run('echo hehe', cwd='/')
extend(
A,
state('X').cmd.run(cwd='/a/b/c'),
state('Y').file('managed', name='a_file.txt'),
state('Z').service.watch('file', 'A')
)
''')
self.assertEqual(len(result), 3)
self.assertEqual(len(result), 4)
self.assertEqual(result['include'],
'some.sls.file another.sls.file more.sls.file'.split())
extend = result['extend']
@ -124,6 +131,10 @@ extend(
self.assertEqual(len(extend['Z']['service']), 1)
self.assertEqual(extend['Z']['service'][0]['watch'][0]['file'], 'A')
self.assertEqual(result['B']['cmd'][0], 'run')
self.assertTrue('A' not in result)
self.assertEqual(extend['A']['cmd'][0], 'run')
def test_cmd_call(self):
result = STATE.call_template_str('''#!pydsl
@ -197,3 +208,88 @@ state('A').cmd.run(name='echo hello world')
self.assertEqual(result['B']['service'][2]['watch'][0]['cmd'], 'A')
def test_ordered_states(self):
if sys.version_info < (2, 7) and not HAS_ORDERED_DICT:
self.skipTest('OrderedDict is not available')
result = render_sls('''
__pydsl__.set(ordered=True)
A = state('A')
state('B').cmd.run('echo bbbb')
A.cmd.run('echo aaa')
state('B').cmd.run(cwd='/')
state('C').cmd.run('echo ccc')
state('B').file.managed(source='/a/b/c')
''')
self.assertEqual(len(result['B']['cmd']), 3)
self.assertEqual(result['A']['cmd'][1]['require'][0]['cmd'], 'B')
self.assertEqual(result['C']['cmd'][1]['require'][0]['cmd'], 'A')
self.assertEqual(result['B']['file'][1]['require'][0]['cmd'], 'C')
def test_pipe_through_stateconf(self):
if sys.version_info < (2, 7) and not HAS_ORDERED_DICT:
self.skipTest('OrderedDict is not available')
dirpath = tempfile.mkdtemp()
output = os.path.join(dirpath, 'output')
try:
xxx = os.path.join(dirpath, 'xxx.sls')
with open(xxx, 'w') as xxx:
xxx.write('''#!stateconf -os yaml . jinja
.X:
cmd.run:
- name: echo X >> {0}
- cwd: /
.Y:
cmd.run:
- name: echo Y >> {1}
- cwd: /
.Z:
cmd.run:
- name: echo Z >> {2}
- cwd: /
'''.format(output, output, output))
yyy = os.path.join(dirpath, 'yyy.sls')
with open(yyy, 'w') as yyy:
yyy.write('''#!pydsl|stateconf -ps
state('.D').cmd.run('echo D >> {0}', cwd='/')
state('.E').cmd.run('echo E >> {1}', cwd='/')
state('.F').cmd.run('echo F >> {2}', cwd='/')
'''.format(output, output, output))
aaa = os.path.join(dirpath, 'aaa.sls')
with open(aaa, 'w') as aaa:
aaa.write('''#!pydsl|stateconf -ps
include('xxx', 'yyy')
# make all states in yyy run BEFORE states in this sls.
extend(state('.start').stateconf.require('stateconf', 'xxx::goal'))
# make all states in xxx run AFTER this sls.
extend(state('.goal').stateconf.require_in('stateconf', 'yyy::start'))
__pydsl__.set(ordered=True)
state('.A').cmd.run('echo A >> {0}', cwd='/')
state('.B').cmd.run('echo B >> {1}', cwd='/')
state('.C').cmd.run('echo C >> {2}', cwd='/')
'''.format(output, output, output))
OPTS['file_roots'] = dict(base=[dirpath])
HIGHSTATE = HighState(OPTS)
HIGHSTATE.state.load_modules()
sys.modules['salt.loaded.int.render.pydsl'].__salt__ = HIGHSTATE.state.functions
high, errors = HIGHSTATE.render_highstate({'base': ['aaa']})
# import pprint
# pprint.pprint(errors)
# pprint.pprint(high)
out = HIGHSTATE.state.call_high(high)
# pprint.pprint(out)
with open(output, 'r') as f:
self.assertEqual(''.join(f.read().split()), "XYZABCDEF")
finally:
shutil.rmtree(dirpath, ignore_errors=True)

22
tests/unit/stateconf_test.py
View File

@ -78,7 +78,7 @@ test1:
test2:
user.present
''' )
self.assertTrue(len(result), 3)
self.assertEqual(len(result), 3)
for args in (result['test1']['pkg.installed'],
result['test2']['user.present'] ):
self.assertTrue(isinstance(args, list))
@ -175,6 +175,23 @@ extend:
self.assertTrue('test.utils::some_state' in result['extend'])
def test_start_state_generation(self):
if sys.version_info < (2, 7) and not HAS_ORDERED_DICT:
self.skipTest('OrderedDict is not available')
result = render_sls('''
A:
cmd.run:
- name: echo hello
- cwd: /
B:
cmd.run:
- name: echo world
- cwd: /
''', sls='test', argline='-so yaml . jinja')
self.assertEqual(len(result), 4)
self.assertEqual(result['test::start']['stateconf.set'][1]['require_in'][0]['cmd'], 'A')
def test_goal_state_generation(self):
result = render_sls('''
{% for sid in "ABCDE": %}
@ -185,7 +202,7 @@ extend:
{% endfor %}
''', sls='test.goalstate', argline='yaml . jinja')
self.assertTrue(len(result), len('ABCDE')+1)
self.assertEqual(len(result), len('ABCDE')+1)
reqs = result['test.goalstate::goal']['stateconf.set'][1]['require']
# note: arg 0 is the name arg.
@ -249,3 +266,4 @@ G:
self.assertEqual(
[i.itervalues().next() for i in goal_args[1]['require']],
list('ABCDEFG'))