From befe4d547013b0067dfe9e2357af30a1fb5ab105 Mon Sep 17 00:00:00 2001 From: Jack Kuan Date: Tue, 15 Jan 2013 23:34:12 -0500 Subject: [PATCH 1/4] Make states created outside of an extend usable inside it. --- salt/modules/pydsl.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/salt/modules/pydsl.py b/salt/modules/pydsl.py index 963540f8db..1b4657f692 100755 --- a/salt/modules/pydsl.py +++ b/salt/modules/pydsl.py @@ -56,9 +56,13 @@ Example of a ``cmd`` state calling a python function:: #TODOs: # +# - add support for implicit ordering. +# # - modify the stateconf renderer so that we can pipe pydsl to -# it to make use of its features, particularly the implicit -# ordering of states using ordered dict. +# it to make use of its features +# +# - support exclude declarations +# - support include declarations with env # # - allow this: # state('X').cmd.run.cwd = '/' @@ -84,6 +88,11 @@ REQUISITES = set("require watch use require_in watch_in use_in".split()) class PyDslError(Exception): pass +class Options(dict): + def __getattr__(self, name): + return self.get(name) + + def sls(sls): return Sls(sls) @@ -97,15 +106,24 @@ class Sls(object): self.includes = [] self.extends = [] self.decls = [] + self.options = Options() + + def set(self, **options): + self.options.update(options) def include(self, *sls_names): self.includes.extend(sls_names) def extend(self, *state_funcs): for f in state_funcs: - self.extends.append(self.all_decls[f.mod._state_id]) - for f in state_funcs: - self.decls.pop() + id = f.mod._state_id + self.extends.append(self.all_decls[id]) + i = len(self.decls) + for decl in reversed(self.decls): + i -= 1 + if decl._id == id: + del self.decls[i] + break def state(self, id=None): if not id: From 9c35844c5661588cd94f532eb7eb6d83453342fe Mon Sep 17 00:00:00 2001 From: Jack Kuan Date: Fri, 18 Jan 2013 12:34:56 -0500 Subject: [PATCH 2/4] Make pydsl work with stateconf. - Added implicit ordering to pydsl.(turned on via __pydsl__.set(ordered=True)) - Fixed a KeyError bug in stateconf renderer that happens during id renaming. - The stateconf renderer now generates a no-op start state as the "first" state in a sls file.(enabled via -s) - Made the stateconf renderer take high state input via the new -p option, so we can pipe pydsl output into it. - Added more and fixed some test cases for stateconf and pydsl. --- salt/modules/pydsl.py | 25 +++++- salt/renderers/stateconf.py | 164 ++++++++++++++++++++++------------- tests/unit/pydsl_test.py | 99 ++++++++++++++++++++- tests/unit/stateconf_test.py | 20 ++++- 4 files changed, 238 insertions(+), 70 deletions(-) diff --git a/salt/modules/pydsl.py b/salt/modules/pydsl.py index 1b4657f692..c8ebd6b917 100755 --- a/salt/modules/pydsl.py +++ b/salt/modules/pydsl.py @@ -107,6 +107,7 @@ class Sls(object): self.extends = [] self.decls = [] self.options = Options() + self.funcs = [] # track the ordering of state func declarations def set(self, **options): self.options.update(options) @@ -115,6 +116,8 @@ class Sls(object): self.includes.extend(sls_names) def extend(self, *state_funcs): + if self.options.ordered and self.last_func(): + raise PyDslError("Can't extend while the ordered option is turned on!") for f in state_funcs: id = f.mod._state_id self.extends.append(self.all_decls[id]) @@ -124,6 +127,10 @@ class Sls(object): if decl._id == id: del self.decls[i] break + try: + self.funcs.remove(f) # untrack it + except ValueError: + pass def state(self, id=None): if not id: @@ -132,10 +139,16 @@ class Sls(object): try: return self.all_decls[id] except KeyError: - self.all_decls[id] = s = StateDeclaration(id) + self.all_decls[id] = s = StateDeclaration(id, self) self.decls.append(s) return s + def last_func(self): + return self.funcs[-1] if self.funcs else None + + def track_func(self, statefunc): + self.funcs.append(statefunc) + def to_highstate(self, slsmod=None): # generate a state that uses the stateconf.set state, which # is a no-op state, to hold a reference to the sls module @@ -177,7 +190,8 @@ class Sls(object): class StateDeclaration(object): - def __init__(self, id=None): + def __init__(self, id, sls): + self._sls = sls self._id = id self._mods = [] @@ -253,6 +267,13 @@ class StateFunction(object): self.name = name self.args = [] + sls = Sls.all_decls[parent_mod._state_id]._sls + if sls.options.ordered: + last_f = sls.last_func() + if last_f: + self.require(last_f.mod) + sls.track_func(self) + def __call__(self, *args, **kws): self.configure(args, kws) return self diff --git a/salt/renderers/stateconf.py b/salt/renderers/stateconf.py index d5e6120751..49ed0a01cc 100644 --- a/salt/renderers/stateconf.py +++ b/salt/renderers/stateconf.py @@ -37,6 +37,9 @@ __opts__ = { 'stateconf_end_marker': r'#\s*-+\s*end of state config\s*-+', # eg, something like "# --- end of state config --" works by default. + 'stateconf_start_state': '.start', + # name of the state id for the generated start state. + 'stateconf_goal_state': '.goal', # name of the state id for the generated goal state. @@ -61,7 +64,7 @@ def __init__(opts): MOD_BASENAME = ospath.basename(__file__) INVALID_USAGE_ERROR = SaltRenderError( 'Invalid use of {0} renderer!\n' - '''Usage: #!{1} [-Go] [options] . [options] + '''Usage: #!{1} [-GoSp] [ [options] . [options]] where an example would be yaml and a might be jinja. Each renderer can be passed its renderer specific options. @@ -72,46 +75,24 @@ Options(for this renderer): -o Indirectly order the states by adding requires such that they will be executed in the order they are defined in the sls. Implies using yaml -o. + + -s Generate the start state that gets inserted as the first state in + the sls. This only makes sense if your high state data dict is ordered. + + -p Assume high state input. This option allows you to pipe high state data + through this renderer. With this option, the use of stateconf.set state + in the sls will have no effect, but other features of the renderer still + apply. + '''.format(MOD_BASENAME, MOD_BASENAME) ) -def render(template_file, env='', sls='', argline='', **kws): +def render(input, env='', sls='', argline='', **kws): + gen_start_state = False no_goal_state = False implicit_require = False - renderers = kws['renderers'] - opts, args = getopt.getopt(argline.split(), 'Go') - argline = ' '.join(args) if args else 'yaml . jinja' - - if ('-G', '') in opts: - no_goal_state = True - - # Split on the first dot surrounded by spaces but not preceded by a - # backslash. A backslash preceded dot will be replaced with just dot. - args = [ - arg.strip().replace('\\.', '.') - for arg in re.split(r'\s+(?> {0} + - cwd: / +.Y: + cmd.run: + - name: echo Y >> {1} + - cwd: / +.Z: + cmd.run: + - name: echo Z >> {2} + - cwd: / +'''.format(output, output, output)) + yyy = os.path.join(dirpath, 'yyy.sls') + with open(yyy, 'w') as yyy: + yyy.write('''#!pydsl|stateconf -ps +state('.D').cmd.run('echo D >> {0}', cwd='/') +state('.E').cmd.run('echo E >> {1}', cwd='/') +state('.F').cmd.run('echo F >> {2}', cwd='/') +'''.format(output, output, output)) + + aaa = os.path.join(dirpath, 'aaa.sls') + with open(aaa, 'w') as aaa: + aaa.write('''#!pydsl|stateconf -ps +include('xxx', 'yyy') + +# make all states in yyy run BEFORE states in this sls. +extend(state('.start').stateconf.require('stateconf', 'xxx::goal')) + +# make all states in xxx run AFTER this sls. +extend(state('.goal').stateconf.require_in('stateconf', 'yyy::start')) + +__pydsl__.set(ordered=True) + +state('.A').cmd.run('echo A >> {0}', cwd='/') +state('.B').cmd.run('echo B >> {1}', cwd='/') +state('.C').cmd.run('echo C >> {2}', cwd='/') +'''.format(output, output, output)) + + OPTS['file_roots'] = dict(base=[dirpath]) + HIGHSTATE = HighState(OPTS) + HIGHSTATE.state.load_modules() + sys.modules['salt.loaded.int.render.pydsl'].__salt__ = HIGHSTATE.state.functions + + high, errors = HIGHSTATE.render_highstate({'base': ['aaa']}) +# import pprint +# pprint.pprint(errors) +# pprint.pprint(high) + out = HIGHSTATE.state.call_high(high) +# pprint.pprint(out) + with open(output, 'r') as f: + self.assertEqual(''.join(f.read().split()), "XYZABCDEF") + + finally: + shutil.rmtree(dirpath, ignore_errors=True) + + + diff --git a/tests/unit/stateconf_test.py b/tests/unit/stateconf_test.py index d18264e039..c2a5821dfe 100644 --- a/tests/unit/stateconf_test.py +++ b/tests/unit/stateconf_test.py @@ -78,7 +78,7 @@ test1: test2: user.present ''' ) - self.assertTrue(len(result), 3) + self.assertEqual(len(result), 3) for args in (result['test1']['pkg.installed'], result['test2']['user.present'] ): self.assertTrue(isinstance(args, list)) @@ -175,6 +175,21 @@ extend: self.assertTrue('test.utils::some_state' in result['extend']) + def test_start_state_generation(self): + result = render_sls(''' +A: + cmd.run: + - name: echo hello + - cwd: / +B: + cmd.run: + - name: echo world + - cwd: / +''', sls='test', argline='-so yaml . jinja') + self.assertEqual(len(result), 4) + self.assertEqual(result['test::start']['stateconf.set'][1]['require_in'][0]['cmd'], 'A') + + def test_goal_state_generation(self): result = render_sls(''' {% for sid in "ABCDE": %} @@ -185,7 +200,7 @@ extend: {% endfor %} ''', sls='test.goalstate', argline='yaml . jinja') - self.assertTrue(len(result), len('ABCDE')+1) + self.assertEqual(len(result), len('ABCDE')+1) reqs = result['test.goalstate::goal']['stateconf.set'][1]['require'] # note: arg 0 is the name arg. @@ -249,3 +264,4 @@ G: self.assertEqual( [i.itervalues().next() for i in goal_args[1]['require']], list('ABCDEFG')) + From 5aaa17cd10714b7303ba1dad59171c6ede062153 Mon Sep 17 00:00:00 2001 From: Jack Kuan Date: Fri, 18 Jan 2013 12:55:49 -0500 Subject: [PATCH 3/4] update todos. --- salt/modules/pydsl.py | 5 ----- salt/renderers/stateconf.py | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/modules/pydsl.py b/salt/modules/pydsl.py index c8ebd6b917..c53c7267a9 100755 --- a/salt/modules/pydsl.py +++ b/salt/modules/pydsl.py @@ -56,11 +56,6 @@ Example of a ``cmd`` state calling a python function:: #TODOs: # -# - add support for implicit ordering. -# -# - modify the stateconf renderer so that we can pipe pydsl to -# it to make use of its features -# # - support exclude declarations # - support include declarations with env # diff --git a/salt/renderers/stateconf.py b/salt/renderers/stateconf.py index 49ed0a01cc..5413df1501 100644 --- a/salt/renderers/stateconf.py +++ b/salt/renderers/stateconf.py @@ -2,6 +2,10 @@ # for a guide to using this module. # # TODO: +# +# - support exclude declarations +# - support include declarations with env +# # - sls meta/info state: Eg, # # sls_info: From a678901c48756f6222273d480aab76c3fa1c6465 Mon Sep 17 00:00:00 2001 From: Jack Kuan Date: Fri, 18 Jan 2013 14:14:42 -0500 Subject: [PATCH 4/4] Fix test cases for py2.6. --- tests/unit/pydsl_test.py | 5 +++++ tests/unit/stateconf_test.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/tests/unit/pydsl_test.py b/tests/unit/pydsl_test.py index 42f720715d..802d54693a 100644 --- a/tests/unit/pydsl_test.py +++ b/tests/unit/pydsl_test.py @@ -9,6 +9,7 @@ from saltunittest import TestCase import salt.loader import salt.config from salt.state import State, HighState +from salt.renderers.yaml import HAS_ORDERED_DICT REQUISITES = ['require', 'require_in', 'use', 'use_in', 'watch', 'watch_in'] @@ -208,6 +209,8 @@ state('A').cmd.run(name='echo hello world') def test_ordered_states(self): + if sys.version_info < (2, 7) and not HAS_ORDERED_DICT: + self.skipTest('OrderedDict is not available') result = render_sls(''' __pydsl__.set(ordered=True) A = state('A') @@ -224,6 +227,8 @@ state('B').file.managed(source='/a/b/c') def test_pipe_through_stateconf(self): + if sys.version_info < (2, 7) and not HAS_ORDERED_DICT: + self.skipTest('OrderedDict is not available') dirpath = tempfile.mkdtemp() output = os.path.join(dirpath, 'output') try: diff --git a/tests/unit/stateconf_test.py b/tests/unit/stateconf_test.py index c2a5821dfe..5463701241 100644 --- a/tests/unit/stateconf_test.py +++ b/tests/unit/stateconf_test.py @@ -176,6 +176,8 @@ extend: def test_start_state_generation(self): + if sys.version_info < (2, 7) and not HAS_ORDERED_DICT: + self.skipTest('OrderedDict is not available') result = render_sls(''' A: cmd.run: