From 22ca5b0b5161a2f6b5895044cfcb085eed4c8607 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 2 Jul 2014 13:45:20 -0600 Subject: [PATCH] Added behavior to master and minion to clean up stale yard keep directories on startup Refactored flo scripts to ensure cleanup runs first Refactored road and lane setup behaviors so the stack creation happens in action not postinitio so that stacks do not start until after cleanup --- salt/daemons/flo/core.py | 107 +++++++++++++++++++++++++++++------ salt/daemons/flo/master.flo | 26 ++++++--- salt/daemons/flo/minion.flo | 31 ++++++---- salt/daemons/test/master.flo | 9 +-- 4 files changed, 131 insertions(+), 42 deletions(-) diff --git a/salt/daemons/flo/core.py b/salt/daemons/flo/core.py index fe806545a9..f246ee9c25 100644 --- a/salt/daemons/flo/core.py +++ b/salt/daemons/flo/core.py @@ -14,6 +14,7 @@ import multiprocessing import traceback import itertools from collections import deque +import shutil # Import salt libs import salt.daemons.masterapi @@ -53,12 +54,61 @@ except ImportError: log = logging.getLogger(__name__) -class SaltRaetRoadStack(ioflo.base.deeding.Deed): +class SaltRaetCleanup(ioflo.base.deeding.Deed): + ''' + Cleanup stray lane keep directories not reaped + + FloScript: + + do salt raet cleanup at enter + + ''' + Ioinits = { + 'opts': '.salt.opts', + 'basedirpath': {'ipath': '.salt.raet.basedirpath', + 'ival': '',}, + } + + def postinitio(self): + ''' + Initialize value of data store share for .salt.raet.basedirpath + Will override if empty value + ''' + if not self.basedirpath.value: # override if empty + self.basedirpath.value = os.path.abspath( + os.path.join(self.opts.value['cachedir'], 'raet')) + + def action(self): + ''' + Should only run once to cleanup stale lane directories. + ''' + basedirpath = self.basedirpath.value + if basedirpath: + console.concise("Cleaning up {0}\n".format(basedirpath)) + dirpaths = [] + prefixes = ['client', 'event', self.opts.value['id']] + for name in os.listdir(basedirpath): + path = os.path.join(basedirpath, name) + if not os.path.isdir(path): + continue + for prefix in prefixes: + if name.startswith(prefix) and len(name) == (len(prefix) + 20): + dirpaths.append(path) + break + + for dirpath in dirpaths: + if os.path.exists(dirpath): + console.concise("Removing directory {0}\n".format(dirpath)) + shutil.rmtree(dirpath) + + + +class SaltRaetRoadStackSetup(ioflo.base.deeding.Deed): ''' Initialize and run raet udp stack for Salt FloScript: - do salt raet road stack + do salt raet road stack setup at enter ''' Ioinits = { @@ -76,24 +126,38 @@ class SaltRaetRoadStack(ioflo.base.deeding.Deed): 'localname': 'master', 'eid': 0, 'sigkey': None, - 'prikey': None}} + 'prikey': None,}}, + 'basedirpath': {'ipath': '.salt.raet.basedirpath', + 'ival': ''}, } def postinitio(self): ''' - Setup stack instance + Assign class defaults + ''' + RoadStack.Bk = raeting.bodyKinds.msgpack + RoadStack.JoinentTimeout = 0.0 + + def action(self): + ''' + enter action + should only run once to setup road stack. + moved from postinitio so can do clean up before stack is initialized + + do salt raet road stack setup at enter ''' - sigkey = self.local.data.sigkey - prikey = self.local.data.prikey name = self.opts.value.get('id', self.local.data.name) localname = self.opts.value.get('id', self.local.data.localname) - dirpath = os.path.abspath( - os.path.join(self.opts.value['cachedir'], 'raet')) + sigkey = self.local.data.sigkey + prikey = self.local.data.prikey auto = self.local.data.auto main = self.local.data.main + eid = self.local.data.eid + ha = (self.opts.value['interface'], self.opts.value['raet_port']) - eid = self.local.data.eid + basedirpath = self.basedirpath.value # must be assigned elsewhere + local = LocalEstate( eid=eid, name=localname, @@ -112,22 +176,19 @@ class SaltRaetRoadStack(ioflo.base.deeding.Deed): localname=localname, auto=auto, main=main, - basedirpath=dirpath, + basedirpath=basedirpath, safe=safe, txMsgs=txMsgs, rxMsgs=rxMsgs, period=3.0, offset=0.5) - self.stack.value.Bk = raeting.bodyKinds.msgpack - self.stack.value.JoinentTimeout = 0.0 - class SaltRaetRoadStackCloser(ioflo.base.deeding.Deed): # pylint: disable=W0232 ''' Closes stack server socket connection FloScript: - salt raet road stack closer at exit + do salt raet road stack closer at exit ''' Ioinits = odict( @@ -497,7 +558,7 @@ class SaltManorLaneSetup(ioflo.base.deeding.Deed): Sets of the LaneStack for the main yard FloScript: - do setup at enter + do salt manor lane setup at enter ''' Ioinits = {'opts': '.salt.opts', @@ -515,19 +576,29 @@ class SaltManorLaneSetup(ioflo.base.deeding.Deed): 'ival': {'name': 'master', 'localname': 'master', 'yid': 0, - 'lanename': 'master'}} + 'lanename': 'master'}}, + 'basedirpath': {'ipath': '.salt.raet.basedirpath', + 'ival': '',}, } def postinitio(self): ''' Set up required objects and queues ''' + pass + + + def action(self): + ''' + Run once at enter + ''' name = "{0}{1}".format(self.opts.value.get('id', self.local.data.name), 'lane') localname = self.opts.value.get('id', self.local.data.localname) lanename = self.opts.value.get('id', self.local.data.lanename) yid = self.local.data.yid - basedirpath = os.path.abspath( - os.path.join(self.opts.value['cachedir'], 'raet')) + basedirpath = self.basedirpath.value # must be assigned elsewhere + + #os.path.abspath(os.path.join(self.opts.value['cachedir'], 'raet')) self.stack.value = LaneStack( name=name, #localname=localname, diff --git a/salt/daemons/flo/master.flo b/salt/daemons/flo/master.flo index 8462ad7e8b..137fa5a6fa 100644 --- a/salt/daemons/flo/master.flo +++ b/salt/daemons/flo/master.flo @@ -9,42 +9,52 @@ init .salt.uxd.stack.local to yid 0 name "master" localname "master" lanename "m framer masterudpstack be active first setup frame setup enter - do salt manor lane setup + do salt raet cleanup + do salt raet road stack setup per inode ".raet.udp.stack" + do salt manor lane setup per inode ".salt.uxd.stack" go spawnmaint + frame spawnmaint enter do fork maint go spawnworkers + frame spawnworkers enter do worker fork go start + frame start - do salt raet road stack per inode ".raet.udp.stack" + bid start inbound + bid start uxdrouter + bid start events + bid start publish + bid start manager + bid start outbound exit do salt raet road stack closer per inode ".raet.udp.stack." do salt raet lane stack closer per inode ".salt.uxd.stack." -framer inbound be active first start +framer inbound be inactive first start frame start do salt raet road stack service rx -framer uxdrouter be active first start +framer uxdrouter be inactive first start frame start do router -framer events be active first start +framer events be inactive first start frame start do eventer -framer publish be active first start +framer publish be inactive first start frame start do salt publisher -framer manager be active first start at 10.0 +framer manager be inactive first start at 10.0 frame start do salt raet road stack manager per inode ".raet.udp.stack" -framer outbound be active first start +framer outbound be inactive first start frame start do salt raet road stack service tx diff --git a/salt/daemons/flo/minion.flo b/salt/daemons/flo/minion.flo index 74ad5b7804..5c6008970f 100644 --- a/salt/daemons/flo/minion.flo +++ b/salt/daemons/flo/minion.flo @@ -10,25 +10,32 @@ init .salt.uxd.stack.local to yid 0 name "minion" localname "minion" lanename "m framer minionudpstack be active first setup frame setup enter - do load modules - go manorsetup - - frame manorsetup - enter + do salt raet cleanup + do salt raet road stack setup per inode ".raet.udp.stack" do salt manor lane setup + go loadmodules + + frame loadmodules + do load modules at enter go start frame start - do salt raet road stack per inode ".raet.udp.stack" + bid start inbound + bid start bootstrap + bid start eventing + bid start functionmanager + bid start manager + bid start outbound + exit do salt raet road stack closer per inode ".raet.udp.stack." do salt raet lane stack closer per inode ".salt.uxd.stack." -framer inbound be active first start +framer inbound be inactive first start frame start do salt raet road stack service rx -framer bootstrap be active first join +framer bootstrap be inactive first join frame join print Joining... enter @@ -78,11 +85,11 @@ framer bootstrap be active first join frame abort bid stop all -framer eventing be active first event +framer eventing be inactive first event frame event do eventer -framer functionmanager be active first checkexec +framer functionmanager be inactive first checkexec frame checkexec do nix executor @@ -90,10 +97,10 @@ framer manager be inactive first start at 10.0 frame start do salt raet road stack manager per inode ".raet.udp.stack" -framer outbound be active first start +framer outbound be inactive first start frame start do salt raet road stack service tx -#framer scheduler be active first start +#framer scheduler be inactive first start # frame start # do schedule diff --git a/salt/daemons/test/master.flo b/salt/daemons/test/master.flo index d96f5cd600..c6154b009b 100644 --- a/salt/daemons/test/master.flo +++ b/salt/daemons/test/master.flo @@ -9,9 +9,10 @@ init .raet.udp.stack.local to eid 1 name "master" host "" port 7530 main true lo framer masterudpstack be active first start frame start - do salt raet road stack per inode ".raet.udp.stack" - exit - do raet road stack closer per inode ".raet.udp.stack." + do salt raet cleanup at enter + do salt raet road stack setup per inode ".raet.udp.stack" at enter + bid start service + do raet road stack closer per inode ".raet.udp.stack." at exit framer printer be active first start frame start @@ -21,7 +22,7 @@ framer printer be active first start frame abort bid stop all -framer service be active first start +framer service be inactive first start frame start do salt raet road stack service