valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-08 17:33:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #16411 from SmithSamuelM/sam_dev1

Browse Source 
Application kind now correctly set for Raet Road Stacks
This commit is contained in:
Samuel Smith 2014-10-06 15:36:01 -06:00
commit 0b471c2fa9
13 changed files with 173 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
salt/cli/__init__.py
View File

@ -410,7 +410,6 @@ class SaltCall(parsers.SaltCallOptionParser):
# Setup file logging!
self.setup_logfile_logger()
#caller = salt.cli.caller.Caller(self.config)
caller = salt.cli.caller.Caller.factory(self.config)
if self.options.doc:

28
salt/cli/caller.py
View File

@ -24,6 +24,8 @@ from salt._compat import string_types
from salt.log import LOG_LEVELS
from salt.utils import print_cli
from salt import daemons
log = logging.getLogger(__name__)
try:
@ -281,18 +283,34 @@ class RAETCaller(ZeroMQCaller):
not already setup such as in salt-call to communicate to-from the minion
'''
mid = opts['id']
role = opts.get('id')
if not role:
emsg = ("Missing role required to setup RAETChannel.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = opts.get('__role') # application kind 'master', 'minion', etc
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAETChannel.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind == 'minion':
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAETChannel.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = opts['sock_dir']
uid = nacling.uuid(size=18)
name = 'caller' + uid
name = 'caller' + nacling.uuid(size=18)
stack = LaneStack(name=name,
lanename=mid,
lanename=lanename,
sockdirpath=sockdirpath)
stack.Pk = raeting.packKinds.pack
stack.addRemote(RemoteYard(stack=stack,
name='manor',
lanename=mid,
lanename=lanename,
dirpath=sockdirpath))
log.debug("Created Caller Jobber Stack {0}\n".format(stack.name))
return stack

26
salt/client/raet/__init__.py
View File

@ -17,6 +17,9 @@ import salt.client
import salt.utils
import salt.syspaths as syspaths
from salt import daemons
log = logging.getLogger(__name__)
@ -51,26 +54,33 @@ class LocalClient(salt.client.LocalClient):
jid=jid,
timeout=timeout,
**kwargs)
kind = self.opts['__role']
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for Raet LocalClient.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind == 'master':
lanename = 'master' # self.opts.value.get('id', self.main.data.lanename)
else: # workers currently are only supported for masters
emsg = ("Invalid application kind '{0}' for client.".format(kind))
lanename = 'master'
else:
emsg = ("Unsupported application kind '{0}' for Raet LocalClient.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = self.opts['sock_dir']
name = 'client' + nacling.uuid(size=18)
stack = LaneStack(
name=name,
lanename=lanename,
sockdirpath=self.opts['sock_dir'])
sockdirpath=sockdirpath)
stack.Pk = raeting.packKinds.pack
router_yard = RemoteYard(
manor_yard = RemoteYard(
stack=stack,
lanename=lanename,
name='manor',
dirpath=self.opts['sock_dir'])
stack.addRemote(router_yard)
route = {'dst': (None, router_yard.name, 'local_cmd'),
dirpath=sockdirpath)
stack.addRemote(manor_yard)
route = {'dst': (None, manor_yard.name, 'local_cmd'),
'src': (None, stack.local.name, None)}
msg = {'route': route, 'load': payload_kwargs}
stack.transmit(msg)

8
salt/daemons/__init__.py
View File

@ -6,3 +6,11 @@ Minion enabling different transports
'''
from collections import namedtuple, OrderedDict
# Python equivalent of an enum
APPL_KINDS = OrderedDict([('master', 0), ('minion', 1), ('syndic', 2), ('call', 3)])
APPL_KIND_NAMES = OrderedDict((v, k) for k, v in APPL_KINDS.iteritems()) # inverse map
ApplKind = namedtuple('ApplKind', APPL_KINDS.keys())
applKinds = ApplKind(**APPL_KINDS)

83
salt/daemons/flo/core.py
View File

@ -25,6 +25,7 @@ from raet.road.estating import RemoteEstate
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
from salt import daemons
from salt.daemons import salting
from salt.exceptions import (
@ -131,14 +132,20 @@ class SaltRaetRoadStackSetup(ioflo.base.deeding.Deed):
do salt raet road stack setup at enter
'''
role = self.opts.value.get('id', self.local.data.role)
kind = self.opts.value['__role'] # application kind
kind = self.opts.value['__role'] # application kind
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}'.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
role = self.opts.value.get('id', '')
if not role:
emsg = ("Missing role required to setup RoadStack.")
log.error(emsg + "\n")
raise ValueError(emsg)
name = "{0}_{1}".format(role, kind)
sigkey = self.local.data.sigkey
prikey = self.local.data.prikey
#name = self.opts.value.get('id', self.local.data.name)
#name = LocalEstate.nameGuid(prefix='road') # name is guid
#name = 'stack_' + role
name = role
main = self.opts.value.get('raet_main', self.local.data.main)
mutable = self.opts.value.get('raet_mutable', self.local.data.mutable)
always = self.opts.value.get('open_mode', False)
@ -165,6 +172,7 @@ class SaltRaetRoadStackSetup(ioflo.base.deeding.Deed):
sigkey=sigkey,
prikey=prikey,
main=main,
kind=daemons.APPL_KINDS[kind],
mutable=mutable,
txMsgs=txMsgs,
rxMsgs=rxMsgs,
@ -586,17 +594,30 @@ class SaltRaetManorLaneSetup(ioflo.base.deeding.Deed):
'''
Run once at enter
'''
name = 'manor'
kind = self.opts.value['__role']
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for manor lane.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind == 'master':
lanename = 'master'
elif kind == 'minion':
role = self.opts.value.get('id', '')
if not role:
emsg = ("Missing role required to setup manor Lane.")
log.error(emsg + "\n")
raise ValueError(emsg)
lanename = "{0}_{1}".format(role, kind)
else:
lanename = self.opts.value.get('id', self.local.data.lanename)
emsg = ("Unsupported application kind = '{0}' for manor Lane.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
name = 'manor'
self.stack.value = LaneStack(
name=name,
lanename=lanename,
uid=0,
sockdirpath=self.opts.value['sock_dir'])
self.stack.value.Pk = raeting.packKinds.pack
self.event_yards.value = set()
@ -1001,20 +1022,38 @@ class SaltRaetNixJobber(ioflo.base.deeding.Deed):
to communicate with the minion manor yard
'''
mid = self.opts['id']
uid = nacling.uuid(size=18)
name = 'jobber' + uid
role = self.opts.get('id', '')
if not role:
emsg = ("Missing role required to setup Jobber Lane.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = self.opts['__role']
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for Jobber lane.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind == 'minion':
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind = '{0}' for Jobber Lane.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath = self.opts['sock_dir']
name = 'jobber' + nacling.uuid(size=18)
stack = LaneStack(
name=name,
lanename=mid,
sockdirpath=self.opts['sock_dir'])
lanename=lanename,
sockdirpath=sockdirpath)
stack.Pk = raeting.packKinds.pack
# add remote for the manor yard
stack.addRemote(RemoteYard(stack=stack,
name='manor',
lanename=mid,
dirpath=self.opts['sock_dir']))
lanename=lanename,
dirpath=sockdirpath))
console.concise("Created Jobber Stack {0}\n".format(stack.name))
return stack
@ -1043,8 +1082,8 @@ class SaltRaetNixJobber(ioflo.base.deeding.Deed):
Pull the queue for functions to execute
'''
while self.fun.value:
exchange = self.fun.value.popleft()
data = exchange.get('pub')
msg = self.fun.value.popleft()
data = msg.get('pub')
match = getattr(
self.matcher,
'{0}_match'.format(
@ -1065,16 +1104,16 @@ class SaltRaetNixJobber(ioflo.base.deeding.Deed):
process = multiprocessing.Process(
target=self.proc_run,
kwargs={'exchange': exchange}
kwargs={'msg': msg}
)
process.start()
process.join()
def proc_run(self, exchange):
def proc_run(self, msg):
'''
Execute the run in a dedicated process
'''
data = exchange['pub']
data = msg['pub']
fn_ = os.path.join(self.proc_dir, data['jid'])
self.opts['__ex_id'] = data['jid']
salt.utils.daemonize_if(self.opts)
@ -1167,7 +1206,7 @@ class SaltRaetNixJobber(ioflo.base.deeding.Deed):
ret['jid'] = data['jid']
ret['fun'] = data['fun']
ret['fun_args'] = data['arg']
self._return_pub(exchange, ret, stack)
self._return_pub(msg, ret, stack)
if data['ret']:
ret['id'] = self.opts['id']
for returner in set(data['ret'].split(',')):

8
salt/daemons/flo/maint.flo
View File

@ -6,7 +6,7 @@ framer jobcleaner be active first setup
frame setup
print Setup Maint
enter
do setup maint
do salt raet maint setup
go modules
frame modules
enter
@ -14,14 +14,14 @@ framer jobcleaner be active first setup
go fsclean
frame fsclean
enter
do fileserver clean
do salt raet maint fileserver clean
go start
frame start
do clear old jobs
do salt raet maint old jobs clear
framer backends be active first start
frame start
do update backends
do salt raet maint backends update
framer scheduler be active first start
frame start

20
salt/daemons/flo/maint.py
View File

@ -16,12 +16,12 @@ import salt.utils.minions
import salt.daemons.masterapi
class ForkMaint(ioflo.base.deeding.Deed):
class SaltRaetMaintFork(ioflo.base.deeding.Deed):
'''
For off the maintinence process from the master router process
FloScript:
do fork maint at enter
do salt raet maint fork at enter
'''
Ioinits = {'opts': '.salt.opts'}
@ -70,12 +70,12 @@ class ForkMaint(ioflo.base.deeding.Deed):
self._fork_maint()
class SetupMaint(ioflo.base.deeding.Deed):
class SaltRaetMaintSetup(ioflo.base.deeding.Deed):
'''
Init loader objects used
FloScript:
do setup maint at enter
do salt raet maint setup at enter
'''
Ioinits = {'opts': '.salt.opts',
@ -95,12 +95,12 @@ class SetupMaint(ioflo.base.deeding.Deed):
self.opts.value)
class FileserverClean(ioflo.base.deeding.Deed):
class SaltRaetMainFileserverClean(ioflo.base.deeding.Deed):
'''
Clear the fileserver backend caches
FloScript:
do fileserver clean at enter
do salt raet maint fileserver clean at enter
'''
Ioinits = {'opts': '.salt.opts'}
@ -112,12 +112,12 @@ class FileserverClean(ioflo.base.deeding.Deed):
salt.daemons.masterapi.clean_fsbackend(self.opts.value)
class ClearOldJobs(ioflo.base.deeding.Deed):
class SaltRaetMainOldJobsClear(ioflo.base.deeding.Deed):
'''
Iterate over the jobs directory and clean out the old jobs
FloScript:
do clear old jobs
do salt raet maint old jobs clear
'''
Ioinits = {'opts': '.salt.opts'}
@ -129,12 +129,12 @@ class ClearOldJobs(ioflo.base.deeding.Deed):
salt.daemons.masterapi.clean_old_jobs(self.opts.value)
class UpdateBackends(ioflo.base.deeding.Deed):
class SaltRaetMainBackendsUpdate(ioflo.base.deeding.Deed):
'''
Update the fileserver and external pillar caches
FloScript:
do update backends
do salt raet maint backends update
'''
Ioinits = {'opts': '.salt.opts',

5
salt/daemons/flo/master.flo
View File

@ -2,9 +2,6 @@
house master
#init .salt.road.manor.local to main true role "master"
#init .salt.lane.manor.local to lanename "master"
framer masterudpstack be active first setup
# Begin the pre-flight checks
frame setup
@ -17,7 +14,7 @@ framer masterudpstack be active first setup
# Create the maintanence frame
frame spawnmaint
enter
do fork maint
do salt raet maint fork
go spawnworkers
# Start forking master workers

3
salt/daemons/flo/minion.flo
View File

@ -26,8 +26,6 @@ framer minionudpstack be active first setup
bid start eventing
# Start the functionmanage framer
bid start functionmanager
# Start the manage framer
bid start manager
# Start the outbound framer
bid start outbound
bid start scheduler
@ -84,6 +82,7 @@ framer bootstrap be inactive first join
go router
frame router
# start the manager framer
bid start manager #start alive presence from minion side
do salt raet router
go pillar if .salt.var.pillar_refresh

19
salt/daemons/flo/worker.py
View File

@ -16,6 +16,7 @@ from raet import raeting
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
from salt import daemons
# Import ioflo libs
import ioflo.base.deeding
@ -115,24 +116,27 @@ class SaltRaetWorkerSetup(ioflo.base.deeding.Deed):
name = "worker{0}".format(self.windex.value)
# master application kind
kind = self.opts.value['__role']
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for Master Worker.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind == 'master':
lanename = 'master' # self.opts.value.get('id',
# self.main.data.lanename)
else: # workers currently are only supported for masters
emsg = ("Invalid application kind '{0}' for worker.".format(kind))
lanename = 'master'
else: # workers currently are only supported for masters
emsg = ("Invalid application kind '{0}' for Master Worker.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
sockdirpath=self.opts.value['sock_dir']
self.stack.value = LaneStack(
name=name,
lanename=lanename,
sockdirpath=self.opts.value['sock_dir'])
sockdirpath=sockdirpath)
self.stack.value.Pk = raeting.packKinds.pack
manor_yard = RemoteYard(
stack=self.stack.value,
name='manor',
lanename=lanename,
dirpath=self.opts.value['sock_dir'])
dirpath=sockdirpath)
self.stack.value.addRemote(manor_yard)
self.remote_loader.value = salt.daemons.masterapi.RemoteFuncs(
self.opts.value)
@ -162,7 +166,6 @@ class SaltRaetWorkerRouter(ioflo.base.deeding.Deed):
'lane_stack': '.salt.lane.manor.stack',
'road_stack': '.salt.road.manor.stack',
'opts': '.salt.opts',
#'windex': '.salt.var.fork.worker.windex',
'worker_verify': '.salt.var.worker_verify',
'remote_loader': '.salt.loader.remote',
'local_loader': '.salt.loader.local',

8
salt/daemons/salting.py
View File

@ -8,7 +8,6 @@ salting.py module of salt specific interfaces to raet
# Import Python libs
import os
from collections import namedtuple, OrderedDict
# Import ioflo libs
from ioflo.base.odicting import odict
@ -21,13 +20,6 @@ from raet.keeping import Keep
from salt.key import RaetKey
# Python equivalent of an enum
APPL_KINDS = OrderedDict([('master', 0), ('minion', 1), ('syndic', 2), ('call', 3)])
APPL_KIND_NAMES = odict((v, k) for k, v in APPL_KINDS.iteritems()) # inverse map
ApplKind = namedtuple('ApplKind', APPL_KINDS.keys())
applKinds = ApplKind(**APPL_KINDS)
class SaltKeep(Keep):
'''
RAET protocol estate on road data persistence for a given estate

30
salt/transport/__init__.py
View File

@ -13,6 +13,8 @@ import salt.utils
import logging
from collections import defaultdict
from salt import daemons
log = logging.getLogger(__name__)
try:
@ -78,7 +80,7 @@ class RAETChannel(Channel):
self.dst = (None, None, 'local_cmd') # runner.py master_call
elif usage == 'salt_call':
self.dst = (None, None, 'remote_cmd') # salt_call caller
else: # everything else
else: # everything else minion
self.dst = (None, None, 'remote_cmd') # normal use case minion to master
self.stack = None
@ -88,29 +90,27 @@ class RAETChannel(Channel):
not already setup such as in salt-call to communicate to-from the minion
'''
kind = self.opts.get('__role', '') # application kind 'master', 'minion', etc
if not kind:
emsg = ("Missing opts['__role']. required to setup RAETChannel.")
role = self.opts.get('id')
if not role:
emsg = ("Missing role required to setup RAETChannel.")
log.error(emsg + "\n")
raise ValueError(emsg)
kind = self.opts.get('__role') # application kind 'master', 'minion', etc
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAETChannel.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind == 'master':
lanename = 'master'
elif kind == 'minion':
role = self.opts.get('id', '')
if not role:
emsg = ("Missing opts['id']. required to setup RAETChannel.")
log.error(emsg + "\n")
raise ValueError(emsg)
lanename = role # add kind later
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application kind '{0}' for RAETChannel "
"Raet.".format(kind))
emsg = ("Unsupported application kind '{0}' for RAETChannel.".format(kind))
log.error(emsg + '\n')
raise ValueError(emsg)
mid = self.opts.get('id', 'master')
uid = nacling.uuid(size=18)
name = 'channel' + uid
name = 'channel' + nacling.uuid(size=18)
stack = LaneStack(name=name,
lanename=lanename,
sockdirpath=self.opts['sock_dir'])

39
salt/utils/raetevent.py
View File

@ -21,6 +21,8 @@ from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard
from salt import daemons
log = logging.getLogger(__name__)
@ -41,32 +43,33 @@ class SaltEvent(object):
self.__prep_stack()
def __prep_stack(self):
kind = self.opts.get('__role', '') # opts optional for master
if kind: # not all uses of Raet SaltEvent has opts defined
if kind not in daemons.APPL_KINDS:
emsg = ("Invalid application kind = '{0}' for RAET SaltEvent.".format(kind))
log.error(emsg + "\n")
raise ValueError(emsg)
if kind != self.node:
emsg = ("Mismatch between node = '{0}' and kind = '{1}' in "
"RAET SaltEvent.".format(self.node, kind))
log.error(emsg + '\n')
raise ValueError(emsg)
if self.node == 'master':
lanename = 'master'
if self.opts:
kind = self.opts.get('__role', '') # opts optional for master
if kind and kind != self.node:
emsg = ("Mismatch between node '{0}' and kind '{1}' in setup "
"of SaltEvent on Raet.".format(self.node, kind))
log.error(emsg + '\n')
raise ValueError(emsg)
elif self.node == 'minion':
role = self.opts.get('id', '') # opts required for minion
if not role:
emsg = ("Missing opts['id'] required by SaltEvent on Raet with "
"node kind {0}.".format(self.node))
emsg = ("Missing role required to setup RAET SaltEvent.")
log.error(emsg + "\n")
raise ValueError(emsg)
if not kind:
emsg = "Missing kind required to setup RAET SaltEvent."
log.error(emsg + '\n')
raise ValueError(emsg)
kind = self.opts.get('__role', '')
if kind != self.node:
emsg = ("Mismatch between node '{0}' and kind '{1}' in setup "
"of SaltEvent on Raet.".format(self.node, kind))
log.error(emsg + '\n')
raise ValueError(emsg)
lanename = role # add '_minion'
lanename = "{0}_{1}".format(role, kind)
else:
emsg = ("Unsupported application node kind '{0}' for SaltEvent "
"Raet.".format(self.node))
emsg = ("Unsupported application node kind '{0}' for RAET SaltEvent.".format(self.node))
log.error(emsg + '\n')
raise ValueError(emsg)