Merge pull request #16339 from cachedout/module_fail_reasons

Module failure reporting
This commit is contained in:
Thomas S Hatch 2014-10-07 10:21:37 -06:00
commit d4eaf2323e
5 changed files with 59 additions and 18 deletions

View File

@ -154,6 +154,15 @@ The ``__virtual__`` function is used to return either a
False is returned then the module is not loaded, if a string is returned then False is returned then the module is not loaded, if a string is returned then
the module is loaded with the name of the string. the module is loaded with the name of the string.
.. note::
Optionally, modules may additionally return a list of reasons that a module could
not be loaded. For example, if a dependency for 'my_mod' was not met, a
__virtual__ function could do as follows:
return False, ['My Module must be installed before this module can be
used.']
This means that the package manager modules can be presented as the ``pkg`` module This means that the package manager modules can be presented as the ``pkg`` module
regardless of what the actual module is named. regardless of what the actual module is named.

View File

@ -103,7 +103,12 @@ class ZeroMQCaller(object):
ret['jid'] ret['jid']
) )
if fun not in self.minion.functions: if fun not in self.minion.functions:
sys.stderr.write('Function {0} is not available\n'.format(fun)) sys.stderr.write('Function {0} is not available.'.format(fun))
mod_name = fun.split('.')[0]
if mod_name in self.minion.function_errors:
sys.stderr.write(' Possible reasons: {0}\n'.format(self.minion.function_errors[mod_name]))
else:
sys.stderr.write('\n')
sys.exit(-1) sys.exit(-1)
try: try:
sdata = { sdata = {

View File

@ -110,7 +110,7 @@ def _create_loader(
) )
def minion_mods(opts, context=None, whitelist=None): def minion_mods(opts, context=None, whitelist=None, include_errors=False):
''' '''
Load execution modules Load execution modules
@ -136,7 +136,8 @@ def minion_mods(opts, context=None, whitelist=None):
functions = load.gen_functions( functions = load.gen_functions(
pack, pack,
whitelist=whitelist, whitelist=whitelist,
provider_overrides=True provider_overrides=True,
include_errors=include_errors
) )
# Enforce dependencies of module functions from "functions" # Enforce dependencies of module functions from "functions"
Depends.enforce_dependencies(functions) Depends.enforce_dependencies(functions)
@ -707,11 +708,12 @@ class Loader(object):
return funcs return funcs
def gen_functions(self, pack=None, virtual_enable=True, whitelist=None, def gen_functions(self, pack=None, virtual_enable=True, whitelist=None,
provider_overrides=False): provider_overrides=False, include_errors=False):
''' '''
Return a dict of functions found in the defined module_dirs Return a dict of functions found in the defined module_dirs
''' '''
funcs = {} funcs = {}
error_funcs = {}
self.load_modules() self.load_modules()
for mod in self.modules: for mod in self.modules:
# If this is a proxy minion then MOST modules cannot work. Therefore, require that # If this is a proxy minion then MOST modules cannot work. Therefore, require that
@ -765,12 +767,16 @@ class Loader(object):
if virtual_enable: if virtual_enable:
# if virtual modules are enabled, we need to look for the # if virtual modules are enabled, we need to look for the
# __virtual__() function inside that module and run it. # __virtual__() function inside that module and run it.
(virtual_ret, virtual_name) = self.process_virtual(mod, (virtual_ret, virtual_name, virtual_errors) = self.process_virtual(
mod,
module_name) module_name)
# if process_virtual returned a non-True value then we are # if process_virtual returned a non-True value then we are
# supposed to not process this module # supposed to not process this module
if virtual_ret is not True: if virtual_ret is not True:
# If a module has information about why it could not be loaded, record it
if virtual_errors:
error_funcs[module_name] = virtual_errors
continue continue
# update our module name to reflect the virtual name # update our module name to reflect the virtual name
@ -807,10 +813,15 @@ class Loader(object):
not str(mod.__name__).startswith('salt.loaded.ext.grain')) not str(mod.__name__).startswith('salt.loaded.ext.grain'))
): ):
mod.__salt__ = funcs mod.__salt__ = funcs
# if include_errors:
# mod.__errors__ = error_funcs
elif not in_pack(pack, '__salt__') and \ elif not in_pack(pack, '__salt__') and \
(str(mod.__name__).startswith('salt.loaded.int.grain') or (str(mod.__name__).startswith('salt.loaded.int.grain') or
str(mod.__name__).startswith('salt.loaded.ext.grain')): str(mod.__name__).startswith('salt.loaded.ext.grain')):
mod.__salt__.update(funcs) mod.__salt__.update(funcs)
# mod.__errors__ = error_funcs
if include_errors:
funcs['_errors'] = error_funcs
return funcs return funcs
def load_modules(self): def load_modules(self):
@ -1024,16 +1035,23 @@ class Loader(object):
# if they are not intended to run on the given platform or are missing # if they are not intended to run on the given platform or are missing
# dependencies. # dependencies.
try: try:
error_reasons = []
if hasattr(mod, '__virtual__') and inspect.isfunction(mod.__virtual__): if hasattr(mod, '__virtual__') and inspect.isfunction(mod.__virtual__):
if self.opts.get('virtual_timer', False): if self.opts.get('virtual_timer', False):
start = time.time() start = time.time()
virtual = mod.__virtual__() virtual = mod.__virtual__()
if isinstance(virtual, tuple):
error_reasons = virtual[1]
virtual = virtual[0]
end = time.time() - start end = time.time() - start
msg = 'Virtual function took {0} seconds for {1}'.format( msg = 'Virtual function took {0} seconds for {1}'.format(
end, module_name) end, module_name)
log.warning(msg) log.warning(msg)
else: else:
virtual = mod.__virtual__() virtual = mod.__virtual__()
if isinstance(virtual, tuple):
error_reasons = virtual[1]
virtual = virtual[0]
# Get the module's virtual name # Get the module's virtual name
virtualname = getattr(mod, '__virtualname__', virtual) virtualname = getattr(mod, '__virtualname__', virtual)
if not virtual: if not virtual:
@ -1054,7 +1072,7 @@ class Loader(object):
) )
) )
return (False, module_name) return (False, module_name, error_reasons)
# At this point, __virtual__ did not return a # At this point, __virtual__ did not return a
# boolean value, let's check for deprecated usage # boolean value, let's check for deprecated usage
@ -1136,9 +1154,9 @@ class Loader(object):
), ),
exc_info=True exc_info=True
) )
return (False, module_name) return (False, module_name, error_reasons)
return (True, module_name) return (True, module_name, [])
def _apply_outputter(self, func, mod): def _apply_outputter(self, func, mod):
''' '''
@ -1158,6 +1176,8 @@ class Loader(object):
gen = self.gen_functions(pack=pack, whitelist=whitelist) gen = self.gen_functions(pack=pack, whitelist=whitelist)
for key, fun in gen.items(): for key, fun in gen.items():
# if the name (after '.') is "name", then rename to mod_name: fun # if the name (after '.') is "name", then rename to mod_name: fun
if key == '_errors':
continue
if key[key.index('.') + 1:] == name: if key[key.index('.') + 1:] == name:
funcs[key[:key.index('.')]] = fun funcs[key[:key.index('.')]] = fun
return funcs return funcs
@ -1216,7 +1236,7 @@ class Loader(object):
continue continue
grains_data.update(ret) grains_data.update(ret)
for key, fun in funcs.items(): for key, fun in funcs.items():
if key.startswith('core.'): if key.startswith('core.') or key == '_errors':
continue continue
try: try:
ret = fun() ret = fun()

View File

@ -278,7 +278,9 @@ class SMinion(object):
self.opts['id'], self.opts['id'],
self.opts['environment'], self.opts['environment'],
).compile_pillar() ).compile_pillar()
self.functions = salt.loader.minion_mods(self.opts) self.functions = salt.loader.minion_mods(self.opts, include_errors=True)
self.function_errors = self.functions['_errors']
self.functions.pop('_errors') # Keep the funcs clean
self.returners = salt.loader.returners(self.opts, self.functions) self.returners = salt.loader.returners(self.opts, self.functions)
self.states = salt.loader.states(self.opts, self.functions) self.states = salt.loader.states(self.opts, self.functions)
self.rend = salt.loader.render(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions)
@ -606,7 +608,7 @@ class Minion(MinionBase):
).compile_pillar() ).compile_pillar()
self.serial = salt.payload.Serial(self.opts) self.serial = salt.payload.Serial(self.opts)
self.mod_opts = self._prep_mod_opts() self.mod_opts = self._prep_mod_opts()
self.functions, self.returners = self._load_modules() self.functions, self.returners, self.function_errors = self._load_modules()
self.matcher = Matcher(self.opts, self.functions) self.matcher = Matcher(self.opts, self.functions)
self.proc_dir = get_proc_dir(opts['cachedir']) self.proc_dir = get_proc_dir(opts['cachedir'])
self.schedule = salt.utils.schedule.Schedule( self.schedule = salt.utils.schedule.Schedule(
@ -815,14 +817,16 @@ class Minion(MinionBase):
log.error('Unable to enforce modules_max_memory because resource is missing') log.error('Unable to enforce modules_max_memory because resource is missing')
self.opts['grains'] = salt.loader.grains(self.opts, force_refresh) self.opts['grains'] = salt.loader.grains(self.opts, force_refresh)
functions = salt.loader.minion_mods(self.opts) functions = salt.loader.minion_mods(self.opts, include_errors=True)
returners = salt.loader.returners(self.opts, functions) returners = salt.loader.returners(self.opts, functions)
errors = functions['_errors']
functions.pop('_errors')
# we're done, reset the limits! # we're done, reset the limits!
if modules_max_memory is True: if modules_max_memory is True:
resource.setrlimit(resource.RLIMIT_AS, old_mem_limit) resource.setrlimit(resource.RLIMIT_AS, old_mem_limit)
return functions, returners return functions, returners, errors
def _fire_master(self, data=None, tag=None, events=None, pretag=None): def _fire_master(self, data=None, tag=None, events=None, pretag=None):
''' '''
@ -952,7 +956,7 @@ class Minion(MinionBase):
''' '''
if isinstance(data['fun'], string_types): if isinstance(data['fun'], string_types):
if data['fun'] == 'sys.reload_modules': if data['fun'] == 'sys.reload_modules':
self.functions, self.returners = self._load_modules() self.functions, self.returners, self.function_errors = self._load_modules()
self.schedule.functions = self.functions self.schedule.functions = self.functions
self.schedule.returners = self.returners self.schedule.returners = self.returners
if isinstance(data['fun'], tuple) or isinstance(data['fun'], list): if isinstance(data['fun'], tuple) or isinstance(data['fun'], list):
@ -1088,6 +1092,9 @@ class Minion(MinionBase):
ret['out'] = 'nested' ret['out'] = 'nested'
else: else:
ret['return'] = '{0!r} is not available.'.format(function_name) ret['return'] = '{0!r} is not available.'.format(function_name)
mod_name = function_name.split('.')[0]
if mod_name in minion_instance.function_errors:
ret['return'] += ' Possible reasons: {0!r}'.format(minion_instance.function_errors[mod_name])
ret['out'] = 'nested' ret['out'] = 'nested'
ret['jid'] = data['jid'] ret['jid'] = data['jid']
@ -1405,7 +1412,7 @@ class Minion(MinionBase):
''' '''
Refresh the functions and returners. Refresh the functions and returners.
''' '''
self.functions, self.returners = self._load_modules(force_refresh) self.functions, self.returners, _ = self._load_modules(force_refresh)
self.schedule.functions = self.functions self.schedule.functions = self.functions
self.schedule.returners = self.returners self.schedule.returners = self.returners
@ -2610,7 +2617,7 @@ class ProxyMinion(Minion):
).compile_pillar() ).compile_pillar()
self.serial = salt.payload.Serial(self.opts) self.serial = salt.payload.Serial(self.opts)
self.mod_opts = self._prep_mod_opts() self.mod_opts = self._prep_mod_opts()
self.functions, self.returners = self._load_modules() self.functions, self.returners, self.function_errors = self._load_modules()
self.matcher = Matcher(self.opts, self.functions) self.matcher = Matcher(self.opts, self.functions)
self.proc_dir = get_proc_dir(opts['cachedir']) self.proc_dir = get_proc_dir(opts['cachedir'])
self.schedule = salt.utils.schedule.Schedule( self.schedule = salt.utils.schedule.Schedule(

View File

@ -296,7 +296,7 @@ def load_states():
for mod in load.modules: for mod in load.modules:
module_name = mod.__name__.rsplit('.', 1)[-1] module_name = mod.__name__.rsplit('.', 1)[-1]
(virtual_ret, virtual_name) = load.process_virtual(mod, module_name) (virtual_ret, virtual_name, _) = load.process_virtual(mod, module_name)
# if the module returned a True value and a new name use that # if the module returned a True value and a new name use that
# otherwise use the default module name # otherwise use the default module name