mirror of
https://github.com/valitydev/salt.git
synced 2024-11-09 01:36:48 +00:00
Remove majority of *old* loader code
This commit is contained in:
Thomas Jackson
parent
db3d1a7426
commit
583fa3d6cc
721
salt/loader.py
721
salt/loader.py
@ -378,13 +378,16 @@ def beacons(opts, context=None):
|
||||
'''
|
||||
Load the beacon modules
|
||||
'''
|
||||
load = _create_loader(opts, 'beacons', 'beacons')
|
||||
if context is None:
|
||||
context = {}
|
||||
pack = {'name': '__context__',
|
||||
'value': context}
|
||||
functions = load.gen_functions(pack)
|
||||
return functions
|
||||
return NewLazyLoader(_module_dirs(opts, 'beacons', 'beacons'),
|
||||
opts,
|
||||
tag='beacons',
|
||||
loaded_base_name='beacons',
|
||||
pack=pack,
|
||||
)
|
||||
|
||||
|
||||
def search(opts, returners, whitelist=None):
|
||||
@ -771,7 +774,7 @@ class NewLazyLoader(salt.utils.lazy.LazyDict):
|
||||
- singletons (per tag)
|
||||
|
||||
# TODO:
|
||||
- change everyone else's calls to clear()
|
||||
- move modules_max_memory into here
|
||||
'''
|
||||
# This class is only a singleton per minion/master pair
|
||||
instances = {}
|
||||
@ -1315,6 +1318,7 @@ class NewLazyLoader(salt.utils.lazy.LazyDict):
|
||||
return (True, module_name, [])
|
||||
|
||||
|
||||
# TODO: remove, we only ever use "gen_module" now
|
||||
class Loader(object):
|
||||
'''
|
||||
Used to load in arbitrary modules from a directory, the Loader can
|
||||
@ -1342,8 +1346,6 @@ class Loader(object):
|
||||
self.opts = self.__prep_mod_opts(opts)
|
||||
self.loaded_base_name = loaded_base_name or LOADED_BASE_NAME
|
||||
self.mod_type_check = mod_type_check or _mod_type
|
||||
if self.opts.get('grains_cache', False):
|
||||
self.serial = salt.payload.Serial(self.opts)
|
||||
|
||||
def __prep_mod_opts(self, opts):
|
||||
'''
|
||||
@ -1356,38 +1358,6 @@ class Loader(object):
|
||||
mod_opts[key] = val
|
||||
return mod_opts
|
||||
|
||||
def call(self, fun, arg=None):
|
||||
'''
|
||||
Call a function in the load path.
|
||||
'''
|
||||
if arg is None:
|
||||
arg = []
|
||||
name = fun[:fun.rindex('.')]
|
||||
try:
|
||||
fn_, path, desc = imp.find_module(name, self.module_dirs)
|
||||
mod = imp.load_module(name, fn_, path, desc)
|
||||
except ImportError:
|
||||
if self.opts.get('cython_enable', True) is True:
|
||||
# The module was not found, try to find a cython module
|
||||
try:
|
||||
import pyximport # pylint: disable=import-error
|
||||
pyximport.install()
|
||||
|
||||
for mod_dir in self.module_dirs:
|
||||
for fn_ in os.listdir(mod_dir):
|
||||
if name == fn_[:fn_.rindex('.')]:
|
||||
# Found it, load the mod and break the loop
|
||||
mod = pyximport.load_module(
|
||||
name, os.path.join(mod_dir, fn_)
|
||||
)
|
||||
return getattr(
|
||||
mod, fun[fun.rindex('.') + 1:])(*arg)
|
||||
except ImportError:
|
||||
log.info('Cython is enabled in options though it\'s not '
|
||||
'present in the system path. Skipping Cython '
|
||||
'modules.')
|
||||
return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)
|
||||
|
||||
def gen_module(self, name, functions, pack=None, virtual_enable=False):
|
||||
'''
|
||||
Load a single module and pack it with the functions passed
|
||||
@ -1492,18 +1462,6 @@ class Loader(object):
|
||||
pass
|
||||
funcs = {}
|
||||
module_name = mod.__name__[mod.__name__.rindex('.') + 1:]
|
||||
if virtual_enable:
|
||||
# if virtual modules are enabled, we need to look for the
|
||||
# __virtual__() function inside that module and run it.
|
||||
(virtual_ret, virtual_name, _) = self.process_virtual(
|
||||
mod,
|
||||
module_name)
|
||||
|
||||
# if process_virtual returned a non-True value then we are
|
||||
# supposed to not process this module
|
||||
if virtual_ret is not True:
|
||||
# If a module has information about why it could not be loaded, record it
|
||||
return False # TODO Support virtual_errors here
|
||||
|
||||
# update our module name to reflect the virtual name
|
||||
if getattr(mod, '__load__', False) is not False:
|
||||
@ -1541,498 +1499,6 @@ class Loader(object):
|
||||
mod.__context__ = context
|
||||
return funcs
|
||||
|
||||
def gen_functions(self, pack=None, virtual_enable=True, whitelist=None,
|
||||
provider_overrides=False, include_errors=False, initial_load=False):
|
||||
'''
|
||||
Return a dict of functions found in the defined module_dirs
|
||||
'''
|
||||
funcs = OrderedDict()
|
||||
error_funcs = {}
|
||||
if not hasattr(self, 'modules'):
|
||||
self.load_modules()
|
||||
for mod in self.modules:
|
||||
# If this is a proxy minion then MOST modules cannot work. Therefore, require that
|
||||
# any module that does work with salt-proxy-minion define __proxyenabled__ as a list
|
||||
# containing the names of the proxy types that the module supports.
|
||||
if not hasattr(mod, 'render') and 'proxy' in self.opts:
|
||||
if not hasattr(mod, '__proxyenabled__'):
|
||||
# This is a proxy minion but this module doesn't support proxy
|
||||
# minions at all
|
||||
continue
|
||||
if not (self.opts['proxy']['proxytype'] in mod.__proxyenabled__ or
|
||||
'*' in mod.__proxyenabled__):
|
||||
# This is a proxy minion, this module supports proxy
|
||||
# minions, but not this particular minion
|
||||
log.debug(mod)
|
||||
continue
|
||||
|
||||
if hasattr(mod, '__opts__'):
|
||||
mod.__opts__.update(self.opts)
|
||||
else:
|
||||
mod.__opts__ = self.opts
|
||||
|
||||
mod.__grains__ = self.grains
|
||||
mod.__pillar__ = self.pillar
|
||||
|
||||
if pack:
|
||||
if isinstance(pack, list):
|
||||
for chunk in pack:
|
||||
if not isinstance(chunk, dict):
|
||||
continue
|
||||
try:
|
||||
setattr(mod, chunk['name'], chunk['value'])
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
setattr(mod, pack['name'], pack['value'])
|
||||
|
||||
# Call a module's initialization method if it exists
|
||||
module_init = getattr(mod, '__init__', None)
|
||||
if inspect.isfunction(module_init):
|
||||
try:
|
||||
module_init(self.opts)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
# Trim the full pathname to just the module
|
||||
# this will be the short name that other salt modules and state
|
||||
# will refer to it as.
|
||||
module_name = mod.__name__.rsplit('.', 1)[-1]
|
||||
|
||||
if virtual_enable:
|
||||
# if virtual modules are enabled, we need to look for the
|
||||
# __virtual__() function inside that module and run it.
|
||||
(virtual_ret, virtual_name, virtual_errors) = self.process_virtual(
|
||||
mod,
|
||||
module_name)
|
||||
|
||||
# if process_virtual returned a non-True value then we are
|
||||
# supposed to not process this module
|
||||
if virtual_ret is not True:
|
||||
# If a module has information about why it could not be loaded, record it
|
||||
if virtual_errors:
|
||||
error_funcs[module_name] = virtual_errors
|
||||
continue
|
||||
|
||||
# update our module name to reflect the virtual name
|
||||
module_name = virtual_name
|
||||
|
||||
if whitelist:
|
||||
# If a whitelist is defined then only load the module if it is
|
||||
# in the whitelist
|
||||
if module_name not in whitelist:
|
||||
continue
|
||||
|
||||
# load the functions from the module and update our dict
|
||||
funcs.update(self.load_functions(mod, module_name))
|
||||
|
||||
# Handle provider overrides
|
||||
if provider_overrides and self.opts.get('providers', False):
|
||||
if isinstance(self.opts['providers'], dict):
|
||||
for mod, provider in six.iteritems(self.opts['providers']):
|
||||
newfuncs = raw_mod(self.opts, provider, funcs)
|
||||
if newfuncs:
|
||||
for newfunc in newfuncs:
|
||||
f_key = '{0}{1}'.format(
|
||||
mod, newfunc[newfunc.rindex('.'):]
|
||||
)
|
||||
funcs[f_key] = newfuncs[newfunc]
|
||||
|
||||
# now that all the functions have been collected, iterate back over
|
||||
# the available modules and inject the special __salt__ namespace that
|
||||
# contains these functions.
|
||||
for mod in self.modules:
|
||||
if not hasattr(mod, '__salt__') or (
|
||||
not in_pack(pack, '__salt__') and
|
||||
(not str(mod.__name__).startswith('salt.loaded.int.grain') and
|
||||
not str(mod.__name__).startswith('salt.loaded.ext.grain'))
|
||||
):
|
||||
mod.__salt__ = funcs
|
||||
# if include_errors:
|
||||
# mod.__errors__ = error_funcs
|
||||
elif not in_pack(pack, '__salt__') and \
|
||||
(str(mod.__name__).startswith('salt.loaded.int.grain') or
|
||||
str(mod.__name__).startswith('salt.loaded.ext.grain')):
|
||||
mod.__salt__.update(funcs)
|
||||
if include_errors:
|
||||
funcs['_errors'] = error_funcs
|
||||
return funcs
|
||||
|
||||
def load_modules(self, initial_load=False):
|
||||
'''
|
||||
Loads all of the modules from module_dirs and returns a list of them
|
||||
'''
|
||||
|
||||
self.modules = []
|
||||
|
||||
log.trace('loading {0} in {1}'.format(self.tag, self.module_dirs))
|
||||
names = OrderedDict()
|
||||
disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))
|
||||
|
||||
cython_enabled = False
|
||||
if self.opts.get('cython_enable', True) is True:
|
||||
try:
|
||||
import pyximport # pylint: disable=import-error
|
||||
pyximport.install()
|
||||
cython_enabled = True
|
||||
except ImportError:
|
||||
log.info('Cython is enabled in the options but not present '
|
||||
'in the system path. Skipping Cython modules.')
|
||||
for mod_dir in self.module_dirs:
|
||||
if not os.path.isabs(mod_dir):
|
||||
log.trace(
|
||||
'Skipping {0}, it is not an absolute path'.format(
|
||||
mod_dir
|
||||
)
|
||||
)
|
||||
continue
|
||||
if not os.path.isdir(mod_dir):
|
||||
log.trace(
|
||||
'Skipping {0}, it is not a directory'.format(
|
||||
mod_dir
|
||||
)
|
||||
)
|
||||
continue
|
||||
for fn_ in os.listdir(mod_dir):
|
||||
if fn_.startswith('_'):
|
||||
# skip private modules
|
||||
# log messages omitted for obviousness
|
||||
continue
|
||||
if fn_.split('.')[0] in disable:
|
||||
log.trace(
|
||||
'Skipping {0}, it is disabled by configuration'.format(
|
||||
fn_
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if fn_.endswith(('.pyc', '.pyo')):
|
||||
non_compiled_filename = '{0}.py'.format(os.path.splitext(fn_)[0])
|
||||
if os.path.exists(os.path.join(mod_dir, non_compiled_filename)):
|
||||
# Let's just process the non compiled python modules
|
||||
continue
|
||||
|
||||
if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
|
||||
or (cython_enabled and fn_.endswith('.pyx'))
|
||||
or os.path.isdir(os.path.join(mod_dir, fn_))):
|
||||
|
||||
extpos = fn_.rfind('.')
|
||||
if extpos > 0:
|
||||
_name = fn_[:extpos]
|
||||
else:
|
||||
_name = fn_
|
||||
|
||||
if _name in names:
|
||||
# Since we load custom modules first, if this logic is true it means
|
||||
# that an internal module was shadowed by an external custom module
|
||||
log.trace(
|
||||
'The {0!r} module from {1!r} was shadowed by '
|
||||
'the module in {2!r}'.format(
|
||||
_name,
|
||||
mod_dir,
|
||||
names[_name],
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
names[_name] = os.path.join(mod_dir, fn_)
|
||||
else:
|
||||
log.trace(
|
||||
'Skipping {0}, it does not end with an expected '
|
||||
'extension'.format(
|
||||
fn_
|
||||
)
|
||||
)
|
||||
failed_loads = {}
|
||||
|
||||
def load_names(names, failhard=False, initial_load=False):
|
||||
for name in names:
|
||||
try:
|
||||
if names[name].endswith('.pyx'):
|
||||
# If there's a name which ends in .pyx it means the above
|
||||
# cython_enabled is True. Continue...
|
||||
mod = pyximport.load_module(
|
||||
'{0}.{1}.{2}.{3}'.format(
|
||||
self.loaded_base_name,
|
||||
self.mod_type_check(names[name]),
|
||||
self.tag,
|
||||
name
|
||||
), names[name], tempfile.gettempdir()
|
||||
)
|
||||
else:
|
||||
fn_, path, desc = imp.find_module(name, self.module_dirs)
|
||||
mod = imp.load_module(
|
||||
'{0}.{1}.{2}.{3}'.format(
|
||||
self.loaded_base_name,
|
||||
self.mod_type_check(path),
|
||||
self.tag,
|
||||
name
|
||||
), fn_, path, desc
|
||||
)
|
||||
if not initial_load:
|
||||
# reload all submodules if necessary
|
||||
submodules = [
|
||||
getattr(mod, sname) for sname in dir(mod) if
|
||||
isinstance(getattr(mod, sname), mod.__class__)
|
||||
]
|
||||
|
||||
# reload only custom "sub"modules i.e. is a submodule in
|
||||
# parent module that are still available on disk (i.e. not
|
||||
# removed during sync_modules)
|
||||
for submodule in submodules:
|
||||
try:
|
||||
smname = '{0}.{1}.{2}'.format(
|
||||
self.loaded_base_name,
|
||||
self.tag,
|
||||
name
|
||||
)
|
||||
smfile = '{0}.py'.format(
|
||||
os.path.splitext(submodule.__file__)[0]
|
||||
)
|
||||
if submodule.__name__.startswith(smname) and \
|
||||
os.path.isfile(smfile):
|
||||
reload(submodule)
|
||||
except AttributeError:
|
||||
continue
|
||||
except ImportError as error:
|
||||
if failhard:
|
||||
log.debug(
|
||||
'Failed to import {0} {1}, this is most likely NOT a '
|
||||
'problem, but the exception was {2}\n'.format(
|
||||
self.tag, name, error
|
||||
),
|
||||
exc_info=True
|
||||
)
|
||||
if not failhard:
|
||||
log.debug('Failed to import {0} {1}. The exeception was {2}. Another attempt will be made to try to resolve dependencies.'.format(
|
||||
self.tag, name, error))
|
||||
failed_loads[name] = path
|
||||
continue
|
||||
except Exception as error:
|
||||
log.warning(
|
||||
'Failed to import {0} {1}, this is due most likely to a '
|
||||
'syntax error. The exception is {2}. Traceback raised:\n'.format(
|
||||
self.tag, name, error
|
||||
),
|
||||
exc_info=True
|
||||
)
|
||||
except SystemExit as error:
|
||||
log.warning(
|
||||
'Failed to import {0} {1} as the module called exit()\n'.format(
|
||||
self.tag, name
|
||||
),
|
||||
exc_info=True
|
||||
)
|
||||
continue
|
||||
self.modules.append(mod)
|
||||
load_names(names, failhard=False, initial_load=initial_load)
|
||||
if failed_loads:
|
||||
load_names(failed_loads, failhard=True)
|
||||
|
||||
def load_functions(self, mod, module_name):
|
||||
'''
|
||||
Load functions returns a dict of all the functions from a module
|
||||
'''
|
||||
funcs = {}
|
||||
|
||||
if getattr(mod, '__load__', False) is not False:
|
||||
log.info(
|
||||
'The functions from module {0!r} are being loaded from '
|
||||
'the provided __load__ attribute'.format(
|
||||
module_name
|
||||
)
|
||||
)
|
||||
|
||||
for attr in getattr(mod, '__load__', dir(mod)):
|
||||
if attr.startswith('_'):
|
||||
# skip private attributes
|
||||
# log messages omitted for obviousness
|
||||
continue
|
||||
|
||||
func = getattr(mod, attr)
|
||||
if not inspect.isfunction(func):
|
||||
# Not a function!? Skip it!!!
|
||||
continue
|
||||
|
||||
# Once confirmed that "func" is a function, add it to the
|
||||
# library of available functions
|
||||
|
||||
# Let's get the function name.
|
||||
# If the module has the __func_alias__ attribute, it must
|
||||
# be a dictionary mapping in the form of(key -> value):
|
||||
# <real-func-name> -> <desired-func-name>
|
||||
#
|
||||
# It default's of course to the found callable attribute
|
||||
# name if no alias is defined.
|
||||
funcname = getattr(mod, '__func_alias__', {}).get(
|
||||
attr, attr
|
||||
)
|
||||
|
||||
# functions are namespaced with their module name, unless
|
||||
# the module_name is None (this is a special case added for
|
||||
# pyobjects), in which case just the function name is used
|
||||
if module_name is None:
|
||||
module_func_name = funcname
|
||||
else:
|
||||
module_func_name = '{0}.{1}'.format(module_name, funcname)
|
||||
|
||||
funcs[module_func_name] = func
|
||||
log.trace(
|
||||
'Added {0} to {1}'.format(module_func_name, self.tag)
|
||||
)
|
||||
self._apply_outputter(func, mod)
|
||||
return funcs
|
||||
|
||||
def process_virtual(self, mod, module_name):
|
||||
'''
|
||||
Given a loaded module and its default name determine its virtual name
|
||||
|
||||
This function returns a tuple. The first value will be either True or
|
||||
False and will indicate if the module should be loaded or not (i.e. if
|
||||
it threw and exception while processing its __virtual__ function). The
|
||||
second value is the determined virtual name, which may be the same as
|
||||
the value provided.
|
||||
|
||||
The default name can be calculated as follows::
|
||||
|
||||
module_name = mod.__name__.rsplit('.', 1)[-1]
|
||||
'''
|
||||
|
||||
# The __virtual__ function will return either a True or False value.
|
||||
# If it returns a True value it can also set a module level attribute
|
||||
# named __virtualname__ with the name that the module should be
|
||||
# referred to as.
|
||||
#
|
||||
# This allows us to have things like the pkg module working on all
|
||||
# platforms under the name 'pkg'. It also allows for modules like
|
||||
# augeas_cfg to be referred to as 'augeas', which would otherwise have
|
||||
# namespace collisions. And finally it allows modules to return False
|
||||
# if they are not intended to run on the given platform or are missing
|
||||
# dependencies.
|
||||
try:
|
||||
error_reasons = []
|
||||
if hasattr(mod, '__virtual__') and inspect.isfunction(mod.__virtual__):
|
||||
if self.opts.get('virtual_timer', False):
|
||||
start = time.time()
|
||||
virtual = mod.__virtual__()
|
||||
if isinstance(virtual, tuple):
|
||||
error_reasons = virtual[1]
|
||||
virtual = virtual[0]
|
||||
end = time.time() - start
|
||||
msg = 'Virtual function took {0} seconds for {1}'.format(
|
||||
end, module_name)
|
||||
log.warning(msg)
|
||||
else:
|
||||
virtual = mod.__virtual__()
|
||||
if isinstance(virtual, tuple):
|
||||
error_reasons = virtual[1]
|
||||
virtual = virtual[0]
|
||||
# Get the module's virtual name
|
||||
virtualname = getattr(mod, '__virtualname__', virtual)
|
||||
if not virtual:
|
||||
# if __virtual__() evaluates to False then the module
|
||||
# wasn't meant for this platform or it's not supposed to
|
||||
# load for some other reason.
|
||||
|
||||
# Some modules might accidentally return None and are
|
||||
# improperly loaded
|
||||
if virtual is None:
|
||||
log.warning(
|
||||
'{0}.__virtual__() is wrongly returning `None`. '
|
||||
'It should either return `True`, `False` or a new '
|
||||
'name. If you\'re the developer of the module '
|
||||
'{1!r}, please fix this.'.format(
|
||||
mod.__name__,
|
||||
module_name
|
||||
)
|
||||
)
|
||||
|
||||
return (False, module_name, error_reasons)
|
||||
|
||||
# At this point, __virtual__ did not return a
|
||||
# boolean value, let's check for deprecated usage
|
||||
# or module renames
|
||||
if virtual is not True and module_name == virtual:
|
||||
# The module was not renamed, it should
|
||||
# have returned True instead
|
||||
#salt.utils.warn_until(
|
||||
# 'Helium',
|
||||
# 'The {0!r} module is NOT renaming itself and is '
|
||||
# 'returning a string. In this case the __virtual__() '
|
||||
# 'function should simply return `True`. This usage will '
|
||||
# 'become an error in Salt Helium'.format(
|
||||
# mod.__name__,
|
||||
# )
|
||||
#)
|
||||
pass
|
||||
|
||||
elif virtual is not True and module_name != virtual:
|
||||
# The module is renaming itself. Updating the module name
|
||||
# with the new name
|
||||
log.trace('Loaded {0} as virtual {1}'.format(
|
||||
module_name, virtual
|
||||
))
|
||||
|
||||
if not hasattr(mod, '__virtualname__'):
|
||||
salt.utils.warn_until(
|
||||
'Hydrogen',
|
||||
'The {0!r} module is renaming itself in it\'s '
|
||||
'__virtual__() function ({1} => {2}). Please '
|
||||
'set it\'s virtual name as the '
|
||||
'\'__virtualname__\' module attribute. '
|
||||
'Example: "__virtualname__ = {2!r}"'.format(
|
||||
mod.__name__,
|
||||
module_name,
|
||||
virtual
|
||||
)
|
||||
)
|
||||
|
||||
if virtualname != virtual:
|
||||
# The __virtualname__ attribute does not match what's
|
||||
# being returned by the __virtual__() function. This
|
||||
# should be considered an error.
|
||||
log.error(
|
||||
'The module {0!r} is showing some bad usage. It\'s '
|
||||
'__virtualname__ attribute is set to {1!r} yet the '
|
||||
'__virtual__() function is returning {2!r}. These '
|
||||
'values should match!'.format(
|
||||
mod.__name__,
|
||||
virtualname,
|
||||
virtual
|
||||
)
|
||||
)
|
||||
|
||||
module_name = virtualname
|
||||
|
||||
# If the __virtual__ function returns True and __virtualname__ is set then use it
|
||||
elif virtual is True and virtualname != module_name:
|
||||
if virtualname is not True:
|
||||
module_name = virtualname
|
||||
|
||||
except KeyError:
|
||||
# Key errors come out of the virtual function when passing
|
||||
# in incomplete grains sets, these can be safely ignored
|
||||
# and logged to debug, still, it includes the traceback to
|
||||
# help debugging.
|
||||
log.debug(
|
||||
'KeyError when loading {0}'.format(module_name),
|
||||
exc_info=True
|
||||
)
|
||||
|
||||
except Exception:
|
||||
# If the module throws an exception during __virtual__()
|
||||
# then log the information and continue to the next.
|
||||
log.error(
|
||||
'Failed to read the virtual function for '
|
||||
'{0}: {1}'.format(
|
||||
self.tag, module_name
|
||||
),
|
||||
exc_info=True
|
||||
)
|
||||
return (False, module_name, error_reasons)
|
||||
|
||||
return (True, module_name, [])
|
||||
|
||||
def _apply_outputter(self, func, mod):
|
||||
'''
|
||||
Apply the __outputter__ variable to the functions
|
||||
@ -2041,174 +1507,3 @@ class Loader(object):
|
||||
outp = mod.__outputter__
|
||||
if func.__name__ in outp:
|
||||
func.__outputter__ = outp[func.__name__]
|
||||
|
||||
def filter_func(self, name, pack=None, whitelist=None):
|
||||
'''
|
||||
Filter a specific function out of the functions, this is used to load
|
||||
the returners for the salt minion
|
||||
'''
|
||||
funcs = {}
|
||||
gen = self.gen_functions(pack=pack, whitelist=whitelist)
|
||||
for key, fun in six.iteritems(gen):
|
||||
# if the name (after '.') is "name", then rename to mod_name: fun
|
||||
if key == '_errors':
|
||||
continue
|
||||
if key[key.index('.') + 1:] == name:
|
||||
funcs[key[:key.index('.')]] = fun
|
||||
return funcs
|
||||
|
||||
def chop_mods(self):
|
||||
'''
|
||||
Chop off the module names so that the raw functions are exposed,
|
||||
used to generate the grains
|
||||
'''
|
||||
funcs = {}
|
||||
for key, fun in six.iteritems(self.gen_functions()):
|
||||
funcs[key[key.rindex('.')] + 1:] = fun
|
||||
return funcs
|
||||
|
||||
|
||||
class LazyLoader(MutableMapping):
|
||||
'''
|
||||
Lazily load things modules. If anyone asks for len or attempts to iterate this
|
||||
will load them all.
|
||||
|
||||
TODO: negative caching? If you ask for 'foo.bar' and it doesn't exist it will
|
||||
look EVERY time unless someone calls load_all()
|
||||
'''
|
||||
def __init__(self,
|
||||
loader,
|
||||
functions=None,
|
||||
pack=None,
|
||||
whitelist=None):
|
||||
# create a dict to store module functions in
|
||||
self._dict = {}
|
||||
|
||||
self.loader = loader
|
||||
if not functions:
|
||||
self.functions = {}
|
||||
else:
|
||||
self.functions = functions
|
||||
self.pack = pack
|
||||
self.whitelist = whitelist
|
||||
|
||||
# have we already loded everything?
|
||||
self.loaded = False
|
||||
|
||||
def _load(self, key):
|
||||
'''
|
||||
Load a single item if you have it
|
||||
'''
|
||||
# if the key doesn't have a '.' then it isn't valid for this mod dict
|
||||
if '.' not in key:
|
||||
raise KeyError
|
||||
mod_key = key.split('.', 1)[0]
|
||||
if self.whitelist:
|
||||
# if the modulename isn't in the whitelist, don't bother
|
||||
if mod_key not in self.whitelist:
|
||||
raise KeyError
|
||||
mod_funcs = self.loader.gen_module(mod_key,
|
||||
self.functions,
|
||||
pack=self.pack,
|
||||
virtual_enable=True
|
||||
)
|
||||
# if you loaded nothing, then we don't have it
|
||||
if mod_funcs is None:
|
||||
# if we couldn't find it, then it could be a virtual or we don't have it
|
||||
# until we have a better way, we have to load them all to know
|
||||
# TODO: maybe do a load until, with some glob match first?
|
||||
self.load_all()
|
||||
return self._dict[key]
|
||||
elif mod_funcs is False: # i.e., the virtual check failed
|
||||
return False
|
||||
self._dict.update(mod_funcs)
|
||||
return True
|
||||
|
||||
def load_all(self):
|
||||
'''
|
||||
Load all of them
|
||||
'''
|
||||
self._dict.update(self.loader.gen_functions(pack=self.pack,
|
||||
whitelist=self.whitelist))
|
||||
self.loaded = True
|
||||
|
||||
def __setitem__(self, key, val):
|
||||
self._dict[key] = val
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._dict[key]
|
||||
|
||||
def __getitem__(self, key):
|
||||
'''
|
||||
Check if the key is ttld out, then do the get
|
||||
'''
|
||||
if key not in self._dict and not self.loaded:
|
||||
# load the item
|
||||
mod_load = self._load(key)
|
||||
if mod_load:
|
||||
log.debug('LazyLoaded {0}'.format(key))
|
||||
return self._dict[key]
|
||||
elif mod_load is False:
|
||||
log.debug('Could not LazyLoad {0}'.format(key))
|
||||
return None
|
||||
else:
|
||||
return self._dict[key]
|
||||
|
||||
def __len__(self):
|
||||
# if not loaded,
|
||||
if not self.loaded:
|
||||
self.load_all()
|
||||
return len(self._dict)
|
||||
|
||||
def __iter__(self):
|
||||
if not self.loaded:
|
||||
self.load_all()
|
||||
return iter(self._dict)
|
||||
|
||||
|
||||
class LazyFilterLoader(LazyLoader):
|
||||
'''
|
||||
Subclass of LazyLoader which filters the module names (for things such as ext_pillar)
|
||||
which have all modules with a single function that we care about
|
||||
'''
|
||||
def __init__(self,
|
||||
loader,
|
||||
name,
|
||||
functions=None,
|
||||
pack=None,
|
||||
whitelist=None):
|
||||
self.name = name
|
||||
LazyLoader.__init__(self,
|
||||
loader,
|
||||
functions=functions,
|
||||
pack=pack,
|
||||
whitelist=whitelist)
|
||||
|
||||
def _load(self, key):
|
||||
if self.whitelist:
|
||||
# if the modulename isn't in the whitelist, don't bother
|
||||
if key not in self.whitelist:
|
||||
raise KeyError
|
||||
mod_funcs = self.loader.gen_module(key,
|
||||
self.functions,
|
||||
pack=self.pack,
|
||||
)
|
||||
# if you loaded nothing, then we don't have it
|
||||
if mod_funcs is None:
|
||||
# if we couldn't find it, then it could be a virtual or we don't have it
|
||||
# until we have a better way, we have to load them all to know
|
||||
# TODO: maybe do a load until, with some glob match first?
|
||||
self.load_all()
|
||||
return self._dict[key]
|
||||
|
||||
# if we got one, now lets check if we have the function name we want
|
||||
for mod_key, mod_fun in six.iteritems(mod_funcs):
|
||||
# if the name (after '.') is "name", then rename to mod_name: fun
|
||||
if mod_key[mod_key.index('.') + 1:] == self.name:
|
||||
self._dict[mod_key[:mod_key.index('.')]] = mod_fun
|
||||
|
||||
def load_all(self):
|
||||
filtered_funcs = self.loader.filter_func(self.name,
|
||||
pack=self.pack,
|
||||
whitelist=self.whitelist)
|
||||
self._dict.update(filtered_funcs)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user