Merge pull request #27731 from cro/dunder_proxy

Add __proxy__ to replace opts['proxymodule']
This commit is contained in:
Erik Johnson 2015-10-12 15:41:22 -05:00
commit 922e2018ef
18 changed files with 175 additions and 184 deletions

View File

@ -162,7 +162,8 @@ additional-builtins=__opts__,
__master_opts__,
__jid_event__,
__instance_id__,
__salt_system_encoding__
__salt_system_encoding__,
__proxy__
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.

View File

@ -225,7 +225,8 @@ additional-builtins=__opts__,
__master_opts__,
__jid_event__,
__instance_id__,
__salt_system_encoding__
__salt_system_encoding__,
__proxy__
# List of strings which can identify a callback function by name. A callback

View File

@ -31,7 +31,7 @@ class SSHState(salt.state.State):
self.wrapper = wrapper
super(SSHState, self).__init__(opts, pillar)
def load_modules(self, data=None):
def load_modules(self, data=None, proxy=None):
'''
Load up the modules for remote compilation via ssh
'''

View File

@ -434,6 +434,10 @@ VALID_OPTS = {
# A master-only copy of the file_roots dictionary, used by the state compiler
'master_roots': dict,
# Add the proxymodule LazyLoader object to opts. This breaks many things
# but this was the default pre 2015.8.2. This should default to
# False in Boron
'add_proxymodule_to_opts': bool,
'git_pillar_base': str,
'git_pillar_branch': str,
'git_pillar_env': str,
@ -1184,6 +1188,7 @@ DEFAULT_MASTER_OPTS = {
DEFAULT_PROXY_MINION_OPTS = {
'conf_file': os.path.join(salt.syspaths.CONFIG_DIR, 'proxy'),
'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'proxy'),
'add_proxymodule_to_opts': True
}
# ----- Salt Cloud Configuration Defaults ----------------------------------->

View File

@ -1370,7 +1370,7 @@ def locale_info():
grains = {}
grains['locale_info'] = {}
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return grains
try:
@ -1399,7 +1399,7 @@ def hostname():
# domain
grains = {}
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return grains
grains['localhost'] = socket.gethostname()
@ -1415,7 +1415,7 @@ def append_domain():
grain = {}
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return grain
if 'append_domain' in __opts__:
@ -1428,7 +1428,7 @@ def ip4():
Return a list of ipv4 addrs
'''
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
return {'ipv4': salt.utils.network.ip_addrs(include_loopback=True)}
@ -1439,7 +1439,7 @@ def fqdn_ip4():
Return a list of ipv4 addrs of fqdn
'''
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
try:
@ -1455,7 +1455,7 @@ def ip6():
Return a list of ipv6 addrs
'''
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
return {'ipv6': salt.utils.network.ip_addrs6(include_loopback=True)}
@ -1466,7 +1466,7 @@ def fqdn_ip6():
Return a list of ipv6 addrs of fqdn
'''
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
try:
@ -1484,7 +1484,7 @@ def ip_interfaces():
# Provides:
# ip_interfaces
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
ret = {}
@ -1511,7 +1511,7 @@ def ip4_interfaces():
# Provides:
# ip_interfaces
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
ret = {}
@ -1535,7 +1535,7 @@ def ip6_interfaces():
# Provides:
# ip_interfaces
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
ret = {}
@ -1678,7 +1678,7 @@ def _hw_data(osdata):
.. versionadded:: 0.9.5
'''
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
grains = {}
@ -1767,7 +1767,7 @@ def _smartos_zone_data():
# hypervisor_uuid
# datacenter
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
grains = {}
@ -1819,7 +1819,7 @@ def get_server_id():
# Provides:
# server_id
if 'proxyminion' in __opts__:
if salt.utils.is_proxy():
return {}
return {'server_id': abs(hash(__opts__.get('id', '')) % (2 ** 31))}

View File

@ -2,13 +2,16 @@
'''
Generate baseline proxy minion grains
'''
from __future__ import absolute_import
import salt.utils
__proxyenabled__ = ['rest_sample']
__virtualname__ = 'rest_sample'
def __virtual__():
if 'proxy' not in __opts__:
if not salt.utils.is_proxy():
return False
else:
return __virtualname__

View File

@ -137,7 +137,8 @@ def minion_mods(
include_errors=False,
initial_load=False,
loaded_base_name=None,
notify=False):
notify=False,
proxy=None):
'''
Load execution modules
@ -173,17 +174,20 @@ def minion_mods(
__salt__['test.ping']()
'''
# TODO Publish documentation for module whitelisting
if context is None:
context = {}
if utils is None:
utils = {}
if proxy is None:
proxy = {}
if not whitelist:
whitelist = opts.get('whitelist_modules', None)
ret = LazyLoader(_module_dirs(opts, 'modules', 'module'),
opts,
tag='module',
pack={'__context__': context, '__utils__': utils},
pack={'__context__': context, '__utils__': utils,
'__proxy__': proxy},
whitelist=whitelist,
loaded_base_name=loaded_base_name)
@ -252,16 +256,19 @@ def engines(opts, functions, runners):
pack=pack)
def proxy(opts, functions, whitelist=None, loaded_base_name=None):
def proxy(opts, functions=None, returners=None, whitelist=None):
'''
Returns the proxy module for this salt-proxy-minion
'''
return LazyLoader(_module_dirs(opts, 'proxy', 'proxy'),
opts,
tag='proxy',
whitelist=whitelist,
pack={'__proxy__': functions},
loaded_base_name=loaded_base_name)
ret = LazyLoader(_module_dirs(opts, 'proxy', 'proxy'),
opts,
tag='proxy',
pack={'__salt__': functions,
'__ret__': returners})
ret.pack['__proxy__'] = ret
return ret
def returners(opts, functions, whitelist=None, context=None):

View File

@ -531,10 +531,10 @@ class SMinion(MinionBase):
self.utils = salt.loader.utils(self.opts)
self.functions = salt.loader.minion_mods(self.opts, utils=self.utils,
include_errors=True)
self.proxy = salt.loader.proxy(self.opts, None)
self.returners = salt.loader.returners(self.opts, self.functions)
self.proxy = salt.loader.proxy(self.opts, self.functions, self.returners, None)
# TODO: remove
self.function_errors = {} # Keep the funcs clean
self.returners = salt.loader.returners(self.opts, self.functions)
self.states = salt.loader.states(self.opts, self.functions, self.utils)
self.rend = salt.loader.render(self.opts, self.functions)
self.matcher = Matcher(self.opts, self.functions)
@ -703,8 +703,9 @@ class Minion(MinionBase):
'upgrade your ZMQ!'
)
# Late setup the of the opts grains, so we can log from the grains
# module
if 'proxyid' not in self.opts:
# module. If this is a proxy, however, we need to init the proxymodule
# before we can get the grains.
if not salt.utils.is_proxy():
self.opts['grains'] = salt.loader.grains(opts)
# TODO: remove?
@ -853,7 +854,7 @@ class Minion(MinionBase):
)
self.event_publisher.handle_publish([event])
def _load_modules(self, force_refresh=False, notify=False):
def _load_modules(self, force_refresh=False, notify=False, proxy=None):
'''
Return the functions and the returners loaded up from the loader
module
@ -879,10 +880,10 @@ class Minion(MinionBase):
self.utils = salt.loader.utils(self.opts)
if self.opts.get('multimaster', False):
s_opts = copy.deepcopy(self.opts)
functions = salt.loader.minion_mods(s_opts, utils=self.utils,
functions = salt.loader.minion_mods(s_opts, utils=self.utils, proxy=proxy,
loaded_base_name=self.loaded_base_name, notify=notify)
else:
functions = salt.loader.minion_mods(self.opts, utils=self.utils, notify=notify)
functions = salt.loader.minion_mods(self.opts, utils=self.utils, notify=notify, proxy=proxy)
returners = salt.loader.returners(self.opts, functions)
errors = {}
if '_errors' in functions:
@ -1330,7 +1331,11 @@ class Minion(MinionBase):
Refresh the functions and returners.
'''
log.debug('Refreshing modules. Notify={0}'.format(notify))
self.functions, self.returners, _ = self._load_modules(force_refresh, notify=notify)
if hasattr(self, 'proxy'):
self.functions, self.returners, _ = self._load_modules(force_refresh, notify=notify, proxy=self.proxy)
else:
self.functions, self.returners, _ = self._load_modules(force_refresh, notify=notify)
self.schedule.functions = self.functions
self.schedule.returners = self.returners
@ -2491,6 +2496,7 @@ class ProxyMinion(Minion):
to know which master they connected to)
'''
log.debug("subclassed _post_master_init")
self.opts['master'] = master
self.opts['pillar'] = yield salt.pillar.get_async_pillar(
@ -2510,23 +2516,36 @@ class ProxyMinion(Minion):
fq_proxyname = self.opts['pillar']['proxy']['proxytype']
self.opts['proxy'] = self.opts['pillar']['proxy']
# We need to do this again, because we are going to throw out a lot of grains.
self.opts['grains'] = salt.loader.grains(self.opts)
self.opts['proxymodule'] = salt.loader.proxy(self.opts, None, loaded_base_name=fq_proxyname)
# Need to load the modules so they get all the dunder variables
self.functions, self.returners, self.function_errors = self._load_modules()
if ('{0}.init'.format(fq_proxyname) not in self.opts['proxymodule']
or '{0}.shutdown'.format(fq_proxyname) not in self.opts['proxymodule']):
# we can then sync any proxymodules down from the master
self.functions['saltutil.sync_proxymodules'](saltenv='base')
# Then load the proxy module
self.proxy = salt.loader.proxy(self.opts)
# And re-load the modules so the __proxy__ variable gets injected
self.functions, self.returners, self.function_errors = self._load_modules(proxy=self.proxy)
self.functions.pack['__proxy__'] = self.proxy
self.proxy.pack['__salt__'] = self.functions
self.proxy.pack['__ret__'] = self.returners
if ('{0}.init'.format(fq_proxyname) not in self.proxy
or '{0}.shutdown'.format(fq_proxyname) not in self.proxy):
log.error('Proxymodule {0} is missing an init() or a shutdown() or both.'.format(fq_proxyname))
log.error('Check your proxymodule. Salt-proxy aborted.')
self._running = False
raise SaltSystemExit(code=-1)
proxy_fn = self.opts['proxymodule'].loaded_base_name + '.init'
proxy_init_fn = self.proxy[fq_proxyname+'.init']
proxy_init_fn(self.opts)
self.opts['grains'] = salt.loader.grains(self.opts)
# Check config 'add_proxymodule_to_opts' Remove this in Boron.
if self.opts['add_proxymodule_to_opts']:
self.opts['proxymodule'] = self.proxy
self.opts['proxymodule'][proxy_fn](self.opts)
# reload ?!?
self.serial = salt.payload.Serial(self.opts)
self.mod_opts = self._prep_mod_opts()
self.matcher = Matcher(self.opts, self.functions)

View File

@ -6,6 +6,7 @@ from __future__ import absolute_import
# Import python libs
import logging
import salt.utils
log = logging.getLogger(__name__)
@ -19,22 +20,22 @@ def __virtual__():
'''
Only work on proxy
'''
if 'proxymodule' in __opts__:
if salt.utils.is_proxy():
return __virtualname__
return False
def list_pkgs(versions_as_list=False, **kwargs):
return __opts__['proxymodule']['rest_sample.package_list']()
return __proxy__['rest_sample.package_list']()
def install(name=None, refresh=False, fromrepo=None,
pkgs=None, sources=None, **kwargs):
return __opts__['proxymodule']['rest_sample.package_install'](name, **kwargs)
return __proxy__['rest_sample.package_install'](name, **kwargs)
def remove(name=None, pkgs=None, **kwargs):
return __opts__['proxymodule']['rest_sample.package_remove'](name)
return __proxy__['rest_sample.package_remove'](name)
def version(*names, **kwargs):
@ -51,7 +52,7 @@ def version(*names, **kwargs):
salt '*' pkg.version <package1> <package2> <package3> ...
'''
if len(names) == 1:
return str(__opts__['proxymodule']['rest_sample.package_status'](names))
return str(__proxy__['rest_sample.package_status'](names))
def installed(
@ -64,7 +65,7 @@ def installed(
sources=None,
**kwargs):
p = __opts__['proxymodule']['rest_sample.package_status'](name)
p = __proxy__['rest_sample.package_status'](name)
if version is None:
if 'ret' in p:
return str(p['ret'])

View File

@ -1,47 +0,0 @@
# -*- coding: utf-8 -*-
'''
Module for interfacing to the REST example
pre-pre-ALPHA QUALITY code.
'''
from __future__ import absolute_import
# Import python libraries
import logging
# Set up logging
log = logging.getLogger(__name__)
# Define the module's virtual name
__virtualname__ = 'rest_example'
__proxyenabled__ = ['rest_example']
def __virtual__():
'''
'''
if 'proxyobject' in __opts__:
return __virtualname__
else:
return False
def grains_refresh():
'''
Refresh the cache.
'''
return __opts__['proxyobject'].grains_refresh()
def ping():
ret = dict()
conn = __opts__['proxyobject']
if conn.ping():
ret['message'] = 'pong'
ret['out'] = True
else:
ret['out'] = False

View File

@ -40,8 +40,8 @@ def get_all():
salt '*' service.get_all
'''
proxy_fn = 'rest_sample'+ '.service_list'
return __opts__['proxymodule'][proxy_fn]()
proxy_fn = 'rest_sample.service_list'
return __proxy__[proxy_fn]()
def list_():
@ -72,8 +72,8 @@ def start(name, sig=None):
salt '*' service.start <service name>
'''
proxy_fn = 'rest_sample'+ '.service_start'
return __opts__['proxymodule'][proxy_fn](name)
proxy_fn = 'rest_sample.service_start'
return __proxy__[proxy_fn](name)
def stop(name, sig=None):
@ -88,8 +88,8 @@ def stop(name, sig=None):
salt '*' service.stop <service name>
'''
proxy_fn = 'rest_sample'+ '.service_stop'
return __opts__['proxymodule'][proxy_fn](name)
proxy_fn = 'rest_sample.service_stop'
return __proxy__[proxy_fn](name)
def restart(name, sig=None):
@ -105,8 +105,8 @@ def restart(name, sig=None):
salt '*' service.restart <service name>
'''
proxy_fn = 'rest_sample'+ '.service_restart'
return __opts__['proxymodule'][proxy_fn](name)
proxy_fn = 'rest_sample.service_restart'
return __proxy__[proxy_fn](name)
def status(name, sig=None):
@ -123,8 +123,8 @@ def status(name, sig=None):
salt '*' service.status <service name>
'''
proxy_fn = 'rest_sample' + '.service_status'
resp = __opts__['proxymodule'][proxy_fn](name)
proxy_fn = 'rest_sample.service_status'
resp = __proxy__[proxy_fn](name)
if resp['comment'] == 'stopped':
return False
if resp['comment'] == 'running':

View File

@ -390,6 +390,25 @@ def sync_returners(saltenv=None, refresh=True):
return ret
def sync_proxymodules(saltenv=None, refresh=False):
'''
Sync the proxy modules from the _proxy directory on the salt master file
server. This function is environment aware, pass the desired environment
to grab the contents of the _returners directory, base is the default
environment.
CLI Example:
.. code-block:: bash
salt '*' saltutil.sync_proxymodules
'''
ret = _sync('proxy', saltenv)
if refresh:
refresh_modules()
return ret
def sync_output(saltenv=None, refresh=True):
'''
Sync the output modules from the _output directory on the salt master file

View File

@ -183,7 +183,10 @@ def low(data, queue=False, **kwargs):
conflict = _check_queue(queue, kwargs)
if conflict is not None:
return conflict
st_ = salt.state.State(__opts__)
try:
st_ = salt.state.State(__opts__, proxy=__proxy__)
except NameError:
st_ = salt.state.State(__opts__)
err = st_.verify_data(data)
if err:
__context__['retcode'] = 1
@ -224,7 +227,11 @@ def high(data, test=False, queue=False, **kwargs):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary'
)
st_ = salt.state.State(__opts__, pillar)
try:
st_ = salt.state.State(__opts__, pillar, proxy=__proxy__)
except NameError:
st_ = salt.state.State(__opts__, pillar)
ret = st_.call_high(data)
_set_retcode(ret)
return ret
@ -283,7 +290,10 @@ def template_str(tem, queue=False, **kwargs):
conflict = _check_queue(queue, kwargs)
if conflict is not None:
return conflict
st_ = salt.state.State(__opts__)
try:
st_ = salt.state.State(__opts__, proxy=__proxy__)
except NameError:
st_ = salt.state.State(__opts__)
ret = st_.call_template_str(tem)
_set_retcode(ret)
return ret
@ -529,7 +539,11 @@ def highstate(test=None,
if 'pillarenv' in kwargs:
opts['pillarenv'] = kwargs['pillarenv']
st_ = salt.state.HighState(opts, pillar, kwargs.get('__pub_jid'))
try:
st_ = salt.state.HighState(opts, pillar, kwargs.get('__pub_jid'), proxy=__proxy__)
except NameError:
st_ = salt.state.HighState(opts, pillar, kwargs.get('__pub_jid'))
st_.push_active()
try:
ret = st_.call_highstate(
@ -681,7 +695,10 @@ def sls(mods,
'{0}.cache.p'.format(kwargs.get('cache_name', 'highstate'))
)
st_ = salt.state.HighState(opts, pillar, kwargs.get('__pub_jid'))
try:
st_ = salt.state.HighState(opts, pillar, kwargs.get('__pub_jid'), proxy=__proxy__)
except NameError:
st_ = salt.state.HighState(opts, pillar, kwargs.get('__pub_jid'))
if kwargs.get('cache'):
if os.path.isfile(cfn):
@ -1104,7 +1121,10 @@ def single(fun, name, test=None, queue=False, **kwargs):
'Pillar data must be formatted as a dictionary'
)
st_ = salt.state.State(opts, pillar)
try:
st_ = salt.state.State(opts, pillar, proxy=__proxy__)
except NameError:
st_ = salt.state.State(opts, pillar)
err = st_.verify_data(kwargs)
if err:
__context__['retcode'] = 1

View File

@ -23,7 +23,7 @@ log = logging.getLogger(__name__)
# Define the module's virtual name
__virtualname__ = 'sys'
__proxyenabled__ = '*'
__proxyenabled__ = ['*']
def __virtual__():

View File

@ -110,11 +110,15 @@ def ping():
salt '*' test.ping
'''
if 'proxymodule' in __opts__:
ping_cmd = __opts__['proxymodule'].loaded_base_name + '.ping'
return __opts__['proxymodule'][ping_cmd]()
else:
if not salt.utils.is_proxy():
return True
else:
ping_cmd = __opts__['proxy']['proxytype'] + '.ping'
if __opts__.get('add_proxymodule_to_opts', False):
return __opts__['proxymodule'][ping_cmd]()
else:
return __proxy__[ping_cmd]()
def sleep(length):

View File

@ -33,12 +33,11 @@ def __virtual__():
log.debug('rest_sample proxy __virtual__() called...')
return True
# Every proxy module needs an 'init', though you can
# just put a 'pass' here if it doesn't need to do anything.
def init(opts):
'''
Every proxy module needs an 'init', though you can
just put a 'pass' here if it doesn't need to do anything.
'''
log.debug('rest_sample proxy init() called...')
# Save the REST URL
@ -49,6 +48,16 @@ def init(opts):
DETAILS['url'] += '/'
def id(opts):
'''
Return a unique ID for this proxy minion. This ID MUST NOT CHANGE.
If it changes while the proxy is running the salt-master will get
really confused and may stop talking to this minion
'''
r = salt.utils.http.query(opts['proxy']['url']+'id', decode_type='json', decode=True)
return r['dict']['id'].encode('ascii', 'ignore')
def grains():
'''
Get the grains from the proxied device

View File

@ -593,14 +593,15 @@ class State(object):
'''
Class used to execute salt states
'''
def __init__(self, opts, pillar=None, jid=None):
def __init__(self, opts, pillar=None, jid=None, proxy=None):
if 'grains' not in opts:
opts['grains'] = salt.loader.grains(opts)
self.opts = opts
self.proxy = proxy
self._pillar_override = pillar
self.opts['pillar'] = self._gather_pillar()
self.state_con = {}
self.load_modules()
self.load_modules(proxy=proxy)
self.active = set()
self.mod_init = set()
self.pre = {}
@ -731,13 +732,15 @@ class State(object):
return ret
return ret
def load_modules(self, data=None):
def load_modules(self, data=None, proxy=None):
'''
Load the modules into the state
'''
log.info('Loading fresh modules for state activity')
self.utils = salt.loader.utils(self.opts)
self.functions = salt.loader.minion_mods(self.opts, self.state_con, utils=self.utils)
self.functions = salt.loader.minion_mods(self.opts, self.state_con,
utils=self.utils,
proxy=proxy)
if isinstance(data, dict):
if data.get('provider', False):
if isinstance(data['provider'], str):
@ -774,7 +777,7 @@ class State(object):
reload(site)
except RuntimeError:
log.error('Error encountered during module reload. Modules were not reloaded.')
self.load_modules()
self.load_modules(proxy=self.proxy)
if not self.opts.get('local', False) and self.opts.get('multiprocessing', True):
self.functions['saltutil.refresh_modules']()
@ -1539,7 +1542,6 @@ class State(object):
initial_ret={'full': state_func_name},
expected_extra_kws=STATE_INTERNAL_KEYWORDS
)
inject_globals = {
# Pass a copy of the running dictionary, the low state chunks and
# the current state dictionaries.
@ -3159,11 +3161,11 @@ class HighState(BaseHighState):
# a stack of active HighState objects during a state.highstate run
stack = []
def __init__(self, opts, pillar=None, jid=None):
def __init__(self, opts, pillar=None, jid=None, proxy=None):
self.opts = opts
self.client = salt.fileclient.get_file_client(self.opts)
BaseHighState.__init__(self, opts)
self.state = State(self.opts, pillar, jid)
self.state = State(self.opts, pillar, jid, proxy=proxy)
self.matcher = salt.minion.Matcher(self.opts)
# tracks all pydsl state declarations globally across sls files
@ -3203,7 +3205,7 @@ class MasterState(State):
def __init__(self, opts, minion):
State.__init__(self, opts)
def load_modules(self, data=None):
def load_modules(self, data=None, proxy=None):
'''
Load the modules into the state
'''

View File

@ -1,53 +0,0 @@
# -*- coding: utf-8 -*-
'''
:codeauthor: :email:`Jayesh Kariya <jayeshk@saltstack.com>`
'''
# Import Python Libs
from __future__ import absolute_import
# Import Salt Testing Libs
from salttesting import TestCase, skipIf
from salttesting.mock import (
MagicMock,
patch,
NO_MOCK,
NO_MOCK_REASON
)
# Import Salt Libs
from salt.modules import rest_sample
# Globals
rest_sample.__opts__ = {}
@skipIf(NO_MOCK, NO_MOCK_REASON)
class RestSampleTestCase(TestCase):
'''
Test cases for salt.modules.rest_sample
'''
# 'grains_refresh' function tests: 1
def test_grains_refresh(self):
'''
Test if it refresh the cache.
'''
mock = MagicMock(return_value=True)
with patch.dict(rest_sample.__opts__, {'proxyobject': mock}):
self.assertTrue(rest_sample.grains_refresh())
# 'ping' function tests: 1
def test_ping(self):
'''
Test if it ping
'''
mock = MagicMock(return_value=True)
with patch.dict(rest_sample.__opts__, {'proxyobject': mock}):
self.assertIsNone(rest_sample.ping())
if __name__ == '__main__':
from integration import run_tests
run_tests(RestSampleTestCase, needs_daemon=False)