Merge pull request #48344 from gtmanfred/proxycaller

allow using proxy minion from cli

doc/ref/clients/index.rst

@@ -88,6 +88,12 @@ Salt Caller
 .. autoclass:: salt.client.Caller
     :members: cmd
 
+Salt Proxy Caller
+-----------------
+
+.. autoclass:: salt.client.ProxyCaller
+    :members: cmd
+
 RunnerClient
 ------------
 

salt/cli/caller.py

@@ -99,7 +99,10 @@ class BaseCaller(object):
         # be imported as part of the salt api doesn't do  a
         # nasty sys.exit() and tick off our developer users
         try:
-            self.minion = salt.minion.SMinion(opts)
+            if self.opts.get('proxyid'):
+                self.minion = salt.minion.SProxyMinion(opts)
+            else:
+                self.minion = salt.minion.SMinion(opts)
         except SaltClientError as exc:
             raise SystemExit(six.text_type(exc))
 
@@ -208,8 +211,24 @@ class BaseCaller(object):
                     'Do you have permissions to '
                     'write to {0} ?\n'.format(proc_fn))
             func = self.minion.functions[fun]
+            data = {
+              'arg': args,
+              'fun': fun
+            }
+            data.update(kwargs)
+            executors = getattr(self.minion, 'module_executors', []) or \
+                        self.opts.get('module_executors', ['direct_call'])
+            if isinstance(executors, six.string_types):
+                executors = [executors]
             try:
                 ret['return'] = func(*args, **kwargs)
+                for name in executors:
+                    fname = '{0}.execute'.format(name)
+                    if fname not in self.minion.executors:
+                        raise SaltInvocationError("Executor '{0}' is not available".format(name))
+                    ret['return'] = self.minion.executors[fname](self.opts, data, func, args, kwargs)
+                    if ret['return'] is not None:
+                        break
             except TypeError as exc:
                 sys.stderr.write('\nPassed invalid arguments: {0}.\n\nUsage:\n'.format(exc))
                 salt.utils.stringutils.print_cli(func.__doc__)

salt/client/__init__.py

@@ -1993,3 +1993,78 @@ class Caller(object):
             salt.utils.args.parse_input(args),
             kwargs)
         return func(*args, **kwargs)
+
+
+class ProxyCaller(object):
+    '''
+    ``ProxyCaller`` is the same interface used by the :command:`salt-call`
+    with the args ``--proxyid <proxyid>`` command-line tool on the Salt Proxy
+    Minion.
+
+    Importing and using ``ProxyCaller`` must be done on the same machine as a
+    Salt Minion and it must be done using the same user that the Salt Minion is
+    running as.
+
+    Usage:
+
+    .. code-block:: python
+
+        import salt.client
+        caller = salt.client.Caller()
+        caller.cmd('test.ping')
+
+    Note, a running master or minion daemon is not required to use this class.
+    Running ``salt-call --local`` simply sets :conf_minion:`file_client` to
+    ``'local'``. The same can be achieved at the Python level by including that
+    setting in a minion config file.
+
+    .. code-block:: python
+
+        import salt.client
+        import salt.config
+        __opts__ = salt.config.proxy_config('/etc/salt/proxy', minion_id='quirky_edison')
+        __opts__['file_client'] = 'local'
+        caller = salt.client.ProxyCaller(mopts=__opts__)
+
+    .. note::
+
+        To use this for calling proxies, the :py:func:`is_proxy functions
+        <salt.utils.platform.is_proxy>` requires that ``--proxyid`` be an
+        argument on the commandline for the script this is used in, or that the
+        string ``proxy`` is in the name of the script.
+    '''
+    def __init__(self, c_path=os.path.join(syspaths.CONFIG_DIR, 'proxy'), mopts=None):
+        # Late-import of the minion module to keep the CLI as light as possible
+        import salt.minion
+        self.opts = mopts or salt.config.proxy_config(c_path)
+        self.sminion = salt.minion.SProxyMinion(self.opts)
+
+    def cmd(self, fun, *args, **kwargs):
+        '''
+        Call an execution module with the given arguments and keyword arguments
+
+        .. code-block:: python
+
+            caller.cmd('test.arg', 'Foo', 'Bar', baz='Baz')
+
+            caller.cmd('event.send', 'myco/myevent/something',
+                data={'foo': 'Foo'}, with_env=['GIT_COMMIT'], with_grains=True)
+        '''
+        func = self.sminion.functions[fun]
+        data = {
+          'arg': args,
+          'fun': fun
+        }
+        data.update(kwargs)
+        executors = getattr(self.sminion, 'module_executors', []) or \
+                    self.opts.get('module_executors', ['direct_call'])
+        if isinstance(executors, six.string_types):
+            executors = [executors]
+        for name in executors:
+            fname = '{0}.execute'.format(name)
+            if fname not in self.sminion.executors:
+                raise SaltInvocationError("Executor '{0}' is not available".format(name))
+            return_data = self.sminion.executors[fname](self.opts, data, func, args, kwargs)
+            if return_data is not None:
+                break
+        return return_data

salt/minion.py

@@ -3787,3 +3787,92 @@ class ProxyMinion(Minion):
                 Minion._thread_multi_return(minion_instance, opts, data)
             else:
                 Minion._thread_return(minion_instance, opts, data)
+
+
+class SProxyMinion(SMinion):
+    '''
+    Create an object that has loaded all of the minion module functions,
+    grains, modules, returners etc.  The SProxyMinion allows developers to
+    generate all of the salt minion functions and present them with these
+    functions for general use.
+    '''
+    def gen_modules(self, initial_load=False):
+        '''
+        Tell the minion to reload the execution modules
+
+        CLI Example:
+
+        .. code-block:: bash
+
+            salt '*' sys.reload_modules
+        '''
+        self.opts['grains'] = salt.loader.grains(self.opts)
+        self.opts['pillar'] = salt.pillar.get_pillar(
+            self.opts,
+            self.opts['grains'],
+            self.opts['id'],
+            saltenv=self.opts['saltenv'],
+            pillarenv=self.opts.get('pillarenv'),
+        ).compile_pillar()
+
+        if 'proxy' not in self.opts['pillar'] and 'proxy' not in self.opts:
+            errmsg = (
+                'No "proxy" configuration key found in pillar or opts '
+                'dictionaries for id {id}. Check your pillar/options '
+                'configuration and contents. Salt-proxy aborted.'
+            ).format(id=self.opts['id'])
+            log.error(errmsg)
+            self._running = False
+            raise SaltSystemExit(code=salt.defaults.exitcodes.EX_GENERIC, msg=errmsg)
+
+        if 'proxy' not in self.opts:
+            self.opts['proxy'] = self.opts['pillar']['proxy']
+
+        # Then load the proxy module
+        self.proxy = salt.loader.proxy(self.opts)
+
+        self.utils = salt.loader.utils(self.opts, proxy=self.proxy)
+
+        self.functions = salt.loader.minion_mods(self.opts, utils=self.utils, notify=False, proxy=self.proxy)
+        self.returners = salt.loader.returners(self.opts, self.functions, proxy=self.proxy)
+        self.matcher = Matcher(self.opts, self.functions)
+        self.functions['sys.reload_modules'] = self.gen_modules
+        self.executors = salt.loader.executors(self.opts, self.functions, proxy=self.proxy)
+
+        fq_proxyname = self.opts['proxy']['proxytype']
+
+        # we can then sync any proxymodules down from the master
+        # we do a sync_all here in case proxy code was installed by
+        # SPM or was manually placed in /srv/salt/_modules etc.
+        self.functions['saltutil.sync_all'](saltenv=self.opts['saltenv'])
+
+        self.functions.pack['__proxy__'] = self.proxy
+        self.proxy.pack['__salt__'] = self.functions
+        self.proxy.pack['__ret__'] = self.returners
+        self.proxy.pack['__pillar__'] = self.opts['pillar']
+
+        # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__
+        self.utils = salt.loader.utils(self.opts, proxy=self.proxy)
+        self.proxy.pack['__utils__'] = self.utils
+
+        # Reload all modules so all dunder variables are injected
+        self.proxy.reload_modules()
+
+        if ('{0}.init'.format(fq_proxyname) not in self.proxy
+                or '{0}.shutdown'.format(fq_proxyname) not in self.proxy):
+            errmsg = 'Proxymodule {0} is missing an init() or a shutdown() or both. '.format(fq_proxyname) + \
+                     'Check your proxymodule.  Salt-proxy aborted.'
+            log.error(errmsg)
+            self._running = False
+            raise SaltSystemExit(code=salt.defaults.exitcodes.EX_GENERIC, msg=errmsg)
+
+        self.module_executors = self.proxy.get('{0}.module_executors'.format(fq_proxyname), lambda: [])()
+        proxy_init_fn = self.proxy[fq_proxyname + '.init']
+        proxy_init_fn(self.opts)
+
+        self.opts['grains'] = salt.loader.grains(self.opts, proxy=self.proxy)
+
+        #  Sync the grains here so the proxy can communicate them to the master
+        self.functions['saltutil.sync_grains'](saltenv='base')
+        self.grains_cache = self.opts['grains']
+        self.ready = True

salt/utils/parsers.py

@@ -1266,6 +1266,28 @@ class ProxyIdMixIn(six.with_metaclass(MixInMeta, object)):
         )
 
 
+class ExecutorsMixIn(six.with_metaclass(MixInMeta, object)):
+    _mixin_prio = 10
+
+    def _mixin_setup(self):
+        self.add_option(
+            '--module-executors',
+            dest='module_executors',
+            default=None,
+            metavar='EXECUTOR_LIST',
+            help=('Set an alternative list of executors to override the one '
+                  'set in minion config.')
+        )
+        self.add_option(
+            '--executor-opts',
+            dest='executor_opts',
+            default=None,
+            metavar='EXECUTOR_OPTS',
+            help=('Set alternate executor options if supported by executor. '
+                  'Options set by minion config are used by default.')
+        )
+
+
 class CacheDirMixIn(six.with_metaclass(MixInMeta, object)):
     _mixin_prio = 40
 
@@ -1879,6 +1901,7 @@ class SaltCMDOptionParser(six.with_metaclass(OptionParserMeta,
                                              ExtendedTargetOptionsMixIn,
                                              OutputOptionsMixIn,
                                              LogLevelMixIn,
+                                             ExecutorsMixIn,
                                              HardCrashMixin,
                                              SaltfileMixIn,
                                              ArgsStdinMixIn,
@@ -2016,22 +2039,6 @@ class SaltCMDOptionParser(six.with_metaclass(OptionParserMeta,
             metavar='RETURNER_KWARGS',
             help=('Set any returner options at the command line.')
         )
-        self.add_option(
-            '--module-executors',
-            dest='module_executors',
-            default=None,
-            metavar='EXECUTOR_LIST',
-            help=('Set an alternative list of executors to override the one '
-                  'set in minion config.')
-        )
-        self.add_option(
-            '--executor-opts',
-            dest='executor_opts',
-            default=None,
-            metavar='EXECUTOR_OPTS',
-            help=('Set alternate executor options if supported by executor. '
-                  'Options set by minion config are used by default.')
-        )
         self.add_option(
             '-d', '--doc', '--documentation',
             dest='doc',
@@ -2572,7 +2579,9 @@ class SaltKeyOptionParser(six.with_metaclass(OptionParserMeta,
 
 class SaltCallOptionParser(six.with_metaclass(OptionParserMeta,
                                               OptionParser,
+                                              ProxyIdMixIn,
                                               ConfigDirMixIn,
+                                              ExecutorsMixIn,
                                               MergeConfigMixIn,
                                               LogLevelMixIn,
                                               OutputOptionsMixIn,
@@ -2732,8 +2741,13 @@ class SaltCallOptionParser(six.with_metaclass(OptionParserMeta,
             self.config['arg'] = self.args[1:]
 
     def setup_config(self):
-        opts = config.minion_config(self.get_config_file_path(),
-                                    cache_minion_id=True)
+        if self.options.proxyid:
+            opts = config.proxy_config(self.get_config_file_path(configfile='proxy'),
+                                       cache_minion_id=True,
+                                       minion_id=self.options.proxyid)
+        else:
+            opts = config.minion_config(self.get_config_file_path(),
+                                        cache_minion_id=True)
 
         if opts.get('transport') == 'raet':
             if not self._find_raet_minion(opts):  # must create caller minion

salt/utils/platform.py

@@ -37,7 +37,10 @@ def is_proxy():
     try:
         # Changed this from 'salt-proxy in main...' to 'proxy in main...'
         # to support the testsuite's temp script that is called 'cli_salt_proxy'
-        if 'proxy' in main.__file__:
+        #
+        # Add '--proxyid' in sys.argv so that salt-call --proxyid
+        # is seen as a proxy minion
+        if 'proxy' in main.__file__ or '--proxyid' in sys.argv:
             ret = True
     except AttributeError:
         pass

tests/integration/proxy/test_shell.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+'''
+Test salt-call --proxyid commands
+
+tests.integration.proxy.test_shell
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+'''
+
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+# Import Salt Libs
+import salt.utils.json as json
+
+# Import salt tests libs
+from tests.support.case import ShellCase
+
+
+class ProxyCallerSimpleTestCase(ShellCase):
+    '''
+    Test salt-call --proxyid <proxyid> commands
+    '''
+    @staticmethod
+    def _load_return(ret):
+        return json.loads('\n'.join(ret))
+
+    def test_can_it_ping(self):
+        '''
+        Ensure the proxy can ping
+        '''
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json test.ping'))
+        self.assertEqual(ret['local'], True)
+
+    def test_list_pkgs(self):
+        '''
+        Package test 1, really just tests that the virtual function capability
+        is working OK.
+        '''
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json pkg.list_pkgs'))
+        self.assertIn('coreutils', ret['local'])
+        self.assertIn('apache', ret['local'])
+        self.assertIn('redbull', ret['local'])
+
+    def test_upgrade(self):
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json pkg.upgrade'))
+        self.assertEqual(ret['local']['coreutils']['new'], '3.0')
+        self.assertEqual(ret['local']['redbull']['new'], '1001.99')
+
+    def test_service_list(self):
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.list'))
+        self.assertIn('ntp', ret['local'])
+
+    def test_service_start(self):
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.start samba'))
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.status samba'))
+        self.assertTrue(ret)
+
+    def test_service_get_all(self):
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.get_all'))
+        self.assertIn('samba', ret['local'])
+
+    def test_grains_items(self):
+        ret = self._load_return(self.run_call('--proxyid proxytest --out=json grains.items'))
+        self.assertEqual(ret['local']['kernel'], 'proxy')
+        self.assertEqual(ret['local']['kernelrelease'], 'proxy')