Merge pull request #46005 from gtmanfred/ssh

pass opts to SSHClient for enable_ssh_minions

salt/client/ssh/__init__.py

@@ -658,7 +658,9 @@ class SSH(object):
             host = next(six.iterkeys(ret))
             self.cache_job(jid, host, ret[host], fun)
             if self.event:
-                _, data = next(six.iteritems(ret))
+                id_, data = next(six.iteritems(ret))
+                if 'id' not in data:
+                    data['id'] = id_
                 data['jid'] = jid  # make the jid in the payload the same as the jid in the tag
                 self.event.fire_event(
                     data,
@@ -769,7 +771,9 @@ class SSH(object):
                         outputter,
                         self.opts)
             if self.event:
-                _, data = next(six.iteritems(ret))
+                id_, data = next(six.iteritems(ret))
+                if 'id' not in data:
+                    data['id'] = id_
                 data['jid'] = jid  # make the jid in the payload the same as the jid in the tag
                 self.event.fire_event(
                     data,

salt/config/__init__.py

@@ -1809,7 +1809,6 @@ DEFAULT_MASTER_OPTS = {
     'schedule': {},
     'auth_events': True,
     'minion_data_cache_events': True,
-    'enable_ssh': False,
     'enable_ssh_minions': False,
 }
 

salt/master.py

@@ -2005,6 +2005,7 @@ class ClearFuncs(object):
         )
         minions = _res.get('minions', list())
         missing = _res.get('missing', list())
+        ssh_minions = _res.get('ssh_minions', False)
 
         # Check for external auth calls and authenticate
         auth_type, err_name, key, sensitive_load_keys = self._prep_auth_info(extra)
@@ -2072,7 +2073,7 @@ class ClearFuncs(object):
         payload = self._prep_pub(minions, jid, clear_load, extra, missing)
 
         # Send it!
-        minions.extend(self._send_ssh_pub(payload))
+        self._send_ssh_pub(payload, ssh_minions=ssh_minions)
         self._send_pub(payload)
 
         return {
@@ -2135,20 +2136,19 @@ class ClearFuncs(object):
             chan = salt.transport.server.PubServerChannel.factory(opts)
             chan.publish(load)
 
-    def _send_ssh_pub(self, load):
+    @property
+    def ssh_client(self):
+        if not hasattr(self, '_ssh_client'):
+            self._ssh_client = salt.client.ssh.client.SSHClient(mopts=self.opts)
+        return self._ssh_client
+
+    def _send_ssh_pub(self, load, ssh_minions=False):
         '''
-        Take a load and send it across the network to connected minions
+        Take a load and send it across the network to ssh minions
         '''
-        minions = []
+        if self.opts['enable_ssh_minions'] is True and ssh_minions is True:
-        if self.opts['enable_ssh_minions'] is True and isinstance(load['tgt'], six.string_types):
+            log.debug('Send payload to ssh minions')
-            # The isinstances makes sure that syndics work
+            threading.Thread(target=self.ssh_client.cmd, kwargs=load).start()
-            log.debug('Use SSHClient for rostered minions')
-            ssh = salt.client.ssh.client.SSHClient()
-            ssh_minions = ssh._prep_ssh(**load).targets.keys()
-            if ssh_minions:
-                minions.extend(ssh_minions)
-                threading.Thread(target=ssh.cmd, kwargs=load).start()
-        return minions
 
     def _prep_pub(self, minions, jid, clear_load, extra, missing):
         '''

salt/utils/minions.py

@@ -13,6 +13,7 @@ import logging
 
 # Import salt libs
 import salt.payload
+import salt.roster
 import salt.utils.data
 import salt.utils.files
 import salt.utils.network
@@ -682,6 +683,13 @@ class CkMinions(object):
                 _res = check_func(expr, delimiter, greedy)
             else:
                 _res = check_func(expr, greedy)
+            _res['ssh_minions'] = False
+            if self.opts.get('enable_ssh_minions', False) is True and isinstance('tgt', six.string_types):
+                roster = salt.roster.Roster(self.opts, self.opts.get('roster', 'flat'))
+                ssh_minions = roster.targets(expr, tgt_type)
+                if ssh_minions:
+                    _res['minions'].extend(ssh_minions)
+                    _res['ssh_minions'] = True
         except Exception:
             log.exception(
                     'Failed matching available minions with %s pattern: %s',

tests/integration/__init__.py

@@ -671,10 +671,15 @@ class TestDaemon(object):
         else:
             os.environ['SSH_DAEMON_RUNNING'] = 'True'
         roster_path = os.path.join(FILES, 'conf/_ssh/roster')
+        syndic_roster_path = os.path.join(FILES, 'conf/_ssh/syndic_roster')
         shutil.copy(roster_path, RUNTIME_VARS.TMP_CONF_DIR)
+        shutil.copy(syndic_roster_path, os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, 'roster'))
         with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'roster'), 'a') as roster:
             roster.write('  user: {0}\n'.format(RUNTIME_VARS.RUNNING_TESTS_USER))
             roster.write('  priv: {0}/{1}'.format(RUNTIME_VARS.TMP_CONF_DIR, 'key_test'))
+        with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, 'roster'), 'a') as roster:
+            roster.write('  user: {0}\n'.format(RUNTIME_VARS.RUNNING_TESTS_USER))
+            roster.write('  priv: {0}/{1}'.format(RUNTIME_VARS.TMP_CONF_DIR, 'key_test'))
         sys.stdout.write(
             ' {LIGHT_GREEN}STARTED!\n{ENDC}'.format(
                 **self.colors

tests/integration/cli/test_batch.py

@@ -19,7 +19,8 @@ class BatchTest(ShellCase):
         Tests executing a simple batch command to help catch regressions
         '''
         ret = 'Executing run on [\'sub_minion\']'
-        cmd = self.run_salt('\'*\' test.echo \'batch testing\' -b 50%')
+
+        cmd = self.run_salt('\'*minion\' test.echo \'batch testing\' -b 50%')
         self.assertIn(ret, cmd)
 
     def test_batch_run_number(self):
@@ -28,7 +29,7 @@ class BatchTest(ShellCase):
         a percentage with full batch CLI call.
         '''
         ret = "Executing run on ['minion', 'sub_minion']"
-        cmd = self.run_salt('\'*\' test.ping --batch-size 2')
+        cmd = self.run_salt('\'*minion\' test.ping --batch-size 2')
         self.assertIn(ret, cmd)
 
     def test_batch_run_grains_targeting(self):
@@ -45,7 +46,7 @@ class BatchTest(ShellCase):
                 os_grain = item
 
         os_grain = os_grain.strip()
-        cmd = self.run_salt('-G \'os:{0}\' -b 25% test.ping'.format(os_grain))
+        cmd = self.run_salt('-C \'G@os:{0} and not localhost\' -b 25% test.ping'.format(os_grain))
         self.assertIn(sub_min_ret, cmd)
         self.assertIn(min_ret, cmd)
 
@@ -53,5 +54,5 @@ class BatchTest(ShellCase):
         '''
         Test that a failed state returns a non-zero exit code in batch mode
         '''
-        cmd = self.run_salt(' "*" state.single test.fail_without_changes name=test_me -b 25%', with_retcode=True)
+        cmd = self.run_salt(' "*minion" state.single test.fail_without_changes name=test_me -b 33%', with_retcode=True)
         self.assertEqual(cmd[-1], 2)

tests/integration/files/conf/_ssh/syndic_roster

@@ -0,0 +1,5 @@
+syndic_localhost:
+  host: 127.0.0.1
+  port: 2827
+  mine_functions:
+    test.arg: ['itworked']

tests/integration/files/conf/master

@@ -102,3 +102,7 @@ autosign_file: autosign_file
 
 # disable discovery for test suite saltstack/salt-jenkins#683
 discovery: false
+
+# enable using ssh minions and regular minions
+enable_ssh_minions: True
+ignore_host_keys: True

tests/integration/files/conf/syndic_master

@@ -23,3 +23,7 @@ tcp_master_workers: 54515
 
 # Syndic Settings
 order_masters: True
+
+# enable using ssh minions and regular minions
+enable_ssh_minions: True
+ignore_host_keys: True

tests/integration/netapi/rest_tornado/test_app.py

@@ -174,7 +174,7 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase):
         # TODO: verify pub function? Maybe look at how we test the publisher
         self.assertEqual(len(ret), 1)
         self.assertIn('jid', ret[0])
-        self.assertEqual(ret[0]['minions'], sorted(['minion', 'sub_minion']))
+        self.assertEqual(ret[0]['minions'], sorted(['minion', 'sub_minion', 'localhost']))
 
     def test_multi_local_async_post(self):
         low = [{'client': 'local_async',
@@ -200,8 +200,8 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase):
         self.assertEqual(len(ret), 2)
         self.assertIn('jid', ret[0])
         self.assertIn('jid', ret[1])
-        self.assertEqual(ret[0]['minions'], sorted(['minion', 'sub_minion']))
+        self.assertEqual(ret[0]['minions'], sorted(['minion', 'sub_minion', 'localhost']))
-        self.assertEqual(ret[1]['minions'], sorted(['minion', 'sub_minion']))
+        self.assertEqual(ret[1]['minions'], sorted(['minion', 'sub_minion', 'localhost']))
 
     def test_multi_local_async_post_multitoken(self):
         low = [{'client': 'local_async',
@@ -235,8 +235,8 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase):
         self.assertIn('jid', ret[0])  # the first 2 are regular returns
         self.assertIn('jid', ret[1])
         self.assertIn('Authentication error occurred.', ret[2])  # bad auth
-        self.assertEqual(ret[0]['minions'], sorted(['minion', 'sub_minion']))
+        self.assertEqual(ret[0]['minions'], sorted(['minion', 'sub_minion', 'localhost']))
-        self.assertEqual(ret[1]['minions'], sorted(['minion', 'sub_minion']))
+        self.assertEqual(ret[1]['minions'], sorted(['minion', 'sub_minion', 'localhost']))
 
     def test_simple_local_async_post_no_tgt(self):
         low = [{'client': 'local_async',
@@ -267,7 +267,7 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase):
                               )
         response_obj = salt.utils.json.loads(response.body)
         self.assertEqual(len(response_obj['return']), 1)
-        self.assertEqual(set(response_obj['return'][0]), set(['minion', 'sub_minion']))
+        self.assertEqual(set(response_obj['return'][0]), set(['localhost', 'minion', 'sub_minion']))
 
     # runner_async tests
     def test_simple_local_runner_async_post(self):
@@ -329,7 +329,7 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase):
         self.assertEqual(response_obj['return'][0]['minion']['id'], 'minion')
 
     def test_post(self):
-        low = [{'tgt': '*',
+        low = [{'tgt': '*minion',
                 'fun': 'test.ping',
                 }]
         response = self.fetch('/minions',
@@ -351,7 +351,7 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase):
     def test_post_with_client(self):
         # get a token for this test
         low = [{'client': 'local_async',
-                'tgt': '*',
+                'tgt': '*minion',
                 'fun': 'test.ping',
                 }]
         response = self.fetch('/minions',

tests/integration/netapi/test_client.py

@@ -35,7 +35,7 @@ class NetapiClientTest(TestCase):
         low.update(self.eauth_creds)
 
         ret = self.netapi.run(low)
-        self.assertEqual(ret, {'minion': True, 'sub_minion': True})
+        self.assertEqual(ret, {'minion': True, 'sub_minion': True, 'localhost': True})
 
     def test_local_async(self):
         low = {'client': 'local_async', 'tgt': '*', 'fun': 'test.ping'}
@@ -47,7 +47,7 @@ class NetapiClientTest(TestCase):
         self.assertIn('jid', ret)
         ret.pop('jid', None)
         ret['minions'] = sorted(ret['minions'])
-        self.assertEqual(ret, {'minions': sorted(['minion', 'sub_minion'])})
+        self.assertEqual(ret, {'minions': sorted(['minion', 'sub_minion', 'localhost'])})
 
     def test_wheel(self):
         low = {'client': 'wheel', 'fun': 'key.list_all'}

tests/integration/runners/test_state.py

@@ -18,6 +18,7 @@ from salt.ext.six.moves import queue
 from tests.support.case import ShellCase
 from tests.support.unit import skipIf
 from tests.support.paths import TMP
+from tests.support.helpers import flaky
 
 # Import Salt Libs
 import salt.utils.platform
@@ -44,6 +45,7 @@ class StateRunnerTest(ShellCase):
         q.put(ret)
         q.task_done()
 
+    @flaky
     def test_orchestrate_output(self):
         '''
         Ensure the orchestrate runner outputs useful state data.

tests/integration/shell/test_cp.py

@@ -55,6 +55,8 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin):
             testfile_contents = fh_.read()
 
         for idx, minion in enumerate(minions):
+            if 'localhost' in minion:
+                continue
             ret = self.run_salt(
                 '--out yaml {0} file.directory_exists {1}'.format(
                     pipes.quote(minion), TMP
@@ -138,7 +140,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin):
 
             ret = self.run_script(
                 self._call_binary_,
-                '--out pprint --config-dir {0} \'*\' {1} {0}/{2}'.format(
+                '--out pprint --config-dir {0} \'*minion\' {1} {0}/{2}'.format(
                     config_dir,
                     fn_,
                     os.path.basename(fn_),

tests/integration/ssh/test_master.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+'''
+Simple Smoke Tests for Connected SSH minions
+'''
+
+# Import Python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+# Import Salt Testing libs
+from tests.support.case import ModuleCase
+from tests.support.helpers import requires_sshd_server
+
+
+@requires_sshd_server
+class SSHMasterTestCase(ModuleCase):
+    '''
+    Test minion blackout functionality
+    '''
+    def test_can_it_ping(self):
+        '''
+        Ensure the proxy can ping
+        '''
+        ret = self.run_function('test.ping', minion_tgt='localhost')
+        self.assertEqual(ret, True)
+
+    def test_service(self):
+        service = 'cron'
+        os_family = self.run_function('grains.get', ['os_family'], minion_tgt='localhost')
+        if os_family == 'RedHat':
+            service = 'crond'
+        elif os_family == 'Arch':
+            service = 'sshd'
+        ret = self.run_function('service.get_all', minion_tgt='localhost')
+        self.assertIn(service, ret)
+        self.run_function('service.stop', [service], minion_tgt='localhost')
+        ret = self.run_function('service.status', [service], minion_tgt='localhost')
+        self.assertFalse(ret)
+        self.run_function('service.start', [service], minion_tgt='localhost')
+        ret = self.run_function('service.status', [service], minion_tgt='localhost')
+        self.assertTrue(ret)
+
+    def test_grains_items(self):
+        ret = self.run_function('grains.items', minion_tgt='localhost')
+        self.assertEqual(ret['kernel'], 'Linux')
+
+    def test_state_apply(self):
+        ret = self.run_function('state.apply', ['core'], minion_tgt='localhost')
+        for key, value in ret.items():
+            self.assertTrue(value['result'])
+
+    def test_state_highstate(self):
+        ret = self.run_function('state.highstate', minion_tgt='localhost')
+        for key, value in ret.items():
+            self.assertTrue(value['result'])