Merge branch '2019.2.1' into fix_test_file

pkg/windows/req.txt

@@ -23,9 +23,9 @@ pyasn1==0.4.5
 pycparser==2.19
 pycrypto==2.6.1
 pycurl==7.43.0.2
+pymssql==2.1.4
 PyMySQL==0.9.3
 pyOpenSSL==18.0.0
-#python-certifi-win32==1.2
 python-dateutil==2.7.5
 python-gnupg==0.4.3
 pythonnet==2.3.0

salt/renderers/gpg.py

@@ -295,7 +295,7 @@ def _decrypt_ciphertext(cipher):
         return decrypted_data
 
 
-def _decrypt_ciphertexts(cipher, translate_newlines=False):
+def _decrypt_ciphertexts(cipher, translate_newlines=False, encoding=None):
     to_bytes = salt.utils.stringutils.to_bytes
     cipher = to_bytes(cipher)
     if translate_newlines:
@@ -314,14 +314,14 @@ def _decrypt_ciphertexts(cipher, translate_newlines=False):
         ret = cipher
 
     try:
-        ret = salt.utils.stringutils.to_unicode(ret)
+        ret = salt.utils.stringutils.to_unicode(ret, encoding=encoding)
     except UnicodeDecodeError:
         # decrypted data contains some sort of binary data - not our problem
         pass
     return ret
 
 
-def _decrypt_object(obj, translate_newlines=False):
+def _decrypt_object(obj, translate_newlines=False, encoding=None):
     '''
     Recursively try to decrypt any object. If the object is a six.string_types
     (string or unicode), and it contains a valid GPG header, decrypt it,
@@ -330,7 +330,7 @@ def _decrypt_object(obj, translate_newlines=False):
     if salt.utils.stringio.is_readable(obj):
         return _decrypt_object(obj.getvalue(), translate_newlines)
     if isinstance(obj, six.string_types):
-        return _decrypt_ciphertexts(obj, translate_newlines=translate_newlines)
+        return _decrypt_ciphertexts(obj, translate_newlines=translate_newlines, encoding=encoding)
     elif isinstance(obj, dict):
         for key, value in six.iteritems(obj):
             obj[key] = _decrypt_object(value,
@@ -355,4 +355,4 @@ def render(gpg_data, saltenv='base', sls='', argline='', **kwargs):
     log.debug('Reading GPG keys from: %s', _get_key_dir())
 
     translate_newlines = kwargs.get('translate_newlines', False)
-    return _decrypt_object(gpg_data, translate_newlines=translate_newlines)
+    return _decrypt_object(gpg_data, translate_newlines=translate_newlines, encoding=kwargs.get('encoding', None))

salt/transport/zeromq.py

@@ -532,9 +532,10 @@ class AsyncZeroMQPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.t
             payload = self.serial.loads(messages[0])
         # 2 includes a header which says who should do it
         elif messages_len == 2:
-            if (self.opts.get('__role') != 'syndic' and messages[0] not in ('broadcast', self.hexid)) or \
+            message_target = salt.utils.stringutils.to_str(messages[0])
-                (self.opts.get('__role') == 'syndic' and messages[0] not in ('broadcast', 'syndic')):
+            if (self.opts.get('__role') != 'syndic' and message_target not in ('broadcast', self.hexid)) or \
-                log.debug('Publish received for not this minion: %s', messages[0])
+                (self.opts.get('__role') == 'syndic' and message_target not in ('broadcast', 'syndic')):
+                log.debug('Publish received for not this minion: %s', message_target)
                 raise tornado.gen.Return(None)
             payload = self.serial.loads(messages[1])
         else:
@@ -916,7 +917,7 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel):
                                 log.trace('Sending filtered data over publisher %s', pub_uri)
                                 # zmq filters are substring match, hash the topic
                                 # to avoid collisions
-                                htopic = salt.utils.stringutils.to_bytes(hashlib.sha1(topic).hexdigest())
+                                htopic = salt.utils.stringutils.to_bytes(hashlib.sha1(salt.utils.stringutils.to_bytes(topic)).hexdigest())
                                 pub_sock.send(htopic, flags=zmq.SNDMORE)
                                 pub_sock.send(payload)
                                 log.trace('Filtered data has been sent')

salt/utils/rsax931.py

@@ -65,7 +65,9 @@ def _init_libcrypto():
     libcrypto = _load_libcrypto()
 
     try:
-        libcrypto.OPENSSL_init_crypto()
+        # If we're greater than OpenSSL 1.1.0, no need to to the init
+        if libcrypto.OpenSSL_version_num < 0x10100000:
+            libcrypto.OPENSSL_init_crypto()
     except AttributeError:
         # Support for OpenSSL < 1.1 (OPENSSL_API_COMPAT < 0x10100000L)
         libcrypto.OPENSSL_no_config()

tests/unit/grains/test_core.py

@@ -141,7 +141,6 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin):
         grains = core._windows_platform_data()
         keys = ['biosversion',
                 'osrelease',
-                'domain',
                 'kernelrelease',
                 'motherboard',
                 'serialnumber',

tests/unit/renderers/test_gpg.py

@@ -143,7 +143,7 @@ class GPGTestCase(TestCase, LoaderModuleMockMixin):
         with patch('salt.renderers.gpg._get_gpg_exec', MagicMock(return_value=True)):
             with patch('salt.renderers.gpg._get_key_dir', MagicMock(return_value=key_dir)):
                 with patch('salt.renderers.gpg._decrypt_ciphertext', MagicMock(return_value=secret)):
-                    self.assertEqual(gpg.render(crypted), expected)
+                    self.assertEqual(gpg.render(crypted, encoding='utf-8'), expected)
 
     def test_render_with_translate_newlines_should_translate_newlines(self):
         key_dir = '/etc/salt/gpgkeys'
@@ -165,6 +165,6 @@ class GPGTestCase(TestCase, LoaderModuleMockMixin):
             with patch('salt.renderers.gpg._get_key_dir', MagicMock(return_value=key_dir)):
                 with patch('salt.renderers.gpg._decrypt_ciphertext', MagicMock(return_value=secret)):
                     self.assertEqual(
-                        gpg.render(crypted, translate_newlines=True),
+                        gpg.render(crypted, translate_newlines=True, encoding='utf-8'),
                         expected,
                     )

tests/unit/states/test_boto_apigateway.py

@@ -229,7 +229,7 @@ deployment1_ret = dict(createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50),
                        description=('{\n'
                                     '    "api_name": "unit test api",\n'
                                     '    "swagger_file": "temp-swagger-sample.yaml",\n'
-                                    '    "swagger_file_md5sum": "693c57997a12a2446bb5c08c793d943c",\n'
+                                    '    "swagger_file_md5sum": "55a948ff90ad80ff747ec91657c7a299",\n'
                                     '    "swagger_info_object": {\n'
                                     '        "description": "salt boto apigateway unit test service",\n'
                                     '        "title": "salt boto apigateway unit test service",\n'
@@ -371,7 +371,7 @@ class TempSwaggerFile(object):
     def __enter__(self):
         self.swaggerfile = 'temp-swagger-sample.yaml'
         with salt.utils.files.fopen(self.swaggerfile, 'w') as fp_:
-            salt.utils.yaml.safe_dump(self.swaggerdict, fp_)
+            salt.utils.yaml.safe_dump(self.swaggerdict, fp_, default_flow_style=False)
         return self.swaggerfile
 
     def __exit__(self, objtype, value, traceback):

tests/unit/transport/test_zeromq.py

@@ -436,7 +436,7 @@ class PubServerChannel(TestCase, AdaptedConfigurationTestCaseMixin):
         del self.process_manager
 
     @staticmethod
-    def _gather_results(opts, pub_uri, results, timeout=120):
+    def _gather_results(opts, pub_uri, results, timeout=120, messages=None):
         '''
         Gather results until then number of seconds specified by timeout passes
         without reveiving a message
@@ -455,6 +455,10 @@ class PubServerChannel(TestCase, AdaptedConfigurationTestCaseMixin):
             except zmq.ZMQError:
                 time.sleep(.01)
             else:
+                if messages:
+                    if messages != 1:
+                        messages -= 1
+                        continue
                 payload = crypticle.loads(serial.loads(payload)['load'])
                 if 'stop' in payload:
                     break
@@ -493,6 +497,94 @@ class PubServerChannel(TestCase, AdaptedConfigurationTestCaseMixin):
         server_channel.pub_close()
         assert len(results) == send_num, (len(results), set(expect).difference(results))
 
+    def test_zeromq_zeromq_filtering_decode_message_no_match(self):
+        '''
+        test AsyncZeroMQPubChannel _decode_messages when
+        zmq_filtering enabled and minion does not match
+        '''
+        message = [b'4f26aeafdb2367620a393c973eddbe8f8b846eb',
+                   b'\x82\xa3enc\xa3aes\xa4load\xda\x00`\xeeR\xcf'
+                   b'\x0eaI#V\x17if\xcf\xae\x05\xa7\xb3bN\xf7\xb2\xe2'
+                   b'\xd0sF\xd1\xd4\xecB\xe8\xaf"/*ml\x80Q3\xdb\xaexg'
+                   b'\x8e\x8a\x8c\xd3l\x03\\,J\xa7\x01i\xd1:]\xe3\x8d'
+                   b'\xf4\x03\x88K\x84\n`\xe8\x9a\xad\xad\xc6\x8ea\x15>'
+                   b'\x92m\x9e\xc7aM\x11?\x18;\xbd\x04c\x07\x85\x99\xa3\xea[\x00D']
+
+        opts = dict(self.master_config, ipc_mode='ipc',
+                    pub_hwm=0, zmq_filtering=True, recon_randomize=False,
+                    recon_default=1, recon_max=2, master_ip='127.0.0.1',
+                    acceptance_wait_time=5, acceptance_wait_time_max=5)
+        opts['master_uri'] = 'tcp://{interface}:{publish_port}'.format(**opts)
+
+        server_channel = salt.transport.zeromq.AsyncZeroMQPubChannel(opts)
+        with patch('salt.crypt.AsyncAuth.crypticle',
+                   MagicMock(return_value={'tgt_type': 'glob', 'tgt': '*',
+                                           'jid': 1})) as mock_test:
+            res = server_channel._decode_messages(message)
+        assert res.result() is None
+
+    def test_zeromq_zeromq_filtering_decode_message(self):
+        '''
+        test AsyncZeroMQPubChannel _decode_messages
+        when zmq_filtered enabled
+        '''
+        message = [b'4f26aeafdb2367620a393c973eddbe8f8b846ebd',
+                   b'\x82\xa3enc\xa3aes\xa4load\xda\x00`\xeeR\xcf'
+                   b'\x0eaI#V\x17if\xcf\xae\x05\xa7\xb3bN\xf7\xb2\xe2'
+                   b'\xd0sF\xd1\xd4\xecB\xe8\xaf"/*ml\x80Q3\xdb\xaexg'
+                   b'\x8e\x8a\x8c\xd3l\x03\\,J\xa7\x01i\xd1:]\xe3\x8d'
+                   b'\xf4\x03\x88K\x84\n`\xe8\x9a\xad\xad\xc6\x8ea\x15>'
+                   b'\x92m\x9e\xc7aM\x11?\x18;\xbd\x04c\x07\x85\x99\xa3\xea[\x00D']
+
+        opts = dict(self.master_config, ipc_mode='ipc',
+                    pub_hwm=0, zmq_filtering=True, recon_randomize=False,
+                    recon_default=1, recon_max=2, master_ip='127.0.0.1',
+                    acceptance_wait_time=5, acceptance_wait_time_max=5)
+        opts['master_uri'] = 'tcp://{interface}:{publish_port}'.format(**opts)
+
+        server_channel = salt.transport.zeromq.AsyncZeroMQPubChannel(opts)
+        with patch('salt.crypt.AsyncAuth.crypticle',
+                   MagicMock(return_value={'tgt_type': 'glob', 'tgt': '*',
+                                           'jid': 1})) as mock_test:
+            res = server_channel._decode_messages(message)
+
+        assert res.result()['enc'] == 'aes'
+
+    @skipIf(salt.utils.platform.is_windows(), 'Skip on Windows OS')
+    def test_zeromq_filtering(self):
+        '''
+        Test sending messags to publisher using UDP
+        with zeromq_filtering enabled
+        '''
+        opts = dict(self.master_config, ipc_mode='ipc',
+                    pub_hwm=0, zmq_filtering=True, acceptance_wait_time=5)
+        server_channel = salt.transport.zeromq.ZeroMQPubServerChannel(opts)
+        server_channel.pre_fork(self.process_manager, kwargs={
+            'log_queue': salt.log.setup.get_multiprocessing_logging_queue()
+        })
+        pub_uri = 'tcp://{interface}:{publish_port}'.format(**server_channel.opts)
+        send_num = 1
+        expect = []
+        results = []
+        gather = threading.Thread(target=self._gather_results,
+                                  args=(self.minion_config, pub_uri, results,),
+                                  kwargs={'messages': 2})
+        gather.start()
+        # Allow time for server channel to start, especially on windows
+        time.sleep(2)
+        expect.append(send_num)
+        load = {'tgt_type': 'glob', 'tgt': '*', 'jid': send_num}
+        with patch('salt.utils.minions.CkMinions.check_minions',
+                   MagicMock(return_value={'minions': ['minion'], 'missing': [],
+                                           'ssh_minions': False})):
+            server_channel.publish(load)
+        server_channel.publish(
+            {'tgt_type': 'glob', 'tgt': '*', 'stop': True}
+        )
+        gather.join()
+        server_channel.pub_close()
+        assert len(results) == send_num, (len(results), set(expect).difference(results))
+
     def test_publish_to_pubserv_tcp(self):
         '''
         Test sending 10K messags to ZeroMQPubServerChannel using TCP transport

tests/unit/utils/test_win_dacl.py

@@ -271,7 +271,6 @@ class WinDaclRegTestCase(TestCase, LoaderModuleMockMixin):
                                                                        'write_owner']}}},
                     'comment': '',
                     'name': self.obj_name,
-                    'pchanges': {'perms': {}},
                     'result': True}
         self.assertDictEqual(result, expected)
 
@@ -334,16 +333,15 @@ class WinDaclRegTestCase(TestCase, LoaderModuleMockMixin):
                 reset=False)
 
         expected = {
-            'changes': {'perms': {}},
+            'changes': {'owner': 'Users',
+                        'perms': {'Backup Operators': {'grant': 'read',
+                                                       'deny': ['delete']},
+                                  'NETWORK SERVICE': {'deny': ['delete',
+                                                               'set_value',
+                                                               'write_dac',
+                                                               'write_owner']}}},
             'comment': '',
             'name': self.obj_name,
-            'pchanges': {'owner': 'Users',
-                         'perms': {'Backup Operators': {'grant': 'read',
-                                                        'deny': ['delete']},
-                                   'NETWORK SERVICE': {'deny': ['delete',
-                                                                'set_value',
-                                                                'write_dac',
-                                                                'write_owner']}}},
             'result': None}
         self.assertDictEqual(result, expected)
 
@@ -573,7 +571,6 @@ class WinDaclFileTestCase(TestCase, LoaderModuleMockMixin):
                                                                'write_data']}}},
             'comment': '',
             'name': self.obj_name,
-            'pchanges': {'perms': {}},
             'result': True}
         self.assertDictEqual(result, expected)
 
@@ -636,16 +633,15 @@ class WinDaclFileTestCase(TestCase, LoaderModuleMockMixin):
                 reset=False)
 
         expected = {
-            'changes': {'perms': {}},
+            'changes': {'owner': 'Users',
+                        'perms': {'Backup Operators': {'grant': 'read',
+                                                       'deny': ['delete']},
+                                  'NETWORK SERVICE': {'deny': ['delete',
+                                                               'set_value',
+                                                               'write_dac',
+                                                               'write_owner']}}},
             'comment': '',
             'name': self.obj_name,
-            'pchanges': {'owner': 'Users',
-                         'perms': {'Backup Operators': {'grant': 'read',
-                                                        'deny': ['delete']},
-                                   'NETWORK SERVICE': {'deny': ['delete',
-                                                                'set_value',
-                                                                'write_dac',
-                                                                'write_owner']}}},
             'result': None}
         self.assertDictEqual(result, expected)