Merge pull request #46769 from dwoz/wincloudtest

Adding windows minion tests for salt cloud

salt/cloud/clouds/ec2.py

@@ -2336,6 +2336,9 @@ def wait_for_instance(
         use_winrm = config.get_cloud_config_value(
             'use_winrm', vm_, __opts__, default=False
         )
+        winrm_verify_ssl = config.get_cloud_config_value(
+            'winrm_verify_ssl', vm_, __opts__, default=True
+        )
 
         if win_passwd and win_passwd == 'auto':
             log.debug('Waiting for auto-generated Windows EC2 password')
@@ -2407,7 +2410,8 @@ def wait_for_instance(
                                                           winrm_port,
                                                           username,
                                                           win_passwd,
-                                                          timeout=ssh_connect_timeout):
+                                                          timeout=ssh_connect_timeout,
+                                                          verify=winrm_verify_ssl):
                 raise SaltCloudSystemExit(
                     'Failed to authenticate against remote windows host'
                 )

salt/utils/cloud.py

@@ -515,7 +515,10 @@ def bootstrap(vm_, opts=None):
             'winrm_port', vm_, opts, default=5986
         )
         deploy_kwargs['winrm_use_ssl'] = salt.config.get_cloud_config_value(
-        'winrm_use_ssl', vm_, opts, default=True
+            'winrm_use_ssl', vm_, opts, default=True
+        )
+        deploy_kwargs['winrm_verify_ssl'] = salt.config.get_cloud_config_value(
+            'winrm_verify_ssl', vm_, opts, default=True
         )
         if saltify_driver:
             deploy_kwargs['port_timeout'] = 1  # No need to wait/retry with Saltify
@@ -843,7 +846,7 @@ def wait_for_winexesvc(host, port, username, password, timeout=900):
         time.sleep(1)
 
 
-def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True):
+def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True, verify=True):
     '''
     Wait until WinRM connection can be established.
     '''
@@ -853,14 +856,20 @@ def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True):
             host, port
         )
     )
+    transport = 'ssl'
+    if not use_ssl:
+        transport = 'plaintext'
     trycount = 0
     while True:
         trycount += 1
         try:
-            transport = 'ssl'
+            winrm_kwargs = {'target': host,
-            if not use_ssl:
+                            'auth': (username, password),
-                transport = 'plaintext'
+                            'transport': transport}
-            s = winrm.Session(host, auth=(username, password), transport=transport)
+            if not verify:
+                log.debug("SSL validation for WinRM disabled.")
+                winrm_kwargs['server_cert_validation'] = 'ignore'
+            s = winrm.Session(**winrm_kwargs)
             if hasattr(s.protocol, 'set_timeout'):
                 s.protocol.set_timeout(15)
             log.trace('WinRM endpoint url: {0}'.format(s.url))
@@ -1008,6 +1017,7 @@ def deploy_windows(host,
                    use_winrm=False,
                    winrm_port=5986,
                    winrm_use_ssl=True,
+                   winrm_verify_ssl=True,
                    **kwargs):
     '''
     Copy the install files to a remote Windows box, and execute them
@@ -1034,7 +1044,8 @@ def deploy_windows(host,
     if HAS_WINRM and use_winrm:
         winrm_session = wait_for_winrm(host=host, port=winrm_port,
                                        username=username, password=password,
-                                       timeout=port_timeout * 60, use_ssl=winrm_use_ssl)
+                                       timeout=port_timeout * 60, use_ssl=winrm_use_ssl,
+                                       verify=winrm_verify_ssl)
         if winrm_session is not None:
             service_available = True
     else:

tests/integration/cloud/providers/test_ec2.py

@@ -8,14 +8,18 @@ from __future__ import absolute_import
 import os
 import random
 import string
+import yaml
 
 # Import Salt Libs
 from salt.config import cloud_providers_config
+import salt.utils
 
 # Import Salt Testing Libs
 from tests.support.case import ShellCase
 from tests.support.paths import FILES
 from tests.support.helpers import expensiveTest
+from tests.support.unit import expectedFailure
+from tests.support import win_installer
 
 # Import Third-Party Libs
 from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin
@@ -39,6 +43,38 @@ class EC2Test(ShellCase):
     '''
     Integration tests for the EC2 cloud provider in Salt-Cloud
     '''
+    TIMEOUT = 500
+
+    def _installer_name(self):
+        '''
+        Determine the downloaded installer name by searching the files
+        directory for the firt file that loosk like an installer.
+        '''
+        for path, dirs, files in os.walk(FILES):
+            for file in files:
+                if file.startswith(win_installer.PREFIX):
+                    return file
+            break
+        return
+
+    def _fetch_latest_installer(self):
+        '''
+        Download the latest Windows installer executable
+        '''
+        name = win_installer.latest_installer_name()
+        path = os.path.join(FILES, name)
+        with salt.utils.fopen(path, 'wb') as fp:
+            win_installer.download_and_verify(fp, name)
+        return name
+
+    def _ensure_installer(self):
+        '''
+        Make sure the testing environment has a Windows installer executbale.
+        '''
+        name = self._installer_name()
+        if name:
+            return name
+        return self._fetch_latest_installer()
 
     @expensiveTest
     def setUp(self):
@@ -90,24 +126,51 @@ class EC2Test(ShellCase):
                 'missing. Check tests/integration/files/conf/cloud.providers.d/{0}.conf'
                 .format(PROVIDER_NAME)
             )
+        self.INSTALLER = self._ensure_installer()
 
-    def test_instance(self):
+    def override_profile_config(self, name, data):
+        conf_path = os.path.join(self.get_config_dir(), 'cloud.profiles.d', 'ec2.conf')
+        with salt.utils.fopen(conf_path, 'r') as fp:
+            conf = yaml.safe_load(fp)
+        conf[name].update(data)
+        with salt.utils.fopen(conf_path, 'w') as fp:
+            yaml.dump(conf, fp)
+
+    def copy_file(self, name):
+        '''
+        Copy a file from tests/integration/files to a test's temporary
+        configuration directory. The path to the file which is created will be
+        returned.
+        '''
+        src = os.path.join(FILES, name)
+        dst = os.path.join(self.get_config_dir(), name)
+        with salt.utils.fopen(src, 'rb') as sfp:
+            with salt.utils.fopen(dst, 'wb') as dfp:
+                dfp.write(sfp.read())
+        return dst
+
+    def _test_instance(self, profile='ec2-test', debug=False, timeout=TIMEOUT):
         '''
         Tests creating and deleting an instance on EC2 (classic)
         '''
+
         # create the instance
-        instance = self.run_cloud('-p ec2-test {0}'.format(INSTANCE_NAME), timeout=500)
+        cmd = '-p {0}'.format(profile)
+        if debug:
+            cmd += ' -l debug'
+        cmd += ' {0}'.format(INSTANCE_NAME)
+        instance = self.run_cloud(cmd, timeout=timeout)
         ret_str = '{0}:'.format(INSTANCE_NAME)
 
         # check if instance returned with salt installed
         try:
             self.assertIn(ret_str, instance)
         except AssertionError:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=timeout)
             raise
 
         # delete the instance
-        delete = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)
+        delete = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=timeout)
         ret_str = '                    shutting-down'
 
         # check if deletion was performed appropriately
@@ -151,6 +214,80 @@ class EC2Test(ShellCase):
         # check if deletion was performed appropriately
         self.assertIn(ret_str, delete)
 
+    def test_instance(self):
+        '''
+        Tests creating and deleting an instance on EC2 (classic)
+        '''
+        self._test_instance('ec2-test')
+
+    @expectedFailure
+    def test_win2012r2_winexe(self):
+        '''
+        Tests creating and deleting a Windows 2012r2instance on EC2 using
+        winexe (classic)
+        '''
+        # TODO: winexe calls hang and the test fails by timing out. The same
+        # same calls succeed when run outside of the test environment.
+        self.override_profile_config(
+            'ec2-win2012-test',
+            {
+                'use_winrm': False,
+                'user_data': self.copy_file('windows-firewall-winexe.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+            },
+        )
+        self._test_instance('ec2-win2012r2-test', debug=True, timeout=500)
+
+    def test_win2012r2_winrm(self):
+        '''
+        Tests creating and deleting a Windows 2012r2 instance on EC2 using
+        winrm (classic)
+        '''
+        self.override_profile_config(
+            'ec2-win2016-test',
+            {
+                'user_data': self.copy_file('windows-firewall.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+                'winrm_ssl_verify': False,
+            }
+
+        )
+        self._test_instance('ec2-win2012r2-test', debug=True, timeout=500)
+
+    @expectedFailure
+    def test_win2016_winexe(self):
+        '''
+        Tests creating and deleting a Windows 2016 instance on EC2 using winrm
+        (classic)
+        '''
+        # TODO: winexe calls hang and the test fails by timing out. The same
+        # same calls succeed when run outside of the test environment.
+        self.override_profile_config(
+            'ec2-win2016-test',
+            {
+                'use_winrm': False,
+                'user_data': self.copy_file('windows-firewall-winexe.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+            },
+        )
+        self._test_instance('ec2-win2016-test', debug=True, timeout=500)
+
+    def test_win2016_winrm(self):
+        '''
+        Tests creating and deleting a Windows 2016 instance on EC2 using winrm
+        (classic)
+        '''
+        self.override_profile_config(
+            'ec2-win2016-test',
+            {
+                'user_data': self.copy_file('windows-firewall.ps1'),
+                'win_installer': self.copy_file(self.INSTALLER),
+                'winrm_ssl_verify': False,
+            }
+
+        )
+        self._test_instance('ec2-win2016-test', debug=True, timeout=500)
+
     def tearDown(self):
         '''
         Clean up after tests
@@ -160,4 +297,4 @@ class EC2Test(ShellCase):
 
         # if test instance is still present, delete it
         if ret_str in query:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=self.TIMEOUT)

tests/integration/files/conf/cloud.profiles.d/ec2.conf

@@ -4,3 +4,31 @@ ec2-test:
   size: t1.micro
   sh_username: ec2-user
   script_args: '-P -Z'
+ec2-win2012r2-test:
+  provider: ec2-config
+  size: t2.micro
+  image: ami-eb1ecd96
+  smb_port: 445
+  win_installer: ''
+  win_username: Administrator
+  win_password: auto
+  userdata_file: ''
+  userdata_template: False
+  use_winrm: True
+  winrm_verify_ssl: False
+  ssh_interface: private_ips
+  deploy: True
+ec2-win2016-test:
+  provider: ec2-config
+  size: t2.micro
+  image: ami-ed14c790
+  smb_port: 445
+  win_installer: ''
+  win_username: Administrator
+  win_password: auto
+  userdata_file: ''
+  userdata_template: False
+  use_winrm: True
+  winrm_verify_ssl: False
+  ssh_interface: private_ips
+  deploy: True

tests/integration/files/windows-firewall-winexe.ps1

@@ -0,0 +1,5 @@
+<powershell>
+New-NetFirewallRule -Name "SMB445" -DisplayName "SMB445" -Protocol TCP -LocalPort 445
+Set-Item (dir wsman:\localhost\Listener\*\Port -Recurse).pspath 445 -Force
+Restart-Service winrm
+</powershell>

tests/integration/files/windows-firewall.ps1

@@ -0,0 +1,33 @@
+<powershell>
+New-NetFirewallRule -Name "SMB445" -DisplayName "SMB445" -Protocol TCP -LocalPort 445
+New-NetFirewallRule -Name "WINRM5986" -DisplayName "WINRM5986" -Protocol TCP -LocalPort 5986
+
+winrm quickconfig -q
+winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="300"}'
+winrm set winrm/config '@{MaxTimeoutms="1800000"}'
+winrm set winrm/config/service/auth '@{Basic="true"}'
+
+$SourceStoreScope = 'LocalMachine'
+$SourceStorename = 'Remote Desktop'
+
+$SourceStore = New-Object  -TypeName System.Security.Cryptography.X509Certificates.X509Store  -ArgumentList $SourceStorename, $SourceStoreScope
+$SourceStore.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadOnly)
+
+$cert = $SourceStore.Certificates | Where-Object  -FilterScript {
+    $_.subject -like '*'
+}
+
+$DestStoreScope = 'LocalMachine'
+$DestStoreName = 'My'
+
+$DestStore = New-Object  -TypeName System.Security.Cryptography.X509Certificates.X509Store  -ArgumentList $DestStoreName, $DestStoreScope
+$DestStore.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite)
+$DestStore.Add($cert)
+
+$SourceStore.Close()
+$DestStore.Close()
+
+winrm create winrm/config/listener?Address=*+Transport=HTTPS  `@`{Hostname=`"($certId)`"`;CertificateThumbprint=`"($cert.Thumbprint)`"`}
+
+Restart-Service winrm
+</powershell>

tests/support/win_installer.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+'''
+    :copyright: Copyright 2013-2017 by the SaltStack Team, see AUTHORS for more details.
+    :license: Apache 2.0, see LICENSE for more details.
+
+
+    tests.support.win_installer
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Fetches the binary Windows installer
+'''
+from __future__ import absolute_import
+import hashlib
+import requests
+import re
+
+PREFIX = 'Salt-Minion-'
+REPO = "https://repo.saltstack.com/windows"
+
+
+def iter_installers(content):
+    '''
+    Parse a list of windows installer links and their corresponding md5
+    checksum links.
+    '''
+    HREF_RE = "<a href=\"(.*?)\">"
+    installer, md5 = None, None
+    for m in re.finditer(HREF_RE, content):
+        x = m.groups()[0]
+        if not x.startswith(PREFIX):
+            continue
+        if x.endswith('zip'):
+            continue
+        if installer:
+            if x != installer + '.md5':
+                raise Exception("Unable to parse response")
+            md5 = x
+            yield installer, md5
+            installer, md5 = None, None
+        else:
+            installer = x
+
+
+def split_installer(name):
+    '''
+    Return a tuple of the salt version, python verison and architecture from an
+    installer name.
+    '''
+    x = name[len(PREFIX):]
+    return x.split('-')[:3]
+
+
+def latest_version(repo=REPO):
+    '''
+    Return the latest version found on the salt repository webpage.
+    '''
+    for name, md5 in iter_installers(requests.get(repo).content):
+        pass
+    return split_installer(name)[0]
+
+
+def installer_name(salt_ver, py_ver='Py2', arch='AMD64'):
+    '''
+    Create an installer file name
+    '''
+    return "Salt-Minion-{}-{}-{}-Setup.exe".format(salt_ver, py_ver, arch)
+
+
+def latest_installer_name(repo=REPO, **kwargs):
+    '''
+    Fetch the latest installer name
+    '''
+    return installer_name(latest_version(repo), **kwargs)
+
+
+def download_and_verify(fp, name, repo=REPO):
+    '''
+    Download an installer and verify it's contents.
+    '''
+    md5 = "{}.md5".format(name)
+    url = lambda x: "{}/{}".format(repo, x)
+    resp = requests.get(url(md5))
+    if resp.status_code != 200:
+        raise Exception("Unable to fetch installer md5")
+    installer_md5 = resp.text.strip().split()[0].lower()
+    resp = requests.get(url(name), stream=True)
+    if resp.status_code != 200:
+        raise Exception("Unable to fetch installer")
+    md5hsh = hashlib.md5()
+    for chunk in resp.iter_content(chunk_size=1024):
+        md5hsh.update(chunk)
+        fp.write(chunk)
+    if md5hsh.hexdigest() != installer_md5:
+        raise Exception("Installer's hash does not match {} != {}".format(
+            md5hsh.hexdigest(), installer_md5
+        ))