Merge pull request #45908 from DimensionDataResearch/fix/issue/45884

Fix for #45884 ("TypeError: can't serialize <NodeImage" when calling salt-cloud with the dimensiondata driver)

salt/cloud/clouds/dimensiondata.py

@@ -32,7 +32,7 @@ from salt.utils.versions import LooseVersion as _LooseVersion
 # Import libcloud
 try:
     import libcloud
-    from libcloud.compute.base import NodeState
+    from libcloud.compute.base import NodeDriver, NodeState
     from libcloud.compute.base import NodeAuthPassword
     from libcloud.compute.types import Provider
     from libcloud.compute.providers import get_driver
@@ -52,12 +52,6 @@ try:
 except ImportError:
     HAS_LIBCLOUD = False
 
-# Import generic libcloud functions
-# from salt.cloud.libcloudfuncs import *
-
-# Import salt libs
-import salt.utils
-
 # Import salt.cloud libs
 from salt.cloud.libcloudfuncs import *  # pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import
 from salt.utils import namespaced_function
@@ -220,7 +214,6 @@ def create(vm_):
 
     log.info('Creating Cloud VM %s', vm_['name'])
     conn = get_conn()
-    rootPw = NodeAuthPassword(vm_['auth'])
 
     location = conn.ex_get_location_by_id(vm_['location'])
     images = conn.list_images(location=location)
@@ -251,15 +244,13 @@ def create(vm_):
     kwargs = {
         'name': vm_['name'],
         'image': image,
-        'auth': rootPw,
         'ex_description': vm_['description'],
         'ex_network_domain': network_domain,
         'ex_vlan': vlan,
         'ex_is_started': vm_['is_started']
     }
 
-    event_data = kwargs.copy()
-    del event_data['auth']
+    event_data = _to_event_data(kwargs)
 
     __utils__['cloud.fire_event'](
         'event',
@@ -270,6 +261,10 @@ def create(vm_):
         transport=__opts__['transport']
     )
 
+    # Initial password (excluded from event payload)
+    initial_password = NodeAuthPassword(vm_['auth'])
+    kwargs['auth'] = initial_password
+
     try:
         data = conn.create_node(**kwargs)
     except Exception as exc:
@@ -283,7 +278,7 @@ def create(vm_):
         return False
 
     try:
-        data = salt.utils.cloud.wait_for_ip(
+        data = __utils__['cloud.wait_for_ip'](
             _query_node_data,
             update_args=(vm_, data),
             timeout=config.get_cloud_config_value(
@@ -309,7 +304,7 @@ def create(vm_):
         ip_address = preferred_ip(vm_, data.public_ips)
     log.debug('Using IP address %s', ip_address)
 
-    if salt.utils.cloud.get_salt_interface(vm_, __opts__) == 'private_ips':
+    if __utils__['cloud.get_salt_interface'](vm_, __opts__) == 'private_ips':
         salt_ip_address = preferred_ip(vm_, data.private_ips)
         log.info('Salt interface set to: %s', salt_ip_address)
     else:
@@ -325,7 +320,7 @@ def create(vm_):
     vm_['ssh_host'] = ip_address
     vm_['password'] = vm_['auth']
 
-    ret = salt.utils.cloud.bootstrap(vm_, __opts__)
+    ret = __utils__['cloud.bootstrap'](vm_, __opts__)
 
     ret.update(data.__dict__)
 
@@ -418,11 +413,13 @@ def create_lb(kwargs=None, call=None):
     log.debug('Network Domain: %s', network_domain.id)
     lb_conn.ex_set_current_network_domain(network_domain.id)
 
+    event_data = _to_event_data(kwargs)
+
     __utils__['cloud.fire_event'](
         'event',
         'create load_balancer',
         'salt/cloud/loadbalancer/creating',
-        args=kwargs,
+        args=event_data,
         sock_dir=__opts__['sock_dir'],
         transport=__opts__['transport']
     )
@@ -431,11 +428,13 @@ def create_lb(kwargs=None, call=None):
         name, port, protocol, algorithm, members
     )
 
+    event_data = _to_event_data(kwargs)
+
     __utils__['cloud.fire_event'](
         'event',
         'created load_balancer',
         'salt/cloud/loadbalancer/created',
-        args=kwargs,
+        args=event_data,
         sock_dir=__opts__['sock_dir'],
         transport=__opts__['transport']
     )
@@ -577,3 +576,46 @@ def get_lb_conn(dd_driver=None):
             'Missing dimensiondata_driver for get_lb_conn method.'
         )
     return get_driver_lb(Provider_lb.DIMENSIONDATA)(user_id, key, region=region)
+
+
+def _to_event_data(obj):
+    '''
+    Convert the specified object into a form that can be serialised by msgpack as event data.
+
+    :param obj: The object to convert.
+    '''
+
+    if obj is None:
+        return None
+    if isinstance(obj, bool):
+        return obj
+    if isinstance(obj, int):
+        return obj
+    if isinstance(obj, float):
+        return obj
+    if isinstance(obj, str):
+        return obj
+    if isinstance(obj, bytes):
+        return obj
+    if isinstance(obj, dict):
+        return obj
+
+    if isinstance(obj, NodeDriver):  # Special case for NodeDriver (cyclic references)
+        return obj.name
+
+    if isinstance(obj, list):
+        return [_to_event_data(item) for item in obj]
+
+    event_data = {}
+    for attribute_name in dir(obj):
+        if attribute_name.startswith('_'):
+            continue
+
+        attribute_value = getattr(obj, attribute_name)
+
+        if callable(attribute_value):  # Strip out methods
+            continue
+
+        event_data[attribute_name] = _to_event_data(attribute_value)
+
+    return event_data

tests/integration/cloud/providers/test_dimensiondata.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+'''
+Integration tests for the Dimension Data cloud provider
+'''
+
+# Import Python Libs
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+import random
+import string
+
+# Import Salt Testing Libs
+from tests.support.case import ShellCase
+from tests.support.paths import FILES
+from tests.support.helpers import expensiveTest
+
+# Import Salt Libs
+from salt.config import cloud_providers_config
+from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin
+
+# Create the cloud instance name to be used throughout the tests
+INSTANCE_NAME = _random_name('CLOUD-TEST-')
+PROVIDER_NAME = 'dimensiondata'
+
+
+def _random_name(size=6):
+    '''
+    Generates a random cloud instance name
+    '''
+    return 'cloud-test-' + ''.join(
+        random.choice(string.ascii_lowercase + string.digits)
+        for x in range(size)
+    )
+
+
+class DimensionDataTest(ShellCase):
+    '''
+    Integration tests for the Dimension Data cloud provider in Salt-Cloud
+    '''
+
+    @expensiveTest
+    def setUp(self):
+        '''
+        Sets up the test requirements
+        '''
+        super(DimensionDataTest, self).setUp()
+
+        # check if appropriate cloud provider and profile files are present
+        profile_str = 'dimensiondata-config'
+        providers = self.run_cloud('--list-providers')
+        if profile_str + ':' not in providers:
+            self.skipTest(
+                'Configuration file for {0} was not found. Check {0}.conf files '
+                'in tests/integration/files/conf/cloud.*.d/ to run these tests.'
+                .format(PROVIDER_NAME)
+            )
+
+        # check if user_id, key, and region are present
+        config = cloud_providers_config(
+            os.path.join(
+                FILES,
+                'conf',
+                'cloud.providers.d',
+                PROVIDER_NAME + '.conf'
+            )
+        )
+
+        user_id = config[profile_str][PROVIDER_NAME]['user_id']
+        key = config[profile_str][PROVIDER_NAME]['key']
+        region = config[profile_str][PROVIDER_NAME]['region']
+
+        if user_id == '' or key == '' or region == '':
+            self.skipTest(
+                'A user Id, password, and a region '
+                'must be provided to run these tests. Check '
+                'tests/integration/files/conf/cloud.providers.d/{0}.conf'
+                .format(PROVIDER_NAME)
+            )
+
+    def test_list_images(self):
+        '''
+        Tests the return of running the --list-images command for the dimensiondata cloud provider
+        '''
+        image_list = self.run_cloud('--list-images {0}'.format(PROVIDER_NAME))
+        self.assertIn(
+            'Ubuntu 14.04 2 CPU',
+            [i.strip() for i in image_list]
+        )
+
+    def test_list_locations(self):
+        '''
+        Tests the return of running the --list-locations command for the dimensiondata cloud provider
+        '''
+        _list_locations = self.run_cloud('--list-locations {0}'.format(PROVIDER_NAME))
+        self.assertIn(
+            'Australia - Melbourne MCP2',
+            [i.strip() for i in _list_locations]
+        )
+
+    def test_list_sizes(self):
+        '''
+        Tests the return of running the --list-sizes command for the dimensiondata cloud provider
+        '''
+        _list_sizes = self.run_cloud('--list-sizes {0}'.format(PROVIDER_NAME))
+        self.assertIn(
+            'default',
+            [i.strip() for i in _list_sizes]
+        )
+
+    def test_instance(self):
+        '''
+        Test creating an instance on Dimension Data's cloud
+        '''
+        # check if instance with salt installed returned
+        try:
+            self.assertIn(
+                INSTANCE_NAME,
+                [i.strip() for i in self.run_cloud('-p dimensiondata-test {0}'.format(INSTANCE_NAME), timeout=500)]
+            )
+        except AssertionError:
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)
+            raise
+
+        # delete the instance
+        try:
+            self.assertIn(
+                'True',
+                [i.strip() for i in self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)]
+            )
+        except AssertionError:
+            raise
+
+        # Final clean-up of created instance, in case something went wrong.
+        # This was originally in a tearDown function, but that didn't make sense
+        # To run this for each test when not all tests create instances.
+        if INSTANCE_NAME in [i.strip() for i in self.run_cloud('--query')]:
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)

tests/integration/files/conf/cloud.profiles.d/dimensiondata.conf

@@ -0,0 +1,11 @@
+dimensiondata-test:
+  provider: dimensiondata-config
+  image: 42816eb2-9846-4483-95c3-7d7fbddebf2c
+  size: default
+  location: AU10
+  is_started: yes
+  description: 'Salt Ubuntu test'
+  network_domain: ''
+  vlan:  ''
+  ssh_interface: private_ips
+  auth: ''

tests/integration/files/conf/cloud.providers.d/dimensiondata.conf

@@ -0,0 +1,5 @@
+dimensiondata-config:
+  driver: dimensiondata
+  user_id: ''
+  key: ''
+  region: 'dd-au'