Change map and profile extends merge strategy to use recursive update

Previously only top-level data elements could be overridden in a map or
extended profile entry. This change adjusts map and profile evaluation
to use a recursive merge using salt.utils.dictupdate.update().

This is a break with compatibility, but in the common case a cloud
profile or map is not structured to truncate configuration but to add
further detail to the profile that it extends.

Update map docs with an example from VMware where we chan now change a
network parameter without having to repeat other device information such
as disk controllers.

doc/topics/cloud/map.rst

@@ -92,25 +92,27 @@ Any top level data element from your profile may be overridden in the map file:
       - web2:
         size: t2.nano
 
-Nested elements cannot be specified individually. Instead, the complete
+As of Salt Nitrogen, nested elements are merged, and can can be specified
-definition for any top level data element must be repeated. In this example a
+individually without having to repeat the complete definition for each top
-separate subnet is assigned to each ec2 instance:
+level data element. In this example a separate MAC is assigned to each VMware
+instance while inheriting device parameters for for disk and network
+configuration:
 
 .. code-block:: yaml
 
-    fedora_small_aws:
+    nyc-vm:
-      - web1:
+      - db1:
-        network_interfaces:
+          devices:
-          - DeviceIndex: 0
+            network:
-            SubnetId: subnet-3bf94a72
+              Network Adapter 1:
-            SecurityGroupId:
+                mac: '44:44:44:44:44:41'
-              - sg-9f644fe5
+      - db2:
-      - web2:
+          devices:
-        network_interfaces:
+            network:
-          - DeviceIndex: 0
+              Network Adapter 1:
-            SubnetId: subnet-c3ad9fa4
+                mac: '44:44:44:44:44:42'
-            SecurityGroupId:
+
-              - sg-9f644fe5
+
 
 A map file may also be used with the various query options:
 

salt/cloud/__init__.py

@@ -31,6 +31,7 @@ import salt.client
 import salt.loader
 import salt.utils
 import salt.utils.cloud
+import salt.utils.dictupdate
 import salt.utils.files
 import salt.syspaths
 from salt.utils import reinit_crypto
@@ -1957,7 +1958,7 @@ class Map(Cloud):
                     if len(overrides['minion']) == 0:
                         del overrides['minion']
 
-                nodedata.update(overrides)
+                nodedata = salt.utils.dictupdate.update(nodedata, overrides)
                 # Add the computed information to the return data
                 ret['create'][nodename] = nodedata
                 # Add the node name to the defined set
@@ -1992,7 +1993,7 @@ class Map(Cloud):
                             break
 
                         log.warning("'{0}' already exists, removing from "
-                                 'the create map.'.format(name))
+                                    'the create map.'.format(name))
 
                         if 'existing' not in ret:
                             ret['existing'] = {}

salt/config/__init__.py

@@ -1709,8 +1709,8 @@ def _validate_opts(opts):
     # We don't expect the user to know this, so we will fix up their path for
     # them if it isn't compliant.
     if (salt.utils.is_windows() and opts.get('transport') == 'raet' and
-             'sock_dir' in opts and
+            'sock_dir' in opts and
-             not opts['sock_dir'].startswith('\\\\.\\mailslot\\')):
+            not opts['sock_dir'].startswith('\\\\.\\mailslot\\')):
         opts['sock_dir'] = (
                 '\\\\.\\mailslot\\' + opts['sock_dir'].replace(':', ''))
 
@@ -2515,9 +2515,10 @@ def apply_vm_profiles_config(providers, overrides, defaults=None):
             vms.pop(profile)
             continue
 
-        extended = vms.get(extends).copy()
+        extended = deepcopy(vms.get(extends))
         extended.pop('profile')
-        extended.update(details)
+        # Merge extended configuration with base profile
+        extended = salt.utils.dictupdate.update(extended, details)
 
         if ':' not in extended['provider']:
             if extended['provider'] not in providers:
@@ -2757,7 +2758,7 @@ def apply_cloud_providers_config(overrides, defaults=None):
                 # Grab a copy of what should be extended
                 extended = providers.get(ext_alias).get(ext_driver).copy()
                 # Merge the data to extend with the details
-                extended.update(details)
+                extended = salt.utils.dictupdate.update(extended, details)
                 # Update the providers dictionary with the merged data
                 providers[alias][driver] = extended
                 # Update name of the driver, now that it's populated with extended information

tests/unit/config/config_test.py

@@ -552,6 +552,33 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn):
                                                           overrides,
                                                           defaults=DEFAULT), ret)
 
+    def test_apply_vm_profiles_config_extend_override_success(self):
+        '''
+        Tests profile extends and recursively merges data elements
+        '''
+        self.maxDiff = None
+        providers = {'test-config': {'ec2': {'profiles': {}, 'driver': 'ec2'}}}
+        overrides = {'Fedora': {'image': 'test-image-2',
+                                'extends': 'dev-instances',
+                                'minion': {'grains': {'stage': 'experimental'}}},
+                     'conf_file': PATH,
+                     'dev-instances': {'ssh_username': 'test_user',
+                                       'provider': 'test-config',
+                                       'minion': {'grains': {'role': 'webserver'}}}}
+        ret = {'Fedora': {'profile': 'Fedora',
+                          'ssh_username': 'test_user',
+                          'image': 'test-image-2',
+                          'minion': {'grains': {'role': 'webserver',
+                                                'stage': 'experimental'}},
+                          'provider': 'test-config:ec2'},
+               'dev-instances': {'profile': 'dev-instances',
+                                 'ssh_username': 'test_user',
+                                 'minion': {'grains': {'role': 'webserver'}},
+                                 'provider': 'test-config:ec2'}}
+        self.assertEqual(sconfig.apply_vm_profiles_config(providers,
+                                                          overrides,
+                                                          defaults=DEFAULT), ret)
+
     # apply_cloud_providers_config tests
 
     def test_apply_cloud_providers_config_same_providers(self):

tests/unit/map_conf_test.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Eric Radman <ericshane@eradman.com>`
+'''
+
+# Import Python libs
+from __future__ import absolute_import
+
+# Import Salt Testing libs
+from salttesting import skipIf, TestCase
+from salttesting.helpers import ensure_in_syspath
+from salttesting.mock import (
+    MagicMock,
+    patch,
+    NO_MOCK,
+    NO_MOCK_REASON,
+)
+
+ensure_in_syspath('../')
+
+# Import Salt libs
+import salt.cloud
+
+EXAMPLE_PROVIDERS = {
+ 'nyc_vcenter': {'vmware': {'driver': 'vmware',
+                            'password': '123456',
+                            'profiles': {'nyc-vm': {'cluster': 'nycvirt',
+                                                    'datastore': 'datastore1',
+                                                    'devices': {'disk': {'Hard disk 1': {'controller': 'SCSI controller 1',
+                                                                                         'size': 20}},
+                                                                 'network': {'Network Adapter 1': {'mac': '44:44:44:44:44:42',
+                                                                                                   'name': 'vlan50',
+                                                                                                   'switch_type': 'standard'}},
+                                                                 'scsi': {'SCSI controller 1': {'type': 'paravirtual'}}},
+                                                    'extra_config': {'mem.hotadd': 'yes'},
+                                                    'folder': 'coreinfra',
+                                                    'image': 'rhel6_64Guest',
+                                                    'memory': '8GB',
+                                                    'num_cpus': 2,
+                                                    'power_on': True,
+                                                    'profile': 'nyc-vm',
+                                                    'provider': 'nyc_vcenter:vmware',
+                                                    'resourcepool': 'Resources'}},
+                            'url': 'vca1.saltstack.com',
+                            'user': 'root'}}
+}
+
+EXAMPLE_PROFILES = {
+ 'nyc-vm': {'cluster': 'nycvirt',
+            'datastore': 'datastore1',
+            'devices': {'disk': {'Hard disk 1': {'controller': 'SCSI controller 1',
+                                                 'size': 20}},
+                        'network': {'Network Adapter 1': {'mac': '44:44:44:44:44:42',
+                                                          'name': 'vlan50',
+                                                          'switch_type': 'standard'}},
+                        'scsi': {'SCSI controller 1': {'type': 'paravirtual'}}},
+            'extra_config': {'mem.hotadd': 'yes'},
+            'folder': 'coreinfra',
+            'image': 'rhel6_64Guest',
+            'memory': '8GB',
+            'num_cpus': 2,
+            'power_on': True,
+            'profile': 'nyc-vm',
+            'provider': 'nyc_vcenter:vmware',
+            'resourcepool': 'Resources'}
+}
+
+EXAMPLE_MAP = {
+ 'nyc-vm': {'db1': {'cpus': 4,
+                    'devices': {'disk': {'Hard disk 1': {'size': 40}},
+                                'network': {'Network Adapter 1': {'mac': '22:4a:b2:92:b3:eb'}}},
+                    'memory': '16GB',
+                    'name': 'db1'}}
+}
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class MapConfTest(TestCase):
+    '''
+    Validate evaluation of salt-cloud map configuration
+    '''
+
+    @patch('salt.cloud.Map.read', MagicMock(return_value=EXAMPLE_MAP))
+    def test_cloud_map_merge_conf(self):
+        self.maxDiff = None
+        '''
+        Ensure that nested values can be selectivly overridden in a map file
+        '''
+        opts = {'extension_modules': '/var/cache/salt/master/extmods',
+                'providers': EXAMPLE_PROVIDERS, 'profiles': EXAMPLE_PROFILES}
+        cloud_map = salt.cloud.Map(opts)
+        merged_profile = {
+         'create': {'db1': {'cluster': 'nycvirt',
+                            'cpus': 4,
+                            'datastore': 'datastore1',
+                            'devices': {'disk': {'Hard disk 1': {'controller': 'SCSI controller 1',
+                                                                 'size': 40}},
+                                        'network': {'Network Adapter 1': {'mac': '22:4a:b2:92:b3:eb',
+                                                                          'name': 'vlan50',
+                                                                          'switch_type': 'standard'}},
+                                        'scsi': {'SCSI controller 1': {'type': 'paravirtual'}}},
+                            'driver': 'vmware',
+                            'extra_config': {'mem.hotadd': 'yes'},
+                            'folder': 'coreinfra',
+                            'image': 'rhel6_64Guest',
+                            'memory': '16GB',
+                            'name': 'db1',
+                            'num_cpus': 2,
+                            'password': '123456',
+                            'power_on': True,
+                            'profile': 'nyc-vm',
+                            'provider': 'nyc_vcenter:vmware',
+                            'resourcepool': 'Resources',
+                            'url': 'vca1.saltstack.com',
+                            'user': 'root'}}
+        }
+        self.assertEqual(cloud_map.map_data(), merged_profile)
+
+
+if __name__ == '__main__':
+    from integration import run_tests
+    run_tests(MapConfTest, needs_daemon=False)