From 9ee2dcfc2ddda99f2ef0726aef4914ce116cbc1b Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 09:54:19 -0500
Subject: [PATCH 01/19] Add userdata_renderer master config param

---
 salt/config/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/salt/config/__init__.py b/salt/config/__init__.py
index 5165cc05d2..f1c37d6815 100644
--- a/salt/config/__init__.py
+++ b/salt/config/__init__.py
@@ -212,6 +212,9 @@ VALID_OPTS = {
     # Rendrerer blacklist. Renderers from this list are disalloed even if specified in whitelist.
     'renderer_blacklist': list,
 
+    # A default renderer for userdata files in salt-cloud
+    'userdata_renderer': str,
+
     # A flag indicating that a highstate run should immediately cease if a failure occurs.
     'failhard': bool,
 
@@ -1320,6 +1323,7 @@ DEFAULT_MASTER_OPTS = {
     'renderer': 'yaml_jinja',
     'renderer_whitelist': [],
     'renderer_blacklist': [],
+    'userdata_renderer': 'jinja',
     'failhard': False,
     'state_top': 'top.sls',
     'state_top_saltenv': None,

From 111188742aa401415d738f8c4207b00bcad00d8c Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 10:23:44 -0500
Subject: [PATCH 02/19] Add documentation for userdata_renderer

---
 doc/ref/configuration/master.rst | 16 ++++++++++++++++
 doc/topics/cloud/aws.rst         |  7 +++++++
 doc/topics/cloud/openstack.rst   |  7 +++++++
 doc/topics/cloud/windows.rst     | 16 ++++++++++++----
 4 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst
index 519fb02692..4cdd2b8fe1 100644
--- a/doc/ref/configuration/master.rst
+++ b/doc/ref/configuration/master.rst
@@ -1351,6 +1351,22 @@ The renderer to use on the minions to render the state data.
 
     renderer: yaml_jinja
 
+.. conf_master:: userdata_renderer
+
+``userdata_renderer``
+---------------------
+
+.. versionadded:: 2016.11.4
+
+Default: ``jinja``
+
+The renderer to use for templating userdata files in salt-cloud, if the
+``userdata_renderer`` is not set in the cloud profile.
+
+.. code-block:: yaml
+
+    userdata_renderer: jinja
+
 .. conf_master:: jinja_trim_blocks
 
 ``jinja_trim_blocks``
diff --git a/doc/topics/cloud/aws.rst b/doc/topics/cloud/aws.rst
index 783b9713b5..4c511e22f8 100644
--- a/doc/topics/cloud/aws.rst
+++ b/doc/topics/cloud/aws.rst
@@ -354,6 +354,13 @@ functionality was added to Salt in the 2015.5.0 release.
       # Pass userdata to the instance to be created
       userdata_file: /etc/salt/my-userdata-file
 
+.. note::
+    As of the 2016.11.0 release, this file can be templated, and as of the
+    2016.11.4 release, the renderer(s) used can be specified in the cloud
+    profile using the ``userdata_renderer`` option. If this option is not set
+    in the cloud profile, salt-cloud will fall back to the
+    :conf_master:`userdata_renderer` master configuration option.
+
 
 EC2 allows a location to be set for servers to be deployed in. Availability
 zones exist inside regions, and may be added to increase specificity.
diff --git a/doc/topics/cloud/openstack.rst b/doc/topics/cloud/openstack.rst
index 11bb1380d1..6a57a673a7 100644
--- a/doc/topics/cloud/openstack.rst
+++ b/doc/topics/cloud/openstack.rst
@@ -154,3 +154,10 @@ cloud-init if available.
 .. code-block:: yaml
 
     userdata_file: /etc/salt/cloud-init/packages.yml
+
+.. note::
+    As of the 2016.11.0 release, this file can be templated, and as of the
+    2016.11.4 release, the renderer(s) used can be specified in the cloud
+    profile using the ``userdata_renderer`` option. If this option is not set
+    in the cloud profile, salt-cloud will fall back to the
+    :conf_master:`userdata_renderer` master configuration option.
diff --git a/doc/topics/cloud/windows.rst b/doc/topics/cloud/windows.rst
index 892b80a7c0..133aa67e6b 100644
--- a/doc/topics/cloud/windows.rst
+++ b/doc/topics/cloud/windows.rst
@@ -75,10 +75,18 @@ profile configuration as `userdata_file`. For instance:
 
     userdata_file: /etc/salt/windows-firewall.ps1
 
-If you are using WinRM on EC2 the HTTPS port for the WinRM service must also be enabled
-in your userdata. By default EC2 Windows images only have insecure HTTP enabled. To
-enable HTTPS and basic authentication required by pywinrm consider the following
-userdata example:
+.. note::
+    As of the 2016.11.0 release, this file can be templated, and as of the
+    2016.11.4 release, the renderer(s) used can be specified in the cloud
+    profile using the ``userdata_renderer`` option. If this option is not set
+    in the cloud profile, salt-cloud will fall back to the
+    :conf_master:`userdata_renderer` master configuration option.
+
+
+If you are using WinRM on EC2 the HTTPS port for the WinRM service must also be
+enabled in your userdata. By default EC2 Windows images only have insecure HTTP
+enabled. To enable HTTPS and basic authentication required by pywinrm consider
+the following userdata example:
 
 .. code-block:: powershell
 

From a85a416c7273dd976e0044181d80881a10bb7132 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 10:25:43 -0500
Subject: [PATCH 03/19] Add userdata_renderer fix info to 2016.11.4 release
 notes

---
 doc/topics/releases/2016.11.4.rst | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/doc/topics/releases/2016.11.4.rst b/doc/topics/releases/2016.11.4.rst
index d83170349c..cf2ab24ee6 100644
--- a/doc/topics/releases/2016.11.4.rst
+++ b/doc/topics/releases/2016.11.4.rst
@@ -21,3 +21,25 @@ makes cache operations faster. It doesn't make much sence for the ``localfs``
 cache driver but helps for more complex drivers like ``consul``.
 For more details see ``memcache_expire_seconds`` and other ``memcache_*``
 options in the master config reverence.
+
+Salt-Cloud Fixes
+================
+
+2016.11.0 added support for templating userdata files for the :mod:`ec2
+<salt.cloud.clouds.ec2>` driver, using the :conf_master:`renderer` option from
+the master config file. However, as the default renderer first evaluates jinja
+templating, followed by loading the data as a YAML dictionary, this results in
+unpredictable results when userdata files are comprised of non-YAML data (which
+they generally are).
+
+2016.11.4 fixes this by setting the default renderer used to template userdata
+files to ``jinja``, and adding a new cloud profile config parameter called
+``userdata_renderer``. If this option is not set, then salt-cloud will fall
+back to using the new master configuration parameter
+:conf_master:`userdata_renderer`.
+
+In addition, the other cloud drivers which support setting a ``userdata_file``
+(:mod:`azurearm <salt.cloud.clouds.azurearm>`, :mod:`nova
+<salt.cloud.clouds.nova>`, and :mod:`openstack <salt.cloud.clouds.openstack>`)
+have had templating support added to bring them to feature parity with ec2's
+implementation of the ``userdata_file`` param.

From eddbd41265ae6e9d953fb95cdda09d468d5e9092 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 10:53:20 -0500
Subject: [PATCH 04/19] Openstack did not have templating support for
 userdata_file before 2016.11.4

---
 doc/topics/cloud/openstack.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/topics/cloud/openstack.rst b/doc/topics/cloud/openstack.rst
index 6a57a673a7..e24224b6a1 100644
--- a/doc/topics/cloud/openstack.rst
+++ b/doc/topics/cloud/openstack.rst
@@ -156,8 +156,8 @@ cloud-init if available.
     userdata_file: /etc/salt/cloud-init/packages.yml
 
 .. note::
-    As of the 2016.11.0 release, this file can be templated, and as of the
-    2016.11.4 release, the renderer(s) used can be specified in the cloud
-    profile using the ``userdata_renderer`` option. If this option is not set
-    in the cloud profile, salt-cloud will fall back to the
-    :conf_master:`userdata_renderer` master configuration option.
+    As of the 2016.11.4 release, this file can be templated. The renderer(s)
+    used can be specified in the cloud profile using the ``userdata_renderer``
+    option. If this option is not set in the cloud profile, salt-cloud will
+    fall back to the :conf_master:`userdata_renderer` master configuration
+    option.

From be8d34c59bd8c0780dd8fd75ab40ad65937d8a3d Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 10:55:42 -0500
Subject: [PATCH 05/19] ec2: Add support for using userdata_renderer to
 template userdata_file

---
 salt/cloud/clouds/ec2.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py
index 0a22919eb4..c265f25c0c 100644
--- a/salt/cloud/clouds/ec2.py
+++ b/salt/cloud/clouds/ec2.py
@@ -1681,6 +1681,9 @@ def request_instance(vm_=None, call=None):
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
+    userdata_renderer = config.get_cloud_config_value(
+        'userdata_renderer', vm_, __opts__, search_global=False, default=None
+    )
     if userdata_file is None:
         userdata = config.get_cloud_config_value(
             'userdata', vm_, __opts__, search_global=False, default=None
@@ -1694,7 +1697,11 @@ def request_instance(vm_=None, call=None):
     if userdata is not None:
         render_opts = __opts__.copy()
         render_opts.update(vm_)
-        renderer = __opts__.get('renderer', 'yaml_jinja')
+        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # master configuration file.
+        renderer = __opts__.get('userdata_renderer', 'jinja') \
+            if userdata_renderer is None
+            else userdata_renderer
         rend = salt.loader.render(render_opts, {})
         blacklist = __opts__['renderer_blacklist']
         whitelist = __opts__['renderer_whitelist']

From cc2186f35a09d5018c98e1582e245ab437d6bf1e Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 10:56:17 -0500
Subject: [PATCH 06/19] Add templating support for other cloud drivers that
 support userdata_file

---
 salt/cloud/clouds/azurearm.py  | 17 +++++++++++++++++
 salt/cloud/clouds/nova.py      | 25 ++++++++++++++++++++++---
 salt/cloud/clouds/openstack.py | 25 ++++++++++++++++++++++---
 3 files changed, 61 insertions(+), 6 deletions(-)

diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py
index eb4d15a6b4..1564a2aaaf 100644
--- a/salt/cloud/clouds/azurearm.py
+++ b/salt/cloud/clouds/azurearm.py
@@ -872,6 +872,9 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
+    userdata_renderer = config.get_cloud_config_value(
+        'userdata_renderer', vm_, __opts__, search_global=False, default=None
+    )
     if userdata_file is None:
         userdata = config.get_cloud_config_value(
             'userdata', vm_, __opts__, search_global=False, default=None
@@ -882,6 +885,20 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
                 userdata = fh_.read()
 
     if userdata is not None:
+        render_opts = __opts__.copy()
+        render_opts.update(vm_)
+        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # master configuration file.
+        renderer = __opts__.get('userdata_renderer', 'jinja') \
+            if userdata_renderer is None
+            else userdata_renderer
+        rend = salt.loader.render(render_opts, {})
+        blacklist = __opts__['renderer_blacklist']
+        whitelist = __opts__['renderer_whitelist']
+        userdata = compile_template(
+            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
+        )
+
         os_kwargs['custom_data'] = base64.b64encode(userdata)
 
     iface_data = create_interface(kwargs=vm_)
diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py
index 27380b8975..c13164458d 100644
--- a/salt/cloud/clouds/nova.py
+++ b/salt/cloud/clouds/nova.py
@@ -645,12 +645,31 @@ def request_instance(vm_=None, call=None):
                 kwargs['files'][src_path] = files[src_path]
 
     userdata_file = config.get_cloud_config_value(
-        'userdata_file', vm_, __opts__, search_global=False
+        'userdata_file', vm_, __opts__, search_global=False, default=None
+    )
+    userdata_renderer = config.get_cloud_config_value(
+        'userdata_renderer', vm_, __opts__, search_global=False, default=None
     )
 
     if userdata_file is not None:
-        with salt.utils.fopen(userdata_file, 'r') as fp:
-            kwargs['userdata'] = fp.read()
+        with salt.utils.fopen(userdata_file, 'r') as fp_:
+            userdata = fp_.read()
+
+        render_opts = __opts__.copy()
+        render_opts.update(vm_)
+        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # master configuration file.
+        renderer = __opts__.get('userdata_renderer', 'jinja') \
+            if userdata_renderer is None
+            else userdata_renderer
+        rend = salt.loader.render(render_opts, {})
+        blacklist = __opts__['renderer_blacklist']
+        whitelist = __opts__['renderer_whitelist']
+        userdata = compile_template(
+            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
+        )
+
+        kwargs['userdata'] = userdata
 
     kwargs['config_drive'] = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, search_global=False
diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py
index b74afaa7ce..4d1a437756 100644
--- a/salt/cloud/clouds/openstack.py
+++ b/salt/cloud/clouds/openstack.py
@@ -526,12 +526,31 @@ def request_instance(vm_=None, call=None):
                 kwargs['ex_files'][src_path] = fp_.read()
 
     userdata_file = config.get_cloud_config_value(
-        'userdata_file', vm_, __opts__, search_global=False
+        'userdata_file', vm_, __opts__, search_global=False, default=None
+    )
+    userdata_renderer = config.get_cloud_config_value(
+        'userdata_renderer', vm_, __opts__, search_global=False, default=None
     )
 
     if userdata_file is not None:
-        with salt.utils.fopen(userdata_file, 'r') as fp:
-            kwargs['ex_userdata'] = fp.read()
+        with salt.utils.fopen(userdata_file, 'r') as fp_:
+            userdata = fp_.read()
+
+        render_opts = __opts__.copy()
+        render_opts.update(vm_)
+        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # master configuration file.
+        renderer = __opts__.get('userdata_renderer', 'jinja') \
+            if userdata_renderer is None
+            else userdata_renderer
+        rend = salt.loader.render(render_opts, {})
+        blacklist = __opts__['renderer_blacklist']
+        whitelist = __opts__['renderer_whitelist']
+        userdata = compile_template(
+            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
+        )
+
+        kwargs['ex_userdata'] = userdata
 
     config_drive = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, default=None, search_global=False

From 50f2b2831fe71e329d915710ce654a67b026b9bb Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 11:11:24 -0500
Subject: [PATCH 07/19] Remove userdata_renderer value

---
 salt/config/__init__.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/salt/config/__init__.py b/salt/config/__init__.py
index f1c37d6815..5165cc05d2 100644
--- a/salt/config/__init__.py
+++ b/salt/config/__init__.py
@@ -212,9 +212,6 @@ VALID_OPTS = {
     # Rendrerer blacklist. Renderers from this list are disalloed even if specified in whitelist.
     'renderer_blacklist': list,
 
-    # A default renderer for userdata files in salt-cloud
-    'userdata_renderer': str,
-
     # A flag indicating that a highstate run should immediately cease if a failure occurs.
     'failhard': bool,
 
@@ -1323,7 +1320,6 @@ DEFAULT_MASTER_OPTS = {
     'renderer': 'yaml_jinja',
     'renderer_whitelist': [],
     'renderer_blacklist': [],
-    'userdata_renderer': 'jinja',
     'failhard': False,
     'state_top': 'top.sls',
     'state_top_saltenv': None,

From a6064fb2e439f48737ca040eafec244afa9ee8b8 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 11:13:44 -0500
Subject: [PATCH 08/19] Rename userdata_renderer -> userdata_template in master
 config docs

---
 doc/ref/configuration/master.rst | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst
index 4cdd2b8fe1..f94f25ba4d 100644
--- a/doc/ref/configuration/master.rst
+++ b/doc/ref/configuration/master.rst
@@ -1351,21 +1351,22 @@ The renderer to use on the minions to render the state data.
 
     renderer: yaml_jinja
 
-.. conf_master:: userdata_renderer
+.. conf_master:: userdata_template
 
-``userdata_renderer``
+``userdata_template``
 ---------------------
 
 .. versionadded:: 2016.11.4
 
-Default: ``jinja``
+Default: ``None``
 
 The renderer to use for templating userdata files in salt-cloud, if the
-``userdata_renderer`` is not set in the cloud profile.
+``userdata_template`` is not set in the cloud profile. If no value is set in
+the cloud profile or master config file, no templating will be performed.
 
 .. code-block:: yaml
 
-    userdata_renderer: jinja
+    userdata_template: jinja
 
 .. conf_master:: jinja_trim_blocks
 

From b580654f858abc363ada4618de1211f6b4d1b6f2 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 11:59:18 -0500
Subject: [PATCH 09/19] Update cloud docs to reflect userdata_renderer ->
 userdata_template

---
 doc/topics/cloud/aws.rst       | 22 +++++++++++++++++-----
 doc/topics/cloud/openstack.rst | 23 +++++++++++++++++------
 doc/topics/cloud/windows.rst   | 26 ++++++++++++++++++++------
 3 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/doc/topics/cloud/aws.rst b/doc/topics/cloud/aws.rst
index 4c511e22f8..10f071a965 100644
--- a/doc/topics/cloud/aws.rst
+++ b/doc/topics/cloud/aws.rst
@@ -355,11 +355,23 @@ functionality was added to Salt in the 2015.5.0 release.
       userdata_file: /etc/salt/my-userdata-file
 
 .. note::
-    As of the 2016.11.0 release, this file can be templated, and as of the
-    2016.11.4 release, the renderer(s) used can be specified in the cloud
-    profile using the ``userdata_renderer`` option. If this option is not set
-    in the cloud profile, salt-cloud will fall back to the
-    :conf_master:`userdata_renderer` master configuration option.
+    From versions 2016.11.0 and 2016.11.3, this file was passed through the
+    master's :conf_master:`renderer` to template it. However, this caused
+    issues with non-YAML data, so templating is no longer performed by default.
+    To template the userdata_file, add a ``userdata_template`` option to the
+    cloud profile:
+
+    .. code-block:: yaml
+
+        my-ec2-config:
+          # Pass userdata to the instance to be created
+          userdata_file: /etc/salt/my-userdata-file
+          userdata_template: jinja
+
+    If no ``userdata_template`` is set in the cloud profile, then the master
+    configuration will be checked for a :conf_master:`userdata_template` value.
+    If this is not set, then no templating will be performed on the
+    userdata_file.
 
 
 EC2 allows a location to be set for servers to be deployed in. Availability
diff --git a/doc/topics/cloud/openstack.rst b/doc/topics/cloud/openstack.rst
index e24224b6a1..46571d20e8 100644
--- a/doc/topics/cloud/openstack.rst
+++ b/doc/topics/cloud/openstack.rst
@@ -153,11 +153,22 @@ cloud-init if available.
 
 .. code-block:: yaml
 
-    userdata_file: /etc/salt/cloud-init/packages.yml
+    my-openstack-config:
+      # Pass userdata to the instance to be created
+      userdata_file: /etc/salt/cloud-init/packages.yml
 
 .. note::
-    As of the 2016.11.4 release, this file can be templated. The renderer(s)
-    used can be specified in the cloud profile using the ``userdata_renderer``
-    option. If this option is not set in the cloud profile, salt-cloud will
-    fall back to the :conf_master:`userdata_renderer` master configuration
-    option.
+    As of the 2016.11.4 release, this file can be templated. To use templating,
+    simply specify a ``userdata_template`` option in the cloud profile:
+
+    .. code-block:: yaml
+
+        my-openstack-config:
+          # Pass userdata to the instance to be created
+          userdata_file: /etc/salt/cloud-init/packages.yml
+          userdata_template: jinja
+
+    If no ``userdata_template`` is set in the cloud profile, then the master
+    configuration will be checked for a :conf_master:`userdata_template` value.
+    If this is not set, then no templating will be performed on the
+    userdata_file.
diff --git a/doc/topics/cloud/windows.rst b/doc/topics/cloud/windows.rst
index 133aa67e6b..bc05b1a6d3 100644
--- a/doc/topics/cloud/windows.rst
+++ b/doc/topics/cloud/windows.rst
@@ -73,14 +73,28 @@ profile configuration as `userdata_file`. For instance:
 
 .. code-block:: yaml
 
-    userdata_file: /etc/salt/windows-firewall.ps1
+    my-ec2-config:
+      # Pass userdata to the instance to be created
+      userdata_file: /etc/salt/windows-firewall.ps1
 
 .. note::
-    As of the 2016.11.0 release, this file can be templated, and as of the
-    2016.11.4 release, the renderer(s) used can be specified in the cloud
-    profile using the ``userdata_renderer`` option. If this option is not set
-    in the cloud profile, salt-cloud will fall back to the
-    :conf_master:`userdata_renderer` master configuration option.
+    From versions 2016.11.0 and 2016.11.3, this file was passed through the
+    master's :conf_master:`renderer` to template it. However, this caused
+    issues with non-YAML data, so templating is no longer performed by default.
+    To template the userdata_file, add a ``userdata_template`` option to the
+    cloud profile:
+
+    .. code-block:: yaml
+
+        my-ec2-config:
+          # Pass userdata to the instance to be created
+          userdata_file: /etc/salt/windows-firewall.ps1
+          userdata_template: jinja
+
+    If no ``userdata_template`` is set in the cloud profile, then the master
+    configuration will be checked for a :conf_master:`userdata_template` value.
+    If this is not set, then no templating will be performed on the
+    userdata_file.
 
 
 If you are using WinRM on EC2 the HTTPS port for the WinRM service must also be

From 79cc253bbf4a22d8ef1fb9bd39bceea78ba0cc46 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 12:00:39 -0500
Subject: [PATCH 10/19] Only template the userdata_file if explicitly
 configured to do so

---
 salt/cloud/clouds/azurearm.py  | 34 ++++++++++++++++++++--------------
 salt/cloud/clouds/ec2.py       | 34 ++++++++++++++++++++--------------
 salt/cloud/clouds/nova.py      | 34 ++++++++++++++++++++--------------
 salt/cloud/clouds/openstack.py | 34 ++++++++++++++++++++--------------
 4 files changed, 80 insertions(+), 56 deletions(-)

diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py
index 1564a2aaaf..a7e95912b4 100644
--- a/salt/cloud/clouds/azurearm.py
+++ b/salt/cloud/clouds/azurearm.py
@@ -872,8 +872,8 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_renderer = config.get_cloud_config_value(
-        'userdata_renderer', vm_, __opts__, search_global=False, default=None
+    userdata_template = config.get_cloud_config_value(
+        'userdata_template', vm_, __opts__, search_global=False, default=None
     )
     if userdata_file is None:
         userdata = config.get_cloud_config_value(
@@ -885,19 +885,25 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
                 userdata = fh_.read()
 
     if userdata is not None:
-        render_opts = __opts__.copy()
-        render_opts.update(vm_)
-        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # Use the cloud profile's userdata_template, otherwise get it from the
         # master configuration file.
-        renderer = __opts__.get('userdata_renderer', 'jinja') \
-            if userdata_renderer is None
-            else userdata_renderer
-        rend = salt.loader.render(render_opts, {})
-        blacklist = __opts__['renderer_blacklist']
-        whitelist = __opts__['renderer_whitelist']
-        userdata = compile_template(
-            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
-        )
+        renderer = __opts__.get('userdata_template') \
+            if userdata_template is None
+            else userdata_template
+        if renderer is not None:
+            render_opts = __opts__.copy()
+            render_opts.update(vm_)
+            rend = salt.loader.render(render_opts, {})
+            blacklist = __opts__['renderer_blacklist']
+            whitelist = __opts__['renderer_whitelist']
+            userdata = compile_template(
+                ':string:',
+                rend,
+                renderer,
+                blacklist,
+                whitelist,
+                input_data=userdata,
+            )
 
         os_kwargs['custom_data'] = base64.b64encode(userdata)
 
diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py
index c265f25c0c..9051965168 100644
--- a/salt/cloud/clouds/ec2.py
+++ b/salt/cloud/clouds/ec2.py
@@ -1681,8 +1681,8 @@ def request_instance(vm_=None, call=None):
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_renderer = config.get_cloud_config_value(
-        'userdata_renderer', vm_, __opts__, search_global=False, default=None
+    userdata_template = config.get_cloud_config_value(
+        'userdata_template', vm_, __opts__, search_global=False, default=None
     )
     if userdata_file is None:
         userdata = config.get_cloud_config_value(
@@ -1695,19 +1695,25 @@ def request_instance(vm_=None, call=None):
                 userdata = fh_.read()
 
     if userdata is not None:
-        render_opts = __opts__.copy()
-        render_opts.update(vm_)
-        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # Use the cloud profile's userdata_template, otherwise get it from the
         # master configuration file.
-        renderer = __opts__.get('userdata_renderer', 'jinja') \
-            if userdata_renderer is None
-            else userdata_renderer
-        rend = salt.loader.render(render_opts, {})
-        blacklist = __opts__['renderer_blacklist']
-        whitelist = __opts__['renderer_whitelist']
-        userdata = compile_template(
-            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
-        )
+        renderer = __opts__.get('userdata_template') \
+            if userdata_template is None
+            else userdata_template
+        if renderer is not None:
+            render_opts = __opts__.copy()
+            render_opts.update(vm_)
+            rend = salt.loader.render(render_opts, {})
+            blacklist = __opts__['renderer_blacklist']
+            whitelist = __opts__['renderer_whitelist']
+            userdata = compile_template(
+                ':string:',
+                rend,
+                renderer,
+                blacklist,
+                whitelist,
+                input_data=userdata,
+            )
 
         params[spot_prefix + 'UserData'] = base64.b64encode(userdata)
 
diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py
index c13164458d..3fe85d3f49 100644
--- a/salt/cloud/clouds/nova.py
+++ b/salt/cloud/clouds/nova.py
@@ -647,27 +647,33 @@ def request_instance(vm_=None, call=None):
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_renderer = config.get_cloud_config_value(
-        'userdata_renderer', vm_, __opts__, search_global=False, default=None
+    userdata_template = config.get_cloud_config_value(
+        'userdata_template', vm_, __opts__, search_global=False, default=None
     )
 
     if userdata_file is not None:
         with salt.utils.fopen(userdata_file, 'r') as fp_:
             userdata = fp_.read()
 
-        render_opts = __opts__.copy()
-        render_opts.update(vm_)
-        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # Use the cloud profile's userdata_template, otherwise get it from the
         # master configuration file.
-        renderer = __opts__.get('userdata_renderer', 'jinja') \
-            if userdata_renderer is None
-            else userdata_renderer
-        rend = salt.loader.render(render_opts, {})
-        blacklist = __opts__['renderer_blacklist']
-        whitelist = __opts__['renderer_whitelist']
-        userdata = compile_template(
-            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
-        )
+        renderer = __opts__.get('userdata_template') \
+            if userdata_template is None
+            else userdata_template
+        if renderer is not None:
+            render_opts = __opts__.copy()
+            render_opts.update(vm_)
+            rend = salt.loader.render(render_opts, {})
+            blacklist = __opts__['renderer_blacklist']
+            whitelist = __opts__['renderer_whitelist']
+            userdata = compile_template(
+                ':string:',
+                rend,
+                renderer,
+                blacklist,
+                whitelist,
+                input_data=userdata,
+            )
 
         kwargs['userdata'] = userdata
 
diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py
index 4d1a437756..f23d74ed1b 100644
--- a/salt/cloud/clouds/openstack.py
+++ b/salt/cloud/clouds/openstack.py
@@ -528,27 +528,33 @@ def request_instance(vm_=None, call=None):
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_renderer = config.get_cloud_config_value(
-        'userdata_renderer', vm_, __opts__, search_global=False, default=None
+    userdata_template = config.get_cloud_config_value(
+        'userdata_template', vm_, __opts__, search_global=False, default=None
     )
 
     if userdata_file is not None:
         with salt.utils.fopen(userdata_file, 'r') as fp_:
             userdata = fp_.read()
 
-        render_opts = __opts__.copy()
-        render_opts.update(vm_)
-        # Use the cloud profile's userdata_renderer, otherwise get it from the
+        # Use the cloud profile's userdata_template, otherwise get it from the
         # master configuration file.
-        renderer = __opts__.get('userdata_renderer', 'jinja') \
-            if userdata_renderer is None
-            else userdata_renderer
-        rend = salt.loader.render(render_opts, {})
-        blacklist = __opts__['renderer_blacklist']
-        whitelist = __opts__['renderer_whitelist']
-        userdata = compile_template(
-            ':string:', rend, renderer, blacklist, whitelist, input_data=userdata,
-        )
+        renderer = __opts__.get('userdata_template') \
+            if userdata_template is None
+            else userdata_template
+        if renderer is not None:
+            render_opts = __opts__.copy()
+            render_opts.update(vm_)
+            rend = salt.loader.render(render_opts, {})
+            blacklist = __opts__['renderer_blacklist']
+            whitelist = __opts__['renderer_whitelist']
+            userdata = compile_template(
+                ':string:',
+                rend,
+                renderer,
+                blacklist,
+                whitelist,
+                input_data=userdata,
+            )
 
         kwargs['ex_userdata'] = userdata
 

From 04f02df5fe24e767241121b592dc0ed129e639ca Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 12:11:47 -0500
Subject: [PATCH 11/19] Try to read compiled template as StringIO

The fact that PyYAML was the last renderer in the render pipe initially,
masked the fact that template renderers return StringIO instances and
not strings. This commit makes sure that we try reading from the
StringIO first and then fallback to just trying to base64 encode the
userdata value as returned by compile_template().
---
 salt/cloud/clouds/azurearm.py  | 10 +++++++++-
 salt/cloud/clouds/ec2.py       | 10 +++++++++-
 salt/cloud/clouds/nova.py      |  9 ++++++++-
 salt/cloud/clouds/openstack.py |  9 ++++++++-
 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py
index a7e95912b4..fe77e8b6ca 100644
--- a/salt/cloud/clouds/azurearm.py
+++ b/salt/cloud/clouds/azurearm.py
@@ -905,7 +905,15 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
                 input_data=userdata,
             )
 
-        os_kwargs['custom_data'] = base64.b64encode(userdata)
+        try:
+            # template renderers like "jinja" should return a StringIO
+            os_kwargs['custom_data'] = \
+                ''.join(base64.b64encode(userdata).readlines())
+        except AttributeError:
+            try:
+                os_kwargs['custom_data'] = base64.b64encode(userdata)
+            except Exception as exc:
+                log.exception('Failed to encode userdata: %s')
 
     iface_data = create_interface(kwargs=vm_)
     vm_['iface_id'] = iface_data['id']
diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py
index 9051965168..7e703d3d66 100644
--- a/salt/cloud/clouds/ec2.py
+++ b/salt/cloud/clouds/ec2.py
@@ -1715,7 +1715,15 @@ def request_instance(vm_=None, call=None):
                 input_data=userdata,
             )
 
-        params[spot_prefix + 'UserData'] = base64.b64encode(userdata)
+        try:
+            # template renderers like "jinja" should return a StringIO
+            params[spot_prefix + 'UserData'] = \
+                ''.join(base64.b64encode(userdata).readlines())
+        except AttributeError:
+            try:
+                params[spot_prefix + 'UserData'] = base64.b64encode(userdata)
+            except Exception as exc:
+                log.exception('Failed to encode userdata: %s')
 
     vm_size = config.get_cloud_config_value(
         'size', vm_, __opts__, search_global=False
diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py
index 3fe85d3f49..d30e15728a 100644
--- a/salt/cloud/clouds/nova.py
+++ b/salt/cloud/clouds/nova.py
@@ -675,7 +675,14 @@ def request_instance(vm_=None, call=None):
                 input_data=userdata,
             )
 
-        kwargs['userdata'] = userdata
+        try:
+            # template renderers like "jinja" should return a StringIO
+            kwargs['userdata'] = ''.join(base64.b64encode(userdata).readlines())
+        except AttributeError:
+            try:
+                kwargs['userdata'] = base64.b64encode(userdata)
+            except Exception as exc:
+                log.exception('Failed to encode userdata: %s')
 
     kwargs['config_drive'] = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, search_global=False
diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py
index f23d74ed1b..7de7854918 100644
--- a/salt/cloud/clouds/openstack.py
+++ b/salt/cloud/clouds/openstack.py
@@ -556,7 +556,14 @@ def request_instance(vm_=None, call=None):
                 input_data=userdata,
             )
 
-        kwargs['ex_userdata'] = userdata
+        try:
+            # template renderers like "jinja" should return a StringIO
+            kwargs['ex_userdata'] = ''.join(base64.b64encode(userdata).readlines())
+        except AttributeError:
+            try:
+                kwargs['ex_userdata'] = base64.b64encode(userdata)
+            except Exception as exc:
+                log.exception('Failed to encode userdata: %s')
 
     config_drive = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, default=None, search_global=False

From a6183d93d3d495d25c20a379de8bab74b3ce6d65 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Thu, 30 Mar 2017 15:00:18 -0500
Subject: [PATCH 12/19] Preserve windows newlines in
 salt.template.compile_template()

Test included.
---
 salt/template.py            |  7 +++++
 tests/unit/template_test.py | 54 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/salt/template.py b/salt/template.py
index 7e4c14040e..1060bc9999 100644
--- a/salt/template.py
+++ b/salt/template.py
@@ -80,6 +80,8 @@ def compile_template(template,
     # Get the list of render funcs in the render pipe line.
     render_pipe = template_shebang(template, renderers, default, blacklist, whitelist, input_data)
 
+    windows_newline = '\r\n' in input_data
+
     input_data = string_io(input_data)
     for render, argline in render_pipe:
         # For GPG renderer, input_data can be an OrderedDict (from YAML) or dict (from py renderer).
@@ -120,6 +122,11 @@ def compile_template(template,
                 # structure. We don't want to log this, so ignore this
                 # exception.
                 pass
+
+    # Preserve newlines from original template
+    if windows_newline and '\r\n' not in ret:
+        return ret.replace('\n', '\r\n')
+
     return ret
 
 
diff --git a/tests/unit/template_test.py b/tests/unit/template_test.py
index 48ff4643dc..d0bbe9c30f 100644
--- a/tests/unit/template_test.py
+++ b/tests/unit/template_test.py
@@ -7,8 +7,9 @@
 from __future__ import absolute_import
 
 # Import Salt Testing libs
-from salttesting import TestCase
+from salttesting import skipIf, TestCase
 from salttesting.helpers import ensure_in_syspath
+from salttesting.mock import NO_MOCK, NO_MOCK_REASON, MagicMock
 
 ensure_in_syspath('../')
 
@@ -29,6 +30,57 @@ class TemplateTestCase(TestCase):
         ret = template.compile_template(['1', '2', '3'], None, None, None, None)
         self.assertDictEqual(ret, {})
 
+    @skipIf(NO_MOCK, NO_MOCK_REASON)
+    def test_compile_template_preserves_windows_newlines(self):
+        '''
+        Test to ensure that a file with Windows newlines, when rendered by a
+        template renderer, does not eat the CR character.
+
+        NOTE: template renderers actually return StringIO instances, but since
+        we're mocking the return from the renderer there's no sense in mocking
+        a StringIO only to have to join its contents to get it back to a string
+        for the purposes of comparing the results.
+        '''
+        input_data_windows = 'foo\r\nbar\r\nbaz\r\n'
+        input_data_non_windows = input_data_windows.replace('\r\n', '\n')
+        renderer = 'test'
+        rend = {renderer: MagicMock(return_value=input_data_non_windows)}
+        blacklist = whitelist = []
+
+        ret = template.compile_template(
+            ':string:',
+            rend,
+            renderer,
+            blacklist,
+            whitelist,
+            input_data=input_data_windows)
+        # Even though the mocked renderer returned a string without the windows
+        # newlines, the compiled template should still have them.
+        self.assertEqual(ret, input_data_windows)
+
+        # Now test that we aren't adding them in unnecessarily.
+        ret = template.compile_template(
+            ':string:',
+            rend,
+            renderer,
+            blacklist,
+            whitelist,
+            input_data=input_data_non_windows)
+        self.assertEqual(ret, input_data_non_windows)
+
+        # Finally, ensure that we're not unnecessarily replacing the \n with
+        # \r\n in the event that the renderer returned a string with the
+        # windows newlines intact.
+        rend[renderer] = MagicMock(return_value=input_data_windows)
+        ret = template.compile_template(
+            ':string:',
+            rend,
+            renderer,
+            blacklist,
+            whitelist,
+            input_data=input_data_windows)
+        self.assertEqual(ret, input_data_windows)
+
     def test_check_render_pipe_str(self):
         '''
         Check that all renderers specified in the pipe string are available.

From b440d0c6792b44717dbb346429275544413f4f92 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 12:33:16 -0500
Subject: [PATCH 13/19] Update 2016.11.4 release notes for userdata_renderer ->
 userdata_template

---
 doc/topics/releases/2016.11.4.rst | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/doc/topics/releases/2016.11.4.rst b/doc/topics/releases/2016.11.4.rst
index cf2ab24ee6..946308b144 100644
--- a/doc/topics/releases/2016.11.4.rst
+++ b/doc/topics/releases/2016.11.4.rst
@@ -32,14 +32,27 @@ templating, followed by loading the data as a YAML dictionary, this results in
 unpredictable results when userdata files are comprised of non-YAML data (which
 they generally are).
 
-2016.11.4 fixes this by setting the default renderer used to template userdata
-files to ``jinja``, and adding a new cloud profile config parameter called
-``userdata_renderer``. If this option is not set, then salt-cloud will fall
-back to using the new master configuration parameter
-:conf_master:`userdata_renderer`.
+2016.11.4 fixes this by only templating the userdata_file when it is explicitly
+configured to do so. This is done by adding a new optional parameter to the
+cloud profile called ``userdata_template``. This option is used in the same way
+as the ``template`` argument in :py:func:`file.managed
+<salt.states.file.managed>` states, it is simply set to the desired templating
+renderer:
+
+.. code-block:: yaml
+
+    my-ec2-config:
+      # Pass userdata to the instance to be created
+      userdata_file: /etc/salt/my-userdata-file
+      userdata_template: jinja
+
+If no ``userdata_template``  option is set in the cloud profile, then
+salt-cloud will check for the presence of the master configuration parameter
+:conf_master:`userdata_renderer`. If this is also not set, then no templating
+will be performed on the userdata_file.
 
 In addition, the other cloud drivers which support setting a ``userdata_file``
 (:mod:`azurearm <salt.cloud.clouds.azurearm>`, :mod:`nova
 <salt.cloud.clouds.nova>`, and :mod:`openstack <salt.cloud.clouds.openstack>`)
-have had templating support added to bring them to feature parity with ec2's
-implementation of the ``userdata_file`` param.
+have had templating support added to bring them to feature parity with the ec2
+driver's implementation of the ``userdata_file`` option.

From 6a6ef0adf8b5f81dff57805fe6d042817483a2aa Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 13:46:10 -0500
Subject: [PATCH 14/19] Move userdata templating to salt.utils.cloud

---
 salt/cloud/clouds/azurearm.py  | 36 ++++---------------------
 salt/cloud/clouds/ec2.py       | 38 ++++-----------------------
 salt/cloud/clouds/nova.py      | 37 ++++----------------------
 salt/cloud/clouds/openstack.py | 37 ++++----------------------
 salt/utils/cloud.py            | 48 ++++++++++++++++++++++++++++++++++
 5 files changed, 68 insertions(+), 128 deletions(-)

diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py
index fe77e8b6ca..fcec348ab5 100644
--- a/salt/cloud/clouds/azurearm.py
+++ b/salt/cloud/clouds/azurearm.py
@@ -872,9 +872,6 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_template = config.get_cloud_config_value(
-        'userdata_template', vm_, __opts__, search_global=False, default=None
-    )
     if userdata_file is None:
         userdata = config.get_cloud_config_value(
             'userdata', vm_, __opts__, search_global=False, default=None
@@ -884,36 +881,13 @@ def request_instance(call=None, kwargs=None):  # pylint: disable=unused-argument
             with salt.utils.fopen(userdata_file, 'r') as fh_:
                 userdata = fh_.read()
 
-    if userdata is not None:
-        # Use the cloud profile's userdata_template, otherwise get it from the
-        # master configuration file.
-        renderer = __opts__.get('userdata_template') \
-            if userdata_template is None
-            else userdata_template
-        if renderer is not None:
-            render_opts = __opts__.copy()
-            render_opts.update(vm_)
-            rend = salt.loader.render(render_opts, {})
-            blacklist = __opts__['renderer_blacklist']
-            whitelist = __opts__['renderer_whitelist']
-            userdata = compile_template(
-                ':string:',
-                rend,
-                renderer,
-                blacklist,
-                whitelist,
-                input_data=userdata,
-            )
+    userdata = salt.utils.cloud.userdata_template(__opts__, vm_, userdata)
 
+    if userdata is not None:
         try:
-            # template renderers like "jinja" should return a StringIO
-            os_kwargs['custom_data'] = \
-                ''.join(base64.b64encode(userdata).readlines())
-        except AttributeError:
-            try:
-                os_kwargs['custom_data'] = base64.b64encode(userdata)
-            except Exception as exc:
-                log.exception('Failed to encode userdata: %s')
+            os_kwargs['custom_data'] = base64.b64encode(userdata)
+        except Exception as exc:
+            log.exception('Failed to encode userdata: %s', exc)
 
     iface_data = create_interface(kwargs=vm_)
     vm_['iface_id'] = iface_data['id']
diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py
index 7e703d3d66..be34ad60fd 100644
--- a/salt/cloud/clouds/ec2.py
+++ b/salt/cloud/clouds/ec2.py
@@ -93,8 +93,6 @@ import salt.utils
 from salt._compat import ElementTree as ET
 import salt.utils.http as http
 import salt.utils.aws as aws
-import salt.loader
-from salt.template import compile_template
 
 # Import salt.cloud libs
 import salt.utils.cloud
@@ -1681,9 +1679,6 @@ def request_instance(vm_=None, call=None):
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_template = config.get_cloud_config_value(
-        'userdata_template', vm_, __opts__, search_global=False, default=None
-    )
     if userdata_file is None:
         userdata = config.get_cloud_config_value(
             'userdata', vm_, __opts__, search_global=False, default=None
@@ -1694,36 +1689,13 @@ def request_instance(vm_=None, call=None):
             with salt.utils.fopen(userdata_file, 'r') as fh_:
                 userdata = fh_.read()
 
-    if userdata is not None:
-        # Use the cloud profile's userdata_template, otherwise get it from the
-        # master configuration file.
-        renderer = __opts__.get('userdata_template') \
-            if userdata_template is None
-            else userdata_template
-        if renderer is not None:
-            render_opts = __opts__.copy()
-            render_opts.update(vm_)
-            rend = salt.loader.render(render_opts, {})
-            blacklist = __opts__['renderer_blacklist']
-            whitelist = __opts__['renderer_whitelist']
-            userdata = compile_template(
-                ':string:',
-                rend,
-                renderer,
-                blacklist,
-                whitelist,
-                input_data=userdata,
-            )
+    userdata = salt.utils.cloud.userdata_template(__opts__, vm_, userdata)
 
+    if userdata is not None:
         try:
-            # template renderers like "jinja" should return a StringIO
-            params[spot_prefix + 'UserData'] = \
-                ''.join(base64.b64encode(userdata).readlines())
-        except AttributeError:
-            try:
-                params[spot_prefix + 'UserData'] = base64.b64encode(userdata)
-            except Exception as exc:
-                log.exception('Failed to encode userdata: %s')
+            params[spot_prefix + 'UserData'] = base64.b64encode(userdata)
+        except Exception as exc:
+            log.exception('Failed to encode userdata: %s', exc)
 
     vm_size = config.get_cloud_config_value(
         'size', vm_, __opts__, search_global=False
diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py
index d30e15728a..290349278e 100644
--- a/salt/cloud/clouds/nova.py
+++ b/salt/cloud/clouds/nova.py
@@ -647,42 +647,15 @@ def request_instance(vm_=None, call=None):
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_template = config.get_cloud_config_value(
-        'userdata_template', vm_, __opts__, search_global=False, default=None
-    )
-
     if userdata_file is not None:
         with salt.utils.fopen(userdata_file, 'r') as fp_:
-            userdata = fp_.read()
-
-        # Use the cloud profile's userdata_template, otherwise get it from the
-        # master configuration file.
-        renderer = __opts__.get('userdata_template') \
-            if userdata_template is None
-            else userdata_template
-        if renderer is not None:
-            render_opts = __opts__.copy()
-            render_opts.update(vm_)
-            rend = salt.loader.render(render_opts, {})
-            blacklist = __opts__['renderer_blacklist']
-            whitelist = __opts__['renderer_whitelist']
-            userdata = compile_template(
-                ':string:',
-                rend,
-                renderer,
-                blacklist,
-                whitelist,
-                input_data=userdata,
+            userdata = salt.utils.cloud.userdata_template(
+                __opts__, vm_, fp_.read()
             )
-
         try:
-            # template renderers like "jinja" should return a StringIO
-            kwargs['userdata'] = ''.join(base64.b64encode(userdata).readlines())
-        except AttributeError:
-            try:
-                kwargs['userdata'] = base64.b64encode(userdata)
-            except Exception as exc:
-                log.exception('Failed to encode userdata: %s')
+            kwargs['userdata'] = base64.b64encode(userdata)
+        except Exception as exc:
+            log.exception('Failed to encode userdata: %s', exc)
 
     kwargs['config_drive'] = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, search_global=False
diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py
index 7de7854918..a686afd858 100644
--- a/salt/cloud/clouds/openstack.py
+++ b/salt/cloud/clouds/openstack.py
@@ -528,42 +528,15 @@ def request_instance(vm_=None, call=None):
     userdata_file = config.get_cloud_config_value(
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
-    userdata_template = config.get_cloud_config_value(
-        'userdata_template', vm_, __opts__, search_global=False, default=None
-    )
-
     if userdata_file is not None:
         with salt.utils.fopen(userdata_file, 'r') as fp_:
-            userdata = fp_.read()
-
-        # Use the cloud profile's userdata_template, otherwise get it from the
-        # master configuration file.
-        renderer = __opts__.get('userdata_template') \
-            if userdata_template is None
-            else userdata_template
-        if renderer is not None:
-            render_opts = __opts__.copy()
-            render_opts.update(vm_)
-            rend = salt.loader.render(render_opts, {})
-            blacklist = __opts__['renderer_blacklist']
-            whitelist = __opts__['renderer_whitelist']
-            userdata = compile_template(
-                ':string:',
-                rend,
-                renderer,
-                blacklist,
-                whitelist,
-                input_data=userdata,
+            userdata = salt.utils.cloud.userdata_template(
+                __opts__, vm_, fp_.read()
             )
-
         try:
-            # template renderers like "jinja" should return a StringIO
-            kwargs['ex_userdata'] = ''.join(base64.b64encode(userdata).readlines())
-        except AttributeError:
-            try:
-                kwargs['ex_userdata'] = base64.b64encode(userdata)
-            except Exception as exc:
-                log.exception('Failed to encode userdata: %s')
+            kwargs['ex_userdata'] = base64.b64encode(userdata)
+        except Exception as exc:
+            log.exception('Failed to encode userdata: %s', exc)
 
     config_drive = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, default=None, search_global=False
diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py
index f7c2ec99c5..7827f51224 100644
--- a/salt/utils/cloud.py
+++ b/salt/utils/cloud.py
@@ -55,6 +55,8 @@ except ImportError:
 import salt.crypt
 import salt.client
 import salt.config
+import salt.loader
+import salt.template
 import salt.utils
 import salt.utils.event
 from salt.utils import vt
@@ -3207,3 +3209,49 @@ def check_key_path_and_mode(provider, key_path):
         return False
 
     return True
+
+
+def userdata_template(opts, vm_, userdata):
+    '''
+    Use the configured templating engine to template the userdata file
+    '''
+    # No userdata, no need to template anything
+    if userdata is None:
+        return userdata
+
+    userdata_template = salt.config.get_cloud_config_value(
+        'userdata_template', vm_, opts, search_global=False, default=None
+    )
+    # Use the cloud profile's userdata_template, otherwise get it from the
+    # master configuration file.
+    renderer = opts.get('userdata_template') \
+        if userdata_template is None \
+        else userdata_template
+    if renderer is None:
+        return userdata
+    else:
+        render_opts = opts.copy()
+        render_opts.update(vm_)
+        rend = salt.loader.render(render_opts, {})
+        blacklist = opts['renderer_blacklist']
+        whitelist = opts['renderer_whitelist']
+        templated = salt.template.compile_template(
+            ':string:',
+            rend,
+            renderer,
+            blacklist,
+            whitelist,
+            input_data=userdata,
+        )
+        if not isinstance(templated, six.string_types):
+            # template renderers like "jinja" should return a StringIO
+            try:
+                templated = ''.join(templated.readlines())
+            except AttributeError:
+                log.warning(
+                    'Templated userdata resulted in non-string result (%s), '
+                    'converting to string', templated
+                )
+                templated = str(templated)
+
+        return templated

From d551b0d857d661174d328e942231e0207e13bdcd Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 13:47:32 -0500
Subject: [PATCH 15/19] Bring in salt.utils.stringio from develop branch

---
 salt/utils/stringio.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 salt/utils/stringio.py

diff --git a/salt/utils/stringio.py b/salt/utils/stringio.py
new file mode 100644
index 0000000000..7bc397f8e7
--- /dev/null
+++ b/salt/utils/stringio.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+'''
+Functions for StringIO objects
+'''
+
+from __future__ import absolute_import
+
+# Import 3rd-party libs
+from salt.ext import six
+
+# Not using six's fake cStringIO since we need to be able to tell if the object
+# is readable, and this can't be done via what six exposes.
+if six.PY2:
+    import StringIO
+    import cStringIO
+    readable_types = (StringIO.StringIO, cStringIO.InputType)
+    writable_types = (StringIO.StringIO, cStringIO.OutputType)
+else:
+    import io
+    readable_types = (io.StringIO,)
+    writable_types = (io.StringIO,)
+
+
+def is_stringio(obj):
+    return isinstance(obj, readable_types)
+
+
+def is_readable(obj):
+    if six.PY2:
+        return isinstance(obj, readable_types)
+    else:
+        return isinstance(obj, readable_types) and obj.readable()
+
+
+def is_writable(obj):
+    if six.PY2:
+        return isinstance(obj, writable_types)
+    else:
+        return isinstance(obj, writable_types) and obj.writable()

From 5f7c5613ce1b2e2b1e6337312cfa0f1f91724174 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 13:47:56 -0500
Subject: [PATCH 16/19] Properly handle renderers which return StringIO objects

---
 salt/template.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/salt/template.py b/salt/template.py
index 1060bc9999..aa3136bbaf 100644
--- a/salt/template.py
+++ b/salt/template.py
@@ -13,9 +13,10 @@ import logging
 
 # Import salt libs
 import salt.utils
+import salt.utils.stringio
 from salt.utils.odict import OrderedDict
-from salt._compat import string_io
-from salt.ext.six import string_types
+from salt.ext import six
+from salt.ext.six.moves import StringIO
 
 log = logging.getLogger(__name__)
 
@@ -57,7 +58,7 @@ def compile_template(template,
 
     if template != ':string:':
         # Template was specified incorrectly
-        if not isinstance(template, string_types):
+        if not isinstance(template, six.string_types):
             log.error('Template was specified incorrectly: {0}'.format(template))
             return ret
         # Template does not exist
@@ -82,7 +83,7 @@ def compile_template(template,
 
     windows_newline = '\r\n' in input_data
 
-    input_data = string_io(input_data)
+    input_data = StringIO(input_data)
     for render, argline in render_pipe:
         # For GPG renderer, input_data can be an OrderedDict (from YAML) or dict (from py renderer).
         # Repress the error.
@@ -124,9 +125,21 @@ def compile_template(template,
                 pass
 
     # Preserve newlines from original template
-    if windows_newline and '\r\n' not in ret:
-        return ret.replace('\n', '\r\n')
+    if windows_newline:
+        if salt.utils.stringio.is_readable(ret):
+            is_stringio = True
+            contents = ret.read()
+        else:
+            is_stringio = False
+            contents = ret
 
+        if isinstance(contents, six.string_types):
+            if '\r\n' not in contents:
+                contents = contents.replace('\n', '\r\n')
+                ret = StringIO(contents) if is_stringio else contents
+            else:
+                if is_stringio:
+                    ret.seek(0)
     return ret
 
 

From 78b4798b1b3b088127c5d621f130f33529384c76 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 14:50:14 -0500
Subject: [PATCH 17/19] Update compile_template test to use StringIO

---
 tests/unit/template_test.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/tests/unit/template_test.py b/tests/unit/template_test.py
index d0bbe9c30f..62c4c13dd2 100644
--- a/tests/unit/template_test.py
+++ b/tests/unit/template_test.py
@@ -15,6 +15,7 @@ ensure_in_syspath('../')
 
 # Import Salt libs
 from salt import template
+from salt.ext.six.moves import StringIO
 
 
 class TemplateTestCase(TestCase):
@@ -35,25 +36,26 @@ class TemplateTestCase(TestCase):
         '''
         Test to ensure that a file with Windows newlines, when rendered by a
         template renderer, does not eat the CR character.
-
-        NOTE: template renderers actually return StringIO instances, but since
-        we're mocking the return from the renderer there's no sense in mocking
-        a StringIO only to have to join its contents to get it back to a string
-        for the purposes of comparing the results.
         '''
+        def _get_rend(renderer, value):
+            '''
+            We need a new MagicMock each time since we're dealing with StringIO
+            objects which are read like files.
+            '''
+            return {renderer: MagicMock(return_value=StringIO(value))}
+
         input_data_windows = 'foo\r\nbar\r\nbaz\r\n'
         input_data_non_windows = input_data_windows.replace('\r\n', '\n')
         renderer = 'test'
-        rend = {renderer: MagicMock(return_value=input_data_non_windows)}
         blacklist = whitelist = []
 
         ret = template.compile_template(
             ':string:',
-            rend,
+            _get_rend(renderer, input_data_non_windows),
             renderer,
             blacklist,
             whitelist,
-            input_data=input_data_windows)
+            input_data=input_data_windows).read()
         # Even though the mocked renderer returned a string without the windows
         # newlines, the compiled template should still have them.
         self.assertEqual(ret, input_data_windows)
@@ -61,24 +63,23 @@ class TemplateTestCase(TestCase):
         # Now test that we aren't adding them in unnecessarily.
         ret = template.compile_template(
             ':string:',
-            rend,
+            _get_rend(renderer, input_data_non_windows),
             renderer,
             blacklist,
             whitelist,
-            input_data=input_data_non_windows)
+            input_data=input_data_non_windows).read()
         self.assertEqual(ret, input_data_non_windows)
 
         # Finally, ensure that we're not unnecessarily replacing the \n with
         # \r\n in the event that the renderer returned a string with the
         # windows newlines intact.
-        rend[renderer] = MagicMock(return_value=input_data_windows)
         ret = template.compile_template(
             ':string:',
-            rend,
+            _get_rend(renderer, input_data_windows),
             renderer,
             blacklist,
             whitelist,
-            input_data=input_data_windows)
+            input_data=input_data_windows).read()
         self.assertEqual(ret, input_data_windows)
 
     def test_check_render_pipe_str(self):

From 73f4c43e2adb1af6cf07672e1b3db02b03749529 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 15:31:13 -0500
Subject: [PATCH 18/19] Allow for userdata_template to be disabled in a
 cloud_profile

This handles the case where a global value has been set and it is
necessary to disable it for a single profile.
---
 doc/topics/cloud/aws.rst       | 10 ++++++++++
 doc/topics/cloud/openstack.rst | 11 +++++++++++
 doc/topics/cloud/windows.rst   | 11 +++++++++++
 salt/utils/cloud.py            |  2 ++
 4 files changed, 34 insertions(+)

diff --git a/doc/topics/cloud/aws.rst b/doc/topics/cloud/aws.rst
index 10f071a965..c2131715f1 100644
--- a/doc/topics/cloud/aws.rst
+++ b/doc/topics/cloud/aws.rst
@@ -373,6 +373,16 @@ functionality was added to Salt in the 2015.5.0 release.
     If this is not set, then no templating will be performed on the
     userdata_file.
 
+    To disable templating in a cloud profile when a
+    :conf_master:`userdata_template` has been set in the master configuration
+    file, simply set ``userdata_template`` to ``False`` in the cloud profile:
+
+    .. code-block:: yaml
+
+        my-ec2-config:
+          # Pass userdata to the instance to be created
+          userdata_file: /etc/salt/my-userdata-file
+          userdata_template: False
 
 EC2 allows a location to be set for servers to be deployed in. Availability
 zones exist inside regions, and may be added to increase specificity.
diff --git a/doc/topics/cloud/openstack.rst b/doc/topics/cloud/openstack.rst
index 46571d20e8..fe3da59d24 100644
--- a/doc/topics/cloud/openstack.rst
+++ b/doc/topics/cloud/openstack.rst
@@ -172,3 +172,14 @@ cloud-init if available.
     configuration will be checked for a :conf_master:`userdata_template` value.
     If this is not set, then no templating will be performed on the
     userdata_file.
+
+    To disable templating in a cloud profile when a
+    :conf_master:`userdata_template` has been set in the master configuration
+    file, simply set ``userdata_template`` to ``False`` in the cloud profile:
+
+    .. code-block:: yaml
+
+        my-openstack-config:
+          # Pass userdata to the instance to be created
+          userdata_file: /etc/salt/cloud-init/packages.yml
+          userdata_template: False
diff --git a/doc/topics/cloud/windows.rst b/doc/topics/cloud/windows.rst
index bc05b1a6d3..84ccd05c74 100644
--- a/doc/topics/cloud/windows.rst
+++ b/doc/topics/cloud/windows.rst
@@ -96,6 +96,17 @@ profile configuration as `userdata_file`. For instance:
     If this is not set, then no templating will be performed on the
     userdata_file.
 
+    To disable templating in a cloud profile when a
+    :conf_master:`userdata_template` has been set in the master configuration
+    file, simply set ``userdata_template`` to ``False`` in the cloud profile:
+
+    .. code-block:: yaml
+
+        my-ec2-config:
+          # Pass userdata to the instance to be created
+          userdata_file: /etc/salt/windows-firewall.ps1
+          userdata_template: False
+
 
 If you are using WinRM on EC2 the HTTPS port for the WinRM service must also be
 enabled in your userdata. By default EC2 Windows images only have insecure HTTP
diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py
index 7827f51224..0485a7476a 100644
--- a/salt/utils/cloud.py
+++ b/salt/utils/cloud.py
@@ -3222,6 +3222,8 @@ def userdata_template(opts, vm_, userdata):
     userdata_template = salt.config.get_cloud_config_value(
         'userdata_template', vm_, opts, search_global=False, default=None
     )
+    if userdata_template is False:
+        return userdata
     # Use the cloud profile's userdata_template, otherwise get it from the
     # master configuration file.
     renderer = opts.get('userdata_template') \

From 84ee6930068abbfdc217ff3079627c6aa0f57a05 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Fri, 31 Mar 2017 16:44:28 -0500
Subject: [PATCH 19/19] Nova and openstack don't accept base64-encoded userdata

This fixes my bad copypasta.
---
 salt/cloud/clouds/nova.py      | 12 ++++++------
 salt/cloud/clouds/openstack.py | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py
index 290349278e..ed9251d4b1 100644
--- a/salt/cloud/clouds/nova.py
+++ b/salt/cloud/clouds/nova.py
@@ -648,14 +648,14 @@ def request_instance(vm_=None, call=None):
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
     if userdata_file is not None:
-        with salt.utils.fopen(userdata_file, 'r') as fp_:
-            userdata = salt.utils.cloud.userdata_template(
-                __opts__, vm_, fp_.read()
-            )
         try:
-            kwargs['userdata'] = base64.b64encode(userdata)
+            with salt.utils.fopen(userdata_file, 'r') as fp_:
+                kwargs['userdata'] = salt.utils.cloud.userdata_template(
+                    __opts__, vm_, fp_.read()
+                )
         except Exception as exc:
-            log.exception('Failed to encode userdata: %s', exc)
+            log.exception(
+                'Failed to read userdata from %s: %s', userdata_file, exc)
 
     kwargs['config_drive'] = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, search_global=False
diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py
index a686afd858..cc936509c7 100644
--- a/salt/cloud/clouds/openstack.py
+++ b/salt/cloud/clouds/openstack.py
@@ -529,14 +529,14 @@ def request_instance(vm_=None, call=None):
         'userdata_file', vm_, __opts__, search_global=False, default=None
     )
     if userdata_file is not None:
-        with salt.utils.fopen(userdata_file, 'r') as fp_:
-            userdata = salt.utils.cloud.userdata_template(
-                __opts__, vm_, fp_.read()
-            )
         try:
-            kwargs['ex_userdata'] = base64.b64encode(userdata)
+            with salt.utils.fopen(userdata_file, 'r') as fp_:
+                kwargs['ex_userdata'] = salt.utils.cloud.userdata_template(
+                    __opts__, vm_, fp_.read()
+                )
         except Exception as exc:
-            log.exception('Failed to encode userdata: %s', exc)
+            log.exception(
+                'Failed to read userdata from %s: %s', userdata_file, exc)
 
     config_drive = config.get_cloud_config_value(
         'config_drive', vm_, __opts__, default=None, search_global=False