From f28d4f267f792e9471281ea4fda2505b84abe3aa Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Wed, 28 Jun 2017 00:21:44 -0700 Subject: [PATCH 01/54] Fix for https://github.com/saltstack/salt/issues/41971. I think this was a typo? I can deploy VPSes again, at least. --- salt/cloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 7cbb91e462..c24dd406bf 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -1179,7 +1179,7 @@ class Cloud(object): 'minion', vm_, self.opts, default={} ) - alias, driver = vm_['provider'].split(':') + alias, driver = vm_['driver'].split(':') fun = '{0}.create'.format(driver) if fun not in self.clouds: log.error( From e59700459fb0a08c1ae9227e70dd3031314a5e6d Mon Sep 17 00:00:00 2001 From: fake-name Date: Tue, 4 Jul 2017 06:50:29 +0200 Subject: [PATCH 02/54] Silly log output thing for dealing with https://github.com/saltstack/salt/issues/42106 --- salt/cloud/clouds/gce.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index 950fa533e6..4f6578e93f 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -53,11 +53,13 @@ import re import pprint import logging import msgpack +import traceback from ast import literal_eval from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs # pylint: disable=import-error +LIBCLOUD_IMPORT_ERRORS = [] try: import libcloud from libcloud.compute.types import Provider @@ -78,6 +80,9 @@ try: libcloud.security.CA_CERTS_PATH.append('/etc/ssl/certs/YaST-CA.pem') HAS_LIBCLOUD = True except ImportError: + LIBCLOUD_IMPORT_ERRORS.append("Failure importing libcloud!") + for line in traceback.format_exc().split("\n"): + LIBCLOUD_IMPORT_ERRORS.append(line) HAS_LIBCLOUD = False # pylint: enable=import-error @@ -154,6 +159,8 @@ def get_dependencies(): ''' Warn if dependencies aren't met. ''' + for line in LIBCLOUD_IMPORT_ERRORS: + log.error(line) return config.check_driver_dependencies( __virtualname__, {'libcloud': HAS_LIBCLOUD} From 95d51a481ce2f91c90d4fb54fda3a1648711c475 Mon Sep 17 00:00:00 2001 From: fake-name Date: Wed, 5 Jul 2017 07:39:25 +0200 Subject: [PATCH 03/54] Validate cache path exists before trying to write files to it (fixes https://github.com/saltstack/salt/issues/42117) --- salt/cloud/clouds/gce.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index 4f6578e93f..c8d1ad195c 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -2623,6 +2623,9 @@ def update_pricing(kwargs=None, call=None): url = 'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json' price_json = http.query(url, decode=True, decode_type='json') + if not os.path.exists(__opts__['cachedir']): + os.makedirs(__opts__['cachedir']) + outfile = os.path.join( __opts__['cachedir'], 'gce-pricing.p' ) From 3b1613937f2d6fcaf9878985cdb670585b13e96b Mon Sep 17 00:00:00 2001 From: fake-name Date: Wed, 5 Jul 2017 07:42:29 +0200 Subject: [PATCH 04/54] Revert "Fix for https://github.com/saltstack/salt/issues/41971." This reverts commit f28d4f267f792e9471281ea4fda2505b84abe3aa. This was discussed, and fixed upstream, so this commit is now invalid. --- salt/cloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 12beed7858..d7f1be2fd4 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -1187,7 +1187,7 @@ class Cloud(object): 'minion', vm_, self.opts, default={} ) - alias, driver = vm_['driver'].split(':') + alias, driver = vm_['provider'].split(':') fun = '{0}.create'.format(driver) if fun not in self.clouds: log.error( From a2d4be0c4ce664113dfa8df30d30be4a1abeac25 Mon Sep 17 00:00:00 2001 From: fake-name Date: Fri, 7 Jul 2017 08:09:40 +0200 Subject: [PATCH 05/54] Move the cache dir creation to the CloudClient() init. --- salt/cloud/__init__.py | 4 ++++ salt/cloud/clouds/gce.py | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index d7f1be2fd4..b06faf2a38 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -185,6 +185,10 @@ class CloudClient(object): else: self.opts = salt.config.cloud_config(path) + # Check the cache-dir exists. If not, create it. + if not os.path.exists(self.opts['cachedir']): + os.makedirs(self.opts['cachedir']) + if pillars: for name, provider in six.iteritems(pillars.pop('providers', {})): driver = provider['driver'] diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index c8d1ad195c..4f6578e93f 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -2623,9 +2623,6 @@ def update_pricing(kwargs=None, call=None): url = 'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json' price_json = http.query(url, decode=True, decode_type='json') - if not os.path.exists(__opts__['cachedir']): - os.makedirs(__opts__['cachedir']) - outfile = os.path.join( __opts__['cachedir'], 'gce-pricing.p' ) From 46e357a5b85117b8945f5588e12899a6ac595b35 Mon Sep 17 00:00:00 2001 From: fake-name Date: Sat, 8 Jul 2017 10:24:35 +0200 Subject: [PATCH 06/54] Add the ability to handle compressed HTTP(s) content in the query() call. This should resolve https://github.com/saltstack/salt/issues/42219 --- salt/utils/http.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/salt/utils/http.py b/salt/utils/http.py index 51c50b00f7..f46fdb7979 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -15,6 +15,9 @@ import os.path import pprint import socket import yaml +import io +import zlib +import gzip import re import ssl @@ -91,6 +94,36 @@ log = logging.getLogger(__name__) USERAGENT = 'Salt/{0}'.format(salt.version.__version__) +def __decompressContent(coding, pgctnt): + ''' + Decompress returned HTTP content depending on the specified encoding. + Currently supports identity/none, deflate, and gzip, which should + cover 99%+ of the content on the internet. + ''' + + log.trace("Decompressing %s byte content with compression type: %s", len(pgctnt), coding) + + if coding == 'deflate': + pgctnt = zlib.decompress(pgctnt, -zlib.MAX_WBITS) + + elif coding == 'gzip': + buf = io.BytesIO(pgctnt) + f = gzip.GzipFile(fileobj=buf) + pgctnt = f.read() + + elif coding == "sdch": + raise ValueError("SDCH compression is not currently supported") + elif coding == "br": + raise ValueError("Brotli compression is not currently supported") + elif coding == "compress": + raise ValueError("LZW compression is not currently supported") + + elif coding == 'identity': + pass + + log.trace("Content size after decompression: %s", len(pgctnt)) + return pgctnt + @jinja_filter('http_query') def query(url, method='GET', @@ -542,6 +575,11 @@ def query(url, log.debug('Response Status Code: {0}'.format(result_status_code)) log.trace('Response Headers: {0}'.format(result_headers)) log.trace('Response Cookies: {0}'.format(sess_cookies)) + + coding = result_headers.get('Content-Encoding', "identity") + + result_text = __decompressContent(coding, result_text) + try: log.trace('Response Text: {0}'.format(result_text)) except UnicodeEncodeError as exc: From ec285082708a477780dac7f26a159e8aa9f72665 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Wed, 12 Jul 2017 20:50:43 -0700 Subject: [PATCH 07/54] Simple fix for requests annoying decompression behaviour. --- salt/utils/http.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index f46fdb7979..826581a011 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -227,6 +227,8 @@ def query(url, log_url = sanitize_url(url_full, hide_fields) log.debug('Requesting URL {0} using {1} method'.format(log_url, method)) + log.debug("Using backend: %s", backend) + if method == 'POST' and log.isEnabledFor(logging.TRACE): # Make sure no secret fields show up in logs if isinstance(data, dict): @@ -575,10 +577,13 @@ def query(url, log.debug('Response Status Code: {0}'.format(result_status_code)) log.trace('Response Headers: {0}'.format(result_headers)) log.trace('Response Cookies: {0}'.format(sess_cookies)) + # log.trace("Content: %s", result_text) coding = result_headers.get('Content-Encoding', "identity") - result_text = __decompressContent(coding, result_text) + # Requests will always decompress the content, and working around that is annoying. + if backend != 'requests': + result_text = __decompressContent(coding, result_text) try: log.trace('Response Text: {0}'.format(result_text)) From 5dc6bec38c3faadafdd94e06e1d2e71669e8074a Mon Sep 17 00:00:00 2001 From: Connor Wolf Date: Thu, 13 Jul 2017 13:09:43 -0700 Subject: [PATCH 08/54] Fix pylint failure, I think? --- salt/utils/http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/utils/http.py b/salt/utils/http.py index 826581a011..c2d033ad21 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -124,6 +124,7 @@ def __decompressContent(coding, pgctnt): log.trace("Content size after decompression: %s", len(pgctnt)) return pgctnt + @jinja_filter('http_query') def query(url, method='GET', From 54e79f89ccc7adb44e17785e0ed9270e101b90f1 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 12 Aug 2017 20:20:55 -0700 Subject: [PATCH 09/54] Change @cachedout asked for: https://github.com/saltstack/salt/pull/42118#discussion_r129429079 --- salt/cloud/clouds/gce.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index 9c4001eaf2..0847a98bdd 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -59,7 +59,7 @@ from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs # pylint: disable=import-error -LIBCLOUD_IMPORT_ERRORS = [] +LIBCLOUD_IMPORT_ERROR = None try: import libcloud from libcloud.compute.types import Provider @@ -79,10 +79,8 @@ try: import libcloud.security libcloud.security.CA_CERTS_PATH.append('/etc/ssl/certs/YaST-CA.pem') HAS_LIBCLOUD = True -except ImportError: - LIBCLOUD_IMPORT_ERRORS.append("Failure importing libcloud!") - for line in traceback.format_exc().split("\n"): - LIBCLOUD_IMPORT_ERRORS.append(line) +except ImportError as e: + LIBCLOUD_IMPORT_ERROR = e HAS_LIBCLOUD = False # pylint: enable=import-error @@ -160,8 +158,8 @@ def get_dependencies(): ''' Warn if dependencies aren't met. ''' - for line in LIBCLOUD_IMPORT_ERRORS: - log.error(line) + if LIBCLOUD_IMPORT_ERROR: + log.error("Failure when importing LibCloud: ", exc_info=LIBCLOUD_IMPORT_ERROR) return config.check_driver_dependencies( __virtualname__, {'libcloud': HAS_LIBCLOUD} From 43be5c161f40b59ffcc439cec8903eeaababa44d Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 12 Aug 2017 20:45:56 -0700 Subject: [PATCH 10/54] Lightsail thing. Non functional so far. --- salt/cloud/clouds/lightsail.py | 96 ++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 salt/cloud/clouds/lightsail.py diff --git a/salt/cloud/clouds/lightsail.py b/salt/cloud/clouds/lightsail.py new file mode 100644 index 0000000000..b1d606e9a0 --- /dev/null +++ b/salt/cloud/clouds/lightsail.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +''' +Pyrax Cloud Module +================== + +PLEASE NOTE: This module is currently in early development, and considered to +be experimental and unstable. It is not recommended for production use. Unless +you are actively developing code in this module, you should use the OpenStack +module instead. +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import salt libs +import salt.utils +import salt.config as config +import salt.utils.boto3 +import salt.utils.compat +import salt.utils.odict as odict + + +import yaml +import salt.ext.six as six +try: + import boto3 # pylint: disable=unused-import + from botocore.exceptions import ClientError + logging.getLogger('boto3').setLevel(logging.CRITICAL) + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + + + +__virtualname__ = 'lightsail' + + +# Only load in this module is the PYRAX configurations are in place +def __virtual__(): + ''' + Check for Pyrax configurations + ''' + if get_configured_provider() is False: + return False + + if get_dependencies() is False: + return False + + return __virtualname__ + +def get_configured_provider(): + ''' + Return the first configured instance. + ''' + return config.is_provider_configured( + __opts__, + __active_provider_name__ or __virtualname__, + ('id', 'key') + ) + + +def get_dependencies(): + ''' + Warn if dependencies aren't met. + ''' + return config.check_driver_dependencies( + __virtualname__, + {'boto3': HAS_BOTO} + ) + + + +def queues_exists(call, kwargs): + conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') + return conn.exists(kwargs['name']) + + +def queues_show(call, kwargs): + conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') + return salt.utils.simple_types_filter(conn.show(kwargs['name']).__dict__) + + +def queues_create(call, kwargs): + conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') + if conn.create(kwargs['name']): + return salt.utils.simple_types_filter(conn.show(kwargs['name']).__dict__) + else: + return {} + + +def queues_delete(call, kwargs): + conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') + if conn.delete(kwargs['name']): + return {} + else: + return salt.utils.simple_types_filter(conn.show(kwargs['name'].__dict__)) From eb3e8adef938156f8757348b38238124aad8c33e Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 12 Aug 2017 22:23:33 -0700 Subject: [PATCH 11/54] Much better fix for https://github.com/saltstack/salt/pull/42118 --- salt/cloud/clouds/gce.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index 0847a98bdd..e064b14812 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -49,6 +49,7 @@ Example Provider Configuration # Import python libs from __future__ import absolute_import import os +import sys import re import pprint import logging @@ -79,8 +80,8 @@ try: import libcloud.security libcloud.security.CA_CERTS_PATH.append('/etc/ssl/certs/YaST-CA.pem') HAS_LIBCLOUD = True -except ImportError as e: - LIBCLOUD_IMPORT_ERROR = e +except ImportError: + LIBCLOUD_IMPORT_ERROR = sys.exc_info() HAS_LIBCLOUD = False # pylint: enable=import-error @@ -160,6 +161,7 @@ def get_dependencies(): ''' if LIBCLOUD_IMPORT_ERROR: log.error("Failure when importing LibCloud: ", exc_info=LIBCLOUD_IMPORT_ERROR) + log.error("Note: The libcloud dependency is called 'apache-libcloud' on PyPi/pip.") return config.check_driver_dependencies( __virtualname__, {'libcloud': HAS_LIBCLOUD} From 5449d37c38809433f84384b64ecd942dcd05005b Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Sat, 12 Aug 2017 22:27:27 -0700 Subject: [PATCH 12/54] Revert "Lightsail thing. Non functional so far." This reverts commit 43be5c161f40b59ffcc439cec8903eeaababa44d. Whoops, meant to put that on a branch --- salt/cloud/clouds/lightsail.py | 96 ---------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 salt/cloud/clouds/lightsail.py diff --git a/salt/cloud/clouds/lightsail.py b/salt/cloud/clouds/lightsail.py deleted file mode 100644 index b1d606e9a0..0000000000 --- a/salt/cloud/clouds/lightsail.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Pyrax Cloud Module -================== - -PLEASE NOTE: This module is currently in early development, and considered to -be experimental and unstable. It is not recommended for production use. Unless -you are actively developing code in this module, you should use the OpenStack -module instead. -''' - -# Import Python Libs -from __future__ import absolute_import - -# Import salt libs -import salt.utils -import salt.config as config -import salt.utils.boto3 -import salt.utils.compat -import salt.utils.odict as odict - - -import yaml -import salt.ext.six as six -try: - import boto3 # pylint: disable=unused-import - from botocore.exceptions import ClientError - logging.getLogger('boto3').setLevel(logging.CRITICAL) - HAS_BOTO = True -except ImportError: - HAS_BOTO = False - - - -__virtualname__ = 'lightsail' - - -# Only load in this module is the PYRAX configurations are in place -def __virtual__(): - ''' - Check for Pyrax configurations - ''' - if get_configured_provider() is False: - return False - - if get_dependencies() is False: - return False - - return __virtualname__ - -def get_configured_provider(): - ''' - Return the first configured instance. - ''' - return config.is_provider_configured( - __opts__, - __active_provider_name__ or __virtualname__, - ('id', 'key') - ) - - -def get_dependencies(): - ''' - Warn if dependencies aren't met. - ''' - return config.check_driver_dependencies( - __virtualname__, - {'boto3': HAS_BOTO} - ) - - - -def queues_exists(call, kwargs): - conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') - return conn.exists(kwargs['name']) - - -def queues_show(call, kwargs): - conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') - return salt.utils.simple_types_filter(conn.show(kwargs['name']).__dict__) - - -def queues_create(call, kwargs): - conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') - if conn.create(kwargs['name']): - return salt.utils.simple_types_filter(conn.show(kwargs['name']).__dict__) - else: - return {} - - -def queues_delete(call, kwargs): - conn = salt.utils.boto3.get_connection('lightsail', 'lightsail') - if conn.delete(kwargs['name']): - return {} - else: - return salt.utils.simple_types_filter(conn.show(kwargs['name'].__dict__)) From 1169c1412a29ad2bcc5d6e021802d5d9668da092 Mon Sep 17 00:00:00 2001 From: Fake-Name Date: Mon, 11 Sep 2017 22:17:26 -0700 Subject: [PATCH 13/54] Switch path verify thingie. --- salt/cloud/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 04230d3899..6613dd68a5 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -35,6 +35,7 @@ import salt.utils.cloud import salt.utils.context import salt.utils.dictupdate import salt.utils.files +import salt.utils.verify import salt.syspaths from salt.template import compile_template @@ -184,9 +185,10 @@ class CloudClient(object): else: self.opts = salt.config.cloud_config(path) + # Check the cache-dir exists. If not, create it. - if not os.path.exists(self.opts['cachedir']): - os.makedirs(self.opts['cachedir']) + v_dirs = [self.opts['cachedir']] + salt.utils.verify.verify_env(v_dirs, salt.utils.get_user()) if pillars: for name, provider in six.iteritems(pillars.pop('providers', {})): From 24d8bd2ac35062aa42359220c79a6960c38be762 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Mon, 9 Oct 2017 14:17:27 -0400 Subject: [PATCH 14/54] Quick fix for caller_ref issues in boto_route53 --- salt/states/boto3_route53.py | 3 +-- salt/states/boto_route53.py | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/salt/states/boto3_route53.py b/salt/states/boto3_route53.py index 0a49dcc3fe..11243941e4 100644 --- a/salt/states/boto3_route53.py +++ b/salt/states/boto3_route53.py @@ -143,8 +143,7 @@ def hosted_zone_present(name, Name=None, PrivateZone=False, Ensure a hosted zone exists with the given attributes. name - The name of the state definition. This will be used as the 'CallerReference' param when - creating the hosted zone to help ensure idempotency. + The name of the state definition. Name The name of the domain. This should be a fully-specified domain, and should terminate with a diff --git a/salt/states/boto_route53.py b/salt/states/boto_route53.py index dddbc2705b..9bd9a08d0a 100644 --- a/salt/states/boto_route53.py +++ b/salt/states/boto_route53.py @@ -74,6 +74,7 @@ passed in as a dict, or as a string to pull from pillars or minion config: # Import Python Libs from __future__ import absolute_import import json +import uuid # Import Salt Libs from salt.utils import SaltInvocationError, exactly_one @@ -359,8 +360,8 @@ def absent( return ret -def hosted_zone_present(name, domain_name=None, private_zone=False, comment='', - vpc_id=None, vpc_name=None, vpc_region=None, +def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=None, + comment='', vpc_id=None, vpc_name=None, vpc_region=None, region=None, key=None, keyid=None, profile=None): ''' Ensure a hosted zone exists with the given attributes. Note that most @@ -374,8 +375,7 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, comment='', - vpc_region (again, supported in boto3 but not boto2). name - The name of the state definition. This will be used as the 'caller_ref' - param if/when creating the hosted zone. + The name of the state definition. domain_name The name of the domain. This should be a fully-specified domain, and @@ -384,12 +384,21 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, comment='', registrar to the Amazon Route 53 delegation servers returned in response to this request. Defaults to the value of name if not provided. - comment - Any comments you want to include about the hosted zone. - private_zone Set True if creating a private hosted zone. + caller_ref + A unique string that identifies the request and that allows create_hosted_zone() calls to be + retried without the risk of executing the operation twice. This helps ensure idempotency + across state calls, but can cause issues if a zone is deleted and then an attempt is made + to recreate it with the same caller_ref. If not provided, a unique UUID will be generated + at each state run, which can potentially lead to duplicate zones being created if the state + is run again while the previous zone creation is still in PENDING status (which can + occasionally take several minutes to clear). Maximum length of 128. + + comment + Any comments you want to include about the hosted zone. + vpc_id When creating a private hosted zone, either the VPC ID or VPC Name to associate with is required. Exclusive with vpe_name. Ignored if passed @@ -475,13 +484,15 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, comment='', 'This may fail...'.format(domain_name)) if create: + if caller_ref is None: + caller_ref = str(uuid.uuid4()) if __opts__['test']: ret['comment'] = 'Route53 Hosted Zone {0} set to be added.'.format( domain_name) ret['result'] = None return ret res = __salt__['boto_route53.create_hosted_zone'](domain_name=domain_name, - caller_ref=name, comment=comment, private_zone=private_zone, + caller_ref=caller_ref, comment=comment, private_zone=private_zone, vpc_id=vpc_id, vpc_region=vpc_region, region=region, key=key, keyid=keyid, profile=profile) if res: From 7cc6ee3e9647d69f4fb1d0efb0d8062ee1059b7a Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Mon, 13 Nov 2017 17:58:52 -0500 Subject: [PATCH 15/54] INFRA-5975 - better fix for https://github.com/saltstack/salt/issues/40655 --- salt/modules/boto_route53.py | 77 +++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/salt/modules/boto_route53.py b/salt/modules/boto_route53.py index ef1e83ce3e..70c25c3429 100644 --- a/salt/modules/boto_route53.py +++ b/salt/modules/boto_route53.py @@ -559,20 +559,46 @@ def delete_record(name, zone, record_type, identifier=None, all_records=False, raise e -def _wait_for_sync(status, conn, wait_for_sync): - if not wait_for_sync: +def _try_func(conn, func, **args): + tries = 30 + while True: + try: + return getattr(conn, func)(**args) + except AttributeError as e: + # Don't include **args in log messages - security concern. + log.error('Function `{0}()` not found for AWS connection object ' + '{1}'.format(func, conn)) + return None + except DNSServerError as e: + if tries and e.code == 'Throttling': + log.debug('Throttled by AWS API. Will retry in 5 seconds') + time.sleep(5) + tries -= 1 + continue + log.error('Failed calling {0}(): {1}'.format(func, str(e))) + return None + + +def _wait_for_sync(status, conn, wait=True): + ### Wait should be a bool or an integer + if wait is True: + wait = 600 + if not wait: return True - retry = 10 - i = 0 - while i < retry: - log.info('Getting route53 status (attempt {0})'.format(i + 1)) + orig_wait = wait + log.info('Waiting up to {0} seconds for Route53 changes to synchronize'.format(orig_wait)) + while wait > 0: change = conn.get_change(status) - log.debug(change.GetChangeResponse.ChangeInfo.Status) - if change.GetChangeResponse.ChangeInfo.Status == 'INSYNC': + current = change.GetChangeResponse.ChangeInfo.Status + if current == 'INSYNC': return True - i = i + 1 - time.sleep(20) - log.error('Timed out waiting for Route53 status update.') + sleep = wait if wait % 60 == wait else 60 + log.info('Sleeping {0} seconds waiting for changes to synch (current status {1})'.format( + sleep, current)) + time.sleep(sleep) + wait -= sleep + continue + log.error('Route53 changes not synced after {0} seconds.'.format(orig_wait)) return False @@ -684,20 +710,15 @@ def create_hosted_zone(domain_name, caller_ref=None, comment='', log.info('Options vpc_id, vpc_name, and vpc_region are ignored ' 'when creating non-private zones.') - retries = 10 - while retries: - try: - # Crazy layers of dereference... - r = conn.create_hosted_zone(**args) - r = r.CreateHostedZoneResponse.__dict__ if hasattr(r, - 'CreateHostedZoneResponse') else {} - return r.get('parent', {}).get('CreateHostedZoneResponse') - except DNSServerError as e: - if retries and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') - time.sleep(3) - retries -= 1 - continue - log.error('Failed to create hosted zone {0}: {1}'.format( - domain_name, e.message)) - return None + r = _try_func(conn, 'create_hosted_zone', **args) + if r is None: + log.error('Failed to create hosted zone {0}'.format(domain_name)) + return None + r = r.get('CreateHostedZoneResponse', {}) + # Pop it since it'll be irrelevant by the time we return + status = r.pop('ChangeInfo', {}).get('Id', '').replace('/change/', '') + synced = _wait_for_sync(status, conn, wait=600) + if not synced: + log.error('Hosted zone {0} not synced after 600 seconds.'.format(domain_name)) + return None + return r From 122f4a6e6d3cc508ca4a7015a96c176487ed8132 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Tue, 14 Nov 2017 13:39:00 +0000 Subject: [PATCH 16/54] Nested outputter colors depending on the function retcode As discussed under #44485, it would be nice that the colors of the nested outputter to depend on the execution function retcode (from the __context__, when available). If the retcode is non-zero, the output will be mostly red to emphasise that's an error. --- salt/output/nested.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/output/nested.py b/salt/output/nested.py index 563bf5768f..fc6c0230ad 100644 --- a/salt/output/nested.py +++ b/salt/output/nested.py @@ -106,12 +106,15 @@ class NestDisplay(object): ) first_line = False elif isinstance(ret, (list, tuple)): + color = self.GREEN + if 'retcode' in __context__ and __context__['retcode']: + color = self.RED for ind in ret: if isinstance(ind, (list, tuple, dict)): out.append( self.ustring( indent, - self.GREEN, + color, '|_' ) ) @@ -121,10 +124,13 @@ class NestDisplay(object): self.display(ind, indent, '- ', out) elif isinstance(ret, dict): if indent: + color = self.CYAN + if 'retcode' in __context__ and __context__['retcode']: + color = self.RED out.append( self.ustring( indent, - self.CYAN, + color, '----------' ) ) @@ -134,13 +140,15 @@ class NestDisplay(object): keys = ret.keys() else: keys = sorted(ret) - + color = self.CYAN + if 'retcode' in __context__ and __context__['retcode']: + color = self.RED for key in keys: val = ret[key] out.append( self.ustring( indent, - self.CYAN, + color, key, suffix=':', prefix=prefix From ccec04ed8620934790ae490613864482946b9569 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Tue, 28 Nov 2017 07:24:48 -0500 Subject: [PATCH 17/54] INFRA-5975 - whitespace cleanup --- salt/modules/boto_route53.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/modules/boto_route53.py b/salt/modules/boto_route53.py index 9e8b2751f3..1d0b57d32f 100644 --- a/salt/modules/boto_route53.py +++ b/salt/modules/boto_route53.py @@ -814,5 +814,3 @@ def create_hosted_zone(domain_name, caller_ref=None, comment='', log.error('Hosted zone {0} not synced after 600 seconds.'.format(domain_name)) return None return r - - \ No newline at end of file From 9b5d8c421bf283bbb498a10f4ec0355ad50ea275 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 29 Nov 2017 16:01:59 -0700 Subject: [PATCH 18/54] Handle unicode values --- salt/ext/win_inet_pton.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index 8c4d7dde3e..1451ce4d2c 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -36,7 +36,7 @@ def inet_pton(address_family, ip_string): # This will catch IP Addresses such as 10.1.2 if address_family == socket.AF_INET: try: - ipaddress.ip_address(ip_string.decode()) + ipaddress.ip_address(ip_string.encode().decode()) except ValueError: raise socket.error('illegal IP address string passed to inet_pton') return socket.inet_aton(ip_string) From 5ac81125856cecb2d22008bd0d8d869dbee86e86 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 1 Dec 2017 16:12:10 -0700 Subject: [PATCH 19/54] Use six to ensure unicode value --- salt/ext/win_inet_pton.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index 1451ce4d2c..49b8d35c68 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -10,6 +10,7 @@ import socket import ctypes import os import ipaddress +import six class sockaddr(ctypes.Structure): @@ -36,7 +37,7 @@ def inet_pton(address_family, ip_string): # This will catch IP Addresses such as 10.1.2 if address_family == socket.AF_INET: try: - ipaddress.ip_address(ip_string.encode().decode()) + ipaddress.ip_address(six.u(ip_string)) except ValueError: raise socket.error('illegal IP address string passed to inet_pton') return socket.inet_aton(ip_string) From df1e6a202b73fc8d39efadd9a905faccbb100959 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Dec 2017 14:02:59 -0700 Subject: [PATCH 20/54] Use salt.ext.six --- salt/ext/win_inet_pton.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index 49b8d35c68..e301085521 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -10,7 +10,7 @@ import socket import ctypes import os import ipaddress -import six +import salt.ext.six as six class sockaddr(ctypes.Structure): From 494835c3f25cb8c358300aed22821d6732cb99fe Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Wed, 6 Dec 2017 01:08:16 +1100 Subject: [PATCH 21/54] I backported develop and applied a long list of fixes to 2016.11 this brings these fixes into 2017.7 - Software was not always being removed, general if & was in the string or msi was downloaded to uninstall the software - pkg.list_upgrades failed. Added support for 'latest' and 'Not Found' for version_cmp() to fix this. - output fixes - pkg.list_available no longer forces a pkg.refresh_db this is no longer required, as by default it will update if older than 6 hours - cmd /s /c is prefixed for all commands i.e. installs and removes. - cmd are now strings, instead of a list when using cmd.run. As windows only supports strings. And the " were being broken --- salt/modules/win_pkg.py | 228 ++++++++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 93 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index c8f699a7d6..5ad15989c0 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -39,10 +39,11 @@ import logging import os import re import time +import sys from functools import cmp_to_key # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error,no-name-in-module from salt.ext.six.moves.urllib.parse import urlparse as _urlparse @@ -50,9 +51,12 @@ from salt.ext.six.moves.urllib.parse import urlparse as _urlparse from salt.exceptions import (CommandExecutionError, SaltInvocationError, SaltRenderError) -import salt.utils +import salt.utils # Can be removed once is_true, get_hash, compare_dicts are moved +import salt.utils.args +import salt.utils.files import salt.utils.pkg import salt.utils.path +import salt.utils.versions import salt.syspaths import salt.payload from salt.exceptions import MinionError @@ -99,7 +103,7 @@ def latest_version(*names, **kwargs): salt '*' pkg.latest_version salt '*' pkg.latest_version ... ''' - if len(names) == 0: + if not names: return '' # Initialize the return dict with empty strings @@ -124,6 +128,8 @@ def latest_version(*names, **kwargs): if name in installed_pkgs: log.trace('Determining latest installed version of %s', name) try: + # installed_pkgs[name] Can be version number or 'Not Found' + # 'Not Found' occurs when version number is not found in the registry latest_installed = sorted( installed_pkgs[name], key=cmp_to_key(_reverse_cmp_pkg_versions) @@ -140,6 +146,8 @@ def latest_version(*names, **kwargs): # get latest available (from winrepo_dir) version of package pkg_info = _get_package_info(name, saltenv=saltenv) log.trace('Raw winrepo pkg_info for {0} is {1}'.format(name, pkg_info)) + + # latest_available can be version number or 'latest' or even 'Not Found' latest_available = _get_latest_pkg_version(pkg_info) if latest_available: log.debug('Latest available version ' @@ -147,9 +155,9 @@ def latest_version(*names, **kwargs): # check, whether latest available version # is newer than latest installed version - if salt.utils.compare_versions(ver1=str(latest_available), - oper='>', - ver2=str(latest_installed)): + if compare_versions(ver1=str(latest_available), + oper='>', + ver2=str(latest_installed)): log.debug('Upgrade of {0} from {1} to {2} ' 'is available'.format(name, latest_installed, @@ -187,11 +195,9 @@ def upgrade_available(name, **kwargs): # Refresh before looking for the latest version available, # same default as latest_version refresh = salt.utils.is_true(kwargs.get('refresh', True)) - - current = version(name, saltenv=saltenv, refresh=refresh).get(name) - latest = latest_version(name, saltenv=saltenv, refresh=False) - - return compare_versions(latest, '>', current) + # if latest_version returns blank, the latest version is already installed or + # their is no package definition. This is a salt standard which could be improved. + return latest_version(name, saltenv=saltenv, refresh=refresh) != '' def list_upgrades(refresh=True, **kwargs): @@ -222,9 +228,13 @@ def list_upgrades(refresh=True, **kwargs): pkgs = {} for pkg in installed_pkgs: if pkg in available_pkgs: + # latest_version() will be blank if the latest version is installed. + # or the package name is wrong. Given we check available_pkgs, this + # should not be the case of wrong package name. + # Note: latest_version() is an expensive way to do this as it + # calls list_pkgs each time. latest_ver = latest_version(pkg, refresh=False, saltenv=saltenv) - install_ver = installed_pkgs[pkg] - if compare_versions(latest_ver, '>', install_ver): + if latest_ver: pkgs[pkg] = latest_ver return pkgs @@ -241,7 +251,7 @@ def list_available(*names, **kwargs): saltenv (str): The salt environment to use. Default ``base``. - refresh (bool): Refresh package metadata. Default ``True``. + refresh (bool): Refresh package metadata. Default ``False``. return_dict_always (bool): Default ``False`` dict when a single package name is queried. @@ -264,11 +274,10 @@ def list_available(*names, **kwargs): return '' saltenv = kwargs.get('saltenv', 'base') - refresh = salt.utils.is_true(kwargs.get('refresh', True)) + refresh = salt.utils.is_true(kwargs.get('refresh', False)) + _refresh_db_conditional(saltenv, force=refresh) return_dict_always = \ salt.utils.is_true(kwargs.get('return_dict_always', False)) - - _refresh_db_conditional(saltenv, force=refresh) if len(names) == 1 and not return_dict_always: pkginfo = _get_package_info(names[0], saltenv=saltenv) if not pkginfo: @@ -293,7 +302,9 @@ def list_available(*names, **kwargs): def version(*names, **kwargs): ''' - Returns a version if the package is installed, else returns an empty string + Returns a string representing the package version or an empty string if not + installed. If more than one package name is specified, a dict of + name/version pairs is returned. Args: name (str): One or more package names @@ -303,10 +314,12 @@ def version(*names, **kwargs): refresh (bool): Refresh package metadata. Default ``False``. Returns: + str: version string when a single packge is specified. dict: The package name(s) with the installed versions. - .. code-block:: cfg + .. code-block:: cfg + {['', '', ]} OR {'': ['', '', ]} CLI Example: @@ -315,19 +328,25 @@ def version(*names, **kwargs): salt '*' pkg.version salt '*' pkg.version - ''' - saltenv = kwargs.get('saltenv', 'base') - installed_pkgs = list_pkgs(refresh=kwargs.get('refresh', False)) - available_pkgs = get_repo_data(saltenv).get('repo') + ''' + # Standard is return empty string even if not a valid name + # TODO: Look at returning an error accross all platforms with + # CommandExecutionError(msg,info={'errors': errors }) + # available_pkgs = get_repo_data(saltenv).get('repo') + # for name in names: + # if name in available_pkgs: + # ret[name] = installed_pkgs.get(name, '') + # + saltenv = kwargs.get('saltenv', 'base') + installed_pkgs = list_pkgs(saltenv=saltenv, refresh=kwargs.get('refresh', False)) + + if len(names) == 1: + return installed_pkgs.get(names[0], '') ret = {} for name in names: - if name in available_pkgs: - ret[name] = installed_pkgs.get(name, '') - else: - ret[name] = 'not available' - + ret[name] = installed_pkgs.get(name, '') return ret @@ -424,7 +443,7 @@ def _get_reg_software(): '(value not set)', '', None] - #encoding = locale.getpreferredencoding() + reg_software = {} hive = 'HKLM' @@ -462,7 +481,7 @@ def _get_reg_software(): def _refresh_db_conditional(saltenv, **kwargs): ''' Internal use only in this module, has a different set of defaults and - returns True or False. And supports check the age of the existing + returns True or False. And supports checking the age of the existing generated metadata db, as well as ensure metadata db exists to begin with Args: @@ -476,8 +495,7 @@ def _refresh_db_conditional(saltenv, **kwargs): failhard (bool): If ``True``, an error will be raised if any repo SLS files failed to - process. If ``False``, no error will be raised, and a dictionary - containing the full results will be returned. + process. Returns: bool: True Fetched or Cache uptodate, False to indicate an issue @@ -615,6 +633,17 @@ def _get_repo_details(saltenv): if contextkey in __context__: (winrepo_source_dir, local_dest, winrepo_file) = __context__[contextkey] else: + ### Remove these warning lines in 2017.7 + ###if 'win_repo_source_dir' in __opts__: + ### salt.utils.warn_until( + ### 'Nitrogen', + ### 'The \'win_repo_source_dir\' config option is deprecated, ' + ### 'please use \'winrepo_source_dir\' instead.' + ### ) + ### winrepo_source_dir = __opts__['win_repo_source_dir'] + ###else: + ### winrepo_source_dir = __opts__['winrepo_source_dir'] + winrepo_source_dir = __opts__['winrepo_source_dir'] winrepo_source_dir = __opts__['winrepo_source_dir'] dirs = [__opts__['cachedir'], 'files', saltenv] url_parts = _urlparse(winrepo_source_dir) @@ -695,8 +724,8 @@ def genrepo(**kwargs): verbose (bool): Return verbose data structure which includes 'success_list', a list - of all sls files and the package names contained within. Default - 'False' + of all sls files and the package names contained within. + Default ``False``. failhard (bool): If ``True``, an error will be raised if any repo SLS files failed @@ -739,11 +768,13 @@ def genrepo(**kwargs): successful_verbose ) serial = salt.payload.Serial(__opts__) + # TODO: 2016.11 has PY2 mode as 'w+b' develop has 'w+' ? PY3 is 'wb+' + # also the reading of this is 'rb' in get_repo_data() mode = 'w+' if six.PY2 else 'wb+' + with salt.utils.fopen(repo_details.winrepo_file, mode) as repo_cache: repo_cache.write(serial.dumps(ret)) - # save reading it back again. ! this breaks due to utf8 issues - #__context__['winrepo.data'] = ret + # For some reason we can not save ret into __context__['winrepo.data'] as this breaks due to utf8 issues successful_count = len(successful_verbose) error_count = len(ret['errors']) if verbose: @@ -778,7 +809,7 @@ def genrepo(**kwargs): return results -def _repo_process_pkg_sls(file, short_path_name, ret, successful_verbose): +def _repo_process_pkg_sls(filename, short_path_name, ret, successful_verbose): renderers = salt.loader.render(__opts__, __salt__) def _failed_compile(msg): @@ -788,7 +819,7 @@ def _repo_process_pkg_sls(file, short_path_name, ret, successful_verbose): try: config = salt.template.compile_template( - file, + filename, renderers, __opts__['renderer'], __opts__.get('renderer_blacklist', ''), @@ -803,7 +834,6 @@ def _repo_process_pkg_sls(file, short_path_name, ret, successful_verbose): if config: revmap = {} errors = [] - pkgname_ok_list = [] for pkgname, versions in six.iteritems(config): if pkgname in ret['repo']: log.error( @@ -812,12 +842,12 @@ def _repo_process_pkg_sls(file, short_path_name, ret, successful_verbose): ) errors.append('package \'{0}\' already defined'.format(pkgname)) break - for version, repodata in six.iteritems(versions): + for version_str, repodata in six.iteritems(versions): # Ensure version is a string/unicode - if not isinstance(version, six.string_types): + if not isinstance(version_str, six.string_types): msg = ( 'package \'{0}\'{{0}}, version number {1} ' - 'is not a string'.format(pkgname, version) + 'is not a string'.format(pkgname, version_str) ) log.error( msg.format(' within \'{0}\''.format(short_path_name)) @@ -829,7 +859,7 @@ def _repo_process_pkg_sls(file, short_path_name, ret, successful_verbose): msg = ( 'package \'{0}\'{{0}}, repo data for ' 'version number {1} is not defined as a dictionary ' - .format(pkgname, version) + .format(pkgname, version_str) ) log.error( msg.format(' within \'{0}\''.format(short_path_name)) @@ -840,8 +870,6 @@ def _repo_process_pkg_sls(file, short_path_name, ret, successful_verbose): if errors: ret.setdefault('errors', {})[short_path_name] = errors else: - if pkgname not in pkgname_ok_list: - pkgname_ok_list.append(pkgname) ret.setdefault('repo', {}).update(config) ret.setdefault('name_map', {}).update(revmap) successful_verbose[short_path_name] = config.keys() @@ -916,7 +944,8 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): to install. (no spaces after the commas) refresh (bool): - Boolean value representing whether or not to refresh the winrepo db + Boolean value representing whether or not to refresh the winrepo db. + Default ``False``. pkgs (list): A list of packages to install from a software repository. All @@ -1051,7 +1080,6 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ''' ret = {} saltenv = kwargs.pop('saltenv', 'base') - refresh = salt.utils.is_true(refresh) # no need to call _refresh_db_conditional as list_pkgs will do it @@ -1072,7 +1100,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): for pkg in pkg_params: pkg_params[pkg] = {'version': pkg_params[pkg]} - if pkg_params is None or len(pkg_params) == 0: + if not pkg_params: log.error('No package definition found') return {} @@ -1114,11 +1142,12 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): version_num = str(version_num) if not version_num: + # following can be version number or latest version_num = _get_latest_pkg_version(pkginfo) # Check if the version is already installed if version_num in old.get(pkg_name, '').split(',') \ - or (old.get(pkg_name) == 'Not Found'): + or (old.get(pkg_name, '') == 'Not Found'): # Desired version number already installed ret[pkg_name] = {'current': version_num} continue @@ -1237,6 +1266,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): log.debug('Source hash matches package hash.') # Get install flags + install_flags = pkginfo[version_num].get('install_flags', '') if options and options.get('extra_install_flags'): install_flags = '{0} {1}'.format( @@ -1244,32 +1274,32 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): options.get('extra_install_flags', '') ) - #Compute msiexec string + # Compute msiexec string use_msiexec, msiexec = _get_msiexec(pkginfo[version_num].get('msiexec', False)) # Build cmd and arguments # cmd and arguments must be separated for use with the task scheduler + cmd_shell = os.getenv('ComSpec', '{0}\\system32\\cmd.exe'.format(os.getenv('WINDIR'))) if use_msiexec: - cmd = msiexec - arguments = ['/i', cached_pkg] + arguments = '"{0}" /I "{1}"'.format(msiexec, cached_pkg) if pkginfo[version_num].get('allusers', True): - arguments.append('ALLUSERS="1"') - arguments.extend(salt.utils.shlex_split(install_flags, posix=False)) + arguments = '{0} ALLUSERS=1'.format(arguments) else: - cmd = cached_pkg - arguments = salt.utils.shlex_split(install_flags, posix=False) + arguments = '"{0}"'.format(cached_pkg) + + if install_flags: + arguments = '{0} {1}'.format(arguments, install_flags) # Install the software # Check Use Scheduler Option if pkginfo[version_num].get('use_scheduler', False): - # Create Scheduled Task __salt__['task.create_task'](name='update-salt-software', user_name='System', force=True, action_type='Execute', - cmd=cmd, - arguments=' '.join(arguments), + cmd=cmd_shell, + arguments='/s /c "{0}"'.format(arguments), start_in=cache_path, trigger_type='Once', start_date='1975-01-01', @@ -1311,14 +1341,10 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): log.error('Scheduled Task failed to run') ret[pkg_name] = {'install status': 'failed'} else: - - # Combine cmd and arguments - cmd = [cmd] - cmd.extend(arguments) - # Launch the command - result = __salt__['cmd.run_all'](cmd, + result = __salt__['cmd.run_all']('"{0}" /s /c "{1}"'.format(cmd_shell, arguments), cache_path, + output_loglevel='trace', python_shell=False, redirect_stderr=True) if not result['retcode']: @@ -1397,14 +1423,17 @@ def remove(name=None, pkgs=None, version=None, **kwargs): .. versionadded:: 0.16.0 Args: - name (str): The name(s) of the package(s) to be uninstalled. Can be a - single package or a comma delimted list of packages, no spaces. + name (str): + The name(s) of the package(s) to be uninstalled. Can be a + single package or a comma delimited list of packages, no spaces. + version (str): The version of the package to be uninstalled. If this option is used to to uninstall multiple packages, then this version will be applied to all targeted packages. Recommended using only when uninstalling a single package. If this parameter is omitted, the latest version will be uninstalled. + pkgs (list): A list of packages to delete. Must be passed as a python list. The ``name`` parameter will be ignored if this option is passed. @@ -1494,7 +1523,6 @@ def remove(name=None, pkgs=None, version=None, **kwargs): removal_targets.append(version_num) for target in removal_targets: - # Get the uninstaller uninstaller = pkginfo[target].get('uninstaller', '') cache_dir = pkginfo[target].get('cache_dir', False) @@ -1519,6 +1547,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # If true, the entire directory will be cached instead of the # individual file. This is useful for installations that are not # single files + if cache_dir and uninstaller.startswith('salt:'): path, _ = os.path.split(uninstaller) __salt__['cp.cache_dir'](path, @@ -1541,6 +1570,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # Compare the hash of the cached installer to the source only if # the file is hosted on salt: + # TODO cp.cache_file does cache and hash checking? So why do it again? if uninstaller.startswith('salt:'): if __salt__['cp.hash_file'](uninstaller, saltenv) != \ __salt__['cp.hash_file'](cached_pkg): @@ -1558,15 +1588,12 @@ def remove(name=None, pkgs=None, version=None, **kwargs): else: # Run the uninstaller directly # (not hosted on salt:, https:, etc.) - cached_pkg = uninstaller + cached_pkg = os.path.expandvars(uninstaller) # Fix non-windows slashes cached_pkg = cached_pkg.replace('/', '\\') cache_path, _ = os.path.split(cached_pkg) - # Get parameters for cmd - expanded_cached_pkg = str(os.path.expandvars(cached_pkg)) - # Get uninstall flags uninstall_flags = pkginfo[target].get('uninstall_flags', '') @@ -1574,30 +1601,30 @@ def remove(name=None, pkgs=None, version=None, **kwargs): uninstall_flags = '{0} {1}'.format( uninstall_flags, kwargs.get('extra_uninstall_flags', '')) - #Compute msiexec string + # Compute msiexec string use_msiexec, msiexec = _get_msiexec(pkginfo[target].get('msiexec', False)) - - # Build cmd and arguments - # cmd and arguments must be separated for use with the task scheduler + cmd_shell = os.getenv('ComSpec', '{0}\\system32\\cmd.exe'.format(os.getenv('WINDIR'))) + # Build Scheduled Task Parameters if use_msiexec: - cmd = msiexec - arguments = ['/x'] - arguments.extend(salt.utils.shlex_split(uninstall_flags, posix=False)) + # Check if uninstaller is set to {guid}, if not we assume its a remote msi file. + # which has already been downloaded. + arguments = '"{0}" /X "{1}"'.format(msiexec, cached_pkg) else: - cmd = expanded_cached_pkg - arguments = salt.utils.shlex_split(uninstall_flags, posix=False) + arguments = '"{0}"'.format(cached_pkg) + + if uninstall_flags: + arguments = '{0} {1}'.format(arguments, uninstall_flags) # Uninstall the software # Check Use Scheduler Option if pkginfo[target].get('use_scheduler', False): - # Create Scheduled Task __salt__['task.create_task'](name='update-salt-software', user_name='System', force=True, action_type='Execute', - cmd=cmd, - arguments=' '.join(arguments), + cmd=cmd_shell, + arguments='/s /c "{0}"'.format(arguments), start_in=cache_path, trigger_type='Once', start_date='1975-01-01', @@ -1610,13 +1637,10 @@ def remove(name=None, pkgs=None, version=None, **kwargs): log.error('Scheduled Task failed to run') ret[pkgname] = {'uninstall status': 'failed'} else: - # Build the install command - cmd = [cmd] - cmd.extend(arguments) - # Launch the command result = __salt__['cmd.run_all']( - cmd, + '"{0}" /s /c "{1}"'.format(cmd_shell, arguments), + output_loglevel='trace', python_shell=False, redirect_stderr=True) if not result['retcode']: @@ -1662,11 +1686,13 @@ def purge(name=None, pkgs=None, version=None, **kwargs): name (str): The name of the package to be deleted. - version (str): The version of the package to be deleted. If this option - is used in combination with the ``pkgs`` option below, then this + version (str): + The version of the package to be deleted. If this option is + used in combination with the ``pkgs`` option below, then this version will be applied to all targeted packages. - pkgs (list): A list of packages to delete. Must be passed as a python + pkgs (list): + A list of packages to delete. Must be passed as a python list. The ``name`` parameter will be ignored if this option is passed. @@ -1800,4 +1826,20 @@ def compare_versions(ver1='', oper='==', ver2=''): salt '*' pkg.compare_versions 1.2 >= 1.3 ''' - return salt.utils.compare_versions(ver1, oper, ver2) + if not ver1: + raise SaltInvocationError('compare_version, ver1 is blank') + if not ver2: + raise SaltInvocationError('compare_version, ver2 is blank') + + # Support version being the special meaning of 'latest' + if ver1 == 'latest': + ver1 = str(sys.maxsize) + if ver2 == 'latest': + ver2 = str(sys.maxsize) + # Support version being the special meaning of 'Not Found' + if ver1 == 'Not Found': + ver1 = '0.0.0.0.0' + if ver2 == 'Not Found': + ver2 = '0.0.0.0.0' + + return salt.utils.compare_versions(ver1, oper, ver2, ignore_epoch=True) From b46f818a57cfca14e4857576b8abe2b1cc4b98c6 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Thu, 7 Dec 2017 01:48:01 +1100 Subject: [PATCH 22/54] Raise a PR to fix 2016 issues commited here, fixed issues with merge. --- salt/modules/win_pkg.py | 48 ++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 90223e334d..15339ad97b 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -195,7 +195,6 @@ def upgrade_available(name, **kwargs): # Refresh before looking for the latest version available, # same default as latest_version refresh = salt.utils.is_true(kwargs.get('refresh', True)) - # if latest_version returns blank, the latest version is already installed or # their is no package definition. This is a salt standard which could be improved. return latest_version(name, saltenv=saltenv, refresh=refresh) != '' @@ -315,9 +314,10 @@ def version(*names, **kwargs): refresh (bool): Refresh package metadata. Default ``False``. Returns: - str: version string when a single package is specified. + str: version string when a single packge is specified. dict: The package name(s) with the installed versions. + .. code-block:: cfg {['', '', ]} OR {'': ['', '', ]} @@ -331,13 +331,13 @@ def version(*names, **kwargs): ''' # Standard is return empty string even if not a valid name - # TODO: Look at returning an error across all platforms with + # TODO: Look at returning an error accross all platforms with # CommandExecutionError(msg,info={'errors': errors }) # available_pkgs = get_repo_data(saltenv).get('repo') # for name in names: # if name in available_pkgs: # ret[name] = installed_pkgs.get(name, '') - + # saltenv = kwargs.get('saltenv', 'base') installed_pkgs = list_pkgs(saltenv=saltenv, refresh=kwargs.get('refresh', False)) @@ -355,6 +355,7 @@ def list_pkgs(versions_as_list=False, **kwargs): List the packages currently installed Args: + version_as_list (bool): Returns the versions as a list Kwargs: saltenv (str): The salt environment to use. Default ``base``. @@ -632,17 +633,6 @@ def _get_repo_details(saltenv): if contextkey in __context__: (winrepo_source_dir, local_dest, winrepo_file) = __context__[contextkey] else: - ### Remove these warning lines in 2017.7 - ###if 'win_repo_source_dir' in __opts__: - ### salt.utils.warn_until( - ### 'Nitrogen', - ### 'The \'win_repo_source_dir\' config option is deprecated, ' - ### 'please use \'winrepo_source_dir\' instead.' - ### ) - ### winrepo_source_dir = __opts__['win_repo_source_dir'] - ###else: - ### winrepo_source_dir = __opts__['winrepo_source_dir'] - winrepo_source_dir = __opts__['winrepo_source_dir'] winrepo_source_dir = __opts__['winrepo_source_dir'] dirs = [__opts__['cachedir'], 'files', saltenv] url_parts = _urlparse(winrepo_source_dir) @@ -1341,12 +1331,11 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ret[pkg_name] = {'install status': 'failed'} else: # Launch the command - result = __salt__['cmd.run_all']( - '"{0}" /s /c "{1}"'.format(cmd_shell, arguments), - cache_path, - output_loglevel='trace', - python_shell=False, - redirect_stderr=True) + result = __salt__['cmd.run_all']('"{0}" /s /c "{1}"'.format(cmd_shell, arguments), + cache_path, + output_loglevel='trace', + python_shell=False, + redirect_stderr=True) if not result['retcode']: ret[pkg_name] = {'install status': 'success'} changed.append(pkg_name) @@ -1594,10 +1583,6 @@ def remove(name=None, pkgs=None, version=None, **kwargs): cached_pkg = cached_pkg.replace('/', '\\') cache_path, _ = os.path.split(cached_pkg) - # Get parameters for cmd - expanded_cached_pkg = str(os.path.expandvars(cached_pkg)) - expanded_cache_path = str(os.path.expandvars(cache_path)) - # Get uninstall flags uninstall_flags = pkginfo[target].get('uninstall_flags', '') @@ -1608,13 +1593,13 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # Compute msiexec string use_msiexec, msiexec = _get_msiexec(pkginfo[target].get('msiexec', False)) cmd_shell = os.getenv('ComSpec', '{0}\\system32\\cmd.exe'.format(os.getenv('WINDIR'))) - - # Build cmd and arguments - # cmd and arguments must be separated for use with the task scheduler + # Build Scheduled Task Parameters if use_msiexec: - arguments = '"{0}" /X "{1}"'.format(msiexec, uninstaller if uninstaller else expanded_cached_pkg) + # Check if uninstaller is set to {guid}, if not we assume its a remote msi file. + # which has already been downloaded. + arguments = '"{0}" /X "{1}"'.format(msiexec, cached_pkg) else: - arguments = '"{0}"'.format(expanded_cached_pkg) + arguments = '"{0}"'.format(cached_pkg) if uninstall_flags: arguments = '{0} {1}'.format(arguments, uninstall_flags) @@ -1629,7 +1614,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): action_type='Execute', cmd=cmd_shell, arguments='/s /c "{0}"'.format(arguments), - start_in=expanded_cache_path, + start_in=cache_path, trigger_type='Once', start_date='1975-01-01', start_time='01:00', @@ -1644,7 +1629,6 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # Launch the command result = __salt__['cmd.run_all']( '"{0}" /s /c "{1}"'.format(cmd_shell, arguments), - expanded_cache_path, output_loglevel='trace', python_shell=False, redirect_stderr=True) From b3d86fd0991b1f218d8d8b83ef6059e326694326 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 6 Dec 2017 22:58:29 +0000 Subject: [PATCH 23/54] Take retcode from the ret fun return, and send it to the outputter as the _retcode kwarg --- salt/cli/caller.py | 12 ++++++++---- salt/output/nested.py | 12 +++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index f681e0cd0e..6ece877c1e 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -145,8 +145,10 @@ class BaseCaller(object): print_ret = ret.get('return', {}) salt.output.display_output( {'local': print_ret}, - out, - self.opts) + out=out, + opts=self.opts, + _retcode=ret.get('retcode', 0)) + # _retcode will be available in the kwargs of the outputter function if self.opts.get('retcode_passthrough', False): sys.exit(ret['retcode']) except SaltInvocationError as err: @@ -372,8 +374,10 @@ class RAETCaller(BaseCaller): self.process.terminate() salt.output.display_output( {'local': print_ret}, - ret.get('out', 'nested'), - self.opts) + out=ret.get('out', 'nested'), + opts=self.opts, + _retcode=ret.get('retcode', 0)) + # _retcode will be available in the kwargs of the outputter function if self.opts.get('retcode_passthrough', False): sys.exit(ret['retcode']) diff --git a/salt/output/nested.py b/salt/output/nested.py index fc6c0230ad..436e5ae1c1 100644 --- a/salt/output/nested.py +++ b/salt/output/nested.py @@ -39,7 +39,7 @@ class NestDisplay(object): ''' Manage the nested display contents ''' - def __init__(self): + def __init__(self, retcode=0): self.__dict__.update( salt.utils.color.get_colors( __opts__.get('color'), @@ -47,6 +47,7 @@ class NestDisplay(object): ) ) self.strip_colors = __opts__.get('strip_colors', True) + self.retcode = retcode def ustring(self, indent, @@ -107,7 +108,7 @@ class NestDisplay(object): first_line = False elif isinstance(ret, (list, tuple)): color = self.GREEN - if 'retcode' in __context__ and __context__['retcode']: + if self.retcode != 0: color = self.RED for ind in ret: if isinstance(ind, (list, tuple, dict)): @@ -125,7 +126,7 @@ class NestDisplay(object): elif isinstance(ret, dict): if indent: color = self.CYAN - if 'retcode' in __context__ and __context__['retcode']: + if self.retcode != 0: color = self.RED out.append( self.ustring( @@ -141,7 +142,7 @@ class NestDisplay(object): else: keys = sorted(ret) color = self.CYAN - if 'retcode' in __context__ and __context__['retcode']: + if self.retcode != 0: color = self.RED for key in keys: val = ret[key] @@ -163,7 +164,8 @@ def output(ret, **kwargs): Display ret data ''' # Prefer kwargs before opts + retcode = kwargs.get('_retcode', 0) base_indent = kwargs.get('nested_indent', 0) \ or __opts__.get('nested_indent', 0) - nest = NestDisplay() + nest = NestDisplay(retcode=retcode) return '\n'.join(nest.display(ret, base_indent, '', [])) From f6112fa89e860e5ac14fe82b2511ee3227b02484 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 7 Dec 2017 11:33:19 +0000 Subject: [PATCH 24/54] Add testcases for the nested outputter --- tests/unit/output/test_nested_out.py | 134 +++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 tests/unit/output/test_nested_out.py diff --git a/tests/unit/output/test_nested_out.py b/tests/unit/output/test_nested_out.py new file mode 100644 index 0000000000..6a7a43a77f --- /dev/null +++ b/tests/unit/output/test_nested_out.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +''' +Unit tests for the Nested outputter +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase + +# Import Salt Libs +import salt.output.nested as nested + + +class NestedOutputterTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.output.nested + ''' + def setup_loader_modules(self): + return { + nested: { + '__opts__': { + 'extension_modules': '', + 'color': True + } + } + } + + def setUp(self): + # The example from the documentation for the test.arg execution function + # Same function from the highstate outputter + self.data = { + 'local': { + 'args': (1, 'two', 3.1), + 'kwargs': { + u'__pub_pid': 25938, + 'wow': { + 'a': 1, + 'b': 'hello' + }, + u'__pub_fun': 'test.arg', + u'__pub_jid': '20171207105927331329', + u'__pub_tgt': 'salt-call', + 'txt': 'hello' + } + } + } + self.addCleanup(delattr, self, 'data') + + def test_output_with_colors(self): + # Should look exacly like that, with the default color scheme: + # + # local: + # ---------- + # args: + # - 1 + # - two + # - 3.1 + # kwargs: + # ---------- + # __pub_fun: + # test.arg + # __pub_jid: + # 20171207105927331329 + # __pub_pid: + # 25938 + # __pub_tgt: + # salt-call + # txt: + # hello + # wow: + # ---------- + # a: + # 1 + # b: + # hello + expected_output_str = ( + '\x1b[0;36mlocal\x1b[0;0m:\n \x1b[0;36m----------\x1b[0;0m\n \x1b[0;36margs\x1b[0;0m:\n' + ' \x1b[0;1;33m- 1\x1b[0;0m\n \x1b[0;32m- two\x1b[0;0m\n \x1b[0;1;33m- 3.1\x1b[0;0m\n' + ' \x1b[0;36mkwargs\x1b[0;0m:\n \x1b[0;36m----------\x1b[0;0m\n' + ' \x1b[0;36m__pub_fun\x1b[0;0m:\n \x1b[0;32mtest.arg\x1b[0;0m\n' + ' \x1b[0;36m__pub_jid\x1b[0;0m:\n \x1b[0;32m20171207105927331329\x1b[0;0m\n' + ' \x1b[0;36m__pub_pid\x1b[0;0m:\n \x1b[0;1;33m25938\x1b[0;0m\n' + ' \x1b[0;36m__pub_tgt\x1b[0;0m:\n \x1b[0;32msalt-call\x1b[0;0m\n' + ' \x1b[0;36mtxt\x1b[0;0m:\n \x1b[0;32mhello\x1b[0;0m\n \x1b[0;36mwow\x1b[0;0m:\n' + ' \x1b[0;36m----------\x1b[0;0m\n \x1b[0;36ma\x1b[0;0m:\n' + ' \x1b[0;1;33m1\x1b[0;0m\n \x1b[0;36mb\x1b[0;0m:\n' + ' \x1b[0;32mhello\x1b[0;0m' + ) + ret = nested.output(self.data) + self.assertEqual(ret, expected_output_str) + + def test_output_with_retcode(self): + # Non-zero retcode should change the colors + # Same output format as above, just different colors + expected_output_str = ( + '\x1b[0;31mlocal\x1b[0;0m:\n \x1b[0;31m----------\x1b[0;0m\n \x1b[0;31margs\x1b[0;0m:\n' + ' \x1b[0;1;33m- 1\x1b[0;0m\n \x1b[0;32m- two\x1b[0;0m\n \x1b[0;1;33m- 3.1\x1b[0;0m\n' + ' \x1b[0;31mkwargs\x1b[0;0m:\n \x1b[0;31m----------\x1b[0;0m\n' + ' \x1b[0;31m__pub_fun\x1b[0;0m:\n \x1b[0;32mtest.arg\x1b[0;0m\n' + ' \x1b[0;31m__pub_jid\x1b[0;0m:\n \x1b[0;32m20171207105927331329\x1b[0;0m\n' + ' \x1b[0;31m__pub_pid\x1b[0;0m:\n \x1b[0;1;33m25938\x1b[0;0m\n' + ' \x1b[0;31m__pub_tgt\x1b[0;0m:\n \x1b[0;32msalt-call\x1b[0;0m\n' + ' \x1b[0;31mtxt\x1b[0;0m:\n \x1b[0;32mhello\x1b[0;0m\n \x1b[0;31mwow\x1b[0;0m:\n' + ' \x1b[0;31m----------\x1b[0;0m\n \x1b[0;31ma\x1b[0;0m:\n' + ' \x1b[0;1;33m1\x1b[0;0m\n \x1b[0;31mb\x1b[0;0m:\n' + ' \x1b[0;32mhello\x1b[0;0m' + ) + # You can notice that in test_output_with_colors the color code is \x1b[0;36m, i.e., GREEN, + # while here the color code is \x1b[0;31m, i.e., RED (failure) + ret = nested.output(self.data, _retcode=1) + self.assertEqual(ret, expected_output_str) + + def test_output_with_indent(self): + # Everything must be indented by exactly two spaces + # (using nested_indent=2 sent to nested.output as kwarg) + expected_output_str = ( + ' \x1b[0;36m----------\x1b[0;0m\n \x1b[0;36mlocal\x1b[0;0m:\n \x1b[0;36m----------\x1b[0;0m\n' + ' \x1b[0;36margs\x1b[0;0m:\n \x1b[0;1;33m- 1\x1b[0;0m\n \x1b[0;32m- two\x1b[0;0m\n' + ' \x1b[0;1;33m- 3.1\x1b[0;0m\n \x1b[0;36mkwargs\x1b[0;0m:\n' + ' \x1b[0;36m----------\x1b[0;0m\n \x1b[0;36m__pub_fun\x1b[0;0m:\n' + ' \x1b[0;32mtest.arg\x1b[0;0m\n \x1b[0;36m__pub_jid\x1b[0;0m:\n' + ' \x1b[0;32m20171207105927331329\x1b[0;0m\n \x1b[0;36m__pub_pid\x1b[0;0m:\n' + ' \x1b[0;1;33m25938\x1b[0;0m\n \x1b[0;36m__pub_tgt\x1b[0;0m:\n' + ' \x1b[0;32msalt-call\x1b[0;0m\n \x1b[0;36mtxt\x1b[0;0m:\n' + ' \x1b[0;32mhello\x1b[0;0m\n \x1b[0;36mwow\x1b[0;0m:\n' + ' \x1b[0;36m----------\x1b[0;0m\n \x1b[0;36ma\x1b[0;0m:\n' + ' \x1b[0;1;33m1\x1b[0;0m\n \x1b[0;36mb\x1b[0;0m:\n' + ' \x1b[0;32mhello\x1b[0;0m' + ) + ret = nested.output(self.data, nested_indent=2) + self.assertEqual(ret, expected_output_str) From 0cf0168493699aaaf1bd273453ab4baed0944ccf Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Fri, 8 Dec 2017 11:07:33 +0000 Subject: [PATCH 25/54] Consider retcode when running from the salt cmd too --- salt/cli/salt.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/cli/salt.py b/salt/cli/salt.py index e4c632c65c..34824e0c74 100644 --- a/salt/cli/salt.py +++ b/salt/cli/salt.py @@ -181,7 +181,7 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): for full_ret in self.local_client.cmd_cli(**kwargs): ret_, out, retcode = self._format_ret(full_ret) ret.update(ret_) - self._output_ret(ret, out) + self._output_ret(ret, out, retcode=retcode) else: if self.options.verbose: kwargs['verbose'] = True @@ -190,7 +190,7 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): try: ret_, out, retcode = self._format_ret(full_ret) retcodes.append(retcode) - self._output_ret(ret_, out) + self._output_ret(ret_, out, retcode=retcode) ret.update(full_ret) except KeyError: errors.append(full_ret) @@ -212,7 +212,7 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): except (SaltInvocationError, EauthAuthenticationError, SaltClientError) as exc: ret = str(exc) - self._output_ret(ret, '') + self._output_ret(ret, '', retcode=1) def _preview_target(self): ''' @@ -352,7 +352,7 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): 'Requested job was still run but output cannot be displayed.\n') salt.output.update_progress(self.config, progress, self.progress_bar, out) - def _output_ret(self, ret, out): + def _output_ret(self, ret, out, retcode=0): ''' Print the output from a single return to the terminal ''' @@ -362,7 +362,10 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): self._print_docs(ret) else: # Determine the proper output method and run it - salt.output.display_output(ret, out, self.config) + salt.output.display_output(ret, + out=out, + opts=self.config, + _retcode=retcode) if not ret: sys.stderr.write('ERROR: No return received\n') sys.exit(2) From f8a95cbe26d5398eb5c96dc6a42c094f800501cd Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sat, 9 Dec 2017 10:15:58 -0600 Subject: [PATCH 26/54] rest_cherrypy: revert addition of eauth user and groups to low chunks These items are not referenced anywhere in the codebase (neither in salt proper nor in any of the associated projects, including the enterprise UI). --- salt/netapi/rest_cherrypy/app.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 1993818898..5aeeb06d18 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -1175,10 +1175,6 @@ class LowDataAdapter(object): for chunk in lowstate: if token: chunk['token'] = token - if cherrypy.session.get('user'): - chunk['__current_eauth_user'] = cherrypy.session.get('user') - if cherrypy.session.get('groups'): - chunk['__current_eauth_groups'] = cherrypy.session.get('groups') if client: chunk['client'] = client @@ -1878,9 +1874,6 @@ class Login(LowDataAdapter): cherrypy.response.headers['X-Auth-Token'] = cherrypy.session.id cherrypy.session['token'] = token['token'] cherrypy.session['timeout'] = (token['expire'] - token['start']) / 60 - cherrypy.session['user'] = token['name'] - if 'groups' in token: - cherrypy.session['groups'] = token['groups'] # Grab eauth config for the current backend for the current user try: From a4f0b41ba285f17646fcdf83f45cfcbb0d585ba1 Mon Sep 17 00:00:00 2001 From: Damon Atkins Date: Tue, 12 Dec 2017 01:16:49 +1100 Subject: [PATCH 27/54] Should be a smaller change set since recent update from 2016.11 --- salt/modules/win_pkg.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index c693667a0f..e6bc024d96 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -195,6 +195,7 @@ def upgrade_available(name, **kwargs): # Refresh before looking for the latest version available, # same default as latest_version refresh = salt.utils.is_true(kwargs.get('refresh', True)) + # if latest_version returns blank, the latest version is already installed or # their is no package definition. This is a salt standard which could be improved. return latest_version(name, saltenv=saltenv, refresh=refresh) != '' @@ -314,7 +315,7 @@ def version(*names, **kwargs): refresh (bool): Refresh package metadata. Default ``False``. Returns: - str: version string when a single packge is specified. + str: version string when a single package is specified. dict: The package name(s) with the installed versions. .. code-block:: cfg @@ -330,13 +331,13 @@ def version(*names, **kwargs): ''' # Standard is return empty string even if not a valid name - # TODO: Look at returning an error accross all platforms with + # TODO: Look at returning an error across all platforms with # CommandExecutionError(msg,info={'errors': errors }) # available_pkgs = get_repo_data(saltenv).get('repo') # for name in names: # if name in available_pkgs: # ret[name] = installed_pkgs.get(name, '') - # + saltenv = kwargs.get('saltenv', 'base') installed_pkgs = list_pkgs(saltenv=saltenv, refresh=kwargs.get('refresh', False)) @@ -1582,6 +1583,8 @@ def remove(name=None, pkgs=None, version=None, **kwargs): cached_pkg = cached_pkg.replace('/', '\\') cache_path, _ = os.path.split(cached_pkg) + # os.path.expandvars is not required as we run everything through cmd.exe /s /c + # Get uninstall flags uninstall_flags = pkginfo[target].get('uninstall_flags', '') @@ -1592,7 +1595,9 @@ def remove(name=None, pkgs=None, version=None, **kwargs): # Compute msiexec string use_msiexec, msiexec = _get_msiexec(pkginfo[target].get('msiexec', False)) cmd_shell = os.getenv('ComSpec', '{0}\\system32\\cmd.exe'.format(os.getenv('WINDIR'))) - # Build Scheduled Task Parameters + + # Build cmd and arguments + # cmd and arguments must be separated for use with the task scheduler if use_msiexec: # Check if uninstaller is set to {guid}, if not we assume its a remote msi file. # which has already been downloaded. From bfd0972d2557422589654450d36dfa1104522069 Mon Sep 17 00:00:00 2001 From: Kirill Goncharov Date: Sun, 10 Dec 2017 12:58:30 +0300 Subject: [PATCH 28/54] Fix TypeError during rbenv ruby installation when rbenv is not found --- salt/modules/rbenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/rbenv.py b/salt/modules/rbenv.py index cdae6e9d9a..98ce90e3b1 100644 --- a/salt/modules/rbenv.py +++ b/salt/modules/rbenv.py @@ -245,7 +245,7 @@ def install_ruby(ruby, runas=None): ret = {} ret = _rbenv_exec(['install', ruby], env=env, runas=runas, ret=ret) - if ret['retcode'] == 0: + if ret is not False and ret['retcode'] == 0: rehash(runas=runas) return ret['stderr'] else: From 3475d3fa01506b8a3b09bf31d2def4e6bc9b25c7 Mon Sep 17 00:00:00 2001 From: Piotr Kruk Date: Mon, 11 Dec 2017 21:37:00 +0100 Subject: [PATCH 29/54] add missing parenthis to keep integration with python3 --- salt/modules/virtualenv_mod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/virtualenv_mod.py b/salt/modules/virtualenv_mod.py index 6c4a3ec761..78e9ee7529 100644 --- a/salt/modules/virtualenv_mod.py +++ b/salt/modules/virtualenv_mod.py @@ -316,7 +316,7 @@ def get_site_packages(venv): ret = __salt__['cmd.exec_code_all']( bin_path, 'from distutils import sysconfig; ' - 'print sysconfig.get_python_lib()' + 'print(sysconfig.get_python_lib())' ) if ret['retcode'] != 0: From adf38cacfbcfa07f5ac690be687d399862beac29 Mon Sep 17 00:00:00 2001 From: Piotr Kruk Date: Mon, 11 Dec 2017 21:37:00 +0100 Subject: [PATCH 30/54] add missing parenthis to keep integration with python3 --- salt/modules/virtualenv_mod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/virtualenv_mod.py b/salt/modules/virtualenv_mod.py index 6c4a3ec761..78e9ee7529 100644 --- a/salt/modules/virtualenv_mod.py +++ b/salt/modules/virtualenv_mod.py @@ -316,7 +316,7 @@ def get_site_packages(venv): ret = __salt__['cmd.exec_code_all']( bin_path, 'from distutils import sysconfig; ' - 'print sysconfig.get_python_lib()' + 'print(sysconfig.get_python_lib())' ) if ret['retcode'] != 0: From c05885a88a9f849162b19d7948c3b67323a43495 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Mon, 11 Dec 2017 16:48:15 -0500 Subject: [PATCH 31/54] INFRA-6108 - boto_elb.present() errors if test=True and ELB doesn't exist --- salt/states/boto_elb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/states/boto_elb.py b/salt/states/boto_elb.py index b93c9dd6eb..b7b16fc0ca 100644 --- a/salt/states/boto_elb.py +++ b/salt/states/boto_elb.py @@ -396,6 +396,9 @@ def present(name, listeners, availability_zones=None, subnets=None, ret['result'] = ret['result'] if _ret['result'] else _ret['result'] if ret['result'] is False: return ret + exists = __salt__['boto_elb.exists'](name, region, key, keyid, profile) + if not exists and __opts__['test']: + return ret if attributes: _ret = _attributes_present(name, attributes, region, key, keyid, profile) From d1317c44e2c8daefb7bc117f2ddc6e1b54e86679 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 12 Dec 2017 11:32:34 -0500 Subject: [PATCH 32/54] Update README with SaltConf18 info Previously we had information for SaltConf16, which is out of date. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index d3917d6abe..d2bd8f78bb 100644 --- a/README.rst +++ b/README.rst @@ -67,8 +67,8 @@ Engage SaltStack `SaltConf`_, **User Groups and Meetups** - SaltStack has a vibrant and `global community`_ of customers, users, developers and enthusiasts. Connect with other -Salted folks in your area of the world, or join `SaltConf16`_, the SaltStack -annual user conference, April 19-21 in Salt Lake City. Please let us know if +Salted folks in your area of the world, or join `SaltConf18`_, the SaltStack +annual user conference, September 10-14 in Salt Lake City. Please let us know if you would like to start a user group or if we should add your existing SaltStack user group to this list by emailing: info@saltstack.com @@ -91,7 +91,7 @@ services`_ offerings. .. _SaltConf: http://www.youtube.com/user/saltstack .. _global community: http://www.meetup.com/pro/saltstack/ -.. _SaltConf16: http://saltconf.com/ +.. _SaltConf18: http://saltconf.com/ .. _SaltStack education offerings: http://saltstack.com/training/ .. _SaltStack Certified Engineer (SSCE): http://saltstack.com/certification/ .. _SaltStack professional services: http://saltstack.com/services/ From d23ac4eabc0f21c425485e1d7a1be46a87f08757 Mon Sep 17 00:00:00 2001 From: mvivaldi Date: Tue, 12 Dec 2017 17:35:46 +0100 Subject: [PATCH 33/54] Fix for the jinja documentation --- doc/topics/jinja/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index 1c0988fa78..a833507269 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -1542,13 +1542,13 @@ Example: .. code-block:: jinja -regex_escape = {{ 'https://example.com?foo=bar%20baz' | regex_escape }} + regex_escape = {{ 'https://example.com?foo=bar%20baz' | regex_escape }} will be rendered as: .. code-block:: text -regex_escape = https\:\/\/example\.com\?foo\=bar\%20baz + regex_escape = https\:\/\/example\.com\?foo\=bar\%20baz Set Theory Filters ------------------ @@ -1566,13 +1566,13 @@ Example: .. code-block:: jinja -unique = {{ ['foo', 'foo', 'bar'] | unique }} + unique = {{ ['foo', 'foo', 'bar'] | unique }} will be rendered as: .. code-block:: text -unique = ['foo', 'bar'] + unique = ['foo', 'bar'] Jinja in Files ============== From 0ff52a93dd923069169928220ca977b520f95692 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 12 Dec 2017 09:54:06 -0700 Subject: [PATCH 34/54] use last entry in acl This will allow for `effective` permissions to be looked at when the package manager updates the system. ``` $ getfacl httpd/ user::rwx group::--- group:webmaster:r-x #effective:--- mask::--- other::--- ``` Fixes #44932 --- salt/modules/linux_acl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/linux_acl.py b/salt/modules/linux_acl.py index 5969b24ea9..9cb7c0d4d7 100644 --- a/salt/modules/linux_acl.py +++ b/salt/modules/linux_acl.py @@ -144,17 +144,17 @@ def _parse_acl(acl, user, group): # Set the permissions fields octal = 0 vals['permissions'] = {} - if 'r' in comps[2]: + if 'r' in comps[-1]: octal += 4 vals['permissions']['read'] = True else: vals['permissions']['read'] = False - if 'w' in comps[2]: + if 'w' in comps[-1]: octal += 2 vals['permissions']['write'] = True else: vals['permissions']['write'] = False - if 'x' in comps[2]: + if 'x' in comps[-1]: octal += 1 vals['permissions']['execute'] = True else: From 66bb7557515c0d2e6241f74644853a97773ddaef Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 12 Dec 2017 09:58:08 -0700 Subject: [PATCH 35/54] add test for effective acls --- tests/unit/modules/test_linux_acl.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/modules/test_linux_acl.py b/tests/unit/modules/test_linux_acl.py index 9fc07569be..f1ec38f56d 100644 --- a/tests/unit/modules/test_linux_acl.py +++ b/tests/unit/modules/test_linux_acl.py @@ -63,6 +63,22 @@ class LinuxAclTestCase(TestCase, LoaderModuleMockMixin): linux_acl.getfacl(*self.files, recursive=True) self.cmdrun.assert_called_once_with('getfacl --absolute-names -R ' + ' '.join(self.quoted_files), python_shell=False) + def test_getfacl__effective_acls(self): + line = 'group:webmaster:r-x #effective:---' + user = 'root' + group = 'root' + expected = { + 'type': 'acl', + 'group': 'webmaster', + 'permissions': { + 'read': False, + 'write': False, + 'execute': False + }, + 'octal': 0, + } + self.assertEqual(linux_acl._parse_acl(line, user, group), expected) + def test_wipefacls_wo_args(self): self.assertRaises(CommandExecutionError, linux_acl.wipefacls) From d66f3a98d713af88a06e7d0637aafa322cb56099 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 12 Dec 2017 17:09:03 -0600 Subject: [PATCH 36/54] Avoid traceback when bogus value in pidfile --- salt/utils/process.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/utils/process.py b/salt/utils/process.py index 027af65edc..8bace77e63 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -142,12 +142,12 @@ def get_pidfile(pidfile): ''' Return the pid from a pidfile as an integer ''' - with salt.utils.fopen(pidfile) as pdf: - pid = pdf.read() - if pid: + try: + with salt.utils.fopen(pidfile) as pdf: + pid = pdf.read().strip() return int(pid) - else: - return + except (OSError, IOError, TypeError, ValueError): + return None def clean_proc(proc, wait_for_kill=10): From 2ac70cfab5eae46f2abe38981cf28fc22dfc0310 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 13 Dec 2017 02:30:19 -0600 Subject: [PATCH 37/54] Fix a race condition in manage runner The runner was not connecting the client event listener to the event bus. This meant that all events between the `client.run_job()` and when `client.get_cli_event_returns()` began polling for events would be lost. Before `get_cli_event_returns` (via LocalClient's `get_iter_returns`) gets around to polling for events, it first checks to see if the job cache has a record of the jid it's querying. When using a custom returner for the job_cache, one which has even a little bit of latency, return events from minions may sneak past before the jid lookup is complete, meaning that `get_iter_returns` will not return them and the manage runner will assume the minion did not respond. Connecting to the event bus before we run the test.ping ensures that we do not miss any of these events. --- salt/runners/manage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index fbf8ea3f23..ca72b09182 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -37,6 +37,7 @@ log = logging.getLogger(__name__) def _ping(tgt, tgt_type, timeout, gather_job_timeout): client = salt.client.get_local_client(__opts__['conf_file']) + client.event.connect_pub(timeout=timeout) pub_data = client.run_job(tgt, 'test.ping', (), tgt_type, '', timeout, '') if not pub_data: From ef749abfc69d8f5354bf38ee00e32aa15898c127 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 13 Dec 2017 03:03:48 -0600 Subject: [PATCH 38/54] No need to manually do connect_pub, use listen=True in run_job --- salt/runners/manage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index ca72b09182..723052a09f 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -37,8 +37,7 @@ log = logging.getLogger(__name__) def _ping(tgt, tgt_type, timeout, gather_job_timeout): client = salt.client.get_local_client(__opts__['conf_file']) - client.event.connect_pub(timeout=timeout) - pub_data = client.run_job(tgt, 'test.ping', (), tgt_type, '', timeout, '') + pub_data = client.run_job(tgt, 'test.ping', (), tgt_type, '', timeout, '', listen=True) if not pub_data: return pub_data From 40665d7b08d597dea319a7cca84c38111e4045fb Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 13 Dec 2017 14:12:07 -0700 Subject: [PATCH 39/54] Skip test_log_created on Windows --- tests/unit/utils/test_parsers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/utils/test_parsers.py b/tests/unit/utils/test_parsers.py index 9f105e690f..9598aae633 100644 --- a/tests/unit/utils/test_parsers.py +++ b/tests/unit/utils/test_parsers.py @@ -489,6 +489,7 @@ class LogSettingsParserTests(TestCase): # Check log file logger self.assertEqual(self.log_setup.log_level_logfile, log_level_logfile) + @skipIf(salt.utils.is_windows(), 'Windows uses a logging listener') def test_log_created(self): ''' Tests that log file is created From c382558bced46fa232431ff6b71cdb8ac7342223 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Wed, 13 Dec 2017 17:06:02 -0500 Subject: [PATCH 40/54] 44303 - resolves #44303 --- salt/modules/boto3_route53.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/boto3_route53.py b/salt/modules/boto3_route53.py index 749655af76..4828ba181f 100644 --- a/salt/modules/boto3_route53.py +++ b/salt/modules/boto3_route53.py @@ -398,7 +398,7 @@ def create_hosted_zone(Name, VPCId=None, VPCName=None, VPCRegion=None, CallerRef if tries and e.response.get('Error', {}).get('Code') == 'Throttling': log.debug('Throttled by AWS API.') time.sleep(3) - retries -= 1 + tries -= 1 continue log.error('Failed to create hosted zone {0}: {1}'.format(Name, str(e))) return [] @@ -448,7 +448,7 @@ def update_hosted_zone_comment(Id=None, Name=None, Comment=None, PrivateZone=Non if tries and e.response.get('Error', {}).get('Code') == 'Throttling': log.debug('Throttled by AWS API.') time.sleep(3) - retries -= 1 + tries -= 1 continue log.error('Failed to update comment on hosted zone {0}: {1}'.format( Name or Id, str(e))) @@ -547,7 +547,7 @@ def associate_vpc_with_hosted_zone(HostedZoneId=None, Name=None, VPCId=None, if tries and e.response.get('Error', {}).get('Code') == 'Throttling': log.debug('Throttled by AWS API.') time.sleep(3) - retries -= 1 + tries -= 1 continue log.error('Failed to associate VPC {0} with hosted zone {1}: {2}'.format( VPCName or VPCId, Name or HostedZoneId, str(e))) @@ -639,7 +639,7 @@ def diassociate_vpc_from_hosted_zone(HostedZoneId=None, Name=None, VPCId=None, if tries and e.response.get('Error', {}).get('Code') == 'Throttling': log.debug('Throttled by AWS API.') time.sleep(3) - retries -= 1 + tries -= 1 continue log.error('Failed to associate VPC {0} with hosted zone {1}: {2}'.format( VPCName or VPCId, Name or HostedZoneId, str(e))) @@ -877,7 +877,7 @@ def change_resource_record_sets(HostedZoneId=None, Name=None, if tries and e.response.get('Error', {}).get('Code') == 'Throttling': log.debug('Throttled by AWS API.') time.sleep(3) - retries -= 1 + tries -= 1 continue log.error('Failed to apply requested changes to the hosted zone {0}: {1}'.format( Name or HostedZoneId, str(e))) From da5e580f8c25a5de46f8d07519c23b31ea7d3927 Mon Sep 17 00:00:00 2001 From: Denys Havrysh Date: Tue, 21 Nov 2017 10:19:43 +0200 Subject: [PATCH 41/54] [DOC] Add doc page reference for schedule util module --- doc/topics/jobs/index.rst | 22 +++ salt/utils/schedule.py | 343 +------------------------------------- 2 files changed, 28 insertions(+), 337 deletions(-) diff --git a/doc/topics/jobs/index.rst b/doc/topics/jobs/index.rst index 3499db9906..a3355b733e 100644 --- a/doc/topics/jobs/index.rst +++ b/doc/topics/jobs/index.rst @@ -211,6 +211,28 @@ localtime. This will schedule the command: ``state.sls httpd test=True`` at 5:00 PM on Monday, Wednesday and Friday, and 3:00 PM on Tuesday and Thursday. +.. code-block:: yaml + + schedule: + job1: + function: state.sls + args: + - httpd + kwargs: + test: True + when: + - 'tea time' + +.. code-block:: yaml + + whens: + tea time: 1:40pm + deployment time: Friday 5:00pm + +The Salt scheduler also allows custom phrases to be used for the `when` +parameter. These `whens` can be stored as either pillar values or +grain values. + .. code-block:: yaml schedule: diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 528d12ccf5..076e6fedaf 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1,344 +1,13 @@ # -*- coding: utf-8 -*- + +# See doc/topics/jobs/index.rst ''' Scheduling routines are located here. To activate the scheduler make the -schedule option available to the master or minion configurations (master config -file or for the minion via config or pillar) - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 3600 - args: - - httpd - kwargs: - test: True - -This will schedule the command: ``state.sls httpd test=True`` every 3600 seconds -(every hour). - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 3600 - args: - - httpd - kwargs: - test: True - splay: 15 - -This will schedule the command: ``state.sls httpd test=True`` every 3600 seconds -(every hour) splaying the time between 0 and 15 seconds. - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 3600 - args: - - httpd - kwargs: - test: True - splay: - start: 10 - end: 15 - -This will schedule the command: ``state.sls httpd test=True`` every 3600 seconds -(every hour) splaying the time between 10 and 15 seconds. - -.. versionadded:: 2014.7.0 - -Frequency of jobs can also be specified using date strings supported by -the Python ``dateutil`` library. This requires the Python ``dateutil`` library -to be installed. - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - args: - - httpd - kwargs: - test: True - when: 5:00pm - -This will schedule the command: ``state.sls httpd test=True`` at 5:00 PM minion -localtime. - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - args: - - httpd - kwargs: - test: True - when: - - Monday 5:00pm - - Tuesday 3:00pm - - Wednesday 5:00pm - - Thursday 3:00pm - - Friday 5:00pm - -This will schedule the command: ``state.sls httpd test=True`` at 5:00 PM on -Monday, Wednesday and Friday, and 3:00 PM on Tuesday and Thursday. - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - args: - - httpd - kwargs: - test: True - when: - - 'tea time' - -.. code-block:: yaml - - whens: - tea time: 1:40pm - deployment time: Friday 5:00pm - -The Salt scheduler also allows custom phrases to be used for the `when` -parameter. These `whens` can be stored as either pillar values or -grain values. - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 3600 - args: - - httpd - kwargs: - test: True - range: - start: 8:00am - end: 5:00pm - -This will schedule the command: ``state.sls httpd test=True`` every 3600 seconds -(every hour) between the hours of 8:00 AM and 5:00 PM. The range parameter must -be a dictionary with the date strings using the ``dateutil`` format. - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 3600 - args: - - httpd - kwargs: - test: True - range: - invert: True - start: 8:00am - end: 5:00pm - -Using the invert option for range, this will schedule the command -``state.sls httpd test=True`` every 3600 seconds (every hour) until the current -time is between the hours of 8:00 AM and 5:00 PM. The range parameter must be -a dictionary with the date strings using the ``dateutil`` format. - -.. code-block:: yaml - - schedule: - job1: - function: pkg.install - kwargs: - pkgs: [{'bar': '>1.2.3'}] - refresh: true - once: '2016-01-07T14:30:00' - -This will schedule the function ``pkg.install`` to be executed once at the -specified time. The schedule entry ``job1`` will not be removed after the job -completes, therefore use ``schedule.delete`` to manually remove it afterwards. - -The default date format is ISO 8601 but can be overridden by also specifying the -``once_fmt`` option, like this: - -.. code-block:: yaml - - schedule: - job1: - function: test.ping - once: 2015-04-22T20:21:00 - once_fmt: '%Y-%m-%dT%H:%M:%S' - -.. versionadded:: 2014.7.0 - -The scheduler also supports ensuring that there are no more than N copies of -a particular routine running. Use this for jobs that may be long-running -and could step on each other or pile up in case of infrastructure outage. - -The default for ``maxrunning`` is 1. - -.. code-block:: yaml - - schedule: - long_running_job: - function: big_file_transfer - jid_include: True - maxrunning: 1 - -.. versionadded:: 2014.7.0 - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - cron: '*/15 * * * *' - args: - - httpd - kwargs: - test: True - -The scheduler also supports scheduling jobs using a cron like format. -This requires the Python ``croniter`` library. - -.. versionadded:: 2015.5.0 - -By default, data about jobs runs from the Salt scheduler is returned to the -master. Setting the ``return_job`` parameter to False will prevent the data -from being sent back to the Salt master. - -.. code-block:: yaml - - schedule: - job1: - function: scheduled_job_function - return_job: False - -.. versionadded:: 2015.5.0 - -It can be useful to include specific data to differentiate a job from other -jobs. Using the metadata parameter special values can be associated with -a scheduled job. These values are not used in the execution of the job, -but can be used to search for specific jobs later if combined with the -``return_job`` parameter. The metadata parameter must be specified as a -dictionary, othewise it will be ignored. - -.. code-block:: yaml - - schedule: - job1: - function: scheduled_job_function - metadata: - foo: bar - -.. versionadded:: 2015.5.0 - -By default any job scheduled based on the startup time of the minion will run -the scheduled job when the minion starts up. Sometimes this is not the desired -situation. Using the ``run_on_start`` parameter set to ``False`` will cause the -scheduler to skip this first run and wait until the next scheduled run: - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 3600 - run_on_start: False - args: - - httpd - kwargs: - test: True - -.. versionadded:: 2015.8.0 - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 15 - until: '12/31/2015 11:59pm' - args: - - httpd - kwargs: - test: True - -Using the until argument, the Salt scheduler allows you to specify -an end time for a scheduled job. If this argument is specified, jobs -will not run once the specified time has passed. Time should be specified -in a format supported by the ``dateutil`` library. -This requires the Python ``dateutil`` library to be installed. - -.. versionadded:: 2015.8.0 - -.. code-block:: yaml - - schedule: - job1: - function: state.sls - seconds: 15 - after: '12/31/2015 11:59pm' - args: - - httpd - kwargs: - test: True - -Using the after argument, the Salt scheduler allows you to specify -an start time for a scheduled job. If this argument is specified, jobs -will not run until the specified time has passed. Time should be specified -in a format supported by the ``dateutil`` library. -This requires the Python ``dateutil`` library to be installed. - -The scheduler also supports ensuring that there are no more than N copies of -a particular routine running. Use this for jobs that may be long-running -and could step on each other or pile up in case of infrastructure outage. - -The default for maxrunning is 1. - -.. code-block:: yaml - - schedule: - long_running_job: - function: big_file_transfer - jid_include: True - maxrunning: 1 - -By default, data about jobs runs from the Salt scheduler is returned to the -master. Setting the ``return_job`` parameter to False will prevent the data -from being sent back to the Salt master. - -.. versionadded:: 2015.5.0 - - schedule: - job1: - function: scheduled_job_function - return_job: False - -Setting the ``return_job`` parameter to 'nocache' prevents the salt master -from storing the job in the master cache. Still, an event is fired on the -master event bus in the form 'salt/job/nocache/ret/myminion'. - -It can be useful to include specific data to differentiate a job from other -jobs. Using the metadata parameter special values can be associated with -a scheduled job. These values are not used in the execution of the job, -but can be used to search for specific jobs later if combined with the -return_job parameter. The metadata parameter must be specified as a -dictionary, othewise it will be ignored. - -.. versionadded:: 2015.5.0 - - schedule: - job1: - function: scheduled_job_function - metadata: - foo: bar +``schedule`` option available to the master or minion configurations (master +config file or for the minion via config or pillar). +Detailed tutorial about scheduling jobs can be found :ref:`here +`. ''' # Import python libs From 5d4d1e29ff1671b7664e0865d602484cff66c6ef Mon Sep 17 00:00:00 2001 From: Jasper Lievisse Adriaanse Date: Mon, 11 Dec 2017 12:03:46 +0100 Subject: [PATCH 42/54] status.meminfo: add OpenBSD support --- salt/modules/status.py | 18 ++++++++++++++++++ tests/unit/modules/test_status.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/salt/modules/status.py b/salt/modules/status.py index 9391a71083..0cf35ea29c 100644 --- a/salt/modules/status.py +++ b/salt/modules/status.py @@ -447,6 +447,9 @@ def meminfo(): .. versionchanged:: 2016.11.4 Added support for AIX + .. versionchanged:: Oxygen + Added support for OpenBSD + CLI Example: .. code-block:: bash @@ -574,10 +577,25 @@ def meminfo(): return ret + def openbsd_meminfo(): + ''' + openbsd specific implementation of meminfo + ''' + vmstat = __salt__['cmd.run']('vmstat').splitlines() + # We're only interested in memory and page values which are printed + # as subsequent fields. + fields = ['active virtual pages', 'free list size', 'page faults', + 'pages reclaimed', 'pages paged in', 'pages paged out', + 'pages freed', 'pages scanned'] + data = vmstat[2].split()[2:10] + ret = dict(zip(fields, data)) + return ret + # dict that return a function that does the right thing per platform get_version = { 'Linux': linux_meminfo, 'FreeBSD': freebsd_meminfo, + 'OpenBSD': openbsd_meminfo, 'AIX': aix_meminfo, } diff --git a/tests/unit/modules/test_status.py b/tests/unit/modules/test_status.py index 80a0e3909c..a8521255a1 100644 --- a/tests/unit/modules/test_status.py +++ b/tests/unit/modules/test_status.py @@ -235,3 +235,34 @@ class StatusTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(status.__salt__, {'cmd.run': MagicMock(return_value=sysctl)}): ret = status.cpuinfo() self.assertDictEqual(ret, m.ret) + + def _set_up_test_meminfo_openbsd(self): + class MockData(object): + ''' + Store mock data + ''' + + m = MockData() + m.ret = { + 'active virtual pages': '355M', + 'free list size': '305M', + 'page faults': '845', + 'pages reclaimed': '1', + 'pages paged in': '2', + 'pages paged out': '3', + 'pages freed': '4', + 'pages scanned': '5' + } + + return m + + def test_meminfo_openbsd(self): + m = self._set_up_test_meminfo_openbsd() + vmstat = ' procs memory page disks traps cpu\n' \ + ' r s avm fre flt re pi po fr sr cd0 sd0 int sys cs us sy id\n' \ + ' 2 103 355M 305M 845 1 2 3 4 5 0 1 21 682 86 1 1 98' + + with patch.dict(status.__grains__, {'kernel': 'OpenBSD'}): + with patch.dict(status.__salt__, {'cmd.run': MagicMock(return_value=vmstat)}): + ret = status.meminfo() + self.assertDictEqual(ret, m.ret) From e4b6002cacf045d40d437201c6cc3c41320d3af4 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Thu, 14 Dec 2017 12:31:24 -0500 Subject: [PATCH 43/54] INFRA-5975 - documentation and formatting fixes --- salt/modules/boto_route53.py | 50 +++++++++----------- salt/states/boto_route53.py | 90 +++++++++++++++--------------------- 2 files changed, 57 insertions(+), 83 deletions(-) diff --git a/salt/modules/boto_route53.py b/salt/modules/boto_route53.py index 1d0b57d32f..1cdb014711 100644 --- a/salt/modules/boto_route53.py +++ b/salt/modules/boto_route53.py @@ -694,27 +694,22 @@ def _wait_for_sync(status, conn, wait=True): return False -def create_hosted_zone(domain_name, caller_ref=None, comment='', - private_zone=False, vpc_id=None, vpc_name=None, - vpc_region=None, region=None, key=None, keyid=None, +def create_hosted_zone(domain_name, caller_ref=None, comment='', private_zone=False, vpc_id=None, + vpc_name=None, vpc_region=None, region=None, key=None, keyid=None, profile=None): ''' - Create a new Route53 Hosted Zone. Returns a Python data structure with - information about the newly created Hosted Zone. + Create a new Route53 Hosted Zone. Returns a Python data structure with information about the + newly created Hosted Zone. domain_name - The name of the domain. This should be a fully-specified domain, and - should terminate with a period. This is the name you have registered - with your DNS registrar. It is also the name you will delegate from your - registrar to the Amazon Route 53 delegation servers returned in response + The name of the domain. This must be fully-qualified, terminating with a period. This is + the name you have registered with your domain registrar. It is also the name you will + delegate from your registrar to the Amazon Route 53 delegation servers returned in response to this request. caller_ref - A unique string that identifies the request and that allows - create_hosted_zone() calls to be retried without the risk of executing - the operation twice. You want to provide this where possible, since - additional calls while the first is in PENDING status will be accepted - and can lead to multiple copies of the zone being created in Route53. + A unique string that identifies the request and that allows create_hosted_zone() calls to + be retried without the risk of executing the operation twice. comment Any comments you want to include about the hosted zone. @@ -723,33 +718,30 @@ def create_hosted_zone(domain_name, caller_ref=None, comment='', Set True if creating a private hosted zone. vpc_id - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_name. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_name. Ignored when creating a non-private zone. vpc_name - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_id. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_id. Ignored when creating a non-private zone. vpc_region - When creating a private hosted zone, the region of the associated VPC is - required. If not provided, an effort will be made to determine it from - vpc_id or vpc_name, if possible. If this fails, you'll need to provide - an explicit value for this option. Ignored if passed for a non-private - zone. + When creating a private hosted zone, the region of the associated VPC is required. If not + provided, an effort will be made to determine it from vpc_id or vpc_name, where possible. + If this fails, you'll need to provide an explicit value for this option. Ignored when + creating a non-private zone. region - Region endpoint to connect to + Region endpoint to connect to. key - AWS key to bind with + AWS key to bind with. keyid - AWS keyid to bind with + AWS keyid to bind with. profile - Dict, or pillar key pointing to a dict, containing AWS region/key/keyid + Dict, or pillar key pointing to a dict, containing AWS region/key/keyid. CLI Example:: diff --git a/salt/states/boto_route53.py b/salt/states/boto_route53.py index 19247516b7..a1789b69d2 100644 --- a/salt/states/boto_route53.py +++ b/salt/states/boto_route53.py @@ -94,20 +94,8 @@ def rr_present(*args, **kwargs): return present(*args, **kwargs) -def present( - name, - value, - zone, - record_type, - ttl=None, - identifier=None, - region=None, - key=None, - keyid=None, - profile=None, - wait_for_sync=True, - split_dns=False, - private_zone=False): +def present(name, value, zone, record_type, ttl=None, identifier=None, region=None, key=None, + keyid=None, profile=None, wait_for_sync=True, split_dns=False, private_zone=False): ''' Ensure the Route53 record is present. @@ -116,8 +104,8 @@ def present( value Value of the record. As a special case, you can pass in: - private: to have the function autodetermine the private IP - public: to have the function autodetermine the public IP + `private:` to have the function determine the private IP + `public:` to have the function autodetermine the public IP zone The zone to create the record in. @@ -141,15 +129,14 @@ def present( Access key to be used. profile - A dict with region, key and keyid, or a pillar key (string) - that contains a dict with region, key and keyid. + A dict with region, key and keyid, or a pillar key (string) that contains a dict + with region, key and keyid. wait_for_sync - Wait for an INSYNC change status from Route53. + Wait for an INSYNC change status from Route53 before returning success. split_dns - Route53 supports a public and private DNS zone with the same - names. + Route53 supports parallel public and private DNS zones with the same name. private_zone If using split_dns, specify if this is the private zone. @@ -361,17 +348,18 @@ def absent( return ret -def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=None, - comment='', vpc_id=None, vpc_name=None, vpc_region=None, - region=None, key=None, keyid=None, profile=None): +def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=None, comment='', + vpc_id=None, vpc_name=None, vpc_region=None, region=None, key=None, + keyid=None, profile=None): ''' - Ensure a hosted zone exists with the given attributes. Note that most - things cannot be modified once a zone is created - it must be deleted and - re-spun to update these attributes: + Ensure a hosted zone exists with the given attributes. Note that most things cannot be + modified once a zone is created - it must be deleted and re-spun to update these attributes. + If you need the ability to update these attributes, please use the newer boto3_route53 + module instead: - private_zone (AWS API limitation). - - comment (the appropriate call exists in the AWS API and in boto3, but - has not, as of this writing, been added to boto2). - - vpc_id (same story - we really need to rewrite this module with boto3) + - comment (the appropriate call exists in the AWS API and in boto3, but has not, as of + this writing, been added to boto2). + - vpc_id (boto3 only) - vpc_name (really just a pointer to vpc_id anyway). - vpc_region (again, supported in boto3 but not boto2). @@ -379,10 +367,9 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N The name of the state definition. domain_name - The name of the domain. This should be a fully-specified domain, and - should terminate with a period. This is the name you have registered - with your DNS registrar. It is also the name you will delegate from your - registrar to the Amazon Route 53 delegation servers returned in response + The name of the domain. This must be fully-qualified, terminating with a period. This is + the name you have registered with your domain registrar. It is also the name you will + delegate from your registrar to the Amazon Route 53 delegation servers returned in response to this request. Defaults to the value of name if not provided. private_zone @@ -393,29 +380,25 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N retried without the risk of executing the operation twice. This helps ensure idempotency across state calls, but can cause issues if a zone is deleted and then an attempt is made to recreate it with the same caller_ref. If not provided, a unique UUID will be generated - at each state run, which can potentially lead to duplicate zones being created if the state - is run again while the previous zone creation is still in PENDING status (which can - occasionally take several minutes to clear). Maximum length of 128. + at each state run, which avoids the risk of the above (transient) error. This option is + generally not needed. Maximum length of 128. comment Any comments you want to include about the hosted zone. vpc_id - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_name. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_name. Ignored when creating a non-private zone. vpc_name - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_id. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_id. Ignored when creating a non-private zone. vpc_region - When creating a private hosted zone, the region of the associated VPC is - required. If not provided, an effort will be made to determine it from - vpc_id or vpc_name, if possible. If this fails, you'll need to provide - an explicit value for this option. Ignored if passed for a non-private - zone. + When creating a private hosted zone, the region of the associated VPC is required. If not + provided, an effort will be made to determine it from vpc_id or vpc_name, where possible. + If this fails, you'll need to provide an explicit value for this option. Ignored when + creating a non-private zone. ''' domain_name = domain_name if domain_name else name @@ -424,23 +407,22 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N # First translaste vpc_name into a vpc_id if possible if private_zone: if not exactly_one((vpc_name, vpc_id)): - raise SaltInvocationError('Either vpc_name or vpc_id is required ' - 'when creating a private zone.') + raise SaltInvocationError('Either vpc_name or vpc_id is required when creating a ' + 'private zone.') vpcs = __salt__['boto_vpc.describe_vpcs']( vpc_id=vpc_id, name=vpc_name, region=region, key=key, keyid=keyid, profile=profile).get('vpcs', []) if vpc_region and vpcs: vpcs = [v for v in vpcs if v['region'] == vpc_region] if not vpcs: - msg = ('Private zone requested but a VPC matching given criteria ' - 'not found.') + msg = ('Private zone requested but a VPC matching given criteria not found.') log.error(msg) ret['result'] = False ret['comment'] = msg return ret if len(vpcs) > 1: - msg = ('Private zone requested but multiple VPCs matching given ' - 'criteria found: {0}.'.format([v['id'] for v in vpcs])) + msg = ('Private zone requested but multiple VPCs matching given criteria found: ' + '{0}.'.format([v['id'] for v in vpcs])) log.error(msg) return None vpc = vpcs[0] From a1184c8c2c106f5b398fdbdea90ca6f5cb888d87 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Thu, 14 Dec 2017 13:15:27 -0500 Subject: [PATCH 44/54] INFRA-5975 - trimmed too much --- salt/states/boto_route53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/boto_route53.py b/salt/states/boto_route53.py index a1789b69d2..1686ae7e8e 100644 --- a/salt/states/boto_route53.py +++ b/salt/states/boto_route53.py @@ -104,7 +104,7 @@ def present(name, value, zone, record_type, ttl=None, identifier=None, region=No value Value of the record. As a special case, you can pass in: - `private:` to have the function determine the private IP + `private:` to have the function autodetermine the private IP `public:` to have the function autodetermine the public IP zone From 2bbe4eb162f0ac4208a7bbb64b0b723f75c377a2 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Thu, 14 Dec 2017 12:31:24 -0500 Subject: [PATCH 45/54] INFRA-5975 - documentation and formatting fixes --- salt/modules/boto_route53.py | 50 +++++++++----------- salt/states/boto_route53.py | 90 +++++++++++++++--------------------- 2 files changed, 57 insertions(+), 83 deletions(-) diff --git a/salt/modules/boto_route53.py b/salt/modules/boto_route53.py index 70c25c3429..7e46f17a50 100644 --- a/salt/modules/boto_route53.py +++ b/salt/modules/boto_route53.py @@ -602,27 +602,22 @@ def _wait_for_sync(status, conn, wait=True): return False -def create_hosted_zone(domain_name, caller_ref=None, comment='', - private_zone=False, vpc_id=None, vpc_name=None, - vpc_region=None, region=None, key=None, keyid=None, +def create_hosted_zone(domain_name, caller_ref=None, comment='', private_zone=False, vpc_id=None, + vpc_name=None, vpc_region=None, region=None, key=None, keyid=None, profile=None): ''' - Create a new Route53 Hosted Zone. Returns a Python data structure with - information about the newly created Hosted Zone. + Create a new Route53 Hosted Zone. Returns a Python data structure with information about the + newly created Hosted Zone. domain_name - The name of the domain. This should be a fully-specified domain, and - should terminate with a period. This is the name you have registered - with your DNS registrar. It is also the name you will delegate from your - registrar to the Amazon Route 53 delegation servers returned in response + The name of the domain. This must be fully-qualified, terminating with a period. This is + the name you have registered with your domain registrar. It is also the name you will + delegate from your registrar to the Amazon Route 53 delegation servers returned in response to this request. caller_ref - A unique string that identifies the request and that allows - create_hosted_zone() calls to be retried without the risk of executing - the operation twice. You want to provide this where possible, since - additional calls while the first is in PENDING status will be accepted - and can lead to multiple copies of the zone being created in Route53. + A unique string that identifies the request and that allows create_hosted_zone() calls to + be retried without the risk of executing the operation twice. comment Any comments you want to include about the hosted zone. @@ -631,33 +626,30 @@ def create_hosted_zone(domain_name, caller_ref=None, comment='', Set True if creating a private hosted zone. vpc_id - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_name. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_name. Ignored when creating a non-private zone. vpc_name - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_id. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_id. Ignored when creating a non-private zone. vpc_region - When creating a private hosted zone, the region of the associated VPC is - required. If not provided, an effort will be made to determine it from - vpc_id or vpc_name, if possible. If this fails, you'll need to provide - an explicit value for this option. Ignored if passed for a non-private - zone. + When creating a private hosted zone, the region of the associated VPC is required. If not + provided, an effort will be made to determine it from vpc_id or vpc_name, where possible. + If this fails, you'll need to provide an explicit value for this option. Ignored when + creating a non-private zone. region - Region endpoint to connect to + Region endpoint to connect to. key - AWS key to bind with + AWS key to bind with. keyid - AWS keyid to bind with + AWS keyid to bind with. profile - Dict, or pillar key pointing to a dict, containing AWS region/key/keyid + Dict, or pillar key pointing to a dict, containing AWS region/key/keyid. CLI Example:: diff --git a/salt/states/boto_route53.py b/salt/states/boto_route53.py index 9bd9a08d0a..e60e772584 100644 --- a/salt/states/boto_route53.py +++ b/salt/states/boto_route53.py @@ -93,20 +93,8 @@ def rr_present(*args, **kwargs): return present(*args, **kwargs) -def present( - name, - value, - zone, - record_type, - ttl=None, - identifier=None, - region=None, - key=None, - keyid=None, - profile=None, - wait_for_sync=True, - split_dns=False, - private_zone=False): +def present(name, value, zone, record_type, ttl=None, identifier=None, region=None, key=None, + keyid=None, profile=None, wait_for_sync=True, split_dns=False, private_zone=False): ''' Ensure the Route53 record is present. @@ -115,8 +103,8 @@ def present( value Value of the record. As a special case, you can pass in: - private: to have the function autodetermine the private IP - public: to have the function autodetermine the public IP + `private:` to have the function determine the private IP + `public:` to have the function autodetermine the public IP zone The zone to create the record in. @@ -140,15 +128,14 @@ def present( Access key to be used. profile - A dict with region, key and keyid, or a pillar key (string) - that contains a dict with region, key and keyid. + A dict with region, key and keyid, or a pillar key (string) that contains a dict + with region, key and keyid. wait_for_sync - Wait for an INSYNC change status from Route53. + Wait for an INSYNC change status from Route53 before returning success. split_dns - Route53 supports a public and private DNS zone with the same - names. + Route53 supports parallel public and private DNS zones with the same name. private_zone If using split_dns, specify if this is the private zone. @@ -360,17 +347,18 @@ def absent( return ret -def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=None, - comment='', vpc_id=None, vpc_name=None, vpc_region=None, - region=None, key=None, keyid=None, profile=None): +def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=None, comment='', + vpc_id=None, vpc_name=None, vpc_region=None, region=None, key=None, + keyid=None, profile=None): ''' - Ensure a hosted zone exists with the given attributes. Note that most - things cannot be modified once a zone is created - it must be deleted and - re-spun to update these attributes: + Ensure a hosted zone exists with the given attributes. Note that most things cannot be + modified once a zone is created - it must be deleted and re-spun to update these attributes. + If you need the ability to update these attributes, please use the newer boto3_route53 + module instead: - private_zone (AWS API limitation). - - comment (the appropriate call exists in the AWS API and in boto3, but - has not, as of this writing, been added to boto2). - - vpc_id (same story - we really need to rewrite this module with boto3) + - comment (the appropriate call exists in the AWS API and in boto3, but has not, as of + this writing, been added to boto2). + - vpc_id (boto3 only) - vpc_name (really just a pointer to vpc_id anyway). - vpc_region (again, supported in boto3 but not boto2). @@ -378,10 +366,9 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N The name of the state definition. domain_name - The name of the domain. This should be a fully-specified domain, and - should terminate with a period. This is the name you have registered - with your DNS registrar. It is also the name you will delegate from your - registrar to the Amazon Route 53 delegation servers returned in response + The name of the domain. This must be fully-qualified, terminating with a period. This is + the name you have registered with your domain registrar. It is also the name you will + delegate from your registrar to the Amazon Route 53 delegation servers returned in response to this request. Defaults to the value of name if not provided. private_zone @@ -392,29 +379,25 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N retried without the risk of executing the operation twice. This helps ensure idempotency across state calls, but can cause issues if a zone is deleted and then an attempt is made to recreate it with the same caller_ref. If not provided, a unique UUID will be generated - at each state run, which can potentially lead to duplicate zones being created if the state - is run again while the previous zone creation is still in PENDING status (which can - occasionally take several minutes to clear). Maximum length of 128. + at each state run, which avoids the risk of the above (transient) error. This option is + generally not needed. Maximum length of 128. comment Any comments you want to include about the hosted zone. vpc_id - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_name. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_name. Ignored when creating a non-private zone. vpc_name - When creating a private hosted zone, either the VPC ID or VPC Name to - associate with is required. Exclusive with vpe_id. Ignored if passed - for a non-private zone. + When creating a private hosted zone, either the VPC ID or VPC Name to associate with is + required. Exclusive with vpe_id. Ignored when creating a non-private zone. vpc_region - When creating a private hosted zone, the region of the associated VPC is - required. If not provided, an effort will be made to determine it from - vpc_id or vpc_name, if possible. If this fails, you'll need to provide - an explicit value for this option. Ignored if passed for a non-private - zone. + When creating a private hosted zone, the region of the associated VPC is required. If not + provided, an effort will be made to determine it from vpc_id or vpc_name, where possible. + If this fails, you'll need to provide an explicit value for this option. Ignored when + creating a non-private zone. ''' domain_name = domain_name if domain_name else name @@ -423,23 +406,22 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N # First translaste vpc_name into a vpc_id if possible if private_zone: if not exactly_one((vpc_name, vpc_id)): - raise SaltInvocationError('Either vpc_name or vpc_id is required ' - 'when creating a private zone.') + raise SaltInvocationError('Either vpc_name or vpc_id is required when creating a ' + 'private zone.') vpcs = __salt__['boto_vpc.describe_vpcs']( vpc_id=vpc_id, name=vpc_name, region=region, key=key, keyid=keyid, profile=profile).get('vpcs', []) if vpc_region and vpcs: vpcs = [v for v in vpcs if v['region'] == vpc_region] if not vpcs: - msg = ('Private zone requested but a VPC matching given criteria ' - 'not found.') + msg = ('Private zone requested but a VPC matching given criteria not found.') log.error(msg) ret['result'] = False ret['comment'] = msg return ret if len(vpcs) > 1: - msg = ('Private zone requested but multiple VPCs matching given ' - 'criteria found: {0}.'.format([v['id'] for v in vpcs])) + msg = ('Private zone requested but multiple VPCs matching given criteria found: ' + '{0}.'.format([v['id'] for v in vpcs])) log.error(msg) return None vpc = vpcs[0] From 255d5d22bd378b352ac65dd32587947eb49d4596 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Thu, 14 Dec 2017 13:15:27 -0500 Subject: [PATCH 46/54] INFRA-5975 - trimmed too much --- salt/states/boto_route53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/boto_route53.py b/salt/states/boto_route53.py index e60e772584..b4662b85de 100644 --- a/salt/states/boto_route53.py +++ b/salt/states/boto_route53.py @@ -103,7 +103,7 @@ def present(name, value, zone, record_type, ttl=None, identifier=None, region=No value Value of the record. As a special case, you can pass in: - `private:` to have the function determine the private IP + `private:` to have the function autodetermine the private IP `public:` to have the function autodetermine the public IP zone From 08e5dfcc57fc344efa5ebee3c32566b8f385ce75 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Thu, 14 Dec 2017 16:51:49 -0500 Subject: [PATCH 47/54] INFRA-5975 - even BETTER documentation! --- salt/modules/boto_route53.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/modules/boto_route53.py b/salt/modules/boto_route53.py index 7e46f17a50..d51c971899 100644 --- a/salt/modules/boto_route53.py +++ b/salt/modules/boto_route53.py @@ -617,7 +617,13 @@ def create_hosted_zone(domain_name, caller_ref=None, comment='', private_zone=Fa caller_ref A unique string that identifies the request and that allows create_hosted_zone() calls to - be retried without the risk of executing the operation twice. + be retried without the risk of executing the operation twice. It can take several minutes + for the change to replicate globally, and change from PENDING to INSYNC status. Thus it's + best to provide some value for this where possible, since duplicate calls while the first + is in PENDING status will be accepted and can lead to multiple copies of the zone being + created. On the other hand, if a zone is created with a given caller_ref, then deleted, + a second attempt to create a zone with the same caller_ref will fail until that caller_ref + is flushed from the Route53 system, which can take upwards of 24 hours. comment Any comments you want to include about the hosted zone. From 5adf8526157705fa33f993db7c0d4439f7de906d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 14 Dec 2017 19:21:00 -0600 Subject: [PATCH 48/54] Add unicode_literals to runner modules --- salt/runners/asam.py | 33 ++++++------ salt/runners/auth.py | 2 +- salt/runners/bgp.py | 6 +-- salt/runners/cache.py | 2 +- salt/runners/cloud.py | 2 +- salt/runners/config.py | 2 +- salt/runners/ddns.py | 2 +- salt/runners/digicertapi.py | 10 ++-- salt/runners/doc.py | 2 +- salt/runners/drac.py | 10 ++-- salt/runners/error.py | 2 +- salt/runners/event.py | 2 +- salt/runners/f5.py | 2 +- salt/runners/fileserver.py | 2 +- salt/runners/git_pillar.py | 6 +-- salt/runners/http.py | 2 +- salt/runners/jobs.py | 2 +- salt/runners/launchd.py | 2 +- salt/runners/lxc.py | 19 +++---- salt/runners/manage.py | 4 +- salt/runners/mattermost.py | 8 +-- salt/runners/mine.py | 2 +- salt/runners/nacl.py | 12 +++-- salt/runners/net.py | 11 ++-- salt/runners/network.py | 12 +++-- salt/runners/pagerduty.py | 8 +-- salt/runners/pillar.py | 2 +- salt/runners/pkg.py | 2 +- salt/runners/queue.py | 3 +- salt/runners/reactor.py | 2 +- salt/runners/salt.py | 7 +-- salt/runners/saltutil.py | 2 +- salt/runners/sdb.py | 2 +- salt/runners/smartos_vmadm.py | 3 +- salt/runners/spacewalk.py | 12 ++--- salt/runners/ssh.py | 2 +- salt/runners/state.py | 2 +- salt/runners/survey.py | 8 +-- salt/runners/test.py | 3 +- salt/runners/thin.py | 3 +- salt/runners/vault.py | 25 +++++---- salt/runners/venafiapi.py | 35 +++++++------ salt/runners/virt.py | 11 ++-- salt/runners/vistara.py | 2 +- salt/runners/winrepo.py | 33 +++++------- tests/integration/runners/test_cache.py | 2 +- tests/integration/runners/test_fileserver.py | 2 +- tests/integration/runners/test_jobs.py | 2 +- tests/integration/runners/test_manage.py | 2 +- .../runners/test_runner_returns.py | 2 +- tests/integration/runners/test_salt.py | 2 +- tests/integration/runners/test_state.py | 52 ++++++++++--------- tests/unit/runners/test_cache.py | 2 +- tests/unit/runners/test_jobs.py | 2 +- tests/unit/runners/test_queue.py | 2 +- tests/unit/runners/test_vault.py | 20 +++---- tests/unit/runners/test_winrepo.py | 14 +++-- 57 files changed, 217 insertions(+), 213 deletions(-) diff --git a/salt/runners/asam.py b/salt/runners/asam.py index 5ffb08a959..1e46ba3d47 100644 --- a/salt/runners/asam.py +++ b/salt/runners/asam.py @@ -28,16 +28,17 @@ master configuration at ``/etc/salt/master`` or ``/etc/salt/master.d/asam.conf`` is not using the defaults. Default is ``protocol: https`` and ``port: 3451``. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals -# Import python libs +# Import Python libs import logging -# Import third party libs +# Import 3rd-party libs +import salt.ext.six as six + HAS_LIBS = False try: import requests - import salt.ext.six as six from salt.ext.six.moves.html_parser import HTMLParser # pylint: disable=E0611 HAS_LIBS = True @@ -90,8 +91,8 @@ def _get_asam_configuration(driver_url=''): if not username or not password: log.error( - "Username or Password has not been specified in the master " - "configuration for {0}".format(asam_server) + 'Username or Password has not been specified in the ' + 'master configuration for %s', asam_server ) return False @@ -107,15 +108,13 @@ def _get_asam_configuration(driver_url=''): if (not driver_url) or (driver_url == asam_server): return ret except Exception as exc: - log.error( - "Exception encountered: {0}".format(exc) - ) + log.error('Exception encountered: %s', exc) return False if driver_url: log.error( - "Configuration for {0} has not been specified in the master " - "configuration".format(driver_url) + 'Configuration for %s has not been specified in the master ' + 'configuration', driver_url ) return False @@ -205,7 +204,7 @@ def remove_platform(name, server_url): html_content = _make_post_request(url, data, auth, verify=False) except Exception as exc: err_msg = "Failed to look up existing platforms on {0}".format(server_url) - log.error("{0}:\n{1}".format(err_msg, exc)) + log.error('%s:\n%s', err_msg, exc) return {name: err_msg} parser = _parse_html_content(html_content) @@ -214,14 +213,14 @@ def remove_platform(name, server_url): if platformset_name: log.debug(platformset_name) data['platformName'] = name - data['platformSetName'] = str(platformset_name) + data['platformSetName'] = six.text_type(platformset_name) data['postType'] = 'platformRemove' data['Submit'] = 'Yes' try: html_content = _make_post_request(url, data, auth, verify=False) except Exception as exc: err_msg = "Failed to delete platform from {1}".format(server_url) - log.error("{0}:\n{1}".format(err_msg, exc)) + log.error('%s:\n%s', err_msg, exc) return {name: err_msg} parser = _parse_html_content(html_content) @@ -263,7 +262,7 @@ def list_platforms(server_url): html_content = _make_post_request(url, data, auth, verify=False) except Exception as exc: err_msg = "Failed to look up existing platforms" - log.error("{0}:\n{1}".format(err_msg, exc)) + log.error('%s:\n%s', err_msg, exc) return {server_url: err_msg} parser = _parse_html_content(html_content) @@ -304,7 +303,7 @@ def list_platform_sets(server_url): html_content = _make_post_request(url, data, auth, verify=False) except Exception as exc: err_msg = "Failed to look up existing platform sets" - log.error("{0}:\n{1}".format(err_msg, exc)) + log.error('%s:\n%s', err_msg, exc) return {server_url: err_msg} parser = _parse_html_content(html_content) @@ -359,7 +358,7 @@ def add_platform(name, platform_set, server_url): html_content = _make_post_request(url, data, auth, verify=False) except Exception as exc: err_msg = "Failed to add platform on {0}".format(server_url) - log.error("{0}:\n{1}".format(err_msg, exc)) + log.error('%s:\n%s', err_msg, exc) return {name: err_msg} platforms = list_platforms(server_url) diff --git a/salt/runners/auth.py b/salt/runners/auth.py index 7e8e64855a..205d308c1a 100644 --- a/salt/runners/auth.py +++ b/salt/runners/auth.py @@ -7,7 +7,7 @@ Authentication runner for creating, deleting, and managing eauth tokens. ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os # Import Salt libs diff --git a/salt/runners/bgp.py b/salt/runners/bgp.py index b460bdcef9..98dc1ce013 100644 --- a/salt/runners/bgp.py +++ b/salt/runners/bgp.py @@ -97,9 +97,7 @@ Configuration - flap_count outputter: yaml ''' -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals # Import third party libs try: @@ -335,7 +333,7 @@ def neighbors(*asns, **kwargs): title_parts = [] if asns: title_parts.append('BGP Neighbors for {asns}'.format( - asns=', '.join([str(asn) for asn in asns]) + asns=', '.join([six.text_type(asn) for asn in asns]) )) if neighbor_ip: title_parts.append('Selecting neighbors having the remote IP address: {ipaddr}'.format(ipaddr=neighbor_ip)) diff --git a/salt/runners/cache.py b/salt/runners/cache.py index cbd7475853..9abfc4508f 100644 --- a/salt/runners/cache.py +++ b/salt/runners/cache.py @@ -2,7 +2,7 @@ ''' Return cached data from minions ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import fnmatch import logging diff --git a/salt/runners/cloud.py b/salt/runners/cloud.py index 33f3f6c6cc..9b84ccb1d6 100644 --- a/salt/runners/cloud.py +++ b/salt/runners/cloud.py @@ -6,7 +6,7 @@ The Salt Cloud Runner This runner wraps the functionality of salt cloud making salt cloud routines available to all internal apis via the runner system ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/runners/config.py b/salt/runners/config.py index 99c784adbf..2c3d8c667b 100644 --- a/salt/runners/config.py +++ b/salt/runners/config.py @@ -3,7 +3,7 @@ This runner is designed to mirror the execution module config.py, but for master settings ''' -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import salt.utils.data import salt.utils.sdb diff --git a/salt/runners/ddns.py b/salt/runners/ddns.py index 6ec805679c..c8738e7325 100644 --- a/salt/runners/ddns.py +++ b/salt/runners/ddns.py @@ -10,7 +10,7 @@ Runner to interact with DNS server and create/delete/update DNS records :codeauthor: Nitin Madhok ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import os diff --git a/salt/runners/digicertapi.py b/salt/runners/digicertapi.py index 70f5cd2027..5b7e293fc3 100644 --- a/salt/runners/digicertapi.py +++ b/salt/runners/digicertapi.py @@ -35,7 +35,7 @@ This API currently only supports RSA key types. Support for other key types wil if interest warrants. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import logging import tempfile @@ -569,7 +569,7 @@ def gen_csr( tmppriv = '{0}/priv'.format(tmpdir) tmpcsr = '{0}/csr'.format(tmpdir) with salt.utils.files.fopen(tmppriv, 'w') as if_: - if_.write(data['private_key']) + if_.write(salt.utils.stringutils.to_str(data['private_key'])) subject = '/C={0}/ST={1}/L={2}/O={3}'.format( org_details['dict']['country'], @@ -597,7 +597,7 @@ def gen_csr( ) with salt.utils.files.fopen(tmpcsr, 'r') as of_: - csr = of_.read() + csr = salt.utils.stringutils.to_unicode(of_.read()) data['minion_id'] = minion_id data['csr'] = csr @@ -643,7 +643,7 @@ def show_organization(domain): }, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -670,7 +670,7 @@ def show_csrs(): }, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) diff --git a/salt/runners/doc.py b/salt/runners/doc.py index 92a06b3a44..af42c99eed 100644 --- a/salt/runners/doc.py +++ b/salt/runners/doc.py @@ -4,7 +4,7 @@ A runner module to collect and display the inline documentation from the various module types ''' # Import Python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import itertools # Import salt libs diff --git a/salt/runners/drac.py b/salt/runners/drac.py index 1ef697e7f1..c7d0b46aa2 100644 --- a/salt/runners/drac.py +++ b/salt/runners/drac.py @@ -14,7 +14,7 @@ configuration file. ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import logging # Import 3rd-party libs @@ -60,7 +60,7 @@ def __connect(hostname, timeout=20, username=None, password=None): try: client.connect(hostname, username=username, password=password, timeout=timeout) except Exception as e: - log.error('Unable to connect to {0}: {1}'.format(hostname, e)) + log.error('Unable to connect to %s: %s', hostname, e) return False return client @@ -108,14 +108,14 @@ def pxe(hostname, timeout=20, username=None, password=None): if isinstance(client, paramiko.SSHClient): for i, cmd in enumerate(_cmds, 1): - log.info('Executing command {0}'.format(i)) + log.info('Executing command %s', i) (stdin, stdout, stderr) = client.exec_command(cmd) if 'successful' in stdout.readline(): - log.info('Executing command: {0}'.format(cmd)) + log.info('Executing command: %s', cmd) else: - log.error('Unable to execute: {0}'.format(cmd)) + log.error('Unable to execute: %s', cmd) return False return True diff --git a/salt/runners/error.py b/salt/runners/error.py index 136f2a18eb..cfb93466c5 100644 --- a/salt/runners/error.py +++ b/salt/runners/error.py @@ -3,7 +3,7 @@ Error generator to enable integration testing of salt runner error handling ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs diff --git a/salt/runners/event.py b/salt/runners/event.py index 55be0ecdd5..e76bf2b7d6 100644 --- a/salt/runners/event.py +++ b/salt/runners/event.py @@ -4,7 +4,7 @@ Module for sending events using the runner system. .. versionadded:: 2016.11.0 ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging diff --git a/salt/runners/f5.py b/salt/runners/f5.py index f64f981691..a9c3299175 100644 --- a/salt/runners/f5.py +++ b/salt/runners/f5.py @@ -17,7 +17,7 @@ Runner to provide F5 Load Balancer functionality username: admin password: secret ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs from salt.exceptions import CommandExecutionError diff --git a/salt/runners/fileserver.py b/salt/runners/fileserver.py index 6322aa7842..e4abc256de 100644 --- a/salt/runners/fileserver.py +++ b/salt/runners/fileserver.py @@ -2,7 +2,7 @@ ''' Directly manage the Salt fileserver plugins ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt libs import salt.fileserver diff --git a/salt/runners/git_pillar.py b/salt/runners/git_pillar.py index 984c7da8cc..b90f98977d 100644 --- a/salt/runners/git_pillar.py +++ b/salt/runners/git_pillar.py @@ -2,7 +2,7 @@ ''' Runner module to directly manage the git external pillar ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -83,8 +83,8 @@ def update(branch=None, repo=None): result = remote.fetch() except Exception as exc: log.error( - 'Exception \'{0}\' caught while fetching git_pillar ' - 'remote \'{1}\''.format(exc, remote.id), + 'Exception \'%s\' caught while fetching git_pillar ' + 'remote \'%s\'', exc, remote.id, exc_info_on_loglevel=logging.DEBUG ) result = False diff --git a/salt/runners/http.py b/salt/runners/http.py index 7104d612b8..3cae7daca1 100644 --- a/salt/runners/http.py +++ b/salt/runners/http.py @@ -5,7 +5,7 @@ like, but also useful for basic http testing. .. versionadded:: 2015.5.0 ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import logging diff --git a/salt/runners/jobs.py b/salt/runners/jobs.py index 13a807bc3a..a16191e757 100644 --- a/salt/runners/jobs.py +++ b/salt/runners/jobs.py @@ -4,7 +4,7 @@ A convenience system to manage jobs, both active and already run ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import fnmatch import logging import os diff --git a/salt/runners/launchd.py b/salt/runners/launchd.py index 18b98f2a05..52496bc295 100644 --- a/salt/runners/launchd.py +++ b/salt/runners/launchd.py @@ -2,7 +2,7 @@ ''' Manage launchd plist files ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import os diff --git a/salt/runners/lxc.py b/salt/runners/lxc.py index bd1f86f971..0e2a1d482a 100644 --- a/salt/runners/lxc.py +++ b/salt/runners/lxc.py @@ -6,7 +6,7 @@ Control Linux Containers via Salt ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import time import os import copy @@ -17,6 +17,7 @@ import salt.client import salt.utils.args import salt.utils.cloud import salt.utils.files +import salt.utils.stringutils import salt.utils.virt import salt.key from salt.utils.odict import OrderedDict as _OrderedDict @@ -267,10 +268,10 @@ def init(names, host=None, saltcloud_mode=False, quiet=False, **kwargs): for host, containers in six.iteritems(data): for name in names: if name in sum(six.itervalues(containers), []): - log.info('Container \'{0}\' already exists' - ' on host \'{1}\',' - ' init can be a NO-OP'.format( - name, host)) + log.info( + 'Container \'%s\' already exists on host \'%s\', init ' + 'can be a NO-OP', name, host + ) if host not in data: ret['comment'] = 'Host \'{0}\' was not found'.format(host) ret['result'] = False @@ -303,8 +304,7 @@ def init(names, host=None, saltcloud_mode=False, quiet=False, **kwargs): ret['result'] = False return ret - log.info('Creating container(s) \'{0}\'' - ' on host \'{1}\''.format(names, host)) + log.info('Creating container(s) \'%s\' on host \'%s\'', names, host) cmds = [] for name in names: @@ -374,10 +374,11 @@ def init(names, host=None, saltcloud_mode=False, quiet=False, **kwargs): fcontent = '' if os.path.exists(key): with salt.utils.files.fopen(key) as fic: - fcontent = fic.read().strip() + fcontent = salt.utils.stringutils.to_unicode(fic.read()).strip() + pub_key = salt.utils.stringutils.to_unicode(pub_key) if pub_key.strip() != fcontent: with salt.utils.files.fopen(key, 'w') as fic: - fic.write(pub_key) + fic.write(salt.utils.stringutils.to_str(pub_key)) fic.flush() mid = j_ret.get('mid', None) if not mid: diff --git a/salt/runners/manage.py b/salt/runners/manage.py index b680671ade..781b3f3f4b 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -5,7 +5,7 @@ and what hosts are down ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import os import operator import re @@ -816,7 +816,7 @@ def bootstrap(version='develop', client_opts['argv'] = ['file.remove', tmp_dir] salt.client.ssh.SSH(client_opts).run() except SaltSystemExit as exc: - log.error(str(exc)) + log.error(six.text_type(exc)) def bootstrap_psexec(hosts='', master=None, version=None, arch='win32', diff --git a/salt/runners/mattermost.py b/salt/runners/mattermost.py index 258e3c2a58..3d176f6cc8 100644 --- a/salt/runners/mattermost.py +++ b/salt/runners/mattermost.py @@ -16,7 +16,7 @@ Module for sending messages to Mattermost ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import json import logging @@ -128,7 +128,7 @@ def post_message(message, if username: parameters['username'] = username parameters['text'] = '```' + message + '```' # pre-formatted, fixed-width text - log.debug('Parameters: {0}'.format(parameters)) + log.debug('Parameters: %s', parameters) result = salt.utils.mattermost.query(api_url=api_url, hook=hook, data='payload={0}'.format(json.dumps(parameters))) @@ -168,8 +168,8 @@ def post_event(event, if not event: log.error('message is a required option.') - log.debug('Event: {0}'.format(str(event))) - log.debug('Event data: {0}'.format(str(event['data']))) + log.debug('Event: %s', event) + log.debug('Event data: %s', event['data']) message = 'tag: {0}\r\n'.format(event['tag']) for key, value in six.iteritems(event['data']): message += '{0}: {1}\r\n'.format(key, value) diff --git a/salt/runners/mine.py b/salt/runners/mine.py index 140a853b23..31776c8bf8 100644 --- a/salt/runners/mine.py +++ b/salt/runners/mine.py @@ -2,7 +2,7 @@ ''' A runner to access data from the salt mine ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python Libs import logging diff --git a/salt/runners/nacl.py b/salt/runners/nacl.py index 15b957c7a3..a872fe0dec 100644 --- a/salt/runners/nacl.py +++ b/salt/runners/nacl.py @@ -151,7 +151,7 @@ Optional small program to encrypt data without needing salt modules. ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import base64 import os @@ -162,6 +162,8 @@ import salt.utils.win_functions import salt.utils.win_dacl import salt.syspaths +# Import 3rd-party libs +from salt.ext import six REQ_ERROR = None try: @@ -209,7 +211,7 @@ def _get_sk(**kwargs): sk_file = config['sk_file'] if not key and sk_file: with salt.utils.files.fopen(sk_file, 'rb') as keyf: - key = str(keyf.read()).rstrip('\n') + key = six.text_type(keyf.read()).rstrip('\n') if key is None: raise Exception('no key or sk_file found') return base64.b64decode(key) @@ -224,10 +226,10 @@ def _get_pk(**kwargs): pk_file = config['pk_file'] if not pubkey and pk_file: with salt.utils.files.fopen(pk_file, 'rb') as keyf: - pubkey = str(keyf.read()).rstrip('\n') + pubkey = six.text_type(keyf.read()).rstrip('\n') if pubkey is None: raise Exception('no pubkey or pk_file found') - pubkey = str(pubkey) + pubkey = six.text_type(pubkey) return base64.b64decode(pubkey) @@ -283,7 +285,7 @@ def keygen(sk_file=None, pk_file=None): if os.path.isfile(sk_file) and not os.path.isfile(pk_file): # generate pk using the sk with salt.utils.files.fopen(sk_file, 'rb') as keyf: - sk = str(keyf.read()).rstrip('\n') + sk = six.text_type(keyf.read()).rstrip('\n') sk = base64.b64decode(sk) kp = libnacl.public.SecretKey(sk) with salt.utils.files.fopen(pk_file, 'w') as keyf: diff --git a/salt/runners/net.py b/salt/runners/net.py index 060d44dcab..18afbf1b66 100644 --- a/salt/runners/net.py +++ b/salt/runners/net.py @@ -68,10 +68,7 @@ Configuration - fxp0 outputter: yaml ''' - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals # Import salt lib import salt.output @@ -340,7 +337,7 @@ def interfaces(device=None, if device: title += ' on device {0}'.format(device) if ipnet: - title += ' that include network {net}'.format(net=str(ipnet)) + title += ' that include network {net}'.format(net=six.text_type(ipnet)) if best: title += ' - only best match returned' @@ -403,7 +400,7 @@ def interfaces(device=None, interf_entry['ips'] = '\n'.join(interf_entry['ips']) if ipnet: inet_ips = [ - str(ip) for ip in ips if _ipnet_belongs(ip) + six.text_type(ip) for ip in ips if _ipnet_belongs(ip) ] # filter and get only IP include ipnet if inet_ips: # if any if best: @@ -600,7 +597,7 @@ def findmac(device=None, mac=None, interface=None, vlan=None, display=_DEFAULT_D napalm_helpers.convert(napalm_helpers.mac, mac_entry.get('mac', '')) == napalm_helpers.convert(napalm_helpers.mac, mac)) or (interface and interface in mac_entry.get('interface', '')) or - (vlan and str(mac_entry.get('vlan', '')) == str(vlan))): + (vlan and six.text_type(mac_entry.get('vlan', '')) == six.text_type(vlan))): rows.append({ 'device': device, 'interface': mac_entry.get('interface'), diff --git a/salt/runners/network.py b/salt/runners/network.py index ed81446989..b4bd170749 100644 --- a/salt/runners/network.py +++ b/salt/runners/network.py @@ -4,14 +4,14 @@ Network tools to run from the Master ''' # Import python libs -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import socket # Import salt libs import salt.utils.files import salt.utils.network +import salt.utils.stringutils log = logging.getLogger(__name__) @@ -33,8 +33,9 @@ def wollist(maclist, bcast='255.255.255.255', destport=9): try: with salt.utils.files.fopen(maclist, 'r') as ifile: for mac in ifile: - wol(mac.strip(), bcast, destport) - print('Waking up {0}'.format(mac.strip())) + mac = salt.utils.stringutils.to_unicode(mac).strip() + wol(mac, bcast, destport) + print('Waking up {0}'.format(mac)) ret.append(mac) except Exception as err: __jid_event__.fire_event({'error': 'Failed to open the MAC file. Error: {0}'.format(err)}, 'progress') @@ -78,7 +79,8 @@ def wolmatch(tgt, tgt_type='glob', bcast='255.255.255.255', destport=9): for iface, mac in minion['hwaddr_interfaces'].items(): if iface == 'lo': continue + mac = mac.strip() wol(mac, bcast, destport) - log.info('Waking up {0}'.format(mac.strip())) + log.info('Waking up %s', mac) ret.append(mac) return ret diff --git a/salt/runners/pagerduty.py b/salt/runners/pagerduty.py index b0791f0ee5..c3bb94d8f8 100644 --- a/salt/runners/pagerduty.py +++ b/salt/runners/pagerduty.py @@ -15,7 +15,7 @@ Runner Module for Firing Events via PagerDuty pagerduty.api_key: F3Rbyjbve43rfFWf2214 pagerduty.subdomain: mysubdomain ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import yaml @@ -24,7 +24,7 @@ import json # Import salt libs import salt.utils.functools import salt.utils.pagerduty -from salt.ext.six import string_types +from salt.ext import six def __virtual__(): @@ -175,9 +175,9 @@ def create_event(service_key=None, description=None, details=None, ''' trigger_url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json' - if isinstance(details, string_types): + if isinstance(details, six.string_types): details = yaml.safe_load(details) - if isinstance(details, string_types): + if isinstance(details, six.string_types): details = {'details': details} ret = json.loads(salt.utils.pagerduty.query( diff --git a/salt/runners/pillar.py b/salt/runners/pillar.py index 7d9ae52efa..adbcf4361c 100644 --- a/salt/runners/pillar.py +++ b/salt/runners/pillar.py @@ -2,7 +2,7 @@ ''' Functions to interact with the pillar compiler on the master ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs import salt.pillar diff --git a/salt/runners/pkg.py b/salt/runners/pkg.py index cbb918726e..fc3d1359b9 100644 --- a/salt/runners/pkg.py +++ b/salt/runners/pkg.py @@ -6,7 +6,7 @@ Package helper functions using ``salt.modules.pkg`` ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs import salt.output diff --git a/salt/runners/queue.py b/salt/runners/queue.py index 1aa1a5fb97..3440335b7f 100644 --- a/salt/runners/queue.py +++ b/salt/runners/queue.py @@ -63,8 +63,7 @@ still running from the last time the process_runner task was executed. ''' # Import python libs -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs import salt.loader diff --git a/salt/runners/reactor.py b/salt/runners/reactor.py index ba21c39579..556e4c6ee8 100644 --- a/salt/runners/reactor.py +++ b/salt/runners/reactor.py @@ -3,7 +3,7 @@ A convenience system to manage reactors ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import logging # Import salt libs diff --git a/salt/runners/salt.py b/salt/runners/salt.py index 1cbae0ac27..4930171347 100644 --- a/salt/runners/salt.py +++ b/salt/runners/salt.py @@ -30,8 +30,7 @@ Execution modules are also available to salt runners: ''' # import python libs -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function, unicode_literals import logging # import salt libs @@ -121,9 +120,7 @@ def execute(tgt, kwarg=kwarg, **kwargs) except SaltClientError as client_error: - log.error('Error while executing {fun} on {tgt} ({tgt_type})'.format(fun=fun, - tgt=tgt, - tgt_type=tgt_type)) + log.error('Error while executing %s on %s (%s)', fun, tgt, tgt_type) log.error(client_error) return {} return ret diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index 1078fd21de..883cc4d1e5 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -6,7 +6,7 @@ managing updates to minions. .. versionadded:: 2016.3.0 ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/runners/sdb.py b/salt/runners/sdb.py index d8a8f37adc..3a2a63e694 100644 --- a/salt/runners/sdb.py +++ b/salt/runners/sdb.py @@ -2,7 +2,7 @@ ''' Runner for setting and querying data via the sdb API on the master ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs import salt.utils.sdb diff --git a/salt/runners/smartos_vmadm.py b/salt/runners/smartos_vmadm.py index 15f8cec6f8..e7091d9f16 100644 --- a/salt/runners/smartos_vmadm.py +++ b/salt/runners/smartos_vmadm.py @@ -3,8 +3,7 @@ Runner for SmartOS minions control vmadm ''' # Import python libs -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs import salt.client diff --git a/salt/runners/spacewalk.py b/salt/runners/spacewalk.py index cb9495934a..b3602abc55 100644 --- a/salt/runners/spacewalk.py +++ b/salt/runners/spacewalk.py @@ -29,7 +29,7 @@ master configuration at ``/etc/salt/master`` or ``/etc/salt/master.d/spacewalk.c not using the defaults. Default is ``protocol: https``. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import atexit @@ -70,7 +70,7 @@ def _get_spacewalk_configuration(spacewalk_url=''): if not username or not password: log.error( 'Username or Password has not been specified in the master ' - 'configuration for {0}'.format(spacewalk_server) + 'configuration for %s', spacewalk_server ) return False @@ -83,15 +83,13 @@ def _get_spacewalk_configuration(spacewalk_url=''): if (not spacewalk_url) or (spacewalk_url == spacewalk_server): return ret except Exception as exc: - log.error( - 'Exception encountered: {0}'.format(exc) - ) + log.error('Exception encountered: %s', exc) return False if spacewalk_url: log.error( - 'Configuration for {0} has not been specified in the master ' - 'configuration'.format(spacewalk_url) + 'Configuration for %s has not been specified in the master ' + 'configuration', spacewalk_url ) return False diff --git a/salt/runners/ssh.py b/salt/runners/ssh.py index a35e924555..a39fc83f74 100644 --- a/salt/runners/ssh.py +++ b/salt/runners/ssh.py @@ -6,7 +6,7 @@ This allows for programmatic use from salt-api, the Reactor, Orchestrate, etc. ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Libs import salt.client.ssh.client diff --git a/salt/runners/state.py b/salt/runners/state.py index f4f5d9e5c7..a0e65a49df 100644 --- a/salt/runners/state.py +++ b/salt/runners/state.py @@ -3,7 +3,7 @@ Execute orchestration functions ''' # Import pytohn libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import logging # Import salt libs diff --git a/salt/runners/survey.py b/salt/runners/survey.py index d7337352f9..9b2354a429 100644 --- a/salt/runners/survey.py +++ b/salt/runners/survey.py @@ -13,7 +13,7 @@ when identifying discrepancies in a large infrastructure managed by salt. ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs import salt.client @@ -101,7 +101,7 @@ def diff(*args, **kwargs): print(k['pool']) print('pool size :\n' '----------') - print(' ' + str(len(k['pool']))) + print(' ' + six.text_type(len(k['pool']))) if is_first_time: is_first_time = False print('pool result :\n' @@ -171,11 +171,11 @@ def _get_pool_results(*args, **kwargs): # hash minion return values as a string for minion in sorted(minions): - digest = hashlib.sha256(str(minions[minion]).encode(__salt_system_encoding__)).hexdigest() + digest = hashlib.sha256(six.text_type(minions[minion]).encode(__salt_system_encoding__)).hexdigest() if digest not in ret: ret[digest] = {} ret[digest]['pool'] = [] - ret[digest]['result'] = str(minions[minion]) + ret[digest]['result'] = six.text_type(minions[minion]) ret[digest]['pool'].append(minion) diff --git a/salt/runners/test.py b/salt/runners/test.py index d5c9c7c465..229c28e82c 100644 --- a/salt/runners/test.py +++ b/salt/runners/test.py @@ -2,8 +2,7 @@ ''' This runner is used only for test purposes and servers no production purpose ''' -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import time from salt.ext import six diff --git a/salt/runners/thin.py b/salt/runners/thin.py index c1172965a6..12b3954a71 100644 --- a/salt/runners/thin.py +++ b/salt/runners/thin.py @@ -8,8 +8,7 @@ system for easy consumption. ''' # Import python libs -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt libs import salt.utils.thin diff --git a/salt/runners/vault.py b/salt/runners/vault.py index 7a3cfe85db..97fa4cd675 100644 --- a/salt/runners/vault.py +++ b/salt/runners/vault.py @@ -7,16 +7,19 @@ Runner functions supporting the Vault modules. Configuration instructions are documented in the execution module docs. ''' - -from __future__ import absolute_import +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals import base64 import logging import string import requests +# Import Salt libs import salt.crypt import salt.exceptions +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -36,8 +39,10 @@ def generate_token(minion_id, signature, impersonated_by_master=False): If the master needs to create a token on behalf of the minion, this is True. This happens when the master generates minion pillars. ''' - log.debug('Token generation request for {0} (impersonated by master: {1})'. - format(minion_id, impersonated_by_master)) + log.debug( + 'Token generation request for %s (impersonated by master: %s)'. + minion_id, impersonated_by_master + ) _validate_signature(minion_id, signature, impersonated_by_master) try: @@ -83,7 +88,7 @@ def generate_token(minion_id, signature, impersonated_by_master=False): 'verify': verify, } except Exception as e: - return {'error': str(e)} + return {'error': six.text_type(e)} def show_policies(minion_id): @@ -114,7 +119,7 @@ def _validate_signature(minion_id, signature, impersonated_by_master): else: public_key = '{0}/minions/{1}'.format(pki_dir, minion_id) - log.trace('Validating signature for {0}'.format(minion_id)) + log.trace('Validating signature for %s', minion_id) signature = base64.b64decode(signature) if not salt.crypt.verify_signature(public_key, minion_id, signature): raise salt.exceptions.AuthenticationError( @@ -143,9 +148,9 @@ def _get_policies(minion_id, config): .lower() # Vault requirement ) except KeyError: - log.warning('Could not resolve policy pattern {0}'.format(pattern)) + log.warning('Could not resolve policy pattern %s', pattern) - log.debug('{0} policies: {1}'.format(minion_id, policies)) + log.debug('%s policies: %s', minion_id, policies) return policies @@ -191,7 +196,7 @@ def _expand_pattern_lists(pattern, **mappings): (value, _) = f.get_field(field_name, None, mappings) if isinstance(value, list): token = '{{{0}}}'.format(field_name) - expanded = [pattern.replace(token, str(elem)) for elem in value] + expanded = [pattern.replace(token, six.text_type(elem)) for elem in value] for expanded_item in expanded: result = _expand_pattern_lists(expanded_item, **mappings) expanded_patterns += result @@ -215,5 +220,5 @@ def _selftoken_expired(): return False except Exception as e: raise salt.exceptions.CommandExecutionError( - 'Error while looking up self token : {0}'.format(str(e)) + 'Error while looking up self token : {0}'.format(six.text_type(e)) ) diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 887b3e5bd0..95b52aad0a 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -29,18 +29,23 @@ file and set the ``api_key`` to it: venafi: api_key: abcdef01-2345-6789-abcd-ef0123456789 ''' -from __future__ import absolute_import -import os +from __future__ import absolute_import, print_function, unicode_literals +import json import logging +import os import tempfile from Crypto.PublicKey import RSA -import json -import salt.syspaths as syspaths + +# Import Salt libs import salt.cache +import salt.syspaths as syspaths import salt.utils.files -from salt.ext import six +import salt.utils.stringutils from salt.exceptions import CommandExecutionError +# Import 3rd-party libs +from salt.ext import six + __virtualname__ = 'venafi' log = logging.getLogger(__name__) @@ -189,7 +194,7 @@ def gen_csr( tmppriv = '{0}/priv'.format(tmpdir) tmpcsr = '{0}/csr'.format(tmpdir) with salt.utils.files.fopen(tmppriv, 'w') as if_: - if_.write(data['private_key']) + if_.write(salt.utils.stringutils.to_str(data['private_key'])) if country is None: country = __opts__.get('venafi', {}).get('country') @@ -233,7 +238,7 @@ def gen_csr( ) with salt.utils.files.fopen(tmpcsr, 'r') as of_: - csr = of_.read() + csr = salt.utils.stringutils.to_unicode(of_.read()) data['minion_id'] = minion_id data['csr'] = csr @@ -386,7 +391,7 @@ def register(email): }, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -413,7 +418,7 @@ def show_company(domain): }, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -440,7 +445,7 @@ def show_csrs(): }, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -468,7 +473,7 @@ def get_zone_id(zone_name): ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -495,7 +500,7 @@ def show_policies(): }, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -522,7 +527,7 @@ def show_zones(): }, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -550,7 +555,7 @@ def show_cert(id_): header_dict={'tppl-api-key': _api_key()}, ) status = data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(data['error']) ) @@ -563,7 +568,7 @@ def show_cert(id_): header_dict={'tppl-api-key': _api_key()}, ) status = csr_data['status'] - if str(status).startswith('4') or str(status).startswith('5'): + if six.text_type(status).startswith('4') or six.text_type(status).startswith('5'): raise CommandExecutionError( 'There was an API error: {0}'.format(csr_data['error']) ) diff --git a/salt/runners/virt.py b/salt/runners/virt.py index 6a9822d5b7..2400e7fe0d 100644 --- a/salt/runners/virt.py +++ b/salt/runners/virt.py @@ -4,15 +4,16 @@ Control virtual machines via Salt ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import os.path import logging # Import Salt libs import salt.client -import salt.utils.files -import salt.utils.cloud import salt.key +import salt.utils.cloud +import salt.utils.files +import salt.utils.stringutils from salt.exceptions import SaltClientError # Import 3rd-party libs @@ -246,7 +247,7 @@ def init( priv_key, pub_key = salt.utils.cloud.gen_keys() accepted_key = os.path.join(__opts__['pki_dir'], 'minions', name) with salt.utils.files.fopen(accepted_key, 'w') as fp_: - fp_.write(pub_key) + fp_.write(salt.utils.stringutils.to_str(pub_key)) client = salt.client.get_local_client(__opts__['conf_file']) @@ -405,7 +406,7 @@ def purge(name, delete_key=True): ret.update(comp) if delete_key: - log.debug('Deleting key {0}'.format(name)) + log.debug('Deleting key %s', name) skey = salt.key.Key(__opts__) skey.delete_key(name) __jid_event__.fire_event({'message': 'Purged VM {0}'.format(name)}, 'progress') diff --git a/salt/runners/vistara.py b/salt/runners/vistara.py index c56cd2636b..609d1ee122 100644 --- a/salt/runners/vistara.py +++ b/salt/runners/vistara.py @@ -20,7 +20,7 @@ For example ``/etc/salt/master.d/_vistara.conf``: ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import json diff --git a/salt/runners/winrepo.py b/salt/runners/winrepo.py index e43523a4be..716ba30e9b 100644 --- a/salt/runners/winrepo.py +++ b/salt/runners/winrepo.py @@ -7,7 +7,7 @@ Runner to manage Windows software repo # salt/modules/win_repo.py # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import os # Import third party libs @@ -79,37 +79,32 @@ def genrepo(opts=None, fire_event=True): ) except SaltRenderError as exc: log.debug( - 'Failed to render {0}.'.format( - os.path.join(root, name) - ) + 'Failed to render %s.', + os.path.join(root, name) ) - log.debug('Error: {0}.'.format(exc)) + log.debug('Error: %s.', exc) continue if config: revmap = {} for pkgname, versions in six.iteritems(config): log.debug( - 'Compiling winrepo data for package \'{0}\'' - .format(pkgname) + 'Compiling winrepo data for package \'%s\'', + pkgname ) for version, repodata in six.iteritems(versions): log.debug( - 'Compiling winrepo data for {0} version {1}' - .format(pkgname, version) + 'Compiling winrepo data for %s version %s', + pkgname, version ) if not isinstance(version, six.string_types): - config[pkgname][str(version)] = \ + config[pkgname][six.text_type(version)] = \ config[pkgname].pop(version) if not isinstance(repodata, dict): - log.debug( - 'Failed to compile {0}.'.format( - os.path.join(root, name) - ) + msg = 'Failed to compile {0}.'.format( + os.path.join(root, name) ) + log.debug(msg) if fire_event: - msg = 'Failed to compile {0}.'.format( - os.path.join(root, name) - ) try: __jid_event__.fire_event( {'error': msg}, @@ -119,8 +114,8 @@ def genrepo(opts=None, fire_event=True): log.error( 'Attempted to fire the an event ' 'with the following error, but ' - 'event firing is not supported: ' - '{0}'.format(msg) + 'event firing is not supported: %s', + msg ) continue revmap[repodata['full_name']] = pkgname diff --git a/tests/integration/runners/test_cache.py b/tests/integration/runners/test_cache.py index 72105a0c41..064e887b05 100644 --- a/tests/integration/runners/test_cache.py +++ b/tests/integration/runners/test_cache.py @@ -3,7 +3,7 @@ Tests for the salt-run command ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.case import ShellCase diff --git a/tests/integration/runners/test_fileserver.py b/tests/integration/runners/test_fileserver.py index 5f18dcbf5e..86dfc94ad6 100644 --- a/tests/integration/runners/test_fileserver.py +++ b/tests/integration/runners/test_fileserver.py @@ -3,7 +3,7 @@ Tests for the fileserver runner ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import contextlib # Import Salt Testing libs diff --git a/tests/integration/runners/test_jobs.py b/tests/integration/runners/test_jobs.py index d5482f1699..e9d6b19cf1 100644 --- a/tests/integration/runners/test_jobs.py +++ b/tests/integration/runners/test_jobs.py @@ -3,7 +3,7 @@ Tests for the salt-run command ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.case import ShellCase diff --git a/tests/integration/runners/test_manage.py b/tests/integration/runners/test_manage.py index d562797f29..2aa03dc19b 100644 --- a/tests/integration/runners/test_manage.py +++ b/tests/integration/runners/test_manage.py @@ -3,7 +3,7 @@ Tests for the salt-run command ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.case import ShellCase diff --git a/tests/integration/runners/test_runner_returns.py b/tests/integration/runners/test_runner_returns.py index 86820a9668..cf45b5b7bb 100644 --- a/tests/integration/runners/test_runner_returns.py +++ b/tests/integration/runners/test_runner_returns.py @@ -3,7 +3,7 @@ Tests for runner_returns ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import errno import os import tempfile diff --git a/tests/integration/runners/test_salt.py b/tests/integration/runners/test_salt.py index f1fa637d41..37488f9a48 100644 --- a/tests/integration/runners/test_salt.py +++ b/tests/integration/runners/test_salt.py @@ -5,7 +5,7 @@ Tests for the salt runner .. versionadded:: 2016.11.0 ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.case import ShellCase diff --git a/tests/integration/runners/test_state.py b/tests/integration/runners/test_state.py index 9ed20b6761..d41c2ea515 100644 --- a/tests/integration/runners/test_state.py +++ b/tests/integration/runners/test_state.py @@ -4,7 +4,7 @@ Tests for the state runner ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import errno import json import os @@ -25,6 +25,10 @@ from tests.support.paths import TMP import salt.utils.platform import salt.utils.event import salt.utils.files +import salt.utils.stringutils + +# Import 3rd-party libs +from salt.ext import six class StateRunnerTest(ShellCase): @@ -92,12 +96,12 @@ class StateRunnerTest(ShellCase): self.run_run('saltutil.sync_modules') ret = json.loads( '\n'.join( - self.run_run(u'state.orchestrate orch.issue43204 --out=json') + self.run_run('state.orchestrate orch.issue43204 --out=json') ) ) # Drill down to the changes dict - state_ret = ret[u'data'][u'master'][u'salt_|-Step01_|-Step01_|-state'][u'changes'] - func_ret = ret[u'data'][u'master'][u'salt_|-Step02_|-runtests_helpers.nonzero_retcode_return_false_|-function'][u'changes'] + state_ret = ret['data']['master']['salt_|-Step01_|-Step01_|-state']['changes'] + func_ret = ret['data']['master']['salt_|-Step02_|-runtests_helpers.nonzero_retcode_return_false_|-function']['changes'] # Remove duration and start time from the results, since they would # vary with each run and that would make it impossible to test. @@ -107,22 +111,22 @@ class StateRunnerTest(ShellCase): self.assertEqual( state_ret, { - u'out': u'highstate', - u'ret': { - u'minion': { - u'test_|-test fail with changes_|-test fail with changes_|-fail_with_changes': { - u'__id__': u'test fail with changes', - u'__run_num__': 0, - u'__sls__': u'orch.issue43204.fail_with_changes', - u'changes': { - u'testing': { - u'new': u'Something pretended to change', - u'old': u'Unchanged' + 'out': 'highstate', + 'ret': { + 'minion': { + 'test_|-test fail with changes_|-test fail with changes_|-fail_with_changes': { + '__id__': 'test fail with changes', + '__run_num__': 0, + '__sls__': 'orch.issue43204.fail_with_changes', + 'changes': { + 'testing': { + 'new': 'Something pretended to change', + 'old': 'Unchanged' } }, - u'comment': u'Failure!', - u'name': u'test fail with changes', - u'result': False, + 'comment': 'Failure!', + 'name': 'test fail with changes', + 'result': False, } } } @@ -131,7 +135,7 @@ class StateRunnerTest(ShellCase): self.assertEqual( func_ret, - {u'out': u'highstate', u'ret': {u'minion': False}} + {'out': 'highstate', 'ret': {'minion': False}} ) def test_orchestrate_target_exists(self): @@ -230,7 +234,7 @@ class StateRunnerTest(ShellCase): while q.empty(): self.run_salt('minion test.ping --static') out = q.get() - self.assertIn(expect, str(out)) + self.assertIn(expect, six.text_type(out)) server_thread.join() @@ -288,14 +292,14 @@ class OrchEventTest(ShellCase): state_sls = os.path.join(self.base_env, 'test_state.sls') with salt.utils.files.fopen(state_sls, 'w') as fp_: - fp_.write(textwrap.dedent(''' + fp_.write(salt.utils.stringutils.to_str(textwrap.dedent(''' date: cmd.run - ''')) + '''))) orch_sls = os.path.join(self.base_env, 'test_orch.sls') with salt.utils.files.fopen(orch_sls, 'w') as fp_: - fp_.write(textwrap.dedent(''' + fp_.write(salt.utils.stringutils.to_str(textwrap.dedent(''' date_cmd: salt.state: - tgt: minion @@ -311,7 +315,7 @@ class OrchEventTest(ShellCase): config.values: salt.wheel - ''')) + '''))) listener = salt.utils.event.get_event( 'master', diff --git a/tests/unit/runners/test_cache.py b/tests/unit/runners/test_cache.py index 442ce41f2c..c493d643be 100644 --- a/tests/unit/runners/test_cache.py +++ b/tests/unit/runners/test_cache.py @@ -4,7 +4,7 @@ unit tests for the cache runner ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin diff --git a/tests/unit/runners/test_jobs.py b/tests/unit/runners/test_jobs.py index dae821c829..0b0b66037d 100644 --- a/tests/unit/runners/test_jobs.py +++ b/tests/unit/runners/test_jobs.py @@ -4,7 +4,7 @@ unit tests for the jobs runner ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin diff --git a/tests/unit/runners/test_queue.py b/tests/unit/runners/test_queue.py index 45ff245bc9..b3a0d1977b 100644 --- a/tests/unit/runners/test_queue.py +++ b/tests/unit/runners/test_queue.py @@ -4,7 +4,7 @@ unit tests for the cache runner ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os # Import Salt Testing Libs diff --git a/tests/unit/runners/test_vault.py b/tests/unit/runners/test_vault.py index e9f751fa99..6a9baa6e97 100644 --- a/tests/unit/runners/test_vault.py +++ b/tests/unit/runners/test_vault.py @@ -4,7 +4,7 @@ Unit tests for the Vault runner ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import Salt Testing Libs @@ -85,9 +85,9 @@ class VaultTest(TestCase, LoaderModuleMockMixin): output = vault._expand_pattern_lists(case, **mappings) # pylint: disable=protected-access diff = set(output).symmetric_difference(set(correct_output)) if len(diff) != 0: - log.debug('Test {0} failed'.format(case)) - log.debug('Expected:\n\t{0}\nGot\n\t{1}'.format(output, correct_output)) - log.debug('Difference:\n\t{0}'.format(diff)) + log.debug('Test %s failed', case) + log.debug('Expected:\n\t%s\nGot\n\t%s', output, correct_output) + log.debug('Difference:\n\t%s', diff) self.assertEqual(output, correct_output) def test_get_policies_for_nonexisting_minions(self): @@ -105,9 +105,9 @@ class VaultTest(TestCase, LoaderModuleMockMixin): output = vault._get_policies(minion_id, test_config) # pylint: disable=protected-access diff = set(output).symmetric_difference(set(correct_output)) if len(diff) != 0: - log.debug('Test {0} failed'.format(case)) - log.debug('Expected:\n\t{0}\nGot\n\t{1}'.format(output, correct_output)) - log.debug('Difference:\n\t{0}'.format(diff)) + log.debug('Test %s failed', case) + log.debug('Expected:\n\t%s\nGot\n\t%s', output, correct_output) + log.debug('Difference:\n\t%s', diff) self.assertEqual(output, correct_output) @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -146,7 +146,7 @@ class VaultTest(TestCase, LoaderModuleMockMixin): output = vault._get_policies('test-minion', test_config) # pylint: disable=protected-access diff = set(output).symmetric_difference(set(correct_output)) if len(diff) != 0: - log.debug('Test {0} failed'.format(case)) - log.debug('Expected:\n\t{0}\nGot\n\t{1}'.format(output, correct_output)) - log.debug('Difference:\n\t{0}'.format(diff)) + log.debug('Test %s failed', case) + log.debug('Expected:\n\t%s\nGot\n\t%s', output, correct_output) + log.debug('Difference:\n\t%s', diff) self.assertEqual(output, correct_output) diff --git a/tests/unit/runners/test_winrepo.py b/tests/unit/runners/test_winrepo.py index 2642de959b..bcb9b6b44e 100644 --- a/tests/unit/runners/test_winrepo.py +++ b/tests/unit/runners/test_winrepo.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import shutil import tempfile @@ -14,15 +14,19 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON # Import salt libs import salt.utils.files +import salt.utils.stringutils import salt.runners.winrepo as winrepo -_WINREPO_SLS = r''' +# Can't use raw string with unicode_literals, since the \u in the uninstaller +# will be interpreted as a unicode code point and the interpreter will raise a +# SyntaxError. +_WINREPO_SLS = ''' winscp_x86: 5.7.5: full_name: 'WinSCP 5.7.5' installer: 'http://heanet.dl.sourceforge.net/project/winscp/WinSCP/5.7.5/winscp575setup.exe' install_flags: '/SP- /verysilent /norestart' - uninstaller: '%PROGRAMFILES%\WinSCP\unins000.exe' + uninstaller: '%PROGRAMFILES%\\WinSCP\\unins000.exe' uninstall_flags: '/verysilent' msiexec: False locale: en_US @@ -31,7 +35,7 @@ winscp_x86: full_name: 'WinSCP 5.7.4' installer: 'http://cznic.dl.sourceforge.net/project/winscp/WinSCP/5.7.4/winscp574setup.exe' install_flags: '/SP- /verysilent /norestart' - uninstaller: '%PROGRAMFILES%\WinSCP\unins000.exe' + uninstaller: '%PROGRAMFILES%\\WinSCP\\unins000.exe' uninstall_flags: '/verysilent' msiexec: False locale: en_US @@ -103,5 +107,5 @@ class WinrepoTest(TestCase, LoaderModuleMockMixin): sls_file = os.path.join(self.winrepo_sls_dir, 'wireshark.sls') # Add a winrepo SLS file with salt.utils.files.fopen(sls_file, 'w') as fp_: - fp_.write(_WINREPO_SLS) + fp_.write(salt.utils.stringutils.to_str(_WINREPO_SLS)) self.assertEqual(winrepo.genrepo(), _WINREPO_GENREPO_DATA) From 7147ac69ac9989a57f04f8507c472fccd20a5a03 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 14 Dec 2017 21:56:49 -0600 Subject: [PATCH 49/54] Add unicode_literals to tops, thorium, proxy, tokens --- salt/modules/dummyproxy_package.py | 7 ++-- salt/modules/dummyproxy_service.py | 2 +- salt/proxy/chronos.py | 2 +- salt/proxy/cimc.py | 2 +- salt/proxy/cisconso.py | 2 +- salt/proxy/dummy.py | 4 +-- salt/proxy/esxcluster.py | 11 +++---- salt/proxy/esxdatacenter.py | 11 +++---- salt/proxy/esxi.py | 13 ++++---- salt/proxy/esxvm.py | 9 +++--- salt/proxy/fx2.py | 2 +- salt/proxy/junos.py | 4 +-- salt/proxy/marathon.py | 2 +- salt/proxy/napalm.py | 26 +++++++-------- salt/proxy/nxos.py | 2 +- salt/proxy/panos.py | 2 +- salt/proxy/philips_hue.py | 11 ++++--- salt/proxy/rest_sample.py | 2 +- salt/proxy/ssh_sample.py | 2 +- salt/proxy/vcenter.py | 11 +++---- salt/thorium/__init__.py | 2 +- salt/thorium/calc.py | 2 +- salt/thorium/check.py | 4 +-- salt/thorium/file.py | 2 +- salt/thorium/key.py | 2 +- salt/thorium/local.py | 2 +- salt/thorium/reg.py | 2 +- salt/thorium/runner.py | 2 +- salt/thorium/status.py | 2 +- salt/thorium/timer.py | 2 +- salt/thorium/wheel.py | 2 +- salt/tokens/localfs.py | 16 ++++++---- salt/tokens/rediscluster.py | 32 +++++++++++++------ salt/tops/cobbler.py | 2 +- salt/tops/ext_nodes.py | 2 +- salt/tops/mongo.py | 20 ++++-------- salt/tops/reclass_adapter.py | 13 ++++---- salt/tops/saltclass.py | 4 +-- salt/tops/varstack.py | 2 +- .../tops/master_tops_test.py | 2 +- tests/integration/proxy/test_simple.py | 2 +- tests/integration/shell/test_master_tops.py | 2 +- tests/integration/shell/test_proxy.py | 2 +- tests/unit/proxy/test_esxcluster.py | 2 +- tests/unit/proxy/test_esxdatacenter.py | 2 +- tests/unit/proxy/test_napalm.py | 2 +- 46 files changed, 131 insertions(+), 125 deletions(-) diff --git a/salt/modules/dummyproxy_package.py b/salt/modules/dummyproxy_package.py index b6e7219cbe..1813bbd9cc 100644 --- a/salt/modules/dummyproxy_package.py +++ b/salt/modules/dummyproxy_package.py @@ -2,12 +2,13 @@ ''' Package support for the dummy proxy used by the test suite ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging import salt.utils.data import salt.utils.platform +from salt.ext import six log = logging.getLogger(__name__) @@ -96,9 +97,9 @@ def installed(name, p = __proxy__['dummy.package_status'](name) if version is None: if 'ret' in p: - return str(p['ret']) + return six.text_type(p['ret']) else: return True else: if p is not None: - return version == str(p) + return version == six.text_type(p) diff --git a/salt/modules/dummyproxy_service.py b/salt/modules/dummyproxy_service.py index 1be19ff582..30794be81c 100644 --- a/salt/modules/dummyproxy_service.py +++ b/salt/modules/dummyproxy_service.py @@ -4,7 +4,7 @@ Provide the service module for the dummy proxy used in integration tests ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import Salt libs diff --git a/salt/proxy/chronos.py b/salt/proxy/chronos.py index 3fbdc49ee8..418ef32be9 100644 --- a/salt/proxy/chronos.py +++ b/salt/proxy/chronos.py @@ -24,7 +24,7 @@ the chronos endpoint: .. versionadded:: 2015.8.2 ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import salt.utils.http diff --git a/salt/proxy/cimc.py b/salt/proxy/cimc.py index 4692a8ef31..56254157d5 100644 --- a/salt/proxy/cimc.py +++ b/salt/proxy/cimc.py @@ -60,7 +60,7 @@ The password used to login to the cimc host. Required. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python Libs import logging diff --git a/salt/proxy/cisconso.py b/salt/proxy/cisconso.py index 5b30452602..4bc94b4c04 100644 --- a/salt/proxy/cisconso.py +++ b/salt/proxy/cisconso.py @@ -173,7 +173,7 @@ responding: ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import Salt Libs diff --git a/salt/proxy/dummy.py b/salt/proxy/dummy.py index 8d7d1fde10..d4e5f3ad6f 100644 --- a/salt/proxy/dummy.py +++ b/salt/proxy/dummy.py @@ -2,7 +2,7 @@ ''' This is a dummy proxy-minion designed for testing the proxy minion subsystem. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import os @@ -194,7 +194,7 @@ def uptodate(): for p in DETAILS['packages']: version_float = float(DETAILS['packages'][p]) version_float = version_float + 1.0 - DETAILS['packages'][p] = str(version_float) + DETAILS['packages'][p] = six.text_type(version_float) return DETAILS['packages'] diff --git a/salt/proxy/esxcluster.py b/salt/proxy/esxcluster.py index af3740d8d5..19a8bc4cf5 100644 --- a/salt/proxy/esxcluster.py +++ b/salt/proxy/esxcluster.py @@ -153,7 +153,7 @@ Look there to find an example structure for Pillar as well as an example # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os @@ -200,13 +200,12 @@ def init(opts): login the protocol and port are cached. ''' - log.debug('Initting esxcluster proxy module in process ' - '{}'.format(os.getpid())) + log.debug('Initting esxcluster proxy module in process %s', os.getpid()) log.debug('Validating esxcluster proxy input') schema = EsxclusterProxySchema.serialize() - log.trace('schema = {}'.format(schema)) + log.trace('schema = %s', schema) proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {})) - log.trace('proxy_conf = {0}'.format(proxy_conf)) + log.trace('proxy_conf = %s', proxy_conf) try: jsonschema.validate(proxy_conf, schema) except jsonschema.exceptions.ValidationError as exc: @@ -253,7 +252,7 @@ def init(opts): username, password = find_credentials() DETAILS['password'] = password except salt.exceptions.SaltSystemExit as err: - log.critical('Error: {0}'.format(err)) + log.critical('Error: %s', err) return False return True diff --git a/salt/proxy/esxdatacenter.py b/salt/proxy/esxdatacenter.py index 5460863d84..b360ce60f5 100644 --- a/salt/proxy/esxdatacenter.py +++ b/salt/proxy/esxdatacenter.py @@ -146,7 +146,7 @@ Look there to find an example structure for Pillar as well as an example ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os @@ -191,13 +191,12 @@ def init(opts): This function gets called when the proxy starts up. All login details are cached. ''' - log.debug('Initting esxdatacenter proxy module in process ' - '{}'.format(os.getpid())) + log.debug('Initting esxdatacenter proxy module in process %s', os.getpid()) log.trace('Validating esxdatacenter proxy input') schema = EsxdatacenterProxySchema.serialize() - log.trace('schema = {}'.format(schema)) + log.trace('schema = %s', schema) proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {})) - log.trace('proxy_conf = {0}'.format(proxy_conf)) + log.trace('proxy_conf = %s', proxy_conf) try: jsonschema.validate(proxy_conf, schema) except jsonschema.exceptions.ValidationError as exc: @@ -244,7 +243,7 @@ def init(opts): username, password = find_credentials() DETAILS['password'] = password except salt.exceptions.SaltSystemExit as err: - log.critical('Error: {0}'.format(err)) + log.critical('Error: %s', err) return False return True diff --git a/salt/proxy/esxi.py b/salt/proxy/esxi.py index 1599c381c6..bf2ecf43d6 100644 --- a/salt/proxy/esxi.py +++ b/salt/proxy/esxi.py @@ -271,7 +271,7 @@ for standing up an ESXi host from scratch. ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os @@ -317,13 +317,12 @@ def init(opts): ESXi devices, the host, login credentials, and, if configured, the protocol and port are cached. ''' - log.debug('Initting esxi proxy module in process \'{}\'' - ''.format(os.getpid())) + log.debug('Initting esxi proxy module in process %s', os.getpid()) log.debug('Validating esxi proxy input') schema = EsxiProxySchema.serialize() - log.trace('esxi_proxy_schema = {}'.format(schema)) + log.trace('esxi_proxy_schema = %s', schema) proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {})) - log.trace('proxy_conf = {0}'.format(proxy_conf)) + log.trace('proxy_conf = %s', proxy_conf) try: jsonschema.validate(proxy_conf, schema) except jsonschema.exceptions.ValidationError as exc: @@ -348,7 +347,7 @@ def init(opts): try: username, password = find_credentials(host) except SaltSystemExit as err: - log.critical('Error: {0}'.format(err)) + log.critical('Error: %s', err) return False # Set configuration details @@ -408,7 +407,7 @@ def init(opts): username, password = find_credentials(DETAILS['vcenter']) DETAILS['password'] = password except SaltSystemExit as err: - log.critical('Error: {0}'.format(err)) + log.critical('Error: %s', err) return False # Save optional diff --git a/salt/proxy/esxvm.py b/salt/proxy/esxvm.py index 24f5b1736a..cbc8ca5f6b 100644 --- a/salt/proxy/esxvm.py +++ b/salt/proxy/esxvm.py @@ -146,7 +146,7 @@ Look there to find an example structure for Pillar as well as an example ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os @@ -182,11 +182,10 @@ def init(opts): This function gets called when the proxy starts up. For login the protocol and port are cached. ''' - log.debug('Initting esxvm proxy module in process ' - '{}'.format(os.getpid())) + log.debug('Initting esxvm proxy module in process %s', os.getpid()) log.debug('Validating esxvm proxy input') proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {})) - log.trace('proxy_conf = {0}'.format(proxy_conf)) + log.trace('proxy_conf = %s', proxy_conf) # TODO json schema validation # Save mandatory fields in cache @@ -230,7 +229,7 @@ def init(opts): username, password = find_credentials() DETAILS['password'] = password except excs.SaltSystemExit as err: - log.critical('Error: {0}'.format(err)) + log.critical('Error: %s', err) return False return True diff --git a/salt/proxy/fx2.py b/salt/proxy/fx2.py index edfa8a28be..cb6ef9d480 100644 --- a/salt/proxy/fx2.py +++ b/salt/proxy/fx2.py @@ -171,7 +171,7 @@ Look there to find an example structure for pillar as well as an example ``.sls`` file for standing up a Dell Chassis from scratch. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/proxy/junos.py b/salt/proxy/junos.py index e3227bb4ae..8346bca349 100644 --- a/salt/proxy/junos.py +++ b/salt/proxy/junos.py @@ -35,7 +35,7 @@ Run the salt proxy via the following command: ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging @@ -191,7 +191,7 @@ def shutdown(opts): This is called when the proxy-minion is exiting to make sure the connection to the device is closed cleanly. ''' - log.debug('Proxy module {0} shutting down!!'.format(opts['id'])) + log.debug('Proxy module %s shutting down!!', opts['id']) try: thisproxy['conn'].close() diff --git a/salt/proxy/marathon.py b/salt/proxy/marathon.py index ae68f0106d..2f6b97b786 100644 --- a/salt/proxy/marathon.py +++ b/salt/proxy/marathon.py @@ -24,7 +24,7 @@ the marathon endpoint: .. versionadded:: 2015.8.2 ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import salt.utils.http diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index 8a5340180e..f932deca05 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -135,7 +135,7 @@ Example using a user-specific library, extending NAPALM's capabilities, e.g. ``c .. versionadded:: 2016.11.0 ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python lib import logging @@ -196,10 +196,10 @@ def alive(opts): # or regular minion is_alive_ret = call('is_alive', **{}) if not is_alive_ret.get('result', False): - log.debug('[{proxyid}] Unable to execute `is_alive`: {comment}'.format( - proxyid=opts.get('id'), - comment=is_alive_ret.get('comment') - )) + log.debug( + '[%s] Unable to execute `is_alive`: %s', + opts.get('id'), is_alive_ret.get('comment') + ) # if `is_alive` is not implemented by the underneath driver, # will consider the connection to be still alive # we don't want overly request connection reestablishment @@ -207,10 +207,7 @@ def alive(opts): # and return False to force reconnection return True flag = is_alive_ret.get('out', {}).get('is_alive', False) - log.debug('Is {proxyid} still alive? {answ}'.format( - proxyid=opts.get('id'), - answ='Yes.' if flag else 'No.' - )) + log.debug('Is %s still alive? %s', opts.get('id'), 'Yes.' if flag else 'No.') return flag @@ -277,13 +274,12 @@ def shutdown(opts): raise Exception('not connected!') NETWORK_DEVICE.get('DRIVER').close() except Exception as error: + port = NETWORK_DEVICE.get('OPTIONAL_ARGS', {}).get('port') log.error( - 'Cannot close connection with {hostname}{port}! Please check error: {error}'.format( - hostname=NETWORK_DEVICE.get('HOSTNAME', '[unknown hostname]'), - port=(':{port}'.format(port=NETWORK_DEVICE.get('OPTIONAL_ARGS', {}).get('port')) - if NETWORK_DEVICE.get('OPTIONAL_ARGS', {}).get('port') else ''), - error=error - ) + 'Cannot close connection with %s%s! Please check error: %s', + NETWORK_DEVICE.get('HOSTNAME', '[unknown hostname]'), + ':{0}'.format(port) if port else '', + error ) return True diff --git a/salt/proxy/nxos.py b/salt/proxy/nxos.py index d78c796dfd..b29a68fe8c 100644 --- a/salt/proxy/nxos.py +++ b/salt/proxy/nxos.py @@ -55,7 +55,7 @@ the :mod:`salt.modules.nxos` execution module. ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import multiprocessing import re diff --git a/salt/proxy/panos.py b/salt/proxy/panos.py index 86ef275a4f..5e7c9c9689 100644 --- a/salt/proxy/panos.py +++ b/salt/proxy/panos.py @@ -185,7 +185,7 @@ The generated XML API key for the Panorama server. Required. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python Libs import logging diff --git a/salt/proxy/philips_hue.py b/salt/proxy/philips_hue.py index 453193f590..9420963d54 100644 --- a/salt/proxy/philips_hue.py +++ b/salt/proxy/philips_hue.py @@ -21,7 +21,7 @@ Philips HUE lamps module for proxy. ''' # pylint: disable=import-error,no-name-in-module,redefined-builtin -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import salt.ext.six.moves.http_client as http_client # Import python libs @@ -29,6 +29,7 @@ import logging import time import json from salt.exceptions import (CommandExecutionError, MinionError) +from salt.ext import six __proxyenabled__ = ['philips_hue'] @@ -188,8 +189,8 @@ def call_lights(*args, **kwargs): res = dict() lights = _get_lights() for dev_id in 'id' in kwargs and _get_devices(kwargs) or sorted(lights.keys()): - if lights.get(str(dev_id)): - res[dev_id] = lights[str(dev_id)] + if lights.get(six.text_type(dev_id)): + res[dev_id] = lights[six.text_type(dev_id)] return res or False @@ -221,7 +222,7 @@ def call_switch(*args, **kwargs): state = kwargs['on'] and Const.LAMP_ON or Const.LAMP_OFF else: # Invert the current state - state = devices[str(dev_id)]['state']['on'] and Const.LAMP_OFF or Const.LAMP_ON + state = devices[six.text_type(dev_id)]['state']['on'] and Const.LAMP_OFF or Const.LAMP_ON out[dev_id] = _set(dev_id, state) return out @@ -247,7 +248,7 @@ def call_blink(*args, **kwargs): pause = kwargs.get('pause', 0) res = dict() for dev_id in 'id' not in kwargs and sorted(devices.keys()) or _get_devices(kwargs): - state = devices[str(dev_id)]['state']['on'] + state = devices[six.text_type(dev_id)]['state']['on'] _set(dev_id, state and Const.LAMP_OFF or Const.LAMP_ON) if pause: time.sleep(pause) diff --git a/salt/proxy/rest_sample.py b/salt/proxy/rest_sample.py index 4b912473c2..eae18edcdb 100644 --- a/salt/proxy/rest_sample.py +++ b/salt/proxy/rest_sample.py @@ -3,7 +3,7 @@ This is a simple proxy-minion designed to connect to and communicate with the bottle-based web service contained in https://github.com/saltstack/salt-contrib/tree/master/proxyminion_rest_example ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/proxy/ssh_sample.py b/salt/proxy/ssh_sample.py index 9311d6656b..fa8db0c5ce 100644 --- a/salt/proxy/ssh_sample.py +++ b/salt/proxy/ssh_sample.py @@ -5,7 +5,7 @@ This can be used as an option when the device does not provide an api over HTTP and doesn't have the python stack to run a minion. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import json diff --git a/salt/proxy/vcenter.py b/salt/proxy/vcenter.py index 5c5ad797d1..2adcba6ade 100644 --- a/salt/proxy/vcenter.py +++ b/salt/proxy/vcenter.py @@ -183,7 +183,7 @@ and communicate with the ESXi host. ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os @@ -228,13 +228,12 @@ def init(opts): This function gets called when the proxy starts up. For login the protocol and port are cached. ''' - log.info('Initting vcenter proxy module in process {0}' - ''.format(os.getpid())) + log.info('Initting vcenter proxy module in process %s', os.getpid()) log.trace('VCenter Proxy Validating vcenter proxy input') schema = VCenterProxySchema.serialize() - log.trace('schema = {}'.format(schema)) + log.trace('schema = %s', schema) proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {})) - log.trace('proxy_conf = {0}'.format(proxy_conf)) + log.trace('proxy_conf = %s', proxy_conf) try: jsonschema.validate(proxy_conf, schema) except jsonschema.exceptions.ValidationError as exc: @@ -281,7 +280,7 @@ def init(opts): username, password = find_credentials() DETAILS['password'] = password except salt.exceptions.SaltSystemExit as err: - log.critical('Error: {0}'.format(err)) + log.critical('Error: %s', err) return False return True diff --git a/salt/thorium/__init__.py b/salt/thorium/__init__.py index 48bef8580b..d8c0e055b1 100644 --- a/salt/thorium/__init__.py +++ b/salt/thorium/__init__.py @@ -10,7 +10,7 @@ The thorium system allows for advanced event tracking and reactions # Add dynamic recompile of thorium ruleset on given interval # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import time import logging diff --git a/salt/thorium/calc.py b/salt/thorium/calc.py index eb387046a0..dd45428d00 100644 --- a/salt/thorium/calc.py +++ b/salt/thorium/calc.py @@ -9,7 +9,7 @@ values are stored and computed, such as averages etc. ''' # import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals try: import statistics diff --git a/salt/thorium/check.py b/salt/thorium/check.py index d7e656d6aa..ea4e99c12d 100644 --- a/salt/thorium/check.py +++ b/salt/thorium/check.py @@ -6,7 +6,7 @@ succeed or fail based on the state of the register, this creates the pattern of having a command execution get gated by a check state via a requisite. ''' # import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import salt.utils.stringutils @@ -252,7 +252,7 @@ def contains(name, count_gte or count_gt or count_ne if count_compare: occurrences = __reg__[name]['val'].count(value) - log.debug('{} appears {} times'.format(value, occurrences)) + log.debug('%s appears %s times', value, occurrences) ret['result'] = True if count_lt: ret['result'] &= occurrences < count_lt diff --git a/salt/thorium/file.py b/salt/thorium/file.py index 41afec83b7..6520ade46e 100644 --- a/salt/thorium/file.py +++ b/salt/thorium/file.py @@ -39,7 +39,7 @@ that can be re-imported into Python. ''' # import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import json diff --git a/salt/thorium/key.py b/salt/thorium/key.py index 7bd6d0b46a..a4f4800b43 100644 --- a/salt/thorium/key.py +++ b/salt/thorium/key.py @@ -5,7 +5,7 @@ The key Thorium State is used to apply changes to the accepted/rejected/pending .. versionadded:: 2016.11.0 ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import time # Import salt libs diff --git a/salt/thorium/local.py b/salt/thorium/local.py index b2b7997d12..e70a26c835 100644 --- a/salt/thorium/local.py +++ b/salt/thorium/local.py @@ -3,7 +3,7 @@ Run remote execution commands via the local client ''' # import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs import salt.client diff --git a/salt/thorium/reg.py b/salt/thorium/reg.py index c5859a8375..65a022145d 100644 --- a/salt/thorium/reg.py +++ b/salt/thorium/reg.py @@ -5,7 +5,7 @@ values are stored and computed, such as averages etc. ''' # import python libs -from __future__ import absolute_import, division +from __future__ import absolute_import, division, print_function, unicode_literals import salt.utils.stringutils __func_alias__ = { diff --git a/salt/thorium/runner.py b/salt/thorium/runner.py index a013212892..d6235d40e7 100644 --- a/salt/thorium/runner.py +++ b/salt/thorium/runner.py @@ -3,7 +3,7 @@ React by calling async runners ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # import salt libs import salt.runner diff --git a/salt/thorium/status.py b/salt/thorium/status.py index 04811a4795..326ae18aad 100644 --- a/salt/thorium/status.py +++ b/salt/thorium/status.py @@ -6,7 +6,7 @@ the active status of minions .. versionadded:: 2016.11.0 ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import time import fnmatch diff --git a/salt/thorium/timer.py b/salt/thorium/timer.py index 9a0141d26f..a8cc5eeeea 100644 --- a/salt/thorium/timer.py +++ b/salt/thorium/timer.py @@ -4,7 +4,7 @@ Allow for flow based timers. These timers allow for a sleep to exist across multiple runs of the flow ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import time diff --git a/salt/thorium/wheel.py b/salt/thorium/wheel.py index 73e73552b4..7c98eff4bd 100644 --- a/salt/thorium/wheel.py +++ b/salt/thorium/wheel.py @@ -3,7 +3,7 @@ React by calling async runners ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # import salt libs import salt.wheel diff --git a/salt/tokens/localfs.py b/salt/tokens/localfs.py index af6dfe2749..021bdb9e50 100644 --- a/salt/tokens/localfs.py +++ b/salt/tokens/localfs.py @@ -4,7 +4,7 @@ Stores eauth tokens in the filesystem of the master. Location is configured by the master config option 'token_dir' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import hashlib import os @@ -14,6 +14,8 @@ import salt.utils.files import salt.utils.path import salt.payload +from salt.ext import six + log = logging.getLogger(__name__) __virtualname__ = 'localfs' @@ -30,10 +32,10 @@ def mk_token(opts, tdata): :returns: tdata with token if successful. Empty dict if failed. ''' hash_type = getattr(hashlib, opts.get('hash_type', 'md5')) - tok = str(hash_type(os.urandom(512)).hexdigest()) + tok = six.text_type(hash_type(os.urandom(512)).hexdigest()) t_path = os.path.join(opts['token_dir'], tok) while os.path.isfile(t_path): - tok = str(hash_type(os.urandom(512)).hexdigest()) + tok = six.text_type(hash_type(os.urandom(512)).hexdigest()) t_path = os.path.join(opts['token_dir'], tok) tdata['token'] = tok serial = salt.payload.Serial(opts) @@ -42,7 +44,8 @@ def mk_token(opts, tdata): with salt.utils.files.fopen(t_path, 'w+b') as fp_: fp_.write(serial.dumps(tdata)) except (IOError, OSError): - log.warning('Authentication failure: can not write token file "{0}".'.format(t_path)) + log.warning( + 'Authentication failure: can not write token file "%s".', t_path) return {} return tdata @@ -64,7 +67,8 @@ def get_token(opts, tok): tdata = serial.loads(fp_.read()) return tdata except (IOError, OSError): - log.warning('Authentication failure: can not read token file "{0}".'.format(t_path)) + log.warning( + 'Authentication failure: can not read token file "%s".', t_path) return {} @@ -81,7 +85,7 @@ def rm_token(opts, tok): os.remove(t_path) return {} except (IOError, OSError): - log.warning('Could not remove token {0}'.format(tok)) + log.warning('Could not remove token %s', tok) def list_tokens(opts): diff --git a/salt/tokens/rediscluster.py b/salt/tokens/rediscluster.py index 5609884f8c..3af121f329 100644 --- a/salt/tokens/rediscluster.py +++ b/salt/tokens/rediscluster.py @@ -15,7 +15,7 @@ Default values for these configs are as follow: :depends: - redis-py-cluster Python package ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals try: @@ -31,6 +31,8 @@ import hashlib import salt.payload +from salt.ext import six + log = logging.getLogger(__name__) __virtualname__ = 'rediscluster' @@ -53,7 +55,10 @@ def _redis_client(opts): try: return rediscluster.StrictRedisCluster(host=redis_host, port=redis_port) except rediscluster.exceptions.RedisClusterException as err: - log.warning("Failed to connect to redis at {0}:{1} - {2}".format(redis_host, redis_port, err)) + log.warning( + 'Failed to connect to redis at %s:%s - %s', + redis_host, redis_port, err + ) return None @@ -71,19 +76,25 @@ def mk_token(opts, tdata): if not redis_client: return {} hash_type = getattr(hashlib, opts.get('hash_type', 'md5')) - tok = str(hash_type(os.urandom(512)).hexdigest()) + tok = six.text_type(hash_type(os.urandom(512)).hexdigest()) try: while redis_client.get(tok) is not None: - tok = str(hash_type(os.urandom(512)).hexdigest()) + tok = six.text_type(hash_type(os.urandom(512)).hexdigest()) except Exception as err: - log.warning("Authentication failure: cannot get token {0} from redis: {1}".format(tok, err)) + log.warning( + 'Authentication failure: cannot get token %s from redis: %s', + tok, err + ) return {} tdata['token'] = tok serial = salt.payload.Serial(opts) try: redis_client.set(tok, serial.dumps(tdata)) except Exception as err: - log.warning("Authentication failure: cannot save token {0} to redis: {1}".format(tok, err)) + log.warning( + 'Authentication failure: cannot save token %s to redis: %s', + tok, err + ) return {} return tdata @@ -104,7 +115,10 @@ def get_token(opts, tok): tdata = serial.loads(redis_client.get(tok)) return tdata except Exception as err: - log.warning("Authentication failure: cannot get token {0} from redis: {1}".format(tok, err)) + log.warning( + 'Authentication failure: cannot get token %s from redis: %s', + tok, err + ) return {} @@ -123,7 +137,7 @@ def rm_token(opts, tok): redis_client.delete(tok) return {} except Exception as err: - log.warning("Could not remove token {0}: {1}".format(tok, err)) + log.warning('Could not remove token %s: %s', tok, err) def list_tokens(opts): @@ -141,5 +155,5 @@ def list_tokens(opts): try: return [k.decode('utf8') for k in redis_client.keys()] except Exception as err: - log.warning("Failed to list keys: {0}".format(err)) + log.warning('Failed to list keys: %s', err) return [] diff --git a/salt/tops/cobbler.py b/salt/tops/cobbler.py index 0996dc7c95..2303242c26 100644 --- a/salt/tops/cobbler.py +++ b/salt/tops/cobbler.py @@ -19,7 +19,7 @@ the Cobbler tops and Cobbler pillar modules. Module Documentation ==================== ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/tops/ext_nodes.py b/salt/tops/ext_nodes.py index 91b239c121..c71668f1f3 100644 --- a/salt/tops/ext_nodes.py +++ b/salt/tops/ext_nodes.py @@ -45,7 +45,7 @@ The above essentially is the same as a top.sls containing the following: - basepackages - database ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/tops/mongo.py b/salt/tops/mongo.py index fdaef38740..54324ca61d 100644 --- a/salt/tops/mongo.py +++ b/salt/tops/mongo.py @@ -41,7 +41,7 @@ Configuring the Mongo Tops Subsystem Module Documentation ==================== ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -102,17 +102,17 @@ def top(**kwargs): states_field = __opts__['master_tops']['mongo'].get('states_field', 'states') environment_field = __opts__['master_tops']['mongo'].get('environment_field', 'environment') - log.info('connecting to {0}:{1} for mongo ext_tops'.format(host, port)) + log.info('connecting to %s:%s for mongo ext_tops', host, port) conn = pymongo.MongoClient(host, port) - log.debug('using database \'{0}\''.format(__opts__['mongo.db'])) + log.debug('using database \'%s\'', __opts__['mongo.db']) mdb = conn[__opts__['mongo.db']] user = __opts__.get('mongo.user') password = __opts__.get('mongo.password') if user and password: - log.debug('authenticating as \'{0}\''.format(user)) + log.debug('authenticating as \'%s\'', user) mdb.authenticate(user, password) # Do the regex string replacement on the minion id @@ -121,10 +121,8 @@ def top(**kwargs): minion_id = re.sub(re_pattern, re_replace, minion_id) log.info( - 'ext_tops.mongo: looking up tops def for {{\'{0}\': \'{1}\'}} ' - 'in mongo'.format( - id_field, minion_id - ) + 'ext_tops.mongo: looking up tops def for {\'%s\': \'%s\'} in mongo', + id_field, minion_id ) result = mdb[collection].find_one({id_field: minion_id}, projection=[states_field, environment_field]) @@ -138,9 +136,5 @@ def top(**kwargs): else: # If we can't find the minion the database it's not necessarily an # error. - log.debug( - 'ext_tops.mongo: no document found in collection {0}'.format( - collection - ) - ) + log.debug('ext_tops.mongo: no document found in collection %s', collection) return {} diff --git a/salt/tops/reclass_adapter.py b/salt/tops/reclass_adapter.py index c078bd2736..45b861a24e 100644 --- a/salt/tops/reclass_adapter.py +++ b/salt/tops/reclass_adapter.py @@ -46,7 +46,7 @@ If you want to run reclass from source, rather than installing it, you can either let the master know via the ``PYTHONPATH`` environment variable, or by setting the configuration option, like in the example above. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # This file cannot be called reclass.py, because then the module import would # not work. Thanks to the __virtual__ function, however, the plugin still @@ -60,6 +60,7 @@ from salt.utils.reclass import ( ) from salt.exceptions import SaltInvocationError +from salt.ext import six # Define the module's virtual name __virtualname__ = 'reclass' @@ -117,7 +118,7 @@ def top(**kwargs): return reclass_top(minion_id, **reclass_opts) except ImportError as e: - if 'reclass' in str(e): + if 'reclass' in six.text_type(e): raise SaltInvocationError( 'master_tops.reclass: cannot find reclass module ' 'in {0}'.format(sys.path) @@ -126,8 +127,8 @@ def top(**kwargs): raise except TypeError as e: - if 'unexpected keyword argument' in str(e): - arg = str(e).split()[-1] + if 'unexpected keyword argument' in six.text_type(e): + arg = six.text_type(e).split()[-1] raise SaltInvocationError( 'master_tops.reclass: unexpected option: {0}'.format(arg) ) @@ -135,11 +136,11 @@ def top(**kwargs): raise except KeyError as e: - if 'reclass' in str(e): + if 'reclass' in six.text_type(e): raise SaltInvocationError('master_tops.reclass: no configuration ' 'found in master config') else: raise except ReclassException as e: - raise SaltInvocationError('master_tops.reclass: {0}'.format(str(e))) + raise SaltInvocationError('master_tops.reclass: {0}'.format(six.text_type(e))) diff --git a/salt/tops/saltclass.py b/salt/tops/saltclass.py index 585641a024..71188b2615 100644 --- a/salt/tops/saltclass.py +++ b/salt/tops/saltclass.py @@ -9,7 +9,7 @@ SaltClass master_tops Module ''' # import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import salt.utils.saltclass as sc @@ -45,7 +45,7 @@ def top(**kwargs): _opts = __opts__['master_tops']['saltclass'] if 'path' not in _opts: path = '/srv/saltclass' - log.warning('path variable unset, using default: {0}'.format(path)) + log.warning('path variable unset, using default: %s', path) else: path = _opts['path'] diff --git a/salt/tops/varstack.py b/salt/tops/varstack.py index 54d84f3b9b..81dfea2ee4 100644 --- a/salt/tops/varstack.py +++ b/salt/tops/varstack.py @@ -43,7 +43,7 @@ managed by salt as if given from a top.sls file. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/tests/integration/files/extension_modules/tops/master_tops_test.py b/tests/integration/files/extension_modules/tops/master_tops_test.py index 3ee47f5361..27cafec407 100644 --- a/tests/integration/files/extension_modules/tops/master_tops_test.py +++ b/tests/integration/files/extension_modules/tops/master_tops_test.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging diff --git a/tests/integration/proxy/test_simple.py b/tests/integration/proxy/test_simple.py index e1af027cf2..bd1cdcf244 100644 --- a/tests/integration/proxy/test_simple.py +++ b/tests/integration/proxy/test_simple.py @@ -4,7 +4,7 @@ Simple Smoke Tests for Connected Proxy Minion ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.case import ModuleCase diff --git a/tests/integration/shell/test_master_tops.py b/tests/integration/shell/test_master_tops.py index a26de9a4e8..e0439e8a2e 100644 --- a/tests/integration/shell/test_master_tops.py +++ b/tests/integration/shell/test_master_tops.py @@ -5,7 +5,7 @@ ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.case import ShellCase diff --git a/tests/integration/shell/test_proxy.py b/tests/integration/shell/test_proxy.py index c9d28e9a11..ad60de2404 100644 --- a/tests/integration/shell/test_proxy.py +++ b/tests/integration/shell/test_proxy.py @@ -7,7 +7,7 @@ ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import salt tests libs diff --git a/tests/unit/proxy/test_esxcluster.py b/tests/unit/proxy/test_esxcluster.py index dee5da74f4..1a2bc5988b 100644 --- a/tests/unit/proxy/test_esxcluster.py +++ b/tests/unit/proxy/test_esxcluster.py @@ -6,7 +6,7 @@ ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import external libs try: diff --git a/tests/unit/proxy/test_esxdatacenter.py b/tests/unit/proxy/test_esxdatacenter.py index fb9da3d401..98104eb146 100644 --- a/tests/unit/proxy/test_esxdatacenter.py +++ b/tests/unit/proxy/test_esxdatacenter.py @@ -6,7 +6,7 @@ ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import external libs try: diff --git a/tests/unit/proxy/test_napalm.py b/tests/unit/proxy/test_napalm.py index 84c4b1534a..14a806d096 100644 --- a/tests/unit/proxy/test_napalm.py +++ b/tests/unit/proxy/test_napalm.py @@ -4,7 +4,7 @@ ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin From c65d4714e52000260512286d8d33ab40b5161975 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 15 Dec 2017 00:17:26 -0600 Subject: [PATCH 50/54] [PY3] Add unicode_literals to renderers and returners --- salt/renderers/cheetah.py | 8 ++-- salt/renderers/dson.py | 2 +- salt/renderers/genshi.py | 6 +-- salt/renderers/gpg.py | 4 +- salt/renderers/hjson.py | 6 +-- salt/renderers/jinja.py | 4 +- salt/renderers/json.py | 6 +-- salt/renderers/json5.py | 6 +-- salt/renderers/mako.py | 2 +- salt/renderers/msgpack.py | 6 +-- salt/renderers/nacl.py | 2 +- salt/renderers/pass.py | 24 +++++----- salt/renderers/py.py | 2 +- salt/renderers/pydsl.py | 4 +- salt/renderers/pyobjects.py | 10 ++--- salt/renderers/stateconf.py | 19 ++++---- salt/renderers/wempy.py | 2 +- salt/renderers/yaml.py | 9 ++-- salt/renderers/yamlex.py | 9 ++-- salt/returners/__init__.py | 5 ++- salt/returners/carbon_return.py | 23 +++++----- salt/returners/cassandra_cql_return.py | 33 ++++++++------ salt/returners/cassandra_return.py | 6 +-- salt/returners/couchbase_return.py | 33 +++++++------- salt/returners/couchdb_return.py | 31 +++++++------ salt/returners/django_return.py | 14 +++--- salt/returners/elasticsearch_return.py | 21 +++++---- salt/returners/etcd_return.py | 12 ++--- salt/returners/highstate_return.py | 36 ++++++++------- salt/returners/hipchat_return.py | 7 +-- salt/returners/influxdb_return.py | 15 ++++--- salt/returners/kafka_return.py | 2 +- salt/returners/librato_return.py | 44 +++++++++++-------- salt/returners/local.py | 2 +- salt/returners/local_cache.py | 39 ++++++++-------- salt/returners/mattermost_returner.py | 12 ++--- salt/returners/memcache_return.py | 4 +- salt/returners/mongo_future_return.py | 4 +- salt/returners/mongo_return.py | 2 +- salt/returners/multi_returner.py | 2 +- salt/returners/mysql.py | 38 ++++++++-------- salt/returners/nagios_return.py | 32 ++++++++------ salt/returners/odbc.py | 2 +- salt/returners/pgjsonb.py | 4 +- salt/returners/postgres.py | 5 ++- salt/returners/postgres_local_cache.py | 26 +++++------ salt/returners/pushover_returner.py | 6 +-- salt/returners/rawfile_json.py | 6 +-- salt/returners/redis_return.py | 4 +- salt/returners/sentry_return.py | 18 +++----- salt/returners/slack_returner.py | 4 +- salt/returners/sms_return.py | 8 ++-- salt/returners/smtp_return.py | 8 ++-- salt/returners/splunk.py | 12 ++--- salt/returners/sqlite3_return.py | 30 ++++++------- salt/returners/syslog_return.py | 6 +-- salt/returners/telegram_return.py | 4 +- salt/returners/xmpp_return.py | 2 +- salt/returners/zabbix_return.py | 2 +- tests/integration/renderers/test_pydsl.py | 5 ++- .../returners/test_librato_return.py | 2 +- tests/unit/renderers/test_gpg.py | 2 +- tests/unit/renderers/test_nacl.py | 2 +- tests/unit/renderers/test_yaml.py | 2 +- tests/unit/renderers/test_yamlex.py | 2 +- tests/unit/returners/test_local_cache.py | 4 +- tests/unit/returners/test_sentry.py | 2 +- tests/unit/returners/test_smtp_return.py | 2 +- 68 files changed, 378 insertions(+), 340 deletions(-) diff --git a/salt/renderers/cheetah.py b/salt/renderers/cheetah.py index ba48913f8f..8374d265c1 100644 --- a/salt/renderers/cheetah.py +++ b/salt/renderers/cheetah.py @@ -3,7 +3,7 @@ Cheetah Renderer for Salt ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import 3rd party libs try: @@ -13,7 +13,7 @@ except ImportError: HAS_LIBS = False # Import salt libs -from salt.ext.six import string_types +from salt.ext import six def render(cheetah_data, saltenv='base', sls='', method='xml', **kws): @@ -25,7 +25,7 @@ def render(cheetah_data, saltenv='base', sls='', method='xml', **kws): if not HAS_LIBS: return {} - if not isinstance(cheetah_data, string_types): + if not isinstance(cheetah_data, six.string_types): cheetah_data = cheetah_data.read() if cheetah_data.startswith('#!'): @@ -33,4 +33,4 @@ def render(cheetah_data, saltenv='base', sls='', method='xml', **kws): if not cheetah_data.strip(): return {} - return str(Template(cheetah_data, searchList=[kws])) + return six.text_type(Template(cheetah_data, searchList=[kws])) diff --git a/salt/renderers/dson.py b/salt/renderers/dson.py index b0678d615a..a0b2a566f9 100644 --- a/salt/renderers/dson.py +++ b/salt/renderers/dson.py @@ -12,7 +12,7 @@ This renderer requires `Dogeon`__ (installable via pip) .. __: https://github.com/soasme/dogeon ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/renderers/genshi.py b/salt/renderers/genshi.py index 3808faa6a7..469e8aefc1 100644 --- a/salt/renderers/genshi.py +++ b/salt/renderers/genshi.py @@ -3,7 +3,7 @@ Genshi Renderer for Salt ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import 3rd party libs try: @@ -15,7 +15,7 @@ except ImportError: HAS_LIBS = False # Import salt libs -from salt.ext.six import string_types +from salt.ext import six def render(genshi_data, saltenv='base', sls='', method='xml', **kws): @@ -40,7 +40,7 @@ def render(genshi_data, saltenv='base', sls='', method='xml', **kws): if not HAS_LIBS: return {} - if not isinstance(genshi_data, string_types): + if not isinstance(genshi_data, six.string_types): genshi_data = genshi_data.read() if genshi_data.startswith('#!'): diff --git a/salt/renderers/gpg.py b/salt/renderers/gpg.py index 7f9b65721c..b0d28294d7 100644 --- a/salt/renderers/gpg.py +++ b/salt/renderers/gpg.py @@ -209,7 +209,7 @@ pillar data like so: ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import re import logging @@ -288,7 +288,7 @@ def _decrypt_ciphertext(cipher, translate_newlines=False): else: if six.PY3 and isinstance(decrypted_data, bytes): decrypted_data = decrypted_data.decode(__salt_system_encoding__) - return str(decrypted_data) + return six.text_type(decrypted_data) def _decrypt_object(obj, translate_newlines=False): diff --git a/salt/renderers/hjson.py b/salt/renderers/hjson.py index e3a7731f73..d1403d59d2 100644 --- a/salt/renderers/hjson.py +++ b/salt/renderers/hjson.py @@ -4,7 +4,7 @@ Hjson Renderer for Salt http://laktak.github.io/hjson/ ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import 3rd party libs try: @@ -14,7 +14,7 @@ except ImportError: HAS_LIBS = False # Import salt libs -from salt.ext.six import string_types +from salt.ext import six def render(hjson_data, saltenv='base', sls='', **kws): @@ -24,7 +24,7 @@ def render(hjson_data, saltenv='base', sls='', **kws): :rtype: A Python data structure ''' - if not isinstance(hjson_data, string_types): + if not isinstance(hjson_data, six.string_types): hjson_data = hjson_data.read() if hjson_data.startswith('#!'): diff --git a/salt/renderers/jinja.py b/salt/renderers/jinja.py index a78619ac23..bf285ff45b 100644 --- a/salt/renderers/jinja.py +++ b/salt/renderers/jinja.py @@ -6,7 +6,7 @@ For Jinja usage information see :ref:`Understanding Jinja ` ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import salt libs @@ -53,7 +53,7 @@ def render(template_file, saltenv='base', sls='', argline='', from_str = argline == '-s' if not from_str and argline: raise SaltRenderError( - 'Unknown renderer option: {opt}'.format(opt=argline) + 'Unknown renderer option: {opt}'.format(opt=argline) ) tmp_data = salt.utils.templates.JINJA(template_file, diff --git a/salt/renderers/json.py b/salt/renderers/json.py index c3db661335..b2c0042c4a 100644 --- a/salt/renderers/json.py +++ b/salt/renderers/json.py @@ -3,14 +3,14 @@ JSON Renderer for Salt ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import salt.utils.json json = salt.utils.json.import_json() # Import salt libs -from salt.ext.six import string_types +from salt.ext import six def render(json_data, saltenv='base', sls='', **kws): @@ -20,7 +20,7 @@ def render(json_data, saltenv='base', sls='', **kws): :rtype: A Python data structure ''' - if not isinstance(json_data, string_types): + if not isinstance(json_data, six.string_types): json_data = json_data.read() if json_data.startswith('#!'): diff --git a/salt/renderers/json5.py b/salt/renderers/json5.py index 0ed200ecd1..7b9df094a9 100644 --- a/salt/renderers/json5.py +++ b/salt/renderers/json5.py @@ -12,7 +12,7 @@ This renderer requires the `json5 python bindings`__, installable via pip. .. __: https://pypi.python.org/pypi/json5 ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -23,7 +23,7 @@ except ImportError: HAS_JSON5 = False # Import salt libs -from salt.ext.six import string_types +from salt.ext import six log = logging.getLogger(__name__) @@ -44,7 +44,7 @@ def render(json_data, saltenv='base', sls='', **kws): :rtype: A Python data structure ''' - if not isinstance(json_data, string_types): + if not isinstance(json_data, six.string_types): json_data = json_data.read() if json_data.startswith('#!'): diff --git a/salt/renderers/mako.py b/salt/renderers/mako.py index 9f8b43077d..777d541703 100644 --- a/salt/renderers/mako.py +++ b/salt/renderers/mako.py @@ -4,7 +4,7 @@ Mako Renderer for Salt ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs from salt.ext import six diff --git a/salt/renderers/msgpack.py b/salt/renderers/msgpack.py index d8d19cfa33..f58d11b85b 100644 --- a/salt/renderers/msgpack.py +++ b/salt/renderers/msgpack.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import third party libs import msgpack # Import salt libs -from salt.ext.six import string_types +from salt.ext import six def render(msgpack_data, saltenv='base', sls='', **kws): @@ -21,7 +21,7 @@ def render(msgpack_data, saltenv='base', sls='', **kws): :rtype: A Python data structure ''' - if not isinstance(msgpack_data, string_types): + if not isinstance(msgpack_data, six.string_types): msgpack_data = msgpack_data.read() if msgpack_data.startswith('#!'): diff --git a/salt/renderers/nacl.py b/salt/renderers/nacl.py index 266fa624ce..5c05383666 100644 --- a/salt/renderers/nacl.py +++ b/salt/renderers/nacl.py @@ -53,7 +53,7 @@ data like so: ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import re import logging diff --git a/salt/renderers/pass.py b/salt/renderers/pass.py index 381ca19448..e4aa3e1334 100644 --- a/salt/renderers/pass.py +++ b/salt/renderers/pass.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -""" +''' Pass Renderer for Salt [pass](https://www.passwordstore.org/) @@ -38,10 +38,10 @@ entries that are of interest for pillar data pass: pkg.installed ``` -""" +''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os from os.path import expanduser @@ -58,9 +58,9 @@ log = logging.getLogger(__name__) def _get_pass_exec(): - """ + ''' Return the pass executable or raise an error - """ + ''' pass_exec = salt.utils.path.which('pass') if pass_exec: return pass_exec @@ -69,12 +69,12 @@ def _get_pass_exec(): def _fetch_secret(pass_path): - """ + ''' Fetch secret from pass based on pass_path. If there is any error, return back the original pass_path value - """ + ''' cmd = "pass show {0}".format(pass_path.strip()) - log.debug('Fetching secret: {0}'.format(cmd)) + log.debug('Fetching secret: %s', cmd) proc = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE) pass_data, pass_error = proc.communicate() @@ -88,9 +88,9 @@ def _fetch_secret(pass_path): def _decrypt_object(obj): - """ + ''' Recursively try to find a pass path (string) that can be handed off to pass - """ + ''' if isinstance(obj, six.string_types): return _fetch_secret(obj) elif isinstance(obj, dict): @@ -103,9 +103,9 @@ def _decrypt_object(obj): def render(pass_info, saltenv='base', sls='', argline='', **kwargs): - """ + ''' Fetch secret from pass based on pass_path - """ + ''' try: _get_pass_exec() except SaltRenderError: diff --git a/salt/renderers/py.py b/salt/renderers/py.py index 94bd9c6999..054afa3f58 100644 --- a/salt/renderers/py.py +++ b/salt/renderers/py.py @@ -97,7 +97,7 @@ Translate to:: return config ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import os diff --git a/salt/renderers/pydsl.py b/salt/renderers/pydsl.py index 20ae7e0152..aa042ecb62 100644 --- a/salt/renderers/pydsl.py +++ b/salt/renderers/pydsl.py @@ -230,7 +230,7 @@ is enabled by setting the ``ordered`` option on ``__pydsl__``. __pydsl__.set(ordered=True) for i in range(10): - i = str(i) + i = six.text_type(i) state(i).cmd.run('echo '+i, cwd='/') state('1').cmd.run('echo one') state('2').cmd.run(name='echo two') @@ -334,7 +334,7 @@ For example: my_mod = sys.modules['salt.loaded.ext.module.my_mod'] ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import types import salt.utils.pydsl as pydsl diff --git a/salt/renderers/pyobjects.py b/salt/renderers/pyobjects.py index 9f0b8430fd..5465ebc5c0 100644 --- a/salt/renderers/pyobjects.py +++ b/salt/renderers/pyobjects.py @@ -295,13 +295,13 @@ TODO ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os import re # Import Salt Libs -from salt.ext.six import exec_ +from salt.ext import six import salt.utils.files import salt.loader from salt.fileclient import get_file_client @@ -384,7 +384,7 @@ def render(template, saltenv='base', sls='', salt_data=True, **kwargs): mod, valid_funcs ) - exec_(mod_cmd, mod_globals, mod_locals) + six.exec_(mod_cmd, mod_globals, mod_locals) _globals[mod_camel] = mod_locals[mod_camel] @@ -459,7 +459,7 @@ def render(template, saltenv='base', sls='', salt_data=True, **kwargs): with salt.utils.files.fopen(state_file) as state_fh: state_contents, state_globals = process_template(state_fh) - exec_(state_contents, state_globals) + six.exec_(state_contents, state_globals) # if no imports have been specified then we are being imported as: import salt://foo.sls # so we want to stick all of the locals from our state file into the template globals @@ -501,6 +501,6 @@ def render(template, saltenv='base', sls='', salt_data=True, **kwargs): Registry.enabled = True # now exec our template using our created scopes - exec_(final_template, _globals) + six.exec_(final_template, _globals) return Registry.salt_data() diff --git a/salt/renderers/stateconf.py b/salt/renderers/stateconf.py index 18057d9360..c47214f32a 100644 --- a/salt/renderers/stateconf.py +++ b/salt/renderers/stateconf.py @@ -27,15 +27,16 @@ A flexible renderer that takes a templating engine and a data format # # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging +import os import re import getopt import copy -from os import path as ospath # Import salt libs import salt.utils.files +import salt.utils.stringutils from salt.exceptions import SaltRenderError # Import 3rd-party libs @@ -75,7 +76,7 @@ def __init__(opts): STATE_NAME = STATE_FUNC.split('.')[0] -MOD_BASENAME = ospath.basename(__file__) +MOD_BASENAME = os.path.basename(__file__) INVALID_USAGE_ERROR = SaltRenderError( 'Invalid use of {0} renderer!\n' '''Usage: #!{1} [-GoSp] [ [options] . [options]] @@ -108,7 +109,7 @@ def render(input, saltenv='base', sls='', argline='', **kws): implicit_require = False def process_sls_data(data, context=None, extract=False): - sls_dir = ospath.dirname(sls.replace('.', ospath.sep)) if '.' in sls else sls + sls_dir = os.path.dirname(sls.replace('.', os.path.sep)) if '.' in sls else sls ctx = dict(sls_dir=sls_dir if sls_dir else '.') if context: @@ -156,8 +157,8 @@ def render(input, saltenv='base', sls='', argline='', **kws): raise except Exception as err: log.exception( - 'Error found while pre-processing the salt file ' - '{0}:\n{1}'.format(sls, err) + 'Error found while pre-processing the salt file %s:\n%s', + sls, err ) from salt.state import State state = State(__opts__) @@ -207,9 +208,9 @@ def render(input, saltenv='base', sls='', argline='', **kws): if isinstance(input, six.string_types): with salt.utils.files.fopen(input, 'r') as ifile: - sls_templ = ifile.read() + sls_templ = salt.utils.stringutils.to_unicode(ifile.read()) else: # assume file-like - sls_templ = input.read() + sls_templ = salt.utils.stringutils.to_unicode(input.read()) # first pass to extract the state configuration match = re.search(__opts__['stateconf_end_marker'], sls_templ) @@ -235,7 +236,7 @@ def render(input, saltenv='base', sls='', argline='', **kws): if log.isEnabledFor(logging.DEBUG): import pprint # FIXME: pprint OrderedDict - log.debug('Rendered sls: {0}'.format(pprint.pformat(data))) + log.debug('Rendered sls: %s', pprint.pformat(data)) return data diff --git a/salt/renderers/wempy.py b/salt/renderers/wempy.py index 9c27e9c952..e762508131 100644 --- a/salt/renderers/wempy.py +++ b/salt/renderers/wempy.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import salt libs from salt.ext import six diff --git a/salt/renderers/yaml.py b/salt/renderers/yaml.py index 2adb667992..93ae327e0f 100644 --- a/salt/renderers/yaml.py +++ b/salt/renderers/yaml.py @@ -5,7 +5,7 @@ YAML Renderer for Salt For YAML usage information see :ref:`Understanding YAML `. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -60,13 +60,12 @@ def render(yaml_data, saltenv='base', sls='', argline='', **kws): if len(warn_list) > 0: for item in warn_list: log.warning( - '{warn} found in {sls} saltenv={env}'.format( - warn=item.message, sls=salt.utils.url.create(sls), env=saltenv - ) + '%s found in %s saltenv=%s', + item.message, salt.utils.url.create(sls), saltenv ) if not data: data = {} - log.debug('Results of YAML rendering: \n{0}'.format(data)) + log.debug('Results of YAML rendering: \n%s', data) def _validate_data(data): ''' diff --git a/salt/renderers/yamlex.py b/salt/renderers/yamlex.py index a80ee6c35e..8e05eab910 100644 --- a/salt/renderers/yamlex.py +++ b/salt/renderers/yamlex.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -24,11 +24,10 @@ def render(sls_data, saltenv='base', sls='', **kws): for item in warn_list: log.warning( - '{warn} found in {sls} saltenv={env}'.format( - warn=item.message, sls=salt.utils.url.create(sls), env=saltenv - ) + '%s found in %s saltenv=%s', + item.message, salt.utils.url.create(sls), saltenv ) - log.debug('Results of SLS rendering: \n{0}'.format(data)) + log.debug('Results of SLS rendering: \n%s', data) return data diff --git a/salt/returners/__init__.py b/salt/returners/__init__.py index 9d35453471..e6e9e11535 100644 --- a/salt/returners/__init__.py +++ b/salt/returners/__init__.py @@ -5,9 +5,10 @@ Returners Directory :func:`get_returner_options` is a general purpose function that returners may use to fetch their configuration options. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging +from salt.ext import six log = logging.getLogger(__name__) @@ -116,7 +117,7 @@ def _fetch_ret_config(ret): return None if 'ret_config' not in ret: return '' - return str(ret['ret_config']) + return six.text_type(ret['ret_config']) def _fetch_option(cfg, ret_config, virtualname, attr_name): diff --git a/salt/returners/carbon_return.py b/salt/returners/carbon_return.py index ba06ad29b6..d8aa510abc 100644 --- a/salt/returners/carbon_return.py +++ b/salt/returners/carbon_return.py @@ -82,7 +82,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import collections import logging import socket @@ -143,7 +143,7 @@ def _carbon(host, port): carbon_sock.connect((host, port)) except socket.error as err: - log.error('Error connecting to {0}:{1}, {2}'.format(host, port, err)) + log.error('Error connecting to %s:%s, %s', host, port, err) raise else: log.debug('Connected to carbon') @@ -176,7 +176,7 @@ def _send_textmetrics(metrics): Format metrics for the carbon plaintext protocol ''' - data = [' '.join(map(str, metric)) for metric in metrics] + [''] + data = [' '.join(map(six.text_type, metric)) for metric in metrics] + [''] return '\n'.join(data) @@ -199,7 +199,10 @@ def _walk(path, value, metrics, timestamp, skip): Whether or not to skip metrics when there's an error casting the value to a float. Defaults to `False`. ''' - log.trace('Carbon return walking path: {0}, value: {1}, metrics: {2}, timestamp: {3}'.format(path, value, metrics, timestamp)) + log.trace( + 'Carbon return walking path: %s, value: %s, metrics: %s, ', + 'timestamp: %s', path, value, metrics, timestamp + ) if isinstance(value, collections.Mapping): for key, val in six.iteritems(value): _walk('{0}.{1}'.format(path, key), val, metrics, timestamp, skip) @@ -232,8 +235,8 @@ def _send(saltdata, metric_base, opts): metric_base_pattern = opts.get('carbon.metric_base_pattern') mode = opts.get('mode').lower() if 'mode' in opts else 'text' - log.debug('Carbon minion configured with host: {0}:{1}'.format(host, port)) - log.debug('Using carbon protocol: {0}'.format(mode)) + log.debug('Carbon minion configured with host: %s:%s', host, port) + log.debug('Using carbon protocol: %s', mode) if not (host and port): log.error('Host or port not defined') @@ -246,10 +249,10 @@ def _send(saltdata, metric_base, opts): handler = _send_picklemetrics if mode == 'pickle' else _send_textmetrics metrics = [] - log.trace('Carbon returning walking data: {0}'.format(saltdata)) + log.trace('Carbon returning walking data: %s', saltdata) _walk(metric_base, saltdata, metrics, timestamp, skip) data = handler(metrics) - log.trace('Carbon inserting data: {0}'.format(data)) + log.trace('Carbon inserting data: %s', data) with _carbon(host, port) as sock: total_sent_bytes = 0 @@ -259,7 +262,7 @@ def _send(saltdata, metric_base, opts): log.error('Bytes sent 0, Connection reset?') return - log.debug('Sent {0} bytes to carbon'.format(sent_bytes)) + log.debug('Sent %s bytes to carbon', sent_bytes) total_sent_bytes += sent_bytes @@ -272,7 +275,7 @@ def event_return(events): opts = _get_options({}) # Pass in empty ret, since this is a list of events opts['skip'] = True for event in events: - log.trace('Carbon returner received event: {0}'.format(event)) + log.trace('Carbon returner received event: %s', event) metric_base = event['tag'] saltdata = event['data'].get('data') _send(saltdata, metric_base, opts) diff --git a/salt/returners/cassandra_cql_return.py b/salt/returners/cassandra_cql_return.py index 5e7e7c0ff9..3fb0c09ad2 100644 --- a/salt/returners/cassandra_cql_return.py +++ b/salt/returners/cassandra_cql_return.py @@ -117,9 +117,7 @@ needs. SaltStack has seen situations where these timeouts can resolve some stacktraces that appear to come from the Datastax Python driver. ''' -from __future__ import absolute_import -# Let's not allow PyLint complain about string substitution -# pylint: disable=W1321,E1321 +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import json @@ -132,6 +130,7 @@ import salt.returners import salt.utils.jid import salt.exceptions from salt.exceptions import CommandExecutionError +from salt.ext import six # Import third party libs try: @@ -210,7 +209,7 @@ def returner(ret): log.critical('Could not insert into salt_returns with Cassandra returner.') raise except Exception as e: - log.critical('Unexpected error while inserting into salt_returns: {0}'.format(str(e))) + log.critical('Unexpected error while inserting into salt_returns: %s', e) raise # Store the last function called by the minion @@ -234,7 +233,10 @@ def returner(ret): log.critical('Could not store minion ID with Cassandra returner.') raise except Exception as e: - log.critical('Unexpected error while inserting minion ID into the minions table: {0}'.format(str(e))) + log.critical( + 'Unexpected error while inserting minion ID into the minions ' + 'table: %s', e + ) raise @@ -258,7 +260,7 @@ def event_return(events): ) VALUES ( ?, ?, ?, ?, ?) ''' - statement_arguments = [str(uuid.uuid1()), + statement_arguments = [six.text_type(uuid.uuid1()), int(time.time() * 1000), json.dumps(data).replace("'", "''"), __opts__['id'], @@ -273,7 +275,8 @@ def event_return(events): log.critical('Could not store events with Cassandra returner.') raise except Exception as e: - log.critical('Unexpected error while inserting into salt_events: {0}'.format(str(e))) + log.critical( + 'Unexpected error while inserting into salt_events: %s', e) raise @@ -302,7 +305,7 @@ def save_load(jid, load, minions=None): log.critical('Could not save load in jids table.') raise except Exception as e: - log.critical('Unexpected error while inserting into jids: {0}'.format(str(e))) + log.critical('Unexpected error while inserting into jids: %s', e) raise @@ -333,7 +336,7 @@ def get_load(jid): log.critical('Could not get load from jids table.') raise except Exception as e: - log.critical('Unexpected error while getting load from jids: {0}'.format(str(e))) + log.critical('Unexpected error while getting load from jids: %s', e) raise return ret @@ -361,7 +364,8 @@ def get_jid(jid): log.critical('Could not select job specific information.') raise except Exception as e: - log.critical('Unexpected error while getting job specific information: {0}'.format(str(e))) + log.critical( + 'Unexpected error while getting job specific information: %s', e) raise return ret @@ -389,7 +393,8 @@ def get_fun(fun): log.critical('Could not get the list of minions.') raise except Exception as e: - log.critical('Unexpected error while getting list of minions: {0}'.format(str(e))) + log.critical( + 'Unexpected error while getting list of minions: %s', e) raise return ret @@ -417,7 +422,8 @@ def get_jids(): log.critical('Could not get a list of all job ids.') raise except Exception as e: - log.critical('Unexpected error while getting list of all job ids: {0}'.format(str(e))) + log.critical( + 'Unexpected error while getting list of all job ids: %s', e) raise return ret @@ -444,7 +450,8 @@ def get_minions(): log.critical('Could not get the list of minions.') raise except Exception as e: - log.critical('Unexpected error while getting list of minions: {0}'.format(str(e))) + log.critical( + 'Unexpected error while getting list of minions: %s', e) raise return ret diff --git a/salt/returners/cassandra_return.py b/salt/returners/cassandra_return.py index d2c0e2b06d..07b4f02fb7 100644 --- a/salt/returners/cassandra_return.py +++ b/salt/returners/cassandra_return.py @@ -20,7 +20,7 @@ Required python modules: pycassa ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import salt libs @@ -68,9 +68,9 @@ def returner(ret): 'id': ret['id']} if isinstance(ret['return'], dict): for key, value in six.iteritems(ret['return']): - columns['return.{0}'.format(key)] = str(value) + columns['return.{0}'.format(key)] = six.text_type(value) else: - columns['return'] = str(ret['return']) + columns['return'] = six.text_type(ret['return']) log.debug(columns) ccf.insert(ret['jid'], columns) diff --git a/salt/returners/couchbase_return.py b/salt/returners/couchbase_return.py index a454d02190..fb108a1a81 100644 --- a/salt/returners/couchbase_return.py +++ b/salt/returners/couchbase_return.py @@ -48,7 +48,7 @@ JID/MINION_ID return: return_data full_ret: full load of job return ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import json import logging @@ -59,11 +59,14 @@ try: except ImportError: HAS_DEPS = False -# Import salt libs +# Import Salt libs import salt.utils.jid import salt.utils.json import salt.utils.minions +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) @@ -168,7 +171,7 @@ def prep_jid(nocache=False, passed_jid=None): cb_ = _get_connection() try: - cb_.add(str(jid), + cb_.add(six.text_type(jid), {'nocache': nocache}, ttl=_get_ttl(), ) @@ -197,10 +200,8 @@ def returner(load): ) except couchbase.exceptions.KeyExistsError: log.error( - 'An extra return was detected from minion {0}, please verify ' - 'the minion, this could be a replay attack'.format( - load['id'] - ) + 'An extra return was detected from minion %s, please verify ' + 'the minion, this could be a replay attack', load['id'] ) return False @@ -212,13 +213,13 @@ def save_load(jid, clear_load, minion=None): cb_ = _get_connection() try: - jid_doc = cb_.get(str(jid)) + jid_doc = cb_.get(six.text_type(jid)) except couchbase.exceptions.NotFoundError: - cb_.add(str(jid), {}, ttl=_get_ttl()) - jid_doc = cb_.get(str(jid)) + cb_.add(six.text_type(jid), {}, ttl=_get_ttl()) + jid_doc = cb_.get(six.text_type(jid)) jid_doc.value['load'] = clear_load - cb_.replace(str(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl()) + cb_.replace(six.text_type(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl()) # if you have a tgt, save that for the UI etc if 'tgt' in clear_load and clear_load['tgt'] != '': @@ -240,9 +241,9 @@ def save_minions(jid, minions, syndic_id=None): # pylint: disable=unused-argume cb_ = _get_connection() try: - jid_doc = cb_.get(str(jid)) + jid_doc = cb_.get(six.text_type(jid)) except couchbase.exceptions.NotFoundError: - log.warning('Could not write job cache file for jid: {0}'.format(jid)) + log.warning('Could not write job cache file for jid: %s', jid) return False # save the minions to a cache so we can see in the UI @@ -252,7 +253,7 @@ def save_minions(jid, minions, syndic_id=None): # pylint: disable=unused-argume ) else: jid_doc.value['minions'] = minions - cb_.replace(str(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl()) + cb_.replace(six.text_type(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl()) def get_load(jid): @@ -262,7 +263,7 @@ def get_load(jid): cb_ = _get_connection() try: - jid_doc = cb_.get(str(jid)) + jid_doc = cb_.get(six.text_type(jid)) except couchbase.exceptions.NotFoundError: return {} @@ -285,7 +286,7 @@ def get_jid(jid): ret = {} - for result in cb_.query(DESIGN_NAME, 'jid_returns', key=str(jid), include_docs=True): + for result in cb_.query(DESIGN_NAME, 'jid_returns', key=six.text_type(jid), include_docs=True): ret[result.value] = result.doc.value return ret diff --git a/salt/returners/couchdb_return.py b/salt/returners/couchdb_return.py index 117b2802f4..499085c420 100644 --- a/salt/returners/couchdb_return.py +++ b/salt/returners/couchdb_return.py @@ -53,7 +53,7 @@ otherwise multi-minion targeting can lead to losing output: ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import time import json @@ -155,11 +155,10 @@ def returner(ret): # Confirm that the response back was simple 'ok': true. if 'ok' not in _response or _response['ok'] is not True: - log.error('Unable to create database "{0}"' - .format(options['db'])) + log.error('Unable to create database \'%s\'', options['db']) log.error('Nothing logged! Lost data.') return - log.info('Created database "{0}"'.format(options['db'])) + log.info('Created database \'%s\'', options['db']) # Call _generate_doc to get a dict object of the document we're going to # shove into the database. @@ -173,7 +172,7 @@ def returner(ret): # Sanity check regarding the response.. if 'ok' not in _response or _response['ok'] is not True: - log.error('Unable to create document: "{0}"'.format(_response)) + log.error('Unable to create document: \'%s\'', _response) log.error('Nothing logged! Lost data.') @@ -184,7 +183,7 @@ def get_jid(jid): options = _get_options(ret=None) _response = _request("GET", options['url'] + options['db'] + '/' + jid) if 'error' in _response: - log.error('Unable to get JID "{0}" : "{1}"'.format(jid, _response)) + log.error('Unable to get JID \'%s\' : \'%s\'', jid, _response) return {} return {_response['id']: _response} @@ -198,8 +197,10 @@ def get_jids(): # Make sure the 'total_rows' is returned.. if not error out. if 'total_rows' not in _response: - log.error('Didn\'t get valid response from requesting all docs: {0}' - .format(_response)) + log.error( + 'Didn\'t get valid response from requesting all docs: %s', + _response + ) return {} # Return the rows. @@ -246,8 +247,10 @@ def get_fun(fun): fun)) # Skip the minion if we got an error.. if 'error' in _response: - log.warning('Got an error when querying for last command by a ' - 'minion: {0}'.format(_response['error'])) + log.warning( + 'Got an error when querying for last command by a minion: %s', + _response['error'] + ) continue # Skip the minion if we didn't get any rows back. ( IE function that @@ -279,7 +282,7 @@ def get_minions(): # Verify that we got a response back. if 'rows' not in _response: - log.error('Unable to get available minions: {0}'.format(_response)) + log.error('Unable to get available minions: %s', _response) return [] # Iterate over the rows to build up a list return it. @@ -354,8 +357,10 @@ def set_salt_view(): options['url'] + options['db'] + "/_design/salt", "application/json", json.dumps(new_doc)) if 'error' in _response: - log.warning("Unable to set the salt design document: {0}" - .format(_response['error'])) + log.warning( + 'Unable to set the salt design document: %s', + _response['error'] + ) return False return True diff --git a/salt/returners/django_return.py b/salt/returners/django_return.py index 8a4517a5ce..90f7e019d1 100644 --- a/salt/returners/django_return.py +++ b/salt/returners/django_return.py @@ -27,7 +27,7 @@ An example Django module that registers a function called ''' # Import Python libraries -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import Salt libraries @@ -61,8 +61,10 @@ def returner(ret): signaled = dispatch.Signal(providing_args=['ret']).send(sender='returner', ret=ret) for signal in signaled: - log.debug('Django returner function \'returner\' signaled {0} ' - 'which responded with {1}'.format(signal[0], signal[1])) + log.debug( + 'Django returner function \'returner\' signaled %s ' + 'which responded with %s', signal[0], signal[1] + ) def save_load(jid, load, minions=None): @@ -74,8 +76,10 @@ def save_load(jid, load, minions=None): sender='save_load', jid=jid, load=load) for signal in signaled: - log.debug('Django returner function \'save_load\' signaled {0} ' - 'which responded with {1}'.format(signal[0], signal[1])) + log.debug( + 'Django returner function \'save_load\' signaled %s ' + 'which responded with %s', signal[0], signal[1] + ) def prep_jid(nocache=False, passed_jid=None): diff --git a/salt/returners/elasticsearch_return.py b/salt/returners/elasticsearch_return.py index e4ffb20f1e..076b06f328 100644 --- a/salt/returners/elasticsearch_return.py +++ b/salt/returners/elasticsearch_return.py @@ -95,7 +95,7 @@ Minion configuration: ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import datetime from datetime import tzinfo, timedelta import uuid @@ -220,14 +220,17 @@ def returner(ret): if job_fun in options['functions_blacklist']: log.info( - 'Won\'t push new data to Elasticsearch, job with jid={0} and ' - 'function={1} which is in the user-defined list of ignored ' - 'functions'.format(job_id, job_fun)) + 'Won\'t push new data to Elasticsearch, job with jid=%s and ' + 'function=%s which is in the user-defined list of ignored ' + 'functions', job_id, job_fun + ) return if ret.get('return', None) is None: - log.info('Won\'t push new data to Elasticsearch, job with jid={0} was ' - 'not succesful'.format(job_id)) + log.info( + 'Won\'t push new data to Elasticsearch, job with jid=%s was ' + 'not succesful', job_id + ) return # Build the index name @@ -258,7 +261,7 @@ def returner(ret): # index data format if options['states_order_output'] and isinstance(ret['return'], dict): index = '{0}-ordered'.format(index) - max_chars = len(str(len(ret['return']))) + max_chars = len(six.text_type(len(ret['return']))) for uid, data in six.iteritems(ret['return']): # Skip keys we've already prefixed @@ -274,7 +277,7 @@ def returner(ret): # Prefix the key with the run order so it can be sorted new_uid = '{0}_|-{1}'.format( - str(data['__run_num__']).zfill(max_chars), + six.text_type(data['__run_num__']).zfill(max_chars), uid, ) @@ -321,7 +324,7 @@ def returner(ret): } if options['debug_returner_payload']: - log.debug('Payload: {0}'.format(data)) + log.debug('elasicsearch payload: %s', data) # Post the payload ret = __salt__['elasticsearch.document_create'](index=index, diff --git a/salt/returners/etcd_return.py b/salt/returners/etcd_return.py index e6e2fddf51..744c5b530b 100644 --- a/salt/returners/etcd_return.py +++ b/salt/returners/etcd_return.py @@ -64,20 +64,22 @@ create the profiles as specified above. Then add: etcd.returner_read_profile: my_etcd_read etcd.returner_write_profile: my_etcd_write ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import json import logging # Import salt libs +import salt.utils.jid try: import salt.utils.etcd_util HAS_LIBS = True except ImportError: HAS_LIBS = False -import salt.utils.jid +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -185,7 +187,7 @@ def get_fun(): client, path = _get_conn(__opts__) items = client.get('/'.join((path, 'minions'))) for item in items.children: - comps = str(item.key).split('/') + comps = six.text_type(item.key).split('/') ret[comps[-1]] = item.value return ret @@ -199,7 +201,7 @@ def get_jids(): items = client.get('/'.join((path, 'jobs'))) for item in items.children: if item.dir is True: - jid = str(item.key).split('/')[-1] + jid = six.text_type(item.key).split('/')[-1] load = client.get('/'.join((item.key, '.load.p'))).value ret[jid] = salt.utils.jid.format_jid_instance(jid, json.loads(load)) return ret @@ -213,7 +215,7 @@ def get_minions(): client, path = _get_conn(__opts__) items = client.get('/'.join((path, 'minions'))) for item in items.children: - comps = str(item.key).split('/') + comps = six.text_type(item.key).split('/') ret.append(comps[-1]) return ret diff --git a/salt/returners/highstate_return.py b/salt/returners/highstate_return.py index 72286b8648..c6d1f7afc3 100644 --- a/salt/returners/highstate_return.py +++ b/salt/returners/highstate_return.py @@ -77,7 +77,7 @@ values at the time of pillar generation, these will contain minion values at the time of execution. ''' -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import logging import json @@ -88,8 +88,10 @@ from email.mime.text import MIMEText import yaml from salt.ext.six.moves import range from salt.ext.six.moves import StringIO +from salt.ext import six import salt.utils.files +import salt.utils.stringutils import salt.returners log = logging.getLogger(__name__) @@ -167,12 +169,12 @@ def _generate_html_table(data, out, level=0, extra_style=''): Generate a single table of data ''' print(''.format( - _lookup_style('table', ['table' + str(level)])), file=out) + _lookup_style('table', ['table' + six.text_type(level)])), file=out) firstone = True - row_style = 'row' + str(level) - cell_style = 'cell' + str(level) + row_style = 'row' + six.text_type(level) + cell_style = 'cell' + six.text_type(level) for subdata in data: first_style = 'first_first' if firstone else 'notfirst_first' @@ -227,7 +229,7 @@ def _generate_html_table(data, out, level=0, extra_style=''): new_extra_style ] ), - cgi.escape(str(value)) + cgi.escape(six.text_type(value)) ), file=out) print('', file=out) elif isinstance(subdata, list): @@ -252,7 +254,7 @@ def _generate_html_table(data, out, level=0, extra_style=''): 'td', [cell_style, first_style, 'value', extra_style] ), - cgi.escape(str(subdata)) + cgi.escape(six.text_type(subdata)) ), file=out) print('', file=out) firstone = False @@ -356,10 +358,10 @@ def _generate_report(ret, setup): unchanged = total - failed - changed - log.debug('highstate total: {0}'.format(total)) - log.debug('highstate failed: {0}'.format(failed)) - log.debug('highstate unchanged: {0}'.format(unchanged)) - log.debug('highstate changed: {0}'.format(changed)) + log.debug('highstate total: %s', total) + log.debug('highstate failed: %s', failed) + log.debug('highstate unchanged: %s', unchanged) + log.debug('highstate changed: %s', changed) # generate report if required if setup.get('report_everything', False) or \ @@ -409,7 +411,7 @@ def _sprinkle(config_str): ''' parts = [x for sub in config_str.split('{') for x in sub.split('}')] for i in range(1, len(parts), 2): - parts[i] = str(__grains__.get(parts[i], '')) + parts[i] = six.text_type(__grains__.get(parts[i], '')) return ''.join(parts) @@ -419,7 +421,7 @@ def _produce_output(report, failed, setup): ''' report_format = setup.get('report_format', 'yaml') - log.debug('highstate output format: {0}'.format(report_format)) + log.debug('highstate output format: %s', report_format) if report_format == 'json': report_text = json.dumps(report) @@ -436,12 +438,12 @@ def _produce_output(report, failed, setup): report_delivery = setup.get('report_delivery', 'file') - log.debug('highstate report_delivery: {0}'.format(report_delivery)) + log.debug('highstate report_delivery: %s', report_delivery) if report_delivery == 'file': output_file = _sprinkle(setup.get('file_output', '/tmp/test.rpt')) with salt.utils.files.fopen(output_file, 'w') as out: - out.write(report_text) + out.write(salt.utils.stringutils.to_str(report_text)) else: msg = MIMEText(report_text, report_format) @@ -473,7 +475,7 @@ def returner(ret): ''' setup = _get_options(ret) - log.debug('highstate setup {0}'.format(setup)) + log.debug('highstate setup %s', setup) report, failed = _generate_report(ret, setup) if report: @@ -491,7 +493,7 @@ def __test_html(): file_output: '/srv/salt/_returners/test.rpt' ''' with salt.utils.files.fopen('test.rpt', 'r') as input_file: - data_text = input_file.read() + data_text = salt.utils.stringutils.to_unicode(input_file.read()) data = yaml.safe_load(data_text) string_file = StringIO() @@ -500,7 +502,7 @@ def __test_html(): result = string_file.read() with salt.utils.files.fopen('test.html', 'w') as output: - output.write(result) + output.write(salt.utils.stringutils.to_str(result)) if __name__ == '__main__': diff --git a/salt/returners/hipchat_return.py b/salt/returners/hipchat_return.py index d53f67a6f0..5aa7bb9829 100644 --- a/salt/returners/hipchat_return.py +++ b/salt/returners/hipchat_return.py @@ -93,7 +93,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return hipchat --return_kwargs '{"room_id": "another-room"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import json @@ -101,6 +101,7 @@ import pprint import logging # pylint: disable=import-error,no-name-in-module +from salt.ext import six from salt.ext.six.moves.urllib.parse import urljoin as _urljoin from salt.ext.six.moves.urllib.parse import urlencode as _urlencode import salt.ext.six.moves.http_client @@ -183,7 +184,7 @@ def _query(function, query_params = {} if room_id: - room_id = 'room/{0}/notification'.format(str(room_id)) + room_id = 'room/{0}/notification'.format(six.text_type(room_id)) else: room_id = 'room/0/notification' @@ -388,7 +389,7 @@ def event_return(events): # TODO: # Pre-process messages to apply individualized colors for various # event types. - log.trace('Hipchat returner received event: {0}'.format(event)) + log.trace('Hipchat returner received event: %s', event) _send_message(_options.get('room_id'), # room_id event['data'], # message _options.get('from_name'), # from_name diff --git a/salt/returners/influxdb_return.py b/salt/returners/influxdb_return.py index e6cafb7cc5..2264adea5a 100644 --- a/salt/returners/influxdb_return.py +++ b/salt/returners/influxdb_return.py @@ -50,7 +50,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return influxdb --return_kwargs '{"db": "another-salt"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import json @@ -113,7 +113,10 @@ def _get_version(host, port, user, password): if influxDBVersionHeader in result.headers: version = result.headers[influxDBVersionHeader] except Exception as ex: - log.critical('Failed to query InfluxDB version from HTTP API within InfluxDB returner: {0}'.format(ex)) + log.critical( + 'Failed to query InfluxDB version from HTTP API within InfluxDB ' + 'returner: %s', ex + ) return version @@ -187,7 +190,7 @@ def returner(ret): try: serv.write_points(req) except Exception as ex: - log.critical('Failed to store return with InfluxDB returner: {0}'.format(ex)) + log.critical('Failed to store return with InfluxDB returner: %s', ex) def save_load(jid, load, minions=None): @@ -224,7 +227,7 @@ def save_load(jid, load, minions=None): try: serv.write_points(req) except Exception as ex: - log.critical('Failed to store load with InfluxDB returner: {0}'.format(ex)) + log.critical('Failed to store load with InfluxDB returner: %s', ex) def save_minions(jid, minions, syndic_id=None): # pylint: disable=unused-argument @@ -241,9 +244,9 @@ def get_load(jid): serv = _get_serv(ret=None) sql = "select load from jids where jid = '{0}'".format(jid) - log.debug(">> Now in get_load {0}".format(jid)) + log.debug(">> Now in get_load %s", jid) data = serv.query(sql) - log.debug(">> Now Data: {0}".format(data)) + log.debug(">> Now Data: %s", data) if data: return data return {} diff --git a/salt/returners/kafka_return.py b/salt/returners/kafka_return.py index 132016125a..b63da11c69 100644 --- a/salt/returners/kafka_return.py +++ b/salt/returners/kafka_return.py @@ -25,7 +25,7 @@ To use the kafka returner, append '--return kafka' to the Salt command, eg; ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import json import logging diff --git a/salt/returners/librato_return.py b/salt/returners/librato_return.py index e14b7c4697..e911a027a2 100644 --- a/salt/returners/librato_return.py +++ b/salt/returners/librato_return.py @@ -31,7 +31,7 @@ by adding more tags to the submission. ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals import logging # Import Salt libs @@ -75,7 +75,7 @@ def _get_options(ret=None): _options['api_url'] = _options.get('api_url', 'metrics-api.librato.com') - log.debug("Retrieved Librato options: {0}".format(_options)) + log.debug('Retrieved Librato options: %s', _options) return _options @@ -112,7 +112,7 @@ def _calculate_runtimes(states): # Count durations results['runtime'] += resultset['duration'] - log.debug("Parsed state metrics: {0}".format(results)) + log.debug('Parsed state metrics: %s', results) return results @@ -125,31 +125,37 @@ def returner(ret): q = librato_conn.new_queue() if ret['fun'] == 'state.highstate': - log.debug("Found returned Highstate data.") + log.debug('Found returned Highstate data.') # Calculate the runtimes and number of failed states. stats = _calculate_runtimes(ret['return']) - log.debug("Batching Metric retcode with {0}".format(ret['retcode'])) - q.add("saltstack.highstate.retcode", ret[ - 'retcode'], tags={'Name': ret['id']}) + log.debug('Batching Metric retcode with %s', ret['retcode']) + q.add('saltstack.highstate.retcode', + ret['retcode'], tags={'Name': ret['id']}) - log.debug("Batching Metric num_failed_jobs with {0}".format( - stats['num_failed_states'])) - q.add("saltstack.highstate.failed_states", + log.debug( + 'Batching Metric num_failed_jobs with %s', + stats['num_failed_states'] + ) + q.add('saltstack.highstate.failed_states', stats['num_failed_states'], tags={'Name': ret['id']}) - log.debug("Batching Metric num_passed_states with {0}".format( - stats['num_passed_states'])) - q.add("saltstack.highstate.passed_states", + log.debug( + 'Batching Metric num_passed_states with %s', + stats['num_passed_states'] + ) + q.add('saltstack.highstate.passed_states', stats['num_passed_states'], tags={'Name': ret['id']}) - log.debug("Batching Metric runtime with {0}".format(stats['runtime'])) - q.add("saltstack.highstate.runtime", + log.debug('Batching Metric runtime with %s', stats['runtime']) + q.add('saltstack.highstate.runtime', stats['runtime'], tags={'Name': ret['id']}) - log.debug("Batching Metric runtime with {0}".format( - stats['num_failed_states'] + stats['num_passed_states'])) - q.add("saltstack.highstate.total_states", stats[ + log.debug( + 'Batching Metric runtime with %s', + stats['num_failed_states'] + stats['num_passed_states'] + ) + q.add('saltstack.highstate.total_states', stats[ 'num_failed_states'] + stats['num_passed_states'], tags={'Name': ret['id']}) - log.info("Sending metrics to Librato.") + log.info('Sending metrics to Librato.') q.submit() diff --git a/salt/returners/local.py b/salt/returners/local.py index d1c9236a2b..414bea9cbd 100644 --- a/salt/returners/local.py +++ b/salt/returners/local.py @@ -11,7 +11,7 @@ To use the local returner, append '--return local' to the salt command. ex: ''' # Import python libs -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, unicode_literals def returner(ret): diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 900eaca0f7..3ab5a938ea 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -3,7 +3,7 @@ Return data to local job cache ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import errno @@ -20,6 +20,7 @@ import salt.utils.atomicfile import salt.utils.files import salt.utils.jid import salt.utils.minions +import salt.utils.stringutils import salt.exceptions # Import 3rd-party libs @@ -108,15 +109,13 @@ def prep_jid(nocache=False, passed_jid=None, recurse_count=0): try: with salt.utils.files.fopen(os.path.join(jid_dir, 'jid'), 'wb+') as fn_: - if six.PY2: - fn_.write(jid) - else: - fn_.write(bytes(jid, 'utf-8')) + fn_.write(salt.utils.stringutils.to_bytes(jid)) if nocache: - with salt.utils.files.fopen(os.path.join(jid_dir, 'nocache'), 'wb+') as fn_: - fn_.write(b'') + with salt.utils.files.fopen(os.path.join(jid_dir, 'nocache'), 'wb+'): + pass except IOError: - log.warning('Could not write out jid file for job {0}. Retrying.'.format(jid)) + log.warning( + 'Could not write out jid file for job %s. Retrying.', jid) time.sleep(0.1) return prep_jid(passed_jid=jid, nocache=nocache, recurse_count=recurse_count+1) @@ -146,16 +145,14 @@ def returner(load): if err.errno == errno.EEXIST: # Minion has already returned this jid and it should be dropped log.error( - 'An extra return was detected from minion {0}, please verify ' - 'the minion, this could be a replay attack'.format( - load['id'] - ) + 'An extra return was detected from minion %s, please verify ' + 'the minion, this could be a replay attack', load['id'] ) return False elif err.errno == errno.ENOENT: log.error( 'An inconsistency occurred, a job was received with a job id ' - 'that is not present in the local cache: {jid}'.format(**load) + '(%s) that is not present in the local cache', load['jid'] ) return False raise @@ -280,8 +277,8 @@ def save_minions(jid, minions, syndic_id=None): serial.dump(minions, wfh) except IOError as exc: log.error( - 'Failed to write minion list {0} to job cache file {1}: {2}' - .format(minions, minions_path, exc) + 'Failed to write minion list %s to job cache file %s: %s', + minions, minions_path, exc ) @@ -351,7 +348,7 @@ def get_jid(jid): with salt.utils.files.fopen(outp, 'rb') as rfh: ret[fn_]['out'] = serial.load(rfh) except Exception as exc: - if 'Permission denied:' in str(exc): + if 'Permission denied:' in six.text_type(exc): raise return ret @@ -459,9 +456,9 @@ def update_endtime(jid, time): if not os.path.exists(jid_dir): os.makedirs(jid_dir) with salt.utils.files.fopen(os.path.join(jid_dir, ENDTIME), 'w') as etfile: - etfile.write(time) + etfile.write(salt.utils.stringutils.to_str(time)) except IOError as exc: - log.warning('Could not write job invocation cache file: {0}'.format(exc)) + log.warning('Could not write job invocation cache file: %s', exc) def get_endtime(jid): @@ -475,7 +472,7 @@ def get_endtime(jid): if not os.path.exists(etpath): return False with salt.utils.files.fopen(etpath, 'r') as etfile: - endtime = etfile.read().strip('\n') + endtime = salt.utils.stringutils.to_unicode(etfile.read()).strip('\n') return endtime @@ -504,7 +501,7 @@ def save_reg(data): with salt.utils.files.fopen(regfile, 'a') as fh_: msgpack.dump(data, fh_) except: - log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) + log.error('Could not write to msgpack file %s', __opts__['outdir']) raise @@ -518,5 +515,5 @@ def load_reg(): with salt.utils.files.fopen(regfile, 'r') as fh_: return msgpack.load(fh_) except: - log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) + log.error('Could not write to msgpack file %s', __opts__['outdir']) raise diff --git a/salt/returners/mattermost_returner.py b/salt/returners/mattermost_returner.py index 07ee8fa955..4f258acfa0 100644 --- a/salt/returners/mattermost_returner.py +++ b/salt/returners/mattermost_returner.py @@ -43,7 +43,7 @@ To override individual configuration items, append --return_kwargs '{'key:': 'va salt '*' test.ping --return mattermost --return_kwargs '{'channel': '#random'}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import logging @@ -89,7 +89,7 @@ def _get_options(ret=None): attrs, __salt__=__salt__, __opts__=__opts__) - log.debug('Options: {0}'.format(_options)) + log.debug('Options: %s', _options) return _options @@ -146,8 +146,8 @@ def event_return(events): is_ok = True for event in events: - log.debug('Event: {0}'.format(str(event))) - log.debug('Event data: {0}'.format(str(event['data']))) + log.debug('Event: %s', event) + log.debug('Event data: %s', event['data']) message = 'tag: {0}\r\n'.format(event['tag']) for key, value in six.iteritems(event['data']): message += '{0}: {1}\r\n'.format(key, value) @@ -183,10 +183,10 @@ def post_message(channel, if username: parameters['username'] = username parameters['text'] = '```' + message + '```' # pre-formatted, fixed-width text - log.debug('Parameters: {0}'.format(parameters)) + log.debug('Parameters: %s', parameters) result = salt.utils.mattermost.query(api_url=api_url, hook=hook, data='payload={0}'.format(json.dumps(parameters))) - log.debug('result {0}'.format(result)) + log.debug('result %s', result) return bool(result) diff --git a/salt/returners/memcache_return.py b/salt/returners/memcache_return.py index c00dcbdf9b..9a9397708d 100644 --- a/salt/returners/memcache_return.py +++ b/salt/returners/memcache_return.py @@ -45,7 +45,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return memcache --return_kwargs '{"host": "hostname.domain.com"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import json @@ -99,7 +99,7 @@ def _get_serv(ret): host = _options.get('host') port = _options.get('port') - log.debug('memcache server: {0}:{1}'.format(host, port)) + log.debug('memcache server: %s:%s', host, port) if not host or not port: log.error('Host or port not defined in salt config') return diff --git a/salt/returners/mongo_future_return.py b/salt/returners/mongo_future_return.py index 0d9c7328b1..a767f112e8 100644 --- a/salt/returners/mongo_future_return.py +++ b/salt/returners/mongo_future_return.py @@ -63,7 +63,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return mongo --return_kwargs '{"db": "another-salt"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -226,7 +226,7 @@ def _safe_copy(dat): for k in dat: r = k.replace('%', '%25').replace('\\', '%5c').replace('$', '%24').replace('.', '%2e') if r != k: - log.debug('converting dict key from {0} to {1} for mongodb'.format(k, r)) + log.debug('converting dict key from %s to %s for mongodb', k, r) ret[r] = _safe_copy(dat[k]) return ret diff --git a/salt/returners/mongo_return.py b/salt/returners/mongo_return.py index f59c25f9f5..9ed314c50e 100644 --- a/salt/returners/mongo_return.py +++ b/salt/returners/mongo_return.py @@ -60,7 +60,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return mongo --return_kwargs '{"db": "another-salt"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/returners/multi_returner.py b/salt/returners/multi_returner.py index f9919083ec..8440b90217 100644 --- a/salt/returners/multi_returner.py +++ b/salt/returners/multi_returner.py @@ -5,7 +5,7 @@ Read/Write multiple returners ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import salt libs diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py index a7bfbed243..a5cfa5d471 100644 --- a/salt/returners/mysql.py +++ b/salt/returners/mysql.py @@ -138,7 +138,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return mysql --return_kwargs '{"db": "another-salt"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Let's not allow PyLint complain about string substitution # pylint: disable=W1321,E1321 @@ -229,7 +229,7 @@ def _get_serv(ret=None, commit=False): conn.ping() connect = False except MySQLdb.connections.OperationalError as exc: - log.debug('OperationalError on ping: {0}'.format(exc)) + log.debug('OperationalError on ping: %s', exc) if connect: log.debug('Generating new MySQL connection pool') @@ -263,7 +263,7 @@ def _get_serv(ret=None, commit=False): yield cursor except MySQLdb.DatabaseError as err: error = err.args - sys.stderr.write(str(error)) + sys.stderr.write(six.text_type(error)) cursor.execute("ROLLBACK") raise err else: @@ -476,8 +476,8 @@ def _purge_jobs(timestamp): cur.execute('COMMIT') except MySQLdb.Error as e: log.error('mysql returner archiver was unable to delete contents of table \'jids\'') - log.error(str(e)) - raise salt.exceptions.Salt(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) try: sql = 'delete from `salt_returns` where alter_time < %s' @@ -485,8 +485,8 @@ def _purge_jobs(timestamp): cur.execute('COMMIT') except MySQLdb.Error as e: log.error('mysql returner archiver was unable to delete contents of table \'salt_returns\'') - log.error(str(e)) - raise salt.exceptions.Salt(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) try: sql = 'delete from `salt_events` where alter_time < %s' @@ -494,8 +494,8 @@ def _purge_jobs(timestamp): cur.execute('COMMIT') except MySQLdb.Error as e: log.error('mysql returner archiver was unable to delete contents of table \'salt_events\'') - log.error(str(e)) - raise salt.exceptions.Salt(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) return True @@ -521,8 +521,8 @@ def _archive_jobs(timestamp): target_tables[table_name] = tmp_table_name except MySQLdb.Error as e: log.error('mysql returner archiver was unable to create the archive tables.') - log.error(str(e)) - raise salt.exceptions.SaltRunnerError(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) try: sql = 'insert into `{0}` select * from `{1}` where jid in (select distinct jid from salt_returns where alter_time < %s)'.format(target_tables['jids'], 'jids') @@ -530,8 +530,8 @@ def _archive_jobs(timestamp): cur.execute('COMMIT') except MySQLdb.Error as e: log.error('mysql returner archiver was unable to copy contents of table \'jids\'') - log.error(str(e)) - raise salt.exceptions.SaltRunnerError(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) except Exception as e: log.error(e) raise @@ -542,8 +542,8 @@ def _archive_jobs(timestamp): cur.execute('COMMIT') except MySQLdb.Error as e: log.error('mysql returner archiver was unable to copy contents of table \'salt_returns\'') - log.error(str(e)) - raise salt.exceptions.SaltRunnerError(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) try: sql = 'insert into `{0}` select * from `{1}` where alter_time < %s'.format(target_tables['salt_events'], 'salt_events') @@ -551,8 +551,8 @@ def _archive_jobs(timestamp): cur.execute('COMMIT') except MySQLdb.Error as e: log.error('mysql returner archiver was unable to copy contents of table \'salt_events\'') - log.error(str(e)) - raise salt.exceptions.SaltRunnerError(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) return _purge_jobs(timestamp) @@ -577,5 +577,5 @@ def clean_old_jobs(): _purge_jobs(stamp) except MySQLdb.Error as e: log.error('Mysql returner was unable to get timestamp for purge/archive of jobs') - log.error(str(e)) - raise salt.exceptions.Salt(str(e)) + log.error(six.text_type(e)) + raise salt.exceptions.SaltRunnerError(six.text_type(e)) diff --git a/salt/returners/nagios_return.py b/salt/returners/nagios_return.py index e022d5e8af..8b9b104de7 100644 --- a/salt/returners/nagios_return.py +++ b/salt/returners/nagios_return.py @@ -48,7 +48,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return nagios --return_kwargs '{"service": "service-name"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import cgi @@ -56,6 +56,7 @@ import logging import salt.returners # pylint: disable=import-error,no-name-in-module,redefined-builtin +from salt.ext import six import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -64,6 +65,13 @@ log = logging.getLogger(__name__) __virtualname__ = 'nagios_nrdp' +def __virtual__(): + ''' + Return virtualname + ''' + return 'nagios.list_plugins' in __salt__ + + def _get_options(ret=None): ''' Get the requests options from salt. @@ -80,7 +88,7 @@ def _get_options(ret=None): __salt__=__salt__, __opts__=__opts__) - log.debug('attrs {0}'.format(attrs)) + log.debug('attrs %s', attrs) if 'checktype' not in _options or _options['checktype'] == '': # default to passive check type _options['checktype'] = '1' @@ -92,7 +100,7 @@ def _get_options(ret=None): _options['checktype'] = '1' # checktype should be a string - _options['checktype'] = str(_options['checktype']) + _options['checktype'] = six.text_type(_options['checktype']) return _options @@ -111,11 +119,11 @@ def _prepare_xml(options=None, state=None): # No service defined then we set the status of the hostname if 'service' in options and options['service'] != '': - xml += "" + xml += "" xml += ""+cgi.escape(options['hostname'], True)+"" xml += ""+cgi.escape(options['service'], True)+"" else: - xml += "" + xml += "" xml += ""+cgi.escape(options['hostname'], True)+"" xml += ""+_state+"" @@ -169,24 +177,20 @@ def _post_data(options=None, xml=None): log.error('No content returned from Nagios NRDP.') return False else: - log.error('Error returned from Nagios NRDP. Status code: {0}.'.format(res.status_code)) + log.error( + 'Error returned from Nagios NRDP. Status code: %s.', + res.status_code + ) return False -def __virtual__(): - ''' - Return virtualname - ''' - return 'nagios.list_plugins' in __salt__ - - def returner(ret): ''' Send a message to Nagios with the data ''' _options = _get_options(ret) - log.debug('_options {0}'.format(_options)) + log.debug('_options %s', _options) _options['hostname'] = ret.get('id') if 'url' not in _options or _options['url'] == '': diff --git a/salt/returners/odbc.py b/salt/returners/odbc.py index 03c114cb10..0f0c431f43 100644 --- a/salt/returners/odbc.py +++ b/salt/returners/odbc.py @@ -123,7 +123,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return odbc --return_kwargs '{"dsn": "dsn-name"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Let's not allow PyLint complain about string substitution # pylint: disable=W1321,E1321 diff --git a/salt/returners/pgjsonb.py b/salt/returners/pgjsonb.py index 6dfbcd7552..e3f85bee3f 100644 --- a/salt/returners/pgjsonb.py +++ b/salt/returners/pgjsonb.py @@ -144,7 +144,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return pgjsonb --return_kwargs '{"db": "another-salt"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Let's not allow PyLint complain about string substitution # pylint: disable=W1321,E1321 @@ -247,7 +247,7 @@ def _get_serv(ret=None, commit=False): yield cursor except psycopg2.DatabaseError as err: error = err.args - sys.stderr.write(str(error)) + sys.stderr.write(six.text_type(error)) cursor.execute("ROLLBACK") raise err else: diff --git a/salt/returners/postgres.py b/salt/returners/postgres.py index 02cbde3ce7..8ccbacbf79 100644 --- a/salt/returners/postgres.py +++ b/salt/returners/postgres.py @@ -126,7 +126,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return postgres --return_kwargs '{"db": "another-salt"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Let's not allow PyLint complain about string substitution # pylint: disable=W1321,E1321 @@ -142,6 +142,7 @@ import salt.returners import salt.exceptions # Import third party libs +from salt.ext import six try: import psycopg2 HAS_POSTGRES = True @@ -209,7 +210,7 @@ def _get_serv(ret=None, commit=False): yield cursor except psycopg2.DatabaseError as err: error = err.args - sys.stderr.write(str(error)) + sys.stderr.write(six.text_type(error)) cursor.execute("ROLLBACK") raise err else: diff --git a/salt/returners/postgres_local_cache.py b/salt/returners/postgres_local_cache.py index 497d8fc56d..95c239a7b5 100644 --- a/salt/returners/postgres_local_cache.py +++ b/salt/returners/postgres_local_cache.py @@ -108,7 +108,7 @@ Required python modules: psycopg2 ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import json import logging import re @@ -148,7 +148,7 @@ def _get_conn(): database=__opts__['master_job_cache.postgres.db'], port=__opts__['master_job_cache.postgres.port']) except psycopg2.OperationalError: - log.error("Could not connect to SQL server: " + str(sys.exc_info()[0])) + log.error('Could not connect to SQL server: %s', sys.exc_info()[0]) return None return conn @@ -232,7 +232,7 @@ def returner(load): sql = '''INSERT INTO salt_returns (fun, jid, return, id, success) VALUES (%s, %s, %s, %s, %s)''' - job_ret = {'return': six.text_type(str(load['return']), 'utf-8', 'replace')} + job_ret = {'return': six.text_type(six.text_type(load['return']), 'utf-8', 'replace')} if 'retcode' in load: job_ret['retcode'] = load['retcode'] if 'success' in load: @@ -288,14 +288,14 @@ def save_load(jid, clear_load, minions=None): sql, ( jid, salt.utils.jid.jid_to_time(jid), - str(clear_load.get("tgt_type")), - str(clear_load.get("cmd")), - str(clear_load.get("tgt")), - str(clear_load.get("kwargs")), - str(clear_load.get("ret")), - str(clear_load.get("user")), - str(json.dumps(clear_load.get("arg"))), - str(clear_load.get("fun")), + six.text_type(clear_load.get("tgt_type")), + six.text_type(clear_load.get("cmd")), + six.text_type(clear_load.get("tgt")), + six.text_type(clear_load.get("kwargs")), + six.text_type(clear_load.get("ret")), + six.text_type(clear_load.get("user")), + six.text_type(json.dumps(clear_load.get("arg"))), + six.text_type(clear_load.get("fun")), ) ) # TODO: Add Metadata support when it is merged from develop @@ -313,7 +313,7 @@ def _escape_jid(jid): ''' Do proper formatting of the jid ''' - jid = str(jid) + jid = six.text_type(jid) jid = re.sub(r"'*", "", jid) return jid @@ -393,7 +393,7 @@ def get_jids(): '''FROM jids''' if __opts__['keep_jobs'] != 0: sql = sql + " WHERE started > NOW() - INTERVAL '" \ - + str(__opts__['keep_jobs']) + "' HOUR" + + six.text_type(__opts__['keep_jobs']) + "' HOUR" cur.execute(sql) ret = {} diff --git a/salt/returners/pushover_returner.py b/salt/returners/pushover_returner.py index ece9d63159..e36d79b058 100644 --- a/salt/returners/pushover_returner.py +++ b/salt/returners/pushover_returner.py @@ -75,7 +75,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return pushover --return_kwargs '{"title": "Salt is awesome!"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import pprint @@ -245,7 +245,7 @@ def returner(ret): sound=sound, token=token) - log.debug('result {0}'.format(result)) + log.debug('pushover result %s', result) if not result['res']: - log.info('Error: {0}'.format(result['message'])) + log.info('Error: %s', result['message']) return diff --git a/salt/returners/rawfile_json.py b/salt/returners/rawfile_json.py index 9755935496..77cbb09db5 100644 --- a/salt/returners/rawfile_json.py +++ b/salt/returners/rawfile_json.py @@ -18,7 +18,7 @@ to restrict the events that are written. ''' # Import python libs -from __future__ import absolute_import, print_function, with_statement +from __future__ import absolute_import, print_function, with_statement, unicode_literals import logging import json @@ -60,7 +60,7 @@ def returner(ret): with salt.utils.files.flopen(opts['filename'], 'a') as logfile: logfile.write(json.dumps(ret)+'\n') except: - log.error('Could not write to rawdata_json file {0}'.format(opts['filename'])) + log.error('Could not write to rawdata_json file %s', opts['filename']) raise @@ -79,5 +79,5 @@ def event_return(events): json.dump(event, logfile) logfile.write('\n') except: - log.error('Could not write to rawdata_json file {0}'.format(opts['filename'])) + log.error('Could not write to rawdata_json file %s', opts['filename']) raise diff --git a/salt/returners/redis_return.py b/salt/returners/redis_return.py index 5eea5bf782..52a304768c 100644 --- a/salt/returners/redis_return.py +++ b/salt/returners/redis_return.py @@ -86,7 +86,7 @@ cluster.skip_full_coverage_check: ``False`` ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import json import logging @@ -305,7 +305,7 @@ def clean_old_jobs(): to_remove.append(ret_key) if len(to_remove) != 0: serv.delete(*to_remove) - log.debug('clean old jobs: {0}'.format(to_remove)) + log.debug('clean old jobs: %s', to_remove) def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument diff --git a/salt/returners/sentry_return.py b/salt/returners/sentry_return.py index b9f8e17c6b..76f09cfacb 100644 --- a/salt/returners/sentry_return.py +++ b/salt/returners/sentry_return.py @@ -41,7 +41,7 @@ tags, allowing tagging of events in the sentry ui. To report only errors to sentry, set report_errors_only: true. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import logging @@ -58,7 +58,7 @@ try: except ImportError: has_raven = False -logger = logging.getLogger(__name__) +log = logging.getLogger(__name__) # Define the module's virtual name __virtualname__ = 'sentry' @@ -81,10 +81,7 @@ def returner(ret): try: _connect_sentry(_get_message(ret), ret) except Exception as err: - logger.error( - 'Can\'t run connect_sentry: {0}'.format(err), - exc_info=True - ) + log.error('Can\'t run connect_sentry: %s', err, exc_info=True) def _ret_is_not_error(result): @@ -164,7 +161,7 @@ def _connect_sentry(message, result): transport=HTTPTransport ) except KeyError as missing_key: - logger.error( + log.error( 'Sentry returner needs key \'%s\' in pillar', missing_key ) @@ -178,12 +175,9 @@ def _connect_sentry(message, result): extra=sentry_data, tags=tags ) - logger.info('Message id {} written to sentry'.format(msgid)) + log.info('Message id %s written to sentry', msgid) except Exception as exc: - logger.error( - 'Can\'t send message to sentry: {0}'.format(exc), - exc_info=True - ) + log.error('Can\'t send message to sentry: %s', exc, exc_info=True) def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument diff --git a/salt/returners/slack_returner.py b/salt/returners/slack_returner.py index 4213027c5f..9a5e8d7cd1 100644 --- a/salt/returners/slack_returner.py +++ b/salt/returners/slack_returner.py @@ -76,7 +76,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return slack --return_kwargs '{"channel": "#random"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import yaml @@ -170,7 +170,7 @@ def _post_message(channel, header_dict={'Content-Type': 'application/x-www-form-urlencoded'}, data=_urlencode(parameters)) - log.debug('result {0}'.format(result)) + log.debug('Slack message post result: %s', result) if result: return True else: diff --git a/salt/returners/sms_return.py b/salt/returners/sms_return.py index 81694115e8..466693400d 100644 --- a/salt/returners/sms_return.py +++ b/salt/returners/sms_return.py @@ -28,7 +28,7 @@ To use the sms returner, append '--return sms' to the salt command. salt '*' test.ping --return sms ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import salt.returners @@ -96,8 +96,10 @@ def returner(ret): ret['id'], ret['fun'], ret['success'], ret['jid'] ), to=receiver, from_=sender) except TwilioRestException as e: - log.error('Twilio [https://www.twilio.com/docs/errors/{0}]'.format( - e.code)) + log.error( + 'Twilio [https://www.twilio.com/docs/errors/%s]', + e.code + ) return False return True diff --git a/salt/returners/smtp_return.py b/salt/returners/smtp_return.py index f630aa18f6..ddd717ffbe 100644 --- a/salt/returners/smtp_return.py +++ b/salt/returners/smtp_return.py @@ -106,7 +106,7 @@ This configuration enables Salt Master to send an email when accepting or reject ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import logging import smtplib @@ -184,7 +184,7 @@ def returner(ret): if not port: port = 25 - log.debug('SMTP port has been set to {0}'.format(port)) + log.debug('SMTP port has been set to %s', port) for field in fields: if field in ret: @@ -198,7 +198,7 @@ def returner(ret): **ret) if isinstance(subject, six.moves.StringIO): subject = subject.read() - log.debug("smtp_return: Subject is '{0}'".format(subject)) + log.debug("smtp_return: Subject is '%s'", subject) template = _options.get('template') if template: @@ -224,7 +224,7 @@ def returner(ret): encrypted_data = gpg.encrypt(content, to_addrs) if encrypted_data.ok: log.debug('smtp_return: Encryption successful') - content = str(encrypted_data) + content = six.text_type(encrypted_data) else: log.error('smtp_return: Encryption failed, only an error message will be sent') content = 'Encryption failed, the return data was not sent.\r\n\r\n{0}\r\n{1}'.format( diff --git a/salt/returners/splunk.py b/salt/returners/splunk.py index af3bb8054e..c949dcbeca 100644 --- a/salt/returners/splunk.py +++ b/salt/returners/splunk.py @@ -17,7 +17,7 @@ Run a test by using ``salt-call test.ping --return splunk`` Written by Scott Pack (github.com/scottjpack) ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import socket @@ -28,6 +28,8 @@ import json import time import logging +from salt.ext import six + _max_content_bytes = 100000 http_event_collector_SSL_verify = False http_event_collector_debug = False @@ -74,7 +76,7 @@ def _send_splunk(event, index_override=None, sourcetype_override=None): ''' #Get Splunk Options opts = _get_options() - logging.info('Options: {0}'.format(json.dumps(opts))) + logging.info('Options: %s', json.dumps(opts)) http_event_collector_key = opts['token'] http_event_collector_host = opts['indexer'] #Set up the collector @@ -94,7 +96,7 @@ def _send_splunk(event, index_override=None, sourcetype_override=None): #Add the event payload.update({"event": event}) - logging.info('Payload: {0}'.format(json.dumps(payload))) + logging.info('Payload: %s', json.dumps(payload)) #fire it off splunk_event.sendEvent(payload) return True @@ -145,7 +147,7 @@ class http_event_collector(object): # If eventtime in epoch not passed as optional argument use current system time in epoch if not eventtime: - eventtime = str(int(time.time())) + eventtime = six.text_type(int(time.time())) # Fill in local hostname if not manually populated if 'host' not in payload: @@ -182,7 +184,7 @@ class http_event_collector(object): # If eventtime in epoch not passed as optional argument use current system time in epoch if not eventtime: - eventtime = str(int(time.time())) + eventtime = six.text_type(int(time.time())) # Update time value on payload if need to use system time data = {"time": eventtime} diff --git a/salt/returners/sqlite3_return.py b/salt/returners/sqlite3_return.py index 1b2159980d..2458b55911 100644 --- a/salt/returners/sqlite3_return.py +++ b/salt/returners/sqlite3_return.py @@ -80,7 +80,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return sqlite3 --return_kwargs '{"db": "/var/lib/salt/another-salt.db"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -91,6 +91,9 @@ import datetime import salt.utils.jid import salt.returners +# Import 3rd-party libs +from salt.ext import six + # Better safe than sorry here. Even though sqlite3 is included in python try: import sqlite3 @@ -141,9 +144,7 @@ def _get_conn(ret=None): if not timeout: raise Exception( 'sqlite3 config option "sqlite3.timeout" is missing') - log.debug('Connecting the sqlite3 database: {0} timeout: {1}'.format( - database, - timeout)) + log.debug('Connecting the sqlite3 database: %s timeout: %s', database, timeout) conn = sqlite3.connect(database, timeout=float(timeout)) return conn @@ -161,7 +162,7 @@ def returner(ret): ''' Insert minion return data into the sqlite3 database ''' - log.debug('sqlite3 returner called with data: {0}'.format(ret)) + log.debug('sqlite3 returner called with data: %s', ret) conn = _get_conn(ret) cur = conn.cursor() sql = '''INSERT INTO salt_returns @@ -171,8 +172,8 @@ def returner(ret): {'fun': ret['fun'], 'jid': ret['jid'], 'id': ret['id'], - 'fun_args': str(ret['fun_args']) if ret.get('fun_args') else None, - 'date': str(datetime.datetime.now()), + 'fun_args': six.text_type(ret['fun_args']) if ret.get('fun_args') else None, + 'date': six.text_type(datetime.datetime.now()), 'full_ret': json.dumps(ret['return']), 'success': ret.get('success', '')}) _close_conn(conn) @@ -182,8 +183,7 @@ def save_load(jid, load, minions=None): ''' Save the load to the specified jid ''' - log.debug('sqlite3 returner called jid:{0} load:{1}' - .format(jid, load)) + log.debug('sqlite3 returner called jid: %s load: %s', jid, load) conn = _get_conn(ret=None) cur = conn.cursor() sql = '''INSERT INTO jids (jid, load) VALUES (:jid, :load)''' @@ -204,7 +204,7 @@ def get_load(jid): ''' Return the load from a specified jid ''' - log.debug('sqlite3 returner called jid: {0}'.format(jid)) + log.debug('sqlite3 returner called jid: %s', jid) conn = _get_conn(ret=None) cur = conn.cursor() sql = '''SELECT load FROM jids WHERE jid = :jid''' @@ -221,18 +221,18 @@ def get_jid(jid): ''' Return the information returned from a specified jid ''' - log.debug('sqlite3 returner called jid: {0}'.format(jid)) + log.debug('sqlite3 returner called jid: %s', jid) conn = _get_conn(ret=None) cur = conn.cursor() sql = '''SELECT id, full_ret FROM salt_returns WHERE jid = :jid''' cur.execute(sql, {'jid': jid}) data = cur.fetchone() - log.debug('query result: {0}'.format(data)) + log.debug('query result: %s', data) ret = {} if data and len(data) > 1: - ret = {str(data[0]): {u'return': json.loads(data[1])}} - log.debug("ret: {0}".format(ret)) + ret = {six.text_type(data[0]): {u'return': json.loads(data[1])}} + log.debug('ret: %s', ret) _close_conn(conn) return ret @@ -241,7 +241,7 @@ def get_fun(fun): ''' Return a dict of the last function called for all minions ''' - log.debug('sqlite3 returner called fun: {0}'.format(fun)) + log.debug('sqlite3 returner called fun: %s', fun) conn = _get_conn(ret=None) cur = conn.cursor() sql = '''SELECT s.id, s.full_ret, s.jid diff --git a/salt/returners/syslog_return.py b/salt/returners/syslog_return.py index f963c751d8..3fbca3e5f1 100644 --- a/salt/returners/syslog_return.py +++ b/salt/returners/syslog_return.py @@ -86,7 +86,7 @@ To override individual configuration items, append implmentation's documentation to determine how to adjust this limit. ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import python libs @@ -148,10 +148,10 @@ def _verify_options(options): for opt_name, opt in bitwise_args: if not hasattr(syslog, opt): - log.error('syslog has no attribute {0}'.format(opt)) + log.error('syslog has no attribute %s', opt) return False if not isinstance(getattr(syslog, opt), int): - log.error('{0} is not a valid syslog {1}'.format(opt, opt_name)) + log.error('%s is not a valid syslog %s', opt, opt_name) return False # Sanity check tag diff --git a/salt/returners/telegram_return.py b/salt/returners/telegram_return.py index ceacaa78a9..42f266ae39 100644 --- a/salt/returners/telegram_return.py +++ b/salt/returners/telegram_return.py @@ -22,7 +22,7 @@ To use the Telegram return, append '--return telegram' to the salt command. salt '*' test.ping --return telegram ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import logging @@ -60,7 +60,7 @@ def _get_options(ret=None): attrs, __salt__=__salt__, __opts__=__opts__) - log.debug('Options: {0}'.format(_options)) + log.debug('Options: %s', _options) return _options diff --git a/salt/returners/xmpp_return.py b/salt/returners/xmpp_return.py index 91f96cdfbb..0587453f2b 100644 --- a/salt/returners/xmpp_return.py +++ b/salt/returners/xmpp_return.py @@ -67,7 +67,7 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return xmpp --return_kwargs '{"recipient": "someone-else@xmpp.example.com"}' ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging diff --git a/salt/returners/zabbix_return.py b/salt/returners/zabbix_return.py index 22f9168c8e..f5747b389f 100644 --- a/salt/returners/zabbix_return.py +++ b/salt/returners/zabbix_return.py @@ -20,7 +20,7 @@ To use the Zabbix returner, append '--return zabbix' to the salt command. ex: ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import os diff --git a/tests/integration/renderers/test_pydsl.py b/tests/integration/renderers/test_pydsl.py index 0d904b4ce0..fc15d238aa 100644 --- a/tests/integration/renderers/test_pydsl.py +++ b/tests/integration/renderers/test_pydsl.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import textwrap @@ -11,6 +11,7 @@ from tests.support.case import ModuleCase # Import Salt libs import salt.utils.files import salt.utils.platform +import salt.utils.stringutils class PyDSLRendererIncludeTestCase(ModuleCase): @@ -49,7 +50,7 @@ class PyDSLRendererIncludeTestCase(ModuleCase): 'hello blue 3 \r\n' with salt.utils.files.fopen('/tmp/output', 'r') as f: - ret = f.read() + ret = salt.utils.stringutils.to_unicode(f.read()) os.remove('/tmp/output') diff --git a/tests/integration/returners/test_librato_return.py b/tests/integration/returners/test_librato_return.py index d9814ad3bf..90fce66346 100644 --- a/tests/integration/returners/test_librato_return.py +++ b/tests/integration/returners/test_librato_return.py @@ -3,7 +3,7 @@ Tests for the librato returner ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging # Import Salt Testing libs diff --git a/tests/unit/renderers/test_gpg.py b/tests/unit/renderers/test_gpg.py index b4b26e3828..71b1d8abdf 100644 --- a/tests/unit/renderers/test_gpg.py +++ b/tests/unit/renderers/test_gpg.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin diff --git a/tests/unit/renderers/test_nacl.py b/tests/unit/renderers/test_nacl.py index 04aa31dd88..de0e702978 100644 --- a/tests/unit/renderers/test_nacl.py +++ b/tests/unit/renderers/test_nacl.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin diff --git a/tests/unit/renderers/test_yaml.py b/tests/unit/renderers/test_yaml.py index 408956d3c7..7eb82b89d0 100644 --- a/tests/unit/renderers/test_yaml.py +++ b/tests/unit/renderers/test_yaml.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin diff --git a/tests/unit/renderers/test_yamlex.py b/tests/unit/renderers/test_yamlex.py index 8c5685c6c4..e3a8af41ff 100644 --- a/tests/unit/renderers/test_yamlex.py +++ b/tests/unit/renderers/test_yamlex.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin diff --git a/tests/unit/returners/test_local_cache.py b/tests/unit/returners/test_local_cache.py index aa7117efb5..aec5efc4b4 100644 --- a/tests/unit/returners/test_local_cache.py +++ b/tests/unit/returners/test_local_cache.py @@ -7,7 +7,7 @@ Unit tests for the Default Job Cache (local_cache). ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import os import shutil import logging @@ -217,7 +217,7 @@ class Local_CacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleM are either present or removed ''' for content in contents: - log.debug('CONTENT {0}'.format(content)) + log.debug('CONTENT %s', content) if status == 'present': check_job_dir = os.path.exists(content) elif status == 'removed': diff --git a/tests/unit/returners/test_sentry.py b/tests/unit/returners/test_sentry.py index c2d4994cc5..ccf235919a 100644 --- a/tests/unit/returners/test_sentry.py +++ b/tests/unit/returners/test_sentry.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.unit import TestCase diff --git a/tests/unit/returners/test_smtp_return.py b/tests/unit/returners/test_smtp_return.py index 92c91e7687..24dea61c05 100644 --- a/tests/unit/returners/test_smtp_return.py +++ b/tests/unit/returners/test_smtp_return.py @@ -8,7 +8,7 @@ ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin From ba0cbea689c4bec893f5b5129b0462b525f14c1a Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Fri, 15 Dec 2017 12:31:08 -0500 Subject: [PATCH 51/54] Lint: Remove extra blank line --- salt/cloud/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index c42522be48..5da1db2837 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -186,7 +186,6 @@ class CloudClient(object): else: self.opts = salt.config.cloud_config(path) - # Check the cache-dir exists. If not, create it. v_dirs = [self.opts['cachedir']] salt.utils.verify.verify_env(v_dirs, salt.utils.get_user()) From 0b88ea7eb46341003f0fe0de99b2d0887578d32c Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Fri, 15 Dec 2017 12:31:43 -0500 Subject: [PATCH 52/54] Lint: Remove unused import --- salt/cloud/clouds/gce.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index 52dc1ae116..5c4d9b1e07 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -54,7 +54,6 @@ import re import pprint import logging import msgpack -import traceback from ast import literal_eval from salt.utils.versions import LooseVersion as _LooseVersion From 4d6abeff6676e5cf0ae4d1efadd1263c0d56e269 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 15 Dec 2017 14:30:13 -0500 Subject: [PATCH 53/54] Fix swap_total for macosx grain detection --- salt/grains/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 5d9d38d497..81962d8c97 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -425,14 +425,14 @@ def _osx_memdata(): sysctl = salt.utils.path.which('sysctl') if sysctl: mem = __salt__['cmd.run']('{0} -n hw.memsize'.format(sysctl)) - swap_total = __salt__['cmd.run']('{0} -n vm.swapusage').split()[2] + swap_total = __salt__['cmd.run']('{0} -n vm.swapusage'.format(sysctl)).split()[2] if swap_total.endswith('K'): _power = 2**10 elif swap_total.endswith('M'): _power = 2**20 elif swap_total.endswith('G'): _power = 2**30 - swap_total = swap_total[:-1] * _power + swap_total = float(swap_total[:-1]) * _power grains['mem_total'] = int(mem) // 1024 // 1024 grains['swap_total'] = int(swap_total) // 1024 // 1024 From ecb1e3cf825b94b38f71fb54f4eef96ce47dbb6f Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 15 Dec 2017 14:32:32 -0500 Subject: [PATCH 54/54] Fix a couple of PY3 test failures in the rsync module Fixes: - unit.modules.test_rsync.RsyncTestCase.test_rsync - unit.modules.test_rsync.RsyncTestCase.test_version --- salt/modules/rsync.py | 2 +- tests/unit/modules/test_rsync.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/rsync.py b/salt/modules/rsync.py index 2ed626b6e2..5fb33960df 100644 --- a/salt/modules/rsync.py +++ b/salt/modules/rsync.py @@ -7,7 +7,7 @@ Wrapper for rsync This data can also be passed into :ref:`pillar `. Options passed into opts will overwrite options passed into pillar. ''' -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # Import python libs import errno diff --git a/tests/unit/modules/test_rsync.py b/tests/unit/modules/test_rsync.py index cf03c722fc..b95011872e 100644 --- a/tests/unit/modules/test_rsync.py +++ b/tests/unit/modules/test_rsync.py @@ -3,7 +3,7 @@ :codeauthor: :email:`Jayesh Kariya ` ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -37,7 +37,7 @@ class RsyncTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(rsync.__salt__, {'config.option': MagicMock(return_value='A'), - 'cmd.run_all': MagicMock(side_effect=[IOError('f'), + 'cmd.run_all': MagicMock(side_effect=[OSError(1, 'f'), 'A'])}): with patch.object(rsync, '_check', return_value=['A']): self.assertRaises(CommandExecutionError, rsync.rsync, 'a', 'b') @@ -48,7 +48,7 @@ class RsyncTestCase(TestCase, LoaderModuleMockMixin): ''' Test for return rsync version ''' - mock = MagicMock(side_effect=[IOError('f'), 'A B C\n']) + mock = MagicMock(side_effect=[OSError(1, 'f'), 'A B C\n']) with patch.dict(rsync.__salt__, {'cmd.run_stdout': mock}): self.assertRaises(CommandExecutionError, rsync.version)