diff --git a/salt/_compat.py b/salt/_compat.py index 9b10646ace..0576210afc 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -2,18 +2,21 @@ ''' Salt compatibility code ''' -# pylint: disable=import-error,unused-import,invalid-name +# pylint: disable=import-error,unused-import,invalid-name,W0231,W0233 # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals, print_function import sys import types +import logging # Import 3rd-party libs -from salt.ext.six import binary_type, string_types, text_type +from salt.exceptions import SaltException +from salt.ext.six import binary_type, string_types, text_type, integer_types from salt.ext.six.moves import cStringIO, StringIO -HAS_XML = True +log = logging.getLogger(__name__) + try: # Python >2.5 import xml.etree.cElementTree as ElementTree @@ -31,11 +34,10 @@ except Exception: import elementtree.ElementTree as ElementTree except Exception: ElementTree = None - HAS_XML = False # True if we are running on Python 3. -PY3 = sys.version_info[0] == 3 +PY3 = sys.version_info.major == 3 if PY3: @@ -45,13 +47,12 @@ else: import exceptions -if HAS_XML: +if ElementTree is not None: if not hasattr(ElementTree, 'ParseError'): class ParseError(Exception): ''' older versions of ElementTree do not have ParseError ''' - pass ElementTree.ParseError = ParseError @@ -61,9 +62,7 @@ def text_(s, encoding='latin-1', errors='strict'): If ``s`` is an instance of ``binary_type``, return ``s.decode(encoding, errors)``, otherwise return ``s`` ''' - if isinstance(s, binary_type): - return s.decode(encoding, errors) - return s + return s.decode(encoding, errors) if isinstance(s, binary_type) else s def bytes_(s, encoding='latin-1', errors='strict'): @@ -71,57 +70,37 @@ def bytes_(s, encoding='latin-1', errors='strict'): If ``s`` is an instance of ``text_type``, return ``s.encode(encoding, errors)``, otherwise return ``s`` ''' + return s.encode(encoding, errors) if isinstance(s, text_type) else s + + +def ascii_native_(s): + ''' + Python 3: If ``s`` is an instance of ``text_type``, return + ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` + + Python 2: If ``s`` is an instance of ``text_type``, return + ``s.encode('ascii')``, otherwise return ``str(s)`` + ''' if isinstance(s, text_type): - return s.encode(encoding, errors) - return s + s = s.encode('ascii') + + return str(s, 'ascii', 'strict') if PY3 else s -if PY3: - def ascii_native_(s): - if isinstance(s, text_type): - s = s.encode('ascii') - return str(s, 'ascii', 'strict') -else: - def ascii_native_(s): - if isinstance(s, text_type): - s = s.encode('ascii') - return str(s) +def native_(s, encoding='latin-1', errors='strict'): + ''' + Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise + return ``str(s, encoding, errors)`` -ascii_native_.__doc__ = ''' -Python 3: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` + Python 2: If ``s`` is an instance of ``text_type``, return + ``s.encode(encoding, errors)``, otherwise return ``str(s)`` + ''' + if PY3: + out = s if isinstance(s, text_type) else str(s, encoding, errors) + else: + out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s) -Python 2: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s)`` -''' - - -if PY3: - def native_(s, encoding='latin-1', errors='strict'): - ''' - If ``s`` is an instance of ``text_type``, return - ``s``, otherwise return ``str(s, encoding, errors)`` - ''' - if isinstance(s, text_type): - return s - return str(s, encoding, errors) -else: - def native_(s, encoding='latin-1', errors='strict'): - ''' - If ``s`` is an instance of ``text_type``, return - ``s.encode(encoding, errors)``, otherwise return ``str(s)`` - ''' - if isinstance(s, text_type): - return s.encode(encoding, errors) - return str(s) - -native_.__doc__ = ''' -Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise -return ``str(s, encoding, errors)`` - -Python 2: If ``s`` is an instance of ``text_type``, return -``s.encode(encoding, errors)``, otherwise return ``str(s)`` -''' + return out def string_io(data=None): # cStringIO can't handle unicode @@ -133,7 +112,199 @@ def string_io(data=None): # cStringIO can't handle unicode except (UnicodeEncodeError, TypeError): return StringIO(data) -if PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress + +try: + if PY3: + import ipaddress + else: + import salt.ext.ipaddress as ipaddress +except ImportError: + ipaddress = None + + +class IPv6AddressScoped(ipaddress.IPv6Address): + ''' + Represent and manipulate single IPv6 Addresses. + Scope-aware version + ''' + def __init__(self, address): + ''' + Instantiate a new IPv6 address object. Scope is moved to an attribute 'scope'. + + Args: + address: A string or integer representing the IP + + Additionally, an integer can be passed, so + IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560) + or, more generally + IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::') + + Raises: + AddressValueError: If address isn't a valid IPv6 address. + + :param address: + ''' + # pylint: disable-all + if not hasattr(self, '_is_packed_binary'): + # This method (below) won't be around for some Python 3 versions + # and we need check this differently anyway + self._is_packed_binary = lambda p: isinstance(p, bytes) + # pylint: enable-all + + if isinstance(address, string_types) and '%' in address: + buff = address.split('%') + if len(buff) != 2: + raise SaltException('Invalid IPv6 address: "{}"'.format(address)) + address, self.__scope = buff + else: + self.__scope = None + + if sys.version_info.major == 2: + ipaddress._BaseAddress.__init__(self, address) + ipaddress._BaseV6.__init__(self, address) + else: + # Python 3.4 fix. Versions higher are simply not affected + # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544 + self._version = 6 + self._max_prefixlen = ipaddress.IPV6LENGTH + + # Efficient constructor from integer. + if isinstance(address, integer_types): + self._check_int_address(address) + self._ip = address + elif self._is_packed_binary(address): + self._check_packed_address(address, 16) + self._ip = ipaddress._int_from_bytes(address, 'big') + else: + address = str(address) + if '/' in address: + raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address)) + self._ip = self._ip_int_from_string(address) + + def _is_packed_binary(self, data): + ''' + Check if data is hexadecimal packed + + :param data: + :return: + ''' + packed = False + if len(data) == 16 and ':' not in data: + try: + packed = bool(int(str(bytearray(data)).encode('hex'), 16)) + except ValueError: + pass + + return packed + + @property + def scope(self): + ''' + Return scope of IPv6 address. + + :return: + ''' + return self.__scope + + def __str__(self): + return text_type(self._string_from_ip_int(self._ip) + + ('%' + self.scope if self.scope is not None else '')) + + +class IPv6InterfaceScoped(ipaddress.IPv6Interface, IPv6AddressScoped): + ''' + Update + ''' + def __init__(self, address): + if isinstance(address, (bytes, int)): + IPv6AddressScoped.__init__(self, address) + self.network = ipaddress.IPv6Network(self._ip) + self._prefixlen = self._max_prefixlen + return + + addr = ipaddress._split_optional_netmask(address) + IPv6AddressScoped.__init__(self, addr[0]) + self.network = ipaddress.IPv6Network(address, strict=False) + self.netmask = self.network.netmask + self._prefixlen = self.network._prefixlen + self.hostmask = self.network.hostmask + + +def ip_address(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Address or IPv6Address object. + + Raises: + ValueError: if the *address* passed isn't either a v4 or a v6 + address + + """ + try: + return ipaddress.IPv4Address(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while parsing IPv4 address: %s', address) + log.debug(err) + + try: + return IPv6AddressScoped(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while parsing IPv6 address: %s', address) + log.debug(err) + + if isinstance(address, bytes): + raise ipaddress.AddressValueError('{} does not appear to be an IPv4 or IPv6 address. ' + 'Did you pass in a bytes (str in Python 2) instead ' + 'of a unicode object?'.format(repr(address))) + + raise ValueError('{} does not appear to be an IPv4 or IPv6 address'.format(repr(address))) + + +def ip_interface(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Interface or IPv6Interface object. + + Raises: + ValueError: if the string passed isn't either a v4 or a v6 + address. + + Notes: + The IPv?Interface classes describe an Address on a particular + Network, so they're basically a combination of both the Address + and Network classes. + + """ + try: + return ipaddress.IPv4Interface(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while getting IPv4 interface for address %s', address) + log.debug(err) + + try: + return ipaddress.IPv6Interface(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while getting IPv6 interface for address %s', address) + log.debug(err) + + raise ValueError('{} does not appear to be an IPv4 or IPv6 interface'.format(address)) + + +if ipaddress: + ipaddress.IPv6Address = IPv6AddressScoped + if sys.version_info.major == 2: + ipaddress.IPv6Interface = IPv6InterfaceScoped + ipaddress.ip_address = ip_address + ipaddress.ip_interface = ip_interface diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index 95ba03c863..00ed2dc969 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -27,11 +27,7 @@ import salt.utils.cloud import salt.config as config import salt.client import salt.ext.six as six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress - +from salt._compat import ipaddress from salt.exceptions import SaltCloudException, SaltCloudSystemExit # Get logging started diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py index a24170c78a..0fe410eb91 100644 --- a/salt/cloud/clouds/vagrant.py +++ b/salt/cloud/clouds/vagrant.py @@ -25,13 +25,8 @@ import tempfile import salt.utils import salt.config as config import salt.client -import salt.ext.six as six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress -from salt.exceptions import SaltCloudException, SaltCloudSystemExit, \ - SaltInvocationError +from salt._compat import ipaddress +from salt.exceptions import SaltCloudException, SaltCloudSystemExit, SaltInvocationError # Get logging started log = logging.getLogger(__name__) diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index 1204bede10..89aba14ce9 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import socket import ctypes import os -import ipaddress +from salt._compat import ipaddress import salt.ext.six as six diff --git a/salt/minion.py b/salt/minion.py index f833920881..3d01481f7f 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -27,10 +27,7 @@ from binascii import crc32 # Import Salt Libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext import six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress from salt.ext.six.moves import range from salt.utils.zeromq import zmq, ZMQDefaultLoop, install_zmq, ZMQ_VERSION_INFO import salt.defaults.exitcodes diff --git a/salt/modules/ipset.py b/salt/modules/ipset.py index 7047e84c29..1a0fa0044d 100644 --- a/salt/modules/ipset.py +++ b/salt/modules/ipset.py @@ -13,10 +13,7 @@ from salt.ext.six.moves import map, range import salt.utils.path # Import third-party libs -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress # Set up logging log = logging.getLogger(__name__) diff --git a/salt/modules/network.py b/salt/modules/network.py index bdf73dfc89..433523681f 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -26,11 +26,7 @@ from salt.exceptions import CommandExecutionError # Import 3rd-party libs from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress - +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/salt/modules/vagrant.py b/salt/modules/vagrant.py index 0592dede55..0f518c2602 100644 --- a/salt/modules/vagrant.py +++ b/salt/modules/vagrant.py @@ -39,11 +39,7 @@ import salt.utils.path import salt.utils.stringutils from salt.exceptions import CommandExecutionError, SaltInvocationError import salt.ext.six as six - -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/salt/modules/win_network.py b/salt/modules/win_network.py index 5e98d17cde..51ed1778bd 100644 --- a/salt/modules/win_network.py +++ b/salt/modules/win_network.py @@ -32,10 +32,7 @@ try: import wmi # pylint: disable=W0611 except ImportError: HAS_DEPENDENCIES = False -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress # Define the module's virtual name __virtualname__ = 'network' diff --git a/salt/pillar/netbox.py b/salt/pillar/netbox.py index 71d99b3bba..21cc3670eb 100644 --- a/salt/pillar/netbox.py +++ b/salt/pillar/netbox.py @@ -53,7 +53,7 @@ import logging # Import Salt libs import salt.utils.http -import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/salt/utils/dns.py b/salt/utils/dns.py index 3687491545..687b9ac4c1 100644 --- a/salt/utils/dns.py +++ b/salt/utils/dns.py @@ -1139,18 +1139,13 @@ def parse_resolv(src='/etc/resolv.conf'): try: (directive, arg) = (line[0].lower(), line[1:]) # Drop everything after # or ; (comments) - arg = list(itertools.takewhile( - lambda x: x[0] not in ('#', ';'), arg)) - + arg = list(itertools.takewhile(lambda x: x[0] not in ('#', ';'), arg)) if directive == 'nameserver': - # Split the scope (interface) if it is present - addr, scope = arg[0].split('%', 1) if '%' in arg[0] else (arg[0], '') + addr = arg[0] try: ip_addr = ipaddress.ip_address(addr) version = ip_addr.version - # Rejoin scope after address validation - if scope: - ip_addr = '%'.join((str(ip_addr), scope)) + ip_addr = str(ip_addr) if ip_addr not in nameservers: nameservers.append(ip_addr) if version == 4 and ip_addr not in ip4_nameservers: diff --git a/salt/utils/minions.py b/salt/utils/minions.py index 721058e416..6ee93c2d73 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -26,10 +26,7 @@ import salt.cache from salt.ext import six # Import 3rd-party libs -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress HAS_RANGE = False try: import seco.range # pylint: disable=import-error diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index a731b7c296..d477f3b692 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -36,10 +36,7 @@ import salt.grains.core as core # Import 3rd-party libs from salt.ext import six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/tests/unit/modules/test_network.py b/tests/unit/modules/test_network.py index d616d8b2ba..9ed307f1dc 100644 --- a/tests/unit/modules/test_network.py +++ b/tests/unit/modules/test_network.py @@ -20,20 +20,11 @@ from tests.support.mock import ( ) # Import Salt Libs -from salt.ext import six import salt.utils.network import salt.utils.path import salt.modules.network as network from salt.exceptions import CommandExecutionError -if six.PY2: - import salt.ext.ipaddress as ipaddress - HAS_IPADDRESS = True -else: - try: - import ipaddress - HAS_IPADDRESS = True - except ImportError: - HAS_IPADDRESS = False +from salt._compat import ipaddress @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -276,7 +267,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(network.connect('host', 'port'), {'comment': ret, 'result': True}) - @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') + @skipIf(bool(ipaddress) is False, 'unable to import \'ipaddress\'') def test_is_private(self): ''' Test for Check if the given IP address is a private address @@ -288,7 +279,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): return_value=True): self.assertTrue(network.is_private('::1')) - @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') + @skipIf(bool(ipaddress) is False, 'unable to import \'ipaddress\'') def test_is_loopback(self): ''' Test for Check if the given IP address is a loopback address diff --git a/tests/unit/utils/test_jinja.py b/tests/unit/utils/test_jinja.py index fea6a2c0b1..1748197936 100644 --- a/tests/unit/utils/test_jinja.py +++ b/tests/unit/utils/test_jinja.py @@ -996,6 +996,10 @@ class TestCustomExtensions(TestCase): ''' Test the `ipaddr` Jinja filter. ''' + rendered = render_jinja_tmpl("{{ '::' | ipaddr }}", + dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) + self.assertEqual(rendered, '::') + rendered = render_jinja_tmpl("{{ '192.168.0.1' | ipaddr }}", dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) self.assertEqual(rendered, '192.168.0.1')