Merge branch '2017.7' into pillar_masterless

doc/topics/grains/index.rst

@@ -80,12 +80,21 @@ same way as in the above example, only without a top-level ``grains:`` key:
 
 .. note::
 
-    The content of ``/etc/salt/grains`` is ignored if you specify grains in the minion config.
+    Grains in ``/etc/salt/grains`` are ignored if you specify the same grains in the minion config.
 
 .. note::
 
     Grains are static, and since they are not often changed, they will need a grains refresh when they are updated. You can do this by calling: ``salt minion saltutil.refresh_modules``
 
+.. note::
+
+    You can equally configure static grains for Proxy Minions.
+    As multiple Proxy Minion processes can run on the same machine, you need
+    to index the files using the Minion ID, under ``/etc/salt/proxy.d/<minion ID>/grains``.
+    For example, the grains for the Proxy Minion ``router1`` can be defined
+    under ``/etc/salt/proxy.d/router1/grains``, while the grains for the
+    Proxy Minion ``switch7`` can be put in ``/etc/salt/proxy.d/switch7/grains``.
+
 Matching Grains in the Top File
 ===============================
 

salt/grains/extra.py

@@ -12,6 +12,7 @@ import logging
 # Import salt libs
 import salt.utils
 
+__proxyenabled__ = ['*']
 log = logging.getLogger(__name__)
 
 
@@ -31,16 +32,33 @@ def config():
     if 'conf_file' not in __opts__:
         return {}
     if os.path.isdir(__opts__['conf_file']):
-        gfn = os.path.join(
-                __opts__['conf_file'],
-                'grains'
-                )
+        if salt.utils.is_proxy():
+            gfn = os.path.join(
+                    __opts__['conf_file'],
+                    'proxy.d',
+                    __opts__['id'],
+                    'grains'
+                    )
+        else:
+            gfn = os.path.join(
+                    __opts__['conf_file'],
+                    'grains'
+                    )
     else:
-        gfn = os.path.join(
-                os.path.dirname(__opts__['conf_file']),
-                'grains'
-                )
+        if salt.utils.is_proxy():
+            gfn = os.path.join(
+                    os.path.dirname(__opts__['conf_file']),
+                    'proxy.d',
+                    __opts__['id'],
+                    'grains'
+                    )
+        else:
+            gfn = os.path.join(
+                    os.path.dirname(__opts__['conf_file']),
+                    'grains'
+                    )
     if os.path.isfile(gfn):
+        log.debug('Loading static grains from %s', gfn)
         with salt.utils.fopen(gfn, 'rb') as fp_:
             try:
                 return yaml.safe_load(fp_.read())

salt/modules/grains.py

@@ -1,6 +1,13 @@
 # -*- coding: utf-8 -*-
 '''
 Return/control aspects of the grains data
+
+Grains set or altered with this module are stored in the 'grains'
+file on the minions. By default, this file is located at: ``/etc/salt/grains``
+
+.. Note::
+
+   This does **NOT** override any grains set in the minion config file.
 '''
 
 # Import python libs

salt/modules/napalm_network.py

@@ -228,7 +228,7 @@ def _config_logic(napalm_device,
 
 
 @proxy_napalm_wrap
-def connected(**kwarvs):  # pylint: disable=unused-argument
+def connected(**kwargs):  # pylint: disable=unused-argument
     '''
     Specifies if the connection to the device succeeded.
 
@@ -932,6 +932,7 @@ def load_config(filename=None,
                 debug=False,
                 replace=False,
                 inherit_napalm_device=None,
+                saltenv='base',
                 **kwargs):  # pylint: disable=unused-argument
     '''
     Applies configuration changes on the device. It can be loaded from a file or from inline string.
@@ -947,10 +948,21 @@ def load_config(filename=None,
     To replace the config, set ``replace`` to ``True``.
 
     filename
-        Path to the file containing the desired configuration. By default is None.
+        Path to the file containing the desired configuration.
+        This can be specified using the absolute path to the file,
+        or using one of the following URL schemes:
+
+        - ``salt://``, to fetch the template from the Salt fileserver.
+        - ``http://`` or ``https://``
+        - ``ftp://``
+        - ``s3://``
+        - ``swift://``
+
+        .. versionchanged:: 2017.7.3
 
     text
         String containing the desired configuration.
+        This argument is ignored when ``filename`` is specified.
 
     test: False
         Dry run? If set as ``True``, will apply the config, discard and return the changes. Default: ``False``
@@ -970,6 +982,11 @@ def load_config(filename=None,
 
         .. versionadded:: 2016.11.2
 
+    saltenv: ``base``
+        Specifies the Salt environment name.
+
+        .. versionadded:: 2017.7.3
+
     :return: a dictionary having the following keys:
 
     * result (bool): if the config was applied successfully. It is ``False`` only in case of failure. In case \
@@ -999,7 +1016,6 @@ def load_config(filename=None,
             'diff': '[edit interfaces xe-0/0/5]+   description "Adding a description";'
         }
     '''
-
     fun = 'load_merge_candidate'
     if replace:
         fun = 'load_replace_candidate'
@@ -1012,21 +1028,28 @@ def load_config(filename=None,
         # compare_config, discard / commit
         # which have to be over the same session
         napalm_device['CLOSE'] = False  # pylint: disable=undefined-variable
+    if filename:
+        text = __salt__['cp.get_file_str'](filename, saltenv=saltenv)
+        if text is False:
+            # When using salt:// or https://, if the resource is not available,
+            #   it will either raise an exception, or return False.
+            ret = {
+                'result': False,
+                'out': None
+            }
+            ret['comment'] = 'Unable to read from {}. Please specify a valid file or text.'.format(filename)
+            log.error(ret['comment'])
+            return ret
     _loaded = salt.utils.napalm.call(
         napalm_device,  # pylint: disable=undefined-variable
         fun,
         **{
-            'filename': filename,
             'config': text
         }
     )
     loaded_config = None
     if debug:
-        if filename:
-            with salt.utils.fopen(filename) as rfh:
-                loaded_config = rfh.read()
-        else:
-            loaded_config = text
+        loaded_config = text
     return _config_logic(napalm_device,  # pylint: disable=undefined-variable
                          _loaded,
                          test=test,

salt/states/file.py

@@ -3836,11 +3836,11 @@ def replace(name,
 
             If you need to match a literal string that contains regex special
             characters, you may want to use salt's custom Jinja filter,
-            ``escape_regex``.
+            ``regex_escape``.
 
             .. code-block:: jinja
 
-                {{ 'http://example.com?foo=bar%20baz' | escape_regex }}
+                {{ 'http://example.com?foo=bar%20baz' | regex_escape }}
 
     repl
         The replacement text

salt/states/grains.py

@@ -4,10 +4,13 @@ Manage grains on the minion
 ===========================
 
 This state allows for grains to be set.
-Grains set or altered this way are stored in the 'grains'
-file on the minions, by default at: /etc/salt/grains
 
-Note: This does NOT override any grains set in the minion file.
+Grains set or altered with this module are stored in the 'grains'
+file on the minions, By default, this file is located at: ``/etc/salt/grains``
+
+.. Note::
+
+   This does **NOT** override any grains set in the minion config file.
 '''
 
 # Import Python libs

salt/utils/jinja.py

@@ -637,11 +637,11 @@ class SerializerExtension(Extension, object):
 
     .. code-block:: jinja
 
-        escape_regex = {{ 'https://example.com?foo=bar%20baz' | escape_regex }}
+        regex_escape = {{ 'https://example.com?foo=bar%20baz' | regex_escape }}
 
     will be rendered as::
 
-        escape_regex = https\\:\\/\\/example\\.com\\?foo\\=bar\\%20baz
+        regex_escape = https\\:\\/\\/example\\.com\\?foo\\=bar\\%20baz
 
     ** Set Theory Filters **
 

tests/integration/files/file/base/requisites/watch_in.sls

@@ -0,0 +1,7 @@
+return_changes:
+  test.succeed_with_changes:
+    - watch_in:
+      - test: watch_states
+
+watch_states:
+  test.succeed_without_changes

tests/integration/files/file/base/requisites/watch_in_failure.sls

@@ -0,0 +1,7 @@
+return_changes:
+  test.fail_with_changes:
+    - watch_in:
+      - test: watch_states
+
+watch_states:
+  test.succeed_without_changes

tests/integration/files/file/base/ssh_state_tests.sls

@@ -1,4 +1,5 @@
+{% set jinja = 'test' %}
 ssh-file-test:
   file.managed:
-    - name: /tmp/test
+    - name: /tmp/{{ jinja }}
     - contents: 'test'

tests/integration/modules/test_state.py

@@ -586,6 +586,33 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin):
         #result = self.normalize_ret(ret)
         #self.assertEqual(expected_result, result)
 
+    def test_watch_in(self):
+        '''
+        test watch_in requisite when there is a success
+        '''
+        ret = self.run_function('state.sls', mods='requisites.watch_in')
+        changes = 'test_|-return_changes_|-return_changes_|-succeed_with_changes'
+        watch = 'test_|-watch_states_|-watch_states_|-succeed_without_changes'
+
+        self.assertEqual(ret[changes]['__run_num__'], 0)
+        self.assertEqual(ret[watch]['__run_num__'], 2)
+
+        self.assertEqual('Watch statement fired.', ret[watch]['comment'])
+        self.assertEqual('Something pretended to change',
+                         ret[changes]['changes']['testing']['new'])
+
+    def test_watch_in_failure(self):
+        '''
+        test watch_in requisite when there is a failure
+        '''
+        ret = self.run_function('state.sls', mods='requisites.watch_in_failure')
+        fail = 'test_|-return_changes_|-return_changes_|-fail_with_changes'
+        watch = 'test_|-watch_states_|-watch_states_|-succeed_without_changes'
+
+        self.assertEqual(False, ret[fail]['result'])
+        self.assertEqual('One or more requisite failed: requisites.watch_in_failure.return_changes',
+                         ret[watch]['comment'])
+
     def normalize_ret(self, ret):
         '''
         Normalize the return to the format that we'll use for result checking

tests/unit/modules/test_napalm_network.py

@@ -0,0 +1,383 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Anthony Shaw <anthonyshaw@apache.org>`
+'''
+
+# Import Python Libs
+from __future__ import absolute_import
+
+from functools import wraps
+
+# Import Salt Testing Libs
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.unit import TestCase, skipIf
+from tests.support.mock import (
+    MagicMock,
+    NO_MOCK,
+    NO_MOCK_REASON
+)
+
+
+# Test data
+TEST_FACTS = {
+    '__opts__': {},
+    'OPTIONAL_ARGS': {},
+    'uptime': 'Forever',
+    'UP': True,
+    'HOSTNAME': 'test-device.com'
+}
+
+TEST_ENVIRONMENT = {
+    'hot': 'yes'
+}
+
+TEST_COMMAND_RESPONSE = {
+    'show run': 'all the command output'
+}
+
+TEST_TRACEROUTE_RESPONSE = {
+    'success': {
+        1: {
+            'probes': {
+                1: {
+                    'rtt': 1.123,
+                    'ip_address': u'206.223.116.21',
+                    'host_name': u'eqixsj-google-gige.google.com'
+                }
+            }
+        }
+    }
+}
+
+TEST_PING_RESPONSE = {
+    'success': {
+        'probes_sent': 5,
+        'packet_loss': 0,
+        'rtt_min': 72.158,
+        'rtt_max': 72.433,
+        'rtt_avg': 72.268,
+        'rtt_stddev': 0.094,
+        'results': [
+            {
+                'ip_address': '1.1.1.1',
+                'rtt': 72.248
+            }
+        ]
+    }
+}
+
+TEST_ARP_TABLE = [
+    {
+        'interface': 'MgmtEth0/RSP0/CPU0/0',
+        'mac': '5C:5E:AB:DA:3C:F0',
+        'ip': '172.17.17.1',
+        'age': 1454496274.84
+    }
+]
+
+TEST_IPADDRS = {
+    'FastEthernet8': {
+        'ipv4': {
+            '10.66.43.169': {
+                'prefix_length': 22
+            }
+        }
+    }
+}
+
+TEST_INTERFACES = {
+    'Management1': {
+        'is_up': False,
+        'is_enabled': False,
+        'description': u'',
+        'last_flapped': -1,
+        'speed': 1000,
+        'mac_address': u'dead:beef:dead',
+    }
+}
+
+TEST_LLDP_NEIGHBORS = {
+    u'Ethernet2':
+        [
+            {
+                'hostname': u'junos-unittest',
+                'port': u'520',
+            }
+        ]
+}
+
+TEST_MAC_TABLE = [
+    {
+        'mac': '00:1C:58:29:4A:71',
+        'interface': 'Ethernet47',
+        'vlan': 100,
+        'static': False,
+        'active': True,
+        'moves': 1,
+        'last_move': 1454417742.58
+    }
+]
+
+TEST_RUNNING_CONFIG = {
+    'one': 'two'
+}
+
+TEST_OPTICS = {
+    'et1': {
+        'physical_channels': {
+            'channel': [
+                {
+                    'index': 0,
+                    'state': {
+                        'input_power': {
+                            'instant': 0.0,
+                            'avg': 0.0,
+                            'min': 0.0,
+                            'max': 0.0,
+                        },
+                        'output_power': {
+                            'instant': 0.0,
+                            'avg': 0.0,
+                            'min': 0.0,
+                            'max': 0.0,
+                        },
+                        'laser_bias_current': {
+                            'instant': 0.0,
+                            'avg': 0.0,
+                            'min': 0.0,
+                            'max': 0.0,
+                        },
+                    }
+                }
+            ]
+        }
+    }
+}
+
+
+class MockNapalmDevice(object):
+    '''Setup a mock device for our tests'''
+    def get_facts(self):
+        return TEST_FACTS
+
+    def get_environment(self):
+        return TEST_ENVIRONMENT
+
+    def get_arp_table(self):
+        return TEST_ARP_TABLE
+
+    def get(self, key, default=None, *args, **kwargs):
+        try:
+            if key == 'DRIVER':
+                return self
+            return TEST_FACTS[key]
+        except KeyError:
+            return default
+
+    def cli(self, commands, *args, **kwargs):
+        assert commands[0] == 'show run'
+        return TEST_COMMAND_RESPONSE
+
+    def traceroute(self, destination, **kwargs):
+        assert destination == 'destination.com'
+        return TEST_TRACEROUTE_RESPONSE
+
+    def ping(self, destination, **kwargs):
+        assert destination == 'destination.com'
+        return TEST_PING_RESPONSE
+
+    def get_config(self, retrieve='all'):
+        assert retrieve == 'running'
+        return TEST_RUNNING_CONFIG
+
+    def get_interfaces_ip(self, **kwargs):
+        return TEST_IPADDRS
+
+    def get_interfaces(self, **kwargs):
+        return TEST_INTERFACES
+
+    def get_lldp_neighbors_detail(self, **kwargs):
+        return TEST_LLDP_NEIGHBORS
+
+    def get_mac_address_table(self, **kwargs):
+        return TEST_MAC_TABLE
+
+    def get_optics(self, **kwargs):
+        return TEST_OPTICS
+
+    def load_merge_candidate(self, filename=None, config=None):
+        assert config == 'new config'
+        return TEST_RUNNING_CONFIG
+
+    def load_replace_candidate(self, filename=None, config=None):
+        assert config == 'new config'
+        return TEST_RUNNING_CONFIG
+
+    def commit_config(self, **kwargs):
+        return TEST_RUNNING_CONFIG
+
+    def discard_config(self, **kwargs):
+        return TEST_RUNNING_CONFIG
+
+    def compare_config(self, **kwargs):
+        return TEST_RUNNING_CONFIG
+
+    def rollback(self, **kwargs):
+        return TEST_RUNNING_CONFIG
+
+
+def mock_proxy_napalm_wrap(func):
+    '''
+    The proper decorator checks for proxy minions. We don't care
+    so just pass back to the origination function
+    '''
+
+    @wraps(func)
+    def func_wrapper(*args, **kwargs):
+        func.__globals__['napalm_device'] = MockNapalmDevice()
+        return func(*args, **kwargs)
+    return func_wrapper
+
+
+import salt.utils.napalm as napalm_utils  # NOQA
+napalm_utils.proxy_napalm_wrap = mock_proxy_napalm_wrap  # pylint: disable=E9502
+
+import salt.modules.napalm_network as napalm_network  # NOQA
+
+
+def true(name):
+    assert name == 'set_ntp_peers'
+    return True
+
+
+def random_hash(source, method):
+    return 12346789
+
+
+def join(*files):
+    return True
+
+
+def get_managed_file(*args, **kwargs):
+    return 'True'
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class NapalmNetworkModuleTestCase(TestCase, LoaderModuleMockMixin):
+
+    def setup_loader_modules(self):
+        module_globals = {
+            '__salt__': {
+                'config.option': MagicMock(return_value={
+                    'test': {
+                        'driver': 'test',
+                        'key': '2orgk34kgk34g'
+                    }
+                }),
+                'file.file_exists': true,
+                'file.join': join,
+                'file.get_managed': get_managed_file,
+                'random.hash': random_hash
+            }
+        }
+
+        return {napalm_network: module_globals}
+
+    def test_connected_pass(self):
+        ret = napalm_network.connected()
+        assert ret['out'] is True
+
+    def test_facts(self):
+        ret = napalm_network.facts()
+        assert ret['out'] == TEST_FACTS
+
+    def test_environment(self):
+        ret = napalm_network.environment()
+        assert ret['out'] == TEST_ENVIRONMENT
+
+    def test_cli_single_command(self):
+        '''
+        Test that CLI works with 1 arg
+        '''
+        ret = napalm_network.cli("show run")
+        assert ret['out'] == TEST_COMMAND_RESPONSE
+
+    def test_cli_multi_command(self):
+        '''
+        Test that CLI works with 2 arg
+        '''
+        ret = napalm_network.cli("show run", "show run")
+        assert ret['out'] == TEST_COMMAND_RESPONSE
+
+    def test_traceroute(self):
+        ret = napalm_network.traceroute('destination.com')
+        assert list(ret['out'].keys())[0] == 'success'
+
+    def test_ping(self):
+        ret = napalm_network.ping('destination.com')
+        assert list(ret['out'].keys())[0] == 'success'
+
+    def test_arp(self):
+        ret = napalm_network.arp()
+        assert ret['out'] == TEST_ARP_TABLE
+
+    def test_ipaddrs(self):
+        ret = napalm_network.ipaddrs()
+        assert ret['out'] == TEST_IPADDRS
+
+    def test_interfaces(self):
+        ret = napalm_network.interfaces()
+        assert ret['out'] == TEST_INTERFACES
+
+    def test_lldp(self):
+        ret = napalm_network.lldp()
+        assert ret['out'] == TEST_LLDP_NEIGHBORS
+
+    def test_mac(self):
+        ret = napalm_network.mac()
+        assert ret['out'] == TEST_MAC_TABLE
+
+    def test_config(self):
+        ret = napalm_network.config('running')
+        assert ret['out'] == TEST_RUNNING_CONFIG
+
+    def test_optics(self):
+        ret = napalm_network.optics()
+        assert ret['out'] == TEST_OPTICS
+
+    def test_load_config(self):
+        ret = napalm_network.load_config(text='new config')
+        assert ret['result']
+
+    def test_load_config_replace(self):
+        ret = napalm_network.load_config(text='new config', replace=True)
+        assert ret['result']
+
+    def test_load_template(self):
+        ret = napalm_network.load_template('set_ntp_peers',
+                                           peers=['192.168.0.1'])
+        assert ret['out'] is None
+
+    def test_commit(self):
+        ret = napalm_network.commit()
+        assert ret['out'] == TEST_RUNNING_CONFIG
+
+    def test_discard_config(self):
+        ret = napalm_network.discard_config()
+        assert ret['out'] == TEST_RUNNING_CONFIG
+
+    def test_compare_config(self):
+        ret = napalm_network.compare_config()
+        assert ret['out'] == TEST_RUNNING_CONFIG
+
+    def test_rollback(self):
+        ret = napalm_network.rollback()
+        assert ret['out'] == TEST_RUNNING_CONFIG
+
+    def test_config_changed(self):
+        ret = napalm_network.config_changed()
+        assert ret == (True, '')
+
+    def test_config_control(self):
+        ret = napalm_network.config_control()
+        assert ret == (True, '')