mirror of
https://github.com/valitydev/salt.git
synced 2024-11-09 01:36:48 +00:00
Merge pull request #41457 from cloudflare/napalm-syslog
Improvements for the napalm syslog engine
This commit is contained in:
Mike Place
GitHub
commit
8ad172f941
@ -37,7 +37,7 @@ The napalm-logs transfers the messages via widely used transport
|
||||
mechanisms such as: ZeroMQ (default), Kafka, etc.
|
||||
|
||||
The user can select the right transport using the ``transport``
|
||||
option in the configuration. The
|
||||
option in the configuration.
|
||||
|
||||
:configuration: Example configuration
|
||||
|
||||
@ -45,67 +45,126 @@ option in the configuration. The
|
||||
|
||||
engines:
|
||||
- napalm_syslog:
|
||||
transport: zmq
|
||||
address: 1.2.3.4
|
||||
port: 49018
|
||||
transport: zmq
|
||||
address: 1.2.3.4
|
||||
port: 49018
|
||||
|
||||
Output object example:
|
||||
:configuration: Configuration example, excluding messages from IOS-XR devices:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
engines:
|
||||
- napalm_syslog:
|
||||
transport: kafka
|
||||
address: 1.2.3.4
|
||||
port: 49018
|
||||
os_blacklist:
|
||||
- iosxr
|
||||
|
||||
Event example:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"error": "BGP_PREFIX_THRESH_EXCEEDED",
|
||||
"ip": "127.0.0.1",
|
||||
"host": "re0.edge01.bjm01",
|
||||
"message_details": {
|
||||
"processId": "2902",
|
||||
napalm/syslog/junos/BGP_PREFIX_THRESH_EXCEEDED/vmx01 {
|
||||
"_stamp": "2017-05-26T10:03:18.653045",
|
||||
"error": "BGP_PREFIX_THRESH_EXCEEDED",
|
||||
"pri": "149",
|
||||
"processName": "rpd",
|
||||
"host": "re0.edge01.bjm01",
|
||||
"time": "12:45:19",
|
||||
"date": "Mar 30",
|
||||
"message": "1.2.3.4 (External AS 15169): Configured maximum prefix-limit threshold(160) exceeded for inet-unicast nlri: 181 (instance master)"
|
||||
},
|
||||
"model_name": "openconfig_bgp",
|
||||
"open_config": {
|
||||
"bgp": {
|
||||
"neighbors": {
|
||||
"neighbor": {
|
||||
"1.2.3.4": {
|
||||
"neighbor-address": "1.2.3.4",
|
||||
"state": {
|
||||
"peer-as": 15169
|
||||
},
|
||||
"afi-safis": {
|
||||
"afi-safi": {
|
||||
"inet": {
|
||||
"state": {
|
||||
"prefixes": {
|
||||
"received": 181
|
||||
"host": "vmx01",
|
||||
"ip": "192.168.140.252",
|
||||
"message_details": {
|
||||
"date": "May 25",
|
||||
"host": "vmx01",
|
||||
"message": "192.168.140.254 (External AS 65001): Configured maximum prefix-limit threshold(22) exceeded for inet-unicast nlri: 28 (instance master)",
|
||||
"pri": "28",
|
||||
"processId": "2957",
|
||||
"processName": "rpd",
|
||||
"tag": "BGP_PREFIX_THRESH_EXCEEDED",
|
||||
"time": "20:50:41"
|
||||
},
|
||||
"model_name": "openconfig_bgp",
|
||||
"open_config": {
|
||||
"bgp": {
|
||||
"neighbors": {
|
||||
"neighbor": {
|
||||
"192.168.140.254": {
|
||||
"afi_safis": {
|
||||
"afi_safi": {
|
||||
"inet": {
|
||||
"afi_safi_name": "inet",
|
||||
"ipv4_unicast": {
|
||||
"prefix_limit": {
|
||||
"state": {
|
||||
"max_prefixes": 22
|
||||
}
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"prefixes": {
|
||||
"received": 28
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"neighbor_address": "192.168.140.254",
|
||||
"state": {
|
||||
"peer_as": 65001
|
||||
}
|
||||
}
|
||||
},
|
||||
"ipv4-unicast": {
|
||||
"prefix-limit": {
|
||||
"state": {
|
||||
"max-prefixes": 160
|
||||
}
|
||||
}
|
||||
},
|
||||
"afi-safi-name": "inet"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"os": "junos",
|
||||
"timestamp": "1490877919"
|
||||
},
|
||||
"os": "junos",
|
||||
"timestamp": "1495741841"
|
||||
}
|
||||
|
||||
To consume the events and eventually react and deploy a configuration
|
||||
changes on the device(s) firing the event, one is able to
|
||||
identify the minion ID, using one of the following alternatives, but not limited to:
|
||||
|
||||
- :mod:`Host grains <salt.grains.napalm.host>` to match the event tag
|
||||
- :mod:`Hostname grains <salt.grains.napalm.hostname>` to match the IP address in the event data
|
||||
- :ref:`Define static grains <static-custom-grains>`
|
||||
- :ref:`Write a grains module <writing-grains>`
|
||||
- :ref:`Targeting minions using pillar data <targeting-pillar>` -- the user
|
||||
can insert certain information in the pillar and then use it to identify minions
|
||||
|
||||
Master configuration example, to match the event and react:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
reactor:
|
||||
- 'napalm/syslog/*/BGP_PREFIX_THRESH_EXCEEDED/*':
|
||||
- salt://increase_prefix_limit_on_thresh_exceeded.sls
|
||||
|
||||
Which matches the events having the error code ``BGP_PREFIX_THRESH_EXCEEDED``
|
||||
from any network operating system, from any host and reacts, executing the
|
||||
``increase_prefix_limit_on_thresh_exceeded.sls`` reactor, found under
|
||||
one of the :conf_master:`file_roots` paths.
|
||||
|
||||
Reactor example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
increase_prefix_limit_on_thresh_exceeded:
|
||||
local.net.load_template:
|
||||
- tgt: "hostname:{{ data['host'] }}"
|
||||
- tgt_type: grain
|
||||
- kwarg:
|
||||
template_name: salt://increase_prefix_limit.jinja
|
||||
openconfig_structure: {{ data['open_config'] }}
|
||||
|
||||
The reactor in the example increases the BGP prefix limit
|
||||
when triggered by an event as above. The minion is matched using the ``host``
|
||||
field from the ``data`` (which is the body of the event), compared to the
|
||||
:mod:`hostname grain <salt.grains.napalm.hostname>` field. When the event
|
||||
occurs, the reactor will execute the
|
||||
:mod:`net.load_template <salt.modules.napalm_network.load_template>` function,
|
||||
sending as arguments the template ``salt://increase_prefix_limit.jinja`` defined
|
||||
by the user in their environment and the complete OpenConfig object under
|
||||
the variable name ``openconfig_structure``. Inside the Jinja template, the user
|
||||
can process the object from ``openconfig_structure`` and define the bussiness
|
||||
logic as required.
|
||||
'''
|
||||
from __future__ import absolute_import
|
||||
|
||||
@ -129,6 +188,7 @@ except ImportError:
|
||||
HAS_NAPALM_LOGS = False
|
||||
|
||||
# Import salt libs
|
||||
import salt.utils
|
||||
import salt.utils.network
|
||||
from salt.utils import event
|
||||
|
||||
@ -154,7 +214,7 @@ def __virtual__():
|
||||
return True
|
||||
|
||||
|
||||
def _zmq(address, port):
|
||||
def _zmq(address, port, **kwargs):
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.SUB)
|
||||
if salt.utils.network.is_ipv6(address):
|
||||
@ -169,11 +229,12 @@ def _zmq(address, port):
|
||||
|
||||
def _get_transport_recv(name='zmq',
|
||||
address='0.0.0.0',
|
||||
port=49017):
|
||||
port=49017,
|
||||
**kwargs):
|
||||
if name not in TRANSPORT_FUN_MAP:
|
||||
log.error('Invalid transport: {0}. Falling back to ZeroMQ.'.format(name))
|
||||
name = 'zmq'
|
||||
return TRANSPORT_FUN_MAP[name](address, port)
|
||||
return TRANSPORT_FUN_MAP[name](address, port, **kwargs)
|
||||
|
||||
|
||||
TRANSPORT_FUN_MAP = {
|
||||
@ -192,7 +253,13 @@ def start(transport='zmq',
|
||||
auth_address='0.0.0.0',
|
||||
auth_port=49018,
|
||||
disable_security=False,
|
||||
certificate=None):
|
||||
certificate=None,
|
||||
os_whitelist=None,
|
||||
os_blacklist=None,
|
||||
error_whitelist=None,
|
||||
error_blacklist=None,
|
||||
host_whitelist=None,
|
||||
host_blacklist=None):
|
||||
'''
|
||||
Listen to napalm-logs and publish events into the Salt event bus.
|
||||
|
||||
@ -203,10 +270,10 @@ def start(transport='zmq',
|
||||
Currently ``zmq`` is the only valid option.
|
||||
|
||||
address: ``0.0.0.0``
|
||||
The address of the publisher.
|
||||
The address of the publisher, as configured on napalm-logs.
|
||||
|
||||
port: ``49017``
|
||||
The port of the publisher.
|
||||
The port of the publisher, as configured on napalm-logs.
|
||||
|
||||
auth_address: ``0.0.0.0``
|
||||
The address used for authentication
|
||||
@ -219,10 +286,31 @@ def start(transport='zmq',
|
||||
Trust unencrypted messages.
|
||||
Strongly discouraged in production.
|
||||
|
||||
certificate
|
||||
certificate: ``None``
|
||||
Absolute path to the SSL certificate.
|
||||
|
||||
os_whitelist: ``None``
|
||||
List of operating systems allowed. By default everything is allowed.
|
||||
|
||||
os_blacklist: ``None``
|
||||
List of operating system to be ignored. Nothing ignored by default.
|
||||
|
||||
error_whitelist: ``None``
|
||||
List of errors allowed.
|
||||
|
||||
error_blacklist: ``None``
|
||||
List of errors ignored.
|
||||
|
||||
host_whitelist: ``None``
|
||||
List of hosts or IPs to be allowed.
|
||||
|
||||
host_blacklist: ``None``
|
||||
List of hosts of IPs to be ignored.
|
||||
'''
|
||||
if not disable_security:
|
||||
if not certificate:
|
||||
log.critical('Please use a certificate, or disable the security.')
|
||||
return
|
||||
priv_key, verify_key = napalm_logs.utils.authenticate(certificate,
|
||||
address=auth_address,
|
||||
port=auth_port)
|
||||
@ -245,14 +333,39 @@ def start(transport='zmq',
|
||||
else:
|
||||
dict_object = napalm_logs.utils.unserialize(raw_object)
|
||||
try:
|
||||
event_os = dict_object['os']
|
||||
if os_blacklist or os_whitelist:
|
||||
valid_os = salt.utils.check_whitelist_blacklist(event_os,
|
||||
whitelist=os_whitelist,
|
||||
blacklist=os_blacklist)
|
||||
if not valid_os:
|
||||
log.info('Ignoring NOS {} as per whitelist/blacklist'.format(event_os))
|
||||
continue
|
||||
event_error = dict_object['error']
|
||||
if error_blacklist or error_whitelist:
|
||||
valid_error = salt.utils.check_whitelist_blacklist(event_error,
|
||||
whitelist=error_whitelist,
|
||||
blacklist=error_blacklist)
|
||||
if not valid_error:
|
||||
log.info('Ignoring error {} as per whitelist/blacklist'.format(event_error))
|
||||
continue
|
||||
event_host = dict_object.get('host') or dict_object.get('ip')
|
||||
if host_blacklist or host_whitelist:
|
||||
valid_host = salt.utils.check_whitelist_blacklist(event_host,
|
||||
whitelist=host_whitelist,
|
||||
blacklist=host_blacklist)
|
||||
if not valid_host:
|
||||
log.info('Ignoring messages from {} as per whitelist/blacklist'.format(event_host))
|
||||
continue
|
||||
tag = 'napalm/syslog/{os}/{error}/{host}'.format(
|
||||
os=dict_object['os'],
|
||||
error=dict_object['error'],
|
||||
host=(dict_object.get('host') or dict_object.get('ip'))
|
||||
os=event_os,
|
||||
error=event_error,
|
||||
host=event_host
|
||||
)
|
||||
except KeyError as kerr:
|
||||
log.warning('Missing keys from the napalm-logs object:', exc_info=True)
|
||||
log.warning(dict_object)
|
||||
continue # jump to the next object in the queue
|
||||
log.debug('Sending event {0}'.format(tag))
|
||||
log.debug(raw_object)
|
||||
if master:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user