diff --git a/doc/ref/clients/index.rst b/doc/ref/clients/index.rst index 3029ffc71d..27fa77c1c0 100644 --- a/doc/ref/clients/index.rst +++ b/doc/ref/clients/index.rst @@ -88,6 +88,12 @@ Salt Caller .. autoclass:: salt.client.Caller :members: cmd +Salt Proxy Caller +----------------- + +.. autoclass:: salt.client.ProxyCaller + :members: cmd + RunnerClient ------------ diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index a6df74aa9d..eeef333d1e 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -187,6 +187,7 @@ execution modules inspectlib.query inspector introspect + iosconfig ipmi ipset iptables @@ -325,6 +326,7 @@ execution modules pcs pdbedit pecl + peeringdb pf philips_hue pillar diff --git a/doc/ref/modules/all/salt.modules.iosconfig.rst b/doc/ref/modules/all/salt.modules.iosconfig.rst new file mode 100644 index 0000000000..77ed5f3106 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.iosconfig.rst @@ -0,0 +1,7 @@ +============================= +salt.modules.iosconfig module +============================= + +.. automodule:: salt.modules.iosconfig + :members: + diff --git a/doc/ref/modules/all/salt.modules.peeringdb.rst b/doc/ref/modules/all/salt.modules.peeringdb.rst new file mode 100644 index 0000000000..c1e9289913 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.peeringdb.rst @@ -0,0 +1,7 @@ +============================= +salt.modules.peeringdb module +============================= + +.. automodule:: salt.modules.peeringdb + :members: + diff --git a/doc/topics/development/contributing.rst b/doc/topics/development/contributing.rst index 37c3294bb8..9a9c6f97ac 100644 --- a/doc/topics/development/contributing.rst +++ b/doc/topics/development/contributing.rst @@ -21,11 +21,21 @@ SaltStack has its own coding style guide that informs contributors on various co approaches. Please review the :ref:`Salt Coding Style ` documentation for information about Salt's particular coding patterns. -Within the :ref:`Salt Coding Style ` documentation, there is a section -about running Salt's ``.pylintrc`` file. SaltStack recommends running the ``.pylintrc`` -file on any files you are changing with your code contribution before submitting a -pull request to Salt's repository. Please see the :ref:`Linting` -documentation for more information. +Within the :ref:`Salt Coding Style ` documentation, there is a +section about running Salt's ``.testing.pylintrc`` file. SaltStack recommends +running the ``.testing.pylintrc`` file on any files you are changing with your +code contribution before submitting a pull request to Salt's repository. Please +see the :ref:`Linting` documentation for more information. + +.. note:: + + There are two pylint files in the ``salt`` directory. One is the + ``.pylintrc`` file and the other is the ``.testing.pylintrc`` file. The + tests that run in Jenkins against GitHub Pull Requests use + ``.testing.pylintrc``. The ``testing.pylintrc`` file is a little less + strict than the ``.pylintrc`` and is used to make it easier for contributors + to submit changes. The ``.pylintrc`` file can be used for linting, but the + ``testing.pylintrc`` is the source of truth when submitting pull requests. .. _github-pull-request: diff --git a/doc/topics/development/conventions/style.rst b/doc/topics/development/conventions/style.rst index fc41660a0d..9ac723f97b 100644 --- a/doc/topics/development/conventions/style.rst +++ b/doc/topics/development/conventions/style.rst @@ -22,21 +22,31 @@ improve Salt)!! Linting ======= -Most Salt style conventions are codified in Salt's ``.pylintrc`` file. Salt's -pylint file has two dependencies: pylint_ and saltpylint_. You can install -these dependencies with ``pip``: +Most Salt style conventions are codified in Salt's ``.testing.pylintrc`` file. +Salt's pylint file has two dependencies: pylint_ and saltpylint_. You can +install these dependencies with ``pip``: .. code-block:: bash pip install pylint pip install saltpylint -The ``.pylintrc`` file is found in the root of the Salt project and can be passed -as an argument to the pylint_ program as follows: +The ``.testing.pylintrc`` file is found in the root of the Salt project and can +be passed as an argument to the pylint_ program as follows: .. code-block:: bash - pylint --rcfile=/path/to/salt/.pylintrc salt/dir/to/lint + pylint --rcfile=/path/to/salt/.testing.pylintrc salt/dir/to/lint + +.. note:: + + There are two pylint files in the ``salt`` directory. One is the + ``.pylintrc`` file and the other is the ``.testing.pylintrc`` file. The + tests that run in Jenkins against GitHub Pull Requests use + ``.testing.pylintrc``. The ``testing.pylintrc`` file is a little less + strict than the ``.pylintrc`` and is used to make it easier for contributors + to submit changes. The ``.pylintrc`` file can be used for linting, but the + ``testing.pylintrc`` is the source of truth when submitting pull requests. .. _pylint: http://www.pylint.org .. _saltpylint: https://github.com/saltstack/salt-pylint diff --git a/doc/topics/releases/fluorine.rst b/doc/topics/releases/fluorine.rst index 4f8c767079..5d8485318f 100644 --- a/doc/topics/releases/fluorine.rst +++ b/doc/topics/releases/fluorine.rst @@ -538,6 +538,11 @@ Module Deprecations function. This is because support for NAPALM native templates has been dropped. +- The :py:mod:`pip ` module has been changed as follows: + + - Support for the ``no_chown`` option has been removed from + :py:func:`pip.install ` function. + - The :py:mod:`trafficserver ` module has been changed as follows: @@ -673,10 +678,17 @@ State Deprecations ` state has been removed. + - The :py:func:`trafficserver.set_var ` state has been removed. Please use :py:func:`trafficserver.config ` instead. +- Support for the ``no_chown`` option in the + :py:func`virtualenv.managed ` function has + been removed. + - The ``win_update`` state module has been removed. It has been replaced by :py:mod:`win_wua `. diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 0d0299647f..b01c6aa888 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -99,7 +99,10 @@ class BaseCaller(object): # be imported as part of the salt api doesn't do a # nasty sys.exit() and tick off our developer users try: - self.minion = salt.minion.SMinion(opts) + if self.opts.get('proxyid'): + self.minion = salt.minion.SProxyMinion(opts) + else: + self.minion = salt.minion.SMinion(opts) except SaltClientError as exc: raise SystemExit(six.text_type(exc)) @@ -210,8 +213,24 @@ class BaseCaller(object): 'Do you have permissions to ' 'write to {0} ?\n'.format(proc_fn)) func = self.minion.functions[fun] + data = { + 'arg': args, + 'fun': fun + } + data.update(kwargs) + executors = getattr(self.minion, 'module_executors', []) or \ + self.opts.get('module_executors', ['direct_call']) + if isinstance(executors, six.string_types): + executors = [executors] try: ret['return'] = func(*args, **kwargs) + for name in executors: + fname = '{0}.execute'.format(name) + if fname not in self.minion.executors: + raise SaltInvocationError("Executor '{0}' is not available".format(name)) + ret['return'] = self.minion.executors[fname](self.opts, data, func, args, kwargs) + if ret['return'] is not None: + break except TypeError as exc: sys.stderr.write('\nPassed invalid arguments: {0}.\n\nUsage:\n'.format(exc)) salt.utils.stringutils.print_cli(func.__doc__) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 60902ccbd6..b49ad934e6 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -1993,3 +1993,78 @@ class Caller(object): salt.utils.args.parse_input(args), kwargs) return func(*args, **kwargs) + + +class ProxyCaller(object): + ''' + ``ProxyCaller`` is the same interface used by the :command:`salt-call` + with the args ``--proxyid `` command-line tool on the Salt Proxy + Minion. + + Importing and using ``ProxyCaller`` must be done on the same machine as a + Salt Minion and it must be done using the same user that the Salt Minion is + running as. + + Usage: + + .. code-block:: python + + import salt.client + caller = salt.client.Caller() + caller.cmd('test.ping') + + Note, a running master or minion daemon is not required to use this class. + Running ``salt-call --local`` simply sets :conf_minion:`file_client` to + ``'local'``. The same can be achieved at the Python level by including that + setting in a minion config file. + + .. code-block:: python + + import salt.client + import salt.config + __opts__ = salt.config.proxy_config('/etc/salt/proxy', minion_id='quirky_edison') + __opts__['file_client'] = 'local' + caller = salt.client.ProxyCaller(mopts=__opts__) + + .. note:: + + To use this for calling proxies, the :py:func:`is_proxy functions + ` requires that ``--proxyid`` be an + argument on the commandline for the script this is used in, or that the + string ``proxy`` is in the name of the script. + ''' + def __init__(self, c_path=os.path.join(syspaths.CONFIG_DIR, 'proxy'), mopts=None): + # Late-import of the minion module to keep the CLI as light as possible + import salt.minion + self.opts = mopts or salt.config.proxy_config(c_path) + self.sminion = salt.minion.SProxyMinion(self.opts) + + def cmd(self, fun, *args, **kwargs): + ''' + Call an execution module with the given arguments and keyword arguments + + .. code-block:: python + + caller.cmd('test.arg', 'Foo', 'Bar', baz='Baz') + + caller.cmd('event.send', 'myco/myevent/something', + data={'foo': 'Foo'}, with_env=['GIT_COMMIT'], with_grains=True) + ''' + func = self.sminion.functions[fun] + data = { + 'arg': args, + 'fun': fun + } + data.update(kwargs) + executors = getattr(self.sminion, 'module_executors', []) or \ + self.opts.get('module_executors', ['direct_call']) + if isinstance(executors, six.string_types): + executors = [executors] + for name in executors: + fname = '{0}.execute'.format(name) + if fname not in self.sminion.executors: + raise SaltInvocationError("Executor '{0}' is not available".format(name)) + return_data = self.sminion.executors[fname](self.opts, data, func, args, kwargs) + if return_data is not None: + break + return return_data diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index de06827f54..b4272e4b55 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -80,13 +80,13 @@ def _ssh_state(chunks, st_kwargs, # Read in the JSON data and return the data structure try: - return salt.utils.json.loads(stdout, object_hook=salt.utils.data.encode_dict) + return salt.utils.data.decode(salt.utils.json.loads(stdout, object_hook=salt.utils.data.encode_dict)) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout - return stdout + return salt.utils.data.decode(stdout) def _set_retcode(ret, highstate=None): diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 84537fab3b..98e4ed3ff6 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -661,6 +661,13 @@ class RemoteFuncs(object): log.error('Invalid file pointer: load[loc] < 0') return False + if load.get('size', 0) > file_recv_max_size: + log.error( + 'Exceeding file_recv_max_size limit: %s', + file_recv_max_size + ) + return False + if len(load['data']) + load.get('loc', 0) > file_recv_max_size: log.error( 'Exceeding file_recv_max_size limit: %s', diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index f61b3f80fe..bf6a636738 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -129,8 +129,6 @@ def serve_file(load, fnd): with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) - if data and six.PY3 and not salt.utils.files.is_binary(fpath): - data = data.decode(__salt_system_encoding__) if gzip and data: data = salt.utils.gzip_util.compress(data, gzip) ret['gzip'] = gzip diff --git a/salt/minion.py b/salt/minion.py index 1c1b55d5f8..92b5e8b765 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -3813,3 +3813,92 @@ class ProxyMinion(Minion): Minion._thread_multi_return(minion_instance, opts, data) else: Minion._thread_return(minion_instance, opts, data) + + +class SProxyMinion(SMinion): + ''' + Create an object that has loaded all of the minion module functions, + grains, modules, returners etc. The SProxyMinion allows developers to + generate all of the salt minion functions and present them with these + functions for general use. + ''' + def gen_modules(self, initial_load=False): + ''' + Tell the minion to reload the execution modules + + CLI Example: + + .. code-block:: bash + + salt '*' sys.reload_modules + ''' + self.opts['grains'] = salt.loader.grains(self.opts) + self.opts['pillar'] = salt.pillar.get_pillar( + self.opts, + self.opts['grains'], + self.opts['id'], + saltenv=self.opts['saltenv'], + pillarenv=self.opts.get('pillarenv'), + ).compile_pillar() + + if 'proxy' not in self.opts['pillar'] and 'proxy' not in self.opts: + errmsg = ( + 'No "proxy" configuration key found in pillar or opts ' + 'dictionaries for id {id}. Check your pillar/options ' + 'configuration and contents. Salt-proxy aborted.' + ).format(id=self.opts['id']) + log.error(errmsg) + self._running = False + raise SaltSystemExit(code=salt.defaults.exitcodes.EX_GENERIC, msg=errmsg) + + if 'proxy' not in self.opts: + self.opts['proxy'] = self.opts['pillar']['proxy'] + + # Then load the proxy module + self.proxy = salt.loader.proxy(self.opts) + + self.utils = salt.loader.utils(self.opts, proxy=self.proxy) + + self.functions = salt.loader.minion_mods(self.opts, utils=self.utils, notify=False, proxy=self.proxy) + self.returners = salt.loader.returners(self.opts, self.functions, proxy=self.proxy) + self.matcher = Matcher(self.opts, self.functions) + self.functions['sys.reload_modules'] = self.gen_modules + self.executors = salt.loader.executors(self.opts, self.functions, proxy=self.proxy) + + fq_proxyname = self.opts['proxy']['proxytype'] + + # we can then sync any proxymodules down from the master + # we do a sync_all here in case proxy code was installed by + # SPM or was manually placed in /srv/salt/_modules etc. + self.functions['saltutil.sync_all'](saltenv=self.opts['saltenv']) + + self.functions.pack['__proxy__'] = self.proxy + self.proxy.pack['__salt__'] = self.functions + self.proxy.pack['__ret__'] = self.returners + self.proxy.pack['__pillar__'] = self.opts['pillar'] + + # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__ + self.utils = salt.loader.utils(self.opts, proxy=self.proxy) + self.proxy.pack['__utils__'] = self.utils + + # Reload all modules so all dunder variables are injected + self.proxy.reload_modules() + + if ('{0}.init'.format(fq_proxyname) not in self.proxy + or '{0}.shutdown'.format(fq_proxyname) not in self.proxy): + errmsg = 'Proxymodule {0} is missing an init() or a shutdown() or both. '.format(fq_proxyname) + \ + 'Check your proxymodule. Salt-proxy aborted.' + log.error(errmsg) + self._running = False + raise SaltSystemExit(code=salt.defaults.exitcodes.EX_GENERIC, msg=errmsg) + + self.module_executors = self.proxy.get('{0}.module_executors'.format(fq_proxyname), lambda: [])() + proxy_init_fn = self.proxy[fq_proxyname + '.init'] + proxy_init_fn(self.opts) + + self.opts['grains'] = salt.loader.grains(self.opts, proxy=self.proxy) + + # Sync the grains here so the proxy can communicate them to the master + self.functions['saltutil.sync_grains'](saltenv='base') + self.grains_cache = self.opts['grains'] + self.ready = True diff --git a/salt/modules/acme.py b/salt/modules/acme.py index 67bab34433..f8f30e0894 100644 --- a/salt/modules/acme.py +++ b/salt/modules/acme.py @@ -23,6 +23,16 @@ eventually falls back to /opt/letsencrypt/letsencrypt-auto Most parameters will fall back to cli.ini defaults if None is given. +DNS plugins +----------- + +This module currently supports the CloudFlare certbot DNS plugin. The DNS +plugin credentials file needs to be passed in using the +``dns_plugin_credentials`` argument. + +Make sure the appropriate certbot plugin for the wanted DNS provider is +installed before using this module. + ''' # Import python libs from __future__ import absolute_import, print_function, unicode_literals @@ -107,7 +117,9 @@ def cert(name, tls_sni_01_port=None, tls_sni_01_address=None, http_01_port=None, - http_01_address=None): + http_01_address=None, + dns_plugin=None, + dns_plugin_credentials=None): ''' Obtain/renew a certificate from an ACME CA, probably Let's Encrypt. @@ -135,6 +147,8 @@ def cert(name, the port Certbot listens on. A conforming ACME server will still attempt to connect on port 80. :param https_01_address: The address the server listens to during http-01 challenge. + :param dns_plugin: Name of a DNS plugin to use (currently only 'cloudflare') + :param dns_plugin_credentials: Path to the credentials file if required by the specified DNS plugin :return: dict with 'result' True/False/None, 'comment' and certificate's expiry date ('not_after') CLI example: @@ -146,6 +160,8 @@ def cert(name, cmd = [LEA, 'certonly', '--non-interactive', '--agree-tos'] + supported_dns_plugins = ['cloudflare'] + cert_file = _cert_file(name, 'cert') if not __salt__['file.file_exists'](cert_file): log.debug('Certificate %s does not exist (yet)', cert_file) @@ -169,6 +185,12 @@ def cert(name, cmd.append('--authenticator webroot') if webroot is not True: cmd.append('--webroot-path {0}'.format(webroot)) + elif dns_plugin in supported_dns_plugins: + if dns_plugin == 'cloudflare': + cmd.append('--dns-cloudflare') + cmd.append('--dns-cloudflare-credentials {0}'.format(dns_plugin_credentials)) + else: + return {'result': False, 'comment': 'DNS plugin \'{0}\' is not supported'.format(dns_plugin)} else: cmd.append('--authenticator standalone') diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 4464883797..e59271c43f 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -2171,6 +2171,9 @@ def retcode(cmd, salt '*' cmd.retcode "grep f" stdin='one\\ntwo\\nthree\\nfour\\nfive\\n' ''' + python_shell = _python_shell_default(python_shell, + kwargs.get('__pub_jid', '')) + ret = _run(cmd, runas=runas, group=group, diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 7f2b985d43..dd067061ee 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -812,6 +812,7 @@ def push(path, keep_symlinks=False, upload_path=None, remove_source=False): load = {'cmd': '_file_recv', 'id': __opts__['id'], 'path': load_path_list, + 'size': os.path.getsize(path), 'tok': auth.gen_token(b'salt')} channel = salt.transport.Channel.factory(__opts__) with salt.utils.files.fopen(path, 'rb') as fp_: diff --git a/salt/modules/file.py b/salt/modules/file.py index d713178885..594aaee4ff 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -2477,10 +2477,10 @@ def blockreplace(path, final output marker_end - The line content identifying a line as the end of the content block. - Note that the whole line containing this marker will be considered, so - whitespace or extra content before or after the marker is included in - final output + The line content identifying the end of the content block. As of + versions 2017.7.5 and 2018.3.1, everything up to the text matching the + marker will be replaced, so it's important to ensure that your marker + includes the beginning of the text you wish to replace. content The content to be used between the two lines identified by marker_start diff --git a/salt/modules/iosconfig.py b/salt/modules/iosconfig.py new file mode 100644 index 0000000000..3d338b2eda --- /dev/null +++ b/salt/modules/iosconfig.py @@ -0,0 +1,452 @@ +# -*- coding: utf-8 -*- +''' +Cisco IOS configuration manipulation helpers + +.. versionadded:: Fluorine + +This module provides a collection of helper functions for Cisco IOS style +configuration manipulation. This module does not have external dependencies +and can be used from any Proxy or regular Minion. +''' +# Import Python Libs +from __future__ import absolute_import, unicode_literals, print_function + +# Import python stdlib +import difflib + +# Import Salt modules +from salt.ext import six +import salt.utils.dictupdate +import salt.utils.dictdiffer +from salt.utils.odict import OrderedDict +from salt.exceptions import SaltException + +# ------------------------------------------------------------------------------ +# module properties +# ------------------------------------------------------------------------------ + +__virtualname__ = 'iosconfig' +__proxyenabled__ = ['*'] + +# ------------------------------------------------------------------------------ +# helper functions -- will not be exported +# ------------------------------------------------------------------------------ + + +def _attach_data_to_path(obj, ele, data): + if ele not in obj: + obj[ele] = OrderedDict() + obj[ele] = data + else: + obj[ele].update(data) + + +def _attach_data_to_path_tags(obj, path, data, list_=False): + if "#list" not in obj: + obj["#list"] = [] + path = [path] + obj_tmp = obj + first = True + while True: + obj_tmp["#text"] = " ".join(path) + path_item = path.pop(0) + if not path: + break + else: + if path_item not in obj_tmp: + obj_tmp[path_item] = OrderedDict() + obj_tmp = obj_tmp[path_item] + + if first and list_: + obj["#list"].append({path_item: obj_tmp}) + first = False + if path_item in obj_tmp: + obj_tmp[path_item].update(data) + else: + obj_tmp[path_item] = data + obj_tmp[path_item]["#standalone"] = True + + +def _parse_text_config(config_lines, + with_tags=False, + current_indent=0, + nested=False): + struct_cfg = OrderedDict() + while config_lines: + line = config_lines.pop(0) + if not line.strip() or line.lstrip().startswith('!'): + # empty or comment + continue + current_line = line.lstrip() + leading_spaces = len(line) - len(current_line) + if leading_spaces > current_indent: + current_block = _parse_text_config(config_lines, + current_indent=leading_spaces, + with_tags=with_tags, + nested=True) + if with_tags: + _attach_data_to_path_tags(struct_cfg, + current_line, + current_block, + nested) + else: + _attach_data_to_path(struct_cfg, current_line, current_block) + elif leading_spaces < current_indent: + config_lines.insert(0, line) + break + else: + if not nested: + current_block = _parse_text_config(config_lines, + current_indent=leading_spaces, + with_tags=with_tags, + nested=True) + if with_tags: + _attach_data_to_path_tags(struct_cfg, + current_line, + current_block, + nested) + else: + _attach_data_to_path(struct_cfg, current_line, current_block) + else: + config_lines.insert(0, line) + break + return struct_cfg + + +def _get_diff_text(old, new): + ''' + Returns the diff of two text blobs. + ''' + diff = difflib.unified_diff(old.splitlines(1), + new.splitlines(1)) + return ''.join([x.replace('\r', '') for x in diff]) + + +def _print_config_text(tree, indentation=0): + ''' + Return the config as text from a config tree. + ''' + config = '' + for key, value in six.iteritems(tree): + config += '{indent}{line}\n'.format(indent=' '*indentation, line=key) + if value: + config += _print_config_text(value, indentation=indentation+1) + return config + +# ------------------------------------------------------------------------------ +# callable functions +# ------------------------------------------------------------------------------ + + +def tree(config=None, + path=None, + with_tags=False, + saltenv='base'): + ''' + Transform Cisco IOS style configuration to structured Python dictionary. + Depending on the value of the ``with_tags`` argument, this function may + provide different views, valuable in different situations. + + config + The configuration sent as text. This argument is ignored when ``path`` + is configured. + + path + Absolute or remote path from where to load the configuration text. This + argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + with_tags: ``False`` + Whether this function should return a detailed view, with tags. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' iosconfig.tree path=salt://path/to/my/config.txt + salt '*' iosconfig.tree path=https://bit.ly/2mAdq7z + ''' + if path: + config = __salt__['cp.get_file_str'](path, saltenv=saltenv) + if config is False: + raise SaltException('{} is not available'.format(path)) + config_lines = config.splitlines() + return _parse_text_config(config_lines, with_tags=with_tags) + + +def clean(config=None, path=None, saltenv='base'): + ''' + Return a clean version of the config, without any special signs (such as + ``!`` as an individual line) or empty lines, but just lines with significant + value in the configuration of the network device. + + config + The configuration sent as text. This argument is ignored when ``path`` + is configured. + + path + Absolute or remote path from where to load the configuration text. This + argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' iosconfig.clean path=salt://path/to/my/config.txt + salt '*' iosconfig.clean path=https://bit.ly/2mAdq7z + ''' + config_tree = tree(config=config, path=path, saltenv=saltenv) + return _print_config_text(config_tree) + + +def merge_tree(initial_config=None, + initial_path=None, + merge_config=None, + merge_path=None, + saltenv='base'): + ''' + Return the merge tree of the ``initial_config`` with the ``merge_config``, + as a Python dictionary. + + initial_config + The initial configuration sent as text. This argument is ignored when + ``initial_path`` is set. + + initial_path + Absolute or remote path from where to load the initial configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + merge_config + The config to be merged into the initial config, sent as text. This + argument is ignored when ``merge_path`` is set. + + merge_path + Absolute or remote path from where to load the merge configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``initial_path`` or ``merge_path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' iosconfig.merge_tree initial_path=salt://path/to/running.cfg merge_path=salt://path/to/merge.cfg + ''' + merge_tree = tree(config=merge_config, + path=merge_path, + saltenv=saltenv) + initial_tree = tree(config=initial_config, + path=initial_path, + saltenv=saltenv) + return salt.utils.dictupdate.merge(initial_tree, merge_tree) + + +def merge_text(initial_config=None, + initial_path=None, + merge_config=None, + merge_path=None, + saltenv='base'): + ''' + Return the merge result of the ``initial_config`` with the ``merge_config``, + as plain text. + + initial_config + The initial configuration sent as text. This argument is ignored when + ``initial_path`` is set. + + initial_path + Absolute or remote path from where to load the initial configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + merge_config + The config to be merged into the initial config, sent as text. This + argument is ignored when ``merge_path`` is set. + + merge_path + Absolute or remote path from where to load the merge configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``initial_path`` or ``merge_path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' iosconfig.merge_text initial_path=salt://path/to/running.cfg merge_path=salt://path/to/merge.cfg + ''' + candidate_tree = merge_tree(initial_config=initial_config, + initial_path=initial_path, + merge_config=merge_config, + merge_path=merge_path, + saltenv=saltenv) + return _print_config_text(candidate_tree) + + +def merge_diff(initial_config=None, + initial_path=None, + merge_config=None, + merge_path=None, + saltenv='base'): + ''' + Return the merge diff, as text, after merging the merge config into the + initial config. + + initial_config + The initial configuration sent as text. This argument is ignored when + ``initial_path`` is set. + + initial_path + Absolute or remote path from where to load the initial configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + merge_config + The config to be merged into the initial config, sent as text. This + argument is ignored when ``merge_path`` is set. + + merge_path + Absolute or remote path from where to load the merge configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``initial_path`` or ``merge_path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' iosconfig.merge_diff initial_path=salt://path/to/running.cfg merge_path=salt://path/to/merge.cfg + ''' + if initial_path: + initial_config = __salt__['cp.get_file_str'](initial_path, saltenv=saltenv) + candidate_config = merge_text(initial_config=initial_config, + merge_config=merge_config, + merge_path=merge_path, + saltenv=saltenv) + clean_running_dict = tree(config=initial_config) + clean_running = _print_config_text(clean_running_dict) + return _get_diff_text(clean_running, candidate_config) + + +def diff_tree(candidate_config=None, + candidate_path=None, + running_config=None, + running_path=None, + saltenv='base'): + ''' + Return the diff, as Python dictionary, between the candidate and the running + configuration. + + candidate_config + The candidate configuration sent as text. This argument is ignored when + ``candidate_path`` is set. + + candidate_path + Absolute or remote path from where to load the candidate configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + running_config + The running configuration sent as text. This argument is ignored when + ``running_path`` is set. + + running_path + Absolute or remote path from where to load the runing configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``candidate_path`` or ``running_path`` is not a + ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' iosconfig.diff_tree candidate_path=salt://path/to/candidate.cfg running_path=salt://path/to/running.cfg + ''' + candidate_tree = tree(config=candidate_config, + path=candidate_path, + saltenv=saltenv) + running_tree = tree(config=running_config, + path=running_path, + saltenv=saltenv) + return salt.utils.dictdiffer.deep_diff(running_tree, candidate_tree) + + +def diff_text(candidate_config=None, + candidate_path=None, + running_config=None, + running_path=None, + saltenv='base'): + ''' + Return the diff, as text, between the candidate and the running config. + + candidate_config + The candidate configuration sent as text. This argument is ignored when + ``candidate_path`` is set. + + candidate_path + Absolute or remote path from where to load the candidate configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + running_config + The running configuration sent as text. This argument is ignored when + ``running_path`` is set. + + running_path + Absolute or remote path from where to load the runing configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``candidate_path`` or ``running_path`` is not a + ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' iosconfig.diff_text candidate_path=salt://path/to/candidate.cfg running_path=salt://path/to/running.cfg + ''' + candidate_text = clean(config=candidate_config, + path=candidate_path, + saltenv=saltenv) + running_text = clean(config=running_config, + path=running_path, + saltenv=saltenv) + return _get_diff_text(running_text, candidate_text) diff --git a/salt/modules/mac_brew.py b/salt/modules/mac_brew.py index 2ea377ae61..beff24dc4a 100644 --- a/salt/modules/mac_brew.py +++ b/salt/modules/mac_brew.py @@ -48,7 +48,7 @@ def _list_taps(): ''' List currently installed brew taps ''' - cmd = 'brew tap' + cmd = 'tap' return _call_brew(cmd)['stdout'].splitlines() @@ -60,7 +60,7 @@ def _tap(tap, runas=None): if tap in _list_taps(): return True - cmd = 'brew tap {0}'.format(tap) + cmd = 'tap {0}'.format(tap) try: _call_brew(cmd) except CommandExecutionError: @@ -85,6 +85,7 @@ def _call_brew(cmd, failhard=True): ''' user = __salt__['file.get_user'](_homebrew_bin()) runas = user if user != __opts__['user'] else None + cmd = '{} {}'.format(salt.utils.path.which('brew'), cmd) result = __salt__['cmd.run_all'](cmd, runas=runas, output_loglevel='trace', @@ -253,7 +254,7 @@ def remove(name=None, pkgs=None, **kwargs): targets = [x for x in pkg_params if x in old] if not targets: return {} - cmd = 'brew uninstall {0}'.format(' '.join(targets)) + cmd = 'uninstall {0}'.format(' '.join(targets)) out = _call_brew(cmd) if out['retcode'] != 0 and out['stderr']: @@ -286,7 +287,7 @@ def refresh_db(): ''' # Remove rtag file to keep multiple refreshes from happening in pkg states salt.utils.pkg.clear_rtag(__opts__) - cmd = 'brew update' + cmd = 'update' if _call_brew(cmd)['retcode']: log.error('Failed to update') return False @@ -309,7 +310,7 @@ def _info(*pkgs): Caveat: If one of the packages does not exist, no packages will be included in the output. ''' - cmd = 'brew info --json=v1 {0}'.format(' '.join(pkgs)) + cmd = 'info --json=v1 {0}'.format(' '.join(pkgs)) brew_result = _call_brew(cmd) if brew_result['retcode']: log.error('Failed to get info about packages: %s', @@ -405,9 +406,9 @@ def install(name=None, pkgs=None, taps=None, options=None, **kwargs): _tap(tap) if options: - cmd = 'brew install {0} {1}'.format(formulas, ' '.join(options)) + cmd = 'install {0} {1}'.format(formulas, ' '.join(options)) else: - cmd = 'brew install {0}'.format(formulas) + cmd = 'install {0}'.format(formulas) out = _call_brew(cmd) if out['retcode'] != 0 and out['stderr']: @@ -441,7 +442,7 @@ def list_upgrades(refresh=True, **kwargs): # pylint: disable=W0613 if refresh: refresh_db() - res = _call_brew(['brew', 'outdated', '--json=v1']) + res = _call_brew('outdated --json=v1') ret = {} try: @@ -501,7 +502,7 @@ def upgrade(refresh=True): if salt.utils.data.is_true(refresh): refresh_db() - result = _call_brew('brew upgrade', failhard=False) + result = _call_brew('upgrade', failhard=False) __context__.pop('pkg.list_pkgs', None) new = list_pkgs() ret = salt.utils.data.compare_dicts(old, new) diff --git a/salt/modules/napalm_acl.py b/salt/modules/napalm_acl.py index ce56572d4a..edafaa6d8c 100644 --- a/salt/modules/napalm_acl.py +++ b/salt/modules/napalm_acl.py @@ -49,7 +49,7 @@ from salt.utils.napalm import proxy_napalm_wrap # ------------------------------------------------------------------------------ __virtualname__ = 'netacl' -__proxyenabled__ = ['napalm'] +__proxyenabled__ = ['*'] # allow napalm proxy only # ------------------------------------------------------------------------------ diff --git a/salt/modules/napalm_bgp.py b/salt/modules/napalm_bgp.py index e6da8fcfc4..89bf03e292 100644 --- a/salt/modules/napalm_bgp.py +++ b/salt/modules/napalm_bgp.py @@ -35,6 +35,7 @@ from salt.utils.napalm import proxy_napalm_wrap __virtualname__ = 'bgp' __proxyenabled__ = ['napalm'] # uses NAPALM-based proxy to interact with network devices +__virtual_aliases__ = ('napalm_bgp',) # ---------------------------------------------------------------------------------------------------------------------- # property functions diff --git a/salt/modules/napalm_mod.py b/salt/modules/napalm_mod.py index 89673732c5..c24924f39b 100644 --- a/salt/modules/napalm_mod.py +++ b/salt/modules/napalm_mod.py @@ -12,7 +12,6 @@ from __future__ import absolute_import, unicode_literals, print_function # Import python stdlib import inspect import logging -log = logging.getLogger(__file__) # import NAPALM utils import salt.utils.napalm @@ -22,6 +21,7 @@ from salt.utils.napalm import proxy_napalm_wrap from salt.ext import six from salt.utils.decorators import depends from salt.exceptions import CommandExecutionError + try: from netmiko import BaseConnection HAS_NETMIKO = True @@ -51,9 +51,11 @@ except ImportError: # ---------------------------------------------------------------------------------------------------------------------- __virtualname__ = 'napalm' -__proxyenabled__ = ['napalm'] +__proxyenabled__ = ['*'] # uses NAPALM-based proxy to interact with network devices +log = logging.getLogger(__file__) + # ---------------------------------------------------------------------------------------------------------------------- # property functions # ---------------------------------------------------------------------------------------------------------------------- @@ -1446,3 +1448,271 @@ def config_filter_lines(parent_regex, child_regex, source='running'): return __salt__['ciscoconfparse.filter_lines'](config=config_txt, parent_regex=parent_regex, child_regex=child_regex) + + +def config_tree(source='running', with_tags=False): + ''' + .. versionadded:: Fluorine + + Transform Cisco IOS style configuration to structured Python dictionary. + Depending on the value of the ``with_tags`` argument, this function may + provide different views, valuable in different situations. + + source: ``running`` + The configuration type to retrieve from the network device. Default: + ``running``. Available options: ``running``, ``startup``, ``candidate``. + + with_tags: ``False`` + Whether this function should return a detailed view, with tags. + + CLI Example: + + .. code-block:: bash + + salt '*' napalm.config_tree + ''' + config_txt = __salt__['net.config'](source=source)['out'][source] + return __salt__['iosconfig.tree'](config=config_txt) + + +def config_merge_tree(source='running', + merge_config=None, + merge_path=None, + saltenv='base'): + ''' + .. versionadded:: Fluorine + + Return the merge tree of the ``initial_config`` with the ``merge_config``, + as a Python dictionary. + + source: ``running`` + The configuration type to retrieve from the network device. Default: + ``running``. Available options: ``running``, ``startup``, ``candidate``. + + merge_config + The config to be merged into the initial config, sent as text. This + argument is ignored when ``merge_path`` is set. + + merge_path + Absolute or remote path from where to load the merge configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``merge_path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' napalm.config_merge_tree merge_path=salt://path/to/merge.cfg + ''' + config_txt = __salt__['net.config'](source=source)['out'][source] + return __salt__['iosconfig.merge_tree'](initial_config=config_txt, + merge_config=merge_config, + merge_path=merge_path, + saltenv=saltenv) + + +def config_merge_text(source='running', + merge_config=None, + merge_path=None, + saltenv='base'): + ''' + .. versionadded:: Fluorine + + Return the merge result of the configuration from ``source`` with the + merge configuration, as plain text (without loading the config on the + device). + + source: ``running`` + The configuration type to retrieve from the network device. Default: + ``running``. Available options: ``running``, ``startup``, ``candidate``. + + merge_config + The config to be merged into the initial config, sent as text. This + argument is ignored when ``merge_path`` is set. + + merge_path + Absolute or remote path from where to load the merge configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``merge_path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' napalm.config_merge_text merge_path=salt://path/to/merge.cfg + ''' + config_txt = __salt__['net.config'](source=source)['out'][source] + return __salt__['iosconfig.merge_text'](initial_config=config_txt, + merge_config=merge_config, + merge_path=merge_path, + saltenv=saltenv) + + +def config_merge_diff(source='running', + merge_config=None, + merge_path=None, + saltenv='base'): + ''' + .. versionadded:: Fluorine + + Return the merge diff, as text, after merging the merge config into the + configuration source requested (without loading the config on the device). + + source: ``running`` + The configuration type to retrieve from the network device. Default: + ``running``. Available options: ``running``, ``startup``, ``candidate``. + + merge_config + The config to be merged into the initial config, sent as text. This + argument is ignored when ``merge_path`` is set. + + merge_path + Absolute or remote path from where to load the merge configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``merge_path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' napalm.config_merge_diff merge_path=salt://path/to/merge.cfg + ''' + config_txt = __salt__['net.config'](source=source)['out'][source] + return __salt__['iosconfig.merge_diff'](initial_config=config_txt, + merge_config=merge_config, + merge_path=merge_path, + saltenv=saltenv) + + +def config_diff_tree(source1='candidate', + candidate_path=None, + source2='running', + running_path=None): + ''' + .. versionadded:: Fluorine + + Return the diff, as Python dictionary, between two different sources. + The sources can be either specified using the ``source1`` and ``source2`` + arguments when retrieving from the managed network device. + + source1: ``candidate`` + The source from where to retrieve the configuration to be compared with. + Available options: ``candidate``, ``running``, ``startup``. Default: + ``candidate``. + + candidate_path + Absolute or remote path from where to load the candidate configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + source2: ``running`` + The source from where to retrieve the configuration to compare with. + Available options: ``candidate``, ``running``, ``startup``. Default: + ``running``. + + running_path + Absolute or remote path from where to load the runing configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``candidate_path`` or ``running_path`` is not a + ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' napalm.config_diff_text + salt '*' napalm.config_diff_text candidate_path=https://bit.ly/2mAdq7z + # Would compare the running config with the configuration available at + # https://bit.ly/2mAdq7z + + CLI Example: + + .. code-block:: bash + + salt '*' napalm.config_diff_tree + salt '*' napalm.config_diff_tree running startup + ''' + get_config = __salt__['net.config']()['out'] + candidate_cfg = get_config[source1] + running_cfg = get_config[source2] + return __salt__['iosconfig.diff_tree'](candidate_config=candidate_cfg, + candidate_path=candidate_path, + running_config=running_cfg, + running_path=running_path) + + +def config_diff_text(source1='candidate', + candidate_path=None, + source2='running', + running_path=None): + ''' + .. versionadded:: Fluorine + + Return the diff, as text, between the two different configuration sources. + The sources can be either specified using the ``source1`` and ``source2`` + arguments when retrieving from the managed network device. + + source1: ``candidate`` + The source from where to retrieve the configuration to be compared with. + Available options: ``candidate``, ``running``, ``startup``. Default: + ``candidate``. + + candidate_path + Absolute or remote path from where to load the candidate configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + source2: ``running`` + The source from where to retrieve the configuration to compare with. + Available options: ``candidate``, ``running``, ``startup``. Default: + ``running``. + + running_path + Absolute or remote path from where to load the runing configuration + text. This argument allows any URI supported by + :py:func:`cp.get_url `), e.g., ``salt://``, + ``https://``, ``s3://``, ``ftp:/``, etc. + + saltenv: ``base`` + Salt fileserver environment from which to retrieve the file. + Ignored if ``candidate_path`` or ``running_path`` is not a + ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' napalm.config_diff_text + salt '*' napalm.config_diff_text candidate_path=https://bit.ly/2mAdq7z + # Would compare the running config with the configuration available at + # https://bit.ly/2mAdq7z + ''' + get_config = __salt__['net.config']()['out'] + candidate_cfg = get_config[source1] + running_cfg = get_config[source2] + return __salt__['iosconfig.diff_text'](candidate_config=candidate_cfg, + candidate_path=candidate_path, + running_config=running_cfg, + running_path=running_path) diff --git a/salt/modules/napalm_network.py b/salt/modules/napalm_network.py index 23cbac98ae..1bbe4dac55 100644 --- a/salt/modules/napalm_network.py +++ b/salt/modules/napalm_network.py @@ -22,6 +22,7 @@ Dependencies # Import Python libs from __future__ import absolute_import, unicode_literals, print_function import logging +import datetime log = logging.getLogger(__name__) @@ -29,6 +30,7 @@ log = logging.getLogger(__name__) import salt.utils.files import salt.utils.napalm import salt.utils.templates +import salt.utils.stringutils # Import 3rd-party libs from salt.ext import six @@ -44,7 +46,8 @@ except ImportError: # ---------------------------------------------------------------------------------------------------------------------- __virtualname__ = 'net' -__proxyenabled__ = ['napalm'] +__proxyenabled__ = ['*'] +__virtual_aliases__ = ('napalm_net',) # uses NAPALM-based proxy to interact with network devices # ---------------------------------------------------------------------------------------------------------------------- @@ -104,6 +107,24 @@ def _filter_dict(input_dict, search_key, search_value): return output_dict +def _safe_dicard_config(loaded_result, napalm_device): + ''' + ''' + log.debug('Discarding the config') + log.debug(loaded_result) + _discarded = discard_config(inherit_napalm_device=napalm_device) + if not _discarded.get('result', False): + loaded_result['comment'] += _discarded['comment'] if _discarded.get('comment') \ + else 'Unable to discard config.' + loaded_result['result'] = False + # make sure it notifies + # that something went wrong + _explicit_close(napalm_device) + __context__['retcode'] = 1 + return loaded_result + return _discarded + + def _explicit_close(napalm_device): ''' Will explicily close the config session with the network device, @@ -126,13 +147,17 @@ def _explicit_close(napalm_device): def _config_logic(napalm_device, loaded_result, test=False, + debug=False, + replace=False, commit_config=True, - loaded_config=None): - + loaded_config=None, + commit_in=None, + commit_at=None, + commit_jid=None, + **kwargs): ''' Builds the config logic for `load_config` and `load_template` functions. ''' - # As the Salt logic is built around independent events # when it comes to configuration changes in the # candidate DB on the network devices, we need to @@ -147,10 +172,14 @@ def _config_logic(napalm_device, # `napalm_device` will be overridden. # See `salt.utils.napalm.proxy_napalm_wrap` decorator. + current_jid = kwargs.get('__pub_jid') + if not current_jid: + current_jid = '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now()) + loaded_result['already_configured'] = False loaded_result['loaded_config'] = '' - if loaded_config: + if debug: loaded_result['loaded_config'] = loaded_config _compare = compare_config(inherit_napalm_device=napalm_device) @@ -173,17 +202,9 @@ def _config_logic(napalm_device, loaded_result['comment'] += '\n' if not len(loaded_result.get('diff', '')) > 0: loaded_result['already_configured'] = True - _discarded = discard_config(inherit_napalm_device=napalm_device) - if not _discarded.get('result', False): - loaded_result['comment'] += _discarded['comment'] if _discarded.get('comment') \ - else 'Unable to discard config.' - loaded_result['result'] = False - # make sure it notifies - # that something went wrong - _explicit_close(napalm_device) - __context__['retcode'] = 1 + discarded = _safe_dicard_config(loaded_result, napalm_device) + if not discarded['result']: return loaded_result - loaded_result['comment'] += 'Configuration discarded.' # loaded_result['result'] = False not necessary # as the result can be true when test=True @@ -194,33 +215,58 @@ def _config_logic(napalm_device, if not test and commit_config: # if not in testing mode and trying to commit + if commit_jid: + log.info('Committing the JID: %s', str(commit_jid)) if len(loaded_result.get('diff', '')) > 0: # if not testing mode # and also the user wants to commit (default) # and there are changes to commit + if commit_in or commit_at: + commit_time = __utils__['timeutil.get_time_at'](time_in=commit_in, + time_at=commit_in) + # schedule job + scheduled_job_name = '__napalm_commit_{}'.format(current_jid) + scheduled = __salt__['schedule.add'](scheduled_job_name, + function='net.load_config', + job_kwargs={ + 'text': loaded_config, + 'commit_jid': current_jid, + 'replace': replace + }, + once=commit_time) + log.debug('Scheduling job') + log.debug(scheduled) + saved = __salt__['schedule.save']() # ensure the schedule is + # persistent cross Minion restart + discarded = _safe_dicard_config(loaded_result, napalm_device) + # discard the changes + if not discarded['result']: + discarded['comment'] += ('Scheduled the job to be executed at {schedule_ts}, ' + 'but was unable to discard the config: \n').format(schedule_ts=commit_time) + return discarded + loaded_result['comment'] = ('Changes discarded for now, and scheduled commit at: {schedule_ts}.\n' + 'The commit ID is: {current_jid}.\n' + 'To discard this commit, you can execute: \n\n' + 'salt {min_id} net.cancel_commit {current_jid}').format(schedule_ts=commit_time, + min_id=__opts__['id'], + current_jid=current_jid) + return loaded_result + log.debug('About to commit:') + log.debug(loaded_result['diff']) _commit = commit(inherit_napalm_device=napalm_device) # calls the function commit, defined below if not _commit.get('result', False): # if unable to commit loaded_result['comment'] += _commit['comment'] if _commit.get('comment') else 'Unable to commit.' loaded_result['result'] = False # unable to commit, something went wrong - _discarded = discard_config(inherit_napalm_device=napalm_device) - # try to discard, thus release the config DB - if not _discarded.get('result', False): - loaded_result['comment'] += '\n' - loaded_result['comment'] += _discarded['comment'] if _discarded.get('comment') \ - else 'Unable to discard config.' + discarded = _safe_dicard_config(loaded_result, napalm_device) + if not discarded['result']: + return loaded_result else: # would like to commit, but there's no change # need to call discard_config() to release the config DB - _discarded = discard_config(inherit_napalm_device=napalm_device) - if not _discarded.get('result', False): - loaded_result['comment'] += _discarded['comment'] if _discarded.get('comment') \ - else 'Unable to discard config.' - loaded_result['result'] = False - # notify if anything goes wrong - _explicit_close(napalm_device) - __context__['retcode'] = 1 + discarded = _safe_dicard_config(loaded_result, napalm_device) + if not discarded['result']: return loaded_result loaded_result['already_configured'] = True loaded_result['comment'] = 'Already configured.' @@ -229,7 +275,6 @@ def _config_logic(napalm_device, __context__['retcode'] = 1 return loaded_result - # ---------------------------------------------------------------------------------------------------------------------- # callable functions # ---------------------------------------------------------------------------------------------------------------------- @@ -1185,6 +1230,9 @@ def load_config(filename=None, commit=True, debug=False, replace=False, + commit_in=None, + commit_at=None, + commit_jid=None, inherit_napalm_device=None, saltenv='base', **kwargs): # pylint: disable=unused-argument @@ -1236,6 +1284,55 @@ def load_config(filename=None, .. versionadded:: 2016.11.2 + commit_in: ``None`` + Commit the changes in a specific number of minutes / hours. Example of + accepted formats: ``5`` (commit in 5 minutes), ``2m`` (commit in 2 + minutes), ``1h`` (commit the changes in 1 hour)`, ``5h30m`` (commit + the changes in 5 hours and 30 minutes). + + .. note:: + This feature works on any platforms, as it does not rely on the + native features of the network operating system. + + .. note:: + After the command is executed and the ``diff`` is not satisfactory, + or for any other reasons you have to discard the commit, you are + able to do so using the + :py:func:`net.cancel_commit ` + execution function, using the commit ID returned by this function. + + .. warning:: + Using this feature, Salt will load the exact configuration you + expect, however the diff may change in time (i.e., if an user + applies a manual configuration change, or a different process or + command changes the configuration in the meanwhile). + + .. versionadded: Fluorine + + commit_at: ``None`` + Commit the changes at a specific time. Example of accepted formats: + ``1am`` (will commit the changes at the next 1AM), ``13:20`` (will + commit at 13:20), ``1:20am``, etc. + + .. note:: + This feature works on any platforms, as it does not rely on the + native features of the network operating system. + + .. note:: + After the command is executed and the ``diff`` is not satisfactory, + or for any other reasons you have to discard the commit, you are + able to do so using the + :py:func:`net.cancel_commit ` + execution function, using the commit ID returned by this function. + + .. warning:: + Using this feature, Salt will load the exact configuration you + expect, however the diff may change in time (i.e., if an user + applies a manual configuration change, or a different process or + command changes the configuration in the meanwhile). + + .. versionadded: Fluorine + saltenv: ``base`` Specifies the Salt environment name. @@ -1302,18 +1399,17 @@ def load_config(filename=None, 'config': text } ) - loaded_config = None - if debug: - if filename: - with salt.utils.files.fopen(filename) as rfh: - loaded_config = salt.utils.stringutils.to_unicode(rfh.read()) - else: - loaded_config = text return _config_logic(napalm_device, # pylint: disable=undefined-variable _loaded, test=test, + debug=debug, + replace=replace, commit_config=commit, - loaded_config=loaded_config) + loaded_config=text, + commit_at=commit_at, + commit_in=commit_in, + commit_jid=commit_jid, + **kwargs) @salt.utils.napalm.proxy_napalm_wrap @@ -1333,6 +1429,8 @@ def load_template(template_name, commit=True, debug=False, replace=False, + commit_in=None, + commit_at=None, inherit_napalm_device=None, # pylint: disable=unused-argument **template_vars): ''' @@ -1458,6 +1556,55 @@ def load_template(template_name, .. versionadded:: 2016.11.2 + commit_in: ``None`` + Commit the changes in a specific number of minutes / hours. Example of + accepted formats: ``5`` (commit in 5 minutes), ``2m`` (commit in 2 + minutes), ``1h`` (commit the changes in 1 hour)`, ``5h30m`` (commit + the changes in 5 hours and 30 minutes). + + .. note:: + This feature works on any platforms, as it does not rely on the + native features of the network operating system. + + .. note:: + After the command is executed and the ``diff`` is not satisfactory, + or for any other reasons you have to discard the commit, you are + able to do so using the + :py:func:`net.cancel_commit ` + execution function, using the commit ID returned by this function. + + .. warning:: + Using this feature, Salt will load the exact configuration you + expect, however the diff may change in time (i.e., if an user + applies a manual configuration change, or a different process or + command changes the configuration in the meanwhile). + + .. versionadded: Fluorine + + commit_at: ``None`` + Commit the changes at a specific time. Example of accepted formats: + ``1am`` (will commit the changes at the next 1AM), ``13:20`` (will + commit at 13:20), ``1:20am``, etc. + + .. note:: + This feature works on any platforms, as it does not rely on the + native features of the network operating system. + + .. note:: + After the command is executed and the ``diff`` is not satisfactory, + or for any other reasons you have to discard the commit, you are + able to do so using the + :py:func:`net.cancel_commit ` + execution function, using the commit ID returned by this function. + + .. warning:: + Using this feature, Salt will load the exact configuration you + expect, however the diff may change in time (i.e., if an user + applies a manual configuration change, or a different process or + command changes the configuration in the meanwhile). + + .. versionadded: Fluorine + defaults: None Default variables/context passed to the template. @@ -1652,9 +1799,7 @@ def load_template(template_name, else: return _loaded # exit - if debug: # all good, but debug mode required - # valid output and debug mode - loaded_config = _rendered + loaded_config = _rendered if _loaded['result']: # all good fun = 'load_merge_candidate' if replace: # replace requested @@ -1706,8 +1851,13 @@ def load_template(template_name, return _config_logic(napalm_device, # pylint: disable=undefined-variable _loaded, test=test, + debug=debug, + replace=replace, commit_config=commit, - loaded_config=loaded_config) + loaded_config=loaded_config, + commit_at=commit_at, + commit_in=commit_in, + **template_vars) @salt.utils.napalm.proxy_napalm_wrap @@ -1863,6 +2013,33 @@ def config_control(inherit_napalm_device=None, **kwargs): # pylint: disable=unu return result, comment +def cancel_commit(jid): + ''' + .. versionadded:: Fluorine + + Cancel a commit scheduled to be executed via the ``commit_in`` and + ``commit_at`` arguments from the + :py:func:`net.load_template ` or + :py:func:`net.load_config `_ API. + +While for GET operations (the functions prefixed by ``get_``) the credentials +are optional, there are some specific details that are visible only to +authenticated users. Moreover, the credentials are required when adding or +updating information. That means, the module can equally work out of the box +without any further configuration with the limitations imposed by the PeeringDB +API. + +For complete API documentation, please refer to https://www.peeringdb.com/apidocs/. + +Configuration (in the opts or Pillar): + +.. code-block:: yaml + + peeringdb: + username: salt + password: 5@1t +''' +from __future__ import absolute_import + +# Import python libs +import logging +log = logging.getLogger(__name__) + +# Import salt modules +import salt.utils.http +try: + from salt.utils import clean_kwargs +except ImportError: + from salt.utils.args import clean_kwargs + +__virtualname__ = 'peeringdb' +__proxyenabled__ = ['*'] + +PEERINGDB_URL = 'https://www.peeringdb.com/api' + + +def __virtual__(): + return __virtualname__ + + +def _get_auth(username=None, + password=None): + peeringdb_cfg = __salt__['config.merge']('peeringdb', default={}) + if not username: + username = peeringdb_cfg.get('username', username) + if not password: + password = peeringdb_cfg.get('password', password) + return username, password + + +def _build_url(endpoint, id=None): + if id: + return '{base}/{endp}/{id}'.format(base=PEERINGDB_URL, endp=endpoint, id=id) + return '{base}/{endp}'.format(base=PEERINGDB_URL, endp=endpoint) + + +def _get_endpoint(endpoint, id=None, **kwargs): + username, password = _get_auth(kwargs.pop('username', None), + kwargs.pop('password', None)) + kwargs = clean_kwargs(**kwargs) + url = _build_url(endpoint, id=id) + ret = { + 'comment': '', + 'result': True, + 'out': None + } + res = salt.utils.http.query(url, + method='GET', + decode=True, + username=username, + password=password, + params=kwargs) + if 'error' in res: + ret.update({ + 'result': False, + 'comment': res['error'] + }) + return ret + ret['out'] = res['dict']['data'] + return ret + + +def get_net(**kwargs): + ''' + Return the details of a network identified using the search filters + specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible networks registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/net/net_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_net id=4224 + salt '*' peeringdb.get_net asn=13335 + salt '*' peeringdb.get_net city='Salt Lake City' + salt '*' peeringdb.get_net name__startswith=GTT + ''' + return _get_endpoint('net', **kwargs) + + +def get_fac(**kwargs): + ''' + Return the details of the facility identified using the search + filters specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible facilities registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/netfac/netfac_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_fac id=1774 + salt '*' peeringdb.get_fac state=UT + ''' + return _get_endpoint('fac', **kwargs) + + +def get_ix(**kwargs): + ''' + Return the details of an IX (Internet Exchange) using the search filters + specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible IXs registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/ix/ix_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_ix id=1 + salt '*' peeringdb.get_ix city='Milwaukee' + ''' + return _get_endpoint('ix', **kwargs) + + +def get_ixfac(**kwargs): + ''' + Return the details of an IX (Internet Exchange) facility using the search + filters specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible IX facilities registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/ixfac/ixfac_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_ixfac id=1 + salt '*' peeringdb.get_ixfac city='Milwaukee' + ''' + return _get_endpoint('ixfac', **kwargs) + + +def get_ixlan(**kwargs): + ''' + Return the details of an IX (Internet Exchange) together with the networks + available in this location (and their details), using the search filters + specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible IX LAN facilities registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/ixlan/ixlan_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_ixlan id=780 + salt '*' peeringdb.get_ixlan city='Milwaukee' + ''' + return _get_endpoint('ixlan', **kwargs) + + +def get_ixpfx(**kwargs): + ''' + Return the details of an IX (Internet Exchange) together with the PeeringDB + IDs of the networks available in this location, using the search filters + specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible IX LAN facilities registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/ixpfx/ixpfx_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_ixpfx id=780 + salt '*' peeringdb.get_ixpfx city='Milwaukee' + ''' + return _get_endpoint('ixpfx', **kwargs) + + +def get_netfac(**kwargs): + ''' + Return the list of facilities used by a particular network, given the ``id`` + or other filters specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible network facilities registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/netfac/netfac_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_netfac id=780 + salt '*' peeringdb.get_netfac city='Milwaukee' + ''' + return _get_endpoint('netfac', **kwargs) + + +def get_netixlan(**kwargs): + ''' + Return the IP addresses used by a particular network at all the IXs where it + is available. The network is selected either via the ``id`` argument or the + other filters specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible IP addresses, of all networks, at all IXs, registered in + PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/netixlan/netixlan_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_netixlan asn=13335 + salt '*' peeringdb.get_netixlan ipaddr4=185.1.114.25 + ''' + return _get_endpoint('netixlan', **kwargs) + + +def get_org(**kwargs): + ''' + Return the details of an organisation together with the networks + available in this location, using the search filters specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible organisations registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/org/org_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_org id=2 + salt '*' peeringdb.get_org city=Duesseldorf + ''' + return _get_endpoint('org', **kwargs) + + +def get_poc(**kwargs): + ''' + Return the details of a person of contact together using the search filters + specified in the query. + + .. note:: + If no ``id`` or filter arguments are specified, it will return all the + possible contacts registered in PeeringDB. + + The available filters are documented at: + https://www.peeringdb.com/apidocs/#!/poc/poc_list + + CLI Example: + + .. code-block:: bash + + salt '*' peeringdb.get_poc id=6721 + salt '*' peeringdb.get_poc email__contains='@cloudflare.com' + ''' + return _get_endpoint('poc', **kwargs) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 68ed8098a3..51e291184b 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -619,12 +619,6 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 editable=git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed upgrade=True no_deps=True ''' - if 'no_chown' in kwargs: - salt.utils.versions.warn_until( - 'Fluorine', - 'The no_chown argument has been deprecated and is no longer used. ' - 'Its functionality was removed in Boron.') - kwargs.pop('no_chown') cmd = _get_pip_bin(bin_env) cmd.append('install') diff --git a/salt/modules/pkgng.py b/salt/modules/pkgng.py index 4ec00d9bf0..dabd817fbf 100644 --- a/salt/modules/pkgng.py +++ b/salt/modules/pkgng.py @@ -678,6 +678,7 @@ def install(name=None, reinstall_requires=False, regex=False, pcre=False, + batch=False, **kwargs): ''' Install package(s) from a repository @@ -801,7 +802,16 @@ def install(name=None, .. code-block:: bash - salt '*' pkg.install pcre=True + + batch + Use BATCH=true for pkg install, skipping all questions. + Be careful when using in production. + + CLI Example: + + .. code-block:: bash + + salt '*' pkg.install batch=True ''' try: pkg_params, pkg_type = __salt__['pkg_resource.parse_targets']( @@ -813,6 +823,7 @@ def install(name=None, if pkg_params is None or len(pkg_params) == 0: return {} + env = {} opts = 'y' if salt.utils.data.is_true(orphan): opts += 'A' @@ -832,6 +843,11 @@ def install(name=None, opts += 'x' if salt.utils.data.is_true(pcre): opts += 'X' + if salt.utils.data.is_true(batch): + env = { + "BATCH": "true", + "ASSUME_ALWAYS_YES": "YES" + } old = list_pkgs(jail=jail, chroot=chroot, root=root) @@ -870,7 +886,8 @@ def install(name=None, out = __salt__['cmd.run_all']( cmd, output_loglevel='trace', - python_shell=False + python_shell=False, + env=env ) if out['retcode'] != 0 and out['stderr']: diff --git a/salt/modules/smbios.py b/salt/modules/smbios.py index c8a0e54a5c..ec7d6497c6 100644 --- a/salt/modules/smbios.py +++ b/salt/modules/smbios.py @@ -29,18 +29,13 @@ from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-bui log = logging.getLogger(__name__) -DMIDECODER = salt.utils.path.which_bin(['dmidecode', 'smbios']) - def __virtual__(): ''' Only work when dmidecode is installed. ''' - if DMIDECODER is None: - log.debug('SMBIOS: neither dmidecode nor smbios found!') - return (False, 'The smbios execution module failed to load: neither dmidecode nor smbios in the path.') - else: - return True + return (bool(salt.utils.path.which_bin(['dmidecode', 'smbios'])), + 'The smbios execution module failed to load: neither dmidecode nor smbios in the path.') def get(string, clean=True): @@ -86,16 +81,12 @@ def get(string, clean=True): val = _dmidecoder('-s {0}'.format(string)).strip() - # Sometimes dmidecode delivers comments in strings. - # Don't. + # Cleanup possible comments in strings. val = '\n'.join([v for v in val.split('\n') if not v.startswith('#')]) + if val.startswith('/dev/mem') or clean and not _dmi_isclean(string, val): + val = None - # handle missing /dev/mem - if val.startswith('/dev/mem'): - return None - - if not clean or _dmi_isclean(string, val): - return val + return val def records(rec_type=None, fields=None, clean=True): @@ -327,7 +318,11 @@ def _dmidecoder(args=None): ''' Call DMIdecode ''' - if args is None: - return salt.modules.cmdmod._run_quiet(DMIDECODER) + dmidecoder = salt.utils.path.which_bin(['dmidecode', 'smbios']) + + if not args: + out = salt.modules.cmdmod._run_quiet(dmidecoder) else: - return salt.modules.cmdmod._run_quiet('{0} {1}'.format(DMIDECODER, args)) + out = salt.modules.cmdmod._run_quiet('{0} {1}'.format(dmidecoder, args)) + + return out diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 874b7db467..41abd09175 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -3191,7 +3191,9 @@ class _policy_info(object): userSid = '{1}\\{0}'.format(userSid[0], userSid[1]) else: userSid = '{0}'.format(userSid[0]) + # TODO: This needs to be more specific except Exception: + log.exception('Handle this explicitly') userSid = win32security.ConvertSidToStringSid(_sid) usernames.append(userSid) return usernames @@ -3210,7 +3212,9 @@ class _policy_info(object): try: sid = win32security.LookupAccountName('', _user)[0] sids.append(sid) + # This needs to be more specific except Exception as e: + log.exception('Handle this explicitly') raise CommandExecutionError(( 'There was an error obtaining the SID of user "{0}". Error ' 'returned: {1}' @@ -3433,7 +3437,9 @@ def _processPolicyDefinitions(policy_def_path='c:\\Windows\\PolicyDefinitions', except lxml.etree.XMLSyntaxError: try: xmltree = _remove_unicode_encoding(admfile) + # TODO: This needs to be more specific except Exception: + log.exception('Handle this explicitly') log.error('A error was found while processing admx ' 'file %s, all policies from this file will ' 'be unavailable via this module', admfile) @@ -3518,7 +3524,9 @@ def _processPolicyDefinitions(policy_def_path='c:\\Windows\\PolicyDefinitions', # see issue #38100 try: xmltree = _remove_unicode_encoding(adml_file) + # TODO: This needs to be more specific except Exception: + log.exception('Handle this explicitly') log.error('An error was found while processing ' 'adml file %s, all policy ' 'language data from this file will be ' @@ -3574,8 +3582,9 @@ def _findOptionValueInSeceditFile(option): if _line.startswith(option): return True, _line.split('=')[1].strip() return True, 'Not Defined' - except Exception as e: - log.debug('error occurred while trying to get secedit data') + # TODO: This needs to be more specific + except Exception: + log.exception('error occurred while trying to get secedit data') return False, None @@ -3605,8 +3614,9 @@ def _importSeceditConfig(infdata): if __salt__['file.file_exists'](_tInfFile): _ret = __salt__['file.remove'](_tInfFile) return True + # TODO: This needs to be more specific except Exception as e: - log.debug('error occurred while trying to import secedit data') + log.exception('error occurred while trying to import secedit data') return False @@ -3668,9 +3678,10 @@ def _addAccountRights(sidObject, user_right): user_rights_list = [user_right] _ret = win32security.LsaAddAccountRights(_polHandle, sidObject, user_rights_list) return True + # TODO: This needs to be more specific except Exception as e: - log.error('Error attempting to add account right, exception was %s', - e) + log.exception('Error attempting to add account right, exception was %s', + e) return False @@ -3684,8 +3695,7 @@ def _delAccountRights(sidObject, user_right): _ret = win32security.LsaRemoveAccountRights(_polHandle, sidObject, False, user_rights_list) return True except Exception as e: - log.error('Error attempting to delete account right, ' - 'exception was %s', e) + log.exception('Error attempting to delete account right') return False @@ -4853,7 +4863,7 @@ def _write_regpol_data(data_to_write, try: reg_pol_header = u'\u5250\u6765\x01\x00' if not os.path.exists(policy_file_path): - ret = __salt__['file.makedirs'](policy_file_path) + __salt__['file.makedirs'](policy_file_path) with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: if not data_to_write.startswith(reg_pol_header.encode('utf-16-le')): pol_file.write(reg_pol_header.encode('utf-16-le')) @@ -4861,11 +4871,12 @@ def _write_regpol_data(data_to_write, try: gpt_ini_data = '' if os.path.exists(gpt_ini_path): - with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + with salt.utils.files.fopen(gpt_ini_path, 'r') as gpt_file: gpt_ini_data = gpt_file.read() if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): gpt_ini_data = '[General]\r\n' + gpt_ini_data - if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): + if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), + gpt_ini_data): # ensure the line contains the ADM guid gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), gpt_ini_data, @@ -4881,9 +4892,10 @@ def _write_regpol_data(data_to_write, general_location = re.search(r'^\[General\]\r\n', gpt_ini_data, re.IGNORECASE | re.MULTILINE) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data = '{0}{1}={2}\r\n{3}'.format( gpt_ini_data[general_location.start():general_location.end()], - gpt_extension, gpt_extension_guid, + gpt_extension, + gpt_extension_guid, gpt_ini_data[general_location.end():]) # https://technet.microsoft.com/en-us/library/cc978247.aspx if _regexSearchRegPolData(r'Version=', gpt_ini_data): @@ -4898,9 +4910,10 @@ def _write_regpol_data(data_to_write, elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): version_nums = (version_nums[0] + 1, version_nums[1]) version_num = struct.unpack(b'>I', struct.pack(b'>2H', *version_nums))[0] - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data = '{0}{1}={2}\r\n{3}'.format( gpt_ini_data[0:version_loc.start()], - 'Version', version_num, + 'Version', + version_num, gpt_ini_data[version_loc.end():]) else: general_location = re.search(r'^\[General\]\r\n', @@ -4910,20 +4923,26 @@ def _write_regpol_data(data_to_write, version_nums = (0, 1) elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): version_nums = (1, 0) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data = '{0}{1}={2}\r\n{3}'.format( gpt_ini_data[general_location.start():general_location.end()], 'Version', - int("{0}{1}".format(six.text_type(version_nums[0]).zfill(4), six.text_type(version_nums[1]).zfill(4)), 16), + int("{0}{1}".format(six.text_type(version_nums[0]).zfill(4), + six.text_type(version_nums[1]).zfill(4)), + 16), gpt_ini_data[general_location.end():]) if gpt_ini_data: - with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: - gpt_file.write(salt.utils.stringutils.to_bytes(gpt_ini_data)) + with salt.utils.files.fopen(gpt_ini_path, 'w') as gpt_file: + gpt_file.write(gpt_ini_data) + # TODO: This needs to be more specific except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( gpt_ini_path, e) + log.exception(msg) raise CommandExecutionError(msg) + # TODO: This needs to be more specific except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format(policy_file_path, e) + log.exception(msg) raise CommandExecutionError(msg) @@ -5321,8 +5340,9 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, policy_data.gpt_ini_path, policy_data.admx_registry_classes[registry_class]['gpt_extension_location'], policy_data.admx_registry_classes[registry_class]['gpt_extension_guid']) + # TODO: This needs to be more specific or removed except Exception: - log.error('Unhandled exception %s occurred while attempting to write Adm Template Policy File') + log.exception('Unhandled exception %s occurred while attempting to write Adm Template Policy File') return False return True @@ -5344,7 +5364,7 @@ def _getScriptSettingsFromIniFile(policy_info): _existingData = deserialize(_existingData.decode('utf-16-le').lstrip('\ufeff')) log.debug('Have deserialized data %s', _existingData) except Exception as error: - log.error('An error occurred attempting to deserialize data for %s', policy_info['Policy']) + log.exception('An error occurred attempting to deserialize data for %s', policy_info['Policy']) raise CommandExecutionError(error) if 'Section' in policy_info['ScriptIni'] and policy_info['ScriptIni']['Section'].lower() in [z.lower() for z in _existingData.keys()]: if 'SettingName' in policy_info['ScriptIni']: @@ -6218,8 +6238,10 @@ def set_(computer_policy=None, user_policy=None, _newModalSetData = dictupdate.update(_existingModalData, _modal_sets[_modal_set]) log.debug('NEW MODAL SET = %s', _newModalSetData) _ret = win32net.NetUserModalsSet(None, _modal_set, _newModalSetData) - except: + # TODO: This needs to be more specific + except Exception: msg = 'An unhandled exception occurred while attempting to set policy via NetUserModalSet' + log.exception(msg) raise CommandExecutionError(msg) if _admTemplateData: _ret = False diff --git a/salt/modules/win_powercfg.py b/salt/modules/win_powercfg.py index 93b3e1ea60..818f979718 100644 --- a/salt/modules/win_powercfg.py +++ b/salt/modules/win_powercfg.py @@ -19,7 +19,6 @@ import logging # Import Salt Libs import salt.utils.platform -import salt.utils.versions log = logging.getLogger(__name__) @@ -74,11 +73,6 @@ def _set_powercfg_value(scheme, sub_group, setting_guid, power, value): ''' Sets the AC/DC values of a setting with the given power for the given scheme ''' - salt.utils.versions.warn_until( - 'Fluorine', - 'This function now expects the timeout value in minutes instead of ' - 'seconds as stated in the documentation. This warning will be removed ' - 'in Salt Fluorine.') if scheme is None: scheme = _get_current_scheme() diff --git a/salt/modules/zfs.py b/salt/modules/zfs.py index 629b3f5bc5..6f6aa04a64 100644 --- a/salt/modules/zfs.py +++ b/salt/modules/zfs.py @@ -422,7 +422,7 @@ def mount(name=None, **kwargs): .. warning:: - Passing '-a' as name is deprecated and will be removed 2 verions after Fluorine. + Passing '-a' as name is deprecated and will be removed in Sodium. CLI Example: @@ -489,7 +489,7 @@ def unmount(name, **kwargs): .. warning:: - Passing '-a' as name is deprecated and will be removed 2 verions after Fluorine. + Passing '-a' as name is deprecated and will be removed in Sodium. CLI Example: @@ -507,7 +507,7 @@ def unmount(name, **kwargs): flags.append('-f') if name in [None, '-a']: # NOTE: still accept '-a' as name for backwards compatibility - # two versions after Fluorine this should just simplify + # until Salt Sodium this should just simplify # this to just set '-a' if name is not set. flags.append('-a') name = None diff --git a/salt/runners/virt.py b/salt/runners/virt.py index 2400e7fe0d..2bdc5d06a2 100644 --- a/salt/runners/virt.py +++ b/salt/runners/virt.py @@ -172,7 +172,9 @@ def init( start=True, disk='default', saltenv='base', - enable_vnc=False): + enable_vnc=False, + seed_cmd='seed.apply', + enable_qcow=False): ''' This routine is used to create a new virtual machine. This routines takes a number of options to determine what the newly created virtual machine @@ -194,14 +196,14 @@ def init( on the salt fileserver, but http, https and ftp can also be used. hypervisor - The hypervisor to use for the new virtual machine. Default is 'kvm'. + The hypervisor to use for the new virtual machine. Default is `kvm`. host The host to use for the new virtual machine, if this is omitted Salt will automatically detect what host to use. seed - Set to False to prevent Salt from seeding the new virtual machine. + Set to `False` to prevent Salt from seeding the new virtual machine. nic The nic profile to use, defaults to the "default" nic profile which @@ -217,6 +219,17 @@ def init( saltenv The Salt environment to use + + enable_vnc + Whether a VNC screen is attached to resulting VM. Default is `False`. + + seed_cmd + If seed is `True`, use this execution module function to seed new VM. + Default is `seed.apply`. + + enable_qcow + Clone disk image as a copy-on-write qcow2 image, using downloaded + `image` as backing file. ''' __jid_event__.fire_event({'message': 'Searching for hosts'}, 'progress') data = query(host, quiet=True) @@ -257,25 +270,29 @@ def init( ) try: cmd_ret = client.cmd_iter( - host, - 'virt.init', - [ - name, - cpu, - mem, - image, - nic, - hypervisor, - start, - disk, - saltenv, - seed, - install, - pub_key, - priv_key, - enable_vnc, - ], - timeout=600) + host, + 'virt.init', + [ + name, + cpu, + mem + ], + timeout=600, + kwarg={ + 'image': image, + 'nic': nic, + 'hypervisor': hypervisor, + 'start': start, + 'disk': disk, + 'saltenv': saltenv, + 'seed': seed, + 'install': install, + 'pub_key': pub_key, + 'priv_key': priv_key, + 'seed_cmd': seed_cmd, + 'enable_vnc': enable_vnc, + 'enable_qcow': enable_qcow, + }) except SaltClientError as client_error: # Fall through to ret error handling below print(client_error) diff --git a/salt/states/acme.py b/salt/states/acme.py index 983e5203b3..b7a44c43d0 100644 --- a/salt/states/acme.py +++ b/salt/states/acme.py @@ -55,7 +55,9 @@ def cert(name, tls_sni_01_port=None, tls_sni_01_address=None, http_01_port=None, - http_01_address=None): + http_01_address=None, + dns_plugin=None, + dns_plugin_credentials=None): ''' Obtain/renew a certificate from an ACME CA, probably Let's Encrypt. @@ -83,6 +85,8 @@ def cert(name, the port Certbot listens on. A conforming ACME server will still attempt to connect on port 80. :param https_01_address: The address the server listens to during http-01 challenge. + :param dns_plugin: Name of a DNS plugin to use (currently only 'cloudflare') + :param dns_plugin_credentials: Path to the credentials file if required by the specified DNS plugin ''' if __opts__['test']: @@ -130,7 +134,9 @@ def cert(name, tls_sni_01_port=tls_sni_01_port, tls_sni_01_address=tls_sni_01_address, http_01_port=http_01_port, - http_01_address=http_01_address + http_01_address=http_01_address, + dns_plugin=dns_plugin, + dns_plugin_credentials=dns_plugin_credentials, ) ret = { diff --git a/salt/states/file.py b/salt/states/file.py index 5d5d4478f6..ec87a6c946 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -1123,8 +1123,7 @@ def _get_template_texts(source_list=None, tmplines = None with salt.utils.files.fopen(rndrd_templ_fn, 'rb') as fp_: tmplines = fp_.read() - if six.PY3: - tmplines = tmplines.decode(__salt_system_encoding__) + tmplines = tmplines.decode(__salt_system_encoding__) tmplines = tmplines.splitlines(True) if not tmplines: msg = 'Failed to read rendered template file {0} ({1})' @@ -4394,9 +4393,15 @@ def blockreplace( A block of content delimited by comments can help you manage several lines entries without worrying about old entries removal. This can help you maintaining an un-managed file containing manual edits. - Note: this function will store two copies of the file in-memory - (the original version and the edited version) in order to detect changes - and only edit the targeted file if necessary. + + .. note:: + This function will store two copies of the file in-memory (the original + version and the edited version) in order to detect changes and only + edit the targeted file if necessary. + + Additionally, you can use :py:func:`file.accumulated + ` and target this state. All accumulated + data dictionaries' content will be added in the content block. name Filesystem path to the file to be edited @@ -4408,12 +4413,10 @@ def blockreplace( final output marker_end - The line content identifying a line as the end of the content block. - Note that the whole line containing this marker will be considered, so - whitespace or extra content before or after the marker is included in - final output. Note: you can use file.accumulated and target this state. - All accumulated data dictionaries content will be added as new lines in - the content + The line content identifying the end of the content block. As of + versions 2017.7.5 and 2018.3.1, everything up to the text matching the + marker will be replaced, so it's important to ensure that your marker + includes the beginning of the text you wish to replace. content The content to be used between the two lines identified by diff --git a/salt/states/netconfig.py b/salt/states/netconfig.py index a15b831a9f..42bbb45b99 100644 --- a/salt/states/netconfig.py +++ b/salt/states/netconfig.py @@ -116,6 +116,8 @@ def managed(name, commit=True, debug=False, replace=False, + commit_in=None, + commit_at=None, **template_vars): ''' @@ -219,6 +221,55 @@ def managed(name, :py:func:`state.apply ` (see below for an example). + commit_in: ``None`` + Commit the changes in a specific number of minutes / hours. Example of + accepted formats: ``5`` (commit in 5 minutes), ``2m`` (commit in 2 + minutes), ``1h`` (commit the changes in 1 hour)`, ``5h30m`` (commit + the changes in 5 hours and 30 minutes). + + .. note:: + This feature works on any platforms, as it does not rely on the + native features of the network operating system. + + .. note:: + After the command is executed and the ``diff`` is not satisfactory, + or for any other reasons you have to discard the commit, you are + able to do so using the + :py:func:`net.cancel_commit ` + execution function, using the commit ID returned by this function. + + .. warning:: + Using this feature, Salt will load the exact configuration you + expect, however the diff may change in time (i.e., if an user + applies a manual configuration change, or a different process or + command changes the configuration in the meanwhile). + + .. versionadded: Fluorine + + commit_at: ``None`` + Commit the changes at a specific time. Example of accepted formats: + ``1am`` (will commit the changes at the next 1AM), ``13:20`` (will + commit at 13:20), ``1:20am``, etc. + + .. note:: + This feature works on any platforms, as it does not rely on the + native features of the network operating system. + + .. note:: + After the command is executed and the ``diff`` is not satisfactory, + or for any other reasons you have to discard the commit, you are + able to do so using the + :py:func:`net.cancel_commit ` + execution function, using the commit ID returned by this function. + + .. warning:: + Using this feature, Salt will load the exact configuration you + expect, however the diff may change in time (i.e., if an user + applies a manual configuration change, or a different process or + command changes the configuration in the meanwhile). + + .. versionadded: Fluorine + replace: False Load and replace the configuration. Default: ``False`` (will apply load merge). @@ -339,6 +390,8 @@ def managed(name, commit = __salt__['config.merge']('commit', commit) replace = __salt__['config.merge']('replace', replace) # this might be a bit risky skip_verify = __salt__['config.merge']('skip_verify', skip_verify) + commit_in = __salt__['config.merge']('commit_in', commit_in) + commit_at = __salt__['config.merge']('commit_at', commit_at) config_update_ret = _update_config(template_name, template_source=template_source, @@ -354,6 +407,8 @@ def managed(name, defaults=defaults, test=test, commit=commit, + commit_in=commit_in, + commit_at=commit_at, debug=debug, replace=replace, **template_vars) diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index bbafaced63..f0050d93be 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -594,13 +594,6 @@ def installed(name, .. _`virtualenv`: http://www.virtualenv.org/en/latest/ ''' - if 'no_chown' in kwargs: - salt.utils.versions.warn_until( - 'Fluorine', - 'The no_chown argument has been deprecated and is no longer used. ' - 'Its functionality was removed in Boron.') - kwargs.pop('no_chown') - if pip_bin and not bin_env: bin_env = pip_bin diff --git a/salt/states/vault.py b/salt/states/vault.py index 238bbe685c..dd28957c7f 100644 --- a/salt/states/vault.py +++ b/salt/states/vault.py @@ -73,7 +73,7 @@ def _create_new_policy(name, rules): payload = {'rules': rules} url = "v1/sys/policy/{0}".format(name) response = __utils__['vault.make_request']('PUT', url, json=payload) - if response.status_code != 204: + if response.status_code not in [200, 204]: return { 'name': name, 'changes': {}, @@ -108,7 +108,7 @@ def _handle_existing_policy(name, new_rules, existing_rules): url = "v1/sys/policy/{0}".format(name) response = __utils__['vault.make_request']('PUT', url, json=payload) - if response.status_code != 204: + if response.status_code not in [200, 204]: return { 'name': name, 'changes': {}, diff --git a/salt/states/virtualenv_mod.py b/salt/states/virtualenv_mod.py index 79b6e8914f..c29d743b2a 100644 --- a/salt/states/virtualenv_mod.py +++ b/salt/states/virtualenv_mod.py @@ -135,13 +135,6 @@ def managed(name, - env_vars: PATH_VAR: '/usr/local/bin/' ''' - if 'no_chown' in kwargs: - salt.utils.versions.warn_until( - 'Fluorine', - 'The no_chown argument has been deprecated and is no longer used. ' - 'Its functionality was removed in Boron.') - kwargs.pop('no_chown') - ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} if 'virtualenv.create' not in __salt__: diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 6d2c3902a1..f256d55b3b 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -832,7 +832,7 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel): log.trace('Sending filtered data over publisher %s', pub_uri) # zmq filters are substring match, hash the topic # to avoid collisions - htopic = hashlib.sha1(topic).hexdigest() + htopic = salt.utils.stringutils.to_bytes(hashlib.sha1(topic).hexdigest()) pub_sock.send(htopic, flags=zmq.SNDMORE) pub_sock.send(payload) log.trace('Filtered data has been sent') @@ -840,7 +840,7 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel): else: # TODO: constants file for "broadcast" log.trace('Sending broadcasted data over publisher %s', pub_uri) - pub_sock.send('broadcast', flags=zmq.SNDMORE) + pub_sock.send(b'broadcast', flags=zmq.SNDMORE) pub_sock.send(payload) log.trace('Broadcasted data has been sent') else: diff --git a/salt/utils/args.py b/salt/utils/args.py index f5e1061af1..75f57f674d 100644 --- a/salt/utils/args.py +++ b/salt/utils/args.py @@ -34,6 +34,12 @@ def clean_kwargs(**kwargs): functions. These keys are useful for tracking what was used to invoke the function call, but they may not be desirable to have if passing the kwargs forward wholesale. + + Usage example: + + .. code-block:: python + + kwargs = __utils__['args.clean_kwargs'](**kwargs) ''' ret = {} for key, val in six.iteritems(kwargs): diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index caf539080f..286657c72c 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -1534,9 +1534,9 @@ class Pygit2(GitProvider): elif tag_ref in refs: tag_obj = self.repo.revparse_single(tag_ref) - if not isinstance(tag_obj, pygit2.Tag): + if not isinstance(tag_obj, pygit2.Commit): log.error( - '%s does not correspond to pygit2.Tag object', + '%s does not correspond to pygit2.Commit object', tag_ref ) else: @@ -1556,9 +1556,10 @@ class Pygit2(GitProvider): exc_info=True ) return None + log.debug('SHA of tag %s: %s', tgt_ref, tag_sha) - if head_sha != target_sha: - if not _perform_checkout(local_ref, branch=False): + if head_sha != tag_sha: + if not _perform_checkout(tag_ref, branch=False): return None # Return the relative root, if present diff --git a/salt/utils/minions.py b/salt/utils/minions.py index 41660b725a..f3e17c376d 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -239,13 +239,12 @@ class CkMinions(object): Retreive complete minion list from PKI dir. Respects cache if configured ''' - opts_role = self.opts.get('__role') - if (opts_role == 'master' and self.opts.get('__cli') == 'salt-run') or (opts_role == 'minion'): - # If we're compiling the pillar directly on the master (or running masterless) - # just return our local ID as that is the only one that is available. - return [self.opts['id']] minions = [] pki_cache_fn = os.path.join(self.opts['pki_dir'], self.acc, '.key_cache') + try: + os.makedirs(os.path.dirname(pki_cache_fn)) + except OSError: + pass try: if self.opts['key_cache'] and os.path.exists(pki_cache_fn): log.debug('Returning cached minion list') diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index c654f033cd..5230261e02 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -1266,6 +1266,28 @@ class ProxyIdMixIn(six.with_metaclass(MixInMeta, object)): ) +class ExecutorsMixIn(six.with_metaclass(MixInMeta, object)): + _mixin_prio = 10 + + def _mixin_setup(self): + self.add_option( + '--module-executors', + dest='module_executors', + default=None, + metavar='EXECUTOR_LIST', + help=('Set an alternative list of executors to override the one ' + 'set in minion config.') + ) + self.add_option( + '--executor-opts', + dest='executor_opts', + default=None, + metavar='EXECUTOR_OPTS', + help=('Set alternate executor options if supported by executor. ' + 'Options set by minion config are used by default.') + ) + + class CacheDirMixIn(six.with_metaclass(MixInMeta, object)): _mixin_prio = 40 @@ -1879,6 +1901,7 @@ class SaltCMDOptionParser(six.with_metaclass(OptionParserMeta, ExtendedTargetOptionsMixIn, OutputOptionsMixIn, LogLevelMixIn, + ExecutorsMixIn, HardCrashMixin, SaltfileMixIn, ArgsStdinMixIn, @@ -2016,22 +2039,6 @@ class SaltCMDOptionParser(six.with_metaclass(OptionParserMeta, metavar='RETURNER_KWARGS', help=('Set any returner options at the command line.') ) - self.add_option( - '--module-executors', - dest='module_executors', - default=None, - metavar='EXECUTOR_LIST', - help=('Set an alternative list of executors to override the one ' - 'set in minion config.') - ) - self.add_option( - '--executor-opts', - dest='executor_opts', - default=None, - metavar='EXECUTOR_OPTS', - help=('Set alternate executor options if supported by executor. ' - 'Options set by minion config are used by default.') - ) self.add_option( '-d', '--doc', '--documentation', dest='doc', @@ -2572,7 +2579,9 @@ class SaltKeyOptionParser(six.with_metaclass(OptionParserMeta, class SaltCallOptionParser(six.with_metaclass(OptionParserMeta, OptionParser, + ProxyIdMixIn, ConfigDirMixIn, + ExecutorsMixIn, MergeConfigMixIn, LogLevelMixIn, OutputOptionsMixIn, @@ -2732,8 +2741,13 @@ class SaltCallOptionParser(six.with_metaclass(OptionParserMeta, self.config['arg'] = self.args[1:] def setup_config(self): - opts = config.minion_config(self.get_config_file_path(), - cache_minion_id=True) + if self.options.proxyid: + opts = config.proxy_config(self.get_config_file_path(configfile='proxy'), + cache_minion_id=True, + minion_id=self.options.proxyid) + else: + opts = config.minion_config(self.get_config_file_path(), + cache_minion_id=True) if opts.get('transport') == 'raet': if not self._find_raet_minion(opts): # must create caller minion diff --git a/salt/utils/path.py b/salt/utils/path.py index 725fe5c828..b1d601e464 100644 --- a/salt/utils/path.py +++ b/salt/utils/path.py @@ -406,5 +406,9 @@ def os_walk(top, *args, **kwargs): This is a helper than ensures that all paths returned from os.walk are unicode. ''' - for item in os.walk(salt.utils.stringutils.to_str(top), *args, **kwargs): + if six.PY2 and salt.utils.platform.is_windows(): + top_query = top + else: + top_query = salt.utils.stringutils.to_str(top) + for item in os.walk(top_query, *args, **kwargs): yield salt.utils.data.decode(item, preserve_tuples=True) diff --git a/salt/utils/platform.py b/salt/utils/platform.py index 8ba56f2023..c93b023f3c 100644 --- a/salt/utils/platform.py +++ b/salt/utils/platform.py @@ -37,7 +37,10 @@ def is_proxy(): try: # Changed this from 'salt-proxy in main...' to 'proxy in main...' # to support the testsuite's temp script that is called 'cli_salt_proxy' - if 'proxy' in main.__file__: + # + # Add '--proxyid' in sys.argv so that salt-call --proxyid + # is seen as a proxy minion + if 'proxy' in main.__file__ or '--proxyid' in sys.argv: ret = True except AttributeError: pass diff --git a/salt/utils/timeutil.py b/salt/utils/timeutil.py new file mode 100644 index 0000000000..74a8226926 --- /dev/null +++ b/salt/utils/timeutil.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +''' +Functions various time manipulations. +''' +from __future__ import absolute_import, print_function, unicode_literals + +# Import Python +import logging +import time +from datetime import datetime +from datetime import timedelta +log = logging.getLogger(__name__) + +# Import Salt modules +from salt.ext import six + + +def get_timestamp_at(time_in=None, time_at=None): + ''' + Computes the timestamp for a future event that may occur in ``time_in`` time + or at ``time_at``. + ''' + if time_in: + if isinstance(time_in, int): + hours = 0 + minutes = time_in + else: + time_in = time_in.replace('h', ':') + time_in = time_in.replace('m', '') + try: + hours, minutes = time_in.split(':') + except ValueError: + hours = 0 + minutes = time_in + if not minutes: + minutes = 0 + hours, minutes = int(hours), int(minutes) + dt = timedelta(hours=hours, minutes=minutes) + time_now = datetime.utcnow() + time_at = time_now + dt + return time.mktime(time_at.timetuple()) + elif time_at: + log.debug('Predicted at specified as {}'.format(time_at)) + if isinstance(time_at, (six.integer_types, float)): + # then it's a timestamp + return time_at + else: + fmts = ('%H%M', '%Hh%M', '%I%p', '%I:%M%p', '%I:%M %p') + # Support different formats for the timestamp + # The current formats accepted are the following: + # + # - 18:30 (and 18h30) + # - 1pm (no minutes, fixed hour) + # - 1:20am (and 1:20am - with or without space) + for fmt in fmts: + try: + log.debug('Trying to match %s', fmt) + dt = datetime.strptime(time_at, fmt) + return time.mktime(dt.timetuple()) + except ValueError: + log.debug('Did not match %s, continue searching', fmt) + continue + msg = '{pat} does not match any of the accepted formats: {fmts}'.format(pat=time_at, + fmts=', '.join(fmts)) + log.error(msg) + raise ValueError(msg) + + +def get_time_at(time_in=None, time_at=None, out_fmt='%Y-%m-%dT%H:%M:%S'): + ''' + Return the time in human readable format for a future event that may occur + in ``time_in`` time, or at ``time_at``. + ''' + dt = get_timestamp_at(time_in=time_in, time_at=time_at) + return time.strftime(out_fmt, time.localtime(dt)) diff --git a/tests/integration/files/file/base/_modules/runtests_helpers.py b/tests/integration/files/file/base/_modules/runtests_helpers.py index cce1a3f544..865ebd55b3 100644 --- a/tests/integration/files/file/base/_modules/runtests_helpers.py +++ b/tests/integration/files/file/base/_modules/runtests_helpers.py @@ -73,6 +73,7 @@ def get_invalid_docs(): 'pkg.expand_repo_def', 'pip.iteritems', 'pip.parse_version', + 'peeringdb.clean_kwargs', 'runtests_decorators.depends', 'runtests_decorators.depends_will_fallback', 'runtests_decorators.missing_depends', diff --git a/tests/integration/files/file/base/issue-46672-a.sls b/tests/integration/files/file/base/issue-46672-a.sls new file mode 100644 index 0000000000..49840a029d --- /dev/null +++ b/tests/integration/files/file/base/issue-46672-a.sls @@ -0,0 +1,3 @@ +echo1: + cmd.run: + - name: "echo 'This is Æ test!'" diff --git a/tests/integration/files/file/base/testfile b/tests/integration/files/file/base/testfile index 0d234303e6..9c09a35e5b 100644 --- a/tests/integration/files/file/base/testfile +++ b/tests/integration/files/file/base/testfile @@ -1,6 +1,6 @@ Scene 24 - + OLD MAN: Ah, hee he he ha! ARTHUR: And this enchanter of whom you speak, he has seen the grail? OLD MAN: Ha ha he he he he! diff --git a/tests/integration/mockbin/su b/tests/integration/mockbin/su deleted file mode 100755 index b442c1baab..0000000000 --- a/tests/integration/mockbin/su +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# "Fake" su command -# -# Just executes the command without changing effective uid. Used in integration -# tests. -while true; do - shift - test "x$1" == "x-c" && break - test "x$1" == "x" && break -done -shift -exec /bin/bash -c "$@" diff --git a/tests/integration/modules/test_state.py b/tests/integration/modules/test_state.py index e22902df8d..3fd1cacf1c 100644 --- a/tests/integration/modules/test_state.py +++ b/tests/integration/modules/test_state.py @@ -1996,6 +1996,20 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): _expected = "cmd_|-echo1_|-echo 'This is Æ test!'_|-run" self.assertIn(_expected, ret) + def test_state_sls_unicode_characters_cmd_output(self): + ''' + test the output from running and echo command with non-ascii + characters. + ''' + ret = self.run_function('state.sls', ['issue-46672-a']) + key = list(ret.keys())[0] + log.debug('== ret %s ==', type(ret)) + _expected = 'This is Æ test!' + if salt.utils.platform.is_windows(): + # Windows cmd.exe will mangle the output using cmd's codepage. + _expected = "'This is ’ test!'" + self.assertEqual(_expected, ret[key]['changes']['stdout']) + def tearDown(self): nonbase_file = os.path.join(TMP, 'nonbase_env') if os.path.isfile(nonbase_file): diff --git a/tests/integration/proxy/test_shell.py b/tests/integration/proxy/test_shell.py new file mode 100644 index 0000000000..0737b639bd --- /dev/null +++ b/tests/integration/proxy/test_shell.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +''' +Test salt-call --proxyid commands + +tests.integration.proxy.test_shell +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +''' + +# Import python libs +from __future__ import absolute_import, print_function, unicode_literals + +# Import Salt Libs +import salt.utils.json as json + +# Import salt tests libs +from tests.support.case import ShellCase + + +class ProxyCallerSimpleTestCase(ShellCase): + ''' + Test salt-call --proxyid commands + ''' + @staticmethod + def _load_return(ret): + return json.loads('\n'.join(ret)) + + def test_can_it_ping(self): + ''' + Ensure the proxy can ping + ''' + ret = self._load_return(self.run_call('--proxyid proxytest --out=json test.ping')) + self.assertEqual(ret['local'], True) + + def test_list_pkgs(self): + ''' + Package test 1, really just tests that the virtual function capability + is working OK. + ''' + ret = self._load_return(self.run_call('--proxyid proxytest --out=json pkg.list_pkgs')) + self.assertIn('coreutils', ret['local']) + self.assertIn('apache', ret['local']) + self.assertIn('redbull', ret['local']) + + def test_upgrade(self): + ret = self._load_return(self.run_call('--proxyid proxytest --out=json pkg.upgrade')) + self.assertEqual(ret['local']['coreutils']['new'], '3.0') + self.assertEqual(ret['local']['redbull']['new'], '1001.99') + + def test_service_list(self): + ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.list')) + self.assertIn('ntp', ret['local']) + + def test_service_start(self): + ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.start samba')) + ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.status samba')) + self.assertTrue(ret) + + def test_service_get_all(self): + ret = self._load_return(self.run_call('--proxyid proxytest --out=json service.get_all')) + self.assertIn('samba', ret['local']) + + def test_grains_items(self): + ret = self._load_return(self.run_call('--proxyid proxytest --out=json grains.items')) + self.assertEqual(ret['local']['kernel'], 'proxy') + self.assertEqual(ret['local']['kernelrelease'], 'proxy') diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index 889073e6b5..5e902a7590 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -30,6 +30,7 @@ from tests.support.helpers import ( with_tempdir, with_tempfile, Webserver, + destructiveTest ) from tests.support.mixins import SaltReturnAssertsMixin @@ -4292,3 +4293,54 @@ class PatchTest(ModuleCase, SaltReturnAssertsMixin): ret = ret[next(iter(ret))] self.assertIn('Patch would not apply cleanly', ret['comment']) self.assertEqual(ret['changes'], {}) + +WIN_TEST_FILE = 'c:/testfile' + + +@destructiveTest +@skipIf(not IS_WINDOWS, 'windows test only') +class WinFileTest(ModuleCase): + ''' + Test for the file state on Windows + ''' + def setUp(self): + self.run_state('file.managed', name=WIN_TEST_FILE, makedirs=True, contents='Only a test') + + def tearDown(self): + self.run_state('file.absent', name=WIN_TEST_FILE) + + def test_file_managed(self): + ''' + Test file.managed on Windows + ''' + self.assertTrue(self.run_state('file.exists', name=WIN_TEST_FILE)) + + def test_file_copy(self): + ''' + Test file.copy on Windows + ''' + ret = self.run_state('file.copy', name='c:/testfile_copy', makedirs=True, source=WIN_TEST_FILE) + self.assertTrue(ret) + + def test_file_comment(self): + ''' + Test file.comment on Windows + ''' + self.run_state('file.comment', name=WIN_TEST_FILE, regex='^Only') + with salt.utils.files.fopen(WIN_TEST_FILE, 'r') as fp_: + self.assertTrue(fp_.read().startswith('#Only')) + + def test_file_replace(self): + ''' + Test file.replace on Windows + ''' + self.run_state('file.replace', name=WIN_TEST_FILE, pattern='test', repl='testing') + with salt.utils.files.fopen(WIN_TEST_FILE, 'r') as fp_: + self.assertIn('testing', fp_.read()) + + def test_file_absent(self): + ''' + Test file.absent on Windows + ''' + ret = self.run_state('file.absent', name=WIN_TEST_FILE) + self.assertTrue(ret) diff --git a/tests/unit/fileserver/test_roots.py b/tests/unit/fileserver/test_roots.py index 91deab475f..f3ab4cedf8 100644 --- a/tests/unit/fileserver/test_roots.py +++ b/tests/unit/fileserver/test_roots.py @@ -12,7 +12,7 @@ import tempfile # Import Salt Testing libs from tests.integration import AdaptedConfigurationTestCaseMixin from tests.support.mixins import LoaderModuleMockMixin -from tests.support.paths import FILES, TMP, TMP_STATE_TREE +from tests.support.paths import BASE_FILES, TMP, TMP_STATE_TREE from tests.support.unit import TestCase, skipIf from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON @@ -20,6 +20,7 @@ from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON import salt.fileserver.roots as roots import salt.fileclient import salt.utils.files +import salt.utils.hashutils import salt.utils.platform try: @@ -63,13 +64,11 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix else: cls.test_symlink_list_file_roots = None cls.tmp_dir = tempfile.mkdtemp(dir=TMP) - full_path_to_file = os.path.join(FILES, 'file', 'base', 'testfile') + full_path_to_file = os.path.join(BASE_FILES, 'testfile') with salt.utils.files.fopen(full_path_to_file, 'rb') as s_fp: with salt.utils.files.fopen(os.path.join(cls.tmp_dir, 'testfile'), 'wb') as d_fp: for line in s_fp: - d_fp.write( - line.rstrip(b'\n').rstrip(b'\r') + os.linesep.encode('utf-8') - ) + d_fp.write(line.rstrip(b'\n').rstrip(b'\r') + b'\n') @classmethod def tearDownClass(cls): @@ -95,7 +94,7 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix ret = roots.find_file('testfile') self.assertEqual('testfile', ret['rel']) - full_path_to_file = os.path.join(FILES, 'file', 'base', 'testfile') + full_path_to_file = os.path.join(BASE_FILES, 'testfile') self.assertEqual(full_path_to_file, ret['path']) def test_serve_file(self): @@ -108,34 +107,9 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix 'rel': 'testfile'} ret = roots.serve_file(load, fnd) - data = 'Scene 24\n\n \n OLD MAN: Ah, hee he he ha!\n ' \ - 'ARTHUR: And this enchanter of whom you speak, he ' \ - 'has seen the grail?\n OLD MAN: Ha ha he he he ' \ - 'he!\n ARTHUR: Where does he live? Old man, where ' \ - 'does he live?\n OLD MAN: He knows of a cave, a ' \ - 'cave which no man has entered.\n ARTHUR: And the ' \ - 'Grail... The Grail is there?\n OLD MAN: Very much ' \ - 'danger, for beyond the cave lies the Gorge\n ' \ - 'of Eternal Peril, which no man has ever crossed.\n ' \ - 'ARTHUR: But the Grail! Where is the Grail!?\n ' \ - 'OLD MAN: Seek you the Bridge of Death.\n ARTHUR: ' \ - 'The Bridge of Death, which leads to the Grail?\n ' \ - 'OLD MAN: Hee hee ha ha!\n\n' - if salt.utils.platform.is_windows(): - data = 'Scene 24\r\n\r\n \r\n OLD MAN: Ah, hee he he ' \ - 'ha!\r\n ARTHUR: And this enchanter of whom you ' \ - 'speak, he has seen the grail?\r\n OLD MAN: Ha ha ' \ - 'he he he he!\r\n ARTHUR: Where does he live? Old ' \ - 'man, where does he live?\r\n OLD MAN: He knows of ' \ - 'a cave, a cave which no man has entered.\r\n ' \ - 'ARTHUR: And the Grail... The Grail is there?\r\n ' \ - 'OLD MAN: Very much danger, for beyond the cave lies ' \ - 'the Gorge\r\n of Eternal Peril, which no man ' \ - 'has ever crossed.\r\n ARTHUR: But the Grail! ' \ - 'Where is the Grail!?\r\n OLD MAN: Seek you the ' \ - 'Bridge of Death.\r\n ARTHUR: The Bridge of Death, ' \ - 'which leads to the Grail?\r\n OLD MAN: Hee hee ha ' \ - 'ha!\r\n\r\n' + with salt.utils.files.fopen( + os.path.join(BASE_FILES, 'testfile'), 'rb') as fp_: + data = fp_.read() self.assertDictEqual( ret, @@ -163,9 +137,9 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix # Hashes are different in Windows. May be how git translates line # endings - hsum = 'baba5791276eb99a7cc498fb1acfbc3b4bd96d24cfe984b4ed6b5be2418731df' - if salt.utils.platform.is_windows(): - hsum = '754aa260e1f3e70f43aaf92149c7d1bad37f708c53304c37660e628d7553f687' + with salt.utils.files.fopen( + os.path.join(BASE_FILES, 'testfile'), 'rb') as fp_: + hsum = salt.utils.hashutils.sha256_digest(fp_.read()) self.assertDictEqual( ret, diff --git a/tests/unit/modules/test_cp.py b/tests/unit/modules/test_cp.py index 587c794a1f..e3eeb05a25 100644 --- a/tests/unit/modules/test_cp.py +++ b/tests/unit/modules/test_cp.py @@ -137,6 +137,8 @@ class CpTestCase(TestCase, LoaderModuleMockMixin): filename = 'c:\\saltines\\test.file' with patch('salt.modules.cp.os.path', MagicMock(isfile=Mock(return_value=True), wraps=cp.os.path)), \ + patch('salt.modules.cp.os.path', + MagicMock(getsize=MagicMock(return_value=10), wraps=cp.os.path)), \ patch.multiple('salt.modules.cp', _auth=MagicMock(**{'return_value.gen_token.return_value': 'token'}), __opts__={'id': 'abc', 'file_buffer_size': 10}), \ @@ -154,6 +156,7 @@ class CpTestCase(TestCase, LoaderModuleMockMixin): cmd='_file_recv', tok='token', path=['saltines', 'test.file'], + size=10, data=b'', # data is empty here because load['data'] is overwritten id='abc' ) diff --git a/tests/unit/modules/test_iosconfig.py b/tests/unit/modules/test_iosconfig.py new file mode 100644 index 0000000000..879d531be5 --- /dev/null +++ b/tests/unit/modules/test_iosconfig.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +''' +Test the iosconfig Execution module. +''' +from __future__ import absolute_import, print_function, unicode_literals + +# Import Python libs +import textwrap + +# Import Salt Testing libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON + +# Import Salt modules +from salt.utils.odict import OrderedDict +import salt.modules.iosconfig as iosconfig + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TestModulesIOSConfig(TestCase, LoaderModuleMockMixin): + + running_config = textwrap.dedent('''\ + interface GigabitEthernet1 + ip address dhcp + negotiation auto + no mop enabled + ! + interface GigabitEthernet2 + ip address 172.20.0.1 255.255.255.0 + shutdown + negotiation auto + ! + interface GigabitEthernet3 + no ip address + shutdown + negotiation auto + !''') + + candidate_config = textwrap.dedent('''\ + interface GigabitEthernet1 + ip address dhcp + negotiation auto + no mop enabled + ! + interface GigabitEthernet2 + no ip address + shutdown + negotiation auto + ! + interface GigabitEthernet3 + no ip address + negotiation auto + ! + router bgp 65000 + bgp log-neighbor-changes + neighbor 1.1.1.1 remote-as 12345 + ! + !''') + + merge_config = textwrap.dedent('''\ + router bgp 65000 + bgp log-neighbor-changes + neighbor 1.1.1.1 remote-as 12345 + ! + ! + virtual-service csr_mgmt + ! + ip forward-protocol nd + !''') + + def setup_loader_modules(self): + return {} + + def test_tree(self): + running_config_tree = OrderedDict([ + (u'interface GigabitEthernet1', OrderedDict([ + (u'ip address dhcp', OrderedDict()), + (u'negotiation auto', OrderedDict()), + (u'no mop enabled', OrderedDict()) + ])), + (u'interface GigabitEthernet2', OrderedDict([ + (u'ip address 172.20.0.1 255.255.255.0', OrderedDict()), + (u'shutdown', OrderedDict()), + (u'negotiation auto', OrderedDict()) + ])), + (u'interface GigabitEthernet3', OrderedDict([ + (u'no ip address', OrderedDict()), + (u'shutdown', OrderedDict()), + (u'negotiation auto', OrderedDict()) + ])) + ]) + tree = iosconfig.tree(config=self.running_config) + self.assertEqual(tree, running_config_tree) + + def test_clean(self): + clean_running_config = textwrap.dedent('''\ + interface GigabitEthernet1 + ip address dhcp + negotiation auto + no mop enabled + interface GigabitEthernet2 + ip address 172.20.0.1 255.255.255.0 + shutdown + negotiation auto + interface GigabitEthernet3 + no ip address + shutdown + negotiation auto + ''') + clean = iosconfig.clean(config=self.running_config) + self.assertEqual(clean, clean_running_config) + + def test_merge_tree(self): + expected_merge_tree = OrderedDict([ + (u'interface GigabitEthernet1', OrderedDict([ + (u'ip address dhcp', OrderedDict()), + (u'negotiation auto', OrderedDict()), + (u'no mop enabled', OrderedDict()) + ])), + (u'interface GigabitEthernet2', OrderedDict([ + (u'ip address 172.20.0.1 255.255.255.0', OrderedDict()), + (u'shutdown', OrderedDict()), + (u'negotiation auto', OrderedDict()) + ])), + (u'interface GigabitEthernet3', OrderedDict([ + (u'no ip address', OrderedDict()), + (u'shutdown', OrderedDict()), + (u'negotiation auto', OrderedDict()) + ])), + (u'router bgp 65000', OrderedDict([ + (u'bgp log-neighbor-changes', OrderedDict()), + (u'neighbor 1.1.1.1 remote-as 12345', OrderedDict()) + ])), + (u'virtual-service csr_mgmt', OrderedDict()), + (u'ip forward-protocol nd', OrderedDict()) + ]) + merge_tree = iosconfig.merge_tree(initial_config=self.running_config, + merge_config=self.merge_config) + self.assertEqual(merge_tree, expected_merge_tree) + + def test_merge_text(self): + extected_merge_text = textwrap.dedent('''\ + interface GigabitEthernet1 + ip address dhcp + negotiation auto + no mop enabled + interface GigabitEthernet2 + ip address 172.20.0.1 255.255.255.0 + shutdown + negotiation auto + interface GigabitEthernet3 + no ip address + shutdown + negotiation auto + router bgp 65000 + bgp log-neighbor-changes + neighbor 1.1.1.1 remote-as 12345 + virtual-service csr_mgmt + ip forward-protocol nd + ''') + merge_text = iosconfig.merge_text(initial_config=self.running_config, + merge_config=self.merge_config) + self.assertEqual(merge_text, extected_merge_text) + + def test_merge_diff(self): + expected_diff = textwrap.dedent('''\ + @@ -10,3 +10,8 @@ + no ip address + shutdown + negotiation auto + +router bgp 65000 + + bgp log-neighbor-changes + + neighbor 1.1.1.1 remote-as 12345 + +virtual-service csr_mgmt + +ip forward-protocol nd + ''') + diff = iosconfig.merge_diff(initial_config=self.running_config, + merge_config=self.merge_config) + self.assertEqual(diff.splitlines()[2:], expected_diff.splitlines()) + + def test_diff_text(self): + expected_diff = textwrap.dedent('''\ + @@ -3,10 +3,12 @@ + negotiation auto + no mop enabled + interface GigabitEthernet2 + - ip address 172.20.0.1 255.255.255.0 + + no ip address + shutdown + negotiation auto + interface GigabitEthernet3 + no ip address + - shutdown + negotiation auto + +router bgp 65000 + + bgp log-neighbor-changes + + neighbor 1.1.1.1 remote-as 12345 + ''') + diff = iosconfig.diff_text(candidate_config=self.candidate_config, + running_config=self.running_config) + self.assertEqual(diff.splitlines()[2:], expected_diff.splitlines()) diff --git a/tests/whitelist.txt b/tests/whitelist.txt index 7913b67dd7..b017ceffdf 100644 --- a/tests/whitelist.txt +++ b/tests/whitelist.txt @@ -43,6 +43,7 @@ integration.states.test_pip_state integration.states.test_pkg integration.states.test_reg integration.states.test_renderers +integration.states.test_file integration.states.test_user integration.utils.testprogram integration.wheel.test_client