From 3be90cc9f4f8c84de2bc9edd251d4aa6f63747c0 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Fri, 5 May 2017 15:32:16 -0600 Subject: [PATCH 01/25] Rescue proxy_auto_tests PR from git rebase hell --- salt/config/__init__.py | 290 +++++++++++++++++++++++++++- salt/modules/dummyproxy_package.py | 94 +++++++++ salt/modules/dummyproxy_service.py | 156 +++++++++++++++ salt/proxy/dummy.py | 239 +++++++++++++++++++++++ salt/scripts.py | 2 +- salt/utils/__init__.py | 4 +- scripts/salt-proxy | 6 +- tests/integration/__init__.py | 58 +++++- tests/integration/files/conf/proxy | 50 +++++ tests/integration/proxy/__init__.py | 1 + tests/integration/proxy/simple.py | 78 ++++++++ tests/runtests.py | 46 ++++- 12 files changed, 1002 insertions(+), 22 deletions(-) create mode 100644 salt/modules/dummyproxy_package.py create mode 100644 salt/modules/dummyproxy_service.py create mode 100644 salt/proxy/dummy.py create mode 100644 tests/integration/files/conf/proxy create mode 100644 tests/integration/proxy/__init__.py create mode 100644 tests/integration/proxy/simple.py diff --git a/salt/config/__init__.py b/salt/config/__init__.py index e286c223f7..aa86e97be1 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1482,19 +1482,249 @@ DEFAULT_MASTER_OPTS = { # ----- Salt Proxy Minion Configuration Defaults -----------------------------------> -# Note DEFAULT_MINION_OPTS -# is loaded first, then if we are setting up a proxy, the config is overwritten with -# these settings. DEFAULT_PROXY_MINION_OPTS = { 'conf_file': os.path.join(salt.syspaths.CONFIG_DIR, 'proxy'), 'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'proxy'), - 'sign_pub_messages': False, 'add_proxymodule_to_opts': False, 'proxy_merge_grains_in_module': False, 'append_minionid_config_dirs': ['cachedir', 'pidfile'], 'default_include': 'proxy.d/*.conf', + 'interface': '0.0.0.0', + 'master': 'salt', + 'master_type': 'str', + 'master_uri_format': 'default', + 'master_port': 4506, + 'master_finger': '', + 'master_shuffle': False, + 'master_alive_interval': 0, + 'master_failback': False, + 'master_failback_interval': 0, + 'verify_master_pubkey_sign': False, + 'sign_pub_messages': False, + 'always_verify_signature': False, + 'master_sign_key_name': 'master_sign', + 'syndic_finger': '', + 'user': salt.utils.get_user(), + 'root_dir': salt.syspaths.ROOT_DIR, + 'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'proxy'), + 'id': '', + 'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'proxy'), + 'cache_jobs': False, + 'grains_cache': False, + 'grains_cache_expiration': 300, + 'grains_deep_merge': False, + 'sock_dir': os.path.join(salt.syspaths.SOCK_DIR, 'proxy'), + 'backup_mode': '', + 'renderer': 'yaml_jinja', + 'renderer_whitelist': [], + 'renderer_blacklist': [], + 'random_startup_delay': 0, + 'failhard': False, + 'autoload_dynamic_modules': True, + 'environment': None, + 'pillarenv': None, + 'pillar_opts': False, + # ``pillar_cache``, ``pillar_cache_ttl`` and ``pillar_cache_backend`` + # are not used on the minion but are unavoidably in the code path + 'pillar_cache': False, + 'pillar_cache_ttl': 3600, + 'pillar_cache_backend': 'disk', + 'extension_modules': os.path.join(salt.syspaths.CACHE_DIR, 'proxy', 'extmods'), + 'state_top': 'top.sls', + 'state_top_saltenv': None, + 'startup_states': '', + 'sls_list': [], + 'top_file': '', + 'thorium_interval': 0.5, + 'thorium_roots': { + 'base': [salt.syspaths.BASE_THORIUM_ROOTS_DIR], + }, + 'file_client': 'remote', + 'use_master_when_local': False, + 'file_roots': { + 'base': [salt.syspaths.BASE_FILE_ROOTS_DIR, + salt.syspaths.SPM_FORMULA_PATH] + }, + 'top_file_merging_strategy': 'merge', + 'env_order': [], + 'default_top': 'base', + 'fileserver_limit_traversal': False, + 'file_recv': False, + 'file_recv_max_size': 100, + 'file_ignore_regex': [], + 'file_ignore_glob': [], + 'fileserver_backend': ['roots'], + 'fileserver_followsymlinks': True, + 'fileserver_ignoresymlinks': False, + 'pillar_roots': { + 'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR, + salt.syspaths.SPM_PILLAR_PATH] + }, + 'on_demand_ext_pillar': ['libvirt', 'virtkey'], + 'git_pillar_base': 'master', + 'git_pillar_branch': 'master', + 'git_pillar_env': '', + 'git_pillar_root': '', + 'git_pillar_ssl_verify': True, + 'git_pillar_global_lock': True, + 'git_pillar_user': '', + 'git_pillar_password': '', + 'git_pillar_insecure_auth': False, + 'git_pillar_privkey': '', + 'git_pillar_pubkey': '', + 'git_pillar_passphrase': '', + 'gitfs_remotes': [], + 'gitfs_mountpoint': '', + 'gitfs_root': '', + 'gitfs_base': 'master', + 'gitfs_user': '', + 'gitfs_password': '', + 'gitfs_insecure_auth': False, + 'gitfs_privkey': '', + 'gitfs_pubkey': '', + 'gitfs_passphrase': '', + 'gitfs_env_whitelist': [], + 'gitfs_env_blacklist': [], + 'gitfs_global_lock': True, + 'gitfs_ssl_verify': True, + 'gitfs_saltenv': [], + 'hash_type': 'sha256', + 'disable_modules': [], + 'disable_returners': [], + 'whitelist_modules': [], + 'module_dirs': [], + 'returner_dirs': [], + 'grains_dirs': [], + 'states_dirs': [], + 'render_dirs': [], + 'outputter_dirs': [], + 'utils_dirs': [], + 'providers': {}, + 'clean_dynamic_modules': True, + 'open_mode': False, + 'auto_accept': True, + 'autosign_timeout': 120, + 'multiprocessing': True, + 'mine_enabled': True, + 'mine_return_job': False, + 'mine_interval': 60, + 'ipc_mode': _DFLT_IPC_MODE, + 'ipc_write_buffer': _DFLT_IPC_WBUFFER, + 'ipv6': None, + 'file_buffer_size': 262144, + 'tcp_pub_port': 4510, + 'tcp_pull_port': 4511, + 'tcp_authentication_retries': 5, + 'log_level': 'warning', + 'log_level_logfile': None, + 'log_datefmt': _DFLT_LOG_DATEFMT, + 'log_datefmt_logfile': _DFLT_LOG_DATEFMT_LOGFILE, + 'log_fmt_console': _DFLT_LOG_FMT_CONSOLE, + 'log_fmt_logfile': _DFLT_LOG_FMT_LOGFILE, + 'log_granular_levels': {}, + 'max_event_size': 1048576, + 'test': False, + 'ext_job_cache': '', + 'cython_enable': False, + 'enable_zip_modules': False, + 'state_verbose': True, + 'state_output': 'full', + 'state_output_diff': False, + 'state_auto_order': True, + 'state_events': False, + 'state_aggregate': False, + 'snapper_states': False, + 'snapper_states_config': 'root', + 'acceptance_wait_time': 10, + 'acceptance_wait_time_max': 0, + 'rejected_retry': False, + 'loop_interval': 1, + 'verify_env': True, + 'grains': {}, + 'permissive_pki_access': False, + 'update_url': False, + 'update_restart_services': [], + 'retry_dns': 30, + 'recon_max': 10000, + 'recon_default': 1000, + 'recon_randomize': True, + 'return_retry_timer': 5, + 'return_retry_timer_max': 10, + 'random_reauth_delay': 10, + 'winrepo_source_dir': 'salt://win/repo-ng/', + 'winrepo_dir': os.path.join(salt.syspaths.BASE_FILE_ROOTS_DIR, 'win', 'repo'), + 'winrepo_dir_ng': os.path.join(salt.syspaths.BASE_FILE_ROOTS_DIR, 'win', 'repo-ng'), + 'winrepo_cachefile': 'winrepo.p', + 'winrepo_cache_expire_max': 21600, + 'winrepo_cache_expire_min': 0, + 'winrepo_remotes': ['https://github.com/saltstack/salt-winrepo.git'], + 'winrepo_remotes_ng': ['https://github.com/saltstack/salt-winrepo-ng.git'], + 'winrepo_branch': 'master', + 'winrepo_ssl_verify': True, + 'winrepo_user': '', + 'winrepo_password': '', + 'winrepo_insecure_auth': False, + 'winrepo_privkey': '', + 'winrepo_pubkey': '', + 'winrepo_passphrase': '', + 'pidfile': os.path.join(salt.syspaths.PIDFILE_DIR, 'salt-proxy.pid'), + 'range_server': 'range:80', + 'reactor_refresh_interval': 60, + 'reactor_worker_threads': 10, + 'reactor_worker_hwm': 10000, + 'engines': [], + 'tcp_keepalive': True, + 'tcp_keepalive_idle': 300, + 'tcp_keepalive_cnt': -1, + 'tcp_keepalive_intvl': -1, + 'modules_max_memory': -1, + 'grains_refresh_every': 0, + 'minion_id_caching': True, + 'keysize': 2048, + 'transport': 'zeromq', + 'auth_timeout': 5, + 'auth_tries': 7, + 'master_tries': _MASTER_TRIES, + 'auth_safemode': False, + 'random_master': False, + 'minion_floscript': os.path.join(FLO_DIR, 'minion.flo'), + 'caller_floscript': os.path.join(FLO_DIR, 'caller.flo'), + 'ioflo_verbose': 0, + 'ioflo_period': 0.1, + 'ioflo_realtime': True, + 'ioflo_console_logdir': '', + 'raet_port': 4510, + 'raet_alt_port': 4511, + 'raet_mutable': False, + 'raet_main': False, + 'raet_clear_remotes': True, + 'raet_clear_remote_masters': True, + 'raet_road_bufcnt': 2, + 'raet_lane_bufcnt': 100, + 'cluster_mode': False, + 'cluster_masters': [], + 'restart_on_error': False, + 'ping_interval': 0, + 'username': None, + 'password': None, + 'zmq_filtering': False, + 'zmq_monitor': False, + 'cache_sreqs': True, + 'cmd_safe': True, + 'sudo_user': '', + 'http_request_timeout': 1 * 60 * 60.0, # 1 hour + 'http_max_body': 100 * 1024 * 1024 * 1024, # 100GB + 'event_match_type': 'startswith', + 'minion_restart_command': [], + 'pub_ret': True, + 'proxy_host': '', + 'proxy_username': '', + 'proxy_password': '', + 'proxy_port': 0, + 'minion_jid_queue_hwm': 100, + 'ssl': None, + 'cache': 'localfs', } - # ----- Salt Cloud Configuration Defaults -----------------------------------> DEFAULT_CLOUD_OPTS = { 'verify_env': True, @@ -1998,6 +2228,56 @@ def minion_config(path, return opts +def proxy_config(path, + env_var='SALT_PROXY_CONFIG', + defaults=None, + cache_minion_id=False, + ignore_config_errors=True, + minion_id=None): + ''' + Reads in the proxy minion configuration file and sets up special options + + This is useful for Minion-side operations, such as the + :py:class:`~salt.client.Caller` class, and manually running the loader + interface. + + .. code-block:: python + + import salt.config + proxy_opts = salt.config.proxy_config('/etc/salt/proxy') + ''' + if path is not None and path.endswith('proxy'): + defaults = DEFAULT_PROXY_MINION_OPTS.copy() + + if not os.environ.get(env_var, None): + # No valid setting was given using the configuration variable. + # Lets see is SALT_CONFIG_DIR is of any use + salt_config_dir = os.environ.get('SALT_CONFIG_DIR', None) + if salt_config_dir: + env_config_file_path = os.path.join(salt_config_dir, 'proxy') + if salt_config_dir and os.path.isfile(env_config_file_path): + # We can get a configuration file using SALT_CONFIG_DIR, let's + # update the environment with this information + os.environ[env_var] = env_config_file_path + + overrides = load_config(path, env_var, DEFAULT_PROXY_MINION_OPTS['conf_file']) + default_include = overrides.get('default_include', + defaults['default_include']) + include = overrides.get('include', []) + + overrides.update(include_config(default_include, path, verbose=False, + exit_on_config_errors=not ignore_config_errors)) + overrides.update(include_config(include, path, verbose=True, + exit_on_config_errors=not ignore_config_errors)) + + opts = apply_minion_config(overrides, defaults, + cache_minion_id=cache_minion_id, + minion_id=minion_id) + apply_sdb(opts) + _validate_opts(opts) + return opts + + def syndic_config(master_config_path, minion_config_path, master_env_var='SALT_MASTER_CONFIG', diff --git a/salt/modules/dummyproxy_package.py b/salt/modules/dummyproxy_package.py new file mode 100644 index 0000000000..64f5dacda1 --- /dev/null +++ b/salt/modules/dummyproxy_package.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +''' +Package support for the dummy proxy used by the test suite +''' +from __future__ import absolute_import + +# Import python libs +import logging +import salt.utils + + +log = logging.getLogger(__name__) + +# Define the module's virtual name +__virtualname__ = 'pkg' + + +def __virtual__(): + ''' + Only work on systems that are a proxy minion + ''' + try: + if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'dummy': + return __virtualname__ + except KeyError: + return (False, 'The dummyproxy_package execution module failed to load. Check the proxy key in pillar or /etc/salt/proxy.') + + return (False, 'The dummyproxy_package execution module failed to load: only works on a dummy proxy minion.') + + +def list_pkgs(versions_as_list=False, **kwargs): + return __proxy__['dummy.package_list']() + + +def install(name=None, refresh=False, fromrepo=None, + pkgs=None, sources=None, **kwargs): + return __proxy__['dummy.package_install'](name, **kwargs) + + +def remove(name=None, pkgs=None, **kwargs): + return __proxy__['dummy.package_remove'](name) + + +def version(*names, **kwargs): + ''' + Returns a string representing the package version or an empty string if not + installed. If more than one package name is specified, a dict of + name/version pairs is returned. + + CLI Example: + + .. code-block:: bash + + salt '*' pkg.version + salt '*' pkg.version ... + ''' + if len(names) == 1: + vers = __proxy__['dummy.package_status'](names[0]) + return vers[names[0]] + else: + results = {} + for n in names: + vers = __proxy__['dummy.package_status'](n) + results.update(vers) + return results + + +def upgrade(name=None, pkgs=None, refresh=True, skip_verify=True, + normalize=True, **kwargs): + old = __proxy__['dummy.package_list']() + new = __proxy__['dummy.uptodate']() + pkg_installed = __proxy__['dummy.upgrade']() + ret = salt.utils.compare_dicts(old, pkg_installed) + return ret + + +def installed(name, + version=None, + refresh=False, + fromrepo=None, + skip_verify=False, + pkgs=None, + sources=None, + **kwargs): + + p = __proxy__['dummy.package_status'](name) + if version is None: + if 'ret' in p: + return str(p['ret']) + else: + return True + else: + if p is not None: + return version == str(p) diff --git a/salt/modules/dummyproxy_service.py b/salt/modules/dummyproxy_service.py new file mode 100644 index 0000000000..c085768b2c --- /dev/null +++ b/salt/modules/dummyproxy_service.py @@ -0,0 +1,156 @@ + +# -*- coding: utf-8 -*- +''' +Provide the service module for the dummy proxy used in integration tests +''' +# Import python libs +from __future__ import absolute_import +import salt.utils + +import logging + +log = logging.getLogger(__name__) + +__func_alias__ = { + 'list_': 'list' +} + + +# Define the module's virtual name +__virtualname__ = 'service' + + +def __virtual__(): + ''' + Only work on systems that are a proxy minion + ''' + try: + if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'dummy': + return __virtualname__ + except KeyError: + return (False, 'The dummyproxy_service execution module failed to load. Check the proxy key in pillar or /etc/salt/proxy.') + + return (False, 'The dummyproxy_service execution module failed to load: only works on the integration testsuite dummy proxy minion.') + + +def get_all(): + ''' + Return a list of all available services + + .. versionadded:: 2016.11.3 + + CLI Example: + + .. code-block:: bash + + salt '*' service.get_all + ''' + proxy_fn = 'dummy.service_list' + return __proxy__[proxy_fn]() + + +def list_(): + ''' + Return a list of all available services. + + .. versionadded:: 2016.11.3 + + CLI Example: + + .. code-block:: bash + + salt '*' service.list + ''' + return get_all() + + +def start(name, sig=None): + ''' + Start the specified service on the dummy + + .. versionadded:: 2016.11.3 + + CLI Example: + + .. code-block:: bash + + salt '*' service.start + ''' + + proxy_fn = 'dummy.service_start' + return __proxy__[proxy_fn](name) + + +def stop(name, sig=None): + ''' + Stop the specified service on the dummy + + .. versionadded:: 2016.11.3 + + CLI Example: + + .. code-block:: bash + + salt '*' service.stop + ''' + proxy_fn = 'dummy.service_stop' + return __proxy__[proxy_fn](name) + + +def restart(name, sig=None): + ''' + Restart the specified service with dummy. + + .. versionadded:: 2016.11.3 + + CLI Example: + + .. code-block:: bash + + salt '*' service.restart + ''' + + proxy_fn = 'dummy.service_restart' + return __proxy__[proxy_fn](name) + + +def status(name, sig=None): + ''' + Return the status for a service via dummy, returns a bool + whether the service is running. + + .. versionadded:: 2016.11.3 + + CLI Example: + + .. code-block:: bash + + salt '*' service.status + ''' + + proxy_fn = 'dummy.service_status' + resp = __proxy__[proxy_fn](name) + if resp['comment'] == 'stopped': + return False + if resp['comment'] == 'running': + return True + + +def running(name, sig=None): + ''' + Return whether this service is running. + + .. versionadded:: 2016.11.3 + + ''' + return status(name).get(name, False) + + +def enabled(name, sig=None): + ''' + Only the 'redbull' service is 'enabled' in the test + + .. versionadded:: 2016.11.3 + + ''' + return name == 'redbull' diff --git a/salt/proxy/dummy.py b/salt/proxy/dummy.py new file mode 100644 index 0000000000..d3404681e6 --- /dev/null +++ b/salt/proxy/dummy.py @@ -0,0 +1,239 @@ +# -*- coding: utf-8 -*- +''' +This is a dummy proxy-minion designed for testing the proxy minion subsystem. +''' +from __future__ import absolute_import + +# Import python libs +import os +import logging +import pickle + +# This must be present or the Salt loader won't load this module +__proxyenabled__ = ['dummy'] + + +# Variables are scoped to this module so we can have persistent data +# across calls to fns in here. +DETAILS = {} + +DETAILS['services'] = {'apache': 'running', 'ntp': 'running', 'samba': 'stopped'} +DETAILS['packages'] = {'coreutils': '1.0', 'apache': '2.4', 'tinc': '1.4', 'redbull': '999.99'} +FILENAME = os.tmpnam() +# Want logging! +log = logging.getLogger(__file__) + + +# This does nothing, it's here just as an example and to provide a log +# entry when the module is loaded. +def __virtual__(): + ''' + Only return if all the modules are available + ''' + log.debug('dummy proxy __virtual__() called...') + return True + + +def _save_state(details): + global FILENAME + pck = open(FILENAME, 'wb') + pickle.dump(details, pck) + pck.close() + + +def _load_state(): + global FILENAME + try: + pck = open(FILENAME, 'r') + DETAILS = pickle.load(pck) + pck.close() + except IOError: + DETAILS = {} + DETAILS['initialized'] = False + _save_state(DETAILS) + + return DETAILS + + +# Every proxy module needs an 'init', though you can +# just put DETAILS['initialized'] = True here if nothing +# else needs to be done. + +def init(opts): + log.debug('dummy proxy init() called...') + DETAILS['initialized'] = True + _save_state(DETAILS) + + +def initialized(): + ''' + Since grains are loaded in many different places and some of those + places occur before the proxy can be initialized, return whether + our init() function has been called + ''' + DETAILS = _load_state() + return DETAILS.get('initialized', False) + + +def grains(): + ''' + Make up some grains + ''' + DETAILS = _load_state() + if 'grains_cache' not in DETAILS: + DETAILS['grains_cache'] = {'dummy_grain_1': 'one', 'dummy_grain_2': 'two', 'dummy_grain_3': 'three', } + _save_state(DETAILS) + + return DETAILS['grains_cache'] + + +def grains_refresh(): + ''' + Refresh the grains + ''' + DETAILS = _load_state() + DETAILS['grains_cache'] = None + _save_state(DETAILS) + return grains() + + +def fns(): + return {'details': 'This key is here because a function in ' + 'grains/rest_sample.py called fns() here in the proxymodule.'} + + +def service_start(name): + ''' + Start a "service" on the dummy server + ''' + DETAILS = _load_state() + DETAILS['services'][name] = 'running' + _save_state(DETAILS) + return 'running' + + +def service_stop(name): + ''' + Stop a "service" on the dummy server + ''' + DETAILS = _load_state() + DETAILS['services'][name] = 'stopped' + _save_state(DETAILS) + return 'stopped' + + +def service_restart(name): + ''' + Restart a "service" on the REST server + ''' + return True + + +def service_list(): + ''' + List "services" on the REST server + ''' + DETAILS = _load_state() + return DETAILS['services'].keys() + + +def service_status(name): + ''' + Check if a service is running on the REST server + ''' + DETAILS = _load_state() + if DETAILS['services'][name] == 'running': + return {'comment': 'running'} + else: + return {'comment': 'stopped'} + + +def package_list(): + ''' + List "packages" installed on the REST server + ''' + DETAILS = _load_state() + return DETAILS['packages'] + + +def package_install(name, **kwargs): + ''' + Install a "package" on the REST server + ''' + DETAILS = _load_state() + if kwargs.get('version', False): + version = kwargs['version'] + else: + version = '1.0' + DETAILS['packages'][name] = version + _save_state(DETAILS) + return {name: version} + + +def upgrade(): + ''' + "Upgrade" packages + ''' + DETAILS = _load_state() + pkgs = uptodate() + DETAILS['packages'] = pkgs + _save_state(DETAILS) + return pkgs + + +def uptodate(): + ''' + Call the REST endpoint to see if the packages on the "server" are up to date. + ''' + DETAILS = _load_state() + for p in DETAILS['packages']: + version_float = float(DETAILS['packages'][p]) + version_float = version_float + 1.0 + DETAILS['packages'][p] = str(version_float) + return DETAILS['packages'] + + +def package_remove(name): + ''' + Remove a "package" on the REST server + ''' + DETAILS = _load_state() + DETAILS['packages'].pop(name) + _save_state(DETAILS) + return DETAILS['packages'] + + +def package_status(name): + ''' + Check the installation status of a package on the REST server + ''' + DETAILS = _load_state() + if name in DETAILS['packages']: + return {name: DETAILS['packages'][name]} + + +def ping(): + ''' + Degenerate ping + ''' + log.debug('dummy proxy returning ping') + return True + + +def shutdown(opts): + ''' + For this proxy shutdown is a no-op + ''' + log.debug('dummy proxy shutdown() called...') + DETAILS = _load_state() + if 'filename' in DETAILS: + os.unlink(DETAILS['filename']) + + +def test_from_state(): + ''' + Test function so we have something to call from a state + :return: + ''' + log.debug('test_from_state called') + return 'testvalue' diff --git a/salt/scripts.py b/salt/scripts.py index 47b8a2e7bd..90683a0985 100644 --- a/salt/scripts.py +++ b/salt/scripts.py @@ -271,7 +271,7 @@ def proxy_minion_process(queue): sys.exit(status) -def salt_proxy_minion(): +def salt_proxy(): ''' Start a proxy minion. ''' diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 53bb225285..25b4ba57db 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1675,7 +1675,9 @@ def is_proxy(): # then this will fail. is_proxy = False try: - if 'salt-proxy' in main.__file__: + # Changed this from 'salt-proxy in main...' to 'proxy in main...' + # to support the testsuite's temp script that is called 'cli_salt_proxy' + if 'proxy' in main.__file__: is_proxy = True except AttributeError: pass diff --git a/scripts/salt-proxy b/scripts/salt-proxy index 8b763816da..9d2a3b2f43 100755 --- a/scripts/salt-proxy +++ b/scripts/salt-proxy @@ -1,9 +1,11 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- ''' This script is used to kick off a salt proxy minion daemon ''' -from salt.scripts import salt_proxy_minion +from __future__ import absolute_import +from salt.scripts import salt_proxy from salt.utils import is_windows from multiprocessing import freeze_support @@ -23,4 +25,4 @@ if __name__ == '__main__': # This handles the bootstrapping code that is included with frozen # scripts. It is a no-op on unfrozen code. freeze_support() - salt_proxy_minion() + salt_proxy() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index d3a901ef8c..5041070aa5 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -650,9 +650,31 @@ class SaltMinion(SaltDaemonScriptBase): return set([self.config['id']]) +class SaltProxy(SaltDaemonScriptBase): + ''' + Class which runs the salt-proxy daemon + ''' + + cli_script_name = 'salt-proxy' + + def get_script_args(self): + #script_args = ['-l', 'debug', '--proxyid', 'testproxy'] + script_args = ['-l', 'quiet', '--proxyid', 'testproxy'] + if salt.utils.is_windows() is False: + script_args.append('--disable-keepalive') + return script_args + + def get_check_ports(self): + if salt.utils.is_windows(): + return set([self.config['tcp_pub_port'], + self.config['tcp_pull_port']]) + else: + return set([self.config['id']]) + + class SaltMaster(SaltDaemonScriptBase): ''' - Class which runs the salt-minion daemon + Class which runs the salt-master daemon ''' cli_script_name = 'salt-master' @@ -791,8 +813,16 @@ class TestDaemon(object): self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - for process in (self.master_process, self.minion_process, self.sub_minion_process, - self.smaster_process, self.syndic_process): + processes_to_start = [self.master_process, self.minion_process, self.sub_minion_process, + self.smaster_process, self.syndic_process] + + if self.parser.options.proxy or (getattr(self.parser.options, 'name', False) + and any(['proxy' in item + for item in self.parser.options.name])): + self.proxy_process = SaltProxy(self.proxy_opts, TMP_CONF_DIR, SCRIPT_DIR) + self.proxy_process.display_name = 'salt-proxy' + processes_to_start.append(self.proxy_process) + for process in processes_to_start: sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( process.display_name, @@ -1020,6 +1050,7 @@ class TestDaemon(object): * syndic * syndic_master * sub_minion + * proxy ''' return RUNTIME_CONFIGS[role] @@ -1106,6 +1137,16 @@ class TestDaemon(object): syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') + # This proxy connects to master + proxy_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'proxy')) + proxy_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') + proxy_opts['user'] = running_tests_user + proxy_opts['config_dir'] = TMP_CONF_DIR + proxy_opts['root_dir'] = os.path.join(TMP, 'rootdir') + proxy_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki') + proxy_opts['hosts.file'] = os.path.join(TMP, 'rootdir', 'hosts') + proxy_opts['aliases.file'] = os.path.join(TMP, 'rootdir', 'aliases') + if transport == 'raet': master_opts['transport'] = 'raet' master_opts['raet_port'] = 64506 @@ -1178,7 +1219,7 @@ class TestDaemon(object): syndic_opts[optname] = optname_path syndic_master_opts[optname] = optname_path - for conf in (master_opts, minion_opts, sub_minion_opts, syndic_opts, syndic_master_opts): + for conf in (master_opts, minion_opts, sub_minion_opts, syndic_opts, syndic_master_opts, proxy_opts): if 'log_handlers_dirs' not in conf: conf['log_handlers_dirs'] = [] conf['log_handlers_dirs'].insert(0, LOG_HANDLERS_DIR) @@ -1186,7 +1227,7 @@ class TestDaemon(object): # ----- Transcribe Configuration ----------------------------------------------------------------------------> for entry in os.listdir(CONF_DIR): - if entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master'): + if entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master', 'proxy'): # These have runtime computed values and will be handled # differently continue @@ -1202,7 +1243,7 @@ class TestDaemon(object): os.path.join(TMP_CONF_DIR, entry) ) - for entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master'): + for entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master', 'proxy'): computed_config = copy.deepcopy(locals()['{0}_opts'.format(entry)]) with salt.utils.fopen(os.path.join(TMP_CONF_DIR, entry), 'w') as fp_: fp_.write(yaml.dump(computed_config, default_flow_style=False)) @@ -1226,6 +1267,7 @@ class TestDaemon(object): # ----- Verify Environment ----------------------------------------------------------------------------------> master_opts = salt.config.master_config(os.path.join(TMP_CONF_DIR, 'master')) minion_opts = salt.config.minion_config(os.path.join(TMP_CONF_DIR, 'minion')) + proxy_opts = salt.config.proxy_config(os.path.join(TMP_CONF_DIR, 'proxy')) syndic_opts = salt.config.syndic_config( os.path.join(TMP_SYNDIC_MINION_CONF_DIR, 'master'), os.path.join(TMP_SYNDIC_MINION_CONF_DIR, 'minion'), @@ -1238,6 +1280,7 @@ class TestDaemon(object): RUNTIME_CONFIGS['syndic'] = freeze(syndic_opts) RUNTIME_CONFIGS['sub_minion'] = freeze(sub_minion_opts) RUNTIME_CONFIGS['syndic_master'] = freeze(syndic_master_opts) + RUNTIME_CONFIGS['proxy'] = freeze(proxy_opts) verify_env([os.path.join(master_opts['pki_dir'], 'minions'), os.path.join(master_opts['pki_dir'], 'minions_pre'), @@ -1284,6 +1327,7 @@ class TestDaemon(object): cls.master_opts = master_opts cls.minion_opts = minion_opts + cls.proxy_opts = proxy_opts cls.sub_minion_opts = sub_minion_opts cls.syndic_opts = syndic_opts cls.syndic_master_opts = syndic_master_opts @@ -1295,6 +1339,8 @@ class TestDaemon(object): ''' self.sub_minion_process.terminate() self.minion_process.terminate() + if hasattr(self, 'proxy_process'): + self.proxy_process.terminate() self.master_process.terminate() try: self.syndic_process.terminate() diff --git a/tests/integration/files/conf/proxy b/tests/integration/files/conf/proxy new file mode 100644 index 0000000000..2ed703377e --- /dev/null +++ b/tests/integration/files/conf/proxy @@ -0,0 +1,50 @@ +# basic config +# Connects to master +master: localhost +master_port: 64506 +interface: 127.0.0.1 +tcp_pub_port: 64510 +tcp_pull_port: 64511 +sock_dir: proxy_sock +id: proxytest +open_mode: True +log_file: proxy.log +log_level_logfile: debug +pidfile: proxy.pid + +# module extension +test.foo: baz +integration.test: True + +# Grains addons +grains: + test_grain: cheese + script: grail + alot: many + planets: + - mercury + - venus + - earth + - mars + level1: + level2: foo + companions: + one: + - susan + - ian + - barbara + +config_test: + spam: eggs + +mine_functions: + test.ping: [] + +# sdb env module +osenv: + driver: env + +add_proxymodule_to_opts: False + +proxy: + proxytype: dummy diff --git a/tests/integration/proxy/__init__.py b/tests/integration/proxy/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/tests/integration/proxy/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/integration/proxy/simple.py b/tests/integration/proxy/simple.py new file mode 100644 index 0000000000..fcfc7a6c0c --- /dev/null +++ b/tests/integration/proxy/simple.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +''' +Simple Smoke Tests for Connected Proxy Minion +''' + +# Import Python libs +from __future__ import absolute_import + +# Import Salt Testing libs +from salttesting.helpers import ensure_in_syspath + +ensure_in_syspath('../') + +# Import Salt libs +import integration + + +class ProxyMinionSimpleTestCase(integration.ModuleCase): + ''' + Test minion blackout functionality + ''' + def test_can_it_ping(self): + ''' + Ensure the proxy can ping + ''' + ret = self.run_function('test.ping', minion_tgt='proxytest') + self.assertEqual(ret, True) + + def test_list_pkgs(self): + ''' + Package test 1, really just tests that the virtual function capability + is working OK. + ''' + ret = self.run_function('pkg.list_pkgs', minion_tgt='proxytest') + self.assertIn('coreutils', ret) + self.assertIn('apache', ret) + self.assertIn('redbull', ret) + + def test_install_pkgs(self): + ''' + Package test 2, really just tests that the virtual function capability + is working OK. + ''' + ret = self.run_function('pkg.install', ['thispkg'], minion_tgt='proxytest') + self.assertEqual(ret['thispkg'], '1.0') + + ret = self.run_function('pkg.list_pkgs', minion_tgt='proxytest') + + self.assertEqual(ret['apache'], '2.4') + self.assertEqual(ret['redbull'], '999.99') + self.assertEqual(ret['thispkg'], '1.0') + + def test_remove_pkgs(self): + ret = self.run_function('pkg.remove', ['apache'], minion_tgt='proxytest') + self.assertNotIn('apache', ret) + + def test_upgrade(self): + ret = self.run_function('pkg.upgrade', minion_tgt='proxytest') + self.assertEqual(ret['coreutils']['new'], '2.0') + self.assertEqual(ret['redbull']['new'], '1000.99') + + def test_service_list(self): + ret = self.run_function('service.list', minion_tgt='proxytest') + self.assertIn('ntp', ret) + + def test_service_stop(self): + ret = self.run_function('service.stop', ['ntp'], minion_tgt='proxytest') + ret = self.run_function('service.status', ['ntp'], minion_tgt='proxytest') + self.assertFalse(ret) + + def test_service_start(self): + ret = self.run_function('service.start', ['samba'], minion_tgt='proxytest') + ret = self.run_function('service.status', ['samba'], minion_tgt='proxytest') + self.assertTrue(ret) + +if __name__ == '__main__': + from integration import run_tests + run_tests(ProxyMinionSimpleTestCase, needs_daemon=True) diff --git a/tests/runtests.py b/tests/runtests.py index 92a2889037..79975e24b9 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -111,6 +111,9 @@ TEST_SUITES = { 'reactor': {'display_name': 'Reactor', 'path': 'integration/reactor'}, + 'proxy': + {'display_name': 'Proxy', + 'path': 'integration/proxy'}, } @@ -119,7 +122,8 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): support_destructive_tests_selection = True source_code_basedir = SALT_ROOT - def _get_suites(self, include_unit=False, include_cloud_provider=False): + def _get_suites(self, include_unit=False, include_cloud_provider=False, + include_proxy=False): ''' Return a set of all test suites except unit and cloud provider tests unless requested @@ -129,24 +133,30 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): suites -= set(['unit']) if not include_cloud_provider: suites -= set(['cloud_provider']) + if not include_proxy: + suites -= set(['proxy']) return suites - def _check_enabled_suites(self, include_unit=False, include_cloud_provider=False): + def _check_enabled_suites(self, include_unit=False, + include_cloud_provider=False, include_proxy=False): ''' Query whether test suites have been enabled ''' suites = self._get_suites(include_unit=include_unit, - include_cloud_provider=include_cloud_provider) + include_cloud_provider=include_cloud_provider, + include_proxy=include_proxy) return any([getattr(self.options, suite) for suite in suites]) - def _enable_suites(self, include_unit=False, include_cloud_provider=False): + def _enable_suites(self, include_unit=False, include_cloud_provider=False, + include_proxy=False): ''' Enable test suites for current test run ''' suites = self._get_suites(include_unit=include_unit, - include_cloud_provider=include_cloud_provider) + include_cloud_provider=include_cloud_provider, + include_proxy=include_proxy) for suite in suites: setattr(self.options, suite, True) @@ -347,6 +357,16 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): help='Run salt-api tests' ) + self.test_selection_group.add_option( + '-P', + '--proxy', + '--proxy-tests', + dest='proxy', + action='store_true', + default=False, + help='Run salt-proxy tests' + ) + def validate_options(self): if self.options.cloud_provider: # Turn on expensive tests execution @@ -367,7 +387,9 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): # When no tests are specifically enumerated on the command line, setup # a default run: +unit -cloud_provider if not self.options.name and not \ - self._check_enabled_suites(include_unit=True, include_cloud_provider=True): + self._check_enabled_suites(include_unit=True, + include_cloud_provider=True, + include_proxy=True): self._enable_suites(include_unit=True) self.start_coverage( @@ -405,6 +427,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Salt daemons started') master_conf = TestDaemon.config('master') minion_conf = TestDaemon.config('minion') + proxy_conf = TestDaemon.config('proxy') sub_minion_conf = TestDaemon.config('sub_minion') syndic_conf = TestDaemon.config('syndic') syndic_master_conf = TestDaemon.config('syndic_master') @@ -445,6 +468,15 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) print('\n') + print_header(' * Proxy Minion configuration values', top=True) + print('interface: {0}'.format(proxy_conf['interface'])) + print('master: {0}'.format(proxy_conf['master'])) + print('master port: {0}'.format(proxy_conf['master_port'])) + if minion_conf['ipc_mode'] == 'tcp': + print('tcp pub port: {0}'.format(proxy_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(proxy_conf['tcp_pull_port'])) + print('\n') + print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location())) @@ -534,7 +566,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): status = [] # Return an empty status if no tests have been enabled - if not self._check_enabled_suites(include_cloud_provider=True) and not self.options.name: + if not self._check_enabled_suites(include_cloud_provider=True, include_proxy=True) and not self.options.name: return status with TestDaemon(self): From 106394c80cfcb88cd13645ab8b1dda3f3a397049 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Sat, 6 May 2017 20:51:33 -0600 Subject: [PATCH 02/25] Lint. --- salt/proxy/dummy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/proxy/dummy.py b/salt/proxy/dummy.py index d3404681e6..a1bb6cf190 100644 --- a/salt/proxy/dummy.py +++ b/salt/proxy/dummy.py @@ -35,14 +35,12 @@ def __virtual__(): def _save_state(details): - global FILENAME pck = open(FILENAME, 'wb') pickle.dump(details, pck) pck.close() def _load_state(): - global FILENAME try: pck = open(FILENAME, 'r') DETAILS = pickle.load(pck) From 7749ceadb6324b2ddae7662c2d6060912e733819 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Thu, 11 May 2017 15:09:27 -0600 Subject: [PATCH 03/25] Change default proxy minion opts so only the proxy-specific ones are listed, and the rest are taken from DEFAULT_MINION_OPTS. --- salt/config/__init__.py | 242 +--------------------------------------- salt/utils/parsers.py | 2 +- 2 files changed, 6 insertions(+), 238 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index aa86e97be1..d5b12f2143 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1482,6 +1482,7 @@ DEFAULT_MASTER_OPTS = { # ----- Salt Proxy Minion Configuration Defaults -----------------------------------> +# These are merged with DEFAULT_MINION_OPTS since many of them also apply here. DEFAULT_PROXY_MINION_OPTS = { 'conf_file': os.path.join(salt.syspaths.CONFIG_DIR, 'proxy'), 'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'proxy'), @@ -1489,241 +1490,9 @@ DEFAULT_PROXY_MINION_OPTS = { 'proxy_merge_grains_in_module': False, 'append_minionid_config_dirs': ['cachedir', 'pidfile'], 'default_include': 'proxy.d/*.conf', - 'interface': '0.0.0.0', - 'master': 'salt', - 'master_type': 'str', - 'master_uri_format': 'default', - 'master_port': 4506, - 'master_finger': '', - 'master_shuffle': False, - 'master_alive_interval': 0, - 'master_failback': False, - 'master_failback_interval': 0, - 'verify_master_pubkey_sign': False, - 'sign_pub_messages': False, - 'always_verify_signature': False, - 'master_sign_key_name': 'master_sign', - 'syndic_finger': '', - 'user': salt.utils.get_user(), - 'root_dir': salt.syspaths.ROOT_DIR, 'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'proxy'), - 'id': '', 'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'proxy'), - 'cache_jobs': False, - 'grains_cache': False, - 'grains_cache_expiration': 300, - 'grains_deep_merge': False, 'sock_dir': os.path.join(salt.syspaths.SOCK_DIR, 'proxy'), - 'backup_mode': '', - 'renderer': 'yaml_jinja', - 'renderer_whitelist': [], - 'renderer_blacklist': [], - 'random_startup_delay': 0, - 'failhard': False, - 'autoload_dynamic_modules': True, - 'environment': None, - 'pillarenv': None, - 'pillar_opts': False, - # ``pillar_cache``, ``pillar_cache_ttl`` and ``pillar_cache_backend`` - # are not used on the minion but are unavoidably in the code path - 'pillar_cache': False, - 'pillar_cache_ttl': 3600, - 'pillar_cache_backend': 'disk', - 'extension_modules': os.path.join(salt.syspaths.CACHE_DIR, 'proxy', 'extmods'), - 'state_top': 'top.sls', - 'state_top_saltenv': None, - 'startup_states': '', - 'sls_list': [], - 'top_file': '', - 'thorium_interval': 0.5, - 'thorium_roots': { - 'base': [salt.syspaths.BASE_THORIUM_ROOTS_DIR], - }, - 'file_client': 'remote', - 'use_master_when_local': False, - 'file_roots': { - 'base': [salt.syspaths.BASE_FILE_ROOTS_DIR, - salt.syspaths.SPM_FORMULA_PATH] - }, - 'top_file_merging_strategy': 'merge', - 'env_order': [], - 'default_top': 'base', - 'fileserver_limit_traversal': False, - 'file_recv': False, - 'file_recv_max_size': 100, - 'file_ignore_regex': [], - 'file_ignore_glob': [], - 'fileserver_backend': ['roots'], - 'fileserver_followsymlinks': True, - 'fileserver_ignoresymlinks': False, - 'pillar_roots': { - 'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR, - salt.syspaths.SPM_PILLAR_PATH] - }, - 'on_demand_ext_pillar': ['libvirt', 'virtkey'], - 'git_pillar_base': 'master', - 'git_pillar_branch': 'master', - 'git_pillar_env': '', - 'git_pillar_root': '', - 'git_pillar_ssl_verify': True, - 'git_pillar_global_lock': True, - 'git_pillar_user': '', - 'git_pillar_password': '', - 'git_pillar_insecure_auth': False, - 'git_pillar_privkey': '', - 'git_pillar_pubkey': '', - 'git_pillar_passphrase': '', - 'gitfs_remotes': [], - 'gitfs_mountpoint': '', - 'gitfs_root': '', - 'gitfs_base': 'master', - 'gitfs_user': '', - 'gitfs_password': '', - 'gitfs_insecure_auth': False, - 'gitfs_privkey': '', - 'gitfs_pubkey': '', - 'gitfs_passphrase': '', - 'gitfs_env_whitelist': [], - 'gitfs_env_blacklist': [], - 'gitfs_global_lock': True, - 'gitfs_ssl_verify': True, - 'gitfs_saltenv': [], - 'hash_type': 'sha256', - 'disable_modules': [], - 'disable_returners': [], - 'whitelist_modules': [], - 'module_dirs': [], - 'returner_dirs': [], - 'grains_dirs': [], - 'states_dirs': [], - 'render_dirs': [], - 'outputter_dirs': [], - 'utils_dirs': [], - 'providers': {}, - 'clean_dynamic_modules': True, - 'open_mode': False, - 'auto_accept': True, - 'autosign_timeout': 120, - 'multiprocessing': True, - 'mine_enabled': True, - 'mine_return_job': False, - 'mine_interval': 60, - 'ipc_mode': _DFLT_IPC_MODE, - 'ipc_write_buffer': _DFLT_IPC_WBUFFER, - 'ipv6': None, - 'file_buffer_size': 262144, - 'tcp_pub_port': 4510, - 'tcp_pull_port': 4511, - 'tcp_authentication_retries': 5, - 'log_level': 'warning', - 'log_level_logfile': None, - 'log_datefmt': _DFLT_LOG_DATEFMT, - 'log_datefmt_logfile': _DFLT_LOG_DATEFMT_LOGFILE, - 'log_fmt_console': _DFLT_LOG_FMT_CONSOLE, - 'log_fmt_logfile': _DFLT_LOG_FMT_LOGFILE, - 'log_granular_levels': {}, - 'max_event_size': 1048576, - 'test': False, - 'ext_job_cache': '', - 'cython_enable': False, - 'enable_zip_modules': False, - 'state_verbose': True, - 'state_output': 'full', - 'state_output_diff': False, - 'state_auto_order': True, - 'state_events': False, - 'state_aggregate': False, - 'snapper_states': False, - 'snapper_states_config': 'root', - 'acceptance_wait_time': 10, - 'acceptance_wait_time_max': 0, - 'rejected_retry': False, - 'loop_interval': 1, - 'verify_env': True, - 'grains': {}, - 'permissive_pki_access': False, - 'update_url': False, - 'update_restart_services': [], - 'retry_dns': 30, - 'recon_max': 10000, - 'recon_default': 1000, - 'recon_randomize': True, - 'return_retry_timer': 5, - 'return_retry_timer_max': 10, - 'random_reauth_delay': 10, - 'winrepo_source_dir': 'salt://win/repo-ng/', - 'winrepo_dir': os.path.join(salt.syspaths.BASE_FILE_ROOTS_DIR, 'win', 'repo'), - 'winrepo_dir_ng': os.path.join(salt.syspaths.BASE_FILE_ROOTS_DIR, 'win', 'repo-ng'), - 'winrepo_cachefile': 'winrepo.p', - 'winrepo_cache_expire_max': 21600, - 'winrepo_cache_expire_min': 0, - 'winrepo_remotes': ['https://github.com/saltstack/salt-winrepo.git'], - 'winrepo_remotes_ng': ['https://github.com/saltstack/salt-winrepo-ng.git'], - 'winrepo_branch': 'master', - 'winrepo_ssl_verify': True, - 'winrepo_user': '', - 'winrepo_password': '', - 'winrepo_insecure_auth': False, - 'winrepo_privkey': '', - 'winrepo_pubkey': '', - 'winrepo_passphrase': '', - 'pidfile': os.path.join(salt.syspaths.PIDFILE_DIR, 'salt-proxy.pid'), - 'range_server': 'range:80', - 'reactor_refresh_interval': 60, - 'reactor_worker_threads': 10, - 'reactor_worker_hwm': 10000, - 'engines': [], - 'tcp_keepalive': True, - 'tcp_keepalive_idle': 300, - 'tcp_keepalive_cnt': -1, - 'tcp_keepalive_intvl': -1, - 'modules_max_memory': -1, - 'grains_refresh_every': 0, - 'minion_id_caching': True, - 'keysize': 2048, - 'transport': 'zeromq', - 'auth_timeout': 5, - 'auth_tries': 7, - 'master_tries': _MASTER_TRIES, - 'auth_safemode': False, - 'random_master': False, - 'minion_floscript': os.path.join(FLO_DIR, 'minion.flo'), - 'caller_floscript': os.path.join(FLO_DIR, 'caller.flo'), - 'ioflo_verbose': 0, - 'ioflo_period': 0.1, - 'ioflo_realtime': True, - 'ioflo_console_logdir': '', - 'raet_port': 4510, - 'raet_alt_port': 4511, - 'raet_mutable': False, - 'raet_main': False, - 'raet_clear_remotes': True, - 'raet_clear_remote_masters': True, - 'raet_road_bufcnt': 2, - 'raet_lane_bufcnt': 100, - 'cluster_mode': False, - 'cluster_masters': [], - 'restart_on_error': False, - 'ping_interval': 0, - 'username': None, - 'password': None, - 'zmq_filtering': False, - 'zmq_monitor': False, - 'cache_sreqs': True, - 'cmd_safe': True, - 'sudo_user': '', - 'http_request_timeout': 1 * 60 * 60.0, # 1 hour - 'http_max_body': 100 * 1024 * 1024 * 1024, # 100GB - 'event_match_type': 'startswith', - 'minion_restart_command': [], - 'pub_ret': True, - 'proxy_host': '', - 'proxy_username': '', - 'proxy_password': '', - 'proxy_port': 0, - 'minion_jid_queue_hwm': 100, - 'ssl': None, - 'cache': 'localfs', } # ----- Salt Cloud Configuration Defaults -----------------------------------> DEFAULT_CLOUD_OPTS = { @@ -2196,9 +1965,6 @@ def minion_config(path, if defaults is None: defaults = DEFAULT_MINION_OPTS.copy() - if path is not None and path.endswith('proxy'): - defaults.update(DEFAULT_PROXY_MINION_OPTS) - if not os.environ.get(env_var, None): # No valid setting was given using the configuration variable. # Lets see is SALT_CONFIG_DIR is of any use @@ -2246,8 +2012,10 @@ def proxy_config(path, import salt.config proxy_opts = salt.config.proxy_config('/etc/salt/proxy') ''' - if path is not None and path.endswith('proxy'): - defaults = DEFAULT_PROXY_MINION_OPTS.copy() + if defaults is None: + defaults = DEFAULT_MINION_OPTS.copy() + + defaults.update(DEFAULT_PROXY_MINION_OPTS) if not os.environ.get(env_var, None): # No valid setting was given using the configuration variable. diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 025326e43a..6747ee0663 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -1766,7 +1766,7 @@ class ProxyMinionOptionParser(six.with_metaclass(OptionParserMeta, except AttributeError: minion_id = None - return config.minion_config(self.get_config_file_path(), + return config.proxy_config(self.get_config_file_path(), cache_minion_id=False, minion_id=minion_id) From 19db038b993261bb2c0ae35683d9256d00631482 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Fri, 12 May 2017 14:01:54 -0600 Subject: [PATCH 04/25] Fix test--use proxy_config instead of minion_config --- tests/unit/utils/parsers_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils/parsers_test.py b/tests/unit/utils/parsers_test.py index 3968898c88..f6cdb2c9c0 100644 --- a/tests/unit/utils/parsers_test.py +++ b/tests/unit/utils/parsers_test.py @@ -547,7 +547,7 @@ class ProxyMinionOptionParserTestCase(LogSettingsParserTests): # Log file self.log_file = '/tmp/salt_proxy_minion_parser_test' # Function to patch - self.config_func = 'salt.config.minion_config' + self.config_func = 'salt.config.proxy_config' # Mock log setup self.setup_log() From bed209c6045abe567522cc92ef3ff445ba73090e Mon Sep 17 00:00:00 2001 From: jmarinaro Date: Fri, 19 May 2017 21:08:12 -0600 Subject: [PATCH 05/25] Fix package name collisions in chocolatey state Fixes #41185 --- salt/states/chocolatey.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/chocolatey.py b/salt/states/chocolatey.py index 210c7294c9..eb7e9bc292 100644 --- a/salt/states/chocolatey.py +++ b/salt/states/chocolatey.py @@ -92,7 +92,7 @@ def installed(name, version=None, source=None, force=False, pre_versions=False, # Determine action # Package not installed - if name not in pre_install: + if name not in [package.split('|')[0].lower() for package in pre_install.splitlines()]: if version: ret['changes'] = {name: 'Version {0} will be installed' ''.format(version)} @@ -193,7 +193,7 @@ def uninstalled(name, version=None, uninstall_args=None, override_args=False): pre_uninstall = __salt__['chocolatey.list'](local_only=True) # Determine if package is installed - if name in pre_uninstall: + if name in [package.split('|')[0].lower() for package in pre_uninstall.splitlines()]: ret['changes'] = {name: '{0} version {1} will be removed' ''.format(name, pre_uninstall[name][0])} else: From 6db31ce52a2cb6e7a1f506e61d53bd94f2b78502 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Sat, 20 May 2017 19:36:07 -0600 Subject: [PATCH 06/25] Fix problem with sysrc on FreeBSD, YAML overeager to coerce to bool and int. Fix problem with sysrc on FreeBSD, YAML overeager to coerce to bool --- salt/modules/sysrc.py | 17 ++++++++-- tests/integration/modules/sysrc.py | 52 ++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/integration/modules/sysrc.py diff --git a/salt/modules/sysrc.py b/salt/modules/sysrc.py index 6f5ac8301b..c2c989b366 100644 --- a/salt/modules/sysrc.py +++ b/salt/modules/sysrc.py @@ -93,6 +93,20 @@ def set_(name, value, **kwargs): if 'jail' in kwargs: cmd += ' -j '+kwargs['jail'] + # This is here because the YAML parser likes to convert the string literals + # YES, NO, Yes, No, True, False, etc. to boolean types. However, in this case, + # we will check to see if that happened and replace it with "YES" or "NO" because + # those items are accepted in sysrc. + if type(value) == bool: + if value: + value = "YES" + else: + value = "NO" + + # This is here for the same reason, except for numbers + if type(value) == int: + value = str(value) + cmd += ' '+name+"=\""+value+"\"" sysrcs = __salt__['cmd.run'](cmd) @@ -105,9 +119,6 @@ def set_(name, value, **kwargs): newval = sysrc.split(': ')[2].split(" -> ")[1] if rcfile not in ret: ret[rcfile] = {} - #ret[rcfile][var] = {} - #ret[rcfile][var]['old'] = oldval - #ret[rcfile][var]['new'] = newval ret[rcfile][var] = newval return ret diff --git a/tests/integration/modules/sysrc.py b/tests/integration/modules/sysrc.py new file mode 100644 index 0000000000..f8c8fa2231 --- /dev/null +++ b/tests/integration/modules/sysrc.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Import python libs +from __future__ import absolute_import +import sys + +# Import Salt Testing libs +from salttesting import skipIf +from salttesting.helpers import ensure_in_syspath, destructiveTest +ensure_in_syspath('../../') + +# Import salt libs +import integration + + +class SysrcModuleTest(integration.ModuleCase): + def setUp(self): + super(SysrcModuleTest, self).setUp() + ret = self.run_function('cmd.has_exec', ['sysrc']) + if not ret: + self.skipTest('sysrc not found') + + @skipIf(not sys.platform.startswith('freebsd'), 'FreeBSD specific') + def test_show(self): + ret = self.run_function('sysrc.get') + self.assertIsInstance(ret, dict, 'sysrc.get returned wrong type, expecting dictionary') + self.assertIn('/etc/rc.conf', ret, 'sysrc.get should have an rc.conf key in it.') + + @skipIf(not sys.platform.startswith('freebsd'), 'FreeBSD specific') + @destructiveTest + def test_set(self): + ret = self.run_function('sysrc.set', ['test_var', '1']) + self.assertIsInstance(ret, dict, 'sysrc.get returned wrong type, expecting dictionary') + self.assertIn('/etc/rc.conf', ret, 'sysrc.set should have an rc.conf key in it.') + self.assertIn('1', ret['/etc/rc.conf']['test_var'], 'sysrc.set should return the value it set.') + ret = self.run_function('sysrc.remove', ['test_var']) + self.assertEqual('test_var removed', ret) + + @skipIf(not sys.platform.startswith('freebsd'), 'FreeBSD specific') + @destructiveTest + def test_set_bool(self): + ret = self.run_function('sysrc.set', ['test_var', True]) + self.assertIsInstance(ret, dict, 'sysrc.get returned wrong type, expecting dictionary') + self.assertIn('/etc/rc.conf', ret, 'sysrc.set should have an rc.conf key in it.') + self.assertIn('YES', ret['/etc/rc.conf']['test_var'], 'sysrc.set should return the value it set.') + ret = self.run_function('sysrc.remove', ['test_var']) + self.assertEqual('test_var removed', ret) + + +if __name__ == '__main__': + from integration import run_tests + run_tests(SysrcModuleTest) From 3d4b8336278fc362dd57344cbee350f5512f38c2 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 21 May 2017 23:37:51 -0500 Subject: [PATCH 07/25] Don't use intermediate file when listing contents of tar.xz file This instead pipes the decompressed tar data directly to tarfile.open() --- salt/modules/archive.py | 45 ++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/salt/modules/archive.py b/salt/modules/archive.py index de7bc76fab..0232000b2c 100644 --- a/salt/modules/archive.py +++ b/salt/modules/archive.py @@ -12,6 +12,7 @@ import os import re import shlex import stat +import subprocess import tarfile import tempfile import zipfile @@ -157,7 +158,10 @@ def list_(name, files = [] links = [] try: - with contextlib.closing(tarfile.open(cached)) as tar_archive: + open_kwargs = {'name': cached} \ + if not isinstance(cached, subprocess.Popen) \ + else {'fileobj': cached.stdout, 'mode': 'r|*'} + with contextlib.closing(tarfile.open(**open_kwargs)) as tar_archive: for member in tar_archive.getmembers(): if member.issym(): links.append(member.name) @@ -168,7 +172,15 @@ def list_(name, return dirs, files, links except tarfile.ReadError: - if not failhard: + if failhard: + if fileobj: + stderr = cached.communicate()[1] + if cached.returncode != 0: + raise CommandExecutionError( + 'Failed to decompress {0}'.format(name), + info={'error': stderr} + ) + else: if not salt.utils.which('tar'): raise CommandExecutionError('\'tar\' command not available') if decompress_cmd is not None: @@ -187,29 +199,12 @@ def list_(name, decompress_cmd = 'xz --decompress --stdout' if decompress_cmd: - fd, decompressed = tempfile.mkstemp() - os.close(fd) - try: - cmd = '{0} {1} > {2}'.format(decompress_cmd, - _quote(cached), - _quote(decompressed)) - result = __salt__['cmd.run_all'](cmd, python_shell=True) - if result['retcode'] != 0: - raise CommandExecutionError( - 'Failed to decompress {0}'.format(name), - info={'error': result['stderr']} - ) - return _list_tar(name, decompressed, None, True) - finally: - try: - os.remove(decompressed) - except OSError as exc: - if exc.errno != errno.ENOENT: - log.warning( - 'Failed to remove intermediate ' - 'decompressed archive %s: %s', - decompressed, exc.__str__() - ) + decompressed = subprocess.Popen( + '{0} {1}'.format(decompress_cmd, _quote(cached)), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return _list_tar(name, decompressed, None, True) raise CommandExecutionError( 'Unable to list contents of {0}. If this is an XZ-compressed tar ' From 788874408ad9c670a9adb3ef3a04a57cff79cbab Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 21 May 2017 23:50:36 -0500 Subject: [PATCH 08/25] Remove '*' from mode This is unnecessary, as the data being piped in is uncompressed. --- salt/modules/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/archive.py b/salt/modules/archive.py index 0232000b2c..0f098b67a8 100644 --- a/salt/modules/archive.py +++ b/salt/modules/archive.py @@ -160,7 +160,7 @@ def list_(name, try: open_kwargs = {'name': cached} \ if not isinstance(cached, subprocess.Popen) \ - else {'fileobj': cached.stdout, 'mode': 'r|*'} + else {'fileobj': cached.stdout, 'mode': 'r|'} with contextlib.closing(tarfile.open(**open_kwargs)) as tar_archive: for member in tar_archive.getmembers(): if member.issym(): From 3192eab128e5b84cbe185b2001a5f2692efeb421 Mon Sep 17 00:00:00 2001 From: Alex Zel Date: Mon, 22 May 2017 09:09:37 +0300 Subject: [PATCH 09/25] Allow HTTP authentication to ES. Add option to authenticate to Elasticsearch instance with optional SSL. --- salt/modules/elasticsearch.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/salt/modules/elasticsearch.py b/salt/modules/elasticsearch.py index d2dd5a7368..ae6a87e751 100644 --- a/salt/modules/elasticsearch.py +++ b/salt/modules/elasticsearch.py @@ -84,13 +84,27 @@ def _get_instance(hosts=None, profile=None): hosts = _profile.get('host', None) if not hosts: hosts = _profile.get('hosts', None) + use_ssl = _profile.get('use_ssl', False) + ca_certs = _profile.get('ca_certs', False) + verify_certs = _profile.get('verify_certs', False) + username = _profile.get('username', None) + password = _profile.get('password', None) if not hosts: hosts = ['127.0.0.1:9200'] if isinstance(hosts, string_types): hosts = [hosts] try: - es = elasticsearch.Elasticsearch(hosts) + if username and password: + es = elasticsearch.Elasticsearch( + hosts, + use_ssl=use_ssl, + ca_certs=ca_certs, + verify_certs=verify_certs, + http_auth=(username, password) + ) + else: + es = elasticsearch.Elasticsearch(hosts) if not es.ping(): raise CommandExecutionError('Could not connect to Elasticsearch host/ cluster {0}, is it unhealthy?'.format(hosts)) except elasticsearch.exceptions.ConnectionError: From 68cb8975208658a8172ad7570ee65a3889b02cec Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 22 May 2017 02:13:56 -0500 Subject: [PATCH 10/25] Replace reference to fileobj This was an argument I had initially added to _list_tar() but later removed in favor of an isinstance check. Since this was removed, this argument no longer exists. --- salt/modules/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/archive.py b/salt/modules/archive.py index 0f098b67a8..e29818385f 100644 --- a/salt/modules/archive.py +++ b/salt/modules/archive.py @@ -173,7 +173,7 @@ def list_(name, except tarfile.ReadError: if failhard: - if fileobj: + if isinstance(cached, subprocess.Popen): stderr = cached.communicate()[1] if cached.returncode != 0: raise CommandExecutionError( From 1f08936d9c3f4ade04adf2bdcdece1d75522869e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 22 May 2017 02:18:28 -0500 Subject: [PATCH 11/25] Remove unused import --- salt/modules/archive.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/archive.py b/salt/modules/archive.py index e29818385f..68d8989a68 100644 --- a/salt/modules/archive.py +++ b/salt/modules/archive.py @@ -14,7 +14,6 @@ import shlex import stat import subprocess import tarfile -import tempfile import zipfile try: from shlex import quote as _quote # pylint: disable=E0611 From 9f314281298aed30914a82ebc2a4c4d8fd2002e5 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 22 May 2017 19:44:48 +0300 Subject: [PATCH 12/25] Update ACL module test to work as non-root. --- salt/utils/__init__.py | 10 +++++++++- tests/integration/modules/test_linux_acl.py | 8 +++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 666c0ecad8..ca94229fd1 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -2879,6 +2879,14 @@ def repack_dictlist(data, return ret +def get_default_group(user): + if HAS_GRP is False or HAS_PWD is False: + # We don't work on platforms that don't have grp and pwd + # Just return an empty list + return None + return grp.getgrgid(pwd.getpwnam(user).pw_gid).gr_name + + def get_group_list(user=None, include_default=True): ''' Returns a list of all of the system group names of which the user @@ -2917,7 +2925,7 @@ def get_group_list(user=None, include_default=True): log.trace('Trying generic group list for \'{0}\''.format(user)) group_names = [g.gr_name for g in grp.getgrall() if user in g.gr_mem] try: - default_group = grp.getgrgid(pwd.getpwnam(user).pw_gid).gr_name + default_group = get_default_group(user) if default_group not in group_names: group_names.append(default_group) except KeyError: diff --git a/tests/integration/modules/test_linux_acl.py b/tests/integration/modules/test_linux_acl.py index 745ccd3853..d8908a93b0 100644 --- a/tests/integration/modules/test_linux_acl.py +++ b/tests/integration/modules/test_linux_acl.py @@ -59,11 +59,13 @@ class LinuxAclModuleTest(ModuleCase, AdaptedConfigurationTestCaseMixin): def test_getfacl_w_single_file_without_acl(self): ret = self.run_function('acl.getfacl', arg=[self.myfile]) + user = salt.utils.get_user() + group = salt.utils.get_default_group(user) self.maxDiff = None self.assertEqual( ret, {self.myfile: {'other': [{'': {'octal': 4, 'permissions': {'read': True, 'write': False, 'execute': False}}}], - 'user': [{'root': {'octal': 6, 'permissions': {'read': True, 'write': True, 'execute': False}}}], - 'group': [{'root': {'octal': 4, 'permissions': {'read': True, 'write': False, 'execute': False}}}], - 'comment': {'owner': 'root', 'group': 'root', 'file': self.myfile}}} + 'user': [{user: {'octal': 6, 'permissions': {'read': True, 'write': True, 'execute': False}}}], + 'group': [{group: {'octal': 4, 'permissions': {'read': True, 'write': False, 'execute': False}}}], + 'comment': {'owner': user, 'group': group, 'file': self.myfile}}} ) From d80fa95f1025aa261b8451ebea09e7534e61fff2 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 22 May 2017 19:47:54 +0300 Subject: [PATCH 13/25] Add venv version of test_pip_installed_removed to work as non-root. --- tests/integration/states/test_pip.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration/states/test_pip.py b/tests/integration/states/test_pip.py index f4cc261066..3325c76bc8 100644 --- a/tests/integration/states/test_pip.py +++ b/tests/integration/states/test_pip.py @@ -35,9 +35,24 @@ from salt.exceptions import CommandExecutionError import salt.ext.six as six +class VirtualEnv(object): + def __init__(self, test, venv_dir): + self.venv_dir = venv_dir + self.test = test + + def __enter__(self): + ret = self.test.run_function('virtualenv.create', [self.venv_dir]) + self.test.assertEqual(ret['retcode'], 0) + + def __exit__(self, exc_type, exc_value, traceback): + if os.path.isdir(self.venv_dir): + shutil.rmtree(self.venv_dir) + + @skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') class PipStateTest(ModuleCase, SaltReturnAssertsMixin): + @skip_if_not_root def test_pip_installed_removed(self): ''' Tests installed and removed states @@ -50,6 +65,17 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pip.removed', name=name) self.assertSaltTrueReturn(ret) + def test_pip_installed_removed_venv(self): + venv_dir = os.path.join( + RUNTIME_VARS.TMP, 'pip_installed_removed' + ) + with VirtualEnv(self, venv_dir): + name = 'pudb' + ret = self.run_state('pip.installed', name=name, bin_env=venv_dir) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pip.removed', name=name, bin_env=venv_dir) + self.assertSaltTrueReturn(ret) + def test_pip_installed_errors(self): venv_dir = os.path.join( RUNTIME_VARS.TMP, 'pip-installed-errors' From 583e3b9a5afedf77a1636a77760a8784d6609f7b Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 22 May 2017 19:52:32 +0300 Subject: [PATCH 14/25] Update pip version in pip upgrade test by compatibility reason. 6.0 version doesn't work in a modern environment. --- tests/integration/states/test_pip.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/states/test_pip.py b/tests/integration/states/test_pip.py index 3325c76bc8..d1809982e2 100644 --- a/tests/integration/states/test_pip.py +++ b/tests/integration/states/test_pip.py @@ -404,7 +404,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): # Let's install a fixed version pip over whatever pip was # previously installed ret = self.run_function( - 'pip.install', ['pip==6.0'], upgrade=True, + 'pip.install', ['pip==8.0'], upgrade=True, bin_env=venv_dir ) try: @@ -418,15 +418,15 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): pprint.pprint(ret) raise - # Let's make sure we have pip 6.0 installed + # Let's make sure we have pip 8.0 installed self.assertEqual( self.run_function('pip.list', ['pip'], bin_env=venv_dir), - {'pip': '6.0'} + {'pip': '8.0.0'} ) # Now the actual pip upgrade pip test ret = self.run_state( - 'pip.installed', name='pip==6.0.7', upgrade=True, + 'pip.installed', name='pip==8.0.1', upgrade=True, bin_env=venv_dir ) try: @@ -434,7 +434,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): self.assertInSaltReturn( 'Installed', ret, - ['changes', 'pip==6.0.7'] + ['changes', 'pip==8.0.1'] ) except AssertionError: import pprint From 4ed0f8ed4e06381e2924e653612129067a4ae68f Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 22 May 2017 19:53:59 +0300 Subject: [PATCH 15/25] Fixed grains module test: Arch release is 'rolling', not an empty str. --- tests/integration/modules/test_grains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/test_grains.py b/tests/integration/modules/test_grains.py index 28e34277c3..352f7730fc 100644 --- a/tests/integration/modules/test_grains.py +++ b/tests/integration/modules/test_grains.py @@ -114,7 +114,7 @@ class TestModulesGrains(ModuleCase): for grain in grains: get_grain = self.run_function('grains.get', [grain]) - if os == 'Arch' and grain in ['osmajorrelease', 'osrelease']: + if os == 'Arch' and grain in ['osmajorrelease']: self.assertEqual(get_grain, '') continue if os == 'Windows' and grain in ['osmajorrelease']: From 3cd0f5ebcff4c8c8206e9839190c046b22cc4121 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Tue, 23 May 2017 11:52:05 +0300 Subject: [PATCH 16/25] Check master shut down correctly. --- tests/integration/shell/test_master.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/shell/test_master.py b/tests/integration/shell/test_master.py index e3c9d5372b..e3433fc38e 100644 --- a/tests/integration/shell/test_master.py +++ b/tests/integration/shell/test_master.py @@ -157,3 +157,16 @@ class MasterTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix stderr=tests.integration.utils.decode_byte_list(stderr) ) master.shutdown() + # Do the test again to check does master shut down correctly + stdout, stderr, status = master.run( + args=['-d'], + catch_stderr=True, + with_retcode=True, + ) + self.assert_exit_status( + status, 'EX_OK', + message='correct usage', + stdout=stdout, + stderr=tests.integration.utils.decode_byte_list(stderr) + ) + master.shutdown() From b02ec97cffd29b2f79aa69091ef2f3bf9831943d Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 21 Apr 2017 16:05:18 -0400 Subject: [PATCH 17/25] add ability to expand disk size on creation of vm in nebula --- doc/topics/cloud/opennebula.rst | 73 +++++++++++++++++++++++++ salt/cloud/clouds/opennebula.py | 95 ++++++++++++++++++++++++++++++++- 2 files changed, 166 insertions(+), 2 deletions(-) diff --git a/doc/topics/cloud/opennebula.rst b/doc/topics/cloud/opennebula.rst index 76abd118c0..64caff0b19 100644 --- a/doc/topics/cloud/opennebula.rst +++ b/doc/topics/cloud/opennebula.rst @@ -103,6 +103,79 @@ and virtual machines are created from templates. Because of this, there is no ne profile. The size of the virtual machine is defined in the template. +You can now change the size of a VM on creation by cloning an image and expanding the size. You can accomplish this by +the following cloud profile settings below. + +.. code-block:: yaml + my-opennebula-profile: + provider: my-opennebula-provider + image: Ubuntu-14.04 + disk: + disk0: + disk_type: clone + size: 8096 + image: centos7-base-image-v2 + disk1: + disk_type: volatile + type: swap + size: 4096 + disk2: + disk_type: volatile + size: 4096 + type: fs + format: ext3 + +There are currently two different disk_types a user can use: volatile and clone. Clone which is required when specifying devices +will clone an image in open nebula and will expand it to the size specified in the profile settings. By default this will clone +the image attached to the template specified in the profile but a user can add the `image` argument under the disk definition. + +For example the profile below will not use Ubuntu-14.04 for the cloned disk image. It will use the centos7-base-image image: + +.. code-block:: yaml + my-opennebula-profile: + provider: my-opennebula-provider + image: Ubuntu-14.04 + disk: + disk0: + disk_type: clone + size: 8096 + image: centos7-base-image + +If you want to use the image attached to the template set in the profile you can simply remove the image argument as show below. +The profile below will clone the image Ubuntu-14.04 and expand the disk to 8GB.: + +.. code-block:: yaml + my-opennebula-profile: + provider: my-opennebula-provider + image: Ubuntu-14.04 + disk: + disk0: + disk_type: clone + size: 8096 + +A user can also currently specify swap or fs disks. Below is an example of this profile setting: + +.. code-block:: yaml + my-opennebula-profile: + provider: my-opennebula-provider + image: Ubuntu-14.04 + disk: + disk0: + disk_type: clone + size: 8096 + disk1: + disk_type: volatile + type: swap + size: 4096 + disk2: + disk_type: volatile + size: 4096 + type: fs + format: ext3 + +The example above will attach both a swap disk and a ext3 filesystem with a size of 4GB. To note if you define other disks you have +to define the image disk to clone because the template will write over the entire 'DISK=[]' template definition on creation. + .. _opennebula-required-settings: Required Settings diff --git a/salt/cloud/clouds/opennebula.py b/salt/cloud/clouds/opennebula.py index 75b04ee39f..6a9616aae1 100644 --- a/salt/cloud/clouds/opennebula.py +++ b/salt/cloud/clouds/opennebula.py @@ -752,6 +752,40 @@ def get_secgroup_id(kwargs=None, call=None): return ret +def get_template_image(kwargs=None, call=None): + ''' + Returns a template's image from the given template name. + + .. versionadded:: oxygen + + .. code-block:: bash + + salt-cloud -f get_template_image opennebula name=my-template-name + ''' + if call == 'action': + raise SaltCloudSystemExit( + 'The get_template_image function must be called with -f or --function.' + ) + + if kwargs is None: + kwargs = {} + + name = kwargs.get('name', None) + if name is None: + raise SaltCloudSystemExit( + 'The get_template_image function requires a \'name\'.' + ) + + try: + ret = list_templates()[name]['template']['disk']['image'] + except KeyError: + raise SaltCloudSystemExit( + 'The image for template \'{1}\' could not be found.'.format(name) + ) + + return ret + + def get_template_id(kwargs=None, call=None): ''' Returns a template's ID from the given template name. @@ -766,7 +800,7 @@ def get_template_id(kwargs=None, call=None): ''' if call == 'action': raise SaltCloudSystemExit( - 'The list_nodes_full function must be called with -f or --function.' + 'The get_template_id function must be called with -f or --function.' ) if kwargs is None: @@ -782,7 +816,7 @@ def get_template_id(kwargs=None, call=None): ret = list_templates()[name]['id'] except KeyError: raise SaltCloudSystemExit( - 'The template \'{0}\' could not be foound.'.format(name) + 'The template \'{0}\' could not be found.'.format(name) ) return ret @@ -881,6 +915,53 @@ def get_vn_id(kwargs=None, call=None): return ret +def _get_device_template(disk, disk_info, template=None): + ''' + Returns the template format to create a disk in open nebula + + .. versionadded:: oxygen + + ''' + def _require_disk_opts(*args): + for arg in args: + if arg not in disk_info: + raise SaltCloudSystemExit( + 'The disk {0} requires a {1}\ + argument'.format(disk, arg) + ) + + _require_disk_opts('disk_type', 'size') + + size = disk_info['size'] + disk_type = disk_info['disk_type'] + + if disk_type == 'clone': + if 'image' in disk_info: + clone_image = disk_info['image'] + else: + clone_image = get_template_image(kwargs={'name': + template}) + + clone_image_id = get_image_id(kwargs={'name': clone_image}) + temp = 'DISK=[IMAGE={0}, IMAGE_ID={1}, CLONE=YES,\ + SIZE={2}]'.format(clone_image, clone_image_id, + size) + return temp + + if disk_type == 'volatile': + _require_disk_opts('type') + v_type = disk_info['type'] + temp = 'DISK=[TYPE={0}, SIZE={1}]'.format(v_type, size) + + if v_type == 'fs': + _require_disk_opts('format') + format = disk_info['format'] + temp = 'DISK=[TYPE={0}, SIZE={1}, FORMAT={2}]'.format(v_type, + size, format) + return temp + #TODO add persistant disk_type + + def create(vm_): r''' Create a single VM from a data dict. @@ -962,7 +1043,17 @@ def create(vm_): template.append('CPU={0}'.format(vm_.get('cpu'))) if vm_.get('vcpu'): template.append('VCPU={0}'.format(vm_.get('vcpu'))) + if vm_.get('disk'): + get_disks = vm_.get('disk') + template_name = vm_['image'] + for disk in get_disks: + template.append(_get_device_template(disk, get_disks[disk], + template=template_name)) template_args = "\n".join(template) + if 'CLONE' not in template_args: + raise SaltCloudSystemExit( + 'Missing an image disk to clone. Must define a clone disk alongside all other disk definitions.' + ) try: server, user, password = _get_xml_rpc() From 42e614bb7d8aac29dfb93d13e7cde5817eb26ecd Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 21 Apr 2017 16:08:52 -0400 Subject: [PATCH 18/25] add title --- doc/topics/cloud/opennebula.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/topics/cloud/opennebula.rst b/doc/topics/cloud/opennebula.rst index 64caff0b19..f3b2084d51 100644 --- a/doc/topics/cloud/opennebula.rst +++ b/doc/topics/cloud/opennebula.rst @@ -102,6 +102,8 @@ OpenNebula uses an image --> template --> virtual machine paradigm where the tem and virtual machines are created from templates. Because of this, there is no need to define a ``size`` in the cloud profile. The size of the virtual machine is defined in the template. +Change Disk Size +================ You can now change the size of a VM on creation by cloning an image and expanding the size. You can accomplish this by the following cloud profile settings below. From 0d3f5da0ec1cfac8c316d5d2e8f1652db2b2ed7f Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 21 Apr 2017 16:10:31 -0400 Subject: [PATCH 19/25] add spaces for code in docs --- doc/topics/cloud/opennebula.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/topics/cloud/opennebula.rst b/doc/topics/cloud/opennebula.rst index f3b2084d51..a453c475f2 100644 --- a/doc/topics/cloud/opennebula.rst +++ b/doc/topics/cloud/opennebula.rst @@ -109,6 +109,7 @@ You can now change the size of a VM on creation by cloning an image and expandin the following cloud profile settings below. .. code-block:: yaml + my-opennebula-profile: provider: my-opennebula-provider image: Ubuntu-14.04 @@ -134,6 +135,7 @@ the image attached to the template specified in the profile but a user can add t For example the profile below will not use Ubuntu-14.04 for the cloned disk image. It will use the centos7-base-image image: .. code-block:: yaml + my-opennebula-profile: provider: my-opennebula-provider image: Ubuntu-14.04 @@ -147,6 +149,7 @@ If you want to use the image attached to the template set in the profile you can The profile below will clone the image Ubuntu-14.04 and expand the disk to 8GB.: .. code-block:: yaml + my-opennebula-profile: provider: my-opennebula-provider image: Ubuntu-14.04 @@ -158,6 +161,7 @@ The profile below will clone the image Ubuntu-14.04 and expand the disk to 8GB.: A user can also currently specify swap or fs disks. Below is an example of this profile setting: .. code-block:: yaml + my-opennebula-profile: provider: my-opennebula-provider image: Ubuntu-14.04 From 858adcf3df27e0f016b5aa42bd7b453d56c3e1b4 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 21 Apr 2017 18:16:19 -0400 Subject: [PATCH 20/25] fix where clone check happens --- salt/cloud/clouds/opennebula.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/cloud/clouds/opennebula.py b/salt/cloud/clouds/opennebula.py index 6a9616aae1..2eef11ec5b 100644 --- a/salt/cloud/clouds/opennebula.py +++ b/salt/cloud/clouds/opennebula.py @@ -1049,11 +1049,12 @@ def create(vm_): for disk in get_disks: template.append(_get_device_template(disk, get_disks[disk], template=template_name)) + if 'CLONE' not in str(template): + raise SaltCloudSystemExit( + 'Missing an image disk to clone. Must define a clone disk alongside all other disk definitions.' + ) + template_args = "\n".join(template) - if 'CLONE' not in template_args: - raise SaltCloudSystemExit( - 'Missing an image disk to clone. Must define a clone disk alongside all other disk definitions.' - ) try: server, user, password = _get_xml_rpc() From 7d390aad6fad5b3cc6a08174dca3521950ddddc0 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 23 May 2017 16:42:51 -0600 Subject: [PATCH 21/25] Pylint fix for config/__init__.py proxy settings --- salt/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index f2369f9d9f..a537141966 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1618,7 +1618,7 @@ DEFAULT_PROXY_MINION_OPTS = { 'proxy_always_alive': True, 'proxy_keep_alive': True, # by default will try to keep alive the connection - 'proxy_keep_alive_interval': 1 # frequency of the proxy keepalive in minutes + 'proxy_keep_alive_interval': 1, # frequency of the proxy keepalive in minutes 'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'proxy'), 'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'proxy'), 'sock_dir': os.path.join(salt.syspaths.SOCK_DIR, 'proxy'), From 5b27c2daf8f01bb3ecdb0e9e2b49b1864e94115c Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 24 May 2017 09:51:00 -0600 Subject: [PATCH 22/25] Comment out/remove proxy pieces from integration tests file This was left over from a merge-forward. The proxy minion testing pieces could not be completely merged-forward due to large conflicts in this file. Please see the comment in PR #41113 for more information. --- tests/integration/__init__.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 108ab8b35c..6a5043c4e7 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -661,7 +661,6 @@ class TestDaemon(object): * syndic * syndic_master * sub_minion - * proxy ''' return RUNTIME_VARS.RUNTIME_CONFIGS[role] @@ -744,14 +743,14 @@ class TestDaemon(object): syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') # This proxy connects to master - proxy_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'proxy')) - proxy_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') - proxy_opts['user'] = running_tests_user - proxy_opts['config_dir'] = TMP_CONF_DIR - proxy_opts['root_dir'] = os.path.join(TMP, 'rootdir') - proxy_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki') - proxy_opts['hosts.file'] = os.path.join(TMP, 'rootdir', 'hosts') - proxy_opts['aliases.file'] = os.path.join(TMP, 'rootdir', 'aliases') + # proxy_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'proxy')) + # proxy_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') + # proxy_opts['user'] = running_tests_user + # proxy_opts['config_dir'] = TMP_CONF_DIR + # proxy_opts['root_dir'] = os.path.join(TMP, 'rootdir') + # proxy_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki') + # proxy_opts['hosts.file'] = os.path.join(TMP, 'rootdir', 'hosts') + # proxy_opts['aliases.file'] = os.path.join(TMP, 'rootdir', 'aliases') if transport == 'raet': master_opts['transport'] = 'raet' @@ -864,7 +863,7 @@ class TestDaemon(object): os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry) ) - for entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master', 'proxy'): + for entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master'): computed_config = copy.deepcopy(locals()['{0}_opts'.format(entry)]) with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry), 'w') as fp_: fp_.write(yaml.dump(computed_config, default_flow_style=False)) @@ -949,7 +948,7 @@ class TestDaemon(object): cls.master_opts = master_opts cls.minion_opts = minion_opts - cls.proxy_opts = proxy_opts + # cls.proxy_opts = proxy_opts cls.sub_minion_opts = sub_minion_opts cls.syndic_opts = syndic_opts cls.syndic_master_opts = syndic_master_opts @@ -961,8 +960,8 @@ class TestDaemon(object): ''' self.sub_minion_process.terminate() self.minion_process.terminate() - if hasattr(self, 'proxy_process'): - self.proxy_process.terminate() + # if hasattr(self, 'proxy_process'): + # self.proxy_process.terminate() self.master_process.terminate() try: self.syndic_process.terminate() From 56ad5629f6c38f010bc6b154db8793e788c3a9da Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 24 May 2017 09:55:35 -0600 Subject: [PATCH 23/25] Disable pylint warnings for uses of "open" in proxy minion dummy file There is an associated "close" for both of the open instances. --- salt/proxy/dummy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/proxy/dummy.py b/salt/proxy/dummy.py index a1bb6cf190..eeb0fb0b3e 100644 --- a/salt/proxy/dummy.py +++ b/salt/proxy/dummy.py @@ -35,14 +35,14 @@ def __virtual__(): def _save_state(details): - pck = open(FILENAME, 'wb') + pck = open(FILENAME, 'wb') # pylint: disable=W8470 pickle.dump(details, pck) pck.close() def _load_state(): try: - pck = open(FILENAME, 'r') + pck = open(FILENAME, 'r') # pylint: disable=W8470 DETAILS = pickle.load(pck) pck.close() except IOError: From 098ed60d1b75325c2c0393ce6cba7a43d10bb011 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 24 May 2017 10:23:57 -0600 Subject: [PATCH 24/25] Rework proxy minion test files to use new import styles --- tests/integration/modules/sysrc.py | 16 ++++------------ tests/integration/proxy/simple.py | 13 ++----------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/tests/integration/modules/sysrc.py b/tests/integration/modules/sysrc.py index f8c8fa2231..50899148fa 100644 --- a/tests/integration/modules/sysrc.py +++ b/tests/integration/modules/sysrc.py @@ -5,15 +5,12 @@ from __future__ import absolute_import import sys # Import Salt Testing libs -from salttesting import skipIf -from salttesting.helpers import ensure_in_syspath, destructiveTest -ensure_in_syspath('../../') - -# Import salt libs -import integration +from tests.support.case import ModuleCase +from tests.support.helpers import destructiveTest +from tests.support.unit import skipIf -class SysrcModuleTest(integration.ModuleCase): +class SysrcModuleTest(ModuleCase): def setUp(self): super(SysrcModuleTest, self).setUp() ret = self.run_function('cmd.has_exec', ['sysrc']) @@ -45,8 +42,3 @@ class SysrcModuleTest(integration.ModuleCase): self.assertIn('YES', ret['/etc/rc.conf']['test_var'], 'sysrc.set should return the value it set.') ret = self.run_function('sysrc.remove', ['test_var']) self.assertEqual('test_var removed', ret) - - -if __name__ == '__main__': - from integration import run_tests - run_tests(SysrcModuleTest) diff --git a/tests/integration/proxy/simple.py b/tests/integration/proxy/simple.py index fcfc7a6c0c..dbca73bafa 100644 --- a/tests/integration/proxy/simple.py +++ b/tests/integration/proxy/simple.py @@ -7,15 +7,10 @@ Simple Smoke Tests for Connected Proxy Minion from __future__ import absolute_import # Import Salt Testing libs -from salttesting.helpers import ensure_in_syspath - -ensure_in_syspath('../') - -# Import Salt libs -import integration +from tests.support.case import ModuleCase -class ProxyMinionSimpleTestCase(integration.ModuleCase): +class ProxyMinionSimpleTestCase(ModuleCase): ''' Test minion blackout functionality ''' @@ -72,7 +67,3 @@ class ProxyMinionSimpleTestCase(integration.ModuleCase): ret = self.run_function('service.start', ['samba'], minion_tgt='proxytest') ret = self.run_function('service.status', ['samba'], minion_tgt='proxytest') self.assertTrue(ret) - -if __name__ == '__main__': - from integration import run_tests - run_tests(ProxyMinionSimpleTestCase, needs_daemon=True) From 537fd763dc06e39dd636ce7f4417c554b7e8f13c Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 24 May 2017 13:03:21 -0600 Subject: [PATCH 25/25] Rename proxy test files to use test_* syntax --- tests/integration/modules/{sysrc.py => test_sysrc.py} | 0 tests/integration/proxy/{simple.py => test_simple.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/integration/modules/{sysrc.py => test_sysrc.py} (100%) rename tests/integration/proxy/{simple.py => test_simple.py} (100%) diff --git a/tests/integration/modules/sysrc.py b/tests/integration/modules/test_sysrc.py similarity index 100% rename from tests/integration/modules/sysrc.py rename to tests/integration/modules/test_sysrc.py diff --git a/tests/integration/proxy/simple.py b/tests/integration/proxy/test_simple.py similarity index 100% rename from tests/integration/proxy/simple.py rename to tests/integration/proxy/test_simple.py