From 29be4bbe117e3834b644e10e545c0c0c78ef1a41 Mon Sep 17 00:00:00 2001 From: Akhter Ali Date: Wed, 22 Jul 2015 15:56:05 -0400 Subject: [PATCH 01/26] Update loader.py --- salt/loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index 130a4a8880..8ca0074083 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -267,7 +267,7 @@ def states(opts, functions, whitelist=None): import salt.config import salt.loader - __opts__ salt.config.minion_config('/etc/salt/minion') + __opts__ = salt.config.minion_config('/etc/salt/minion') statemods = salt.loader.states(__opts__, None) ''' load = _create_loader(opts, 'states', 'states') @@ -356,7 +356,7 @@ def grains(opts, force_refresh=False): import salt.config import salt.loader - __opts__ salt.config.minion_config('/etc/salt/minion') + __opts__ = salt.config.minion_config('/etc/salt/minion') __grains__ = salt.loader.grains(__opts__) print __grains__['id'] ''' From f8b2f8079fc7c2c479c47e4864731f8ca2eb56a6 Mon Sep 17 00:00:00 2001 From: Metin OSMAN Date: Thu, 23 Jul 2015 13:13:01 +0200 Subject: [PATCH 02/26] Add the ability to specify a base pattern for metrics path used by the carbon returner --- doc/man/salt.7 | 39 +++++++++++++++++++++++++++++++++ salt/returners/carbon_return.py | 23 ++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/doc/man/salt.7 b/doc/man/salt.7 index 61a4703835..29a3fc4440 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -108465,6 +108465,44 @@ carbon.mode: pickle .UNINDENT .UNINDENT .sp +You can also specify the pattern used for the metric base path (except for virt modules metrics): +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +carbon.metric_base_pattern: carbon.[minion_id].[module].[function] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +These tokens can used : +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[module]: salt module +[function]: salt function +[minion_id]: minion id +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Default is : +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +carbon.metric_base_pattern: [module].[function].[minion_id] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp Carbon settings may also be configured as: .INDENT 0.0 .INDENT 3.5 @@ -108476,6 +108514,7 @@ Carbon settings may also be configured as: port: skip_on_error: True mode: (pickle|text) + metric_base_pattern: | [module].[function].[minion_id] To use the carbon returner, append \(aq\-\-return carbon\(aq to the salt command. ex: diff --git a/salt/returners/carbon_return.py b/salt/returners/carbon_return.py index 365807b0e2..875be2125b 100644 --- a/salt/returners/carbon_return.py +++ b/salt/returners/carbon_return.py @@ -17,6 +17,18 @@ the pickle protocol, set ``carbon.mode`` to ``pickle``:: carbon.mode: pickle +You can also specify the pattern used for the metric base path (except for virt modules metrics): + carbon.metric_base_pattern: carbon.[minion_id].[module].[function] + +These tokens can used : + [module]: salt module + [function]: salt function + [minion_id]: minion id + +Default is : + carbon.metric_base_pattern: [module].[function].[minion_id] + + Carbon settings may also be configured as:: carbon: @@ -24,6 +36,7 @@ Carbon settings may also be configured as:: port: skip_on_error: True mode: (pickle|text) + metric_base_pattern: | [module].[function].[minion_id] To use the carbon returner, append '--return carbon' to the salt command. ex: @@ -163,12 +176,14 @@ def returner(ret): port = c_cfg.get('port', cfg('carbon.port', None)) skip = c_cfg.get('skip_on_error', cfg('carbon.skip_on_error', False)) mode = c_cfg.get('mode', cfg('carbon.mode', 'text')).lower() + metric_base_pattern = c_cfg.get('metric_base_pattern', cfg('carbon.metric_base_pattern', None)) else: cfg = __opts__ host = cfg.get('cabon.host', None) port = cfg.get('cabon.port', None) skip = cfg.get('carbon.skip_on_error', False) mode = cfg.get('carbon.mode', 'text').lower() + metric_base_pattern = cfg('carbon.metric_base_pattern', None) log.debug('Carbon minion configured with host: {0}:{1}'.format(host, port)) log.debug('Using carbon protocol: {0}'.format(mode)) @@ -190,7 +205,13 @@ def returner(ret): # module since then we will get stable metric bases even if the VM is # migrate from host to host if not metric_base.startswith('virt.'): - metric_base += '.' + ret['id'].replace('.', '_') + minion_id = ret['id'].replace('.', '_') + if metric_base_pattern is not None: + [module, function] = ret['fun'].split('.') + metric_base = metric_base_pattern.replace("[module]", module).replace("[function]", function).replace("[minion_id]", minion_id) + else: + metric_base += '.' + minion_id + log.debug('Carbon metric_base : %s', metric_base) metrics = [] _walk(metric_base, saltdata, metrics, timestamp, skip) From c95886c9a727e2ae5c0e2cc15f9186abfc00d5a0 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 24 Jul 2015 10:57:01 -0600 Subject: [PATCH 03/26] Ensure prior alignment with master_type in 2014.7 --- salt/config.py | 2 +- salt/minion.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/config.py b/salt/config.py index 63912c02f8..4160c56967 100644 --- a/salt/config.py +++ b/salt/config.py @@ -271,7 +271,7 @@ VALID_OPTS = { DEFAULT_MINION_OPTS = { 'interface': '0.0.0.0', 'master': 'salt', - 'master_type': 'standard', + 'master_type': 'str', 'master_port': '4506', 'master_finger': '', 'master_shuffle': False, diff --git a/salt/minion.py b/salt/minion.py index 996e210344..95b37cd979 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -686,7 +686,7 @@ class Minion(MinionBase): (possibly failed) master will then be removed from the list of masters. ''' # check if master_type was altered from its default - if opts['master_type'] != 'standard': + if opts['master_type'] != 'str': # check for a valid keyword if opts['master_type'] == 'func': # split module and function and try loading the module From 9ec3ae96d4041222bba3c71688b24acc53707955 Mon Sep 17 00:00:00 2001 From: Peter Tripp Date: Fri, 26 Jun 2015 16:21:45 -0700 Subject: [PATCH 04/26] Add file as supported protocol for file source_hash. Fixes #23764. Conflicts: salt/modules/file.py --- salt/modules/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index feb5f63864..b838700d59 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -2556,7 +2556,7 @@ def get_managed( if not source_sum: return '', {}, 'Source file {0} not found'.format(source) elif source_hash: - protos = ['salt', 'http', 'https', 'ftp', 'swift', 's3'] + protos = ['salt', 'http', 'https', 'ftp', 'swift', 's3', 'file'] if salt._compat.urlparse(source_hash).scheme in protos: # The source_hash is a file on a server hash_fn = __salt__['cp.cache_file'](source_hash, saltenv) From 3aa5045138f3b42def6f733894ea1d918609c080 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 28 Jul 2015 17:15:31 -0600 Subject: [PATCH 05/26] Clean up stacktrace when referenced map file doesn't exist --- salt/cloud/__init__.py | 8 ++++---- salt/cloud/cli.py | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 8f698a4072..8c3dd56d99 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -1648,11 +1648,11 @@ class Map(Cloud): return {} if not os.path.isfile(self.opts['map']): - raise SaltCloudNotFound( - 'The specified map file does not exist: {0}\n'.format( - self.opts['map'] - ) + log.error( + 'The specified map file does not exist: \'{0}\''.format( + self.opts['map']) ) + raise SaltCloudNotFound() try: renderer = self.opts.get('renderer', 'yaml_jinja') rend = salt.loader.render(self.opts, {}) diff --git a/salt/cloud/cli.py b/salt/cloud/cli.py index 544f947a7d..85896f76cd 100644 --- a/salt/cloud/cli.py +++ b/salt/cloud/cli.py @@ -83,7 +83,11 @@ class SaltCloud(parsers.SaltCloudParser): self.exit(salt.defaults.exitcodes.EX_OK) log.info('salt-cloud starting') - mapper = salt.cloud.Map(self.config) + try: + mapper = salt.cloud.Map(self.config) + except SaltCloudException as exc: + msg = 'There was an error generating the mapper.' + self.handle_exception(msg, exc) names = self.config.get('names', None) if names is not None: From ad7fdda68bb4720392b808ded250e7abbdb1c38b Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 29 Jul 2015 16:23:08 -0600 Subject: [PATCH 06/26] Adder allusers="1" when installing msi --- salt/modules/win_pkg.py | 42 ++++- salt/states/pkg.py | 392 +++++++++++++++++++++------------------- 2 files changed, 240 insertions(+), 194 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index ebfa49f719..ed9b43a0a0 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -44,6 +44,16 @@ def __virtual__(): return False +def normalize_name(name): + ''' + This function needed for the pkg state module + :param str name: + :return: name + :rtype str: + ''' + return name + + def latest_version(*names, **kwargs): ''' Return the latest version of the named package available for upgrade or @@ -386,12 +396,28 @@ def refresh_db(saltenv='base'): def install(name=None, refresh=False, pkgs=None, saltenv='base', **kwargs): ''' - Install the passed package + Install the passed package from the winrepo - Return a dict containing the new package names and versions:: + :param name: The name of the package to install + :type name: str or None - {'': {'old': '', - 'new': ''}} + :param bool refresh: Boolean value representing whether or not to refresh + the winrepo db + + :param pkgs: A list of packages to install from a software repository. + All packages listed under ``pkgs`` will be installed via a single + command. + :type pkgs: list or None + + :param str saltenv: The salt environment to use. Default is ``base``. + + :param dict kwargs: Any additional argument that may be passed from the + state module. If they don't apply, they are ignored. + + :return: Return a dict containing the new package names and versions:: + + {'': {'old': '', + 'new': ''}} CLI Example: @@ -450,6 +476,9 @@ def install(name=None, refresh=False, pkgs=None, saltenv='base', **kwargs): if not cached_pkg: # It's not cached. Cache it, mate. cached_pkg = __salt__['cp.cache_file'](installer, saltenv) + if not cached_pkg: + return 'Unable to cache file {0} from saltenv: {1}'\ + .format(installer, saltenv) if __salt__['cp.hash_file'](installer, saltenv) != \ __salt__['cp.hash_file'](cached_pkg): cached_pkg = __salt__['cp.cache_file'](installer, saltenv) @@ -458,6 +487,9 @@ def install(name=None, refresh=False, pkgs=None, saltenv='base', **kwargs): cached_pkg = cached_pkg.replace('/', '\\') msiexec = pkginfo[version_num].get('msiexec') + allusers = pkginfo[version_num].get('allusers') + if allusers is None: + allusers = True install_flags = '{0} {1}'.format(pkginfo[version_num]['install_flags'], options and options.get('extra_install_flags') or "") cmd = [] @@ -465,6 +497,8 @@ def install(name=None, refresh=False, pkgs=None, saltenv='base', **kwargs): cmd.extend(['msiexec', '/i']) cmd.append(cached_pkg) cmd.extend(install_flags.split()) + if msiexec and allusers: + cmd.append('ALLUSERS="1"') __salt__['cmd.run'](cmd, output_loglevel='trace', python_shell=False) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 0bd3daeaf8..14f5816fd2 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -144,8 +144,6 @@ def _find_remove_targets(name=None, _normalize_name = __salt__.get('pkg.normalize_name', lambda pkgname: pkgname) to_remove = {_normalize_name(name): version} - cver = cur_pkgs.get(name, []) - version_spec = False # Find out which packages will be targeted in the call to pkg.remove # Check current versions against specified versions @@ -487,14 +485,61 @@ def installed( Ensure that the package is installed, and that it is the correct version (if specified). - name + :param str name: The name of the package to be installed. This parameter is ignored if either "pkgs" or "sources" is used. Additionally, please note that this option can only be used to install packages from a software repository. To install a package file manually, use the "sources" option detailed below. - fromrepo + :param str version: + Install a specific version of a package. This option is ignored if + either "pkgs" or "sources" is used. Currently, this option is supported + for the following pkg providers: :mod:`apt `, + :mod:`ebuild `, + :mod:`pacman `, + :mod:`yumpkg `, and + :mod:`zypper `. The version number includes the + release designation where applicable, to allow Salt to target a + specific release of a given version. When in doubt, using the + ``pkg.latest_version`` function for an uninstalled package will tell + you the version available. + + .. code-block:: bash + + # salt myminion pkg.latest_version httpd + myminion: + 2.2.15-30.el6.centos + + Also, while this function is not yet implemented for all pkg frontends, + :mod:`pkg.list_repo_pkgs ` will + show all versions available in the various repositories for a given + package, irrespective of whether or not it is installed. + + .. code-block:: bash + + # salt myminion pkg.list_repo_pkgs httpd + myminion: + ---------- + base: + |_ + ---------- + httpd: + 2.2.15-29.el6.centos + updates: + |_ + ---------- + httpd: + 2.2.15-30.el6.centos + + The version strings returned by either of these functions can be used + as version specifiers in pkg states. + + :param bool refresh: + Update the repo database of available packages prior to installing the + requested package. + + :param str fromrepo: Specify a repository from which to install .. note:: @@ -548,88 +593,124 @@ def installed( **4:0.8.10-0ubuntu0.12.04.1** either ``precise-updates`` or ``precise-security`` could be used for the ``fromrepo`` value. - skip_verify + :param bool skip_verify: Skip the GPG verification check for the package to be installed - skip_suggestions + :param bool skip_suggestions: Force strict package naming. Disables lookup of package alternatives. .. versionadded:: 2014.1.1 - version - Install a specific version of a package. This option is ignored if - either "pkgs" or "sources" is used. Currently, this option is supported - for the following pkg providers: :mod:`apt `, + :param list pkgs: + A list of packages to install from a software repository. All packages + listed under ``pkgs`` will be installed via a single command. + + Example: + + .. code-block:: yaml + + mypkgs: + pkg.installed: + - pkgs: + - foo + - bar + - baz + - hold: True + + ``NOTE:`` For :mod:`apt `, :mod:`ebuild `, - :mod:`pacman `, - :mod:`yumpkg `, and - :mod:`zypper `. The version number includes the - release designation where applicable, to allow Salt to target a - specific release of a given version. When in doubt, using the - ``pkg.latest_version`` function for an uninstalled package will tell - you the version available. + :mod:`pacman `, :mod:`yumpkg `, + and :mod:`zypper `, version numbers can be specified + in the ``pkgs`` argument. For example: - .. code-block:: bash + .. code-block:: yaml - # salt myminion pkg.latest_version httpd - myminion: - 2.2.15-30.el6.centos + mypkgs: + pkg.installed: + - pkgs: + - foo + - bar: 1.2.3-4 + - baz - Also, while this function is not yet implemented for all pkg frontends, - :mod:`pkg.list_repo_pkgs ` will - show all versions available in the various repositories for a given - package, irrespective of whether or not it is installed. + Additionally, :mod:`ebuild `, + :mod:`pacman ` and + :mod:`zypper ` support the ``<``, ``<=``, ``>=``, and + ``>`` operators for more control over what versions will be installed. For - .. code-block:: bash + Example: - # salt myminion pkg.list_repo_pkgs httpd - myminion: - ---------- - base: - |_ - ---------- - httpd: - 2.2.15-29.el6.centos - updates: - |_ - ---------- - httpd: - 2.2.15-30.el6.centos + .. code-block:: yaml - The version strings returned by either of these functions can be used - as version specifiers in pkg states. + mypkgs: + pkg.installed: + - pkgs: + - foo + - bar: '>=1.2.3-4' + - baz - refresh - Update the repo database of available packages prior to installing the - requested package. + ``NOTE:`` When using comparison operators, the expression must be enclosed + in quotes to avoid a YAML render error. - hold - Force the package to be held at the current installed version. - Currently works with YUM & APT based systems. + With :mod:`ebuild ` is also possible to specify a + use flag list and/or if the given packages should be in + package.accept_keywords file and/or the overlay from which you want the + package to be installed. + + For example: + + .. code-block:: yaml + + mypkgs: + pkg.installed: + - pkgs: + - foo: '~' + - bar: '~>=1.2:slot::overlay[use,-otheruse]' + - baz + + **Multiple Package Installation Options: (not supported in Windows or + pkgng)** + + :param list sources: + A list of packages to install, along with the source URI or local path + from which to install each package. In the example below, ``foo``, + ``bar``, ``baz``, etc. refer to the name of the package, as it would + appear in the output of the ``pkg.version`` or ``pkg.list_pkgs`` salt + CLI commands. + + .. code-block:: yaml + + mypkgs: + pkg.installed: + - sources: + - foo: salt://rpms/foo.rpm + - bar: http://somesite.org/bar.rpm + - baz: ftp://someothersite.org/baz.rpm + - qux: /minion/path/to/qux.rpm + + :param bool allow_updates: + Allow the package to be updated outside Salt's control (e.g. auto + updates on Windows). This means a package on the Minion can have a newer + version than the latest available in the repository without enforcing a + re-installation of the package. .. versionadded:: 2014.7.0 - allow_updates - Allow the package to be updated outside Salt's control (e.g. auto updates on Windows). - This means a package on the Minion can have a newer version than the latest available - in the repository without enforcing a re-installation of the package. + Example: - .. versionadded:: 2014.7.0 + .. code-block:: yaml - Example: + httpd: + pkg.installed: + - fromrepo: mycustomrepo + - skip_verify: True + - skip_suggestions: True + - version: 2.0.6~ubuntu3 + - refresh: True + - allow_updates: True + - hold: False - .. code-block:: yaml + :param bool pkg_verify: - httpd: - pkg.installed: - - fromrepo: mycustomrepo - - skip_verify: True - - skip_suggestions: True - - version: 2.0.6~ubuntu3 - - refresh: True - - hold: False - - pkg_verify .. versionadded:: 2014.7.0 For requested packages that are already installed and would not be targeted for @@ -639,27 +720,27 @@ def installed( passed to pkg.verify (see example below). Currently, this option is supported for the following pkg providers: :mod:`yumpkg `. - Examples: + Examples: - .. code-block:: yaml + .. code-block:: yaml - httpd: - pkg.installed: - - version: 2.2.15-30.el6.centos - - pkg_verify: True + httpd: + pkg.installed: + - version: 2.2.15-30.el6.centos + - pkg_verify: True - .. code-block:: yaml + .. code-block:: yaml - mypkgs: - pkg.installed: - - pkgs: - - foo - - bar: 1.2.3-4 - - baz - - pkg_verify: - - ignore_types: [config,doc] + mypkgs: + pkg.installed: + - pkgs: + - foo + - bar: 1.2.3-4 + - baz + - pkg_verify: + - ignore_types: [config,doc] - normalize + :param bool normalize: Normalize the package name by removing the architecture. Default is True. This is useful for poorly created packages which might include the architecture as an actual part of the name such as kernel modules @@ -667,133 +748,64 @@ def installed( .. versionadded:: 2014.7.0 - Example: + Example: - .. code-block:: yaml + .. code-block:: yaml - gpfs.gplbin-2.6.32-279.31.1.el6.x86_64: - pkg.installed: - - normalize: False + gpfs.gplbin-2.6.32-279.31.1.el6.x86_64: + pkg.installed: + - normalize: False - **Multiple Package Installation Options: (not supported in Windows or - pkgng)** + :param kwargs: + These are specific to each OS. If it does not apply to the execution + module for your OS, it is ignored. - pkgs - A list of packages to install from a software repository. All packages - listed under ``pkgs`` will be installed via a single command. + :param bool hold: + Force the package to be held at the current installed version. + Currently works with YUM & APT based systems. - Example: + .. versionadded:: 2014.7.0 - .. code-block:: yaml + :param list names: + A list of packages to install from a software repository. Each package + will be installed individually by the package manager. - mypkgs: - pkg.installed: - - pkgs: - - foo - - bar - - baz - - hold: True + .. warning:: - ``NOTE:`` For :mod:`apt `, - :mod:`ebuild `, - :mod:`pacman `, :mod:`yumpkg `, - and :mod:`zypper `, version numbers can be specified - in the ``pkgs`` argument. For example: + Unlike ``pkgs``, the ``names`` parameter cannot specify a version. + In addition, it makes a separate call to the package management + frontend to install each package, whereas ``pkgs`` makes just a + single call. It is therefore recommended to use ``pkgs`` instead of + ``names`` to install multiple packages, both for the additional + features and the performance improvement that it brings. - .. code-block:: yaml + :param bool install_recommends: + Whether to install the packages marked as recommended. Default is True. + Currently only works with APT based systems. - mypkgs: - pkg.installed: - - pkgs: - - foo - - bar: 1.2.3-4 - - baz + .. versionadded:: 2015.5.0 - Additionally, :mod:`ebuild `, - :mod:`pacman ` and - :mod:`zypper ` support the ``<``, ``<=``, ``>=``, and - ``>`` operators for more control over what versions will be installed. For - example: + .. code-block:: yaml - .. code-block:: yaml + httpd: + pkg.installed: + - install_recommends: False - mypkgs: - pkg.installed: - - pkgs: - - foo - - bar: '>=1.2.3-4' - - baz + :param bool only_upgrade: + Only upgrade the packages, if they are already installed. Default is + False. Currently only works with APT based systems. - ``NOTE:`` When using comparison operators, the expression must be enclosed - in quotes to avoid a YAML render error. + .. versionadded:: 2015.5.0 - With :mod:`ebuild ` is also possible to specify a use - flag list and/or if the given packages should be in package.accept_keywords - file and/or the overlay from which you want the package to be installed. - For example: + .. code-block:: yaml - .. code-block:: yaml + httpd: + pkg.installed: + - only_upgrade: True - mypkgs: - pkg.installed: - - pkgs: - - foo: '~' - - bar: '~>=1.2:slot::overlay[use,-otheruse]' - - baz - - names - A list of packages to install from a software repository. Each package - will be installed individually by the package manager. - - .. warning:: - - Unlike ``pkgs``, the ``names`` parameter cannot specify a version. - In addition, it makes a separate call to the package management - frontend to install each package, whereas ``pkgs`` makes just a - single call. It is therefore recommended to use ``pkgs`` instead of - ``names`` to install multiple packages, both for the additional - features and the performance improvement that it brings. - - sources - A list of packages to install, along with the source URI or local path - from which to install each package. In the example below, ``foo``, - ``bar``, ``baz``, etc. refer to the name of the package, as it would - appear in the output of the ``pkg.version`` or ``pkg.list_pkgs`` salt - CLI commands. - - .. code-block:: yaml - - mypkgs: - pkg.installed: - - sources: - - foo: salt://rpms/foo.rpm - - bar: http://somesite.org/bar.rpm - - baz: ftp://someothersite.org/baz.rpm - - qux: /minion/path/to/qux.rpm - - install_recommends - Whether to install the packages marked as recommended. Default is True. - Currently only works with APT based systems. - - .. versionadded:: 2015.5.0 - - .. code-block:: yaml - - httpd: - pkg.installed: - - install_recommends: False - - only_upgrade - Only upgrade the packages, if they are already installed. Default is False. - Currently only works with APT based systems. - - .. versionadded:: 2015.5.0 - - .. code-block:: yaml - - httpd: - pkg.installed: - - only_upgrade: True + :return: + A dictionary containing the state of the software installation + :rtype dict: ''' if isinstance(pkgs, list) and len(pkgs) == 0: From a25d0eabefe31b8e27d71f0625473fc41c077b09 Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Thu, 30 Jul 2015 03:02:19 -0600 Subject: [PATCH 07/26] update syndic doc to conform to style --- doc/topics/topology/syndic.rst | 78 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/doc/topics/topology/syndic.rst b/doc/topics/topology/syndic.rst index bcacd57102..dedb467a66 100644 --- a/doc/topics/topology/syndic.rst +++ b/doc/topics/topology/syndic.rst @@ -7,9 +7,9 @@ Salt Syndic The Salt Syndic interface is a powerful tool which allows for the construction of Salt command topologies. A basic Salt setup has a Salt Master commanding a group of Salt Minions. The Syndic interface is a special passthrough -minion, it is run on a master and connects to another master, then the master -that the Syndic minion is listening to can control the minions attached to -the master running the syndic. +Minion, it is run on a Master and connects to another Master, then the Master +that the Syndic Minion is listening to can control the Minions attached to +the Master running the Syndic. The intent for supporting many layouts is not presented with the intent of supposing the use of any single topology, but to allow a more flexible method @@ -18,23 +18,23 @@ of controlling many systems. Configuring the Syndic ====================== -Since the Syndic only needs to be attached to a higher level master the -configuration is very simple. On a master that is running a syndic to connect -to a higher level master the :conf_master:`syndic_master` option needs to be -set in the master config file. The ``syndic_master`` option contains the -hostname or IP address of the master server that can control the master that -the syndic is running on. +Since the Syndic only needs to be attached to a higher level Master the +configuration is very simple. On a Master that is running a Syndic to connect +to a higher level Master the :conf_master:`syndic_master` option needs to be +set in the Master config file. The ``syndic_master`` option contains the +hostname or IP address of the Master server that can control the Master that +the Syndic is running on. -The master that the syndic connects to sees the syndic as an ordinary minion, -and treats it as such. the higher level master will need to accept the syndic's -minion key like any other minion. This master will also need to set the +The Master that the Syndic connects to sees the Syndic as an ordinary Minion, +and treats it as such. the higher level Master will need to accept the Syndic's +Minion key like any other Minion. This Master will also need to set the :conf_master:`order_masters` value in the configuration to ``True``. The -``order_masters`` option in the config on the higher level master is very -important, to control a syndic extra information needs to be sent with the +``order_masters`` option in the config on the higher level Master is very +important, to control a Syndic extra information needs to be sent with the publications, the ``order_masters`` option makes sure that the extra data is sent out. -To sum up, you have those configuration options available on the master side: +To sum up, you have those configuration options available on the Master side: - :conf_master:`syndic_master`: MasterOfMaster ip/address - :conf_master:`syndic_master_port`: MasterOfMaster ret_port @@ -42,13 +42,13 @@ To sum up, you have those configuration options available on the master side: - :conf_master:`syndic_pidfile`: path to the pidfile (absolute or not) Each Syndic must provide its own ``file_roots`` directory. Files will not be -automatically transferred from the master-master. +automatically transferred from the Master-Master. Running the Syndic ================== -The Syndic is a separate daemon that needs to be started on the master that is -controlled by a higher master. Starting the Syndic daemon is the same as +The Syndic is a separate daemon that needs to be started on the Master that is +controlled by a higher Master. Starting the Syndic daemon is the same as starting the other Salt daemons. .. code-block:: bash @@ -58,9 +58,9 @@ starting the other Salt daemons. .. note:: If you have an exceptionally large infrastructure or many layers of - syndics, you may find that the CLI doesn't wait long enough for the syndics + Syndics, you may find that the CLI doesn't wait long enough for the Syndics to return their events. If you think this is the case, you can set the - :conf_master:`syndic_wait` value in the upper master config. The default + :conf_master:`syndic_wait` value in the upper Master config. The default value is ``1``, and should work for the majority of deployments. @@ -68,38 +68,38 @@ Topology ======== The ``salt-syndic`` is little more than a command and event forwarder. When a -command is issued from a higher-level master, it will be received by the -configured syndics on lower-level masters, and propagated to to their minions, -and other syndics that are bound to them further down in the hierarchy. When -events and job return data are generated by minions, they aggregated back, -through the same syndic(s), to the master which issued the command. +command is issued from a higher-level Master, it will be received by the +configured Syndics on lower-level Masters, and propagated to to their Minions, +and other Syndics that are bound to them further down in the hierarchy. When +events and job return data are generated by Minions, they aggregated back, +through the same Syndic(s), to the Master which issued the command. -The master sitting at the top of the hierarchy (the Master of Masters) will *not* +The Master sitting at the top of the hierarchy (the Master of Masters) will *not* be running the ``salt-syndic`` daemon. It will have the ``salt-master`` -daemon running, and optionally, the ``salt-minion`` daemon. Each syndic -connected to an upper-level master will have both the ``salt-master`` and the +daemon running, and optionally, the ``salt-minion`` daemon. Each Syndic +connected to an upper-level Master will have both the ``salt-master`` and the ``salt-syndic`` daemon running, and optionally, the ``salt-minion`` daemon. -Nodes on the lowest points of the hierarchy (minions which do not propagate +Nodes on the lowest points of the hierarchy (Minions which do not propagate data to another level) will only have the ``salt-minion`` daemon running. There is no need for either ``salt-master`` or ``salt-syndic`` to be running on a -standard minion. +standard Minion. Syndic and the CLI ================== -In order for the high-level master to return information from minions that are -below the syndic(s), the CLI requires a short wait time in order to allow the -syndic(s) to gather responses from their minions. This value is defined in the +In order for the high-level Master to return information from Minions that are +below the Syndic(s), the CLI requires a short wait time in order to allow the +Syndic(s) to gather responses from their Minions. This value is defined in the ``syndic_wait`` and has a default of five seconds. -While it is possible to run a syndic without a minion installed on the same machine, -it is recommended, for a faster CLI response time, to do so. Without a minion -installed on the syndic, the timeout value of ``syndic_wait`` increases -significantly - about three-fold. With a minion installed on the syndic, the CLI +While it is possible to run a Syndic without a Minion installed on the same machine, +it is recommended, for a faster CLI response time, to do so. Without a Minion +installed on the Syndic, the timeout value of ``syndic_wait`` increases +significantly - about three-fold. With a Minion installed on the Syndic, the CLI timeout resides at the value defined in ``syndic_wait``. .. note:: - To reduce the amount of time the CLI waits for minions to respond, install a minion - on the syndic or tune the value of the ``syndic_wait`` configuration. + To reduce the amount of time the CLI waits for Minions to respond, install a Minion + on the Syndic or tune the value of the ``syndic_wait`` configuration. From 1fd4837bebaf2c4603dfe842e91ec4b6938bbc7b Mon Sep 17 00:00:00 2001 From: Julien Barbot Date: Wed, 29 Jul 2015 12:24:24 +0200 Subject: [PATCH 08/26] Fix get_managed() in file.py module for local files This fixes the 'Unable to manage file: string indices must be integers, not str' error message problem. source_sum must be a dictionary not a string. --- salt/modules/file.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index d5d6a9d984..ef03c86079 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -2929,9 +2929,11 @@ def get_managed( return '', {}, 'Source file {0} not found'.format(source) # if its a local file elif urlparsed_source.scheme == 'file': - source_sum = get_hash(urlparsed_source.path) + file_sum = get_hash(urlparsed_source.path, form='sha256') + source_sum = {'hsum': file_sum, 'hash_type': 'sha256'} elif source.startswith('/'): - source_sum = get_hash(source) + file_sum = get_hash(source, form='sha256') + source_sum = {'hsum': file_sum, 'hash_type': 'sha256'} elif source_hash: protos = ('salt', 'http', 'https', 'ftp', 'swift', 's3') if _urlparse(source_hash).scheme in protos: From c6715e0404180f82d9e335f7efaa3ba06af69851 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 30 Jul 2015 10:19:16 -0600 Subject: [PATCH 09/26] Protect against passing a map file in addition to VM names with --destroy Fixes #24036 --- salt/cloud/cli.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/salt/cloud/cli.py b/salt/cloud/cli.py index 85896f76cd..79a767acfb 100644 --- a/salt/cloud/cli.py +++ b/salt/cloud/cli.py @@ -166,12 +166,21 @@ class SaltCloud(parsers.SaltCloudParser): elif self.options.destroy and (self.config.get('names', None) or self.config.get('map', None)): - if self.config.get('map', None): - log.info('Applying map from {0!r}.'.format(self.config['map'])) + map_file = self.config.get('map', None) + names = self.config.get('names', ()) + if map_file is not None: + if names != (): + msg = 'Supplying a mapfile, \'{0}\', in addition to instance names {1} ' \ + 'with the \'--destroy\' or \'-d\' function is not supported. ' \ + 'Please choose to delete either the entire map file or individual ' \ + 'instances.'.format(map_file, names) + self.handle_exception(msg, SaltCloudSystemExit) + + log.info('Applying map from \'{0}\'.'.format(map_file)) matching = mapper.delete_map(query='list_nodes') else: matching = mapper.get_running_by_names( - self.config.get('names', ()), + names, profile=self.options.profile ) From 7d68e49d98caf3f351b6cbcc7753b0474ba74ad4 Mon Sep 17 00:00:00 2001 From: Akhter Ali Date: Fri, 24 Jul 2015 10:07:28 -0400 Subject: [PATCH 10/26] Update schedule.py This is regarding issue #25577 There is technically nothing wrong with the documentation already placed, it's just keeping up with consistency. --- salt/states/schedule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/schedule.py b/salt/states/schedule.py index c61f0bdd28..ca0fef3929 100644 --- a/salt/states/schedule.py +++ b/salt/states/schedule.py @@ -62,7 +62,7 @@ Management of the Salt scheduler - args: - httpd - kwargs: - test: True + test: True - when: - Monday 5:00pm - Tuesday 3:00pm From 0211972fd7b8e20a0ffc59f727ccf748042d627d Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 30 Jul 2015 13:05:29 -0600 Subject: [PATCH 11/26] Whitespace fix --- salt/cloud/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/cli.py b/salt/cloud/cli.py index 79a767acfb..08aab8f037 100644 --- a/salt/cloud/cli.py +++ b/salt/cloud/cli.py @@ -175,7 +175,7 @@ class SaltCloud(parsers.SaltCloudParser): 'Please choose to delete either the entire map file or individual ' \ 'instances.'.format(map_file, names) self.handle_exception(msg, SaltCloudSystemExit) - + log.info('Applying map from \'{0}\'.'.format(map_file)) matching = mapper.delete_map(query='list_nodes') else: From db2129b1995f2c0fda0da16679a5e4a9607dbb38 Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Thu, 30 Jul 2015 14:39:15 -0600 Subject: [PATCH 12/26] Minor doc bug fixes Refs #24042 Refs #25650 Refs #21296 Refs #23788 --- conf/master | 11 +++++++--- doc/ref/configuration/master.rst | 35 ++++++++++++++++++++++++++++++++ doc/topics/eauth/index.rst | 5 +++++ salt/modules/drac.py | 4 ++-- salt/modules/state.py | 6 +++--- 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/conf/master b/conf/master index fbffc66c62..3067a2fd76 100644 --- a/conf/master +++ b/conf/master @@ -354,12 +354,12 @@ # If this is set to True the first newline after a Jinja block is removed # (block, not variable tag!). Defaults to False, corresponds to the Jinja # environment init variable "trim_blocks". -# jinja_trim_blocks: False +#jinja_trim_blocks: False # # If this is set to True leading spaces and tabs are stripped from the start # of a line to a block. Defaults to False, corresponds to the Jinja # environment init variable "lstrip_blocks". -# jinja_lstrip_blocks: False +#jinja_lstrip_blocks: False # The failhard option tells the minions to stop immediately after the first # failure detected in the state execution, defaults to False @@ -379,7 +379,7 @@ #state_output: full # Automatically aggregate all states that have support for mod_aggregate by -# setting to True. Or pass a list of state module names to automatically +# setting to 'True'. Or pass a list of state module names to automatically # aggregate just those types. # # state_aggregate: @@ -387,6 +387,11 @@ # #state_aggregate: False +# Send progress events as each function in a state run completes execution +# by setting to 'True'. Progress events are in the format +# 'salt/job//prog//'. +#state_events: False + ##### File Server settings ##### ########################################## # Salt runs a lightweight file server written in zeromq to deliver files to diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index b8f8c4820a..bab2daaab1 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -906,6 +906,41 @@ If set to 'changes', the output will be full unless the state didn't change. state_output: full +.. conf_master:: state_aggregate + +``state_aggregate`` +------------------- + +Default: ``False`` + +Automatically aggregate all states that have support for mod_aggregate by +setting to ``True``. Or pass a list of state module names to automatically +aggregate just those types. + +.. code-block:: yaml + + state_aggregate: + - pkg + +.. code-block:: yaml + + state_aggregate: True + +.. conf_master:: state_events + +``state_events`` +---------------- + +Default: ``False`` + +Send progress events as each function in a state run completes execution +by setting to ``True``. Progress events are in the format +``salt/job//prog//``. + +.. code-block:: yaml + + state_events: True + .. conf_master:: yaml_utf8 ``yaml_utf8`` diff --git a/doc/topics/eauth/index.rst b/doc/topics/eauth/index.rst index 815acc043d..d19c11df22 100644 --- a/doc/topics/eauth/index.rst +++ b/doc/topics/eauth/index.rst @@ -73,6 +73,11 @@ append a ``%`` to the ID: - '*': - 'pkg.*' +.. warning:: + All users that have external authentication privileges are allowed to run + :mod:`saltutil.findjob `. Be aware + that this could inadvertently expose some data such as minion IDs. + .. _salt-token-generation: Tokens diff --git a/salt/modules/drac.py b/salt/modules/drac.py index 131622dbc0..4e0f323e62 100644 --- a/salt/modules/drac.py +++ b/salt/modules/drac.py @@ -64,7 +64,7 @@ def system_info(): .. code-block:: bash - salt dell drac.getsysinfo + salt dell drac.system_info ''' drac = {} section = '' @@ -85,7 +85,7 @@ def network_info(): .. code-block:: bash - salt dell drac.getniccfg + salt dell drac.network_info ''' cmd = __salt__['cmd.run_all']('racadm getniccfg') diff --git a/salt/modules/state.py b/salt/modules/state.py index a2cf2f7765..fdb1453425 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -101,9 +101,9 @@ def _wait(jid): def running(concurrent=False): ''' - Return a dict of state return data if a state function is already running. - This function is used to prevent multiple state calls from being run at - the same time. + Return a list of strings that contain state return data if a state function is + already running. This function is used to prevent multiple state calls from being + run at the same time. CLI Example: From 3f3db4bd8e10b3bcfe2729ace984075e8806fb52 Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Thu, 30 Jul 2015 15:43:21 -0600 Subject: [PATCH 13/26] Additions for #24042 --- conf/master | 2 +- doc/ref/configuration/master.rst | 2 +- doc/topics/event/master_events.rst | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/conf/master b/conf/master index 3067a2fd76..aaafb6f4da 100644 --- a/conf/master +++ b/conf/master @@ -389,7 +389,7 @@ # Send progress events as each function in a state run completes execution # by setting to 'True'. Progress events are in the format -# 'salt/job//prog//'. +# 'salt/job//prog//'. #state_events: False ##### File Server settings ##### diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index bab2daaab1..b7cd1bdc40 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -935,7 +935,7 @@ Default: ``False`` Send progress events as each function in a state run completes execution by setting to ``True``. Progress events are in the format -``salt/job//prog//``. +``salt/job//prog//``. .. code-block:: yaml diff --git a/doc/topics/event/master_events.rst b/doc/topics/event/master_events.rst index 35614387cf..d07bc09c9a 100644 --- a/doc/topics/event/master_events.rst +++ b/doc/topics/event/master_events.rst @@ -86,6 +86,15 @@ Job events :var fun: The function the minion ran. E.g., ``test.ping``. :var return: The data returned from the execution module. +.. salt:event:: salt/job//prog// + + Fired each time a each function in a state run completes execution. Must be + enabled using the :conf_master:`state_events` option. + + :var data: The data returned from the state module function. + :var id: The minion ID. + :var jid: The job ID. + .. _event-master_presence: Presence events From 4795952847d295728c346e98f3edb3d3454c7a9a Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Thu, 30 Jul 2015 04:19:39 -0600 Subject: [PATCH 14/26] rework syndic doc --- doc/topics/topology/syndic.rst | 219 +++++++++++++++++++++++---------- 1 file changed, 152 insertions(+), 67 deletions(-) diff --git a/doc/topics/topology/syndic.rst b/doc/topics/topology/syndic.rst index dedb467a66..025cbc56d8 100644 --- a/doc/topics/topology/syndic.rst +++ b/doc/topics/topology/syndic.rst @@ -4,102 +4,187 @@ Salt Syndic =========== -The Salt Syndic interface is a powerful tool which allows for the construction -of Salt command topologies. A basic Salt setup has a Salt Master commanding a -group of Salt Minions. The Syndic interface is a special passthrough -Minion, it is run on a Master and connects to another Master, then the Master -that the Syndic Minion is listening to can control the Minions attached to -the Master running the Syndic. +The most basic or typical Salt topology consists of a single Master node +controlling a group of Minion nodes. An intermediate node type, called Syndic, +when used offers greater structural flexibility and scalability in the +construction of Salt topologies than topologies constructed only out of Master +and Minion node types. -The intent for supporting many layouts is not presented with the intent of -supposing the use of any single topology, but to allow a more flexible method -of controlling many systems. +A Syndic node can be thought of as a special passthrough Minion node. A Syndic +node consists of a ``salt-syndic`` daemon and a ``salt-master`` daemon running +on the same system. The ``salt-master`` daemon running on the Syndic node +controls a group of lower level Minion nodes and the ``salt-syndic`` daemon +connects higher level Master node, sometimes called a Master of Masters. + +The ``salt-syndic`` daemon relays publications and events between the Master +node and the local ``salt-master`` daemon. This gives the Master node control +over the Minion nodes attached to the ``salt-master`` daemon running on the +Syndic node. Configuring the Syndic ====================== -Since the Syndic only needs to be attached to a higher level Master the -configuration is very simple. On a Master that is running a Syndic to connect -to a higher level Master the :conf_master:`syndic_master` option needs to be -set in the Master config file. The ``syndic_master`` option contains the -hostname or IP address of the Master server that can control the Master that -the Syndic is running on. +To setup a Salt Syndic you need to tell the Syndic node and its Master node +about each other. If your Master node is located at ``10.10.0.1``, then your +configurations would be: -The Master that the Syndic connects to sees the Syndic as an ordinary Minion, -and treats it as such. the higher level Master will need to accept the Syndic's -Minion key like any other Minion. This Master will also need to set the -:conf_master:`order_masters` value in the configuration to ``True``. The -``order_masters`` option in the config on the higher level Master is very -important, to control a Syndic extra information needs to be sent with the -publications, the ``order_masters`` option makes sure that the extra data is -sent out. +On the Syndic node: -To sum up, you have those configuration options available on the Master side: +.. code-block:: yaml - - :conf_master:`syndic_master`: MasterOfMaster ip/address - - :conf_master:`syndic_master_port`: MasterOfMaster ret_port - - :conf_master:`syndic_log_file`: path to the logfile (absolute or not) - - :conf_master:`syndic_pidfile`: path to the pidfile (absolute or not) + # /etc/salt/master + syndic_master: 10.10.0.1 # may be either an IP address or a hostname -Each Syndic must provide its own ``file_roots`` directory. Files will not be -automatically transferred from the Master-Master. +.. code-block:: yaml + + # /etc/salt/minion + + # id is shared by the salt-syndic daemon and a possible salt-minion daemon + # on the Syndic node + id: my_syndic + +On the Master node: + +.. code-block:: yaml + + # /etc/salt/master + order_masters: True + +The :conf_master:`syndic_master` option tells the Syndic node where to find the +Master node in the same way that the :conf_minion:`master` option tells a +Minion node where to find a Master node. + +The :conf_minion:`id` option is used by the ``salt-syndic`` daemon to identify +with the Master node and if unset will default to the hostname or IP address of +the Syndic just as with a Minion. + +The :conf_master:`order_masters` option configures the Master node to send +extra information with its publications that is needed by Syndic nodes +connected directly to it. + +.. note:: + + Each Syndic must provide its own ``file_roots`` directory. Files will not + be automatically transferred from the Master node. Running the Syndic ================== -The Syndic is a separate daemon that needs to be started on the Master that is -controlled by a higher Master. Starting the Syndic daemon is the same as -starting the other Salt daemons. +The ``salt-syndic`` daemon is a separate process that needs to be started in +addition to the ``salt-master`` daemon running on the Syndic node. Starting +the ``salt-syndic`` daemon is the same as starting the other Salt daemons. + +The Master node in many ways sees the Syndic as an ordinary Minion node. In +particular, the Master will need to accept the Syndic's Minion key as it would +for any other Minion. + +On the Syndic node: .. code-block:: bash # salt-syndic + or + # service salt-syndic start -.. note:: +On the Master node: - If you have an exceptionally large infrastructure or many layers of - Syndics, you may find that the CLI doesn't wait long enough for the Syndics - to return their events. If you think this is the case, you can set the - :conf_master:`syndic_wait` value in the upper Master config. The default - value is ``1``, and should work for the majority of deployments. +.. code-block:: bash + # salt-key -a my_syndic + +The Master node will now be able to control the Minion nodes connected to the +Syndic. Only the Syndic key will be listed in the Master node's key registry +but this also means that key activity between the Syndic's Minions and the +Syndic does not encumber the Master node. In this way, the Syndic's key on the +Master node can be thought of as a placeholder for the keys of all the Minion +and Syndic nodes beneath it, giving the Master node a clear, high level +structural view on the Salt cluster. + +On the Master node: + +.. code-block:: bash + + # salt-key -L + Accepted Keys: + my_syndic + Denied Keys: + Unaccepted Keys: + Rejected Keys: + + # salt '*' test.ping + minion_1: + True + minion_2: + True + minion_4: + True + minion_3: + True Topology ======== -The ``salt-syndic`` is little more than a command and event forwarder. When a -command is issued from a higher-level Master, it will be received by the -configured Syndics on lower-level Masters, and propagated to to their Minions, -and other Syndics that are bound to them further down in the hierarchy. When -events and job return data are generated by Minions, they aggregated back, -through the same Syndic(s), to the Master which issued the command. +A Master node (a node which is itself not a Syndic to another higher level +Master node) must run a ``salt-master`` daemon and optionally a ``salt-minion`` +daemon. -The Master sitting at the top of the hierarchy (the Master of Masters) will *not* -be running the ``salt-syndic`` daemon. It will have the ``salt-master`` -daemon running, and optionally, the ``salt-minion`` daemon. Each Syndic -connected to an upper-level Master will have both the ``salt-master`` and the -``salt-syndic`` daemon running, and optionally, the ``salt-minion`` daemon. +A Syndic node must run ``salt-syndic`` and ``salt-master`` daemons and +optionally a ``salt-minion`` daemon. -Nodes on the lowest points of the hierarchy (Minions which do not propagate -data to another level) will only have the ``salt-minion`` daemon running. There -is no need for either ``salt-master`` or ``salt-syndic`` to be running on a -standard Minion. +A Minion node must run a ``salt-minion`` daemon. -Syndic and the CLI -================== +When a ``salt-master`` daemon issues a command, it will be received by the +Syndic and Minion nodes directly connected to it. A Minion node will process +the command in the way it ordinarily would. On a Syndic node, the +``salt-syndic`` daemon will relay the command to the ``salt-master`` daemon +running on the Syndic node, which then propagates the command to to the Minions +and Syndics connected to it. -In order for the high-level Master to return information from Minions that are -below the Syndic(s), the CLI requires a short wait time in order to allow the -Syndic(s) to gather responses from their Minions. This value is defined in the -``syndic_wait`` and has a default of five seconds. +When events and job return data are generated by ``salt-minion`` daemons, they +are aggregated by the ``salt-master`` daemon they are connected to, which +``salt-master`` daemon then relays the data back through its ``salt-syndic`` +daemon until the data reaches the Master or Syndic node that issued the command. -While it is possible to run a Syndic without a Minion installed on the same machine, -it is recommended, for a faster CLI response time, to do so. Without a Minion -installed on the Syndic, the timeout value of ``syndic_wait`` increases -significantly - about three-fold. With a Minion installed on the Syndic, the CLI -timeout resides at the value defined in ``syndic_wait``. +Syndic wait +=========== .. note:: - To reduce the amount of time the CLI waits for Minions to respond, install a Minion - on the Syndic or tune the value of the ``syndic_wait`` configuration. + To reduce the amount of time the CLI waits for Minions to respond, install + a Minion on the Syndic or tune the value of the ``syndic_wait`` + configuration. + +While it is possible to run a Syndic without a Minion installed on the same +system, it is recommended, for a faster CLI response time, to do so. Without a +Minion installed on the Syndic node, the timeout value of ``syndic_wait`` +increases significantly - about three-fold. With a Minion installed on the +Syndic, the CLI timeout resides at the value defined in ``syndic_wait``. + +.. note:: + + If you have a very large infrastructure or many layers of Syndics, you may + find that the CLI doesn't wait long enough for the Syndics to return their + events. If you think this is the case, you can set the + :conf_master:`syndic_wait` value in the Master configs on the Master or + Syndic nodes from which commands are executed. The default value is ``5``, + and should work for the majority of deployments. + +In order for a Master or Syndic node to return information from Minions that +are below their Syndics, the CLI requires a short wait time in order to allow +the Syndics to gather responses from their Minions. This value is defined in +the :conf_master:`syndic_wait` config option and has a default of five seconds. + +Syndic config options +===================== + +These are the options that can be used to configure a Syndic node. Note that +other than ``id``, Syndic config options are placed in the Master config on the +Syndic node. + + - :conf_minion:`id`: Syndic id (shared by the ``salt-syndic`` daemon with a + potential ``salt-minion`` daemon on the same system) + - :conf_master:`syndic_master`: Master node IP address or hostname + - :conf_master:`syndic_master_port`: Master node ret_port + - :conf_master:`syndic_log_file`: path to the logfile (absolute or not) + - :conf_master:`syndic_pidfile`: path to the pidfile (absolute or not) + - :conf_master:`syndic_wait`: time in seconds to wait on returns from this syndic From 333fbdde306debb989177cca7357db17faa7debd Mon Sep 17 00:00:00 2001 From: TheBigBear Date: Thu, 30 Jul 2015 20:33:15 +0100 Subject: [PATCH 15/26] Update 7-zip msi un-installer instructions The instructions as-are do not actually work. The uninstaller flag takes the windows GUID of the msi product code. And the various ```install_flags``` and ```uninstall_flags``` do NOT need leading, or trailing spaces. @twangboy can confirm this is a better example (also see salt issue #25778 https://github.com/saltstack/salt/issues/25778) --- doc/topics/windows/windows-package-manager.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 750b3a7639..e36cf5e88c 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -160,11 +160,10 @@ project's wiki_: installer: salt://win/repo/7zip/7z920-x64.msi full_name: 7-Zip 9.20 (x64 edition) reboot: False - install_flags: ' /q ' + install_flags: '/qn /norestart' msiexec: True - uninstaller: salt://win/repo/7zip/7z920-x64.msi - uninstall_flags: ' /qn' - + uninstaller: '{23170F69-40C1-2702-0920-000001000000}' + uninstall_flags: '/qn /norestart' Generate Repo Cache File From c994d22696e84e348c79f7fd4506d5758984d498 Mon Sep 17 00:00:00 2001 From: TheBigBear Date: Fri, 31 Jul 2015 16:31:33 +0100 Subject: [PATCH 16/26] Minor update to msi un-installer info The ```uninstaller`` string can be either the windows GUID (aka msi prodcut code) OR the URL to the msi file. Both options work to uninstall a msi installed software. --- doc/topics/windows/windows-package-manager.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index e36cf5e88c..8e06a84f66 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -165,6 +165,20 @@ project's wiki_: uninstaller: '{23170F69-40C1-2702-0920-000001000000}' uninstall_flags: '/qn /norestart' +Alternatively the ``uninstaller`` can also simply repeat the URL of the msi file. + +.. code-block:: yaml + + 7zip: + 9.20.00.0: + installer: salt://win/repo/7zip/7z920-x64.msi + full_name: 7-Zip 9.20 (x64 edition) + reboot: False + install_flags: '/qn /norestart' + msiexec: True + uninstaller: salt://win/repo/7zip/7z920-x64.msi + uninstall_flags: '/qn /norestart' + Generate Repo Cache File ======================== From 8074c545ea85b9dec651f64fb4f40d4cfd8114f1 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 31 Jul 2015 09:40:25 -0600 Subject: [PATCH 17/26] Handle non-ascii in state log Closes #25810 --- salt/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 7a2c419fac..2e07e62dcf 100644 --- a/salt/state.py +++ b/salt/state.py @@ -205,8 +205,8 @@ def format_log(ret): for pkg in chg: old = chg[pkg]['old'] or 'absent' new = chg[pkg]['new'] or 'absent' - msg += '{0} changed from {1} to ' \ - '{2}\n'.format(pkg, old, new) + msg += '{0!r} changed from {1!r} to ' \ + '{2!r}\n'.format(pkg, old, new) if not msg: msg = str(ret['changes']) if ret['result'] is True or ret['result'] is None: From e797739a1b38255c57ee6789f727b1ba3fd5dd4d Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 31 Jul 2015 10:15:12 -0600 Subject: [PATCH 18/26] Removed normalize_name function --- salt/modules/win_pkg.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index ed9b43a0a0..10b2e8626d 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -44,16 +44,6 @@ def __virtual__(): return False -def normalize_name(name): - ''' - This function needed for the pkg state module - :param str name: - :return: name - :rtype str: - ''' - return name - - def latest_version(*names, **kwargs): ''' Return the latest version of the named package available for upgrade or From cf7479aa0a5e3b89fb11ad2f4a0dcf45a94a644b Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 31 Jul 2015 09:48:26 -0700 Subject: [PATCH 19/26] Pass actual renderers to the Reactor's Compiler Weird issue, from looking at the old versions of the code it looks like it was also broken (at least on class init) but something afterwards came in and changed the pointer (SCARY!). This removes the need to load the renderers twice (bonus!) in addition to fixing the reported issue. Fixes #25852 --- salt/state.py | 4 ++-- salt/utils/reactor.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/state.py b/salt/state.py index 0de65d0f90..4a7dcfd4f3 100644 --- a/salt/state.py +++ b/salt/state.py @@ -245,9 +245,9 @@ class Compiler(object): ''' Class used to compile and manage the High Data structure ''' - def __init__(self, opts): + def __init__(self, opts, renderers): self.opts = opts - self.rend = salt.loader.render(self.opts, {}) + self.rend = renderers def render_template(self, template, **kwargs): ''' diff --git a/salt/utils/reactor.py b/salt/utils/reactor.py index 3ba5a371b1..6e45256f49 100644 --- a/salt/utils/reactor.py +++ b/salt/utils/reactor.py @@ -29,11 +29,10 @@ class Reactor(multiprocessing.Process, salt.state.Compiler): ''' def __init__(self, opts): multiprocessing.Process.__init__(self) - salt.state.Compiler.__init__(self, opts) - - local_minion_opts = self.opts.copy() + local_minion_opts = opts.copy() local_minion_opts['file_client'] = 'local' self.minion = salt.minion.MasterMinion(local_minion_opts) + salt.state.Compiler.__init__(self, opts, self.minion.rend) def render_reaction(self, glob_ref, tag, data): ''' From ac0a8ff711a377b855087d06f4ad92abf50cdaa0 Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Fri, 31 Jul 2015 15:25:15 -0600 Subject: [PATCH 20/26] Doc on using syndic with multimaster --- doc/ref/configuration/master.rst | 13 ++++++++++++- doc/topics/highavailability/index.rst | 9 +++++++++ doc/topics/topology/syndic.rst | 20 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index b7cd1bdc40..a3af999f9a 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -2089,10 +2089,21 @@ master, specify the higher level master with this configuration value. syndic_master: masterofmasters +You can optionally connect a syndic to multiple higher level masters by +setting the 'syndic_master' value to a list: + +.. code-block:: yaml + + syndic_master: + - masterofmasters1 + - masterofmasters2 + +Each higher level master must be set up in a multimaster configuration. + .. conf_master:: syndic_master_port ``syndic_master_port`` ------------------------ +---------------------- Default: ``4506`` diff --git a/doc/topics/highavailability/index.rst b/doc/topics/highavailability/index.rst index bc0f4c4787..f0263e7656 100644 --- a/doc/topics/highavailability/index.rst +++ b/doc/topics/highavailability/index.rst @@ -63,3 +63,12 @@ of Masters" nodes can control multiple segments underneath them. Syndics are covered in depth in :doc:`Salt Syndic `. +Syndic with Multimaster +======================= + +.. versionadded:: 2015.5.0 + +Syndic with Multimaster lets you connect a syndic to multiple masters to provide +an additional layer of redundancy in a syndic configuration. + +Syndics are covered in depth in :doc:`Salt Syndic `. \ No newline at end of file diff --git a/doc/topics/topology/syndic.rst b/doc/topics/topology/syndic.rst index 025cbc56d8..04624b82ee 100644 --- a/doc/topics/topology/syndic.rst +++ b/doc/topics/topology/syndic.rst @@ -67,6 +67,26 @@ connected directly to it. Each Syndic must provide its own ``file_roots`` directory. Files will not be automatically transferred from the Master node. +Configuring the Syndic with Multimaster +======================================= + +.. versionadded:: 2015.5.0 + +Syndic with Multimaster lets you connect a syndic to multiple masters to provide +an additional layer of redundancy in a syndic configuration. + +Higher level masters should first be configured in a multimaster configuration. +See :doc:`Multimaster Tutorial `. + +On the syndic, the :conf_master:`syndic_master` option is populated with +a list of the higher level masters. + +Since each syndic is connected to each master, jobs sent from any master are +forwarded to minions that are connected to each syndic. If the ``master_id`` value +is set in the master config on the higher level masters, job results are returned +to the master that originated the request in a best effort fashion. Events/jobs +without a ``master_id`` are returned to any available master. + Running the Syndic ================== From 464f7a404c76998b1205af5d6b83bfcad2408e2a Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Fri, 31 Jul 2015 17:19:21 -0600 Subject: [PATCH 21/26] add timelib to dependency versions Related to #25850. --- salt/version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/version.py b/salt/version.py index b5c0f72df9..b8abd0fba0 100644 --- a/salt/version.py +++ b/salt/version.py @@ -543,6 +543,7 @@ def versions_information(include_salt_cloud=False): ('ZMQ', 'zmq', 'zmq_version'), ('Mako', 'mako', '__version__'), ('Tornado', 'tornado', 'version'), + ('timelib', 'timelib', 'version'), ] if include_salt_cloud: From 7e121de9070b3dbb8745f5706565cfc01a25e808 Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Fri, 31 Jul 2015 17:34:07 -0600 Subject: [PATCH 22/26] Update minion.rst --- doc/ref/configuration/minion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 4223774516..5320ca8e51 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -530,7 +530,7 @@ be able to execute a certain module. The sys module is built into the minion and cannot be disabled. This setting can also tune the minion, as all modules are loaded into ram -disabling modules will lover the minion's ram footprint. +disabling modules will lower the minion's ram footprint. .. code-block:: yaml From 7f20454427f6612f1f74dfe8dfdd80d6f985ff75 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sun, 2 Aug 2015 11:11:32 -0700 Subject: [PATCH 23/26] If we're unable to fire an event, log the cause so we know what happened --- salt/modules/event.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/modules/event.py b/salt/modules/event.py index b003c236bc..a150a8c2ef 100644 --- a/salt/modules/event.py +++ b/salt/modules/event.py @@ -6,7 +6,10 @@ master to the minion and vice-versa. from __future__ import absolute_import # Import Python libs import collections +import logging import os +import sys +import traceback # Import salt libs import salt.crypt @@ -16,6 +19,7 @@ import salt.transport import salt.ext.six as six __proxyenabled__ = ['*'] +log = logging.getLogger(__name__) def _dict_subset(keys, master_dict): @@ -71,6 +75,9 @@ def fire_master(data, tag, preload=None): return salt.utils.event.MinionEvent(__opts__).fire_event( {'data': data, 'tag': tag, 'events': None, 'pretag': None}, 'fire_master') except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + log.debug(lines) return False @@ -93,6 +100,9 @@ def fire(data, tag): return event.fire_event(data, tag) except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + log.debug(lines) return False From 0f7f9637b49636fd4c451bbda9a4ed45fbee172e Mon Sep 17 00:00:00 2001 From: Petr Demin Date: Thu, 30 Jul 2015 17:49:30 +0300 Subject: [PATCH 24/26] #25863 fix - state.pkg: do preflight check only for non-installed packages --- salt/states/pkg.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 14f5816fd2..3871a9a97a 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -302,7 +302,12 @@ def _find_install_targets(name=None, # Takes extra time. Disable for improved performance if not skip_suggestions: # Perform platform-specific pre-flight checks - problems = _preflight_check(desired, **kwargs) + not_installed = dict([ + (name, version) + for name, version in desired.items() + if not (name in cur_pkgs and version in (None, cur_pkgs[name])) + ]) + problems = _preflight_check(not_installed, **kwargs) comments = [] if problems.get('no_suggest'): comments.append( From 82b7e14a1fdda114aa6769008d2c3e0b483f1b28 Mon Sep 17 00:00:00 2001 From: confidential Date: Fri, 31 Jul 2015 09:52:35 -0500 Subject: [PATCH 25/26] adding missing format string --- salt/modules/artifactory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/artifactory.py b/salt/modules/artifactory.py index e0f25619d8..e831ea77a4 100644 --- a/salt/modules/artifactory.py +++ b/salt/modules/artifactory.py @@ -259,7 +259,7 @@ def __save_artifact(artifact_url, target_file): } if os.path.isfile(target_file): - log.debug("File %s already exists, checking checksum...") + log.debug("File {0} already exists, checking checksum...".format(target_file)) checksum_url = artifact_url + ".sha1" checksum_success, artifact_sum, checksum_comment = __download(checksum_url) From 56e43c8f887e68ce5d1501684318e77478bd8e0a Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 3 Aug 2015 10:48:42 -0600 Subject: [PATCH 26/26] Fix lint --- salt/returners/carbon_return.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/returners/carbon_return.py b/salt/returners/carbon_return.py index a94e851644..aabcf43dca 100644 --- a/salt/returners/carbon_return.py +++ b/salt/returners/carbon_return.py @@ -221,7 +221,7 @@ def returner(ret): host = _options.get('host') port = _options.get('port') skip = _options.get('skip') - metric_base_pattern = options.get('carbon.metric_base_pattern') + metric_base_pattern = _options.get('carbon.metric_base_pattern') if 'mode' in _options: mode = _options.get('mode').lower()