From 7b470e15f67eeda8ebee38676ca3d547072b8d33 Mon Sep 17 00:00:00 2001 From: Pablo Ruiz Date: Sat, 17 Jun 2017 03:16:36 +0200 Subject: [PATCH 1/4] Addresses rh_ip's shortcomings when configuring balance-rr bonds with miimon monitoring, as well as adding support for 'advertise' ethtool's setting. --- salt/modules/rh_ip.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/salt/modules/rh_ip.py b/salt/modules/rh_ip.py index e9dd2da6ae..1afc1a56e5 100644 --- a/salt/modules/rh_ip.py +++ b/salt/modules/rh_ip.py @@ -46,7 +46,7 @@ def __virtual__(): _ETHTOOL_CONFIG_OPTS = [ 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', - 'gso', 'gro', 'lro' + 'gso', 'gro', 'lro', 'advertise' ] _RH_CONFIG_OPTS = [ 'domain', 'peerdns', 'peerntp', 'defroute', @@ -158,6 +158,18 @@ def _parse_ethtool_opts(opts, iface): else: _raise_error_iface(iface, opts['speed'], valid) + if 'advertise' in opts: + valid = [ + '0x001', '0x002', '0x004', '0x008', '0x010', '0x020', + '0x20000', '0x8000', '0x1000', '0x40000', '0x80000', + '0x200000', '0x400000', '0x800000', '0x1000000', + '0x2000000', '0x4000000' + ] + if str(opts['advertise']) in valid: + config.update({'advertise': opts['advertise']}) + else: + _raise_error_iface(iface, 'advertise', valid) + valid = _CONFIG_TRUE + _CONFIG_FALSE for option in ('rx', 'tx', 'sg', 'tso', 'ufo', 'gso', 'gro', 'lro'): if option in opts: @@ -268,7 +280,11 @@ def _parse_settings_bond_0(opts, iface, bond_def): function will log what the Interface, Setting and what it was expecting. ''' - bond = {'mode': '0'} + + # balance-rr shares miimon settings with balance-xor + bond = _parse_settings_bond_1(opts, iface, bond_def) + + bond.update({'mode': '0'}) # ARP targets in n.n.n.n form valid = ['list of ips (up to 16)'] @@ -285,7 +301,7 @@ def _parse_settings_bond_0(opts, iface, bond_def): _raise_error_iface(iface, 'arp_ip_target', valid) else: _raise_error_iface(iface, 'arp_ip_target', valid) - else: + elif not 'miimon' in opts: _raise_error_iface(iface, 'arp_ip_target', valid) if 'arp_interval' in opts: From 8d9596a22e489eb686ee797a43f8f5c6383194ed Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 20 Jun 2017 13:45:44 -0600 Subject: [PATCH 2/4] Fix lint error --- salt/modules/rh_ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/rh_ip.py b/salt/modules/rh_ip.py index 1afc1a56e5..58759afc54 100644 --- a/salt/modules/rh_ip.py +++ b/salt/modules/rh_ip.py @@ -301,7 +301,7 @@ def _parse_settings_bond_0(opts, iface, bond_def): _raise_error_iface(iface, 'arp_ip_target', valid) else: _raise_error_iface(iface, 'arp_ip_target', valid) - elif not 'miimon' in opts: + elif 'miimon' not in opts: _raise_error_iface(iface, 'arp_ip_target', valid) if 'arp_interval' in opts: From 1279f56c5617022371837879351bad077df8c824 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 9 May 2017 10:34:26 -0500 Subject: [PATCH 3/4] gitfs: Add two new options to affect saltenv mapping This adds 2 new config options: - gitfs_disable_saltenv_mapping - This allows a user to disable the environment mapping logic and rely solely on per-saltenv configuration - gitfs_ref_types - This allows individual ref types to be ignored It also includes a performance improvement in retrieving tree objects matching the desired ref name. Before, we would iterate through the different refs and check if the name matched the desired ref's name. Now we just check for a head/tag ref based on the ref path in the gitdir (e.g. refs/remotes/origin/master or refs/tags/v1.2.3). This prevents us from needlessly iterating through all refs until a match is found, which does not scale well when the number of branches and tags is larger. --- doc/ref/configuration/master.rst | 54 ++++++++++++++++++++++++++++++++ doc/topics/releases/oxygen.rst | 24 ++++++++++++++ doc/topics/tutorials/gitfs.rst | 22 ++++++++++--- salt/config/__init__.py | 6 ++++ salt/fileserver/gitfs.py | 9 ++++-- salt/utils/gitfs.py | 49 ++++++++++++++++++++++++----- 6 files changed, 150 insertions(+), 14 deletions(-) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 6a9bd7ae94..1705a603c4 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -2056,6 +2056,60 @@ gitfs remotes. - dev: - ref: develop +.. conf_master:: gitfs_disable_saltenv_mapping + +``gitfs_disable_saltenv_mapping`` +********************************* + +.. versionadded:: Oxygen + +Default: ``False`` + +When set to ``True``, all saltenv mapping logic is disregarded (aside from +which branch/tag is mapped to the ``base`` saltenv). To use any other +environments, they must then be defined using :ref:`per-saltenv configuration +parameters `. + +.. code-block:: yaml + + gitfs_disable_saltenv_mapping: True + +.. note:: + This is is a global configuration option, see :ref:`here + ` for examples of configuring it for individual + repositories. + +.. conf_master:: gitfs_ref_types + +``gitfs_ref_types`` +******************* + +.. versionadded:: Oxygen + +Default: ``['branch', 'tag', 'sha']`` + +This option defines what types of refs are mapped to fileserver environments +(i.e. saltenvs). It also sets the order of preference when there are +ambiguously-named refs (i.e. when a branch and tag both have the same name). +The below example disables mapping of both tags and SHAs, so that only branches +are mapped as saltenvs: + +.. code-block:: yaml + + gitfs_ref_types: + - branch + +.. note:: + This is is a global configuration option, see :ref:`here + ` for examples of configuring it for individual + repositories. + +.. note:: + ``sha`` is special in that it will not show up when listing saltenvs (e.g. + with the :py:func:`fileserver.envs ` runner), + but works within states and with :py:func:`cp.cache_file + ` to retrieve a file from a specific git SHA. + .. conf_master:: gitfs_saltenv_whitelist ``gitfs_saltenv_whitelist`` diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index f39fecafcc..51f7332388 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -25,6 +25,30 @@ by any master tops matches that are not matched via a top file. To make master tops matches execute first, followed by top file matches, set the new :conf_minion:`master_tops_first` minion config option to ``True``. +New GitFS Features +------------------ + +Two new features which affect how GitFS maps branches/tags to fileserver +environments (i.e. ``saltenvs``) have been added: + +1. It is now possible to completely turn off Salt's default mapping logic + (aside from the mapping of the ``base`` saltenv). This can be triggered + using the new :conf_master:`gitfs_disable_saltenv_mapping` config option. + + .. note:: + When this is disabled, only the ``base`` saltenv and any configured + using :ref:`per-saltenv configuration parameters + ` will be available. + +2. The types of refs which Salt will use as saltenvs can now be controlled. In + previous releases, branches and tags were both mapped as environments, and + individual commit SHAs could be specified as saltenvs in states (and when + caching files using :py:func:`cp.cache_file `). + Using the new :conf_master:`gitfs_ref_types` config option, the types of + refs which are used as saltenvs can be restricted. This makes it possible to + ignore all tags and use branches only, and also to keep SHAs from being made + available as saltenvs. + Configuration Option Deprecations --------------------------------- diff --git a/doc/topics/tutorials/gitfs.rst b/doc/topics/tutorials/gitfs.rst index fb23173dfb..b81d9eca14 100644 --- a/doc/topics/tutorials/gitfs.rst +++ b/doc/topics/tutorials/gitfs.rst @@ -330,6 +330,8 @@ configured gitfs remotes): * :conf_master:`gitfs_privkey` (**pygit2 only**, new in 2014.7.0) * :conf_master:`gitfs_passphrase` (**pygit2 only**, new in 2014.7.0) * :conf_master:`gitfs_refspecs` (new in 2017.7.0) +* :conf_master:`gitfs_disable_saltenv_mapping` (new in Oxygen) +* :conf_master:`gitfs_ref_types` (new in Oxygen) .. note:: pygit2 only supports disabling SSL verification in versions 0.23.2 and @@ -355,11 +357,17 @@ tremendous amount of customization. Here's some example usage: - root: other/salt - mountpoint: salt://other/bar - base: salt-base + - ref_types: + - branch - http://foo.com/baz.git: - root: salt/states - user: joe - password: mysupersecretpassword - insecure_auth: True + - disable_saltenv_mapping: True + - saltenv: + - foo: + - ref: foo - http://foo.com/quux.git: - all_saltenvs: master @@ -391,18 +399,24 @@ In the example configuration above, the following is true: will only serve files from the ``salt/states`` directory (and its subdirectories). -3. The first and fourth remotes will have files located under the root of the +3. The third remote will only serve files from branches, and not from tags or + SHAs. + +4. The fourth remote will only have two saltenvs available: ``base`` (pointed + at ``develop``), and ``foo`` (pointed at ``foo``). + +5. The first and fourth remotes will have files located under the root of the Salt fileserver namespace (``salt://``). The files from the second remote will be located under ``salt://bar``, while the files from the third remote will be located under ``salt://other/bar``. -4. The second and third remotes reference the same repository and unique names +6. The second and third remotes reference the same repository and unique names need to be declared for duplicate gitfs remotes. -5. The fourth remote overrides the default behavior of :ref:`not authenticating +7. The fourth remote overrides the default behavior of :ref:`not authenticating to insecure (non-HTTPS) remotes `. -6. Because ``all_saltenvs`` is configured for the fifth remote, files from the +8. Because ``all_saltenvs`` is configured for the fifth remote, files from the branch/tag ``master`` will appear in every fileserver environment. .. note:: diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 6f16fa6cf9..9995b95f12 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -611,7 +611,9 @@ VALID_OPTS = { 'gitfs_ssl_verify': bool, 'gitfs_global_lock': bool, 'gitfs_saltenv': list, + 'gitfs_ref_types': list, 'gitfs_refspecs': list, + 'gitfs_disable_saltenv_mapping': bool, 'hgfs_remotes': list, 'hgfs_mountpoint': str, 'hgfs_root': str, @@ -1180,7 +1182,9 @@ DEFAULT_MINION_OPTS = { 'gitfs_global_lock': True, 'gitfs_ssl_verify': True, 'gitfs_saltenv': [], + 'gitfs_ref_types': ['branch', 'tag', 'sha'], 'gitfs_refspecs': _DFLT_REFSPECS, + 'gitfs_disable_saltenv_mapping': False, 'hash_type': 'sha256', 'disable_modules': [], 'disable_returners': [], @@ -1410,7 +1414,9 @@ DEFAULT_MASTER_OPTS = { 'gitfs_global_lock': True, 'gitfs_ssl_verify': True, 'gitfs_saltenv': [], + 'gitfs_ref_types': ['branch', 'tag', 'sha'], 'gitfs_refspecs': _DFLT_REFSPECS, + 'gitfs_disable_saltenv_mapping': False, 'hgfs_remotes': [], 'hgfs_mountpoint': '', 'hgfs_root': '', diff --git a/salt/fileserver/gitfs.py b/salt/fileserver/gitfs.py index fec489857d..1182ec69be 100644 --- a/salt/fileserver/gitfs.py +++ b/salt/fileserver/gitfs.py @@ -48,9 +48,12 @@ Walkthrough `. from __future__ import absolute_import import logging -PER_REMOTE_OVERRIDES = ('base', 'mountpoint', 'root', 'ssl_verify', - 'saltenv_whitelist', 'saltenv_blacklist', - 'env_whitelist', 'env_blacklist', 'refspecs') +PER_REMOTE_OVERRIDES = ( + 'base', 'mountpoint', 'root', 'ssl_verify', + 'saltenv_whitelist', 'saltenv_blacklist', + 'env_whitelist', 'env_blacklist', 'refspecs', + 'disable_saltenv_mapping', 'ref_types' +) PER_REMOTE_ONLY = ('all_saltenvs', 'name', 'saltenv') # Auth support (auth params can be global or per-remote, too) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index 00320afb23..b80a138249 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -24,6 +24,7 @@ import salt.utils import salt.utils.itertools import salt.utils.url import salt.fileserver +from salt.config import DEFAULT_MASTER_OPTS as __DEFAULT_MASTER_OPTS from salt.utils.odict import OrderedDict from salt.utils.process import os_is_running as pid_exists from salt.exceptions import ( @@ -39,6 +40,8 @@ from salt.utils.versions import LooseVersion as _LooseVersion import salt.ext.six as six VALID_PROVIDERS = ('pygit2', 'gitpython') +VALID_REF_TYPES = __DEFAULT_MASTER_OPTS['gitfs_ref_types'] + # Optional per-remote params that can only be used on a per-remote basis, and # thus do not have defaults in salt/config.py. PER_REMOTE_ONLY = ('name',) @@ -116,11 +119,13 @@ def enforce_types(key, val): non_string_params = { 'ssl_verify': bool, 'insecure_auth': bool, + 'disable_saltenv_mapping': bool, 'env_whitelist': 'stringlist', 'env_blacklist': 'stringlist', 'saltenv_whitelist': 'stringlist', 'saltenv_blacklist': 'stringlist', 'refspecs': 'stringlist', + 'ref_types': 'stringlist', } def _find_global(key): @@ -333,6 +338,23 @@ class GitProvider(object): # params as attributes delattr(self, 'conf') + # Normalize components of the ref_types configuration and check for + # invalid configuration. + if hasattr(self, 'ref_types'): + self.ref_types = [x.lower() for x in self.ref_types] + invalid_ref_types = [x for x in self.ref_types + if x not in VALID_REF_TYPES] + if invalid_ref_types: + log.critical( + 'The following ref_types for %s remote \'%s\' are ' + 'invalid: %s. The supported values are: %s', + self.role, + self.id, + ', '.join(invalid_ref_types), + ', '.join(VALID_REF_TYPES), + ) + failhard(self.role) + if not isinstance(self.url, six.string_types): log.critical( 'Invalid %s remote \'%s\'. Remotes must be strings, you ' @@ -384,15 +406,18 @@ class GitProvider(object): else: env_set.add('base' if rname == self.base else rname) + use_branches = 'branch' in self.ref_types + use_tags = 'tag' in self.ref_types + ret = set() for ref in refs: ref = re.sub('^refs/', '', ref) rtype, rname = ref.split('/', 1) - if rtype == 'remotes': + if rtype == 'remotes' and use_branches: parted = rname.partition('/') rname = parted[2] if parted[2] else parted[0] _check_ref(ret, rname) - elif rtype == 'tags': + elif rtype == 'tags' and use_tags: _check_ref(ret, rname) return ret @@ -424,9 +449,9 @@ class GitProvider(object): return None # Return the all_saltenvs branch/tag if it is configured + per_saltenv_ref = _get_per_saltenv(tgt_env) try: all_saltenvs_ref = self.all_saltenvs - per_saltenv_ref = _get_per_saltenv(tgt_env) if per_saltenv_ref and all_saltenvs_ref != per_saltenv_ref: log.debug( 'The per-saltenv configuration has mapped the ' @@ -444,8 +469,16 @@ class GitProvider(object): if tgt_env == 'base': return self.base + elif self.disable_saltenv_mapping: + if per_saltenv_ref is None: + log.debug( + 'saltenv mapping is diabled for %s remote \'%s\' ' + 'and saltenv \'%s\' is not explicitly mapped', + self.role, self.id, tgt_env + ) + return per_saltenv_ref else: - return _get_per_saltenv(tgt_env) or tgt_env + return per_saltenv_ref or tgt_env if name in saltenv_conf: return strip_sep(saltenv_conf[name]) @@ -833,7 +866,7 @@ class GitProvider(object): if tgt_ref is None: return None - for ref_type in ('branch', 'tag', 'sha'): + for ref_type in self.ref_types: try: func_name = 'get_tree_from_{0}'.format(ref_type) func = getattr(self, func_name) @@ -1137,7 +1170,7 @@ class GitPython(GitProvider): ''' tree = self.get_tree(tgt_env) if not tree: - # Branch/tag/SHA not found + # Branch/tag/SHA not found in repo return None, None, None blob = None depth = 0 @@ -1759,6 +1792,7 @@ class Pygit2(GitProvider): def setup_callbacks(self): ''' + Assign attributes for pygit2 callbacks ''' # pygit2 radically changed fetching in 0.23.2 pygit2_version = pygit2.__version__ @@ -2490,7 +2524,8 @@ class GitFS(GitBase): ret = set() for repo in self.remotes: repo_envs = set() - repo_envs.update(repo.envs()) + if not repo.disable_saltenv_mapping: + repo_envs.update(repo.envs()) for env_list in six.itervalues(repo.saltenv_revmap): repo_envs.update(env_list) ret.update([x for x in repo_envs if repo.env_is_exposed(x)]) From 798567b766ffff6c78be114603640e974463c1a3 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 15 May 2017 13:30:34 -0500 Subject: [PATCH 4/4] Add new gitfs config options to mocked opts --- tests/unit/fileserver/test_gitfs.py | 108 +++++++++++++++------------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index 64532ecf3b..affa22eb05 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -43,31 +43,34 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): self.tmp_sock_dir = tempfile.mkdtemp(dir=TMP) return { gitfs: { - '__opts__': {'cachedir': self.tmp_cachedir, - 'sock_dir': self.tmp_sock_dir, - 'gitfs_root': 'salt', - 'fileserver_backend': ['git'], - 'gitfs_base': 'master', - 'fileserver_events': True, - 'transport': 'zeromq', - 'gitfs_mountpoint': '', - 'gitfs_saltenv': [], - 'gitfs_env_whitelist': [], - 'gitfs_env_blacklist': [], - 'gitfs_saltenv_whitelist': [], - 'gitfs_saltenv_blacklist': [], - 'gitfs_user': '', - 'gitfs_password': '', - 'gitfs_insecure_auth': False, - 'gitfs_privkey': '', - 'gitfs_pubkey': '', - 'gitfs_passphrase': '', - 'gitfs_refspecs': [ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/tags/*:refs/tags/*' - ], - 'gitfs_ssl_verify': True, - '__role': 'master' + '__opts__': { + 'cachedir': self.tmp_cachedir, + 'sock_dir': self.tmp_sock_dir, + 'gitfs_root': 'salt', + 'fileserver_backend': ['git'], + 'gitfs_base': 'master', + 'fileserver_events': True, + 'transport': 'zeromq', + 'gitfs_mountpoint': '', + 'gitfs_saltenv': [], + 'gitfs_env_whitelist': [], + 'gitfs_env_blacklist': [], + 'gitfs_saltenv_whitelist': [], + 'gitfs_saltenv_blacklist': [], + 'gitfs_user': '', + 'gitfs_password': '', + 'gitfs_insecure_auth': False, + 'gitfs_privkey': '', + 'gitfs_pubkey': '', + 'gitfs_passphrase': '', + 'gitfs_refspecs': [ + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*' + ], + 'gitfs_ssl_verify': True, + 'gitfs_disable_saltenv_mapping': False, + 'gitfs_ref_types': ['branch', 'tag', 'sha'], + '__role': 'master', } } } @@ -164,32 +167,35 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): self.tmp_repo_dir = os.path.join(TMP, 'gitfs_root') return { gitfs: { - '__opts__': {'cachedir': self.tmp_cachedir, - 'sock_dir': self.tmp_sock_dir, - 'gitfs_remotes': ['file://' + self.tmp_repo_dir], - 'gitfs_root': '', - 'fileserver_backend': ['git'], - 'gitfs_base': 'master', - 'fileserver_events': True, - 'transport': 'zeromq', - 'gitfs_mountpoint': '', - 'gitfs_saltenv': [], - 'gitfs_env_whitelist': [], - 'gitfs_env_blacklist': [], - 'gitfs_saltenv_whitelist': [], - 'gitfs_saltenv_blacklist': [], - 'gitfs_user': '', - 'gitfs_password': '', - 'gitfs_insecure_auth': False, - 'gitfs_privkey': '', - 'gitfs_pubkey': '', - 'gitfs_passphrase': '', - 'gitfs_refspecs': [ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/tags/*:refs/tags/*' - ], - 'gitfs_ssl_verify': True, - '__role': 'master' + '__opts__': { + 'cachedir': self.tmp_cachedir, + 'sock_dir': self.tmp_sock_dir, + 'gitfs_remotes': ['file://' + self.tmp_repo_dir], + 'gitfs_root': '', + 'fileserver_backend': ['git'], + 'gitfs_base': 'master', + 'fileserver_events': True, + 'transport': 'zeromq', + 'gitfs_mountpoint': '', + 'gitfs_saltenv': [], + 'gitfs_env_whitelist': [], + 'gitfs_env_blacklist': [], + 'gitfs_saltenv_whitelist': [], + 'gitfs_saltenv_blacklist': [], + 'gitfs_user': '', + 'gitfs_password': '', + 'gitfs_insecure_auth': False, + 'gitfs_privkey': '', + 'gitfs_pubkey': '', + 'gitfs_passphrase': '', + 'gitfs_refspecs': [ + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*' + ], + 'gitfs_ssl_verify': True, + 'gitfs_disable_saltenv_mapping': False, + 'gitfs_ref_types': ['branch', 'tag', 'sha'], + '__role': 'master', } } }