diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 71f0af391d..e42a1c2297 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -57,6 +57,7 @@ _DFLT_LOG_FMT_LOGFILE = ( '%(asctime)s,%(msecs)03d [%(name)-17s:%(lineno)-4d][%(levelname)-8s][%(process)d] %(message)s' ) _DFLT_REFSPECS = ['+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/tags/*'] +DEFAULT_INTERVAL = 60 if salt.utils.platform.is_windows(): # Since an 'ipc_mode' of 'ipc' will never work on Windows due to lack of @@ -643,6 +644,16 @@ VALID_OPTS = { # Frequency of the proxy_keep_alive, in minutes 'proxy_keep_alive_interval': int, + + # Update intervals + 'roots_update_interval': int, + 'azurefs_update_interval': int, + 'gitfs_update_interval': int, + 'hgfs_update_interval': int, + 'minionfs_update_interval': int, + 's3fs_update_interval': int, + 'svnfs_update_interval': int, + 'git_pillar_base': six.string_types, 'git_pillar_branch': six.string_types, 'git_pillar_env': six.string_types, @@ -1275,6 +1286,16 @@ DEFAULT_MINION_OPTS = { 'decrypt_pillar_delimiter': ':', 'decrypt_pillar_default': 'gpg', 'decrypt_pillar_renderers': ['gpg'], + + # Update intervals + 'roots_update_interval': DEFAULT_INTERVAL, + 'azurefs_update_interval': DEFAULT_INTERVAL, + 'gitfs_update_interval': DEFAULT_INTERVAL, + 'hgfs_update_interval': DEFAULT_INTERVAL, + 'minionfs_update_interval': DEFAULT_INTERVAL, + 's3fs_update_interval': DEFAULT_INTERVAL, + 'svnfs_update_interval': DEFAULT_INTERVAL, + 'git_pillar_base': 'master', 'git_pillar_branch': 'master', 'git_pillar_env': '', @@ -1525,6 +1546,16 @@ DEFAULT_MASTER_OPTS = { 'pillarenv': None, 'default_top': 'base', 'file_client': 'local', + + # Update intervals + 'roots_update_interval': DEFAULT_INTERVAL, + 'azurefs_update_interval': DEFAULT_INTERVAL, + 'gitfs_update_interval': DEFAULT_INTERVAL, + 'hgfs_update_interval': DEFAULT_INTERVAL, + 'minionfs_update_interval': DEFAULT_INTERVAL, + 's3fs_update_interval': DEFAULT_INTERVAL, + 'svnfs_update_interval': DEFAULT_INTERVAL, + 'git_pillar_base': 'master', 'git_pillar_branch': 'master', 'git_pillar_env': '', @@ -3677,6 +3708,15 @@ def apply_minion_config(overrides=None, ) opts['saltenv'] = opts['environment'] + for idx, val in enumerate(opts['fileserver_backend']): + if val in ('git', 'hg', 'svn', 'minion'): + new_val = val + 'fs' + log.debug( + 'Changed %s to %s in minion opts\' fileserver_backend list', + val, new_val + ) + opts['fileserver_backend'][idx] = new_val + opts['__cli'] = os.path.basename(sys.argv[0]) # No ID provided. Will getfqdn save us? @@ -3843,6 +3883,15 @@ def apply_master_config(overrides=None, defaults=None): ) opts['saltenv'] = opts['environment'] + for idx, val in enumerate(opts['fileserver_backend']): + if val in ('git', 'hg', 'svn', 'minion'): + new_val = val + 'fs' + log.debug( + 'Changed %s to %s in master opts\' fileserver_backend list', + val, new_val + ) + opts['fileserver_backend'][idx] = new_val + if len(opts['sock_dir']) > len(opts['cachedir']) + 10: opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix') diff --git a/salt/fileserver/gitfs.py b/salt/fileserver/gitfs.py index 477e6c40b2..c7eef46ba2 100644 --- a/salt/fileserver/gitfs.py +++ b/salt/fileserver/gitfs.py @@ -5,13 +5,17 @@ Git Fileserver Backend With this backend, branches and tags in a remote git repository are exposed to salt as different environments. -To enable, add ``git`` to the :conf_master:`fileserver_backend` option in the +To enable, add ``gitfs`` to the :conf_master:`fileserver_backend` option in the Master config file. .. code-block:: yaml fileserver_backend: - - git + - gitfs + +.. note:: + ``git`` also works here. Prior to the Oxygen release, *only* ``git`` + would work. The Git fileserver backend supports both pygit2_ and GitPython_, to provide the Python interface to git. If both are present, the order of preference for which @@ -52,7 +56,7 @@ PER_REMOTE_OVERRIDES = ( 'base', 'mountpoint', 'root', 'ssl_verify', 'saltenv_whitelist', 'saltenv_blacklist', 'env_whitelist', 'env_blacklist', 'refspecs', - 'disable_saltenv_mapping', 'ref_types' + 'disable_saltenv_mapping', 'ref_types', 'update_interval', ) PER_REMOTE_ONLY = ('all_saltenvs', 'name', 'saltenv') @@ -68,7 +72,7 @@ from salt.exceptions import FileserverConfigError log = logging.getLogger(__name__) # Define the module's virtual name -__virtualname__ = 'git' +__virtualname__ = 'gitfs' def _gitfs(init_remotes=True): @@ -122,11 +126,18 @@ def lock(remote=None): return _gitfs().lock(remote=remote) -def update(): +def update(remotes=None): ''' Execute a git fetch on all of the repos ''' - _gitfs().update() + _gitfs().update(remotes) + + +def update_intervals(): + ''' + Returns the update intervals for each configured remote + ''' + return _gitfs().update_intervals() def envs(ignore_cache=False): diff --git a/salt/fileserver/hgfs.py b/salt/fileserver/hgfs.py index bc7d2c32e7..a632774886 100644 --- a/salt/fileserver/hgfs.py +++ b/salt/fileserver/hgfs.py @@ -2,13 +2,17 @@ ''' Mercurial Fileserver Backend -To enable, add ``hg`` to the :conf_master:`fileserver_backend` option in the +To enable, add ``hgfs`` to the :conf_master:`fileserver_backend` option in the Master config file. .. code-block:: yaml fileserver_backend: - - hg + - hgfs + +.. note:: + ``hg`` also works here. Prior to the Oxygen release, *only* ``hg`` would + work. After enabling this backend, branches, bookmarks, and tags in a remote mercurial repository are exposed to salt as different environments. This diff --git a/salt/fileserver/minionfs.py b/salt/fileserver/minionfs.py index 89a52aa538..6d9be0b5a6 100644 --- a/salt/fileserver/minionfs.py +++ b/salt/fileserver/minionfs.py @@ -6,15 +6,19 @@ The :mod:`cp.push ` function allows Minions to push files up to the Master. Using this backend, these pushed files are exposed to other Minions via the Salt fileserver. -To enable minionfs, :conf_master:`file_recv` needs to be set to ``True`` in -the master config file (otherwise :mod:`cp.push ` will -not be allowed to push files to the Master), and ``minion`` must be added to -the :conf_master:`fileserver_backends` list. +To enable minionfs, :conf_master:`file_recv` needs to be set to ``True`` in the +master config file (otherwise :mod:`cp.push ` will not be +allowed to push files to the Master), and ``minionfs`` must be added to the +:conf_master:`fileserver_backends` list. .. code-block:: yaml fileserver_backend: - - minion + - minionfs + +.. note:: + ``minion`` also works here. Prior to the Oxygen release, *only* ``minion`` + would work. Other minionfs settings include: :conf_master:`minionfs_whitelist`, :conf_master:`minionfs_blacklist`, :conf_master:`minionfs_mountpoint`, and @@ -46,7 +50,7 @@ log = logging.getLogger(__name__) # Define the module's virtual name -__virtualname__ = 'minion' +__virtualname__ = 'minionfs' def __virtual__(): diff --git a/salt/fileserver/svnfs.py b/salt/fileserver/svnfs.py index a4b3bab204..7ecd4a474b 100644 --- a/salt/fileserver/svnfs.py +++ b/salt/fileserver/svnfs.py @@ -4,13 +4,17 @@ Subversion Fileserver Backend After enabling this backend, branches and tags in a remote subversion repository are exposed to salt as different environments. To enable this -backend, add ``svn`` to the :conf_master:`fileserver_backend` option in the +backend, add ``svnfs`` to the :conf_master:`fileserver_backend` option in the Master config file. .. code-block:: yaml fileserver_backend: - - svn + - svnfs + +.. note:: + ``svn`` also works here. Prior to the Oxygen release, *only* ``svn`` would + work. This backend assumes a standard svn layout with directories for ``branches``, ``tags``, and ``trunk``, at the repository root. diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index bcf2a2e283..9013bab374 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -138,6 +138,7 @@ def enforce_types(key, val): 'saltenv_blacklist': 'stringlist', 'refspecs': 'stringlist', 'ref_types': 'stringlist', + 'update_interval': int, } def _find_global(key): @@ -160,14 +161,21 @@ def enforce_types(key, val): return six.text_type(val) expected = non_string_params[key] - if expected is bool: - return val - elif expected == 'stringlist': + if expected == 'stringlist': if not isinstance(val, (six.string_types, list)): val = six.text_type(val) if isinstance(val, six.string_types): return [x.strip() for x in val.split(',')] return [six.text_type(x) for x in val] + else: + try: + return expected(val) + except Exception as exc: + log.error( + 'Failed to enforce type for key=%s with val=%s, falling back ' + 'to a string', key, val + ) + return six.text_type(val) def failhard(role): @@ -398,7 +406,8 @@ class GitProvider(object): hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5')) if six.PY3: - # We loaded this data from yaml configuration files, so, its safe to use UTF-8 + # We loaded this data from yaml configuration files, so, its safe + # to use UTF-8 self.hash = hash_type(self.id.encode('utf-8')).hexdigest() else: self.hash = hash_type(self.id).hexdigest() @@ -2166,16 +2175,15 @@ class GitBase(object): cachedir_map = {} for repo in self.remotes: cachedir_map.setdefault(repo.cachedir, []).append(repo.id) + collisions = [x for x in cachedir_map if len(cachedir_map[x]) > 1] if collisions: for dirname in collisions: log.critical( - 'The following {0} remotes have conflicting cachedirs: ' - '{1}. Resolve this using a per-remote parameter called ' - '\'name\'.'.format( - self.role, - ', '.join(cachedir_map[dirname]) - ) + 'The following %s remotes have conflicting cachedirs: ' + '%s. Resolve this using a per-remote parameter called ' + '\'name\'.', + self.role, ', '.join(cachedir_map[dirname]) ) failhard(self.role) @@ -2262,27 +2270,39 @@ class GitBase(object): errors.extend(failed) return cleared, errors - def fetch_remotes(self): + def fetch_remotes(self, remotes=None): ''' Fetch all remotes and return a boolean to let the calling function know whether or not any remotes were updated in the process of fetching ''' + if remotes is None: + remotes = [] + elif not isinstance(remotes, list): + log.error( + 'Invalid \'remotes\' argument (%s) for fetch_remotes. ' + 'Must be a list of strings', remotes + ) + remotes = [] + changed = False for repo in self.remotes: - try: - if repo.fetch(): - # We can't just use the return value from repo.fetch() - # because the data could still have changed if old remotes - # were cleared above. Additionally, we're running this in a - # loop and later remotes without changes would override - # this value and make it incorrect. - changed = True - except Exception as exc: - log.error( - 'Exception caught while fetching %s remote \'%s\': %s', - self.role, repo.id, exc, - exc_info=True - ) + name = getattr(repo, 'name', None) + if not remotes or (repo.id, name) in remotes: + try: + if repo.fetch(): + # We can't just use the return value from repo.fetch() + # because the data could still have changed if old + # remotes were cleared above. Additionally, we're + # running this in a loop and later remotes without + # changes would override this value and make it + # incorrect. + changed = True + except Exception as exc: + log.error( + 'Exception caught while fetching %s remote \'%s\': %s', + self.role, repo.id, exc, + exc_info=True + ) return changed def lock(self, remote=None): @@ -2307,8 +2327,13 @@ class GitBase(object): errors.extend(failed) return locked, errors - def update(self): + def update(self, remotes=None): ''' + .. versionchanged:: Oxygen + The remotes argument was added. This being a list of remote URLs, + it will only update matching remotes. This actually matches on + repo.id + Execute a git fetch on all of the repos and perform maintenance on the fileserver cache. ''' @@ -2317,7 +2342,7 @@ class GitBase(object): 'backend': 'gitfs'} data['changed'] = self.clear_old_remotes() - if self.fetch_remotes(): + if self.fetch_remotes(remotes=remotes): data['changed'] = True # A masterless minion will need a new env cache file even if no changes @@ -2358,6 +2383,18 @@ class GitBase(object): # Hash file won't exist if no files have yet been served up pass + def update_intervals(self): + ''' + Returns a dictionary mapping remote IDs to their intervals, designed to + be used for variable update intervals in salt.master.FileserverUpdate. + + A remote's ID is defined here as a tuple of the GitPython/Pygit2 + object's "id" and "name" attributes, with None being assumed as the + "name" value if the attribute is not present. + ''' + return {(repo.id, getattr(repo, 'name', None)): repo.update_interval + for repo in self.remotes} + def verify_provider(self): ''' Determine which provider to use diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index bc0e0aca6d..6a671cf540 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -63,7 +63,7 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): 'cachedir': self.tmp_cachedir, 'sock_dir': TMP_SOCK_DIR, 'gitfs_root': 'salt', - 'fileserver_backend': ['git'], + 'fileserver_backend': ['gitfs'], 'gitfs_base': 'master', 'fileserver_events': True, 'transport': 'zeromq', @@ -86,6 +86,7 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): 'gitfs_ssl_verify': True, 'gitfs_disable_saltenv_mapping': False, 'gitfs_ref_types': ['branch', 'tag', 'sha'], + 'gitfs_update_interval': 60, '__role': 'master', } } @@ -195,7 +196,7 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): 'sock_dir': TMP_SOCK_DIR, 'gitfs_remotes': ['file://' + TMP_REPO_DIR], 'gitfs_root': '', - 'fileserver_backend': ['git'], + 'fileserver_backend': ['gitfs'], 'gitfs_base': 'master', 'fileserver_events': True, 'transport': 'zeromq', @@ -218,6 +219,7 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): 'gitfs_ssl_verify': True, 'gitfs_disable_saltenv_mapping': False, 'gitfs_ref_types': ['branch', 'tag', 'sha'], + 'gitfs_update_interval': 60, '__role': 'master', } }