Add update_interval to gitfs, "fs" to several virtual names

This also adds "fs" to all of the fileserver virtualnames which did not
already have them. This both A) normalizes the usage of the non-roots
backends, and B) makes it easier for the FileserverUpdate process to get
the default update interval for each backend from the opts.
This commit is contained in:
Erik Johnson 2017-12-04 23:57:28 -06:00
parent e192b422fb
commit 2d6760ee40
No known key found for this signature in database
GPG Key ID: 5E5583C437808F3F
7 changed files with 156 additions and 45 deletions

View File

@ -57,6 +57,7 @@ _DFLT_LOG_FMT_LOGFILE = (
'%(asctime)s,%(msecs)03d [%(name)-17s:%(lineno)-4d][%(levelname)-8s][%(process)d] %(message)s'
)
_DFLT_REFSPECS = ['+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/tags/*']
DEFAULT_INTERVAL = 60
if salt.utils.platform.is_windows():
# Since an 'ipc_mode' of 'ipc' will never work on Windows due to lack of
@ -643,6 +644,16 @@ VALID_OPTS = {
# Frequency of the proxy_keep_alive, in minutes
'proxy_keep_alive_interval': int,
# Update intervals
'roots_update_interval': int,
'azurefs_update_interval': int,
'gitfs_update_interval': int,
'hgfs_update_interval': int,
'minionfs_update_interval': int,
's3fs_update_interval': int,
'svnfs_update_interval': int,
'git_pillar_base': six.string_types,
'git_pillar_branch': six.string_types,
'git_pillar_env': six.string_types,
@ -1275,6 +1286,16 @@ DEFAULT_MINION_OPTS = {
'decrypt_pillar_delimiter': ':',
'decrypt_pillar_default': 'gpg',
'decrypt_pillar_renderers': ['gpg'],
# Update intervals
'roots_update_interval': DEFAULT_INTERVAL,
'azurefs_update_interval': DEFAULT_INTERVAL,
'gitfs_update_interval': DEFAULT_INTERVAL,
'hgfs_update_interval': DEFAULT_INTERVAL,
'minionfs_update_interval': DEFAULT_INTERVAL,
's3fs_update_interval': DEFAULT_INTERVAL,
'svnfs_update_interval': DEFAULT_INTERVAL,
'git_pillar_base': 'master',
'git_pillar_branch': 'master',
'git_pillar_env': '',
@ -1525,6 +1546,16 @@ DEFAULT_MASTER_OPTS = {
'pillarenv': None,
'default_top': 'base',
'file_client': 'local',
# Update intervals
'roots_update_interval': DEFAULT_INTERVAL,
'azurefs_update_interval': DEFAULT_INTERVAL,
'gitfs_update_interval': DEFAULT_INTERVAL,
'hgfs_update_interval': DEFAULT_INTERVAL,
'minionfs_update_interval': DEFAULT_INTERVAL,
's3fs_update_interval': DEFAULT_INTERVAL,
'svnfs_update_interval': DEFAULT_INTERVAL,
'git_pillar_base': 'master',
'git_pillar_branch': 'master',
'git_pillar_env': '',
@ -3677,6 +3708,15 @@ def apply_minion_config(overrides=None,
)
opts['saltenv'] = opts['environment']
for idx, val in enumerate(opts['fileserver_backend']):
if val in ('git', 'hg', 'svn', 'minion'):
new_val = val + 'fs'
log.debug(
'Changed %s to %s in minion opts\' fileserver_backend list',
val, new_val
)
opts['fileserver_backend'][idx] = new_val
opts['__cli'] = os.path.basename(sys.argv[0])
# No ID provided. Will getfqdn save us?
@ -3843,6 +3883,15 @@ def apply_master_config(overrides=None, defaults=None):
)
opts['saltenv'] = opts['environment']
for idx, val in enumerate(opts['fileserver_backend']):
if val in ('git', 'hg', 'svn', 'minion'):
new_val = val + 'fs'
log.debug(
'Changed %s to %s in master opts\' fileserver_backend list',
val, new_val
)
opts['fileserver_backend'][idx] = new_val
if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')

View File

@ -5,13 +5,17 @@ Git Fileserver Backend
With this backend, branches and tags in a remote git repository are exposed to
salt as different environments.
To enable, add ``git`` to the :conf_master:`fileserver_backend` option in the
To enable, add ``gitfs`` to the :conf_master:`fileserver_backend` option in the
Master config file.
.. code-block:: yaml
fileserver_backend:
- git
- gitfs
.. note::
``git`` also works here. Prior to the Oxygen release, *only* ``git``
would work.
The Git fileserver backend supports both pygit2_ and GitPython_, to provide the
Python interface to git. If both are present, the order of preference for which
@ -52,7 +56,7 @@ PER_REMOTE_OVERRIDES = (
'base', 'mountpoint', 'root', 'ssl_verify',
'saltenv_whitelist', 'saltenv_blacklist',
'env_whitelist', 'env_blacklist', 'refspecs',
'disable_saltenv_mapping', 'ref_types'
'disable_saltenv_mapping', 'ref_types', 'update_interval',
)
PER_REMOTE_ONLY = ('all_saltenvs', 'name', 'saltenv')
@ -68,7 +72,7 @@ from salt.exceptions import FileserverConfigError
log = logging.getLogger(__name__)
# Define the module's virtual name
__virtualname__ = 'git'
__virtualname__ = 'gitfs'
def _gitfs(init_remotes=True):
@ -122,11 +126,18 @@ def lock(remote=None):
return _gitfs().lock(remote=remote)
def update():
def update(remotes=None):
'''
Execute a git fetch on all of the repos
'''
_gitfs().update()
_gitfs().update(remotes)
def update_intervals():
'''
Returns the update intervals for each configured remote
'''
return _gitfs().update_intervals()
def envs(ignore_cache=False):

View File

@ -2,13 +2,17 @@
'''
Mercurial Fileserver Backend
To enable, add ``hg`` to the :conf_master:`fileserver_backend` option in the
To enable, add ``hgfs`` to the :conf_master:`fileserver_backend` option in the
Master config file.
.. code-block:: yaml
fileserver_backend:
- hg
- hgfs
.. note::
``hg`` also works here. Prior to the Oxygen release, *only* ``hg`` would
work.
After enabling this backend, branches, bookmarks, and tags in a remote
mercurial repository are exposed to salt as different environments. This

View File

@ -6,15 +6,19 @@ The :mod:`cp.push <salt.modules.cp.push>` function allows Minions to push files
up to the Master. Using this backend, these pushed files are exposed to other
Minions via the Salt fileserver.
To enable minionfs, :conf_master:`file_recv` needs to be set to ``True`` in
the master config file (otherwise :mod:`cp.push <salt.modules.cp.push>` will
not be allowed to push files to the Master), and ``minion`` must be added to
the :conf_master:`fileserver_backends` list.
To enable minionfs, :conf_master:`file_recv` needs to be set to ``True`` in the
master config file (otherwise :mod:`cp.push <salt.modules.cp.push>` will not be
allowed to push files to the Master), and ``minionfs`` must be added to the
:conf_master:`fileserver_backends` list.
.. code-block:: yaml
fileserver_backend:
- minion
- minionfs
.. note::
``minion`` also works here. Prior to the Oxygen release, *only* ``minion``
would work.
Other minionfs settings include: :conf_master:`minionfs_whitelist`,
:conf_master:`minionfs_blacklist`, :conf_master:`minionfs_mountpoint`, and
@ -46,7 +50,7 @@ log = logging.getLogger(__name__)
# Define the module's virtual name
__virtualname__ = 'minion'
__virtualname__ = 'minionfs'
def __virtual__():

View File

@ -4,13 +4,17 @@ Subversion Fileserver Backend
After enabling this backend, branches and tags in a remote subversion
repository are exposed to salt as different environments. To enable this
backend, add ``svn`` to the :conf_master:`fileserver_backend` option in the
backend, add ``svnfs`` to the :conf_master:`fileserver_backend` option in the
Master config file.
.. code-block:: yaml
fileserver_backend:
- svn
- svnfs
.. note::
``svn`` also works here. Prior to the Oxygen release, *only* ``svn`` would
work.
This backend assumes a standard svn layout with directories for ``branches``,
``tags``, and ``trunk``, at the repository root.

View File

@ -138,6 +138,7 @@ def enforce_types(key, val):
'saltenv_blacklist': 'stringlist',
'refspecs': 'stringlist',
'ref_types': 'stringlist',
'update_interval': int,
}
def _find_global(key):
@ -160,14 +161,21 @@ def enforce_types(key, val):
return six.text_type(val)
expected = non_string_params[key]
if expected is bool:
return val
elif expected == 'stringlist':
if expected == 'stringlist':
if not isinstance(val, (six.string_types, list)):
val = six.text_type(val)
if isinstance(val, six.string_types):
return [x.strip() for x in val.split(',')]
return [six.text_type(x) for x in val]
else:
try:
return expected(val)
except Exception as exc:
log.error(
'Failed to enforce type for key=%s with val=%s, falling back '
'to a string', key, val
)
return six.text_type(val)
def failhard(role):
@ -398,7 +406,8 @@ class GitProvider(object):
hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5'))
if six.PY3:
# We loaded this data from yaml configuration files, so, its safe to use UTF-8
# We loaded this data from yaml configuration files, so, its safe
# to use UTF-8
self.hash = hash_type(self.id.encode('utf-8')).hexdigest()
else:
self.hash = hash_type(self.id).hexdigest()
@ -2166,16 +2175,15 @@ class GitBase(object):
cachedir_map = {}
for repo in self.remotes:
cachedir_map.setdefault(repo.cachedir, []).append(repo.id)
collisions = [x for x in cachedir_map if len(cachedir_map[x]) > 1]
if collisions:
for dirname in collisions:
log.critical(
'The following {0} remotes have conflicting cachedirs: '
'{1}. Resolve this using a per-remote parameter called '
'\'name\'.'.format(
self.role,
', '.join(cachedir_map[dirname])
)
'The following %s remotes have conflicting cachedirs: '
'%s. Resolve this using a per-remote parameter called '
'\'name\'.',
self.role, ', '.join(cachedir_map[dirname])
)
failhard(self.role)
@ -2262,27 +2270,39 @@ class GitBase(object):
errors.extend(failed)
return cleared, errors
def fetch_remotes(self):
def fetch_remotes(self, remotes=None):
'''
Fetch all remotes and return a boolean to let the calling function know
whether or not any remotes were updated in the process of fetching
'''
if remotes is None:
remotes = []
elif not isinstance(remotes, list):
log.error(
'Invalid \'remotes\' argument (%s) for fetch_remotes. '
'Must be a list of strings', remotes
)
remotes = []
changed = False
for repo in self.remotes:
try:
if repo.fetch():
# We can't just use the return value from repo.fetch()
# because the data could still have changed if old remotes
# were cleared above. Additionally, we're running this in a
# loop and later remotes without changes would override
# this value and make it incorrect.
changed = True
except Exception as exc:
log.error(
'Exception caught while fetching %s remote \'%s\': %s',
self.role, repo.id, exc,
exc_info=True
)
name = getattr(repo, 'name', None)
if not remotes or (repo.id, name) in remotes:
try:
if repo.fetch():
# We can't just use the return value from repo.fetch()
# because the data could still have changed if old
# remotes were cleared above. Additionally, we're
# running this in a loop and later remotes without
# changes would override this value and make it
# incorrect.
changed = True
except Exception as exc:
log.error(
'Exception caught while fetching %s remote \'%s\': %s',
self.role, repo.id, exc,
exc_info=True
)
return changed
def lock(self, remote=None):
@ -2307,8 +2327,13 @@ class GitBase(object):
errors.extend(failed)
return locked, errors
def update(self):
def update(self, remotes=None):
'''
.. versionchanged:: Oxygen
The remotes argument was added. This being a list of remote URLs,
it will only update matching remotes. This actually matches on
repo.id
Execute a git fetch on all of the repos and perform maintenance on the
fileserver cache.
'''
@ -2317,7 +2342,7 @@ class GitBase(object):
'backend': 'gitfs'}
data['changed'] = self.clear_old_remotes()
if self.fetch_remotes():
if self.fetch_remotes(remotes=remotes):
data['changed'] = True
# A masterless minion will need a new env cache file even if no changes
@ -2358,6 +2383,18 @@ class GitBase(object):
# Hash file won't exist if no files have yet been served up
pass
def update_intervals(self):
'''
Returns a dictionary mapping remote IDs to their intervals, designed to
be used for variable update intervals in salt.master.FileserverUpdate.
A remote's ID is defined here as a tuple of the GitPython/Pygit2
object's "id" and "name" attributes, with None being assumed as the
"name" value if the attribute is not present.
'''
return {(repo.id, getattr(repo, 'name', None)): repo.update_interval
for repo in self.remotes}
def verify_provider(self):
'''
Determine which provider to use

View File

@ -63,7 +63,7 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin):
'cachedir': self.tmp_cachedir,
'sock_dir': TMP_SOCK_DIR,
'gitfs_root': 'salt',
'fileserver_backend': ['git'],
'fileserver_backend': ['gitfs'],
'gitfs_base': 'master',
'fileserver_events': True,
'transport': 'zeromq',
@ -86,6 +86,7 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin):
'gitfs_ssl_verify': True,
'gitfs_disable_saltenv_mapping': False,
'gitfs_ref_types': ['branch', 'tag', 'sha'],
'gitfs_update_interval': 60,
'__role': 'master',
}
}
@ -195,7 +196,7 @@ class GitFSTest(TestCase, LoaderModuleMockMixin):
'sock_dir': TMP_SOCK_DIR,
'gitfs_remotes': ['file://' + TMP_REPO_DIR],
'gitfs_root': '',
'fileserver_backend': ['git'],
'fileserver_backend': ['gitfs'],
'gitfs_base': 'master',
'fileserver_events': True,
'transport': 'zeromq',
@ -218,6 +219,7 @@ class GitFSTest(TestCase, LoaderModuleMockMixin):
'gitfs_ssl_verify': True,
'gitfs_disable_saltenv_mapping': False,
'gitfs_ref_types': ['branch', 'tag', 'sha'],
'gitfs_update_interval': 60,
'__role': 'master',
}
}