mirror of
https://github.com/valitydev/salt.git
synced 2024-11-07 00:55:19 +00:00
Add update_interval to gitfs, "fs" to several virtual names
This also adds "fs" to all of the fileserver virtualnames which did not already have them. This both A) normalizes the usage of the non-roots backends, and B) makes it easier for the FileserverUpdate process to get the default update interval for each backend from the opts.
This commit is contained in:
parent
e192b422fb
commit
2d6760ee40
@ -57,6 +57,7 @@ _DFLT_LOG_FMT_LOGFILE = (
|
||||
'%(asctime)s,%(msecs)03d [%(name)-17s:%(lineno)-4d][%(levelname)-8s][%(process)d] %(message)s'
|
||||
)
|
||||
_DFLT_REFSPECS = ['+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/tags/*']
|
||||
DEFAULT_INTERVAL = 60
|
||||
|
||||
if salt.utils.platform.is_windows():
|
||||
# Since an 'ipc_mode' of 'ipc' will never work on Windows due to lack of
|
||||
@ -643,6 +644,16 @@ VALID_OPTS = {
|
||||
|
||||
# Frequency of the proxy_keep_alive, in minutes
|
||||
'proxy_keep_alive_interval': int,
|
||||
|
||||
# Update intervals
|
||||
'roots_update_interval': int,
|
||||
'azurefs_update_interval': int,
|
||||
'gitfs_update_interval': int,
|
||||
'hgfs_update_interval': int,
|
||||
'minionfs_update_interval': int,
|
||||
's3fs_update_interval': int,
|
||||
'svnfs_update_interval': int,
|
||||
|
||||
'git_pillar_base': six.string_types,
|
||||
'git_pillar_branch': six.string_types,
|
||||
'git_pillar_env': six.string_types,
|
||||
@ -1275,6 +1286,16 @@ DEFAULT_MINION_OPTS = {
|
||||
'decrypt_pillar_delimiter': ':',
|
||||
'decrypt_pillar_default': 'gpg',
|
||||
'decrypt_pillar_renderers': ['gpg'],
|
||||
|
||||
# Update intervals
|
||||
'roots_update_interval': DEFAULT_INTERVAL,
|
||||
'azurefs_update_interval': DEFAULT_INTERVAL,
|
||||
'gitfs_update_interval': DEFAULT_INTERVAL,
|
||||
'hgfs_update_interval': DEFAULT_INTERVAL,
|
||||
'minionfs_update_interval': DEFAULT_INTERVAL,
|
||||
's3fs_update_interval': DEFAULT_INTERVAL,
|
||||
'svnfs_update_interval': DEFAULT_INTERVAL,
|
||||
|
||||
'git_pillar_base': 'master',
|
||||
'git_pillar_branch': 'master',
|
||||
'git_pillar_env': '',
|
||||
@ -1525,6 +1546,16 @@ DEFAULT_MASTER_OPTS = {
|
||||
'pillarenv': None,
|
||||
'default_top': 'base',
|
||||
'file_client': 'local',
|
||||
|
||||
# Update intervals
|
||||
'roots_update_interval': DEFAULT_INTERVAL,
|
||||
'azurefs_update_interval': DEFAULT_INTERVAL,
|
||||
'gitfs_update_interval': DEFAULT_INTERVAL,
|
||||
'hgfs_update_interval': DEFAULT_INTERVAL,
|
||||
'minionfs_update_interval': DEFAULT_INTERVAL,
|
||||
's3fs_update_interval': DEFAULT_INTERVAL,
|
||||
'svnfs_update_interval': DEFAULT_INTERVAL,
|
||||
|
||||
'git_pillar_base': 'master',
|
||||
'git_pillar_branch': 'master',
|
||||
'git_pillar_env': '',
|
||||
@ -3677,6 +3708,15 @@ def apply_minion_config(overrides=None,
|
||||
)
|
||||
opts['saltenv'] = opts['environment']
|
||||
|
||||
for idx, val in enumerate(opts['fileserver_backend']):
|
||||
if val in ('git', 'hg', 'svn', 'minion'):
|
||||
new_val = val + 'fs'
|
||||
log.debug(
|
||||
'Changed %s to %s in minion opts\' fileserver_backend list',
|
||||
val, new_val
|
||||
)
|
||||
opts['fileserver_backend'][idx] = new_val
|
||||
|
||||
opts['__cli'] = os.path.basename(sys.argv[0])
|
||||
|
||||
# No ID provided. Will getfqdn save us?
|
||||
@ -3843,6 +3883,15 @@ def apply_master_config(overrides=None, defaults=None):
|
||||
)
|
||||
opts['saltenv'] = opts['environment']
|
||||
|
||||
for idx, val in enumerate(opts['fileserver_backend']):
|
||||
if val in ('git', 'hg', 'svn', 'minion'):
|
||||
new_val = val + 'fs'
|
||||
log.debug(
|
||||
'Changed %s to %s in master opts\' fileserver_backend list',
|
||||
val, new_val
|
||||
)
|
||||
opts['fileserver_backend'][idx] = new_val
|
||||
|
||||
if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
|
||||
opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')
|
||||
|
||||
|
@ -5,13 +5,17 @@ Git Fileserver Backend
|
||||
With this backend, branches and tags in a remote git repository are exposed to
|
||||
salt as different environments.
|
||||
|
||||
To enable, add ``git`` to the :conf_master:`fileserver_backend` option in the
|
||||
To enable, add ``gitfs`` to the :conf_master:`fileserver_backend` option in the
|
||||
Master config file.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
fileserver_backend:
|
||||
- git
|
||||
- gitfs
|
||||
|
||||
.. note::
|
||||
``git`` also works here. Prior to the Oxygen release, *only* ``git``
|
||||
would work.
|
||||
|
||||
The Git fileserver backend supports both pygit2_ and GitPython_, to provide the
|
||||
Python interface to git. If both are present, the order of preference for which
|
||||
@ -52,7 +56,7 @@ PER_REMOTE_OVERRIDES = (
|
||||
'base', 'mountpoint', 'root', 'ssl_verify',
|
||||
'saltenv_whitelist', 'saltenv_blacklist',
|
||||
'env_whitelist', 'env_blacklist', 'refspecs',
|
||||
'disable_saltenv_mapping', 'ref_types'
|
||||
'disable_saltenv_mapping', 'ref_types', 'update_interval',
|
||||
)
|
||||
PER_REMOTE_ONLY = ('all_saltenvs', 'name', 'saltenv')
|
||||
|
||||
@ -68,7 +72,7 @@ from salt.exceptions import FileserverConfigError
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Define the module's virtual name
|
||||
__virtualname__ = 'git'
|
||||
__virtualname__ = 'gitfs'
|
||||
|
||||
|
||||
def _gitfs(init_remotes=True):
|
||||
@ -122,11 +126,18 @@ def lock(remote=None):
|
||||
return _gitfs().lock(remote=remote)
|
||||
|
||||
|
||||
def update():
|
||||
def update(remotes=None):
|
||||
'''
|
||||
Execute a git fetch on all of the repos
|
||||
'''
|
||||
_gitfs().update()
|
||||
_gitfs().update(remotes)
|
||||
|
||||
|
||||
def update_intervals():
|
||||
'''
|
||||
Returns the update intervals for each configured remote
|
||||
'''
|
||||
return _gitfs().update_intervals()
|
||||
|
||||
|
||||
def envs(ignore_cache=False):
|
||||
|
@ -2,13 +2,17 @@
|
||||
'''
|
||||
Mercurial Fileserver Backend
|
||||
|
||||
To enable, add ``hg`` to the :conf_master:`fileserver_backend` option in the
|
||||
To enable, add ``hgfs`` to the :conf_master:`fileserver_backend` option in the
|
||||
Master config file.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
fileserver_backend:
|
||||
- hg
|
||||
- hgfs
|
||||
|
||||
.. note::
|
||||
``hg`` also works here. Prior to the Oxygen release, *only* ``hg`` would
|
||||
work.
|
||||
|
||||
After enabling this backend, branches, bookmarks, and tags in a remote
|
||||
mercurial repository are exposed to salt as different environments. This
|
||||
|
@ -6,15 +6,19 @@ The :mod:`cp.push <salt.modules.cp.push>` function allows Minions to push files
|
||||
up to the Master. Using this backend, these pushed files are exposed to other
|
||||
Minions via the Salt fileserver.
|
||||
|
||||
To enable minionfs, :conf_master:`file_recv` needs to be set to ``True`` in
|
||||
the master config file (otherwise :mod:`cp.push <salt.modules.cp.push>` will
|
||||
not be allowed to push files to the Master), and ``minion`` must be added to
|
||||
the :conf_master:`fileserver_backends` list.
|
||||
To enable minionfs, :conf_master:`file_recv` needs to be set to ``True`` in the
|
||||
master config file (otherwise :mod:`cp.push <salt.modules.cp.push>` will not be
|
||||
allowed to push files to the Master), and ``minionfs`` must be added to the
|
||||
:conf_master:`fileserver_backends` list.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
fileserver_backend:
|
||||
- minion
|
||||
- minionfs
|
||||
|
||||
.. note::
|
||||
``minion`` also works here. Prior to the Oxygen release, *only* ``minion``
|
||||
would work.
|
||||
|
||||
Other minionfs settings include: :conf_master:`minionfs_whitelist`,
|
||||
:conf_master:`minionfs_blacklist`, :conf_master:`minionfs_mountpoint`, and
|
||||
@ -46,7 +50,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Define the module's virtual name
|
||||
__virtualname__ = 'minion'
|
||||
__virtualname__ = 'minionfs'
|
||||
|
||||
|
||||
def __virtual__():
|
||||
|
@ -4,13 +4,17 @@ Subversion Fileserver Backend
|
||||
|
||||
After enabling this backend, branches and tags in a remote subversion
|
||||
repository are exposed to salt as different environments. To enable this
|
||||
backend, add ``svn`` to the :conf_master:`fileserver_backend` option in the
|
||||
backend, add ``svnfs`` to the :conf_master:`fileserver_backend` option in the
|
||||
Master config file.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
fileserver_backend:
|
||||
- svn
|
||||
- svnfs
|
||||
|
||||
.. note::
|
||||
``svn`` also works here. Prior to the Oxygen release, *only* ``svn`` would
|
||||
work.
|
||||
|
||||
This backend assumes a standard svn layout with directories for ``branches``,
|
||||
``tags``, and ``trunk``, at the repository root.
|
||||
|
@ -138,6 +138,7 @@ def enforce_types(key, val):
|
||||
'saltenv_blacklist': 'stringlist',
|
||||
'refspecs': 'stringlist',
|
||||
'ref_types': 'stringlist',
|
||||
'update_interval': int,
|
||||
}
|
||||
|
||||
def _find_global(key):
|
||||
@ -160,14 +161,21 @@ def enforce_types(key, val):
|
||||
return six.text_type(val)
|
||||
|
||||
expected = non_string_params[key]
|
||||
if expected is bool:
|
||||
return val
|
||||
elif expected == 'stringlist':
|
||||
if expected == 'stringlist':
|
||||
if not isinstance(val, (six.string_types, list)):
|
||||
val = six.text_type(val)
|
||||
if isinstance(val, six.string_types):
|
||||
return [x.strip() for x in val.split(',')]
|
||||
return [six.text_type(x) for x in val]
|
||||
else:
|
||||
try:
|
||||
return expected(val)
|
||||
except Exception as exc:
|
||||
log.error(
|
||||
'Failed to enforce type for key=%s with val=%s, falling back '
|
||||
'to a string', key, val
|
||||
)
|
||||
return six.text_type(val)
|
||||
|
||||
|
||||
def failhard(role):
|
||||
@ -398,7 +406,8 @@ class GitProvider(object):
|
||||
|
||||
hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5'))
|
||||
if six.PY3:
|
||||
# We loaded this data from yaml configuration files, so, its safe to use UTF-8
|
||||
# We loaded this data from yaml configuration files, so, its safe
|
||||
# to use UTF-8
|
||||
self.hash = hash_type(self.id.encode('utf-8')).hexdigest()
|
||||
else:
|
||||
self.hash = hash_type(self.id).hexdigest()
|
||||
@ -2166,16 +2175,15 @@ class GitBase(object):
|
||||
cachedir_map = {}
|
||||
for repo in self.remotes:
|
||||
cachedir_map.setdefault(repo.cachedir, []).append(repo.id)
|
||||
|
||||
collisions = [x for x in cachedir_map if len(cachedir_map[x]) > 1]
|
||||
if collisions:
|
||||
for dirname in collisions:
|
||||
log.critical(
|
||||
'The following {0} remotes have conflicting cachedirs: '
|
||||
'{1}. Resolve this using a per-remote parameter called '
|
||||
'\'name\'.'.format(
|
||||
self.role,
|
||||
', '.join(cachedir_map[dirname])
|
||||
)
|
||||
'The following %s remotes have conflicting cachedirs: '
|
||||
'%s. Resolve this using a per-remote parameter called '
|
||||
'\'name\'.',
|
||||
self.role, ', '.join(cachedir_map[dirname])
|
||||
)
|
||||
failhard(self.role)
|
||||
|
||||
@ -2262,27 +2270,39 @@ class GitBase(object):
|
||||
errors.extend(failed)
|
||||
return cleared, errors
|
||||
|
||||
def fetch_remotes(self):
|
||||
def fetch_remotes(self, remotes=None):
|
||||
'''
|
||||
Fetch all remotes and return a boolean to let the calling function know
|
||||
whether or not any remotes were updated in the process of fetching
|
||||
'''
|
||||
if remotes is None:
|
||||
remotes = []
|
||||
elif not isinstance(remotes, list):
|
||||
log.error(
|
||||
'Invalid \'remotes\' argument (%s) for fetch_remotes. '
|
||||
'Must be a list of strings', remotes
|
||||
)
|
||||
remotes = []
|
||||
|
||||
changed = False
|
||||
for repo in self.remotes:
|
||||
try:
|
||||
if repo.fetch():
|
||||
# We can't just use the return value from repo.fetch()
|
||||
# because the data could still have changed if old remotes
|
||||
# were cleared above. Additionally, we're running this in a
|
||||
# loop and later remotes without changes would override
|
||||
# this value and make it incorrect.
|
||||
changed = True
|
||||
except Exception as exc:
|
||||
log.error(
|
||||
'Exception caught while fetching %s remote \'%s\': %s',
|
||||
self.role, repo.id, exc,
|
||||
exc_info=True
|
||||
)
|
||||
name = getattr(repo, 'name', None)
|
||||
if not remotes or (repo.id, name) in remotes:
|
||||
try:
|
||||
if repo.fetch():
|
||||
# We can't just use the return value from repo.fetch()
|
||||
# because the data could still have changed if old
|
||||
# remotes were cleared above. Additionally, we're
|
||||
# running this in a loop and later remotes without
|
||||
# changes would override this value and make it
|
||||
# incorrect.
|
||||
changed = True
|
||||
except Exception as exc:
|
||||
log.error(
|
||||
'Exception caught while fetching %s remote \'%s\': %s',
|
||||
self.role, repo.id, exc,
|
||||
exc_info=True
|
||||
)
|
||||
return changed
|
||||
|
||||
def lock(self, remote=None):
|
||||
@ -2307,8 +2327,13 @@ class GitBase(object):
|
||||
errors.extend(failed)
|
||||
return locked, errors
|
||||
|
||||
def update(self):
|
||||
def update(self, remotes=None):
|
||||
'''
|
||||
.. versionchanged:: Oxygen
|
||||
The remotes argument was added. This being a list of remote URLs,
|
||||
it will only update matching remotes. This actually matches on
|
||||
repo.id
|
||||
|
||||
Execute a git fetch on all of the repos and perform maintenance on the
|
||||
fileserver cache.
|
||||
'''
|
||||
@ -2317,7 +2342,7 @@ class GitBase(object):
|
||||
'backend': 'gitfs'}
|
||||
|
||||
data['changed'] = self.clear_old_remotes()
|
||||
if self.fetch_remotes():
|
||||
if self.fetch_remotes(remotes=remotes):
|
||||
data['changed'] = True
|
||||
|
||||
# A masterless minion will need a new env cache file even if no changes
|
||||
@ -2358,6 +2383,18 @@ class GitBase(object):
|
||||
# Hash file won't exist if no files have yet been served up
|
||||
pass
|
||||
|
||||
def update_intervals(self):
|
||||
'''
|
||||
Returns a dictionary mapping remote IDs to their intervals, designed to
|
||||
be used for variable update intervals in salt.master.FileserverUpdate.
|
||||
|
||||
A remote's ID is defined here as a tuple of the GitPython/Pygit2
|
||||
object's "id" and "name" attributes, with None being assumed as the
|
||||
"name" value if the attribute is not present.
|
||||
'''
|
||||
return {(repo.id, getattr(repo, 'name', None)): repo.update_interval
|
||||
for repo in self.remotes}
|
||||
|
||||
def verify_provider(self):
|
||||
'''
|
||||
Determine which provider to use
|
||||
|
@ -63,7 +63,7 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin):
|
||||
'cachedir': self.tmp_cachedir,
|
||||
'sock_dir': TMP_SOCK_DIR,
|
||||
'gitfs_root': 'salt',
|
||||
'fileserver_backend': ['git'],
|
||||
'fileserver_backend': ['gitfs'],
|
||||
'gitfs_base': 'master',
|
||||
'fileserver_events': True,
|
||||
'transport': 'zeromq',
|
||||
@ -86,6 +86,7 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin):
|
||||
'gitfs_ssl_verify': True,
|
||||
'gitfs_disable_saltenv_mapping': False,
|
||||
'gitfs_ref_types': ['branch', 'tag', 'sha'],
|
||||
'gitfs_update_interval': 60,
|
||||
'__role': 'master',
|
||||
}
|
||||
}
|
||||
@ -195,7 +196,7 @@ class GitFSTest(TestCase, LoaderModuleMockMixin):
|
||||
'sock_dir': TMP_SOCK_DIR,
|
||||
'gitfs_remotes': ['file://' + TMP_REPO_DIR],
|
||||
'gitfs_root': '',
|
||||
'fileserver_backend': ['git'],
|
||||
'fileserver_backend': ['gitfs'],
|
||||
'gitfs_base': 'master',
|
||||
'fileserver_events': True,
|
||||
'transport': 'zeromq',
|
||||
@ -218,6 +219,7 @@ class GitFSTest(TestCase, LoaderModuleMockMixin):
|
||||
'gitfs_ssl_verify': True,
|
||||
'gitfs_disable_saltenv_mapping': False,
|
||||
'gitfs_ref_types': ['branch', 'tag', 'sha'],
|
||||
'gitfs_update_interval': 60,
|
||||
'__role': 'master',
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user