mirror of
https://github.com/valitydev/salt.git
synced 2024-11-06 16:45:27 +00:00
Merge pull request #31521 from terminalmage/issue24753
salt-ssh: Fix race condition when caching files to build the thin tarball
This commit is contained in:
Mike Place
commit
060a60fd90
@ -119,7 +119,7 @@ def salt_refs(data, ret=None):
|
||||
return ret
|
||||
|
||||
|
||||
def prep_trans_tar(file_client, chunks, file_refs, pillar=None):
|
||||
def prep_trans_tar(file_client, chunks, file_refs, pillar=None, id_=None):
|
||||
'''
|
||||
Generate the execution package from the saltenv file refs and a low state
|
||||
data structure
|
||||
@ -142,6 +142,7 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None):
|
||||
if pillar:
|
||||
with salt.utils.fopen(pillarfn, 'w+') as fp_:
|
||||
fp_.write(json.dumps(pillar))
|
||||
cachedir = os.path.join('salt-ssh', id_)
|
||||
for saltenv in file_refs:
|
||||
file_refs[saltenv].extend(sync_refs)
|
||||
env_root = os.path.join(gendir, saltenv)
|
||||
@ -150,7 +151,7 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None):
|
||||
for ref in file_refs[saltenv]:
|
||||
for name in ref:
|
||||
short = name[7:]
|
||||
path = file_client.cache_file(name, saltenv)
|
||||
path = file_client.cache_file(name, saltenv, cachedir=cachedir)
|
||||
if path:
|
||||
tgt = os.path.join(env_root, short)
|
||||
tgt_dir = os.path.dirname(tgt)
|
||||
@ -158,7 +159,7 @@ def prep_trans_tar(file_client, chunks, file_refs, pillar=None):
|
||||
os.makedirs(tgt_dir)
|
||||
shutil.copy(path, tgt)
|
||||
continue
|
||||
files = file_client.cache_dir(name, saltenv)
|
||||
files = file_client.cache_dir(name, saltenv, cachedir=cachedir)
|
||||
if files:
|
||||
for filename in files:
|
||||
fn = filename[filename.find(short) + len(short):]
|
||||
|
||||
@ -7,6 +7,7 @@ from __future__ import absolute_import
|
||||
# Import salt libs
|
||||
import salt.client.ssh
|
||||
import logging
|
||||
import os
|
||||
from salt.exceptions import CommandExecutionError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@ -33,7 +34,10 @@ def get_file(path,
|
||||
if template is not None:
|
||||
(path, dest) = _render_filenames(path, dest, saltenv, template)
|
||||
|
||||
src = __context__['fileclient'].cache_file(path, saltenv)
|
||||
src = __context__['fileclient'].cache_file(
|
||||
path,
|
||||
saltenv,
|
||||
cachedir=os.path.join('salt-ssh', __salt__.kwargs['id_']))
|
||||
single = salt.client.ssh.Single(
|
||||
__opts__,
|
||||
'',
|
||||
@ -46,7 +50,10 @@ def get_dir(path, dest, saltenv='base'):
|
||||
'''
|
||||
Transfer a directory down
|
||||
'''
|
||||
src = __context__['fileclient'].cache_dir(path, saltenv)
|
||||
src = __context__['fileclient'].cache_dir(
|
||||
path,
|
||||
saltenv,
|
||||
cachedir=os.path.join('salt-ssh', __salt__.kwargs['id_']))
|
||||
src = ' '.join(src)
|
||||
single = salt.client.ssh.Single(
|
||||
__opts__,
|
||||
@ -60,7 +67,10 @@ def get_url(path, dest, saltenv='base'):
|
||||
'''
|
||||
retrive a URL
|
||||
'''
|
||||
src = __context__['fileclient'].get_url(path, saltenv)
|
||||
src = __context__['fileclient'].get_url(
|
||||
path,
|
||||
saltenv,
|
||||
cachedir=os.path.join('salt-ssh', __salt__.kwargs['id_']))
|
||||
single = salt.client.ssh.Single(
|
||||
__opts__,
|
||||
'',
|
||||
|
||||
@ -94,11 +94,13 @@ def sls(mods, saltenv='base', test=None, exclude=None, env=None, **kwargs):
|
||||
__opts__.get('extra_filerefs', '')
|
||||
)
|
||||
)
|
||||
# Create the tar containing the state pkg and relevant files.
|
||||
trans_tar = salt.client.ssh.state.prep_trans_tar(
|
||||
__context__['fileclient'],
|
||||
chunks,
|
||||
file_refs,
|
||||
__pillar__)
|
||||
__pillar__,
|
||||
id_=st_kwargs['id_'])
|
||||
trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
|
||||
cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
|
||||
__opts__['thin_dir'],
|
||||
@ -161,11 +163,13 @@ def low(data, **kwargs):
|
||||
__opts__.get('extra_filerefs', '')
|
||||
)
|
||||
)
|
||||
# Create the tar containing the state pkg and relevant files.
|
||||
trans_tar = salt.client.ssh.state.prep_trans_tar(
|
||||
__context__['fileclient'],
|
||||
chunks,
|
||||
file_refs,
|
||||
__pillar__)
|
||||
__pillar__,
|
||||
id_=st_kwargs['id_'])
|
||||
trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
|
||||
cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format(
|
||||
__opts__['thin_dir'],
|
||||
@ -225,11 +229,13 @@ def high(data, **kwargs):
|
||||
__opts__.get('extra_filerefs', '')
|
||||
)
|
||||
)
|
||||
# Create the tar containing the state pkg and relevant files.
|
||||
trans_tar = salt.client.ssh.state.prep_trans_tar(
|
||||
__context__['fileclient'],
|
||||
chunks,
|
||||
file_refs,
|
||||
__pillar__)
|
||||
__pillar__,
|
||||
id_=st_kwargs['id_'])
|
||||
trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
|
||||
cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format(
|
||||
__opts__['thin_dir'],
|
||||
@ -317,11 +323,13 @@ def highstate(test=None, **kwargs):
|
||||
for chunk in chunks:
|
||||
if not isinstance(chunk, dict):
|
||||
return chunks
|
||||
# Create the tar containing the state pkg and relevant files.
|
||||
trans_tar = salt.client.ssh.state.prep_trans_tar(
|
||||
__context__['fileclient'],
|
||||
chunks,
|
||||
file_refs,
|
||||
__pillar__)
|
||||
__pillar__,
|
||||
id_=st_kwargs['id_'])
|
||||
trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
|
||||
cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
|
||||
__opts__['thin_dir'],
|
||||
@ -388,11 +396,13 @@ def top(topfn, test=None, **kwargs):
|
||||
__opts__.get('extra_filerefs', '')
|
||||
)
|
||||
)
|
||||
# Create the tar containing the state pkg and relevant files.
|
||||
trans_tar = salt.client.ssh.state.prep_trans_tar(
|
||||
__context__['fileclient'],
|
||||
chunks,
|
||||
file_refs,
|
||||
__pillar__)
|
||||
__pillar__,
|
||||
id_=st_kwargs['id_'])
|
||||
trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
|
||||
cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
|
||||
__opts__['thin_dir'],
|
||||
@ -605,12 +615,13 @@ def single(fun, name, test=None, **kwargs):
|
||||
)
|
||||
)
|
||||
|
||||
# Create the tar containing the state pkg and relevant files
|
||||
# Create the tar containing the state pkg and relevant files.
|
||||
trans_tar = salt.client.ssh.state.prep_trans_tar(
|
||||
__context__['fileclient'],
|
||||
chunks,
|
||||
file_refs,
|
||||
__pillar__)
|
||||
__pillar__,
|
||||
id_=st_kwargs['id_'])
|
||||
|
||||
# Create a hash so we can verify the tar on the target system
|
||||
trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
|
||||
|
||||
@ -89,7 +89,7 @@ class Client(object):
|
||||
return filelist
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _cache_loc(self, path, saltenv='base', env=None):
|
||||
def _cache_loc(self, path, saltenv='base', env=None, cachedir=None):
|
||||
'''
|
||||
Return the local location to cache the file, cache dirs will be made
|
||||
'''
|
||||
@ -103,10 +103,12 @@ class Client(object):
|
||||
# Backwards compatibility
|
||||
saltenv = env
|
||||
|
||||
dest = os.path.join(self.opts['cachedir'],
|
||||
'files',
|
||||
saltenv,
|
||||
path)
|
||||
if cachedir is None:
|
||||
cachedir = self.opts['cachedir']
|
||||
elif not os.path.isabs(cachedir):
|
||||
cachedir = os.path.join(self.opts['cachedir'], cachedir)
|
||||
|
||||
dest = os.path.join(cachedir, 'files', saltenv, path)
|
||||
destdir = os.path.dirname(dest)
|
||||
cumask = os.umask(63)
|
||||
if not os.path.isdir(destdir):
|
||||
@ -124,7 +126,8 @@ class Client(object):
|
||||
makedirs=False,
|
||||
saltenv='base',
|
||||
gzip=None,
|
||||
env=None):
|
||||
env=None,
|
||||
cachedir=None):
|
||||
'''
|
||||
Copies a file from the local files or master depending on
|
||||
implementation
|
||||
@ -137,7 +140,7 @@ class Client(object):
|
||||
'''
|
||||
raise NotImplementedError
|
||||
|
||||
def cache_file(self, path, saltenv='base', env=None):
|
||||
def cache_file(self, path, saltenv='base', env=None, cachedir=None):
|
||||
'''
|
||||
Pull a file down from the file server and store it in the minion
|
||||
file cache
|
||||
@ -152,9 +155,9 @@ class Client(object):
|
||||
# Backwards compatibility
|
||||
saltenv = env
|
||||
|
||||
return self.get_url(path, '', True, saltenv)
|
||||
return self.get_url(path, '', True, saltenv, cachedir=cachedir)
|
||||
|
||||
def cache_files(self, paths, saltenv='base', env=None):
|
||||
def cache_files(self, paths, saltenv='base', env=None, cachedir=None):
|
||||
'''
|
||||
Download a list of files stored on the master and put them in the
|
||||
minion file cache
|
||||
@ -173,10 +176,10 @@ class Client(object):
|
||||
if isinstance(paths, str):
|
||||
paths = paths.split(',')
|
||||
for path in paths:
|
||||
ret.append(self.cache_file(path, saltenv))
|
||||
ret.append(self.cache_file(path, saltenv, cachedir=cachedir))
|
||||
return ret
|
||||
|
||||
def cache_master(self, saltenv='base', env=None):
|
||||
def cache_master(self, saltenv='base', env=None, cachedir=None):
|
||||
'''
|
||||
Download and cache all files on a master in a specified environment
|
||||
'''
|
||||
@ -192,11 +195,14 @@ class Client(object):
|
||||
|
||||
ret = []
|
||||
for path in self.file_list(saltenv):
|
||||
ret.append(self.cache_file('salt://{0}'.format(path), saltenv))
|
||||
ret.append(
|
||||
self.cache_file(
|
||||
'salt://{0}'.format(path), saltenv, cachedir=cachedir)
|
||||
)
|
||||
return ret
|
||||
|
||||
def cache_dir(self, path, saltenv='base', include_empty=False,
|
||||
include_pat=None, exclude_pat=None, env=None):
|
||||
include_pat=None, exclude_pat=None, env=None, cachedir=None):
|
||||
'''
|
||||
Download all of the files in a subdir of the master
|
||||
'''
|
||||
@ -230,7 +236,8 @@ class Client(object):
|
||||
if fn_.strip() and fn_.startswith(path):
|
||||
if salt.utils.check_include_exclude(
|
||||
fn_, include_pat, exclude_pat):
|
||||
fn_ = self.cache_file('salt://' + fn_, saltenv)
|
||||
fn_ = self.cache_file(
|
||||
'salt://' + fn_, saltenv, cachedir=cachedir)
|
||||
if fn_:
|
||||
ret.append(fn_)
|
||||
|
||||
@ -244,11 +251,12 @@ class Client(object):
|
||||
# prefix = ''
|
||||
# else:
|
||||
# prefix = separated[0]
|
||||
dest = salt.utils.path_join(
|
||||
self.opts['cachedir'],
|
||||
'files',
|
||||
saltenv
|
||||
)
|
||||
if cachedir is None:
|
||||
cachedir = self.opts['cachedir']
|
||||
elif not os.path.isabs(cachedir):
|
||||
cachedir = os.path.join(self.opts['cachdir'], cachedir)
|
||||
|
||||
dest = salt.utils.path_join(cachedir, 'files', saltenv)
|
||||
for fn_ in self.file_list_emptydirs(saltenv):
|
||||
if fn_.startswith(path):
|
||||
minion_dir = '{0}/{1}'.format(dest, fn_)
|
||||
@ -340,7 +348,7 @@ class Client(object):
|
||||
|
||||
return {}
|
||||
|
||||
def is_cached(self, path, saltenv='base', env=None):
|
||||
def is_cached(self, path, saltenv='base', env=None, cachedir=None):
|
||||
'''
|
||||
Returns the full path to a file if it is cached locally on the minion
|
||||
otherwise returns a blank string
|
||||
@ -367,12 +375,14 @@ class Client(object):
|
||||
self.opts['cachedir'], 'localfiles', path.lstrip('|/'))
|
||||
filesdest = os.path.join(
|
||||
self.opts['cachedir'], 'files', saltenv, path.lstrip('|/'))
|
||||
extrndest = self._extrn_path(path, saltenv)
|
||||
extrndest = self._extrn_path(path, saltenv, cachedir=cachedir)
|
||||
|
||||
if os.path.exists(filesdest):
|
||||
return salt.utils.url.escape(filesdest) if escaped else filesdest
|
||||
elif os.path.exists(localsfilesdest):
|
||||
return salt.utils.url.escape(localsfilesdest) if escaped else localsfilesdest
|
||||
return salt.utils.url.escape(localsfilesdest) \
|
||||
if escaped \
|
||||
else localsfilesdest
|
||||
elif os.path.exists(extrndest):
|
||||
return extrndest
|
||||
|
||||
@ -430,7 +440,7 @@ class Client(object):
|
||||
states.append(path.replace('/', '.')[:-4])
|
||||
return states
|
||||
|
||||
def get_state(self, sls, saltenv):
|
||||
def get_state(self, sls, saltenv, cachedir=None):
|
||||
'''
|
||||
Get a state file from the master and store it in the local minion
|
||||
cache return the location of the file
|
||||
@ -439,12 +449,13 @@ class Client(object):
|
||||
sls = sls.replace('.', '/')
|
||||
for path in ['salt://{0}.sls'.format(sls),
|
||||
'/'.join(['salt:/', sls, 'init.sls'])]:
|
||||
dest = self.cache_file(path, saltenv)
|
||||
dest = self.cache_file(path, saltenv, cachedir=cachedir)
|
||||
if dest:
|
||||
return {'source': path, 'dest': dest}
|
||||
return {}
|
||||
|
||||
def get_dir(self, path, dest='', saltenv='base', gzip=None, env=None):
|
||||
def get_dir(self, path, dest='', saltenv='base', gzip=None, env=None,
|
||||
cachedir=None):
|
||||
'''
|
||||
Get a directory recursively from the salt-master
|
||||
'''
|
||||
@ -487,15 +498,15 @@ class Client(object):
|
||||
self.get_file(
|
||||
'salt://{0}'.format(fn_),
|
||||
'{0}/{1}'.format(dest, minion_relpath),
|
||||
True, saltenv, gzip
|
||||
True, saltenv, gzip, cachedir=cachedir
|
||||
)
|
||||
)
|
||||
# Replicate empty dirs from master
|
||||
try:
|
||||
for fn_ in self.file_list_emptydirs(saltenv):
|
||||
if fn_.startswith(path):
|
||||
# Prevent an empty dir "salt://foobar/" from matching a path of
|
||||
# "salt://foo"
|
||||
# Prevent an empty dir "salt://foobar/" from matching a
|
||||
# path of "salt://foo"
|
||||
try:
|
||||
if fn_[len(path)] != '/':
|
||||
continue
|
||||
@ -513,7 +524,8 @@ class Client(object):
|
||||
ret.sort()
|
||||
return ret
|
||||
|
||||
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
|
||||
def get_url(self, url, dest, makedirs=False, saltenv='base', env=None,
|
||||
no_cache=False, cachedir=None):
|
||||
'''
|
||||
Get a single file from a URL.
|
||||
'''
|
||||
@ -538,7 +550,8 @@ class Client(object):
|
||||
return url_data.path
|
||||
|
||||
if url_data.scheme == 'salt':
|
||||
return self.get_file(url, dest, makedirs, saltenv)
|
||||
return self.get_file(
|
||||
url, dest, makedirs, saltenv, cachedir=cachedir)
|
||||
if dest:
|
||||
destdir = os.path.dirname(dest)
|
||||
if not os.path.isdir(destdir):
|
||||
@ -547,7 +560,7 @@ class Client(object):
|
||||
else:
|
||||
return ''
|
||||
elif not no_cache:
|
||||
dest = self._extrn_path(url, saltenv)
|
||||
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
|
||||
destdir = os.path.dirname(dest)
|
||||
if not os.path.isdir(destdir):
|
||||
os.makedirs(destdir)
|
||||
@ -575,7 +588,9 @@ class Client(object):
|
||||
location=s3_opt('location'))
|
||||
return dest
|
||||
except Exception as exc:
|
||||
raise MinionError('Could not fetch from {0}. Exception: {1}'.format(url, exc))
|
||||
raise MinionError(
|
||||
'Could not fetch from {0}. Exception: {1}'.format(url, exc)
|
||||
)
|
||||
if url_data.scheme == 'ftp':
|
||||
try:
|
||||
ftp = ftplib.FTP(url_data.hostname)
|
||||
@ -652,6 +667,7 @@ class Client(object):
|
||||
makedirs=False,
|
||||
saltenv='base',
|
||||
env=None,
|
||||
cachedir=None,
|
||||
**kwargs):
|
||||
'''
|
||||
Cache a file then process it as a template
|
||||
@ -668,7 +684,7 @@ class Client(object):
|
||||
|
||||
kwargs['saltenv'] = saltenv
|
||||
url_data = urlparse(url)
|
||||
sfn = self.cache_file(url, saltenv)
|
||||
sfn = self.cache_file(url, saltenv, cachedir=cachedir)
|
||||
if not os.path.exists(sfn):
|
||||
return ''
|
||||
if template in salt.utils.templates.TEMPLATE_REGISTRY:
|
||||
@ -690,7 +706,7 @@ class Client(object):
|
||||
return ''
|
||||
if not dest:
|
||||
# No destination passed, set the dest as an extrn_files cache
|
||||
dest = self._extrn_path(url, saltenv)
|
||||
dest = self._extrn_path(url, saltenv, cachedir=cachedir)
|
||||
# If Salt generated the dest name, create any required dirs
|
||||
makedirs = True
|
||||
|
||||
@ -704,7 +720,7 @@ class Client(object):
|
||||
shutil.move(data['data'], dest)
|
||||
return dest
|
||||
|
||||
def _extrn_path(self, url, saltenv):
|
||||
def _extrn_path(self, url, saltenv, cachedir=None):
|
||||
'''
|
||||
Return the extn_filepath for a given url
|
||||
'''
|
||||
@ -714,8 +730,13 @@ class Client(object):
|
||||
else:
|
||||
netloc = url_data.netloc
|
||||
|
||||
if cachedir is None:
|
||||
cachedir = self.opts['cachedir']
|
||||
elif not os.path.isabs(cachedir):
|
||||
cachedir = os.path.join(self.opts['cachedir'], cachedir)
|
||||
|
||||
return salt.utils.path_join(
|
||||
self.opts['cachedir'],
|
||||
cachedir,
|
||||
'extrn_files',
|
||||
saltenv,
|
||||
netloc,
|
||||
@ -756,7 +777,8 @@ class LocalClient(Client):
|
||||
makedirs=False,
|
||||
saltenv='base',
|
||||
gzip=None,
|
||||
env=None):
|
||||
env=None,
|
||||
cachedir=None):
|
||||
'''
|
||||
Copies a file from the local files directory into :param:`dest`
|
||||
gzip compression settings are ignored for local files
|
||||
@ -973,14 +995,14 @@ class RemoteClient(Client):
|
||||
makedirs=False,
|
||||
saltenv='base',
|
||||
gzip=None,
|
||||
env=None):
|
||||
env=None,
|
||||
cachedir=None):
|
||||
'''
|
||||
Get a single file from the salt-master
|
||||
path must be a salt server location, aka, salt://path/to/file, if
|
||||
dest is omitted, then the downloaded file will be placed in the minion
|
||||
cache
|
||||
'''
|
||||
|
||||
# Check if file exists on server, before creating files and
|
||||
# directories
|
||||
hash_server = self.hash_file(path, saltenv)
|
||||
@ -1007,7 +1029,8 @@ class RemoteClient(Client):
|
||||
dest2check = dest
|
||||
if not dest2check:
|
||||
rel_path = self._check_proto(path)
|
||||
with self._cache_loc(rel_path, saltenv) as cache_dest:
|
||||
with self._cache_loc(
|
||||
rel_path, saltenv, cachedir=cachedir) as cache_dest:
|
||||
dest2check = cache_dest
|
||||
|
||||
if dest2check and os.path.isfile(dest2check):
|
||||
@ -1056,7 +1079,10 @@ class RemoteClient(Client):
|
||||
if not data['data']:
|
||||
if not fn_ and data['dest']:
|
||||
# This is a 0 byte file on the master
|
||||
with self._cache_loc(data['dest'], saltenv) as cache_dest:
|
||||
with self._cache_loc(
|
||||
data['dest'],
|
||||
saltenv,
|
||||
cachedir=cachedir) as cache_dest:
|
||||
dest = cache_dest
|
||||
with salt.utils.fopen(cache_dest, 'wb+') as ofile:
|
||||
ofile.write(data['data'])
|
||||
@ -1071,7 +1097,10 @@ class RemoteClient(Client):
|
||||
continue
|
||||
break
|
||||
if not fn_:
|
||||
with self._cache_loc(data['dest'], saltenv) as cache_dest:
|
||||
with self._cache_loc(
|
||||
data['dest'],
|
||||
saltenv,
|
||||
cachedir=cachedir) as cache_dest:
|
||||
dest = cache_dest
|
||||
# If a directory was formerly cached at this path, then
|
||||
# remove it to avoid a traceback trying to write the file
|
||||
|
||||
333
tests/integration/fileclient_test.py
Normal file
333
tests/integration/fileclient_test.py
Normal file
@ -0,0 +1,333 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
:codeauthor: :email:`Erik Johnson <erik@saltstack.com>`
|
||||
'''
|
||||
|
||||
# Import Salt Testing libs
|
||||
from salttesting import skipIf
|
||||
from salttesting.helpers import ensure_in_syspath
|
||||
from salttesting.mock import patch, NO_MOCK, NO_MOCK_REASON
|
||||
|
||||
ensure_in_syspath('../..')
|
||||
|
||||
# Import Python libs
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
|
||||
# Import salt libs
|
||||
import integration
|
||||
import salt.utils
|
||||
from salt import fileclient
|
||||
from salt.ext import six
|
||||
from salttesting.helpers import ensure_in_syspath, destructiveTest
|
||||
ensure_in_syspath('..')
|
||||
|
||||
SALTENVS = ('base', 'dev')
|
||||
FS_ROOT = os.path.join(integration.TMP, 'fileclient_fs_root')
|
||||
CACHE_ROOT = os.path.join(integration.TMP, 'fileclient_cache_root')
|
||||
SUBDIR = 'subdir'
|
||||
SUBDIR_FILES = ('foo.txt', 'bar.txt', 'baz.txt')
|
||||
|
||||
|
||||
def _get_file_roots():
|
||||
return dict(
|
||||
[(x, [os.path.join(FS_ROOT, x)]) for x in SALTENVS]
|
||||
)
|
||||
|
||||
|
||||
fileclient.__opts__ = {}
|
||||
MOCKED_OPTS = {
|
||||
'file_roots': _get_file_roots(),
|
||||
'fileserver_backend': ['roots'],
|
||||
'cachedir': CACHE_ROOT,
|
||||
'file_client': 'local',
|
||||
}
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@skipIf(NO_MOCK, NO_MOCK_REASON)
|
||||
@destructiveTest
|
||||
class FileclientTest(integration.ModuleCase):
|
||||
'''
|
||||
Tests for the fileclient. The LocalClient is the only thing we can test as
|
||||
it is the only way we can mock the fileclient (the tests run from the
|
||||
minion process, so the master cannot be mocked from test code).
|
||||
'''
|
||||
|
||||
def setUp(self):
|
||||
'''
|
||||
No need to add a dummy foo.txt to muddy up the github repo, just make
|
||||
our own fileserver root on-the-fly.
|
||||
'''
|
||||
def _new_dir(path):
|
||||
'''
|
||||
Add a new dir at ``path`` using os.makedirs. If the directory
|
||||
already exists, remove it recursively and then try to create it
|
||||
again.
|
||||
'''
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
# Just in case a previous test was interrupted, remove the
|
||||
# directory and try adding it again.
|
||||
shutil.rmtree(path)
|
||||
os.makedirs(path)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Crete the FS_ROOT
|
||||
for saltenv in SALTENVS:
|
||||
saltenv_root = os.path.join(FS_ROOT, saltenv)
|
||||
# Make sure we have a fresh root dir for this saltenv
|
||||
_new_dir(saltenv_root)
|
||||
|
||||
path = os.path.join(saltenv_root, 'foo.txt')
|
||||
with salt.utils.fopen(path, 'w') as fp_:
|
||||
fp_.write(
|
||||
'This is a test file in the \'{0}\' saltenv.\n'
|
||||
.format(saltenv)
|
||||
)
|
||||
|
||||
subdir_abspath = os.path.join(saltenv_root, SUBDIR)
|
||||
os.makedirs(subdir_abspath)
|
||||
for subdir_file in SUBDIR_FILES:
|
||||
path = os.path.join(subdir_abspath, subdir_file)
|
||||
with salt.utils.fopen(path, 'w') as fp_:
|
||||
fp_.write(
|
||||
'This is file \'{0}\' in subdir \'{1} from saltenv '
|
||||
'\'{2}\''.format(subdir_file, SUBDIR, saltenv)
|
||||
)
|
||||
|
||||
# Create the CACHE_ROOT
|
||||
_new_dir(CACHE_ROOT)
|
||||
|
||||
def tearDown(self):
|
||||
'''
|
||||
Remove the directories created for these tests
|
||||
'''
|
||||
shutil.rmtree(FS_ROOT)
|
||||
shutil.rmtree(CACHE_ROOT)
|
||||
|
||||
def test_cache_dir(self):
|
||||
'''
|
||||
Ensure entire directory is cached to correct location
|
||||
'''
|
||||
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
|
||||
patched_opts.update(MOCKED_OPTS)
|
||||
|
||||
with patch.dict(fileclient.__opts__, patched_opts):
|
||||
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
|
||||
for saltenv in SALTENVS:
|
||||
self.assertTrue(
|
||||
client.cache_dir(
|
||||
'salt://{0}'.format(SUBDIR),
|
||||
saltenv,
|
||||
cachedir=None
|
||||
)
|
||||
)
|
||||
for subdir_file in SUBDIR_FILES:
|
||||
cache_loc = os.path.join(fileclient.__opts__['cachedir'],
|
||||
'files',
|
||||
saltenv,
|
||||
SUBDIR,
|
||||
subdir_file)
|
||||
# Double check that the content of the cached file
|
||||
# identifies it as being from the correct saltenv. The
|
||||
# setUp function creates the file with the name of the
|
||||
# saltenv mentioned in the file, so a simple 'in' check is
|
||||
# sufficient here. If opening the file raises an exception,
|
||||
# this is a problem, so we are not catching the exception
|
||||
# and letting it be raised so that the test fails.
|
||||
with salt.utils.fopen(cache_loc) as fp_:
|
||||
content = fp_.read()
|
||||
log.debug('cache_loc = %s', cache_loc)
|
||||
log.debug('content = %s', content)
|
||||
self.assertTrue(subdir_file in content)
|
||||
self.assertTrue(SUBDIR in content)
|
||||
self.assertTrue(saltenv in content)
|
||||
|
||||
def test_cache_dir_with_alternate_cachedir_and_absolute_path(self):
|
||||
'''
|
||||
Ensure entire directory is cached to correct location when an alternate
|
||||
cachedir is specified and that cachedir is an absolute path
|
||||
'''
|
||||
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
|
||||
patched_opts.update(MOCKED_OPTS)
|
||||
alt_cachedir = os.path.join(integration.TMP, 'abs_cachedir')
|
||||
|
||||
with patch.dict(fileclient.__opts__, patched_opts):
|
||||
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
|
||||
for saltenv in SALTENVS:
|
||||
self.assertTrue(
|
||||
client.cache_dir(
|
||||
'salt://{0}'.format(SUBDIR),
|
||||
saltenv,
|
||||
cachedir=alt_cachedir
|
||||
)
|
||||
)
|
||||
for subdir_file in SUBDIR_FILES:
|
||||
cache_loc = os.path.join(alt_cachedir,
|
||||
'files',
|
||||
saltenv,
|
||||
SUBDIR,
|
||||
subdir_file)
|
||||
# Double check that the content of the cached file
|
||||
# identifies it as being from the correct saltenv. The
|
||||
# setUp function creates the file with the name of the
|
||||
# saltenv mentioned in the file, so a simple 'in' check is
|
||||
# sufficient here. If opening the file raises an exception,
|
||||
# this is a problem, so we are not catching the exception
|
||||
# and letting it be raised so that the test fails.
|
||||
with salt.utils.fopen(cache_loc) as fp_:
|
||||
content = fp_.read()
|
||||
log.debug('cache_loc = %s', cache_loc)
|
||||
log.debug('content = %s', content)
|
||||
self.assertTrue(subdir_file in content)
|
||||
self.assertTrue(SUBDIR in content)
|
||||
self.assertTrue(saltenv in content)
|
||||
|
||||
def test_cache_dir_with_alternate_cachedir_and_relative_path(self):
|
||||
'''
|
||||
Ensure entire directory is cached to correct location when an alternate
|
||||
cachedir is specified and that cachedir is a relative path
|
||||
'''
|
||||
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
|
||||
patched_opts.update(MOCKED_OPTS)
|
||||
alt_cachedir = 'foo'
|
||||
|
||||
with patch.dict(fileclient.__opts__, patched_opts):
|
||||
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
|
||||
for saltenv in SALTENVS:
|
||||
self.assertTrue(
|
||||
client.cache_dir(
|
||||
'salt://{0}'.format(SUBDIR),
|
||||
saltenv,
|
||||
cachedir=alt_cachedir
|
||||
)
|
||||
)
|
||||
for subdir_file in SUBDIR_FILES:
|
||||
cache_loc = os.path.join(fileclient.__opts__['cachedir'],
|
||||
alt_cachedir,
|
||||
'files',
|
||||
saltenv,
|
||||
SUBDIR,
|
||||
subdir_file)
|
||||
# Double check that the content of the cached file
|
||||
# identifies it as being from the correct saltenv. The
|
||||
# setUp function creates the file with the name of the
|
||||
# saltenv mentioned in the file, so a simple 'in' check is
|
||||
# sufficient here. If opening the file raises an exception,
|
||||
# this is a problem, so we are not catching the exception
|
||||
# and letting it be raised so that the test fails.
|
||||
with salt.utils.fopen(cache_loc) as fp_:
|
||||
content = fp_.read()
|
||||
log.debug('cache_loc = %s', cache_loc)
|
||||
log.debug('content = %s', content)
|
||||
self.assertTrue(subdir_file in content)
|
||||
self.assertTrue(SUBDIR in content)
|
||||
self.assertTrue(saltenv in content)
|
||||
|
||||
def test_cache_file(self):
|
||||
'''
|
||||
Ensure file is cached to correct location
|
||||
'''
|
||||
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
|
||||
patched_opts.update(MOCKED_OPTS)
|
||||
|
||||
with patch.dict(fileclient.__opts__, patched_opts):
|
||||
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
|
||||
for saltenv in SALTENVS:
|
||||
self.assertTrue(
|
||||
client.cache_file('salt://foo.txt', saltenv, cachedir=None)
|
||||
)
|
||||
cache_loc = os.path.join(
|
||||
fileclient.__opts__['cachedir'], 'files', saltenv, 'foo.txt')
|
||||
# Double check that the content of the cached file identifies
|
||||
# it as being from the correct saltenv. The setUp function
|
||||
# creates the file with the name of the saltenv mentioned in
|
||||
# the file, so a simple 'in' check is sufficient here. If
|
||||
# opening the file raises an exception, this is a problem, so
|
||||
# we are not catching the exception and letting it be raised so
|
||||
# that the test fails.
|
||||
with salt.utils.fopen(cache_loc) as fp_:
|
||||
content = fp_.read()
|
||||
log.debug('cache_loc = %s', cache_loc)
|
||||
log.debug('content = %s', content)
|
||||
self.assertTrue(saltenv in content)
|
||||
|
||||
def test_cache_file_with_alternate_cachedir_and_absolute_path(self):
|
||||
'''
|
||||
Ensure file is cached to correct location when an alternate cachedir is
|
||||
specified and that cachedir is an absolute path
|
||||
'''
|
||||
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
|
||||
patched_opts.update(MOCKED_OPTS)
|
||||
alt_cachedir = os.path.join(integration.TMP, 'abs_cachedir')
|
||||
|
||||
with patch.dict(fileclient.__opts__, patched_opts):
|
||||
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
|
||||
for saltenv in SALTENVS:
|
||||
self.assertTrue(
|
||||
client.cache_file('salt://foo.txt',
|
||||
saltenv,
|
||||
cachedir=alt_cachedir)
|
||||
)
|
||||
cache_loc = os.path.join(alt_cachedir,
|
||||
'files',
|
||||
saltenv,
|
||||
'foo.txt')
|
||||
# Double check that the content of the cached file identifies
|
||||
# it as being from the correct saltenv. The setUp function
|
||||
# creates the file with the name of the saltenv mentioned in
|
||||
# the file, so a simple 'in' check is sufficient here. If
|
||||
# opening the file raises an exception, this is a problem, so
|
||||
# we are not catching the exception and letting it be raised so
|
||||
# that the test fails.
|
||||
with salt.utils.fopen(cache_loc) as fp_:
|
||||
content = fp_.read()
|
||||
log.debug('cache_loc = %s', cache_loc)
|
||||
log.debug('content = %s', content)
|
||||
self.assertTrue(saltenv in content)
|
||||
|
||||
def test_cache_file_with_alternate_cachedir_and_relative_path(self):
|
||||
'''
|
||||
Ensure file is cached to correct location when an alternate cachedir is
|
||||
specified and that cachedir is a relative path
|
||||
'''
|
||||
patched_opts = dict((x, y) for x, y in six.iteritems(self.minion_opts))
|
||||
patched_opts.update(MOCKED_OPTS)
|
||||
alt_cachedir = 'foo'
|
||||
|
||||
with patch.dict(fileclient.__opts__, patched_opts):
|
||||
client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
|
||||
for saltenv in SALTENVS:
|
||||
self.assertTrue(
|
||||
client.cache_file('salt://foo.txt',
|
||||
saltenv,
|
||||
cachedir=alt_cachedir)
|
||||
)
|
||||
cache_loc = os.path.join(fileclient.__opts__['cachedir'],
|
||||
alt_cachedir,
|
||||
'files',
|
||||
saltenv,
|
||||
'foo.txt')
|
||||
# Double check that the content of the cached file identifies
|
||||
# it as being from the correct saltenv. The setUp function
|
||||
# creates the file with the name of the saltenv mentioned in
|
||||
# the file, so a simple 'in' check is sufficient here. If
|
||||
# opening the file raises an exception, this is a problem, so
|
||||
# we are not catching the exception and letting it be raised so
|
||||
# that the test fails.
|
||||
with salt.utils.fopen(cache_loc) as fp_:
|
||||
content = fp_.read()
|
||||
log.debug('cache_loc = %s', cache_loc)
|
||||
log.debug('content = %s', content)
|
||||
self.assertTrue(saltenv in content)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
integration.run_tests(FileclientTest)
|
||||
Loading…
Reference in New Issue
Block a user