valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 08:58:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
7c7353c5f6
salt/tests/support/gitfs.py
rallytime 201031fa8a
Merge branch '2017.7' into '2018.3' 
Conflicts:
  - 	doc/man/salt.7
  - 	doc/ref/configuration/minion.rst
  - 	salt/cloud/clouds/ec2.py
  - 	salt/returners/highstate_return.py
  - 	tests/support/gitfs.py
  - 	tests/unit/loader/test_loader.py
2018-08-10 13:12:14 -04:00

585 lines
20 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
'''
Base classes for gitfs/git_pillar integration tests
'''
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import copy
import errno
import logging
import os
import psutil
import shutil
import signal
import tempfile
import textwrap
import time
# Import Salt libs
import salt.utils.files
import salt.utils.path
import salt.utils.yaml
from salt.fileserver import gitfs
from salt.pillar import git_pillar
from salt.ext.six.moves import range # pylint: disable=redefined-builtin
# Import Salt Testing libs
from tests.support.case import ModuleCase
from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin
from tests.support.paths import TMP
from tests.support.helpers import (
get_unused_localhost_port,
requires_system_grains,
)
from tests.support.mock import patch
log = logging.getLogger(__name__)
USERNAME = 'gitpillaruser'
PASSWORD = 'saltrules'
_OPTS = {
'__role': 'minion',
'environment': None,
'pillarenv': None,
'hash_type': 'sha256',
'file_roots': {},
'state_top': 'top.sls',
'state_top_saltenv': None,
'renderer': 'yaml_jinja',
'renderer_whitelist': [],
'renderer_blacklist': [],
'pillar_merge_lists': False,
'git_pillar_base': 'master',
'git_pillar_branch': 'master',
'git_pillar_env': '',
'git_pillar_root': '',
'git_pillar_ssl_verify': True,
'git_pillar_global_lock': True,
'git_pillar_user': '',
'git_pillar_password': '',
'git_pillar_insecure_auth': False,
'git_pillar_privkey': '',
'git_pillar_pubkey': '',
'git_pillar_passphrase': '',
'git_pillar_refspecs': [
'+refs/heads/*:refs/remotes/origin/*',
'+refs/tags/*:refs/tags/*',
],
'git_pillar_includes': True,
}
PROC_TIMEOUT = 10
NOTSET = object()
class ProcessManager(object):
'''
Functions used both to set up self-contained SSH/HTTP servers for testing
'''
wait = 10
def find_proc(self, name=None, search=None):
def _search(proc):
return any([search in x for x in proc.cmdline()])
if name is None and search is None:
raise ValueError('one of name or search is required')
for proc in psutil.process_iter():
if name is not None:
if search is None:
if name in proc.name():
return proc
elif name in proc.name() and _search(proc):
return proc
else:
if _search(proc):
return proc
return None
def wait_proc(self, name=None, search=None, timeout=PROC_TIMEOUT):
for idx in range(1, self.wait + 1):
proc = self.find_proc(name=name, search=search)
if proc is not None:
return proc
else:
if idx != self.wait:
log.debug(
'Waiting for %s process (%d of %d)',
name, idx, self.wait
)
time.sleep(1)
else:
log.debug(
'Failed fo find %s process after %d seconds',
name, self.wait
)
raise Exception(
'Unable to find {0} process running from temp config file {1} '
'using psutil'.format(name, search)
)
class SSHDMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin):
'''
Functions to stand up an SSHD server to serve up git repos for tests.
'''
sshd_proc = None
@classmethod
def prep_server(cls):
cls.sshd_config_dir = tempfile.mkdtemp(dir=TMP)
cls.sshd_config = os.path.join(cls.sshd_config_dir, 'sshd_config')
cls.sshd_port = get_unused_localhost_port()
cls.url = 'ssh://{username}@127.0.0.1:{port}/~/repo.git'.format(
username=cls.username,
port=cls.sshd_port)
home = '/root/.ssh'
cls.ext_opts = {
'url': cls.url,
'privkey_nopass': os.path.join(home, cls.id_rsa_nopass),
'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'),
'privkey_withpass': os.path.join(home, cls.id_rsa_withpass),
'pubkey_withpass': os.path.join(home, cls.id_rsa_withpass + '.pub'),
'passphrase': cls.passphrase}
def spawn_server(self):
ret = self.run_function(
'state.apply',
mods='git_pillar.ssh',
pillar={'git_pillar': {'git_ssh': self.git_ssh,
'id_rsa_nopass': self.id_rsa_nopass,
'id_rsa_withpass': self.id_rsa_withpass,
'sshd_bin': self.sshd_bin,
'sshd_port': self.sshd_port,
'sshd_config_dir': self.sshd_config_dir,
'master_user': self.master_opts['user'],
'user': self.username}}
)
try:
self.sshd_proc = self.wait_proc(name='sshd',
search=self.sshd_config)
finally:
# Do the assert after we check for the PID so that we can track
# it regardless of whether or not something else in the SLS
# failed (but the SSH server still started).
self.assertSaltTrueReturn(ret)
class WebserverMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin):
'''
Functions to stand up an nginx + uWSGI + git-http-backend webserver to
serve up git repos for tests.
'''
nginx_proc = uwsgi_proc = None
@classmethod
def prep_server(cls):
'''
Set up all the webserver paths. Designed to be run once in a
setUpClass function.
'''
cls.root_dir = tempfile.mkdtemp(dir=TMP)
cls.config_dir = os.path.join(cls.root_dir, 'config')
cls.nginx_conf = os.path.join(cls.config_dir, 'nginx.conf')
cls.uwsgi_conf = os.path.join(cls.config_dir, 'uwsgi.yml')
cls.git_dir = os.path.join(cls.root_dir, 'git')
cls.repo_dir = os.path.join(cls.git_dir, 'repos')
cls.venv_dir = os.path.join(cls.root_dir, 'venv')
cls.uwsgi_bin = os.path.join(cls.venv_dir, 'bin', 'uwsgi')
cls.nginx_port = cls.uwsgi_port = get_unused_localhost_port()
while cls.uwsgi_port == cls.nginx_port:
# Ensure we don't hit a corner case in which two sucessive calls to
# get_unused_localhost_port() return identical port numbers.
cls.uwsgi_port = get_unused_localhost_port()
cls.url = 'http://127.0.0.1:{port}/repo.git'.format(port=cls.nginx_port)
cls.ext_opts = {'url': cls.url}
# Add auth params if present (if so this will trigger the spawned
# server to turn on HTTP basic auth).
for credential_param in ('user', 'password'):
if hasattr(cls, credential_param):
cls.ext_opts[credential_param] = getattr(cls, credential_param)
@requires_system_grains
def spawn_server(self, grains):
auth_enabled = hasattr(self, 'username') and hasattr(self, 'password')
pillar = {'git_pillar': {'config_dir': self.config_dir,
'git_dir': self.git_dir,
'venv_dir': self.venv_dir,
'root_dir': self.root_dir,
'nginx_port': self.nginx_port,
'uwsgi_port': self.uwsgi_port,
'auth_enabled': auth_enabled}}
# Different libexec dir for git backend on Debian-based systems
git_core = '/usr/libexec/git-core' \
if grains['os_family'] in ('RedHat') \
else '/usr/lib/git-core'
pillar['git_pillar']['git-http-backend'] = os.path.join(
git_core,
'git-http-backend')
ret = self.run_function(
'state.apply',
mods='git_pillar.http',
pillar=pillar)
if not os.path.exists(pillar['git_pillar']['git-http-backend']):
self.fail(
'{0} not found. Either git is not installed, or the test '
'class needs to be updated.'.format(
pillar['git_pillar']['git-http-backend']
)
)
try:
self.nginx_proc = self.wait_proc(name='nginx',
search=self.nginx_conf)
self.uwsgi_proc = self.wait_proc(name='uwsgi',
search=self.uwsgi_conf)
finally:
# Do the assert after we check for the PID so that we can track
# it regardless of whether or not something else in the SLS
# failed (but the webserver still started).
self.assertSaltTrueReturn(ret)
class GitTestBase(ModuleCase):
'''
Base class for all gitfs/git_pillar tests. Must be subclassed and paired
with either SSHDMixin or WebserverMixin to provide the server.
'''
case = port = bare_repo = admin_repo = None
maxDiff = None
git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com'
ext_opts = {}
# We need to temporarily skip pygit2 tests on EL7 until the EPEL packager
# updates pygit2 to bring it up-to-date with libgit2.
@requires_system_grains
def is_el7(self, grains):
return grains['os_family'] == 'RedHat' and grains['osmajorrelease'] == 7
# Cent OS 6 has too old a version of git to handle the make_repo code, as
# it lacks the -c option for git itself.
@requires_system_grains
def is_pre_el7(self, grains):
return grains['os_family'] == 'RedHat' and grains['osmajorrelease'] < 7
@classmethod
def setUpClass(cls):
cls.prep_server()
def setUp(self):
# Make the test class available to the tearDownClass so we can clean up
# after ourselves. This (and the gated block below) prevent us from
# needing to spend the extra time creating an ssh server and user and
# then tear them down separately for each test.
self.update_class(self)
if self.is_pre_el7(): # pylint: disable=E1120
self.skipTest(
'RHEL < 7 has too old a version of git to run these tests')
@classmethod
def update_class(cls, case):
'''
Make the test class available to the tearDownClass. Note that this
cannot be defined in a parent class and inherited, as this will cause
the parent class to be modified.
'''
if getattr(cls, 'case') is None:
setattr(cls, 'case', case)
def make_repo(self, root_dir, user='root'):
raise NotImplementedError()
class GitFSTestBase(GitTestBase, LoaderModuleMockMixin):
'''
Base class for all gitfs tests
'''
@requires_system_grains
def setup_loader_modules(self, grains): # pylint: disable=W0221
return {
gitfs: {
'__opts__': copy.copy(_OPTS),
'__grains__': grains,
}
}
def make_repo(self, root_dir, user='root'):
raise NotImplementedError()
class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
'''
Base class for all git_pillar tests
'''
@requires_system_grains
def setup_loader_modules(self, grains): # pylint: disable=W0221
return {
git_pillar: {
'__opts__': copy.copy(_OPTS),
'__grains__': grains,
}
}
def get_pillar(self, ext_pillar_conf):
'''
Run git_pillar with the specified configuration
'''
cachedir = tempfile.mkdtemp(dir=TMP)
self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True)
ext_pillar_opts = {'optimization_order': [0, 1, 2]}
ext_pillar_opts.update(
salt.utils.yaml.safe_load(
ext_pillar_conf.format(
cachedir=cachedir,
extmods=os.path.join(cachedir, 'extmods'),
**self.ext_opts
)
)
)
with patch.dict(git_pillar.__opts__, ext_pillar_opts):
return git_pillar.ext_pillar(
'minion',
{},
*ext_pillar_opts['ext_pillar'][0]['git']
)
def make_repo(self, root_dir, user='root'):
self.bare_repo = os.path.join(root_dir, 'repo.git')
self.admin_repo = os.path.join(root_dir, 'admin')
for dirname in (self.bare_repo, self.admin_repo):
shutil.rmtree(dirname, ignore_errors=True)
# Create bare repo
self.run_function(
'git.init',
[self.bare_repo],
user=user,
bare=True)
# Clone bare repo
self.run_function(
'git.clone',
[self.admin_repo],
url=self.bare_repo,
user=user)
def _push(branch, message):
self.run_function(
'git.add',
[self.admin_repo, '.'],
user=user)
self.run_function(
'git.commit',
[self.admin_repo, message],
user=user,
git_opts=self.git_opts,
)
self.run_function(
'git.push',
[self.admin_repo],
remote='origin',
ref=branch,
user=user,
)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_:
fp_.write(textwrap.dedent('''\
base:
'*':
- foo
'''))
with salt.utils.files.fopen(
os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_:
fp_.write(textwrap.dedent('''\
branch: master
mylist:
- master
mydict:
master: True
nested_list:
- master
nested_dict:
master: True
'''))
# Add another file to be referenced using git_pillar_includes
with salt.utils.files.fopen(
os.path.join(self.admin_repo, 'bar.sls'), 'w') as fp_:
fp_.write('included_pillar: True\n')
_push('master', 'initial commit')
# Do the same with different values for "dev" branch
self.run_function(
'git.checkout',
[self.admin_repo],
user=user,
opts='-b dev')
# The bar.sls shouldn't be in any branch but master
self.run_function(
'git.rm',
[self.admin_repo, 'bar.sls'],
user=user)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_:
fp_.write(textwrap.dedent('''\
dev:
'*':
- foo
'''))
with salt.utils.files.fopen(
os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_:
fp_.write(textwrap.dedent('''\
branch: dev
mylist:
- dev
mydict:
dev: True
nested_list:
- dev
nested_dict:
dev: True
'''))
_push('dev', 'add dev branch')
# Create just a top file in a separate repo, to be mapped to the base
# env and referenced using git_pillar_includes
self.run_function(
'git.checkout',
[self.admin_repo],
user=user,
opts='-b top_only')
# The top.sls should be the only file in this branch
self.run_function(
'git.rm',
[self.admin_repo, 'foo.sls'],
user=user)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_:
fp_.write(textwrap.dedent('''\
base:
'*':
- bar
'''))
_push('top_only', 'add top_only branch')
class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin):
'''
Base class for GitPython and Pygit2 SSH tests
'''
id_rsa_nopass = id_rsa_withpass = None
git_ssh = '/tmp/git_ssh'
@classmethod
def tearDownClass(cls):
if cls.case.sshd_proc is not None:
cls.case.sshd_proc.send_signal(signal.SIGTERM)
cls.case.run_state('user.absent', name=cls.username, purge=True)
for dirname in (cls.sshd_config_dir, cls.case.admin_repo,
cls.case.bare_repo):
if dirname is not None:
shutil.rmtree(dirname, ignore_errors=True)
ssh_dir = os.path.expanduser('~/.ssh')
for filename in (cls.id_rsa_nopass,
cls.id_rsa_nopass + '.pub',
cls.id_rsa_withpass,
cls.id_rsa_withpass + '.pub',
cls.git_ssh):
try:
os.remove(os.path.join(ssh_dir, filename))
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
def setUp(self):
'''
Create the SSH server and user, and create the git repo
'''
super(GitPillarSSHTestBase, self).setUp()
self.sshd_proc = self.find_proc(name='sshd',
search=self.sshd_config)
self.sshd_bin = salt.utils.path.which('sshd')
if self.sshd_proc is None:
self.spawn_server()
known_hosts_ret = self.run_function(
'ssh.set_known_host',
user=self.master_opts['user'],
hostname='127.0.0.1',
port=self.sshd_port,
enc='ssh-rsa',
fingerprint='fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46',
hash_known_hosts=False,
fingerprint_hash_type='md5',
)
if 'error' in known_hosts_ret:
raise Exception(
'Failed to add key to {0} user\'s known_hosts '
'file: {1}'.format(
self.master_opts['user'],
known_hosts_ret['error']
)
)
root_dir = os.path.expanduser('~{0}'.format(self.username))
if root_dir.startswith('~'):
self.fail(
'Unable to resolve homedir for user \'{0}\''.format(
self.username
)
)
self.make_repo(root_dir, user=self.username)
def get_pillar(self, ext_pillar_conf):
'''
Wrap the parent class' get_pillar() func in logic that temporarily
changes the GIT_SSH to use our custom script, ensuring that the
passphraselsess key is used to auth without needing to modify the root
user's ssh config file.
'''
orig_git_ssh = os.environ.pop('GIT_SSH', NOTSET)
os.environ['GIT_SSH'] = self.git_ssh
try:
return super(GitPillarSSHTestBase, self).get_pillar(ext_pillar_conf)
finally:
os.environ.pop('GIT_SSH', None)
if orig_git_ssh is not NOTSET:
os.environ['GIT_SSH'] = orig_git_ssh
class GitPillarHTTPTestBase(GitPillarTestBase, WebserverMixin):
'''
Base class for GitPython and Pygit2 HTTP tests
'''
@classmethod
def tearDownClass(cls):
for proc in (cls.case.nginx_proc, cls.case.uwsgi_proc):
if proc is not None:
try:
proc.send_signal(signal.SIGQUIT)
except psutil.NoSuchProcess:
pass
shutil.rmtree(cls.root_dir, ignore_errors=True)
def setUp(self):
'''
Create and start the webserver, and create the git repo
'''
super(GitPillarHTTPTestBase, self).setUp()
self.nginx_proc = self.find_proc(name='nginx',
search=self.nginx_conf)
self.uwsgi_proc = self.find_proc(name='uwsgi',
search=self.uwsgi_conf)
if self.nginx_proc is None and self.uwsgi_proc is None:
self.spawn_server() # pylint: disable=E1120
self.make_repo(self.repo_dir)
Reference in New Issue View Git Blame Copy Permalink