mirror of
https://github.com/valitydev/salt.git
synced 2024-11-07 00:55:19 +00:00
1327 lines
54 KiB
Python
1327 lines
54 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
'''
|
|
Set up the Salt integration test suite
|
|
'''
|
|
|
|
# Import Python libs
|
|
from __future__ import absolute_import, print_function
|
|
import os
|
|
import re
|
|
import sys
|
|
import copy
|
|
import time
|
|
import stat
|
|
import errno
|
|
import signal
|
|
import shutil
|
|
import pprint
|
|
import atexit
|
|
import socket
|
|
import logging
|
|
import tempfile
|
|
import threading
|
|
import subprocess
|
|
import multiprocessing
|
|
from datetime import datetime, timedelta
|
|
try:
|
|
import pwd
|
|
except ImportError:
|
|
pass
|
|
|
|
# Import salt tests support dirs
|
|
from tests.support.paths import * # pylint: disable=wildcard-import
|
|
from tests.support.processes import * # pylint: disable=wildcard-import
|
|
from tests.support.unit import TestCase
|
|
from tests.support.case import ShellTestCase
|
|
from tests.support.parser import PNUM, print_header, SaltTestcaseParser
|
|
from tests.support.helpers import requires_sshd_server, RedirectStdStreams
|
|
from tests.support.paths import ScriptPathMixin
|
|
from tests.support.mixins import CheckShellBinaryNameAndVersionMixin, ShellCaseCommonTestsMixin
|
|
from tests.support.mixins import AdaptedConfigurationTestCaseMixin, SaltClientTestCaseMixin
|
|
from tests.support.mixins import SaltMinionEventAssertsMixin, SaltReturnAssertsMixin
|
|
from tests.support.runtests import RUNTIME_VARS
|
|
# Import Salt libs
|
|
import salt
|
|
import salt.config
|
|
import salt.minion
|
|
import salt.runner
|
|
import salt.output
|
|
import salt.version
|
|
import salt.utils.color
|
|
import salt.utils.files
|
|
import salt.utils.path
|
|
import salt.utils.platform
|
|
import salt.utils.process
|
|
import salt.utils.stringutils
|
|
import salt.utils.yaml
|
|
import salt.log.setup as salt_log_setup
|
|
from salt.utils.verify import verify_env
|
|
from salt.utils.immutabletypes import freeze
|
|
from salt.utils.nb_popen import NonBlockingPopen
|
|
from salt.exceptions import SaltClientError
|
|
|
|
try:
|
|
import salt.master
|
|
except ImportError:
|
|
# Not required for raet tests
|
|
pass
|
|
|
|
# Import 3rd-party libs
|
|
import msgpack
|
|
from salt.ext import six
|
|
from salt.ext.six.moves import cStringIO
|
|
|
|
try:
|
|
import salt.ext.six.moves.socketserver as socketserver
|
|
except ImportError:
|
|
import socketserver
|
|
|
|
from tornado import gen
|
|
from tornado import ioloop
|
|
|
|
# Import salt tests support libs
|
|
from tests.support.processes import SaltMaster, SaltMinion, SaltSyndic
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_RUNTESTS_PORTS = {}
|
|
|
|
|
|
def get_unused_localhost_port():
|
|
'''
|
|
Return a random unused port on localhost
|
|
'''
|
|
usock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
|
|
usock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
usock.bind(('127.0.0.1', 0))
|
|
port = usock.getsockname()[1]
|
|
if port in (54505, 54506, 64505, 64506, 64510, 64511, 64520, 64521):
|
|
# These ports are hardcoded in the test configuration
|
|
port = get_unused_localhost_port()
|
|
usock.close()
|
|
return port
|
|
|
|
DARWIN = True if sys.platform.startswith('darwin') else False
|
|
BSD = True if 'bsd' in sys.platform else False
|
|
|
|
if DARWIN and port in _RUNTESTS_PORTS:
|
|
port = get_unused_localhost_port()
|
|
usock.close()
|
|
return port
|
|
|
|
_RUNTESTS_PORTS[port] = usock
|
|
|
|
if DARWIN or BSD:
|
|
usock.close()
|
|
|
|
return port
|
|
|
|
|
|
def close_open_sockets(sockets_dict):
|
|
for port in list(sockets_dict):
|
|
sock = sockets_dict.pop(port)
|
|
sock.close()
|
|
|
|
|
|
atexit.register(close_open_sockets, _RUNTESTS_PORTS)
|
|
|
|
|
|
SALT_LOG_PORT = get_unused_localhost_port()
|
|
|
|
|
|
class ThreadingMixIn(socketserver.ThreadingMixIn):
|
|
daemon_threads = True
|
|
|
|
|
|
class ThreadedSocketServer(ThreadingMixIn, socketserver.TCPServer):
|
|
|
|
allow_reuse_address = True
|
|
|
|
def server_activate(self):
|
|
self.shutting_down = threading.Event()
|
|
socketserver.TCPServer.server_activate(self)
|
|
#super(ThreadedSocketServer, self).server_activate()
|
|
|
|
def server_close(self):
|
|
if hasattr(self, 'shutting_down'):
|
|
self.shutting_down.set()
|
|
socketserver.TCPServer.server_close(self)
|
|
#super(ThreadedSocketServer, self).server_close()
|
|
|
|
|
|
class SocketServerRequestHandler(socketserver.StreamRequestHandler):
|
|
def handle(self):
|
|
unpacker = msgpack.Unpacker(encoding='utf-8')
|
|
while not self.server.shutting_down.is_set():
|
|
try:
|
|
wire_bytes = self.request.recv(1024)
|
|
if not wire_bytes:
|
|
break
|
|
unpacker.feed(wire_bytes)
|
|
for record_dict in unpacker:
|
|
record = logging.makeLogRecord(record_dict)
|
|
logger = logging.getLogger(record.name)
|
|
logger.handle(record)
|
|
del record_dict
|
|
except (EOFError, KeyboardInterrupt, SystemExit):
|
|
break
|
|
except socket.error as exc:
|
|
try:
|
|
if exc.errno == errno.WSAECONNRESET:
|
|
# Connection reset on windows
|
|
break
|
|
except AttributeError:
|
|
# We're not on windows
|
|
pass
|
|
log.exception(exc)
|
|
except Exception as exc:
|
|
log.exception(exc)
|
|
|
|
|
|
class TestDaemon(object):
|
|
'''
|
|
Set up the master and minion daemons, and run related cases
|
|
'''
|
|
MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 120
|
|
|
|
def __init__(self, parser):
|
|
self.parser = parser
|
|
self.colors = salt.utils.color.get_colors(self.parser.options.no_colors is False)
|
|
if salt.utils.platform.is_windows():
|
|
# There's no shell color support on windows...
|
|
for key in self.colors:
|
|
self.colors[key] = ''
|
|
|
|
def __enter__(self):
|
|
'''
|
|
Start a master and minion
|
|
'''
|
|
# Setup the multiprocessing logging queue listener
|
|
salt_log_setup.setup_multiprocessing_logging_listener(
|
|
self.master_opts
|
|
)
|
|
|
|
# Set up PATH to mockbin
|
|
self._enter_mockbin()
|
|
|
|
if self.parser.options.transport == 'zeromq':
|
|
self.start_zeromq_daemons()
|
|
elif self.parser.options.transport == 'raet':
|
|
self.start_raet_daemons()
|
|
elif self.parser.options.transport == 'tcp':
|
|
self.start_tcp_daemons()
|
|
|
|
self.minion_targets = set(['minion', 'sub_minion'])
|
|
self.pre_setup_minions()
|
|
self.setup_minions()
|
|
|
|
if getattr(self.parser.options, 'ssh', False):
|
|
self.prep_ssh()
|
|
|
|
if self.parser.options.sysinfo:
|
|
try:
|
|
print_header(
|
|
'~~~~~~~ Versions Report ', inline=True,
|
|
width=getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
except TypeError:
|
|
print_header('~~~~~~~ Versions Report ', inline=True)
|
|
|
|
print('\n'.join(salt.version.versions_report()))
|
|
|
|
try:
|
|
print_header(
|
|
'~~~~~~~ Minion Grains Information ', inline=True,
|
|
width=getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
except TypeError:
|
|
print_header('~~~~~~~ Minion Grains Information ', inline=True)
|
|
|
|
grains = self.client.cmd('minion', 'grains.items')
|
|
|
|
minion_opts = self.minion_opts.copy()
|
|
minion_opts['color'] = self.parser.options.no_colors is False
|
|
salt.output.display_output(grains, 'grains', minion_opts)
|
|
|
|
try:
|
|
print_header(
|
|
'=', sep='=', inline=True,
|
|
width=getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
except TypeError:
|
|
print_header('', sep='=', inline=True)
|
|
|
|
try:
|
|
return self
|
|
finally:
|
|
self.post_setup_minions()
|
|
|
|
def start_daemon(self, cls, opts, start_fun):
|
|
def start(cls, opts, start_fun):
|
|
salt.utils.process.appendproctitle('{0}-{1}'.format(self.__class__.__name__, cls.__name__))
|
|
daemon = cls(opts)
|
|
getattr(daemon, start_fun)()
|
|
process = multiprocessing.Process(target=start,
|
|
args=(cls, opts, start_fun))
|
|
process.start()
|
|
return process
|
|
|
|
def start_zeromq_daemons(self):
|
|
'''
|
|
Fire up the daemons used for zeromq tests
|
|
'''
|
|
self.log_server = ThreadedSocketServer(('localhost', SALT_LOG_PORT), SocketServerRequestHandler)
|
|
self.log_server_process = threading.Thread(target=self.log_server.serve_forever)
|
|
self.log_server_process.daemon = True
|
|
self.log_server_process.start()
|
|
try:
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}Starting salt-master ... {ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
self.master_process = start_daemon(
|
|
daemon_name='salt-master',
|
|
daemon_id=self.master_opts['id'],
|
|
daemon_log_prefix='salt-master/{}'.format(self.master_opts['id']),
|
|
daemon_cli_script_name='master',
|
|
daemon_config=self.master_opts,
|
|
daemon_config_dir=RUNTIME_VARS.TMP_CONF_DIR,
|
|
daemon_class=SaltMaster,
|
|
bin_dir_path=SCRIPT_DIR,
|
|
fail_hard=True,
|
|
start_timeout=60)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_GREEN}Starting salt-master ... STARTED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
except (RuntimeWarning, RuntimeError):
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_RED}Starting salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
try:
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}Starting salt-minion ... {ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
self.minion_process = start_daemon(
|
|
daemon_name='salt-minion',
|
|
daemon_id=self.master_opts['id'],
|
|
daemon_log_prefix='salt-minion/{}'.format(self.minion_opts['id']),
|
|
daemon_cli_script_name='minion',
|
|
daemon_config=self.minion_opts,
|
|
daemon_config_dir=RUNTIME_VARS.TMP_CONF_DIR,
|
|
daemon_class=SaltMinion,
|
|
bin_dir_path=SCRIPT_DIR,
|
|
fail_hard=True,
|
|
start_timeout=60)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_GREEN}Starting salt-minion ... STARTED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
except (RuntimeWarning, RuntimeError):
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_RED}Starting salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
try:
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}Starting sub salt-minion ... {ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
self.sub_minion_process = start_daemon(
|
|
daemon_name='sub salt-minion',
|
|
daemon_id=self.master_opts['id'],
|
|
daemon_log_prefix='sub-salt-minion/{}'.format(self.sub_minion_opts['id']),
|
|
daemon_cli_script_name='minion',
|
|
daemon_config=self.sub_minion_opts,
|
|
daemon_config_dir=RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR,
|
|
daemon_class=SaltMinion,
|
|
bin_dir_path=SCRIPT_DIR,
|
|
fail_hard=True,
|
|
start_timeout=60)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_GREEN}Starting sub salt-minion ... STARTED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
except (RuntimeWarning, RuntimeError):
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_RED}Starting sub salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
try:
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}Starting syndic salt-master ... {ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
self.smaster_process = start_daemon(
|
|
daemon_name='salt-smaster',
|
|
daemon_id=self.syndic_master_opts['id'],
|
|
daemon_log_prefix='salt-smaster/{}'.format(self.syndic_master_opts['id']),
|
|
daemon_cli_script_name='master',
|
|
daemon_config=self.syndic_master_opts,
|
|
daemon_config_dir=RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR,
|
|
daemon_class=SaltMaster,
|
|
bin_dir_path=SCRIPT_DIR,
|
|
fail_hard=True,
|
|
start_timeout=60)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_GREEN}Starting syndic salt-master ... STARTED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
except (RuntimeWarning, RuntimeError):
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_RED}Starting syndic salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
try:
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}Starting salt-syndic ... {ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
self.syndic_process = start_daemon(
|
|
daemon_name='salt-syndic',
|
|
daemon_id=self.syndic_opts['id'],
|
|
daemon_log_prefix='salt-syndic/{}'.format(self.syndic_opts['id']),
|
|
daemon_cli_script_name='syndic',
|
|
daemon_config=self.syndic_opts,
|
|
daemon_config_dir=RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR,
|
|
daemon_class=SaltSyndic,
|
|
bin_dir_path=SCRIPT_DIR,
|
|
fail_hard=True,
|
|
start_timeout=60)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_GREEN}Starting salt-syndic ... STARTED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
except (RuntimeWarning, RuntimeError):
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_RED}Starting salt-syndic ... FAILED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
if self.parser.options.proxy:
|
|
try:
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}Starting salt-proxy ... {ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
self.proxy_process = start_daemon(
|
|
daemon_name='salt-proxy',
|
|
daemon_id=self.master_opts['id'],
|
|
daemon_log_prefix='salt-proxy/{}'.format(self.proxy_opts['id']),
|
|
daemon_cli_script_name='proxy',
|
|
daemon_config=self.proxy_opts,
|
|
daemon_config_dir=RUNTIME_VARS.TMP_CONF_DIR,
|
|
daemon_class=SaltProxy,
|
|
bin_dir_path=SCRIPT_DIR,
|
|
fail_hard=True,
|
|
start_timeout=60)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_GREEN}Starting salt-proxy ... STARTED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
except (RuntimeWarning, RuntimeError):
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_RED}Starting salt-proxy ... FAILED!\n{ENDC}'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
def start_raet_daemons(self):
|
|
'''
|
|
Fire up the raet daemons!
|
|
'''
|
|
import salt.daemons.flo
|
|
self.master_process = self.start_daemon(salt.daemons.flo.IofloMaster,
|
|
self.master_opts,
|
|
'start')
|
|
|
|
self.minion_process = self.start_daemon(salt.daemons.flo.IofloMinion,
|
|
self.minion_opts,
|
|
'tune_in')
|
|
|
|
self.sub_minion_process = self.start_daemon(salt.daemons.flo.IofloMinion,
|
|
self.sub_minion_opts,
|
|
'tune_in')
|
|
# Wait for the daemons to all spin up
|
|
time.sleep(5)
|
|
|
|
# self.smaster_process = self.start_daemon(salt.daemons.flo.IofloMaster,
|
|
# self.syndic_master_opts,
|
|
# 'start')
|
|
|
|
# no raet syndic daemon yet
|
|
|
|
start_tcp_daemons = start_zeromq_daemons
|
|
|
|
def prep_ssh(self):
|
|
'''
|
|
Generate keys and start an ssh daemon on an alternate port
|
|
'''
|
|
sys.stdout.write(
|
|
' * {LIGHT_GREEN}Starting {0} ... {ENDC}'.format(
|
|
'SSH server',
|
|
**self.colors
|
|
)
|
|
)
|
|
keygen = salt.utils.path.which('ssh-keygen')
|
|
sshd = salt.utils.path.which('sshd')
|
|
|
|
if not (keygen and sshd):
|
|
print('WARNING: Could not initialize SSH subsystem. Tests for salt-ssh may break!')
|
|
return
|
|
if not os.path.exists(RUNTIME_VARS.TMP_CONF_DIR):
|
|
os.makedirs(RUNTIME_VARS.TMP_CONF_DIR)
|
|
|
|
# Generate client key
|
|
pub_key_test_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'key_test.pub')
|
|
priv_key_test_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'key_test')
|
|
if os.path.exists(pub_key_test_file):
|
|
os.remove(pub_key_test_file)
|
|
if os.path.exists(priv_key_test_file):
|
|
os.remove(priv_key_test_file)
|
|
keygen_process = subprocess.Popen(
|
|
[keygen, '-t',
|
|
'ecdsa',
|
|
'-b',
|
|
'521',
|
|
'-C',
|
|
'"$(whoami)@$(hostname)-$(date -I)"',
|
|
'-f',
|
|
'key_test',
|
|
'-P',
|
|
''],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
close_fds=True,
|
|
cwd=RUNTIME_VARS.TMP_CONF_DIR
|
|
)
|
|
_, keygen_err = keygen_process.communicate()
|
|
if keygen_err:
|
|
print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_err)))
|
|
sshd_config_path = os.path.join(FILES, 'conf/_ssh/sshd_config')
|
|
shutil.copy(sshd_config_path, RUNTIME_VARS.TMP_CONF_DIR)
|
|
auth_key_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'key_test.pub')
|
|
|
|
# Generate server key
|
|
server_key_dir = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'server')
|
|
if not os.path.exists(server_key_dir):
|
|
os.makedirs(server_key_dir)
|
|
server_dsa_priv_key_file = os.path.join(server_key_dir, 'ssh_host_dsa_key')
|
|
server_dsa_pub_key_file = os.path.join(server_key_dir, 'ssh_host_dsa_key.pub')
|
|
server_ecdsa_priv_key_file = os.path.join(server_key_dir, 'ssh_host_ecdsa_key')
|
|
server_ecdsa_pub_key_file = os.path.join(server_key_dir, 'ssh_host_ecdsa_key.pub')
|
|
server_ed25519_priv_key_file = os.path.join(server_key_dir, 'ssh_host_ed25519_key')
|
|
server_ed25519_pub_key_file = os.path.join(server_key_dir, 'ssh_host.ed25519_key.pub')
|
|
|
|
for server_key_file in (server_dsa_priv_key_file,
|
|
server_dsa_pub_key_file,
|
|
server_ecdsa_priv_key_file,
|
|
server_ecdsa_pub_key_file,
|
|
server_ed25519_priv_key_file,
|
|
server_ed25519_pub_key_file):
|
|
if os.path.exists(server_key_file):
|
|
os.remove(server_key_file)
|
|
|
|
keygen_process_dsa = subprocess.Popen(
|
|
[keygen, '-t',
|
|
'dsa',
|
|
'-b',
|
|
'1024',
|
|
'-C',
|
|
'"$(whoami)@$(hostname)-$(date -I)"',
|
|
'-f',
|
|
'ssh_host_dsa_key',
|
|
'-P',
|
|
''],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
close_fds=True,
|
|
cwd=server_key_dir
|
|
)
|
|
_, keygen_dsa_err = keygen_process_dsa.communicate()
|
|
if keygen_dsa_err:
|
|
print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_dsa_err)))
|
|
|
|
keygen_process_ecdsa = subprocess.Popen(
|
|
[keygen, '-t',
|
|
'ecdsa',
|
|
'-b',
|
|
'521',
|
|
'-C',
|
|
'"$(whoami)@$(hostname)-$(date -I)"',
|
|
'-f',
|
|
'ssh_host_ecdsa_key',
|
|
'-P',
|
|
''],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
close_fds=True,
|
|
cwd=server_key_dir
|
|
)
|
|
_, keygen_escda_err = keygen_process_ecdsa.communicate()
|
|
if keygen_escda_err:
|
|
print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_escda_err)))
|
|
|
|
keygen_process_ed25519 = subprocess.Popen(
|
|
[keygen, '-t',
|
|
'ed25519',
|
|
'-b',
|
|
'521',
|
|
'-C',
|
|
'"$(whoami)@$(hostname)-$(date -I)"',
|
|
'-f',
|
|
'ssh_host_ed25519_key',
|
|
'-P',
|
|
''],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
close_fds=True,
|
|
cwd=server_key_dir
|
|
)
|
|
_, keygen_ed25519_err = keygen_process_ed25519.communicate()
|
|
if keygen_ed25519_err:
|
|
print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_ed25519_err)))
|
|
|
|
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'sshd_config'), 'a') as ssh_config:
|
|
ssh_config.write('AuthorizedKeysFile {0}\n'.format(auth_key_file))
|
|
if not keygen_dsa_err:
|
|
ssh_config.write('HostKey {0}\n'.format(server_dsa_priv_key_file))
|
|
if not keygen_escda_err:
|
|
ssh_config.write('HostKey {0}\n'.format(server_ecdsa_priv_key_file))
|
|
if not keygen_ed25519_err:
|
|
ssh_config.write('HostKey {0}\n'.format(server_ed25519_priv_key_file))
|
|
|
|
self.sshd_pidfile = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'sshd.pid')
|
|
self.sshd_process = subprocess.Popen(
|
|
[sshd, '-f', 'sshd_config', '-oPidFile={0}'.format(self.sshd_pidfile)],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
close_fds=True,
|
|
cwd=RUNTIME_VARS.TMP_CONF_DIR
|
|
)
|
|
_, sshd_err = self.sshd_process.communicate()
|
|
if sshd_err:
|
|
print('sshd had errors on startup: {0}'.format(salt.utils.stringutils.to_str(sshd_err)))
|
|
else:
|
|
os.environ['SSH_DAEMON_RUNNING'] = 'True'
|
|
roster_path = os.path.join(FILES, 'conf/_ssh/roster')
|
|
shutil.copy(roster_path, RUNTIME_VARS.TMP_CONF_DIR)
|
|
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'roster'), 'a') as roster:
|
|
roster.write(' user: {0}\n'.format(RUNTIME_VARS.RUNNING_TESTS_USER))
|
|
roster.write(' priv: {0}/{1}'.format(RUNTIME_VARS.TMP_CONF_DIR, 'key_test'))
|
|
sys.stdout.write(
|
|
' {LIGHT_GREEN}STARTED!\n{ENDC}'.format(
|
|
**self.colors
|
|
)
|
|
)
|
|
|
|
@classmethod
|
|
def config(cls, role):
|
|
'''
|
|
Return a configuration for a master/minion/syndic.
|
|
|
|
Currently these roles are:
|
|
* master
|
|
* minion
|
|
* syndic
|
|
* syndic_master
|
|
* sub_minion
|
|
* proxy
|
|
'''
|
|
return RUNTIME_VARS.RUNTIME_CONFIGS[role]
|
|
|
|
@classmethod
|
|
def config_location(cls):
|
|
return RUNTIME_VARS.TMP_CONF_DIR
|
|
|
|
@property
|
|
def client(self):
|
|
'''
|
|
Return a local client which will be used for example to ping and sync
|
|
the test minions.
|
|
|
|
This client is defined as a class attribute because its creation needs
|
|
to be deferred to a latter stage. If created it on `__enter__` like it
|
|
previously was, it would not receive the master events.
|
|
'''
|
|
if 'runtime_client' not in RUNTIME_VARS.RUNTIME_CONFIGS:
|
|
RUNTIME_VARS.RUNTIME_CONFIGS['runtime_client'] = salt.client.get_local_client(
|
|
mopts=self.master_opts
|
|
)
|
|
return RUNTIME_VARS.RUNTIME_CONFIGS['runtime_client']
|
|
|
|
@classmethod
|
|
def transplant_configs(cls, transport='zeromq'):
|
|
if os.path.isdir(RUNTIME_VARS.TMP_CONF_DIR):
|
|
shutil.rmtree(RUNTIME_VARS.TMP_CONF_DIR)
|
|
os.makedirs(RUNTIME_VARS.TMP_CONF_DIR)
|
|
os.makedirs(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR)
|
|
os.makedirs(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR)
|
|
os.makedirs(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR)
|
|
print(' * Transplanting configuration files to \'{0}\''.format(RUNTIME_VARS.TMP_CONF_DIR))
|
|
tests_known_hosts_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'salt_ssh_known_hosts')
|
|
with salt.utils.files.fopen(tests_known_hosts_file, 'w') as known_hosts:
|
|
known_hosts.write('')
|
|
|
|
# This master connects to syndic_master via a syndic
|
|
master_opts = salt.config._read_conf_file(os.path.join(RUNTIME_VARS.CONF_DIR, 'master'))
|
|
master_opts['known_hosts_file'] = tests_known_hosts_file
|
|
master_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache')
|
|
master_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
|
|
master_opts['config_dir'] = RUNTIME_VARS.TMP_CONF_DIR
|
|
master_opts['root_dir'] = os.path.join(TMP, 'rootdir')
|
|
master_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'master')
|
|
|
|
# This is the syndic for master
|
|
# Let's start with a copy of the syndic master configuration
|
|
syndic_opts = copy.deepcopy(master_opts)
|
|
# Let's update with the syndic configuration
|
|
syndic_opts.update(salt.config._read_conf_file(os.path.join(RUNTIME_VARS.CONF_DIR, 'syndic')))
|
|
syndic_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache')
|
|
syndic_opts['config_dir'] = RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR
|
|
|
|
# This minion connects to master
|
|
minion_opts = salt.config._read_conf_file(os.path.join(RUNTIME_VARS.CONF_DIR, 'minion'))
|
|
minion_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache')
|
|
minion_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
|
|
minion_opts['config_dir'] = RUNTIME_VARS.TMP_CONF_DIR
|
|
minion_opts['root_dir'] = os.path.join(TMP, 'rootdir')
|
|
minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki')
|
|
minion_opts['hosts.file'] = os.path.join(TMP, 'rootdir', 'hosts')
|
|
minion_opts['aliases.file'] = os.path.join(TMP, 'rootdir', 'aliases')
|
|
|
|
# This sub_minion also connects to master
|
|
sub_minion_opts = salt.config._read_conf_file(os.path.join(RUNTIME_VARS.CONF_DIR, 'sub_minion'))
|
|
sub_minion_opts['cachedir'] = os.path.join(TMP, 'rootdir-sub-minion', 'cache')
|
|
sub_minion_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
|
|
sub_minion_opts['config_dir'] = RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR
|
|
sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion')
|
|
sub_minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir-sub-minion', 'pki', 'minion')
|
|
sub_minion_opts['hosts.file'] = os.path.join(TMP, 'rootdir', 'hosts')
|
|
sub_minion_opts['aliases.file'] = os.path.join(TMP, 'rootdir', 'aliases')
|
|
|
|
# This is the master of masters
|
|
syndic_master_opts = salt.config._read_conf_file(os.path.join(RUNTIME_VARS.CONF_DIR, 'syndic_master'))
|
|
syndic_master_opts['cachedir'] = os.path.join(TMP, 'rootdir-syndic-master', 'cache')
|
|
syndic_master_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
|
|
syndic_master_opts['config_dir'] = RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR
|
|
syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master')
|
|
syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master')
|
|
|
|
# This proxy connects to master
|
|
proxy_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'proxy'))
|
|
proxy_opts['cachedir'] = os.path.join(TMP, 'rootdir-proxy', 'cache')
|
|
# proxy_opts['user'] = running_tests_user
|
|
proxy_opts['config_dir'] = RUNTIME_VARS.TMP_CONF_DIR
|
|
proxy_opts['root_dir'] = os.path.join(TMP, 'rootdir-proxy')
|
|
proxy_opts['pki_dir'] = os.path.join(TMP, 'rootdir-proxy', 'pki')
|
|
proxy_opts['hosts.file'] = os.path.join(TMP, 'rootdir-proxy', 'hosts')
|
|
proxy_opts['aliases.file'] = os.path.join(TMP, 'rootdir-proxy', 'aliases')
|
|
|
|
if transport == 'raet':
|
|
master_opts['transport'] = 'raet'
|
|
master_opts['raet_port'] = 64506
|
|
minion_opts['transport'] = 'raet'
|
|
minion_opts['raet_port'] = 64510
|
|
sub_minion_opts['transport'] = 'raet'
|
|
sub_minion_opts['raet_port'] = 64520
|
|
# syndic_master_opts['transport'] = 'raet'
|
|
|
|
if transport == 'tcp':
|
|
master_opts['transport'] = 'tcp'
|
|
minion_opts['transport'] = 'tcp'
|
|
sub_minion_opts['transport'] = 'tcp'
|
|
syndic_master_opts['transport'] = 'tcp'
|
|
proxy_opts['transport'] = 'tcp'
|
|
|
|
# Set up config options that require internal data
|
|
master_opts['pillar_roots'] = syndic_master_opts['pillar_roots'] = {
|
|
'base': [
|
|
RUNTIME_VARS.TMP_PILLAR_TREE,
|
|
os.path.join(FILES, 'pillar', 'base'),
|
|
]
|
|
}
|
|
minion_opts['pillar_roots'] = {
|
|
'base': [
|
|
RUNTIME_VARS.TMP_PILLAR_TREE,
|
|
os.path.join(FILES, 'pillar', 'base'),
|
|
]
|
|
}
|
|
master_opts['file_roots'] = syndic_master_opts['file_roots'] = {
|
|
'base': [
|
|
os.path.join(FILES, 'file', 'base'),
|
|
# Let's support runtime created files that can be used like:
|
|
# salt://my-temp-file.txt
|
|
RUNTIME_VARS.TMP_STATE_TREE
|
|
],
|
|
# Alternate root to test __env__ choices
|
|
'prod': [
|
|
os.path.join(FILES, 'file', 'prod'),
|
|
RUNTIME_VARS.TMP_PRODENV_STATE_TREE
|
|
]
|
|
}
|
|
minion_opts['file_roots'] = {
|
|
'base': [
|
|
os.path.join(FILES, 'file', 'base'),
|
|
# Let's support runtime created files that can be used like:
|
|
# salt://my-temp-file.txt
|
|
RUNTIME_VARS.TMP_STATE_TREE
|
|
],
|
|
# Alternate root to test __env__ choices
|
|
'prod': [
|
|
os.path.join(FILES, 'file', 'prod'),
|
|
RUNTIME_VARS.TMP_PRODENV_STATE_TREE
|
|
]
|
|
}
|
|
master_opts.setdefault('reactor', []).append(
|
|
{
|
|
'salt/minion/*/start': [
|
|
os.path.join(FILES, 'reactor-sync-minion.sls')
|
|
],
|
|
}
|
|
)
|
|
for opts_dict in (master_opts, syndic_master_opts):
|
|
if 'ext_pillar' not in opts_dict:
|
|
opts_dict['ext_pillar'] = []
|
|
if salt.utils.platform.is_windows():
|
|
opts_dict['ext_pillar'].append(
|
|
{'cmd_yaml': 'type {0}'.format(os.path.join(FILES, 'ext.yaml'))})
|
|
else:
|
|
opts_dict['ext_pillar'].append(
|
|
{'cmd_yaml': 'cat {0}'.format(os.path.join(FILES, 'ext.yaml'))})
|
|
|
|
# all read, only owner write
|
|
autosign_file_permissions = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
|
|
for opts_dict in (master_opts, syndic_master_opts):
|
|
# We need to copy the extension modules into the new master root_dir or
|
|
# it will be prefixed by it
|
|
new_extension_modules_path = os.path.join(opts_dict['root_dir'], 'extension_modules')
|
|
if not os.path.exists(new_extension_modules_path):
|
|
shutil.copytree(
|
|
os.path.join(
|
|
INTEGRATION_TEST_DIR, 'files', 'extension_modules'
|
|
),
|
|
new_extension_modules_path
|
|
)
|
|
opts_dict['extension_modules'] = os.path.join(opts_dict['root_dir'], 'extension_modules')
|
|
|
|
# Copy the autosign_file to the new master root_dir
|
|
new_autosign_file_path = os.path.join(opts_dict['root_dir'], 'autosign_file')
|
|
shutil.copyfile(
|
|
os.path.join(INTEGRATION_TEST_DIR, 'files', 'autosign_file'),
|
|
new_autosign_file_path
|
|
)
|
|
os.chmod(new_autosign_file_path, autosign_file_permissions)
|
|
|
|
# Point the config values to the correct temporary paths
|
|
for name in ('hosts', 'aliases'):
|
|
optname = '{0}.file'.format(name)
|
|
optname_path = os.path.join(TMP, name)
|
|
master_opts[optname] = optname_path
|
|
minion_opts[optname] = optname_path
|
|
sub_minion_opts[optname] = optname_path
|
|
syndic_opts[optname] = optname_path
|
|
syndic_master_opts[optname] = optname_path
|
|
proxy_opts[optname] = optname_path
|
|
|
|
master_opts['runtests_conn_check_port'] = get_unused_localhost_port()
|
|
minion_opts['runtests_conn_check_port'] = get_unused_localhost_port()
|
|
sub_minion_opts['runtests_conn_check_port'] = get_unused_localhost_port()
|
|
syndic_opts['runtests_conn_check_port'] = get_unused_localhost_port()
|
|
syndic_master_opts['runtests_conn_check_port'] = get_unused_localhost_port()
|
|
proxy_opts['runtests_conn_check_port'] = get_unused_localhost_port()
|
|
|
|
for conf in (master_opts, minion_opts, sub_minion_opts, syndic_opts, syndic_master_opts, proxy_opts):
|
|
if 'engines' not in conf:
|
|
conf['engines'] = []
|
|
conf['engines'].append({'salt_runtests': {}})
|
|
|
|
if 'engines_dirs' not in conf:
|
|
conf['engines_dirs'] = []
|
|
|
|
conf['engines_dirs'].insert(0, ENGINES_DIR)
|
|
|
|
if 'log_handlers_dirs' not in conf:
|
|
conf['log_handlers_dirs'] = []
|
|
conf['log_handlers_dirs'].insert(0, LOG_HANDLERS_DIR)
|
|
conf['runtests_log_port'] = SALT_LOG_PORT
|
|
|
|
# ----- Transcribe Configuration ---------------------------------------------------------------------------->
|
|
for entry in os.listdir(RUNTIME_VARS.CONF_DIR):
|
|
if entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master', 'proxy'):
|
|
# These have runtime computed values and will be handled
|
|
# differently
|
|
continue
|
|
entry_path = os.path.join(RUNTIME_VARS.CONF_DIR, entry)
|
|
if os.path.isfile(entry_path):
|
|
shutil.copy(
|
|
entry_path,
|
|
os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry)
|
|
)
|
|
elif os.path.isdir(entry_path):
|
|
shutil.copytree(
|
|
entry_path,
|
|
os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry)
|
|
)
|
|
|
|
for entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master', 'proxy'):
|
|
computed_config = copy.deepcopy(locals()['{0}_opts'.format(entry)])
|
|
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry), 'w') as fp_:
|
|
salt.utils.yaml.safe_dump(computed_config, fp_, default_flow_style=False)
|
|
sub_minion_computed_config = copy.deepcopy(sub_minion_opts)
|
|
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'minion'), 'w') as wfh:
|
|
salt.utils.yaml.safe_dump(sub_minion_computed_config, wfh, default_flow_style=False)
|
|
shutil.copyfile(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'master'), os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'master'))
|
|
|
|
syndic_master_computed_config = copy.deepcopy(syndic_master_opts)
|
|
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, 'master'), 'w') as wfh:
|
|
salt.utils.yaml.safe_dump(syndic_master_computed_config, wfh, default_flow_style=False)
|
|
syndic_computed_config = copy.deepcopy(syndic_opts)
|
|
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'minion'), 'w') as wfh:
|
|
salt.utils.yaml.safe_dump(syndic_computed_config, wfh, default_flow_style=False)
|
|
shutil.copyfile(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'master'), os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'master'))
|
|
# <---- Transcribe Configuration -----------------------------------------------------------------------------
|
|
|
|
# ----- Verify Environment ---------------------------------------------------------------------------------->
|
|
master_opts = salt.config.master_config(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'master'))
|
|
minion_opts = salt.config.minion_config(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'minion'))
|
|
syndic_opts = salt.config.syndic_config(
|
|
os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'master'),
|
|
os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'minion'),
|
|
)
|
|
sub_minion_opts = salt.config.minion_config(os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'minion'))
|
|
syndic_master_opts = salt.config.master_config(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, 'master'))
|
|
proxy_opts = salt.config.proxy_config(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'proxy'))
|
|
|
|
RUNTIME_VARS.RUNTIME_CONFIGS['master'] = freeze(master_opts)
|
|
RUNTIME_VARS.RUNTIME_CONFIGS['minion'] = freeze(minion_opts)
|
|
RUNTIME_VARS.RUNTIME_CONFIGS['syndic'] = freeze(syndic_opts)
|
|
RUNTIME_VARS.RUNTIME_CONFIGS['sub_minion'] = freeze(sub_minion_opts)
|
|
RUNTIME_VARS.RUNTIME_CONFIGS['syndic_master'] = freeze(syndic_master_opts)
|
|
RUNTIME_VARS.RUNTIME_CONFIGS['proxy'] = freeze(proxy_opts)
|
|
|
|
verify_env([os.path.join(master_opts['pki_dir'], 'minions'),
|
|
os.path.join(master_opts['pki_dir'], 'minions_pre'),
|
|
os.path.join(master_opts['pki_dir'], 'minions_rejected'),
|
|
os.path.join(master_opts['pki_dir'], 'minions_denied'),
|
|
os.path.join(master_opts['cachedir'], 'jobs'),
|
|
os.path.join(master_opts['cachedir'], 'raet'),
|
|
os.path.join(master_opts['root_dir'], 'cache', 'tokens'),
|
|
os.path.join(syndic_master_opts['pki_dir'], 'minions'),
|
|
os.path.join(syndic_master_opts['pki_dir'], 'minions_pre'),
|
|
os.path.join(syndic_master_opts['pki_dir'], 'minions_rejected'),
|
|
os.path.join(syndic_master_opts['cachedir'], 'jobs'),
|
|
os.path.join(syndic_master_opts['cachedir'], 'raet'),
|
|
os.path.join(syndic_master_opts['root_dir'], 'cache', 'tokens'),
|
|
os.path.join(master_opts['pki_dir'], 'accepted'),
|
|
os.path.join(master_opts['pki_dir'], 'rejected'),
|
|
os.path.join(master_opts['pki_dir'], 'pending'),
|
|
os.path.join(syndic_master_opts['pki_dir'], 'accepted'),
|
|
os.path.join(syndic_master_opts['pki_dir'], 'rejected'),
|
|
os.path.join(syndic_master_opts['pki_dir'], 'pending'),
|
|
os.path.join(syndic_master_opts['cachedir'], 'raet'),
|
|
|
|
os.path.join(minion_opts['pki_dir'], 'accepted'),
|
|
os.path.join(minion_opts['pki_dir'], 'rejected'),
|
|
os.path.join(minion_opts['pki_dir'], 'pending'),
|
|
os.path.join(minion_opts['cachedir'], 'raet'),
|
|
os.path.join(sub_minion_opts['pki_dir'], 'accepted'),
|
|
os.path.join(sub_minion_opts['pki_dir'], 'rejected'),
|
|
os.path.join(sub_minion_opts['pki_dir'], 'pending'),
|
|
os.path.join(sub_minion_opts['cachedir'], 'raet'),
|
|
os.path.dirname(master_opts['log_file']),
|
|
minion_opts['extension_modules'],
|
|
sub_minion_opts['extension_modules'],
|
|
sub_minion_opts['pki_dir'],
|
|
master_opts['sock_dir'],
|
|
syndic_master_opts['sock_dir'],
|
|
sub_minion_opts['sock_dir'],
|
|
minion_opts['sock_dir'],
|
|
RUNTIME_VARS.TMP_STATE_TREE,
|
|
RUNTIME_VARS.TMP_PILLAR_TREE,
|
|
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
|
|
TMP,
|
|
],
|
|
RUNTIME_VARS.RUNNING_TESTS_USER,
|
|
root_dir=master_opts['root_dir'],
|
|
)
|
|
|
|
cls.master_opts = master_opts
|
|
cls.minion_opts = minion_opts
|
|
# cls.proxy_opts = proxy_opts
|
|
cls.sub_minion_opts = sub_minion_opts
|
|
cls.syndic_opts = syndic_opts
|
|
cls.syndic_master_opts = syndic_master_opts
|
|
cls.proxy_opts = proxy_opts
|
|
# <---- Verify Environment -----------------------------------------------------------------------------------
|
|
|
|
def __exit__(self, type, value, traceback):
|
|
'''
|
|
Kill the minion and master processes
|
|
'''
|
|
self.sub_minion_process.terminate()
|
|
self.minion_process.terminate()
|
|
if hasattr(self, 'proxy_process'):
|
|
self.proxy_process.terminate()
|
|
self.master_process.terminate()
|
|
try:
|
|
self.syndic_process.terminate()
|
|
except AttributeError:
|
|
pass
|
|
try:
|
|
self.smaster_process.terminate()
|
|
except AttributeError:
|
|
pass
|
|
#salt.utils.process.clean_proc(self.sub_minion_process, wait_for_kill=50)
|
|
#self.sub_minion_process.join()
|
|
#salt.utils.process.clean_proc(self.minion_process, wait_for_kill=50)
|
|
#self.minion_process.join()
|
|
#salt.utils.process.clean_proc(self.master_process, wait_for_kill=50)
|
|
#self.master_process.join()
|
|
#try:
|
|
# salt.utils.process.clean_proc(self.syndic_process, wait_for_kill=50)
|
|
# self.syndic_process.join()
|
|
#except AttributeError:
|
|
# pass
|
|
#try:
|
|
# salt.utils.process.clean_proc(self.smaster_process, wait_for_kill=50)
|
|
# self.smaster_process.join()
|
|
#except AttributeError:
|
|
# pass
|
|
self.log_server.server_close()
|
|
self.log_server.shutdown()
|
|
self._exit_mockbin()
|
|
self._exit_ssh()
|
|
self.log_server_process.join()
|
|
# Shutdown the multiprocessing logging queue listener
|
|
salt_log_setup.shutdown_multiprocessing_logging()
|
|
salt_log_setup.shutdown_multiprocessing_logging_listener(daemonizing=True)
|
|
|
|
def pre_setup_minions(self):
|
|
'''
|
|
Subclass this method for additional minion setups.
|
|
'''
|
|
|
|
def setup_minions(self):
|
|
'''
|
|
Minions setup routines
|
|
'''
|
|
|
|
def post_setup_minions(self):
|
|
'''
|
|
Subclass this method to execute code after the minions have been setup
|
|
'''
|
|
|
|
def _enter_mockbin(self):
|
|
path = os.environ.get('PATH', '')
|
|
path_items = path.split(os.pathsep)
|
|
if MOCKBIN not in path_items:
|
|
path_items.insert(0, MOCKBIN)
|
|
os.environ['PATH'] = os.pathsep.join(path_items)
|
|
|
|
def _exit_ssh(self):
|
|
if hasattr(self, 'sshd_process'):
|
|
try:
|
|
self.sshd_process.kill()
|
|
except OSError as exc:
|
|
if exc.errno != 3:
|
|
raise
|
|
with salt.utils.files.fopen(self.sshd_pidfile) as fhr:
|
|
try:
|
|
os.kill(int(fhr.read()), signal.SIGKILL)
|
|
except OSError as exc:
|
|
if exc.errno != 3:
|
|
raise
|
|
|
|
def _exit_mockbin(self):
|
|
path = os.environ.get('PATH', '')
|
|
path_items = path.split(os.pathsep)
|
|
try:
|
|
path_items.remove(MOCKBIN)
|
|
except ValueError:
|
|
pass
|
|
os.environ['PATH'] = os.pathsep.join(path_items)
|
|
|
|
@classmethod
|
|
def clean(cls):
|
|
'''
|
|
Clean out the tmp files
|
|
'''
|
|
def remove_readonly(func, path, excinfo):
|
|
# Give full permissions to owner
|
|
os.chmod(path, stat.S_IRWXU)
|
|
func(path)
|
|
|
|
for dirname in (TMP, RUNTIME_VARS.TMP_STATE_TREE,
|
|
RUNTIME_VARS.TMP_PILLAR_TREE, RUNTIME_VARS.TMP_PRODENV_STATE_TREE):
|
|
if os.path.isdir(dirname):
|
|
shutil.rmtree(six.text_type(dirname), onerror=remove_readonly)
|
|
|
|
def wait_for_jid(self, targets, jid, timeout=120):
|
|
time.sleep(1) # Allow some time for minions to accept jobs
|
|
now = datetime.now()
|
|
expire = now + timedelta(seconds=timeout)
|
|
job_finished = False
|
|
while now <= expire:
|
|
running = self.__client_job_running(targets, jid)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
if not running and job_finished is False:
|
|
# Let's not have false positives and wait one more seconds
|
|
job_finished = True
|
|
elif not running and job_finished is True:
|
|
return True
|
|
elif running and job_finished is True:
|
|
job_finished = False
|
|
|
|
if job_finished is False:
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}[Quit in {0}]{ENDC} Waiting for {1}'.format(
|
|
'{0}'.format(expire - now).rsplit('.', 1)[0],
|
|
', '.join(running),
|
|
**self.colors
|
|
)
|
|
)
|
|
sys.stdout.flush()
|
|
time.sleep(1)
|
|
now = datetime.now()
|
|
else: # pylint: disable=W0120
|
|
sys.stdout.write(
|
|
'\n {LIGHT_RED}*{ENDC} ERROR: Failed to get information '
|
|
'back\n'.format(**self.colors)
|
|
)
|
|
sys.stdout.flush()
|
|
return False
|
|
|
|
def __client_job_running(self, targets, jid):
|
|
running = self.client.cmd(
|
|
list(targets), 'saltutil.running', tgt_type='list'
|
|
)
|
|
return [
|
|
k for (k, v) in six.iteritems(running) if v and v[0]['jid'] == jid
|
|
]
|
|
|
|
def wait_for_minion_connections(self, targets, timeout):
|
|
salt.utils.process.appendproctitle('WaitForMinionConnections')
|
|
sys.stdout.write(
|
|
' {LIGHT_BLUE}*{ENDC} Waiting at most {0} for minions({1}) to '
|
|
'connect back\n'.format(
|
|
(timeout > 60 and
|
|
timedelta(seconds=timeout) or
|
|
'{0} secs'.format(timeout)),
|
|
', '.join(targets),
|
|
**self.colors
|
|
)
|
|
)
|
|
sys.stdout.flush()
|
|
expected_connections = set(targets)
|
|
now = datetime.now()
|
|
expire = now + timedelta(seconds=timeout)
|
|
while now <= expire:
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns', PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' * {LIGHT_YELLOW}[Quit in {0}]{ENDC} Waiting for {1}'.format(
|
|
'{0}'.format(expire - now).rsplit('.', 1)[0],
|
|
', '.join(expected_connections),
|
|
**self.colors
|
|
)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
try:
|
|
responses = self.client.cmd(
|
|
list(expected_connections), 'test.ping', tgt_type='list',
|
|
)
|
|
# we'll get this exception if the master process hasn't finished starting yet
|
|
except SaltClientError:
|
|
time.sleep(0.1)
|
|
now = datetime.now()
|
|
continue
|
|
for target in responses:
|
|
if target not in expected_connections:
|
|
# Someone(minion) else "listening"?
|
|
continue
|
|
expected_connections.remove(target)
|
|
sys.stdout.write(
|
|
'\r{0}\r'.format(
|
|
' ' * getattr(self.parser.options, 'output_columns',
|
|
PNUM)
|
|
)
|
|
)
|
|
sys.stdout.write(
|
|
' {LIGHT_GREEN}*{ENDC} {0} connected.\n'.format(
|
|
target, **self.colors
|
|
)
|
|
)
|
|
sys.stdout.flush()
|
|
|
|
if not expected_connections:
|
|
return
|
|
|
|
time.sleep(1)
|
|
now = datetime.now()
|
|
else: # pylint: disable=W0120
|
|
print(
|
|
'\n {LIGHT_RED}*{ENDC} WARNING: Minions failed to connect '
|
|
'back. Tests requiring them WILL fail'.format(**self.colors)
|
|
)
|
|
try:
|
|
print_header(
|
|
'=', sep='=', inline=True,
|
|
width=getattr(self.parser.options, 'output_columns', PNUM)
|
|
|
|
)
|
|
except TypeError:
|
|
print_header('=', sep='=', inline=True)
|
|
raise SystemExit()
|
|
|
|
def sync_minion_modules_(self, modules_kind, targets, timeout=None):
|
|
if not timeout:
|
|
timeout = 120
|
|
# Let's sync all connected minions
|
|
print(
|
|
' {LIGHT_BLUE}*{ENDC} Syncing minion\'s {1} '
|
|
'(saltutil.sync_{1})'.format(
|
|
', '.join(targets),
|
|
modules_kind,
|
|
**self.colors
|
|
)
|
|
)
|
|
syncing = set(targets)
|
|
jid_info = self.client.run_job(
|
|
list(targets), 'saltutil.sync_{0}'.format(modules_kind),
|
|
tgt_type='list',
|
|
timeout=999999999999999,
|
|
)
|
|
|
|
if self.wait_for_jid(targets, jid_info['jid'], timeout) is False:
|
|
print(
|
|
' {LIGHT_RED}*{ENDC} WARNING: Minions failed to sync {0}. '
|
|
'Tests requiring these {0} WILL fail'.format(
|
|
modules_kind, **self.colors)
|
|
)
|
|
raise SystemExit()
|
|
|
|
while syncing:
|
|
rdata = self.client.get_full_returns(jid_info['jid'], syncing, 1)
|
|
if rdata:
|
|
for name, output in six.iteritems(rdata):
|
|
if not output['ret']:
|
|
# Already synced!?
|
|
syncing.remove(name)
|
|
continue
|
|
|
|
if isinstance(output['ret'], six.string_types):
|
|
# An errors has occurred
|
|
print(
|
|
' {LIGHT_RED}*{ENDC} {0} Failed to sync {2}: '
|
|
'{1}'.format(
|
|
name, output['ret'],
|
|
modules_kind,
|
|
**self.colors)
|
|
)
|
|
return False
|
|
|
|
print(
|
|
' {LIGHT_GREEN}*{ENDC} Synced {0} {2}: '
|
|
'{1}'.format(
|
|
name,
|
|
', '.join(output['ret']),
|
|
modules_kind, **self.colors
|
|
)
|
|
)
|
|
# Synced!
|
|
try:
|
|
syncing.remove(name)
|
|
except KeyError:
|
|
print(
|
|
' {LIGHT_RED}*{ENDC} {0} already synced??? '
|
|
'{1}'.format(name, output, **self.colors)
|
|
)
|
|
return True
|
|
|
|
def sync_minion_states(self, targets, timeout=None):
|
|
salt.utils.process.appendproctitle('SyncMinionStates')
|
|
self.sync_minion_modules_('states', targets, timeout=timeout)
|
|
|
|
def sync_minion_modules(self, targets, timeout=None):
|
|
salt.utils.process.appendproctitle('SyncMinionModules')
|
|
self.sync_minion_modules_('modules', targets, timeout=timeout)
|
|
|
|
def sync_minion_grains(self, targets, timeout=None):
|
|
salt.utils.process.appendproctitle('SyncMinionGrains')
|
|
self.sync_minion_modules_('grains', targets, timeout=timeout)
|