mirror of
https://github.com/valitydev/salt.git
synced 2024-11-08 09:23:56 +00:00
b6cb78364f
While running the tests within a VirtualBox(vagrant) machine, if the salt source is mounted using shared folders, any tests involving hard links WILL fail. Any tests involving symlinks, can be made to work by setting some properties on the VirtualBox shared folder. By moving the tests `tmp` directory to `tempfile.gettempdir()` we avoid this false test case errors.
423 lines
13 KiB
Python
Executable File
423 lines
13 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
'''
|
|
Discover all instances of unittest.TestCase in this directory.
|
|
'''
|
|
# Import python libs
|
|
import sys
|
|
import os
|
|
import logging
|
|
import optparse
|
|
import resource
|
|
import tempfile
|
|
|
|
# Import salt libs
|
|
try:
|
|
import console
|
|
width, height = console.getTerminalSize()
|
|
PNUM = width
|
|
except:
|
|
PNUM = 70
|
|
import saltunittest
|
|
from integration import TestDaemon
|
|
|
|
try:
|
|
import xmlrunner
|
|
except ImportError:
|
|
xmlrunner = None
|
|
|
|
TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__)))
|
|
|
|
|
|
try:
|
|
import coverage
|
|
# Cover any subprocess
|
|
coverage.process_startup()
|
|
# Setup coverage
|
|
code_coverage = coverage.coverage(
|
|
branch=True,
|
|
source=[os.path.join(os.path.dirname(TEST_DIR), 'salt')],
|
|
)
|
|
except ImportError:
|
|
code_coverage = None
|
|
|
|
|
|
REQUIRED_OPEN_FILES = 2048
|
|
|
|
TEST_RESULTS = []
|
|
|
|
def print_header(header, sep='~', top=True, bottom=True, inline=False,
|
|
centered=False):
|
|
if top and not inline:
|
|
print(sep * PNUM)
|
|
|
|
if centered and not inline:
|
|
fmt = u'{0:^{width}}'
|
|
elif inline and not centered:
|
|
fmt = u'{0:{sep}<{width}}'
|
|
elif inline and centered:
|
|
fmt = u'{0:{sep}^{width}}'
|
|
else:
|
|
fmt = u'{0}'
|
|
print(fmt.format(header, sep=sep, width=PNUM))
|
|
|
|
if bottom and not inline:
|
|
print(sep * PNUM)
|
|
|
|
|
|
def run_suite(opts, path, display_name, suffix='[!_]*.py'):
|
|
'''
|
|
Execute a unit test suite
|
|
'''
|
|
loader = saltunittest.TestLoader()
|
|
if opts.name:
|
|
tests = loader.loadTestsFromName(display_name)
|
|
else:
|
|
tests = loader.discover(path, suffix, TEST_DIR)
|
|
|
|
header = '{0} Tests'.format(display_name)
|
|
print_header('Starting {0}'.format(header))
|
|
|
|
if opts.xmlout:
|
|
runner = xmlrunner.XMLTestRunner(output='test-reports').run(tests)
|
|
else:
|
|
runner = saltunittest.TextTestRunner(
|
|
verbosity=opts.verbosity
|
|
).run(tests)
|
|
TEST_RESULTS.append((header, runner))
|
|
return runner.wasSuccessful()
|
|
|
|
|
|
def run_integration_suite(opts, suite_folder, display_name):
|
|
'''
|
|
Run an integration test suite
|
|
'''
|
|
path = os.path.join(TEST_DIR, 'integration', suite_folder)
|
|
return run_suite(opts, path, display_name)
|
|
|
|
|
|
def run_integration_tests(opts):
|
|
'''
|
|
Execute the integration tests suite
|
|
'''
|
|
smax_open_files, hmax_open_files = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
if smax_open_files < REQUIRED_OPEN_FILES:
|
|
print('~' * PNUM)
|
|
print('Max open files setting is too low({0}) for running the tests'.format(smax_open_files))
|
|
print('Trying to raise the limit to {0}'.format(REQUIRED_OPEN_FILES))
|
|
if hmax_open_files < 4096:
|
|
hmax_open_files = 4096 # Decent default?
|
|
try:
|
|
resource.setrlimit(
|
|
resource.RLIMIT_NOFILE,
|
|
(REQUIRED_OPEN_FILES, hmax_open_files)
|
|
)
|
|
except Exception, err:
|
|
print('ERROR: Failed to raise the max open files setting -> {0}'.format(err))
|
|
print('Please issue the following command on your console:')
|
|
print(' ulimit -n {0}'.format(REQUIRED_OPEN_FILES))
|
|
sys.exit(1)
|
|
finally:
|
|
print('~' * PNUM)
|
|
|
|
print_header('Setting up Salt daemons to execute tests', top=False)
|
|
status = []
|
|
if not any([opts.client, opts.module, opts.runner,
|
|
opts.shell, opts.state, opts.name]):
|
|
return status
|
|
with TestDaemon(clean=opts.clean):
|
|
if opts.name:
|
|
for name in opts.name:
|
|
results = run_suite(opts, '', name)
|
|
status.append(results)
|
|
if opts.runner:
|
|
status.append(run_integration_suite(opts, 'runners', 'Runner'))
|
|
if opts.module:
|
|
status.append(run_integration_suite(opts, 'modules', 'Module'))
|
|
if opts.state:
|
|
status.append(run_integration_suite(opts, 'states', 'State'))
|
|
if opts.client:
|
|
status.append(run_integration_suite(opts, 'client', 'Client'))
|
|
if opts.shell:
|
|
status.append(run_integration_suite(opts, 'shell', 'Shell'))
|
|
return status
|
|
|
|
|
|
def run_unit_tests(opts):
|
|
'''
|
|
Execute the unit tests
|
|
'''
|
|
if not opts.unit:
|
|
return [True]
|
|
status = []
|
|
results = run_suite(
|
|
opts, os.path.join(TEST_DIR, 'unit'), 'Unit', '*_test.py')
|
|
status.append(results)
|
|
return status
|
|
|
|
|
|
def parse_opts():
|
|
'''
|
|
Parse command line options for running specific tests
|
|
'''
|
|
parser = optparse.OptionParser()
|
|
parser.add_option('-m',
|
|
'--module',
|
|
'--module-tests',
|
|
dest='module',
|
|
default=False,
|
|
action='store_true',
|
|
help='Run tests for modules')
|
|
parser.add_option('-S',
|
|
'--state',
|
|
'--state-tests',
|
|
dest='state',
|
|
default=False,
|
|
action='store_true',
|
|
help='Run tests for states')
|
|
parser.add_option('-c',
|
|
'--client',
|
|
'--client-tests',
|
|
dest='client',
|
|
default=False,
|
|
action='store_true',
|
|
help='Run tests for client')
|
|
parser.add_option('-s',
|
|
'--shell',
|
|
dest='shell',
|
|
default=False,
|
|
action='store_true',
|
|
help='Run shell tests')
|
|
parser.add_option('-r',
|
|
'--runner',
|
|
dest='runner',
|
|
default=False,
|
|
action='store_true',
|
|
help='Run runner tests')
|
|
parser.add_option('-u',
|
|
'--unit',
|
|
'--unit-tests',
|
|
dest='unit',
|
|
default=False,
|
|
action='store_true',
|
|
help='Run unit tests')
|
|
parser.add_option('-v',
|
|
'--verbose',
|
|
dest='verbosity',
|
|
default=1,
|
|
action='count',
|
|
help='Verbose test runner output')
|
|
parser.add_option('-x',
|
|
'--xml',
|
|
dest='xmlout',
|
|
default=False,
|
|
action='store_true',
|
|
help='XML test runner output')
|
|
parser.add_option('-n',
|
|
'--name',
|
|
dest='name',
|
|
action='append',
|
|
default=[],
|
|
help='Specific test name to run')
|
|
parser.add_option('--clean',
|
|
dest='clean',
|
|
default=True,
|
|
action='store_true',
|
|
help=('Clean up test environment before and after '
|
|
'integration testing (default behaviour)'))
|
|
parser.add_option('--no-clean',
|
|
dest='clean',
|
|
action='store_false',
|
|
help=('Don\'t clean up test environment before and after '
|
|
'integration testing (speed up test process)'))
|
|
parser.add_option('--run-destructive',
|
|
action='store_true',
|
|
default=False,
|
|
help='Run destructive tests. These tests can include adding or '
|
|
'removing users from your system for example. Default: '
|
|
'%default'
|
|
)
|
|
parser.add_option('--no-report',
|
|
default=False,
|
|
action='store_true',
|
|
help='Do NOT show the overall tests result'
|
|
)
|
|
|
|
parser.add_option('--coverage',
|
|
default=False,
|
|
action='store_true',
|
|
help='Run tests and report code coverage'
|
|
)
|
|
|
|
options, _ = parser.parse_args()
|
|
|
|
if options.xmlout and xmlrunner is None:
|
|
parser.error('\'--xml\' is not available. The xmlrunner library '
|
|
'is not installed.')
|
|
|
|
if options.coverage and code_coverage is None:
|
|
parser.error(
|
|
'Cannot run tests with coverage report. '
|
|
'Please install coverage>=3.5.3'
|
|
)
|
|
elif options.coverage:
|
|
coverage_version = tuple(
|
|
[int(part) for part in coverage.__version__.split('.')]
|
|
)
|
|
if coverage_version < (3, 5, 3):
|
|
# Should we just print the error instead of exiting?
|
|
parser.error(
|
|
'Versions lower than 3.5.3 of the coverage library are know '
|
|
'to produce incorrect results. Please consider upgrading...'
|
|
)
|
|
|
|
if any((options.module, options.client, options.shell, options.unit,
|
|
options.state, options.runner, options.name,
|
|
os.geteuid() is not 0, not options.run_destructive)):
|
|
parser.error(
|
|
'No sense in generating the tests coverage report when not '
|
|
'running the full test suite, including the destructive '
|
|
'tests, as \'root\'. It would only produce incorrect '
|
|
'results.'
|
|
)
|
|
|
|
# Update environ so that any subprocess started on test are also
|
|
# included in the report
|
|
os.environ['COVERAGE_PROCESS_START'] = '1'
|
|
|
|
# Setup logging
|
|
formatter = logging.Formatter(
|
|
'%(asctime)s,%(msecs)03.0f [%(name)-5s:%(lineno)-4d]'
|
|
'[%(levelname)-8s] %(message)s',
|
|
datefmt='%H:%M:%S'
|
|
)
|
|
logfile = os.path.join(tempfile.gettempdir(), 'salt-runtests.log')
|
|
filehandler = logging.FileHandler(
|
|
mode='w', # Not preserved between re-runs
|
|
filename=logfile
|
|
)
|
|
filehandler.setLevel(logging.DEBUG)
|
|
filehandler.setFormatter(formatter)
|
|
logging.root.addHandler(filehandler)
|
|
logging.root.setLevel(logging.DEBUG)
|
|
|
|
print_header('Logging tests on {0}'.format(logfile), bottom=False)
|
|
|
|
# With greater verbosity we can also log to the console
|
|
if options.verbosity > 2:
|
|
consolehandler = logging.StreamHandler(stream=sys.stderr)
|
|
consolehandler.setLevel(logging.INFO) # -vv
|
|
consolehandler.setFormatter(formatter)
|
|
if options.verbosity > 3:
|
|
consolehandler.setLevel(logging.DEBUG) # -vvv
|
|
|
|
logging.root.addHandler(consolehandler)
|
|
|
|
os.environ['DESTRUCTIVE_TESTS'] = str(options.run_destructive)
|
|
|
|
if not any((options.module, options.client,
|
|
options.shell, options.unit,
|
|
options.state, options.runner,
|
|
options.name)):
|
|
options.module = True
|
|
options.client = True
|
|
options.shell = True
|
|
options.unit = True
|
|
options.runner = True
|
|
options.state = True
|
|
return options
|
|
|
|
|
|
if __name__ == '__main__':
|
|
opts = parse_opts()
|
|
if opts.coverage:
|
|
code_coverage.start()
|
|
|
|
overall_status = []
|
|
status = run_integration_tests(opts)
|
|
overall_status.extend(status)
|
|
status = run_unit_tests(opts)
|
|
overall_status.extend(status)
|
|
false_count = overall_status.count(False)
|
|
|
|
if opts.no_report:
|
|
if opts.coverage:
|
|
code_coverage.stop()
|
|
code_coverage.save()
|
|
|
|
if false_count > 0:
|
|
sys.exit(1)
|
|
else:
|
|
sys.exit(0)
|
|
|
|
print
|
|
print_header(u' Overall Tests Report ', sep=u'=', centered=True, inline=True)
|
|
|
|
no_problems_found = True
|
|
for (name, results) in TEST_RESULTS:
|
|
if not results.failures and not results.errors and not results.skipped:
|
|
continue
|
|
|
|
no_problems_found = False
|
|
|
|
print_header(u'\u22c6\u22c6\u22c6 {0} '.format(name), sep=u'\u22c6', inline=True)
|
|
if results.skipped:
|
|
print_header(u' -------- Skipped Tests ', sep='-', inline=True)
|
|
maxlen = len(max([tc.id() for (tc, reason) in results.skipped], key=len))
|
|
fmt = u' \u2192 {0: <{maxlen}} \u2192 {1}'
|
|
for tc, reason in results.skipped:
|
|
print(fmt.format(tc.id(), reason, maxlen=maxlen))
|
|
print_header(u' ', sep='-', inline=True)
|
|
|
|
if results.errors:
|
|
print_header(u' -------- Tests with Errors ', sep='-', inline=True)
|
|
for tc, reason in results.errors:
|
|
print_header(u' \u2192 {0} '.format(tc.id()), sep=u'.', inline=True)
|
|
for line in reason.rstrip().splitlines():
|
|
print(' {0}'.format(line.rstrip()))
|
|
print_header(u' ', sep=u'.', inline=True)
|
|
print_header(u' ', sep='-', inline=True)
|
|
|
|
if results.failures:
|
|
print_header(u' -------- Failed Tests ', sep='-', inline=True)
|
|
for tc, reason in results.failures:
|
|
print_header(u' \u2192 {0} '.format(tc.id()), sep=u'.', inline=True)
|
|
for line in reason.rstrip().splitlines():
|
|
print(' {0}'.format(line.rstrip()))
|
|
print_header(u' ', sep=u'.', inline=True)
|
|
print_header(u' ', sep='-', inline=True)
|
|
|
|
print_header(u'', sep=u'\u22c6', inline=True)
|
|
|
|
if no_problems_found:
|
|
print_header(
|
|
u'\u22c6\u22c6\u22c6 No Problems Found While Running Tests ',
|
|
sep=u'\u22c6', inline=True
|
|
)
|
|
|
|
print_header(' Overall Tests Report ', sep='=', centered=True, inline=True)
|
|
|
|
if opts.coverage:
|
|
print('Stopping and saving coverage info')
|
|
code_coverage.stop()
|
|
code_coverage.save()
|
|
|
|
report_dir = os.path.join(os.path.dirname(__file__), 'coverage-report')
|
|
print(
|
|
'\nGenerating Coverage HTML Report Under {0!r} ...'.format(
|
|
report_dir
|
|
)
|
|
),
|
|
sys.stdout.flush()
|
|
|
|
if os.path.isdir(report_dir):
|
|
import shutil
|
|
shutil.rmtree(report_dir)
|
|
code_coverage.html_report(directory=report_dir)
|
|
print('Done.\n')
|
|
|
|
if false_count > 0:
|
|
sys.exit(1)
|
|
else:
|
|
sys.exit(0)
|