support one more old plugin path style

This commit is contained in:
Alexey Lavrenuke 2016-12-26 19:00:21 +03:00
parent 08e007657c
commit c6d04eb9fa
2 changed files with 130 additions and 96 deletions

View File

@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='yandextank',
version='1.8.29-4',
version='1.8.29-5',
description='a performance measurement tool',
longer_description='''
Yandex.Tank is a performance measurement and load testing automatization tool.

View File

@ -31,11 +31,15 @@ from ..plugins.Aggregator import Plugin as AggregatorPlugin
from ..plugins.Monitoring import Plugin as MonitoringPlugin
from ..plugins.Telegraf import Plugin as TelegrafPlugin
logger = logging.getLogger(__name__)
class Job(object):
def __init__(self, name, description, task, version, config_copy, monitoring_plugin, aggregator_plugin, tank,
generator_plugin=None):
# type: (unicode, unicode, unicode, unicode, unicode, MonitoringPlugin, AggregatorPlugin, GeneratorPlugin) -> Job
# type: (unicode, unicode, unicode, unicode, unicode, MonitoringPlugin,
# AggregatorPlugin, GeneratorPlugin) -> Job
self.name = name
self.description = description
self.task = task
@ -77,7 +81,6 @@ class TankCore(object):
LOCK_DIR = '/var/lock'
def __init__(self, artifacts_base_dir=None, artifacts_dir_name=None):
self.log = logging.getLogger(__name__)
self.config = ConfigManager()
self.status = {}
self.plugins = []
@ -107,7 +110,7 @@ class TankCore(object):
def load_configs(self, configs):
""" Tells core to load configs set into options storage """
self.log.info("Loading configs...")
logger.info("Loading configs...")
self.config.load_files(configs)
dotted_options = []
for option, value in self.config.get_options(self.SECTION):
@ -126,13 +129,15 @@ class TankCore(object):
"""
Tells core to take plugin options and instantiate plugin classes
"""
self.log.info("Loading plugins...")
logger.info("Loading plugins...")
if not self.artifacts_base_dir:
self.artifacts_base_dir = os.path.expanduser(self.get_option(self.SECTION, "artifacts_base_dir", '.'))
self.artifacts_base_dir = os.path.expanduser(
self.get_option(self.SECTION, "artifacts_base_dir", '.'))
if not self.artifacts_dir_name:
self.artifacts_dir_name = self.get_option(self.SECTION, "artifacts_dir", "")
self.artifacts_dir_name = self.get_option(
self.SECTION, "artifacts_dir", "")
self.taskset_path = self.get_option(self.SECTION, 'taskset_path',
'taskset')
@ -141,36 +146,59 @@ class TankCore(object):
options = self.config.get_options(self.SECTION, self.PLUGIN_PREFIX)
for (plugin_name, plugin_path) in options:
if not plugin_path:
self.log.debug("Seems the plugin '%s' was disabled",
plugin_name)
logger.debug("Seems the plugin '%s' was disabled",
plugin_name)
continue
self.log.debug("Loading plugin %s from %s", plugin_name,
plugin_path)
logger.debug("Loading plugin %s from %s", plugin_name,
plugin_path)
# FIXME cleanup an old deprecated plugin path format
if '/' in plugin_path:
self.log.warning("Deprecated plugin path format: %s\n"
"Should be in pythonic format. Example:\n"
" plugin_jmeter=yandextank.plugins.JMeter",
plugin_path)
logger.warning("Deprecated plugin path format: %s\n"
"Should be in pythonic format. Example:\n"
" plugin_jmeter=yandextank.plugins.JMeter",
plugin_path)
if plugin_path.startswith("Tank/Plugins/"):
plugin_path = "yandextank.plugins." + \
plugin_path.split('/')[-1].split('.')[0]
self.log.warning("Converted plugin path to %s",
plugin_path)
logger.warning("Converted plugin path to %s",
plugin_path)
else:
raise ValueError(
"Couldn't convert plugin path to new format:\n %s" %
plugin_path)
plugin = il.import_module(plugin_path)
try:
plugin = il.import_module(plugin_path)
except ImportError:
if plugin_path.startswith("yatank_internal_"):
logger.warning("Deprecated plugin path format: %s\n"
"Tank plugins are now orginized using"
" namespace packages. Example:\n"
" plugin_jmeter=yandextank.plugins.JMeter",
plugin_path)
plugin_path = plugin_path.replace(
"yatank_internal_", "yandextank.plugins.")
if plugin_path.startswith("yatank_"):
logger.warning("Deprecated plugin path format: %s\n"
"Tank plugins are now orginized using"
" namespace packages. Example:\n"
" plugin_jmeter=yandextank.plugins.JMeter",
plugin_path)
plugin_path = plugin_path.replace(
"yatank_", "yandextank.plugins.")
logger.warning("Patched plugin path: %s", plugin_path)
plugin = il.import_module(plugin_path)
try:
instance = getattr(plugin, 'Plugin')(self)
except:
self.log.warning("Deprecated plugin classname: %s. Should be 'Plugin'", plugin)
instance = getattr(plugin, plugin_path.split('.')[-1] + 'Plugin')(self)
logger.warning(
"Deprecated plugin classname: %s. Should be 'Plugin'", plugin)
instance = getattr(plugin, plugin_path.split(
'.')[-1] + 'Plugin')(self)
self.plugins.append(instance)
self.log.debug("Plugin instances: %s", self.plugins)
logger.debug("Plugin instances: %s", self.plugins)
def plugins_configure(self):
""" Call configure() on all plugins """
@ -180,7 +208,7 @@ class TankCore(object):
os.chmod(self.artifacts_base_dir, 0o755)
self.log.info("Configuring plugins...")
logger.info("Configuring plugins...")
if self.taskset_affinity != '':
self.taskset(os.getpid(), self.taskset_path, self.taskset_affinity)
@ -188,39 +216,43 @@ class TankCore(object):
try:
mon = self.get_plugin_of_type(TelegrafPlugin)
except KeyError:
self.log.debug("Telegraf plugin not found:", exc_info=True)
logger.debug("Telegraf plugin not found:", exc_info=True)
try:
mon = self.get_plugin_of_type(MonitoringPlugin)
except KeyError:
self.log.debug("Monitoring plugin not found:", exc_info=True)
logger.debug("Monitoring plugin not found:", exc_info=True)
mon = None
# aggregator plugin
try:
aggregator = self.get_plugin_of_type(AggregatorPlugin)
except KeyError:
self.log.warning("Aggregator plugin not found:", exc_info=True)
logger.warning("Aggregator plugin not found:", exc_info=True)
aggregator = None
# generator plugin
try:
gen = self.get_plugin_of_type(GeneratorPlugin)
except KeyError:
self.log.warning("Load generator not found:", exc_info=True)
logger.warning("Load generator not found:", exc_info=True)
gen = None
self.job = Job(name=self.get_option(self.SECTION_META, "job_name", 'none').decode('utf8'),
description=self.get_option(self.SECTION_META, "job_dsc", '').decode('utf8'),
task=self.get_option(self.SECTION_META, 'task', 'dir').decode('utf8'),
version=self.get_option(self.SECTION_META, 'ver', '').decode('utf8'),
config_copy=self.get_option(self.SECTION_META, 'copy_config_to', 'config_copy'),
description=self.get_option(
self.SECTION_META, "job_dsc", '').decode('utf8'),
task=self.get_option(
self.SECTION_META, 'task', 'dir').decode('utf8'),
version=self.get_option(
self.SECTION_META, 'ver', '').decode('utf8'),
config_copy=self.get_option(
self.SECTION_META, 'copy_config_to', 'config_copy'),
monitoring_plugin=mon,
aggregator_plugin=aggregator,
generator_plugin=gen,
tank=socket.getfqdn())
for plugin in self.plugins:
self.log.debug("Configuring %s", plugin)
logger.debug("Configuring %s", plugin)
plugin.configure()
self.config.flush()
if self.flush_config_to:
@ -228,20 +260,20 @@ class TankCore(object):
def plugins_prepare_test(self):
""" Call prepare_test() on all plugins """
self.log.info("Preparing test...")
logger.info("Preparing test...")
self.publish("core", "stage", "prepare")
for plugin in self.plugins:
self.log.debug("Preparing %s", plugin)
logger.debug("Preparing %s", plugin)
plugin.prepare_test()
if self.flush_config_to:
self.config.flush(self.flush_config_to)
def plugins_start_test(self):
""" Call start_test() on all plugins """
self.log.info("Starting test...")
logger.info("Starting test...")
self.publish("core", "stage", "start")
for plugin in self.plugins:
self.log.debug("Starting %s", plugin)
logger.debug("Starting %s", plugin)
plugin.start_test()
if self.flush_config_to:
self.config.flush(self.flush_config_to)
@ -251,7 +283,7 @@ class TankCore(object):
Call is_test_finished() on all plugins 'till one of them initiates exit
"""
self.log.info("Waiting for test to finish...")
logger.info("Waiting for test to finish...")
self.publish("core", "stage", "shoot")
if not self.plugins:
raise RuntimeError("It's strange: we have no plugins loaded...")
@ -259,16 +291,16 @@ class TankCore(object):
while not self.interrupted:
begin_time = time.time()
for plugin in self.plugins:
self.log.debug("Polling %s", plugin)
logger.debug("Polling %s", plugin)
retcode = plugin.is_test_finished()
if retcode >= 0:
return retcode
end_time = time.time()
diff = end_time - begin_time
self.log.debug("Polling took %s", diff)
self.log.debug("Tank status:\n%s",
json.dumps(self.status,
indent=2))
logger.debug("Polling took %s", diff)
logger.debug("Tank status:\n%s",
json.dumps(self.status,
indent=2))
# screen refresh every 0.5 s
if diff < 0.5:
time.sleep(0.5 - diff)
@ -276,19 +308,19 @@ class TankCore(object):
def plugins_end_test(self, retcode):
""" Call end_test() on all plugins """
self.log.info("Finishing test...")
logger.info("Finishing test...")
self.publish("core", "stage", "end")
for plugin in self.plugins:
self.log.debug("Finalize %s", plugin)
logger.debug("Finalize %s", plugin)
try:
self.log.debug("RC before: %s", retcode)
logger.debug("RC before: %s", retcode)
plugin.end_test(retcode)
self.log.debug("RC after: %s", retcode)
logger.debug("RC after: %s", retcode)
except Exception as ex:
self.log.error("Failed finishing plugin %s: %s", plugin, ex)
self.log.debug("Failed finishing plugin: %s",
traceback.format_exc(ex))
logger.error("Failed finishing plugin %s: %s", plugin, ex)
logger.debug("Failed finishing plugin: %s",
traceback.format_exc(ex))
if not retcode:
retcode = 1
@ -300,20 +332,20 @@ class TankCore(object):
"""
Call post_process() on all plugins
"""
self.log.info("Post-processing test...")
logger.info("Post-processing test...")
self.publish("core", "stage", "post_process")
for plugin in self.plugins:
self.log.debug("Post-process %s", plugin)
logger.debug("Post-process %s", plugin)
try:
self.log.debug("RC before: %s", retcode)
logger.debug("RC before: %s", retcode)
retcode = plugin.post_process(retcode)
self.log.debug("RC after: %s", retcode)
logger.debug("RC after: %s", retcode)
except Exception as ex:
self.log.error("Failed post-processing plugin %s: %s", plugin,
ex)
self.log.debug("Failed post-processing plugin: %s",
traceback.format_exc(ex))
logger.error("Failed post-processing plugin %s: %s", plugin,
ex)
logger.debug("Failed post-processing plugin: %s",
traceback.format_exc(ex))
if not retcode:
retcode = 1
@ -331,21 +363,21 @@ class TankCore(object):
shell=True,
poll_period=0.1,
catch_out=True)
self.log.debug('taskset stdout: %s', stdout)
logger.debug('taskset stdout: %s', stdout)
if retcode != 0:
raise KeyError(stderr)
else:
self.log.info("Enabled taskset for pid %s with affinity %s",
str(pid), affinity)
logger.info("Enabled taskset for pid %s with affinity %s",
str(pid), affinity)
def __collect_artifacts(self):
self.log.debug("Collecting artifacts")
self.log.info("Artifacts dir: %s", self.artifacts_dir)
logger.debug("Collecting artifacts")
logger.info("Artifacts dir: %s", self.artifacts_dir)
for filename, keep in self.artifact_files.items():
try:
self.__collect_file(filename, keep)
except Exception as ex:
self.log.warn("Failed to collect file %s: %s", filename, ex)
logger.warn("Failed to collect file %s: %s", filename, ex)
def get_option(self, section, option, default=None):
"""
@ -353,7 +385,7 @@ class TankCore(object):
and `set` if default specified.
"""
if not self.config.config.has_section(section):
self.log.debug("No section '%s', adding", section)
logger.debug("No section '%s', adding", section)
self.config.config.add_section(section)
try:
@ -365,13 +397,13 @@ class TankCore(object):
self.config.flush()
value = default.strip()
else:
self.log.warn(
logger.warn(
"Mandatory option %s was not found in section %s", option,
section)
raise ex
if len(value) > 1 and value[0] == '`' and value[-1] == '`':
self.log.debug("Expanding shell option %s", value)
logger.debug("Expanding shell option %s", value)
retcode, stdout, stderr = execute(value[1:-1], True, 0.1, True)
if retcode or stderr:
raise ValueError("Error expanding option %s, RC: %s" %
@ -393,13 +425,13 @@ class TankCore(object):
"""
Retrieve a plugin of desired class, KeyError raised otherwise
"""
self.log.debug("Searching for plugin: %s", plugin_class)
logger.debug("Searching for plugin: %s", plugin_class)
matches = [plugin
for plugin in self.plugins
if isinstance(plugin, plugin_class)]
if len(matches) > 0:
if len(matches) > 1:
self.log.debug(
logger.debug(
"More then one plugin of type %s found. Using first one.",
plugin_class)
return matches[-1]
@ -412,14 +444,14 @@ class TankCore(object):
Move or copy single file to artifacts dir
"""
dest = self.artifacts_dir + '/' + os.path.basename(filename)
self.log.debug("Collecting file: %s to %s", filename, dest)
logger.debug("Collecting file: %s to %s", filename, dest)
if not filename or not os.path.exists(filename):
self.log.warning("File not found to collect: %s", filename)
logger.warning("File not found to collect: %s", filename)
return
if os.path.exists(dest):
# FIXME: 3 find a way to store artifacts anyway
self.log.warning("File already exists: %s", dest)
logger.warning("File already exists: %s", dest)
return
if keep_original:
@ -434,8 +466,8 @@ class TankCore(object):
Add file to be stored as result artifact on post-process phase
"""
if filename:
self.log.debug("Adding artifact file to collect (keep=%s): %s",
keep_original, filename)
logger.debug("Adding artifact file to collect (keep=%s): %s",
keep_original, filename)
self.artifact_files[filename] = keep_original
def apply_shorthand_options(self, options, default_section='DEFAULT'):
@ -443,13 +475,13 @@ class TankCore(object):
try:
section = option_str[:option_str.index('.')]
option = option_str[
option_str.index('.') + 1:option_str.index('=')]
option_str.index('.') + 1:option_str.index('=')]
except ValueError:
section = default_section
option = option_str[:option_str.index('=')]
value = option_str[option_str.index('=') + 1:]
self.log.debug("Override option: %s => [%s] %s=%s", option_str,
section, option, value)
logger.debug("Override option: %s => [%s] %s=%s", option_str,
section, option, value)
self.set_option(section, option, value)
def get_lock_dir(self):
@ -473,7 +505,7 @@ class TankCore(object):
def release_lock(self):
self.config.file = None
if self.lock_file and os.path.exists(self.lock_file):
self.log.debug("Releasing lock: %s", self.lock_file)
logger.debug("Releasing lock: %s", self.lock_file)
os.remove(self.lock_file)
def __there_is_locks(self):
@ -482,25 +514,25 @@ class TankCore(object):
for filename in os.listdir(lock_dir):
if fnmatch.fnmatch(filename, 'lunapark_*.lock'):
full_name = os.path.join(lock_dir, filename)
self.log.warn("Lock file present: %s", full_name)
logger.warn("Lock file present: %s", full_name)
try:
info = ConfigParser.ConfigParser()
info.read(full_name)
pid = info.get(TankCore.SECTION, self.PID_OPTION)
if not pid_exists(int(pid)):
self.log.debug("Lock PID %s not exists, ignoring and "
"trying to remove", pid)
logger.debug("Lock PID %s not exists, ignoring and "
"trying to remove", pid)
try:
os.remove(full_name)
except Exception as exc:
self.log.debug("Failed to delete lock %s: %s",
full_name, exc)
logger.debug("Failed to delete lock %s: %s",
full_name, exc)
else:
retcode = True
except Exception as exc:
self.log.warn("Failed to load info from lock %s: %s",
full_name, exc)
logger.warn("Failed to load info from lock %s: %s",
full_name, exc)
retcode = True
return retcode
@ -523,16 +555,16 @@ class TankCore(object):
"""
Call close() for all plugins
"""
self.log.info("Close allocated resources...")
logger.info("Close allocated resources...")
for plugin in self.plugins:
self.log.debug("Close %s", plugin)
logger.debug("Close %s", plugin)
try:
plugin.close()
except Exception as ex:
self.log.error("Failed closing plugin %s: %s", plugin, ex)
self.log.debug("Failed closing plugin: %s",
traceback.format_exc(ex))
logger.error("Failed closing plugin %s: %s", plugin, ex)
logger.debug("Failed closing plugin: %s",
traceback.format_exc(ex))
@property
def artifacts_dir(self):
@ -549,9 +581,11 @@ class TankCore(object):
@staticmethod
def get_user_agent():
tank_agent = 'YandexTank/{}'.format(pkg_resources.require('yandextank')[0].version)
tank_agent = 'YandexTank/{}'.format(
pkg_resources.require('yandextank')[0].version)
py_info = sys.version_info
python_agent = 'Python/{}.{}.{}'.format(py_info[0], py_info[1], py_info[2])
python_agent = 'Python/{}.{}.{}'.format(
py_info[0], py_info[1], py_info[2])
os_agent = 'OS/{}'.format(platform.platform())
return ' '.join((tank_agent, python_agent, os_agent))
@ -561,18 +595,18 @@ class ConfigManager(object):
def __init__(self):
self.file = None
self.log = logging.getLogger(__name__)
logger = logging.getLogger(__name__)
self.config = ConfigParser.ConfigParser()
def load_files(self, configs):
""" Read configs set into storage """
self.log.debug("Reading configs: %s", configs)
logger.debug("Reading configs: %s", configs)
config_filenames = [resource.resource_filename(config)
for config in configs]
try:
self.config.read(config_filenames)
except Exception as ex:
self.log.error("Can't load configs: %s", ex)
logger.error("Can't load configs: %s", ex)
raise ex
def flush(self, filename=None):
@ -593,10 +627,10 @@ class ConfigManager(object):
res += [(option[len(prefix):],
self.config.get(section, option))]
except NoSectionError as ex:
self.log.warning("No section: %s", ex)
logger.warning("No section: %s", ex)
self.log.debug("Section: [%s] prefix: '%s' options:\n%s", section,
prefix, res)
logger.debug("Section: [%s] prefix: '%s' options:\n%s", section,
prefix, res)
return res
def find_sections(self, prefix):