From bd8ea9a8e113cc216ff18183f7ea9a031a972489 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Thu, 13 Oct 2016 15:23:42 +0300 Subject: [PATCH 1/6] Update install.rst --- docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index c8f7c65..989b949 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -14,7 +14,7 @@ Docker container .. code-block:: bash - docker run -v $(pwd):/var/loadtest -v $HOME/.ssh:/home/yandextank/.ssh -it direvius/yandex-tank + docker run -v $(pwd):/var/loadtest -v $HOME/.ssh:/root/.ssh -it direvius/yandex-tank .. note:: From a3e2e4507a89d735cbb653ad454f252a9fdaddc3 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Thu, 13 Oct 2016 15:24:33 +0300 Subject: [PATCH 2/6] Update install.rst --- docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index 989b949..10b4991 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -14,7 +14,7 @@ Docker container .. code-block:: bash - docker run -v $(pwd):/var/loadtest -v $HOME/.ssh:/root/.ssh -it direvius/yandex-tank + docker run -v $(pwd):/var/loadtest -v $HOME/.ssh:/root/.ssh --net host -it direvius/yandex-tank .. note:: From b589f22938a8d426d61c747a0a8b92fb8eab8af5 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Fri, 14 Oct 2016 20:07:44 +0300 Subject: [PATCH 3/6] JsonReport plugin started; writing phantom info to core job --- yandextank/common/exceptions.py | 8 ++++ yandextank/core/tankcore.py | 26 +++++++++--- yandextank/plugins/JsonReport/__init__.py | 1 + yandextank/plugins/JsonReport/plugin.py | 52 +++++++++++++++++++++++ yandextank/plugins/Phantom/plugin.py | 8 ++-- 5 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 yandextank/plugins/JsonReport/__init__.py create mode 100644 yandextank/plugins/JsonReport/plugin.py diff --git a/yandextank/common/exceptions.py b/yandextank/common/exceptions.py index c2893a1..5eaee83 100644 --- a/yandextank/common/exceptions.py +++ b/yandextank/common/exceptions.py @@ -3,3 +3,11 @@ class PluginImplementationError(RuntimeError): Error in plugin implementation """ pass + + +class PluginNotPrepared(Exception): + """ + Can't find plugin's info in core.job + """ + def __init__(self, msg): + self.message = "%s\n%s" % (self.__doc__, msg) \ No newline at end of file diff --git a/yandextank/core/tankcore.py b/yandextank/core/tankcore.py index b9eeeb9..7fc3bab 100644 --- a/yandextank/core/tankcore.py +++ b/yandextank/core/tankcore.py @@ -11,9 +11,12 @@ import tempfile import time import traceback import uuid - import configparser +from builtins import str + from configparser import NoSectionError +from yandextank.common.exceptions import PluginNotPrepared + from ..common.util import update_status, execute, pid_exists from ..common.resource import manager as resource @@ -33,11 +36,22 @@ class Job(object): self.monitoring_plugin = monitoring_plugin self.aggregator_plugin = aggregator_plugin self.tank = tank + self._phantom_info = None def subscribe_plugin(self, plugin): self.aggregator_plugin.add_result_listener(plugin) self.monitoring_plugin.add_listener(plugin) + @property + def phantom_info(self): + if self._phantom_info is None: + raise PluginNotPrepared + return self._phantom_info + + @phantom_info.setter + def phantom_info(self, info): + self._phantom_info = info + class TankCore(object): """ @@ -172,11 +186,11 @@ class TankCore(object): self.log.warning("Aggregator plugin not found:", exc_info=True) aggregator = None - self.job = Job(name=self.get_option(self.SECTION_META, 'job_name', ''), - description=self.get_option(self.SECTION_META, 'job_dsc', ''), - task=self.get_option(self.SECTION_META, 'task', ''), - version=self.get_option(self.SECTION_META, 'version', ''), - config_copy=self.get_option(self.SECTION_META, 'config_copy', 'config_copy'), + self.job = Job(name=str(self.get_option(self.SECTION_META, "job_name", 'none').decode('utf8')), + description=str(self.get_option(self.SECTION_META, "job_dsc", '').decode('utf8')), + task=str(self.get_option(self.SECTION_META, 'task', 'dir')), + version=str(self.get_option(self.SECTION_META, 'ver', '')), + config_copy=self.get_option(self.SECTION_META, 'copy_config_to', 'config_copy'), monitoring_plugin=mon, aggregator_plugin=aggregator, tank=socket.getfqdn()) diff --git a/yandextank/plugins/JsonReport/__init__.py b/yandextank/plugins/JsonReport/__init__.py new file mode 100644 index 0000000..35c21bd --- /dev/null +++ b/yandextank/plugins/JsonReport/__init__.py @@ -0,0 +1 @@ +from .plugin import Plugin diff --git a/yandextank/plugins/JsonReport/plugin.py b/yandextank/plugins/JsonReport/plugin.py new file mode 100644 index 0000000..34b9be3 --- /dev/null +++ b/yandextank/plugins/JsonReport/plugin.py @@ -0,0 +1,52 @@ +# TODO: make the next two lines unnecessary +# pylint: disable=line-too-long +# pylint: disable=missing-docstring + +import logging + +from ..Aggregator import Plugin as AggregatorPlugin +from ..Monitoring import Plugin as MonitoringPlugin +from ..Telegraf import Plugin as TelegrafPlugin +from ...common.interfaces import AbstractPlugin, MonitoringDataListener, AggregateResultListener + +logger = logging.getLogger(__name__) # pylint: disable=C0103 + + +class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener): + # pylint:disable=R0902 + """ Yandex Overload analytics service client (https://overload.yandex.net) """ + SECTION = 'json_report' + + def get_available_options(self): + return [] + + def configure(self): + try: + aggregator = self.core.get_plugin_of_type(AggregatorPlugin) + except KeyError: + logger.debug("Aggregator plugin not found", exc_info=True) + else: + aggregator.add_result_listener(self) + + try: + self.mon = self.core.get_plugin_of_type(TelegrafPlugin) + except KeyError: + logger.debug("Telegraf plugin not found:", exc_info=True) + try: + self.mon = self.core.get_plugin_of_type(MonitoringPlugin) + except KeyError: + logger.debug("Monitoring plugin not found:", exc_info=True) + + if self.mon and self.mon.monitoring: + self.mon.monitoring.add_listener(self) + + def on_aggregated_data(self, data, stats): + """ + @data: aggregated data + @stats: stats about gun + """ + print(data) + print(stats) + + def monitoring_data(self, data_list): + print(data_list) \ No newline at end of file diff --git a/yandextank/plugins/Phantom/plugin.py b/yandextank/plugins/Phantom/plugin.py index 5253a73..92dc773 100644 --- a/yandextank/plugins/Phantom/plugin.py +++ b/yandextank/plugins/Phantom/plugin.py @@ -92,11 +92,7 @@ class Plugin(AbstractPlugin): self.phantom.read_config() def prepare_test(self): - aggregator = None - try: - aggregator = self.core.get_plugin_of_type(AggregatorPlugin) - except Exception as ex: - logger.warning("No aggregator found: %s", ex) + aggregator = self.core.job.aggregator_plugin if not self.config and not self.phout_import_mode: @@ -140,6 +136,8 @@ class Plugin(AbstractPlugin): logger.debug("Console not found: %s", ex) console = None + self.core.job.phantom_info = self.phantom.get_info() + if console and aggregator: widget = PhantomProgressBarWidget(self) console.add_info_widget(widget) From e0ceec51cac565d444165d43b173fa9700119d30 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Mon, 24 Oct 2016 21:02:38 +0300 Subject: [PATCH 4/6] reraise scenario exceptions after setting error code --- yandextank/plugins/Bfg/guns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yandextank/plugins/Bfg/guns.py b/yandextank/plugins/Bfg/guns.py index 6a6cbe9..09101de 100644 --- a/yandextank/plugins/Bfg/guns.py +++ b/yandextank/plugins/Bfg/guns.py @@ -45,6 +45,7 @@ class AbstractGun(AbstractPlugin): data_item["proto_code"] = 500 if data_item["net_code"] == 0: data_item["net_code"] == 1 + raise finally: if data_item.get("interval_real") is None: data_item["interval_real"] = int((time.time() - start_time) * From c55c375b0ee41f1b868eeaac5008dcfb9f9fce87 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Tue, 25 Oct 2016 20:18:30 +0300 Subject: [PATCH 5/6] json reporter ready, py3 compatible --- setup.py | 2 +- yandextank/core/tankcore.py | 6 +-- yandextank/plugins/Aggregator/plugin.py | 2 +- yandextank/plugins/JsonReport/plugin.py | 46 +++++++++++----------- yandextank/plugins/TipsAndTricks/plugin.py | 5 +-- yandextank/stepper/format.py | 6 +-- yandextank/stepper/main.py | 2 +- 7 files changed, 35 insertions(+), 34 deletions(-) diff --git a/setup.py b/setup.py index 71347ac..c5a9973 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ analytic tools for the results they produce. entry_points={ 'console_scripts': [ 'yandex-tank = yandextank.core.cli:main', - 'yandex-tank-check-ssh = yandextank.core.util:check_ssh_connection', + 'yandex-tank-check-ssh = yandextank.common.util:check_ssh_connection', ], }, package_data={ diff --git a/yandextank/core/tankcore.py b/yandextank/core/tankcore.py index 9e585bc..dd90482 100644 --- a/yandextank/core/tankcore.py +++ b/yandextank/core/tankcore.py @@ -199,8 +199,8 @@ class TankCore(object): self.log.warning("Load generator not found:", exc_info=True) gen = None - self.job = Job(name=str(self.get_option(self.SECTION_META, "job_name", 'none').decode('utf8')), - description=str(self.get_option(self.SECTION_META, "job_dsc", '').decode('utf8')), + self.job = Job(name=str(self.get_option(self.SECTION_META, "job_name", 'none')), + description=str(self.get_option(self.SECTION_META, "job_dsc", '')), task=str(self.get_option(self.SECTION_META, 'task', 'dir')), version=str(self.get_option(self.SECTION_META, 'ver', '')), config_copy=self.get_option(self.SECTION_META, 'copy_config_to', 'config_copy'), @@ -563,7 +563,7 @@ class ConfigManager(object): filename = self.file if filename: - with open(filename, 'wb') as handle: + with open(filename, 'w') as handle: self.config.write(handle) def get_options(self, section, prefix=''): diff --git a/yandextank/plugins/Aggregator/plugin.py b/yandextank/plugins/Aggregator/plugin.py index 0e1f070..f167634 100644 --- a/yandextank/plugins/Aggregator/plugin.py +++ b/yandextank/plugins/Aggregator/plugin.py @@ -50,7 +50,7 @@ class Plugin(AbstractPlugin): def configure(self): self.aggregator_config = json.loads(resource_string( - __name__, 'config/phout.json')) + __name__, 'config/phout.json').decode('utf8')) verbose_histogram_option = self.get_option("verbose_histogram", "0") self.verbose_histogram = ( verbose_histogram_option.lower() == "true") or ( diff --git a/yandextank/plugins/JsonReport/plugin.py b/yandextank/plugins/JsonReport/plugin.py index d46e866..d7a2349 100644 --- a/yandextank/plugins/JsonReport/plugin.py +++ b/yandextank/plugins/JsonReport/plugin.py @@ -3,6 +3,7 @@ # pylint: disable=missing-docstring import logging +import os from ..Aggregator import Plugin as AggregatorPlugin from ..Monitoring import Plugin as MonitoringPlugin @@ -14,40 +15,41 @@ logger = logging.getLogger(__name__) # pylint: disable=C0103 class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener): # pylint:disable=R0902 - """ Yandex Overload analytics service client (https://overload.yandex.net) """ SECTION = 'json_report' def get_available_options(self): return [] def configure(self): - # try: - # aggregator = self.core.get_plugin_of_type(AggregatorPlugin) - # except KeyError: - # logger.debug("Aggregator plugin not found", exc_info=True) - # else: - # aggregator.add_result_listener(self) - # - # try: - # self.mon = self.core.get_plugin_of_type(TelegrafPlugin) - # except KeyError: - # logger.debug("Telegraf plugin not found:", exc_info=True) - # try: - # self.mon = self.core.get_plugin_of_type(MonitoringPlugin) - # except KeyError: - # logger.debug("Monitoring plugin not found:", exc_info=True) - # - # if self.mon and self.mon.monitoring: - # self.mon.monitoring.add_listener(self) + self.monitoring_logger = self.create_file_logger('monitoring', + self.get_option('monitoring_log', + 'monitoring.log')) + self.aggregator_data_logger = self.create_file_logger('aggregator_data', + self.get_option('test_data_log', + 'test_data.log')) + self.stats_logger = self.create_file_logger('stats', + self.get_option('test_stats_log', + 'test_stats.log')) self.core.job.subscribe_plugin(self) + def create_file_logger(self, logger_name, file_name, formatter=None): + loggr = logging.getLogger(logger_name) + loggr.setLevel(logging.INFO) + handler = logging.FileHandler(os.path.join(self.core.artifacts_dir, file_name), mode='w') + handler.setLevel(logging.INFO) + if formatter: + handler.setFormatter(formatter) + loggr.addHandler(handler) + loggr.propagate = False + return loggr + def on_aggregated_data(self, data, stats): """ @data: aggregated data @stats: stats about gun """ - print(data) - print(stats) + self.aggregator_data_logger.info(data) + self.stats_logger.info(stats) def monitoring_data(self, data_list): - print(data_list) \ No newline at end of file + self.monitoring_logger.info(data_list) \ No newline at end of file diff --git a/yandextank/plugins/TipsAndTricks/plugin.py b/yandextank/plugins/TipsAndTricks/plugin.py index 38c5660..14e9f8e 100644 --- a/yandextank/plugins/TipsAndTricks/plugin.py +++ b/yandextank/plugins/TipsAndTricks/plugin.py @@ -19,12 +19,11 @@ class Plugin(AbstractPlugin, AbstractInfoWidget): def __init__(self, core): AbstractPlugin.__init__(self, core) AbstractInfoWidget.__init__(self) - self.lines = resource_stream(__name__, "config/tips.txt").readlines() + self.lines = [l.decode('utf-8') for l in resource_stream(__name__, "config/tips.txt").readlines()] self.disable = 0 line = random.choice(self.lines) - self.section = line[:line.index(':')] - self.tip = line[line.index(':') + 1:].strip() + self.section, self.tip = [_.strip() for _ in line.split(':', 1)] self.probability = 0.0 @staticmethod diff --git a/yandextank/stepper/format.py b/yandextank/stepper/format.py index 2f23184..c2ce3bf 100644 --- a/yandextank/stepper/format.py +++ b/yandextank/stepper/format.py @@ -29,9 +29,9 @@ class StpdReader(object): def __iter__(self): def read_chunk_header(ammo_file): chunk_header = '' - while chunk_header is '': - line = ammo_file.readline() - if line is '': + while chunk_header in '': + line = ammo_file.readline().decode('utf8') + if line in '': return line # EOF chunk_header = line.strip('\r\n') return chunk_header diff --git a/yandextank/stepper/main.py b/yandextank/stepper/main.py index 91d9081..1155115 100644 --- a/yandextank/stepper/main.py +++ b/yandextank/stepper/main.py @@ -253,7 +253,7 @@ class StepperWrapper(object): hashed_str += sep + \ ';'.join(self.uris) + sep + ';'.join(self.headers) self.log.debug("stpd-hash source: %s", hashed_str) - hasher.update(hashed_str) + hasher.update(hashed_str.encode('utf8')) if not os.path.exists(self.cache_dir): os.makedirs(self.cache_dir) stpd = self.cache_dir + '/' + \ From b26479ea409a0ce1e46b25a3e30a35daa269e3e9 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Tue, 25 Oct 2016 20:41:09 +0300 Subject: [PATCH 6/6] plugin options; StpdReader fix --- yandextank/plugins/JsonReport/plugin.py | 2 +- yandextank/stepper/format.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yandextank/plugins/JsonReport/plugin.py b/yandextank/plugins/JsonReport/plugin.py index d7a2349..82b4256 100644 --- a/yandextank/plugins/JsonReport/plugin.py +++ b/yandextank/plugins/JsonReport/plugin.py @@ -18,7 +18,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener): SECTION = 'json_report' def get_available_options(self): - return [] + return ['monitoring_log', 'test_data_log', 'test_stats_log'] def configure(self): self.monitoring_logger = self.create_file_logger('monitoring', diff --git a/yandextank/stepper/format.py b/yandextank/stepper/format.py index c2ce3bf..0d32d95 100644 --- a/yandextank/stepper/format.py +++ b/yandextank/stepper/format.py @@ -29,9 +29,9 @@ class StpdReader(object): def __iter__(self): def read_chunk_header(ammo_file): chunk_header = '' - while chunk_header in '': + while not chunk_header: line = ammo_file.readline().decode('utf8') - if line in '': + if not line: return line # EOF chunk_header = line.strip('\r\n') return chunk_header