From d5f9dcf5dd0a8f6d4f7274a1732181f084da326e Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Fri, 24 Mar 2017 15:56:06 +0300 Subject: [PATCH 01/26] version, little fix --- setup.py | 2 +- yandextank/core/tankcore.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e6609a5..ab46057 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='yandextank', - version='1.8.33', + version='1.9.0', description='a performance measurement tool', longer_description=''' Yandex.Tank is a performance measurement and load testing automatization tool. diff --git a/yandextank/core/tankcore.py b/yandextank/core/tankcore.py index c0a6d77..23ec75a 100644 --- a/yandextank/core/tankcore.py +++ b/yandextank/core/tankcore.py @@ -172,6 +172,12 @@ class TankCore(object): raise ValueError( "Couldn't convert plugin path to new format:\n %s" % plugin_path) + if plugin_path is "yandextank.plugins.Overload": + logger.warning( + "Deprecated plugin name: 'yandextank.plugins.Overload'\n" + "There is a new generic plugin now.\n" + "Correcting to 'yandextank.plugins.DataUploader overload'") + plugin_path = "yandextank.plugins.DataUploader overload" try: plugin = il.import_module(plugin_path) except ImportError: From 464ac64b81e87fe875e69bc071dc089bf84d1c64 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Fri, 24 Mar 2017 15:59:03 +0300 Subject: [PATCH 02/26] gitter badge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b2c56e9..e08d271 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Yandex Tank [![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/yandex/yandex-tank?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +# Yandex Tank [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/yandex/yandex-tank?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://secure.travis-ci.org/yandex/yandex-tank.png?branch=master)](http://travis-ci.org/yandex/yandex-tank) @@ -22,7 +22,7 @@ Yandex.Tank is an extensible open source load testing tool for advanced linux us Installation at [ReadTheDocs](http://yandextank.readthedocs.org/en/latest/install.html). ## Get help -Chat with authors and other performance specialists: [![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/yandex/yandex-tank?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +Chat with authors and other performance specialists: [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/yandex/yandex-tank?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) Documentation at [ReadTheDocs](https://yandextank.readthedocs.org/en/latest/). From 1aea496a9722d4535786fec9d58d8b4a1d2e8bec Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Fri, 24 Mar 2017 17:05:08 +0300 Subject: [PATCH 03/26] version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab46057..e6609a5 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='yandextank', - version='1.9.0', + version='1.8.33', description='a performance measurement tool', longer_description=''' Yandex.Tank is a performance measurement and load testing automatization tool. From 4127eeeb94bd93375bf86c739d31bf6e3ce178c6 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Fri, 24 Mar 2017 17:06:41 +0300 Subject: [PATCH 04/26] version again --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e6609a5..4ed9bd8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='yandextank', - version='1.8.33', + version='1.8.34', description='a performance measurement tool', longer_description=''' Yandex.Tank is a performance measurement and load testing automatization tool. From 0f5c69e520039b680c77c74401ff19ea769df7c0 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Tue, 28 Mar 2017 17:09:16 +0300 Subject: [PATCH 05/26] register plugins with names; get job number --- yandextank/core/tankcore.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/yandextank/core/tankcore.py b/yandextank/core/tankcore.py index 23ec75a..35cb610 100644 --- a/yandextank/core/tankcore.py +++ b/yandextank/core/tankcore.py @@ -89,7 +89,7 @@ class TankCore(object): def __init__(self, artifacts_base_dir=None, artifacts_dir_name=None): self.config = ConfigManager() self.status = {} - self.plugins = [] + self.plugins = {} self.artifacts_dir_name = artifacts_dir_name self._artifacts_dir = None self.artifact_files = {} @@ -211,7 +211,7 @@ class TankCore(object): instance = getattr( plugin, plugin_path.split('.')[-1] + 'Plugin')(self) - self.plugins.append(instance) + self.register_plugin(plugin_name, instance) logger.debug("Plugin instances: %s", self.plugins) @@ -257,7 +257,7 @@ class TankCore(object): generator_plugin=gen, tank=socket.getfqdn()) - for plugin in self.plugins: + for plugin in self.plugins.values(): logger.debug("Configuring %s", plugin) plugin.configure() self.config.flush() @@ -268,7 +268,7 @@ class TankCore(object): """ Call prepare_test() on all plugins """ logger.info("Preparing test...") self.publish("core", "stage", "prepare") - for plugin in self.plugins: + for plugin in self.plugins.values(): logger.debug("Preparing %s", plugin) plugin.prepare_test() if self.flush_config_to: @@ -278,7 +278,7 @@ class TankCore(object): """ Call start_test() on all plugins """ logger.info("Starting test...") self.publish("core", "stage", "start") - for plugin in self.plugins: + for plugin in self.plugins.values(): logger.debug("Starting %s", plugin) plugin.start_test() if self.flush_config_to: @@ -297,7 +297,7 @@ class TankCore(object): while not self.interrupted: begin_time = time.time() - for plugin in self.plugins: + for plugin in self.plugins.values(): logger.debug("Polling %s", plugin) retcode = plugin.is_test_finished() if retcode >= 0: @@ -316,7 +316,7 @@ class TankCore(object): logger.info("Finishing test...") self.publish("core", "stage", "end") - for plugin in self.plugins: + for plugin in self.plugins.values(): logger.debug("Finalize %s", plugin) try: logger.debug("RC before: %s", retcode) @@ -340,7 +340,7 @@ class TankCore(object): logger.info("Post-processing test...") self.publish("core", "stage", "post_process") - for plugin in self.plugins: + for plugin in self.plugins.values(): logger.debug("Post-process %s", plugin) try: logger.debug("RC before: %s", retcode) @@ -430,7 +430,7 @@ class TankCore(object): Retrieve a plugin of desired class, KeyError raised otherwise """ logger.debug("Searching for plugin: %s", plugin_class) - matches = [plugin for plugin in self.plugins if isinstance(plugin, plugin_class)] + matches = [plugin for plugin in self.plugins.values() if isinstance(plugin, plugin_class)] if len(matches) > 0: if len(matches) > 1: logger.debug( @@ -440,6 +440,10 @@ class TankCore(object): else: raise KeyError("Requested plugin type not found: %s" % plugin_class) + def get_jobno(self, plugin_name='plugin_lunapark'): + uploader_plugin = self.plugins[plugin_name] + return uploader_plugin.lp_job.number + def __collect_file(self, filename, keep_original=False): """ Move or copy single file to artifacts dir @@ -561,7 +565,7 @@ class TankCore(object): """ logger.info("Close allocated resources...") - for plugin in self.plugins: + for plugin in self.plugins.values(): logger.debug("Close %s", plugin) try: plugin.close() @@ -594,6 +598,14 @@ class TankCore(object): os_agent = 'OS/{}'.format(platform.platform()) return ' '.join((tank_agent, python_agent, os_agent)) + def register_plugin(self, plugin_name, instance): + if self.plugins.get(plugin_name, None) is not None: + logger.warning('Plugins\' names should diverse') + logger.warning('Renaming {0} to {0}1 for {1}'.format(plugin_name, instance)) + plugin_name += '1' + self.plugins[plugin_name] = instance + + class ConfigManager(object): """ Option storage class """ From 82cb3d9c09b73341bae157a0388bd53fdeaaa073 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Thu, 30 Mar 2017 16:16:03 +0300 Subject: [PATCH 06/26] 423 interpretation fix --- yandextank/core/tankcore.py | 2 +- yandextank/plugins/DataUploader/client.py | 23 ++++++++++++++--------- yandextank/plugins/DataUploader/plugin.py | 15 +++++++++++---- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/yandextank/core/tankcore.py b/yandextank/core/tankcore.py index 35cb610..01497e4 100644 --- a/yandextank/core/tankcore.py +++ b/yandextank/core/tankcore.py @@ -211,7 +211,7 @@ class TankCore(object): instance = getattr( plugin, plugin_path.split('.')[-1] + 'Plugin')(self) - self.register_plugin(plugin_name, instance) + self.register_plugin(self.PLUGIN_PREFIX+plugin_name, instance) logger.debug("Plugin instances: %s", self.plugins) diff --git a/yandextank/plugins/DataUploader/client.py b/yandextank/plugins/DataUploader/client.py index f4589b9..ca8c9dc 100644 --- a/yandextank/plugins/DataUploader/client.py +++ b/yandextank/plugins/DataUploader/client.py @@ -144,7 +144,9 @@ class APIClient(object): response_callback=lambda x: x, writer=False, trace=False, - json=None): + json=None, + maintenance_timeouts=None, + maintenance_msg=None): url = urljoin(self.base_url, path) if json: request = requests.Request( @@ -153,7 +155,8 @@ class APIClient(object): request = requests.Request( http_method, url, data=data, headers={'User-Agent': self.user_agent}, params=self.params) network_timeouts = self.network_timeouts() - maintenance_timeouts = self.maintenance_timeouts() + maintenance_timeouts = maintenance_timeouts or self.maintenance_timeouts() + maintenance_msg = maintenance_msg or "%s is under maintenance" % (self._base_url) while True: try: response = self.__send_single_request(request, trace=trace) @@ -172,8 +175,8 @@ class APIClient(object): except self.UnderMaintenance as e: try: timeout = next(maintenance_timeouts) - logger.warn( - "%s is under maintenance, will retry in %ss..." % (self._base_url, timeout)) + logger.warn(maintenance_msg) + logger.warn("Retrying in %ss..." % timeout) time.sleep(timeout) continue except StopIteration: @@ -223,13 +226,15 @@ class APIClient(object): except StopIteration: raise e - def __get(self, addr, trace=False): + def __get(self, addr, trace=False, maintenance_timeouts=None, maintenance_msg=None): return self.__make_api_request( 'GET', addr, trace=trace, - response_callback=lambda r: json.loads( - r.content.decode('utf8'))) + response_callback=lambda r: json.loads(r.content.decode('utf8')), + maintenance_timeouts=maintenance_timeouts, + maintenance_msg=maintenance_msg + ) def __post_raw(self, addr, txt_data, trace=False): return self.__make_api_request( @@ -554,11 +559,11 @@ class APIClient(object): res = self.__get(addr, trace=trace) return res[0] - def lock_target(self, target, duration, trace=False): + def lock_target(self, target, duration, trace=False, maintenance_timeouts=None, maintenance_msg=None): addr = "api/server/lock.json?action=lock&" + \ "address=%s&duration=%s&jobno=None" % \ (target, int(duration)) - res = self.__get(addr, trace=trace) + res = self.__get(addr, trace=trace, maintenance_timeouts=maintenance_timeouts, maintenance_msg=maintenance_msg) return res[0] def unlock_target(self, target): diff --git a/yandextank/plugins/DataUploader/plugin.py b/yandextank/plugins/DataUploader/plugin.py index 2162029..7320486 100644 --- a/yandextank/plugins/DataUploader/plugin.py +++ b/yandextank/plugins/DataUploader/plugin.py @@ -790,12 +790,20 @@ class LPJob(object): self.number, text, trace=self.log_other_requests) def lock_target(self, lock_target, lock_target_duration, ignore, strict): + lock_wait_timeout = 10 + maintenance_timeouts = iter([0]) if ignore else iter(lambda: lock_wait_timeout, 0) while True: try: self.api_client.lock_target( lock_target, lock_target_duration, - trace=self.log_other_requests) + trace=self.log_other_requests, + maintenance_timeouts=maintenance_timeouts, + maintenance_msg="Target is locked.\nManual unlock link: %s%s" % ( + self.api_client.base_url, + self.api_client.get_manual_unlock_link(lock_target) + ) + ) return True except (APIClient.NotAvailable, APIClient.StoppedFromOnline) as e: logger.info('Target is not locked due to %s', e.message) @@ -809,12 +817,11 @@ class LPJob(object): return False except APIClient.UnderMaintenance: logger.info('Target is locked') - logger.info("Manual unlock link: %s%s", self.api_client.base_url, - self.api_client.get_manual_unlock_link(lock_target)) if ignore: logger.info('ignore_target_locks = 1') return False - time.sleep(10) + logger.info("Manual unlock link: %s%s", self.api_client.base_url, + self.api_client.get_manual_unlock_link(lock_target)) continue def set_imbalance_and_dsc(self, rps, comment): From fd0223cb8e6dd8acd9f0ee5a1406cdb28bd2a4be Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Thu, 30 Mar 2017 17:38:52 +0300 Subject: [PATCH 07/26] more properties --- yandextank/plugins/DataUploader/plugin.py | 50 ++++++++++++++++------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/yandextank/plugins/DataUploader/plugin.py b/yandextank/plugins/DataUploader/plugin.py index 7320486..2085608 100644 --- a/yandextank/plugins/DataUploader/plugin.py +++ b/yandextank/plugins/DataUploader/plugin.py @@ -62,7 +62,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, self.mon = None self.regression_component = None self.retcode = -1 - self.target = None + self._target = None self.task_name = '' self.token_file = None self.version_tested = None @@ -72,6 +72,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, self.backend_type = BackendTypes.identify_backend(self.SECTION) self._task = None self._api_token = '' + self._lp_job = None @staticmethod def get_key(): @@ -181,9 +182,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, os.getcwd()) def prepare_test(self): - info = self.core.job.generator_plugin.get_info() - self.target = info.address - logger.info("Detected target: %s", self.target) + info = self.generator_info port = info.port instances = info.instances if info.ammo_file.startswith( @@ -191,25 +190,23 @@ class Plugin(AbstractPlugin, AggregateResultListener, ammo_path = info.ammo_file else: ammo_path = os.path.realpath(info.ammo_file) - loadscheme = [] if isinstance(info.rps_schedule, - str) else info.rps_schedule duration = int(info.duration) if duration: self.lock_target_duration = duration loop_count = info.loop_count - self.lp_job = self.__get_lp_job(self.target, port, loadscheme) + lp_job = self.lp_job self.locked_targets = self.check_and_lock_targets(strict=bool( int(self.get_option('strict_lock', '0'))), ignore=self.ignore_target_lock) try: - if self.lp_job._number: - self.make_symlink(self.lp_job._number) + if lp_job._number: + self.make_symlink(lp_job._number) self.check_task_is_open() else: self.check_task_is_open() - self.lp_job.create() - self.make_symlink(self.lp_job.number) + lp_job.create() + self.make_symlink(lp_job.number) except (APIClient.JobNotCreated, APIClient.NotAvailable, APIClient.NetworkError) as e: logger.error(e.message) logger.error( @@ -221,7 +218,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, return cmdline = ' '.join(sys.argv) - self.lp_job.edit_metainfo( + lp_job.edit_metainfo( instances=instances, ammo_path=ammo_path, loop_count=loop_count, @@ -555,10 +552,22 @@ class Plugin(AbstractPlugin, AggregateResultListener, user_agent=self._get_user_agent(), api_token=self.api_token) - def __get_lp_job(self, target, port, loadscheme): + @property + def lp_job(self): + if self._lp_job is None: + self._lp_job = self.__get_lp_job() + return self._lp_job + + def __get_lp_job(self): api_client = self.__get_api_client() + + info = self.generator_info + port = info.port + loadscheme = [] if isinstance(info.rps_schedule, + str) else info.rps_schedule + return LPJob(client=api_client, - target_host=target, + target_host=self.target, target_port=port, number=self.get_option('jobno', ''), token=self.get_option('upload_token', ''), @@ -619,6 +628,19 @@ class Plugin(AbstractPlugin, AggregateResultListener, ) raise RuntimeError("API token error") + @property + def generator_info(self): + if self._generator_info is None: + self._generator_info = self.core.job.generator_plugin.get_info() + return self._generator_info + + @property + def target(self): + if self._target is None: + self._target = self.generator_info.address + logger.info("Detected target: %s", self.target) + return self._target + class JobInfoWidget(AbstractInfoWidget): def __init__(self, sender): From e2b145e627bea34ea2c3bbb92c72b5d8e2700df4 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Thu, 30 Mar 2017 17:54:19 +0300 Subject: [PATCH 08/26] flake8 --- yandextank/core/tankcore.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yandextank/core/tankcore.py b/yandextank/core/tankcore.py index 01497e4..c387b8e 100644 --- a/yandextank/core/tankcore.py +++ b/yandextank/core/tankcore.py @@ -211,7 +211,7 @@ class TankCore(object): instance = getattr( plugin, plugin_path.split('.')[-1] + 'Plugin')(self) - self.register_plugin(self.PLUGIN_PREFIX+plugin_name, instance) + self.register_plugin(self.PLUGIN_PREFIX + plugin_name, instance) logger.debug("Plugin instances: %s", self.plugins) @@ -606,7 +606,6 @@ class TankCore(object): self.plugins[plugin_name] = instance - class ConfigManager(object): """ Option storage class """ From fb9822cbaeb3dfdd72565ab38334c27b63cee8af Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Thu, 30 Mar 2017 19:15:44 +0300 Subject: [PATCH 09/26] publish jobno --- yandextank/core/tankcore.py | 4 +--- yandextank/plugins/DataUploader/plugin.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/yandextank/core/tankcore.py b/yandextank/core/tankcore.py index c387b8e..bf8b3ee 100644 --- a/yandextank/core/tankcore.py +++ b/yandextank/core/tankcore.py @@ -600,9 +600,7 @@ class TankCore(object): def register_plugin(self, plugin_name, instance): if self.plugins.get(plugin_name, None) is not None: - logger.warning('Plugins\' names should diverse') - logger.warning('Renaming {0} to {0}1 for {1}'.format(plugin_name, instance)) - plugin_name += '1' + logger.exception('Plugins\' names should diverse') self.plugins[plugin_name] = instance diff --git a/yandextank/plugins/DataUploader/plugin.py b/yandextank/plugins/DataUploader/plugin.py index 2085608..38ed121 100644 --- a/yandextank/plugins/DataUploader/plugin.py +++ b/yandextank/plugins/DataUploader/plugin.py @@ -207,6 +207,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, self.check_task_is_open() lp_job.create() self.make_symlink(lp_job.number) + self.core.publish(self.SECTION, 'jobno', lp_job.number) except (APIClient.JobNotCreated, APIClient.NotAvailable, APIClient.NetworkError) as e: logger.error(e.message) logger.error( From 9724c44642226df863af31ad98bf3a59fa5ed154 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Tue, 4 Apr 2017 20:19:27 +0300 Subject: [PATCH 10/26] Update core_and_modules.rst --- docs/core_and_modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/core_and_modules.rst b/docs/core_and_modules.rst index f6a3a89..0ee9d75 100644 --- a/docs/core_and_modules.rst +++ b/docs/core_and_modules.rst @@ -804,7 +804,7 @@ Example: :: [tank] ; plugin is disabled by default, enable it: - plugin_overload=yandextank.plugins.Overload + plugin_uploader=yandextank.plugins.DataUploader overload [overload] token_file=token.txt From 314c9116177728e8e8e1efee397d16b4f3004a44 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Tue, 4 Apr 2017 20:20:16 +0300 Subject: [PATCH 11/26] Update core_and_modules.rst --- docs/core_and_modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/core_and_modules.rst b/docs/core_and_modules.rst index a719673..c3dfa80 100644 --- a/docs/core_and_modules.rst +++ b/docs/core_and_modules.rst @@ -786,7 +786,7 @@ Example: :: [tank] ; plugin is disabled by default, enable it: - plugin_overload=yandextank.plugins.Overload + plugin_uploader=yandextank.plugins.DataUploader overload [overload] token_file=token.txt From 82a69232dad947571fd24a9b20048e2aa636e5a1 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Wed, 5 Apr 2017 16:37:25 +0300 Subject: [PATCH 12/26] fix lunapark-2697 --- yandextank/plugins/DataUploader/client.py | 2 +- yandextank/plugins/DataUploader/plugin.py | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/yandextank/plugins/DataUploader/client.py b/yandextank/plugins/DataUploader/client.py index ca8c9dc..17ff9cb 100644 --- a/yandextank/plugins/DataUploader/client.py +++ b/yandextank/plugins/DataUploader/client.py @@ -594,5 +594,5 @@ class OverloadClient(APIClient): def send_status(self, jobno, upload_token, status, trace=False): return - def lock_target(self, target, duration, trace=False): + def lock_target(self, target, duration, trace=False, **kwargs): return diff --git a/yandextank/plugins/DataUploader/plugin.py b/yandextank/plugins/DataUploader/plugin.py index 38ed121..c9ca914 100644 --- a/yandextank/plugins/DataUploader/plugin.py +++ b/yandextank/plugins/DataUploader/plugin.py @@ -817,16 +817,12 @@ class LPJob(object): maintenance_timeouts = iter([0]) if ignore else iter(lambda: lock_wait_timeout, 0) while True: try: - self.api_client.lock_target( - lock_target, - lock_target_duration, - trace=self.log_other_requests, - maintenance_timeouts=maintenance_timeouts, - maintenance_msg="Target is locked.\nManual unlock link: %s%s" % ( - self.api_client.base_url, - self.api_client.get_manual_unlock_link(lock_target) - ) - ) + self.api_client.lock_target(lock_target, lock_target_duration, trace=self.log_other_requests, + maintenance_timeouts=maintenance_timeouts, + maintenance_msg="Target is locked.\nManual unlock link: %s%s" % ( + self.api_client.base_url, + self.api_client.get_manual_unlock_link(lock_target) + )) return True except (APIClient.NotAvailable, APIClient.StoppedFromOnline) as e: logger.info('Target is not locked due to %s', e.message) From d249fd2d9f18ed7070f56c5a965e651e26ea59bc Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Wed, 5 Apr 2017 17:56:17 +0300 Subject: [PATCH 13/26] update pandora plugin for new pandora --- yandextank/plugins/Pandora/plugin.py | 74 ++++++---------------------- yandextank/plugins/Pandora/reader.py | 6 ++- 2 files changed, 19 insertions(+), 61 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 4980edb..0e6e04d 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -5,7 +5,7 @@ import time from ...common.interfaces import AbstractPlugin, AbstractInfoWidget, GeneratorPlugin -from .config import PoolConfig, PandoraConfig, parse_schedule +#from .config import PoolConfig, PandoraConfig, parse_schedule from .reader import PandoraStatsReader from ..Aggregator import Plugin as AggregatorPlugin from ..Console import Plugin as ConsolePlugin @@ -36,7 +36,7 @@ class Plugin(AbstractPlugin, GeneratorPlugin): def get_available_options(self): opts = [ - "pandora_cmd", "buffered_seconds", "ammo", "loop", "sample_log", + "pandora_cmd", "buffered_seconds", "config_file", "startup_schedule", "user_schedule", "gun_type", "custom_config" ] @@ -47,66 +47,20 @@ class Plugin(AbstractPlugin, GeneratorPlugin): self.pandora_cmd = self.get_option("pandora_cmd", "pandora") self.buffered_seconds = int( self.get_option("buffered_seconds", self.buffered_seconds)) - - pool_config = PoolConfig() - - ammo = self.get_option("ammo", "") - if ammo: - pool_config.set_ammo(ammo) - loop_limit = int(self.get_option("loop", "0")) - pool_config.set_loop(loop_limit) - - self.sample_log = self.get_option("sample_log", "") - if not self.sample_log: - self.sample_log = self.core.mkstemp(".samples", "results_") - self.core.add_artifact_file(self.sample_log) - pool_config.set_sample_log(self.sample_log) - - startup_schedule = self.get_option("startup_schedule", "") - if startup_schedule: - pool_config.set_startup_schedule(parse_schedule(startup_schedule)) - else: - raise RuntimeError("startup_schedule not specified") - - user_schedule = self.get_option("user_schedule", "") - if user_schedule: - pool_config.set_user_schedule(parse_schedule(user_schedule)) - else: - raise RuntimeError("user_schedule not specified") - - shared_schedule = bool(int(self.get_option("shared_schedule", "1"))) - pool_config.set_shared_schedule(shared_schedule) - - target = self.get_option("target", "localhost:3000") - pool_config.set_target(target) - - gun_type = self.get_option("gun_type", "http") - if gun_type == 'https': - pool_config.set_ssl(True) - logger.info("SSL is on") - gun_type = "http" - logger.info("Pandora gun type is: %s", gun_type) - pool_config.set_gun_type(gun_type) - - ammo_type = self.get_option("ammo_type", "jsonline/http") - logger.info("Pandora ammo type is: %s", ammo_type) - pool_config.set_ammo_type(ammo_type) - - self.pandora_config = PandoraConfig() - self.pandora_config.add_pool(pool_config) - - self.pandora_config_file = self.get_option("config_file", "") - if not self.pandora_config_file: - if self.custom_config: - raise RuntimeError( - "You said you would like to use custom config," - " but you didn't specify it") + self.core.add_artifact_file("./phout.log") + config_content = self.get_option("config_content", "") + if config_content: self.pandora_config_file = self.core.mkstemp( - ".json", "pandora_config_") - self.core.add_artifact_file(self.pandora_config_file) - if not self.custom_config: + ".yml", "pandora_config_") with open(self.pandora_config_file, 'w') as config_file: - config_file.write(self.pandora_config.json()) + config_file.write(config_content) + else: + self.pandora_config_file = self.get_option("config_file", "") + if not self.pandora_config_file: + raise RuntimeError( + "neither pandora config content" + "nor pandora config file is specified") + self.core.add_artifact_file(self.pandora_config_file) def prepare_test(self): aggregator = None diff --git a/yandextank/plugins/Pandora/reader.py b/yandextank/plugins/Pandora/reader.py index f9af536..b95c71f 100644 --- a/yandextank/plugins/Pandora/reader.py +++ b/yandextank/plugins/Pandora/reader.py @@ -7,8 +7,12 @@ logger = logging.getLogger(__name__) class PandoraStatsReader(object): # TODO: maybe make stats collection asyncronous + def __init__(self): + self.closed = False def next(self): + if self.closed: + raise StopIteration() try: pandora_response = requests.get("http://localhost:1234/debug/vars") pandora_stat = pandora_response.json() @@ -40,7 +44,7 @@ class PandoraStatsReader(object): }] def close(self): - pass + self.closed = True def __iter__(self): return self From 4f631343373b52948d5e0ecb5993800fb0e5c46b Mon Sep 17 00:00:00 2001 From: Timur Torubarov Date: Wed, 5 Apr 2017 19:10:02 +0300 Subject: [PATCH 14/26] added tank exit codes --- docs/core_and_modules.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/core_and_modules.rst b/docs/core_and_modules.rst index 0ee9d75..895ce0e 100644 --- a/docs/core_and_modules.rst +++ b/docs/core_and_modules.rst @@ -105,6 +105,41 @@ Example: except Exception, ex: logger.error('Error trying to perform a test: %s', ex) +exit codes +========== +.. code-block:: json + + { + "0": "completed", + "1": "interrupted_generic_interrupt", + "2": "interrupted", + "3": "interrupted_active_task_not_found ", + "4": "interrupted_no_ammo_file", + "5": "interrupted_address_not_specified", + "6": "interrupted_cpu_or_disk_overload", + "7": "interrupted_unknown_config_parameter", + "8": "interrupted_stop_via_web", + "9": "interrupted", + "11": "interrupted_job_number_error", + "12": "interrupted_phantom_error", + "13": "interrupted_job_metainfo_error", + "14": "interrupted_target_monitoring_error", + "15": "interrupted_target_info_error", + "21": "autostop_time", + "22": "autostop_http", + "23": "autostop_net", + "24": "autostop_instances", + "25": "autostop_total_time", + "26": "autostop_total_http", + "27": "autostop_total_net", + "28": "autostop_negative_http", + "29": "autostop_negative_net", + "30": "autostop_http_trend", + "31": "autostop_metric_higher", + "32": "autostop_metric_lower" + } + + *************** Load Generators *************** From c800c54d7ca4f6f16e078a7866f61d6c5d0ca7fd Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Wed, 5 Apr 2017 19:43:14 +0300 Subject: [PATCH 15/26] some more --- yandextank/plugins/Pandora/plugin.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 0e6e04d..88063bb 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -3,9 +3,9 @@ import logging import subprocess import time -from ...common.interfaces import AbstractPlugin, AbstractInfoWidget, GeneratorPlugin +from ...common.interfaces import AbstractPlugin, \ + AbstractInfoWidget, GeneratorPlugin -#from .config import PoolConfig, PandoraConfig, parse_schedule from .reader import PandoraStatsReader from ..Aggregator import Plugin as AggregatorPlugin from ..Console import Plugin as ConsolePlugin @@ -37,13 +37,11 @@ class Plugin(AbstractPlugin, GeneratorPlugin): def get_available_options(self): opts = [ "pandora_cmd", "buffered_seconds", - "config_file", "startup_schedule", "user_schedule", "gun_type", - "custom_config" + "config_content", "config_file", ] return opts def configure(self): - self.custom_config = self.get_option("custom_config", "0") == "1" self.pandora_cmd = self.get_option("pandora_cmd", "pandora") self.buffered_seconds = int( self.get_option("buffered_seconds", self.buffered_seconds)) From d1731c9bdca33e5b68b90a723840f5632e35d4ec Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Wed, 5 Apr 2017 20:28:45 +0300 Subject: [PATCH 16/26] fix bugs --- yandextank/plugins/Pandora/plugin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 88063bb..7022c65 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -29,6 +29,7 @@ class Plugin(AbstractPlugin, GeneratorPlugin): self.process_stderr = None self.process_start_time = None self.custom_config = False + self.sample_log = "./phout.log" @staticmethod def get_key(): @@ -45,11 +46,13 @@ class Plugin(AbstractPlugin, GeneratorPlugin): self.pandora_cmd = self.get_option("pandora_cmd", "pandora") self.buffered_seconds = int( self.get_option("buffered_seconds", self.buffered_seconds)) - self.core.add_artifact_file("./phout.log") + with open(self.sample_log, 'w'): + pass + self.core.add_artifact_file(self.sample_log) config_content = self.get_option("config_content", "") if config_content: self.pandora_config_file = self.core.mkstemp( - ".yml", "pandora_config_") + ".json", "pandora_config_") with open(self.pandora_config_file, 'w') as config_file: config_file.write(config_content) else: From 2752c9ef99edba6eec74d4f38194bdc60abfc3db Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Wed, 5 Apr 2017 20:37:38 +0300 Subject: [PATCH 17/26] docs --- docs/core_and_modules.rst | 61 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/docs/core_and_modules.rst b/docs/core_and_modules.rst index 0ee9d75..5e8c185 100644 --- a/docs/core_and_modules.rst +++ b/docs/core_and_modules.rst @@ -691,40 +691,37 @@ Disable phantom first (unless you really want to keep it active alongside at you ; Pandora config section: [pandora] - ; ammo file name - ammo=ammo.jsonline + ; Pandora executable path + pandora_cmd=/usr/bin/pandora - ; loop limit - loop=1000 + ; Pandora config contents (json) + config_content = { + "pools": [ + { + "name": "dummy pool", + "gun": {"type": "log"}, + "ammo": { + "type": "dummy/log", + "AmmoLimit": 10000000 + }, + "result": { + "type": "log/phout", + "destination": "./phout.log" + }, + "shared-limits": false, + "user-limiter": { + "type": "unlimited" + }, + "startup-limiter": { + "type": "periodic", + "batch": 1, + "max": 5, + "period": "0.5s" + } + }]} - ; each user will maintain this schedule - user_schedule = periodic(1, 1, 100) - - ; users are started using this schedule - startup_schedule = periodic(1, 1, 100) - - ; if shared_schedule is false, then each user is independent, - ; in other case they all hold to a common schedule - shared_schedule = 0 - - ; target host and port - target=localhost:3000 - - -Ammo format ------------ - -Pandora currently supports only one ammo format: ``jsonline``, i.e. one json doc per line. - -Example: -:: - - {"uri": "/00", "method": "GET", "headers": {"Host": "example.org", "User-Agent": "Pandora/0.0.1"}, "host": "example.org"} - {"uri": "/01", "method": "GET", "headers": {"Host": "example.org", "User-Agent": "Pandora/0.0.1"}, "host": "example.org"} - {"tag": "mytag", "uri": "/02", "method": "GET", "headers": {"Host": "example.org", "User-Agent": "Pandora/0.0.1"}, "host": "example.org"} - {"uri": "/03", "method": "GET", "headers": {"Host": "example.org", "User-Agent": "Pandora/0.0.1"}, "host": "example.org"} - -Each json doc describes an HTTP request. Some of them may have a tag field, it will be used as other tags in other ammo formats. + ; OR config file (yaml or json) + config_file = pandora_config.yml Schedules --------- From 4293e79d2c39f45ba57e622089f80b2879669d3c Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Wed, 5 Apr 2017 20:46:14 +0300 Subject: [PATCH 18/26] don't move pandora's config --- yandextank/plugins/Pandora/plugin.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 7022c65..dda45d6 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -49,19 +49,26 @@ class Plugin(AbstractPlugin, GeneratorPlugin): with open(self.sample_log, 'w'): pass self.core.add_artifact_file(self.sample_log) + + self.pandora_config_file = self.core.mkstemp( + ".json", "pandora_config_") + self.core.add_artifact_file(self.pandora_config_file) + config_content = self.get_option("config_content", "") if config_content: - self.pandora_config_file = self.core.mkstemp( - ".json", "pandora_config_") with open(self.pandora_config_file, 'w') as config_file: config_file.write(config_content) else: - self.pandora_config_file = self.get_option("config_file", "") - if not self.pandora_config_file: + config_file = self.get_option("config_file", "") + if not config_file: raise RuntimeError( "neither pandora config content" "nor pandora config file is specified") - self.core.add_artifact_file(self.pandora_config_file) + else: + with open(config_file, 'rb') as config: + config_content = config.read() + with open(self.pandora_config_file, 'wb') as config_file: + config_file.write(config_content) def prepare_test(self): aggregator = None From 47e6a36e3d0dc8c9580c71c718533832156bedca Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Wed, 5 Apr 2017 20:54:04 +0300 Subject: [PATCH 19/26] expvar enable/disable --- docs/core_and_modules.rst | 3 +++ yandextank/plugins/Pandora/plugin.py | 5 ++++- yandextank/plugins/Pandora/reader.py | 11 ++++++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/core_and_modules.rst b/docs/core_and_modules.rst index 5e8c185..da58a5c 100644 --- a/docs/core_and_modules.rst +++ b/docs/core_and_modules.rst @@ -694,6 +694,9 @@ Disable phantom first (unless you really want to keep it active alongside at you ; Pandora executable path pandora_cmd=/usr/bin/pandora + ; Enable/disable expvar monitoring + expvar = 1 ; default + ; Pandora config contents (json) config_content = { "pools": [ diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index dda45d6..ec3f26c 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -30,6 +30,7 @@ class Plugin(AbstractPlugin, GeneratorPlugin): self.process_start_time = None self.custom_config = False self.sample_log = "./phout.log" + self.expvar = True @staticmethod def get_key(): @@ -39,10 +40,12 @@ class Plugin(AbstractPlugin, GeneratorPlugin): opts = [ "pandora_cmd", "buffered_seconds", "config_content", "config_file", + "expvar" ] return opts def configure(self): + self.expvar = self.get_option("expvar", "1") == "1" self.pandora_cmd = self.get_option("pandora_cmd", "pandora") self.buffered_seconds = int( self.get_option("buffered_seconds", self.buffered_seconds)) @@ -82,7 +85,7 @@ class Plugin(AbstractPlugin, GeneratorPlugin): "Linking sample and stats readers to aggregator. Reading samples from %s", self.sample_log) aggregator.reader = PhantomReader(self.sample_log) - aggregator.stats_reader = PandoraStatsReader() + aggregator.stats_reader = PandoraStatsReader(self.expvar) try: console = self.core.get_plugin_of_type(ConsolePlugin) diff --git a/yandextank/plugins/Pandora/reader.py b/yandextank/plugins/Pandora/reader.py index b95c71f..07e8e80 100644 --- a/yandextank/plugins/Pandora/reader.py +++ b/yandextank/plugins/Pandora/reader.py @@ -7,12 +7,21 @@ logger = logging.getLogger(__name__) class PandoraStatsReader(object): # TODO: maybe make stats collection asyncronous - def __init__(self): + def __init__(self, expvar): self.closed = False + self.expvar = expvar def next(self): if self.closed: raise StopIteration() + if not self.expvar: + return [{ + 'ts': int(time.time() - 1), + 'metrics': { + 'instances': 0, + 'reqps': 0 + } + }] try: pandora_response = requests.get("http://localhost:1234/debug/vars") pandora_stat = pandora_response.json() From 5f5741073faf3831ed2acab8e64d2cad0952c7af Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Wed, 5 Apr 2017 23:02:30 +0300 Subject: [PATCH 20/26] fix 2705 & 2706 --- yandextank/plugins/DataUploader/client.py | 3 +++ yandextank/plugins/Telegraf/collector.py | 9 +++++++++ yandextank/plugins/Telegraf/plugin.py | 4 +--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/yandextank/plugins/DataUploader/client.py b/yandextank/plugins/DataUploader/client.py index 17ff9cb..aa53bd8 100644 --- a/yandextank/plugins/DataUploader/client.py +++ b/yandextank/plugins/DataUploader/client.py @@ -596,3 +596,6 @@ class OverloadClient(APIClient): def lock_target(self, target, duration, trace=False, **kwargs): return + + def unlock_target(self, *args, **kwargs): + return \ No newline at end of file diff --git a/yandextank/plugins/Telegraf/collector.py b/yandextank/plugins/Telegraf/collector.py index ab522d5..6b77380 100644 --- a/yandextank/plugins/Telegraf/collector.py +++ b/yandextank/plugins/Telegraf/collector.py @@ -135,8 +135,17 @@ class MonitoringCollector(object): data = copy.deepcopy(self.__collected_data) self.__collected_data = [] for listener in self.listeners: + # 2nd deep copy to ensure each listener gets it's own copy listener.monitoring_data(copy.deepcopy(data)) + def not_empty(self): + return len(self.__collected_data) > 0 + + def send_rest_data(self): + while self.not_empty(): + logger.info("Sending monitoring data rests...") + self.send_collected_data() + class StdOutPrintMon(MonitoringDataListener): """Simple listener, writing data to stdout""" diff --git a/yandextank/plugins/Telegraf/plugin.py b/yandextank/plugins/Telegraf/plugin.py index 90657e7..c52e29d 100644 --- a/yandextank/plugins/Telegraf/plugin.py +++ b/yandextank/plugins/Telegraf/plugin.py @@ -225,9 +225,7 @@ class Plugin(AbstractPlugin): for log in self.monitoring.artifact_files: self.core.add_artifact_file(log) - while self.monitoring.__collected_data: - logger.info("Sending monitoring data rests...") - self.monitoring.send_collected_data() + self.monitoring.send_rest_data() if self.mon_saver: self.mon_saver.close() return retcode From 403a030836939d3ea2e2c3dc19cfe4def897c646 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Wed, 5 Apr 2017 23:04:42 +0300 Subject: [PATCH 21/26] pep8 newline --- yandextank/plugins/DataUploader/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yandextank/plugins/DataUploader/client.py b/yandextank/plugins/DataUploader/client.py index aa53bd8..eead2b4 100644 --- a/yandextank/plugins/DataUploader/client.py +++ b/yandextank/plugins/DataUploader/client.py @@ -598,4 +598,4 @@ class OverloadClient(APIClient): return def unlock_target(self, *args, **kwargs): - return \ No newline at end of file + return From eace9847f4e5d1800155bceea9945125309ed829 Mon Sep 17 00:00:00 2001 From: Nurlan Nugumanov Date: Wed, 5 Apr 2017 21:11:40 +0300 Subject: [PATCH 22/26] removed console snapshot sending --- yandextank/plugins/DataUploader/client.py | 7 ------- yandextank/plugins/DataUploader/plugin.py | 10 ---------- 2 files changed, 17 deletions(-) diff --git a/yandextank/plugins/DataUploader/client.py b/yandextank/plugins/DataUploader/client.py index 17ff9cb..3f7c7ff 100644 --- a/yandextank/plugins/DataUploader/client.py +++ b/yandextank/plugins/DataUploader/client.py @@ -547,13 +547,6 @@ class APIClient(object): except StopIteration: raise e - def send_console(self, jobno, console, trace=False): - if trace: - logger.debug("Sending console view [%s]: %s", len(console), - console[:64]) - addr = "api/job/%s/console.txt" % jobno - self.__post_raw(addr, console, trace=trace) - def is_target_locked(self, target, trace=False): addr = "api/server/lock.json?action=check&address=%s" % target res = self.__get(addr, trace=trace) diff --git a/yandextank/plugins/DataUploader/plugin.py b/yandextank/plugins/DataUploader/plugin.py index c9ca914..7a89033 100644 --- a/yandextank/plugins/DataUploader/plugin.py +++ b/yandextank/plugins/DataUploader/plugin.py @@ -467,12 +467,6 @@ class Plugin(AbstractPlugin, AggregateResultListener, with open(os.path.join(self.core.artifacts_dir, 'saved_conf.ini'), 'w') as f: config.write(f) - def send_console(self, text): - try: - self.lp_job.send_console(text) - except Exception: # pylint: disable=W0703 - logger.debug("Can't send console snapshot: %s", exc_info=True) - def parse_lock_targets(self): # prepare target lock list locks_list_cfg = self.get_option('lock_targets', 'auto').strip() @@ -808,10 +802,6 @@ class LPJob(object): self.api_client.push_monitoring_data( self.number, self.token, data, trace=self.log_monitoring_requests) - def send_console(self, text): - return self.api_client.send_console( - self.number, text, trace=self.log_other_requests) - def lock_target(self, lock_target, lock_target_duration, ignore, strict): lock_wait_timeout = 10 maintenance_timeouts = iter([0]) if ignore else iter(lambda: lock_wait_timeout, 0) From 88146ac52cd99d3c379889ab2eaace1a6ec24580 Mon Sep 17 00:00:00 2001 From: Nurlan Nugumanov Date: Wed, 5 Apr 2017 21:56:18 +0300 Subject: [PATCH 23/26] removed send_console completely --- yandextank/plugins/Console/plugin.py | 4 ---- yandextank/plugins/DataUploader/plugin.py | 1 - 2 files changed, 5 deletions(-) diff --git a/yandextank/plugins/Console/plugin.py b/yandextank/plugins/Console/plugin.py index 7cb9682..f305c24 100644 --- a/yandextank/plugins/Console/plugin.py +++ b/yandextank/plugins/Console/plugin.py @@ -21,7 +21,6 @@ class Plugin(AbstractPlugin, AggregateResultListener): self.screen = None self.render_exception = None self.console_markup = None - self.remote_translator = None self.info_panel_width = '33' self.short_only = 0 # these three provide non-blocking console output @@ -71,9 +70,6 @@ class Plugin(AbstractPlugin, AggregateResultListener): sys.stdout.write(self.__console_view) sys.stdout.write(self.console_markup.TOTAL_RESET) - if self.remote_translator: - self.remote_translator.send_console(self.__console_view) - def is_test_finished(self): if not self.__writer_thread: self.__writer_event = threading.Event() diff --git a/yandextank/plugins/DataUploader/plugin.py b/yandextank/plugins/DataUploader/plugin.py index 7a89033..0516557 100644 --- a/yandextank/plugins/DataUploader/plugin.py +++ b/yandextank/plugins/DataUploader/plugin.py @@ -240,7 +240,6 @@ class Plugin(AbstractPlugin, AggregateResultListener, if console: console.add_info_widget(JobInfoWidget(self)) - console.remote_translator = self self.set_option('target_host', self.target) self.set_option('target_port', port) From 942ac28f13641375df50d60ff188e5872319ccc0 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Thu, 6 Apr 2017 14:02:14 +0300 Subject: [PATCH 24/26] removed extra deepcopy --- yandextank/plugins/Telegraf/collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yandextank/plugins/Telegraf/collector.py b/yandextank/plugins/Telegraf/collector.py index 6b77380..90d360e 100644 --- a/yandextank/plugins/Telegraf/collector.py +++ b/yandextank/plugins/Telegraf/collector.py @@ -132,10 +132,10 @@ class MonitoringCollector(object): def send_collected_data(self): """sends pending data set to listeners""" - data = copy.deepcopy(self.__collected_data) + data = self.__collected_data self.__collected_data = [] for listener in self.listeners: - # 2nd deep copy to ensure each listener gets it's own copy + # deep copy to ensure each listener gets it's own copy listener.monitoring_data(copy.deepcopy(data)) def not_empty(self): From 920e9fa1d8143b637bf43d7439ae7e55ccd2c155 Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Thu, 6 Apr 2017 14:05:28 +0300 Subject: [PATCH 25/26] copy pandora config with its own extension --- yandextank/plugins/Pandora/plugin.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index ec3f26c..190243b 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -53,12 +53,11 @@ class Plugin(AbstractPlugin, GeneratorPlugin): pass self.core.add_artifact_file(self.sample_log) - self.pandora_config_file = self.core.mkstemp( - ".json", "pandora_config_") - self.core.add_artifact_file(self.pandora_config_file) - config_content = self.get_option("config_content", "") if config_content: + self.pandora_config_file = self.core.mkstemp( + ".json", "pandora_config_") + self.core.add_artifact_file(self.pandora_config_file) with open(self.pandora_config_file, 'w') as config_file: config_file.write(config_content) else: @@ -68,6 +67,10 @@ class Plugin(AbstractPlugin, GeneratorPlugin): "neither pandora config content" "nor pandora config file is specified") else: + extension = config_file.rsplit(".", 1)[1] + self.pandora_config_file = self.core.mkstemp( + "." + extension, "pandora_config_") + self.core.add_artifact_file(self.pandora_config_file) with open(config_file, 'rb') as config: config_content = config.read() with open(self.pandora_config_file, 'wb') as config_file: From e88f9b42eba9fb34419fd9b05cd190add989c4f6 Mon Sep 17 00:00:00 2001 From: Arseniy Fomchenko Date: Fri, 7 Apr 2017 14:12:01 +0300 Subject: [PATCH 26/26] fix 2707 --- yandextank/plugins/JsonReport/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yandextank/plugins/JsonReport/plugin.py b/yandextank/plugins/JsonReport/plugin.py index ea98ec6..963491d 100644 --- a/yandextank/plugins/JsonReport/plugin.py +++ b/yandextank/plugins/JsonReport/plugin.py @@ -53,7 +53,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener): if data ] - def end_test(self, retcode): + def post_process(self, retcode): self.data_and_stats_stream.close() self.monitoring_stream.close() return retcode