diff --git a/setup.py b/setup.py index 48bd01a..8a7f2a8 100644 --- a/setup.py +++ b/setup.py @@ -45,33 +45,18 @@ analytic tools for the results they produce. 'yandex-tank = yandextank.core.cli:main', ], }, - package_data={}, + package_data={ + 'yandextank.plugins.GraphiteUploader': ['config/*'], + 'yandextank.plugins.JMeter': ['config/*'], + 'yandextank.plugins.Monitoring': ['config/*'], + 'yandextank.plugins.Phantom': ['config/*'], + 'yandextank.plugins.TipsAndTricks': ['config/*'], + }, # TODO: move them all to resources maybe data_files=[ ('/etc/yandex-tank', [ 'config/00-base.ini', ]), - ('/etc/yandex-tank/JMeter', [ - 'config/JMeter/jmeter_argentum.xml', - 'config/JMeter/jmeter_var_template.xml', - 'config/JMeter/jmeter_writer.xml', - ]), - ('/etc/yandex-tank/GraphiteUploader', [ - 'config/GraphiteUploader/graphite-js.tpl', - 'config/GraphiteUploader/graphite.tpl', - ]), - ('/etc/yandex-tank/Monitoring', [ - 'config/Monitoring/agent.cfg', - 'config/Monitoring/monitoring_default_config.xml', - ]), - ('/etc/yandex-tank/Phantom', [ - 'config/Phantom/phantom.conf.tpl', - 'config/Phantom/phantom_benchmark_additional.tpl', - 'config/Phantom/phantom_benchmark_main.tpl', - ]), - ('/etc/yandex-tank/TipsAndTricks', [ - 'config/TipsAndTricks/tips.txt', - ]), ('/etc/bash_completion.d', [ 'data/yandex-tank.completion' ]), diff --git a/yandextank/plugins/GraphiteUploader/__init__.py b/yandextank/plugins/GraphiteUploader/__init__.py new file mode 100644 index 0000000..309c266 --- /dev/null +++ b/yandextank/plugins/GraphiteUploader/__init__.py @@ -0,0 +1 @@ +from plugin import * diff --git a/yandextank/plugins/charts.coffee b/yandextank/plugins/GraphiteUploader/charts.coffee similarity index 100% rename from yandextank/plugins/charts.coffee rename to yandextank/plugins/GraphiteUploader/charts.coffee diff --git a/config/GraphiteUploader/graphite-js.tpl b/yandextank/plugins/GraphiteUploader/config/graphite-js.tpl similarity index 100% rename from config/GraphiteUploader/graphite-js.tpl rename to yandextank/plugins/GraphiteUploader/config/graphite-js.tpl diff --git a/config/GraphiteUploader/graphite.tpl b/yandextank/plugins/GraphiteUploader/config/graphite.tpl similarity index 100% rename from config/GraphiteUploader/graphite.tpl rename to yandextank/plugins/GraphiteUploader/config/graphite.tpl diff --git a/yandextank/plugins/graphite.coffee b/yandextank/plugins/GraphiteUploader/graphite.coffee similarity index 100% rename from yandextank/plugins/graphite.coffee rename to yandextank/plugins/GraphiteUploader/graphite.coffee diff --git a/yandextank/plugins/GraphiteUploader.py b/yandextank/plugins/GraphiteUploader/plugin.py similarity index 84% rename from yandextank/plugins/GraphiteUploader.py rename to yandextank/plugins/GraphiteUploader/plugin.py index c3557dc..4a0392e 100644 --- a/yandextank/plugins/GraphiteUploader.py +++ b/yandextank/plugins/GraphiteUploader/plugin.py @@ -1,6 +1,8 @@ '''Graphite Uploader plugin that sends aggregated data to Graphite server''' -from Aggregator import AggregateResultListener, AggregatorPlugin +from pkg_resources import resource_string +from yandextank.plugins.Aggregator import \ + AggregateResultListener, AggregatorPlugin from yandextank.core import AbstractPlugin import logging import socket @@ -10,6 +12,7 @@ import datetime class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener): + '''Graphite data uploader''' SECTION = 'graphite' @@ -30,7 +33,7 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener): self.start_time = start_time.strftime("%H:%M%%20%Y%m%d") def end_test(self, retcode): - end_time = datetime.datetime.now() + datetime.timedelta(minutes = 1) + end_time = datetime.datetime.now() + datetime.timedelta(minutes=1) self.end_time = end_time.strftime("%H:%M%%20%Y%m%d") return retcode @@ -38,16 +41,22 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener): '''Read configuration''' self.address = self.get_option("address", "") if self.address == "": - self.log.warning("Graphite uploader is not configured and will not send any data") + self.log.warning( + "Graphite uploader is not configured and will not send any data") else: port = self.get_option("port", "2003") self.web_port = self.get_option("web_port", "8080") self.prefix = self.get_option("prefix", "one_sec.yandex_tank") - default_template = "/etc/yandex-tank/GraphiteUploader/graphite.tpl" - if self.get_option("js", "1") == "1": - default_template = "/etc/yandex-tank/GraphiteUploader/graphite-js.tpl" - self.template = self.get_option("template", default_template) - self.graphite_client = GraphiteClient(self.prefix, self.address, port) + specified_template = self.get_option("template", "") + if specified_template != "": + self.template = open(specified_template, 'r').read() + else: + default_template = "graphite.tpl" + if self.get_option("js", "1") == "1": + default_template = "graphite-js.tpl" + self.template = resource_string(__name__, 'config/' + default_template) + self.graphite_client = GraphiteClient( + self.prefix, self.address, port) aggregator = self.core.get_plugin_of_type(AggregatorPlugin) aggregator.add_result_listener(self) @@ -55,7 +64,7 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener): """ @data: SecondAggregateData """ - #TODO: Use ts from data + # TODO: Use ts from data if self.graphite_client: results = {} overall = GraphiteUploaderPlugin.__flatten( @@ -72,12 +81,11 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener): def post_process(self, retcode): if self.graphite_client: - template = open(self.template, 'r').read() graphite_html = self.core.mkstemp(".html", "graphite_") self.core.add_artifact_file(graphite_html) with open(graphite_html, 'w') as graphite_html_file: graphite_html_file.write( - string.Template(template).safe_substitute( + string.Template(self.template).safe_substitute( host=self.address, width=1000, height=400, @@ -119,6 +127,7 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener): class GraphiteClient(object): + '''Graphite client that writes metrics to Graphite server''' def __init__(self, prefix, address, port): diff --git a/yandextank/plugins/JMeter/__init__.py b/yandextank/plugins/JMeter/__init__.py new file mode 100644 index 0000000..309c266 --- /dev/null +++ b/yandextank/plugins/JMeter/__init__.py @@ -0,0 +1 @@ +from plugin import * diff --git a/config/JMeter/jmeter_argentum.xml b/yandextank/plugins/JMeter/config/jmeter_argentum.xml similarity index 100% rename from config/JMeter/jmeter_argentum.xml rename to yandextank/plugins/JMeter/config/jmeter_argentum.xml diff --git a/config/JMeter/jmeter_var_template.xml b/yandextank/plugins/JMeter/config/jmeter_var_template.xml similarity index 100% rename from config/JMeter/jmeter_var_template.xml rename to yandextank/plugins/JMeter/config/jmeter_var_template.xml diff --git a/config/JMeter/jmeter_writer.xml b/yandextank/plugins/JMeter/config/jmeter_writer.xml similarity index 100% rename from config/JMeter/jmeter_writer.xml rename to yandextank/plugins/JMeter/config/jmeter_writer.xml diff --git a/yandextank/plugins/JMeter.py b/yandextank/plugins/JMeter/plugin.py similarity index 85% rename from yandextank/plugins/JMeter.py rename to yandextank/plugins/JMeter/plugin.py index 3626597..c681c86 100644 --- a/yandextank/plugins/JMeter.py +++ b/yandextank/plugins/JMeter/plugin.py @@ -7,15 +7,18 @@ import time import datetime import json -from Aggregator import AbstractReader, AggregatorPlugin, \ +from pkg_resources import resource_string +from yandextank.plugins.Aggregator import AbstractReader, AggregatorPlugin, \ AggregateResultListener, SecondAggregateDataItem -from ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget +from yandextank.plugins.ConsoleOnline import \ + ConsoleOnlinePlugin, AbstractInfoWidget from yandextank.core import AbstractPlugin import yandextank.core as tankcore -import ConsoleScreen +import yandextank.plugins.ConsoleScreen as ConsoleScreen class JMeterPlugin(AbstractPlugin): + """ JMeter tank plugin """ SECTION = 'jmeter' @@ -52,7 +55,8 @@ class JMeterPlugin(AbstractPlugin): self.get_option('buffered_seconds', '3'))) self.core.add_artifact_file(self.jmeter_log, True) self.use_argentum = eval(self.get_option('use_argentum', 'False')) - self.jmx = self.__add_jmeter_components(self.original_jmx, self.jtl_file, self._get_variables()) + self.jmx = self.__add_jmeter_components( + self.original_jmx, self.jtl_file, self._get_variables()) self.core.add_artifact_file(self.jmx) def prepare_test(self): @@ -84,7 +88,8 @@ class JMeterPlugin(AbstractPlugin): aggregator.add_result_listener(widget) def start_test(self): - self.log.info("Starting %s with arguments: %s", self.jmeter_path, self.args) + self.log.info( + "Starting %s with arguments: %s", self.jmeter_path, self.args) self.jmeter_process = subprocess.Popen(self.args, executable=self.jmeter_path, preexec_fn=os.setsid, close_fds=True) # stderr=subprocess.PIPE, stdout=subprocess.PIPE, self.start_time = time.time() @@ -92,14 +97,16 @@ class JMeterPlugin(AbstractPlugin): def is_test_finished(self): retcode = self.jmeter_process.poll() if retcode is not None: - self.log.info("JMeter process finished with exit code: %s", retcode) + self.log.info( + "JMeter process finished with exit code: %s", retcode) return retcode else: return -1 def end_test(self, retcode): if self.jmeter_process: - self.log.info("Terminating jmeter process group with PID %s", self.jmeter_process.pid) + self.log.info( + "Terminating jmeter process group with PID %s", self.jmeter_process.pid) try: os.killpg(self.jmeter_process.pid, signal.SIGTERM) except OSError, exc: @@ -123,24 +130,23 @@ class JMeterPlugin(AbstractPlugin): except Exception, exc: raise RuntimeError("Failed to find the end of JMX XML: %s" % exc) - tpl_filepath = '/etc/yandex-tank/JMeter/jmeter_writer.xml' + tpl_resource = 'jmeter_writer.xml' if self.use_argentum: - self.log.warn("You are using argentum aggregator for JMeter. Be careful.") - tpl_filepath = '/etc/yandex-tank/JMeter/jmeter_argentum.xml' + self.log.warn( + "You are using argentum aggregator for JMeter. Be careful.") + tpl_resource = 'jmeter_argentum.xml' - with open(tpl_filepath, 'r') as tpl_file: - tpl = tpl_file.read() - - with open('/etc/yandex-tank/JMeter/jmeter_var_template.xml', 'r') as tpl_file: - udv_tpl = tpl_file.read() + tpl = resource_string(__name__, 'config/' + tpl_resource) + udv_tpl = resource_string(__name__, 'config/jmeter_var_template.xml') udv_set = [] for var_name, var_value in variables.iteritems(): udv_set.append(udv_tpl % (var_name, var_name, var_value)) try: - new_file = self.core.mkstemp('.jmx', 'modified_', os.path.dirname(os.path.realpath(jmx))) + new_file = self.core.mkstemp( + '.jmx', 'modified_', os.path.dirname(os.path.realpath(jmx))) except OSError, exc: self.log.debug("Can't create modified jmx near original: %s", exc) new_file = self.core.mkstemp('.jmx', 'modified_') @@ -167,6 +173,7 @@ class JMeterPlugin(AbstractPlugin): class JMeterReader(AbstractReader): + """ JTL files reader """ KNOWN_EXC = { "java.net.NoRouteToHostException": 113, @@ -224,10 +231,12 @@ class JMeterReader(AbstractReader): return None else: # good json-object. parse it! - second_ag = self.get_zero_sample(datetime.datetime.fromtimestamp(second['second'])) + second_ag = self.get_zero_sample( + datetime.datetime.fromtimestamp(second['second'])) second_ag.overall.avg_connect_time = 0 second_ag.overall.avg_send_time = 0 - second_ag.overall.avg_receive_time = second['avg_rt'] - second['avg_lt'] + second_ag.overall.avg_receive_time = second[ + 'avg_rt'] - second['avg_lt'] second_ag.overall.avg_response_time = second['avg_rt'] second_ag.overall.avg_latency = second['avg_lt'] second_ag.overall.RPS = second['th'] @@ -242,7 +251,8 @@ class JMeterReader(AbstractReader): second_ag.overall.http_codes = rc_map for percentile in second['percentile'].keys(): - second_ag.overall.quantiles[int(float(percentile))] = second['percentile'][percentile] + second_ag.overall.quantiles[ + int(float(percentile))] = second['percentile'][percentile] second_ag.cumulative.quantiles[int(float(percentile))] = second['cumulative_percentile'][ percentile] @@ -251,13 +261,18 @@ class JMeterReader(AbstractReader): for sampler in second['samplers'].keys(): sampler_ag_data_item = SecondAggregateDataItem() sampler_ag_data_item.case = sampler - sampler_ag_data_item.active_threads = second['active_threads'] - sampler_ag_data_item.RPS = int(second['samplers'][sampler]) - sampler_ag_data_item.times_dist = second['sampler_interval_dist'][sampler] + sampler_ag_data_item.active_threads = second[ + 'active_threads'] + sampler_ag_data_item.RPS = int( + second['samplers'][sampler]) + sampler_ag_data_item.times_dist = second[ + 'sampler_interval_dist'][sampler] - sampler_ag_data_item.quantiles = second['sampler_percentile'][sampler] + sampler_ag_data_item.quantiles = second[ + 'sampler_percentile'][sampler] - sampler_ag_data_item.avg_response_time = second['sampler_avg_rt'][sampler] + sampler_ag_data_item.avg_response_time = second[ + 'sampler_avg_rt'][sampler] second_ag.cases[sampler] = sampler_ag_data_item return second_ag return None @@ -279,7 +294,8 @@ class JMeterReader(AbstractReader): # self.log.warning("Wrong jtl line, skipped: %s", line) continue cur_time = int(data[0]) / 1000 - netcode = '0' if data[4] == 'true' else self.exc_to_net(data[3]) + netcode = '0' if data[ + 4] == 'true' else self.exc_to_net(data[3]) if not cur_time in self.data_buffer.keys(): if self.data_queue and self.data_queue[0] >= cur_time: @@ -290,7 +306,8 @@ class JMeterReader(AbstractReader): self.data_queue.append(cur_time) self.data_buffer[cur_time] = [] # marker, threads, overallRT, httpCode, netCode - data_item = [data[2], int(data[7]), int(data[1]), self.exc_to_http(data[3]), netcode] + data_item = [ + data[2], int(data[7]), int(data[1]), self.exc_to_http(data[3]), netcode] # bytes: sent received data_item += [0, int(data[5])] # connect send latency receive @@ -315,7 +332,8 @@ class JMeterReader(AbstractReader): if exc in self.KNOWN_EXC.keys(): return self.KNOWN_EXC[exc] else: - self.log.warning("Not known Java exception, consider adding it to dictionary: %s", param1) + self.log.warning( + "Not known Java exception, consider adding it to dictionary: %s", param1) return '1' def exc_to_http(self, param1): @@ -333,6 +351,7 @@ class JMeterReader(AbstractReader): # =============================================================================== class JMeterInfoWidget(AbstractInfoWidget, AggregateResultListener): + """ Right panel widget with JMeter test info """ def __init__(self, jmeter): @@ -364,6 +383,7 @@ class JMeterInfoWidget(AbstractInfoWidget, AggregateResultListener): template += " Duration: %s\n" template += "Active Threads: %s\n" template += " Responses/s: %s" - data = (os.path.basename(self.jmeter.original_jmx), duration, self.active_threads, self.RPS) + data = (os.path.basename(self.jmeter.original_jmx), + duration, self.active_threads, self.RPS) return template % data diff --git a/yandextank/plugins/Monitoring/__init__.py b/yandextank/plugins/Monitoring/__init__.py index 34bf345..309c266 100644 --- a/yandextank/plugins/Monitoring/__init__.py +++ b/yandextank/plugins/Monitoring/__init__.py @@ -1,4 +1 @@ -''' -Package contains all original tank tool plugins -''' from plugin import * diff --git a/config/Monitoring/agent.cfg b/yandextank/plugins/Monitoring/config/agent.cfg similarity index 100% rename from config/Monitoring/agent.cfg rename to yandextank/plugins/Monitoring/config/agent.cfg diff --git a/config/Monitoring/monitoring_default_config.xml b/yandextank/plugins/Monitoring/config/monitoring_default_config.xml similarity index 100% rename from config/Monitoring/monitoring_default_config.xml rename to yandextank/plugins/Monitoring/config/monitoring_default_config.xml diff --git a/yandextank/plugins/Monitoring/plugin.py b/yandextank/plugins/Monitoring/plugin.py index 297934c..4858d41 100644 --- a/yandextank/plugins/Monitoring/plugin.py +++ b/yandextank/plugins/Monitoring/plugin.py @@ -6,6 +6,7 @@ import traceback import fnmatch import datetime +from pkg_resources import resource_string from collector import MonitoringCollector, \ MonitoringDataListener, MonitoringDataDecoder from yandextank.plugins.ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget @@ -63,7 +64,10 @@ class MonitoringPlugin(AbstractPlugin): self.monitoring = None if self.config == 'auto': - self.config = '/etc/yandex-tank/Monitoring/monitoring_default_config.xml' + default_config = resource_string(__name__, 'config/monitoring_default_config.xml') + self.config = self.core.mkstemp(".xml", "monitoring_default_") + with open(self.config, 'w') as cfg_file: + cfg_file.write(default_config) try: autostop = self.core.get_plugin_of_type(AutostopPlugin) diff --git a/yandextank/plugins/PhantomUtils.py b/yandextank/plugins/Phantom/PhantomUtils.py similarity index 98% rename from yandextank/plugins/PhantomUtils.py rename to yandextank/plugins/Phantom/PhantomUtils.py index 5eb41e9..59d1a17 100644 --- a/yandextank/plugins/PhantomUtils.py +++ b/yandextank/plugins/Phantom/PhantomUtils.py @@ -4,11 +4,11 @@ import copy import logging import traceback import multiprocessing -import os import re import socket import string +from pkg_resources import resource_string from yandextank.stepper import StepperWrapper @@ -103,9 +103,7 @@ class PhantomConfig: filename = self.core.mkstemp(".conf", "phantom_") self.core.add_artifact_file(filename) self.log.debug("Generating phantom config: %s", filename) - tpl_file = open("/etc/yandex-tank/Phantom/phantom.conf.tpl", 'r') - template_str = tpl_file.read() - tpl_file.close() + template_str = resource_string(__name__, "config/phantom.conf.tpl") tpl = string.Template(template_str) config = tpl.substitute(kwargs) @@ -310,9 +308,7 @@ class StreamConfig: fname = 'phantom_benchmark_main.tpl' else: fname = 'phantom_benchmark_additional.tpl' - tplf = open('/etc/yandex-tank/Phantom/' + fname, 'r') - template_str = tplf.read() - tplf.close() + template_str = template_str = resource_string(__name__, "config/" + fname) tpl = string.Template(template_str) config = tpl.substitute(kwargs) diff --git a/yandextank/plugins/Phantom/__init__.py b/yandextank/plugins/Phantom/__init__.py new file mode 100644 index 0000000..309c266 --- /dev/null +++ b/yandextank/plugins/Phantom/__init__.py @@ -0,0 +1 @@ +from plugin import * diff --git a/config/Phantom/phantom.conf.tpl b/yandextank/plugins/Phantom/config/phantom.conf.tpl similarity index 100% rename from config/Phantom/phantom.conf.tpl rename to yandextank/plugins/Phantom/config/phantom.conf.tpl diff --git a/config/Phantom/phantom_benchmark_additional.tpl b/yandextank/plugins/Phantom/config/phantom_benchmark_additional.tpl similarity index 100% rename from config/Phantom/phantom_benchmark_additional.tpl rename to yandextank/plugins/Phantom/config/phantom_benchmark_additional.tpl diff --git a/config/Phantom/phantom_benchmark_main.tpl b/yandextank/plugins/Phantom/config/phantom_benchmark_main.tpl similarity index 100% rename from config/Phantom/phantom_benchmark_main.tpl rename to yandextank/plugins/Phantom/config/phantom_benchmark_main.tpl diff --git a/yandextank/plugins/Phantom.py b/yandextank/plugins/Phantom/plugin.py similarity index 98% rename from yandextank/plugins/Phantom.py rename to yandextank/plugins/Phantom/plugin.py index 0a0b138..f5e3064 100644 --- a/yandextank/plugins/Phantom.py +++ b/yandextank/plugins/Phantom/plugin.py @@ -8,11 +8,12 @@ import sys import time import datetime -import ConsoleScreen -from Aggregator import AggregatorPlugin, AggregateResultListener, \ - AbstractReader -from Autostop import AutostopPlugin, AbstractCriteria -from ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget +from yandextank.plugins import ConsoleScreen +from yandextank.plugins.Aggregator import \ + AggregatorPlugin, AggregateResultListener, AbstractReader +from yandextank.plugins.Autostop import AutostopPlugin, AbstractCriteria +from yandextank.plugins.ConsoleOnline import \ + ConsoleOnlinePlugin, AbstractInfoWidget from PhantomUtils import PhantomConfig from yandextank.core import AbstractPlugin import yandextank.core as tankcore diff --git a/yandextank/plugins/TipsAndTricks/__init__.py b/yandextank/plugins/TipsAndTricks/__init__.py new file mode 100644 index 0000000..309c266 --- /dev/null +++ b/yandextank/plugins/TipsAndTricks/__init__.py @@ -0,0 +1 @@ +from plugin import * diff --git a/config/TipsAndTricks/tips.txt b/yandextank/plugins/TipsAndTricks/config/tips.txt similarity index 100% rename from config/TipsAndTricks/tips.txt rename to yandextank/plugins/TipsAndTricks/config/tips.txt diff --git a/yandextank/plugins/TipsAndTricks.py b/yandextank/plugins/TipsAndTricks/plugin.py similarity index 87% rename from yandextank/plugins/TipsAndTricks.py rename to yandextank/plugins/TipsAndTricks/plugin.py index f1502e8..373ef8d 100644 --- a/yandextank/plugins/TipsAndTricks.py +++ b/yandextank/plugins/TipsAndTricks/plugin.py @@ -2,7 +2,9 @@ Plugin showing tool learning hints in console ''' -from ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget +from pkg_resources import resource_stream +from yandextank.plugins.ConsoleOnline import \ + ConsoleOnlinePlugin, AbstractInfoWidget from yandextank.core import AbstractPlugin import random import textwrap @@ -17,7 +19,7 @@ class TipsAndTricksPlugin(AbstractPlugin, AbstractInfoWidget): def __init__(self, core): AbstractPlugin.__init__(self, core) AbstractInfoWidget.__init__(self) - lines = open('/etc/yandex-tank/TipsAndTricks/tips.txt').readlines() + lines = resource_stream(__name__, "config/tips.txt").readlines() line = random.choice(lines) self.section = line[:line.index(':')] self.tip = line[line.index(':') + 1:].strip()