move plugins config files to into python modules and use pkg_resources

setup.py

@@ -45,33 +45,18 @@ analytic tools for the results they produce.
             'yandex-tank = yandextank.core.cli:main',
         ],
     },
-    package_data={},
+    package_data={
+        'yandextank.plugins.GraphiteUploader': ['config/*'],
+        'yandextank.plugins.JMeter': ['config/*'],
+        'yandextank.plugins.Monitoring': ['config/*'],
+        'yandextank.plugins.Phantom': ['config/*'],
+        'yandextank.plugins.TipsAndTricks': ['config/*'],
+    },
     # TODO: move them all to resources maybe
     data_files=[
         ('/etc/yandex-tank', [
             'config/00-base.ini',
         ]),
-        ('/etc/yandex-tank/JMeter', [
-            'config/JMeter/jmeter_argentum.xml',
-            'config/JMeter/jmeter_var_template.xml',
-            'config/JMeter/jmeter_writer.xml',
-        ]),
-        ('/etc/yandex-tank/GraphiteUploader', [
-            'config/GraphiteUploader/graphite-js.tpl',
-            'config/GraphiteUploader/graphite.tpl',
-        ]),
-        ('/etc/yandex-tank/Monitoring', [
-            'config/Monitoring/agent.cfg',
-            'config/Monitoring/monitoring_default_config.xml',
-        ]),
-        ('/etc/yandex-tank/Phantom', [
-            'config/Phantom/phantom.conf.tpl',
-            'config/Phantom/phantom_benchmark_additional.tpl',
-            'config/Phantom/phantom_benchmark_main.tpl',
-        ]),
-        ('/etc/yandex-tank/TipsAndTricks', [
-            'config/TipsAndTricks/tips.txt',
-        ]),
         ('/etc/bash_completion.d', [
             'data/yandex-tank.completion'
         ]),

yandextank/plugins/GraphiteUploader/__init__.py

@@ -0,0 +1 @@
+from plugin import *

yandextank/plugins/GraphiteUploader/plugin.py

@@ -1,6 +1,8 @@
 '''Graphite Uploader plugin that sends aggregated data to Graphite server'''
 
-from Aggregator import AggregateResultListener, AggregatorPlugin
+from pkg_resources import resource_string
+from yandextank.plugins.Aggregator import \
+    AggregateResultListener, AggregatorPlugin
 from yandextank.core import AbstractPlugin
 import logging
 import socket
@@ -10,6 +12,7 @@ import datetime
 
 
 class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
+
     '''Graphite data uploader'''
 
     SECTION = 'graphite'
@@ -30,7 +33,7 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
         self.start_time = start_time.strftime("%H:%M%%20%Y%m%d")
 
     def end_test(self, retcode):
-        end_time = datetime.datetime.now() + datetime.timedelta(minutes = 1)
+        end_time = datetime.datetime.now() + datetime.timedelta(minutes=1)
         self.end_time = end_time.strftime("%H:%M%%20%Y%m%d")
         return retcode
 
@@ -38,16 +41,22 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
         '''Read configuration'''
         self.address = self.get_option("address", "")
         if self.address == "":
-            self.log.warning("Graphite uploader is not configured and will not send any data")
+            self.log.warning(
+                "Graphite uploader is not configured and will not send any data")
         else:
             port = self.get_option("port", "2003")
             self.web_port = self.get_option("web_port", "8080")
             self.prefix = self.get_option("prefix", "one_sec.yandex_tank")
-            default_template = "/etc/yandex-tank/GraphiteUploader/graphite.tpl"
-            if self.get_option("js", "1") == "1":
-                default_template = "/etc/yandex-tank/GraphiteUploader/graphite-js.tpl"
-            self.template = self.get_option("template", default_template)
-            self.graphite_client = GraphiteClient(self.prefix, self.address, port)
+            specified_template = self.get_option("template", "")
+            if specified_template != "":
+                self.template = open(specified_template, 'r').read()
+            else:
+                default_template = "graphite.tpl"
+                if self.get_option("js", "1") == "1":
+                    default_template = "graphite-js.tpl"
+                self.template = resource_string(__name__, 'config/' + default_template)
+            self.graphite_client = GraphiteClient(
+                self.prefix, self.address, port)
             aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
             aggregator.add_result_listener(self)
 
@@ -55,7 +64,7 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
         """
         @data: SecondAggregateData
         """
-        #TODO: Use ts from data
+        # TODO: Use ts from data
         if self.graphite_client:
             results = {}
             overall = GraphiteUploaderPlugin.__flatten(
@@ -72,12 +81,11 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
 
     def post_process(self, retcode):
         if self.graphite_client:
-            template = open(self.template, 'r').read()
             graphite_html = self.core.mkstemp(".html", "graphite_")
             self.core.add_artifact_file(graphite_html)
             with open(graphite_html, 'w') as graphite_html_file:
                 graphite_html_file.write(
-                    string.Template(template).safe_substitute(
+                    string.Template(self.template).safe_substitute(
                         host=self.address,
                         width=1000,
                         height=400,
@@ -119,6 +127,7 @@ class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
 
 
 class GraphiteClient(object):
+
     '''Graphite client that writes metrics to Graphite server'''
 
     def __init__(self, prefix, address, port):

yandextank/plugins/JMeter/__init__.py

@@ -0,0 +1 @@
+from plugin import *

yandextank/plugins/JMeter/plugin.py

@@ -7,15 +7,18 @@ import time
 import datetime
 import json
 
-from Aggregator import AbstractReader, AggregatorPlugin, \
+from pkg_resources import resource_string
+from yandextank.plugins.Aggregator import AbstractReader, AggregatorPlugin, \
     AggregateResultListener, SecondAggregateDataItem
-from ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget
+from yandextank.plugins.ConsoleOnline import \
+    ConsoleOnlinePlugin, AbstractInfoWidget
 from yandextank.core import AbstractPlugin
 import yandextank.core as tankcore
-import ConsoleScreen
+import yandextank.plugins.ConsoleScreen as ConsoleScreen
 
 
 class JMeterPlugin(AbstractPlugin):
+
     """ JMeter tank plugin """
     SECTION = 'jmeter'
 
@@ -52,7 +55,8 @@ class JMeterPlugin(AbstractPlugin):
                                                       self.get_option('buffered_seconds', '3')))
         self.core.add_artifact_file(self.jmeter_log, True)
         self.use_argentum = eval(self.get_option('use_argentum', 'False'))
-        self.jmx = self.__add_jmeter_components(self.original_jmx, self.jtl_file, self._get_variables())
+        self.jmx = self.__add_jmeter_components(
+            self.original_jmx, self.jtl_file, self._get_variables())
         self.core.add_artifact_file(self.jmx)
 
     def prepare_test(self):
@@ -84,7 +88,8 @@ class JMeterPlugin(AbstractPlugin):
                 aggregator.add_result_listener(widget)
 
     def start_test(self):
-        self.log.info("Starting %s with arguments: %s", self.jmeter_path, self.args)
+        self.log.info(
+            "Starting %s with arguments: %s", self.jmeter_path, self.args)
         self.jmeter_process = subprocess.Popen(self.args, executable=self.jmeter_path, preexec_fn=os.setsid,
                                                close_fds=True)  # stderr=subprocess.PIPE, stdout=subprocess.PIPE,
         self.start_time = time.time()
@@ -92,14 +97,16 @@ class JMeterPlugin(AbstractPlugin):
     def is_test_finished(self):
         retcode = self.jmeter_process.poll()
         if retcode is not None:
-            self.log.info("JMeter process finished with exit code: %s", retcode)
+            self.log.info(
+                "JMeter process finished with exit code: %s", retcode)
             return retcode
         else:
             return -1
 
     def end_test(self, retcode):
         if self.jmeter_process:
-            self.log.info("Terminating jmeter process group with PID %s", self.jmeter_process.pid)
+            self.log.info(
+                "Terminating jmeter process group with PID %s", self.jmeter_process.pid)
             try:
                 os.killpg(self.jmeter_process.pid, signal.SIGTERM)
             except OSError, exc:
@@ -123,24 +130,23 @@ class JMeterPlugin(AbstractPlugin):
         except Exception, exc:
             raise RuntimeError("Failed to find the end of JMX XML: %s" % exc)
 
-        tpl_filepath = '/etc/yandex-tank/JMeter/jmeter_writer.xml'
+        tpl_resource = 'jmeter_writer.xml'
 
         if self.use_argentum:
-            self.log.warn("You are using argentum aggregator for JMeter. Be careful.")
-            tpl_filepath = '/etc/yandex-tank/JMeter/jmeter_argentum.xml'
+            self.log.warn(
+                "You are using argentum aggregator for JMeter. Be careful.")
+            tpl_resource = 'jmeter_argentum.xml'
 
-        with open(tpl_filepath, 'r') as tpl_file:
-            tpl = tpl_file.read()
-
-        with open('/etc/yandex-tank/JMeter/jmeter_var_template.xml', 'r') as tpl_file:
-            udv_tpl = tpl_file.read()
+        tpl = resource_string(__name__, 'config/' + tpl_resource)
+        udv_tpl = resource_string(__name__, 'config/jmeter_var_template.xml')
 
         udv_set = []
         for var_name, var_value in variables.iteritems():
             udv_set.append(udv_tpl % (var_name, var_name, var_value))
 
         try:
-            new_file = self.core.mkstemp('.jmx', 'modified_', os.path.dirname(os.path.realpath(jmx)))
+            new_file = self.core.mkstemp(
+                '.jmx', 'modified_', os.path.dirname(os.path.realpath(jmx)))
         except OSError, exc:
             self.log.debug("Can't create modified jmx near original: %s", exc)
             new_file = self.core.mkstemp('.jmx', 'modified_')
@@ -167,6 +173,7 @@ class JMeterPlugin(AbstractPlugin):
 
 
 class JMeterReader(AbstractReader):
+
     """ JTL files reader """
     KNOWN_EXC = {
         "java.net.NoRouteToHostException": 113,
@@ -224,10 +231,12 @@ class JMeterReader(AbstractReader):
                     return None
                 else:
                     # good json-object. parse it!
-                    second_ag = self.get_zero_sample(datetime.datetime.fromtimestamp(second['second']))
+                    second_ag = self.get_zero_sample(
+                        datetime.datetime.fromtimestamp(second['second']))
                     second_ag.overall.avg_connect_time = 0
                     second_ag.overall.avg_send_time = 0
-                    second_ag.overall.avg_receive_time = second['avg_rt'] - second['avg_lt']
+                    second_ag.overall.avg_receive_time = second[
+                        'avg_rt'] - second['avg_lt']
                     second_ag.overall.avg_response_time = second['avg_rt']
                     second_ag.overall.avg_latency = second['avg_lt']
                     second_ag.overall.RPS = second['th']
@@ -242,7 +251,8 @@ class JMeterReader(AbstractReader):
                     second_ag.overall.http_codes = rc_map
 
                     for percentile in second['percentile'].keys():
-                        second_ag.overall.quantiles[int(float(percentile))] = second['percentile'][percentile]
+                        second_ag.overall.quantiles[
+                            int(float(percentile))] = second['percentile'][percentile]
                         second_ag.cumulative.quantiles[int(float(percentile))] = second['cumulative_percentile'][
                             percentile]
 
@@ -251,13 +261,18 @@ class JMeterReader(AbstractReader):
                     for sampler in second['samplers'].keys():
                         sampler_ag_data_item = SecondAggregateDataItem()
                         sampler_ag_data_item.case = sampler
-                        sampler_ag_data_item.active_threads = second['active_threads']
-                        sampler_ag_data_item.RPS = int(second['samplers'][sampler])
-                        sampler_ag_data_item.times_dist = second['sampler_interval_dist'][sampler]
+                        sampler_ag_data_item.active_threads = second[
+                            'active_threads']
+                        sampler_ag_data_item.RPS = int(
+                            second['samplers'][sampler])
+                        sampler_ag_data_item.times_dist = second[
+                            'sampler_interval_dist'][sampler]
 
-                        sampler_ag_data_item.quantiles = second['sampler_percentile'][sampler]
+                        sampler_ag_data_item.quantiles = second[
+                            'sampler_percentile'][sampler]
 
-                        sampler_ag_data_item.avg_response_time = second['sampler_avg_rt'][sampler]
+                        sampler_ag_data_item.avg_response_time = second[
+                            'sampler_avg_rt'][sampler]
                         second_ag.cases[sampler] = sampler_ag_data_item
                 return second_ag
         return None
@@ -279,7 +294,8 @@ class JMeterReader(AbstractReader):
                     # self.log.warning("Wrong jtl line, skipped: %s", line)
                     continue
                 cur_time = int(data[0]) / 1000
-                netcode = '0' if data[4] == 'true' else self.exc_to_net(data[3])
+                netcode = '0' if data[
+                    4] == 'true' else self.exc_to_net(data[3])
 
                 if not cur_time in self.data_buffer.keys():
                     if self.data_queue and self.data_queue[0] >= cur_time:
@@ -290,7 +306,8 @@ class JMeterReader(AbstractReader):
                         self.data_queue.append(cur_time)
                         self.data_buffer[cur_time] = []
                 # marker, threads, overallRT, httpCode, netCode
-                data_item = [data[2], int(data[7]), int(data[1]), self.exc_to_http(data[3]), netcode]
+                data_item = [
+                    data[2], int(data[7]), int(data[1]), self.exc_to_http(data[3]), netcode]
                 # bytes:     sent    received
                 data_item += [0, int(data[5])]
                 # connect    send    latency    receive
@@ -315,7 +332,8 @@ class JMeterReader(AbstractReader):
         if exc in self.KNOWN_EXC.keys():
             return self.KNOWN_EXC[exc]
         else:
-            self.log.warning("Not known Java exception, consider adding it to dictionary: %s", param1)
+            self.log.warning(
+                "Not known Java exception, consider adding it to dictionary: %s", param1)
             return '1'
 
     def exc_to_http(self, param1):
@@ -333,6 +351,7 @@ class JMeterReader(AbstractReader):
 # ===============================================================================
 
 class JMeterInfoWidget(AbstractInfoWidget, AggregateResultListener):
+
     """ Right panel widget with JMeter test info """
 
     def __init__(self, jmeter):
@@ -364,6 +383,7 @@ class JMeterInfoWidget(AbstractInfoWidget, AggregateResultListener):
         template += "      Duration: %s\n"
         template += "Active Threads: %s\n"
         template += "   Responses/s: %s"
-        data = (os.path.basename(self.jmeter.original_jmx), duration, self.active_threads, self.RPS)
+        data = (os.path.basename(self.jmeter.original_jmx),
+                duration, self.active_threads, self.RPS)
 
         return template % data

yandextank/plugins/Monitoring/__init__.py

@@ -1,4 +1 @@
-'''
-Package contains all original tank tool plugins
-'''
 from plugin import *

yandextank/plugins/Monitoring/plugin.py

@@ -6,6 +6,7 @@ import traceback
 import fnmatch
 import datetime
 
+from pkg_resources import resource_string
 from collector import MonitoringCollector, \
     MonitoringDataListener, MonitoringDataDecoder
 from yandextank.plugins.ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget
@@ -63,7 +64,10 @@ class MonitoringPlugin(AbstractPlugin):
             self.monitoring = None
 
         if self.config == 'auto':
-            self.config = '/etc/yandex-tank/Monitoring/monitoring_default_config.xml'
+            default_config = resource_string(__name__, 'config/monitoring_default_config.xml')
+            self.config = self.core.mkstemp(".xml", "monitoring_default_")
+            with open(self.config, 'w') as cfg_file:
+                cfg_file.write(default_config)
 
         try:
             autostop = self.core.get_plugin_of_type(AutostopPlugin)

yandextank/plugins/Phantom/PhantomUtils.py

@@ -4,11 +4,11 @@ import copy
 import logging
 import traceback
 import multiprocessing
-import os
 import re
 import socket
 import string
 
+from pkg_resources import resource_string
 from yandextank.stepper import StepperWrapper
 
 
@@ -103,9 +103,7 @@ class PhantomConfig:
         filename = self.core.mkstemp(".conf", "phantom_")
         self.core.add_artifact_file(filename)
         self.log.debug("Generating phantom config: %s", filename)
-        tpl_file = open("/etc/yandex-tank/Phantom/phantom.conf.tpl", 'r')
-        template_str = tpl_file.read()
-        tpl_file.close()
+        template_str = resource_string(__name__, "config/phantom.conf.tpl")
         tpl = string.Template(template_str)
         config = tpl.substitute(kwargs)
 
@@ -310,9 +308,7 @@ class StreamConfig:
             fname = 'phantom_benchmark_main.tpl'
         else:
             fname = 'phantom_benchmark_additional.tpl'
-        tplf = open('/etc/yandex-tank/Phantom/' + fname, 'r')
-        template_str = tplf.read()
-        tplf.close()
+        template_str = template_str = resource_string(__name__, "config/" + fname)
         tpl = string.Template(template_str)
         config = tpl.substitute(kwargs)
 

yandextank/plugins/Phantom/__init__.py

@@ -0,0 +1 @@
+from plugin import *

yandextank/plugins/Phantom/plugin.py

@@ -8,11 +8,12 @@ import sys
 import time
 import datetime
 
-import ConsoleScreen
-from Aggregator import AggregatorPlugin, AggregateResultListener, \
-    AbstractReader
-from Autostop import AutostopPlugin, AbstractCriteria
-from ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget
+from yandextank.plugins import ConsoleScreen
+from yandextank.plugins.Aggregator import \
+    AggregatorPlugin, AggregateResultListener, AbstractReader
+from yandextank.plugins.Autostop import AutostopPlugin, AbstractCriteria
+from yandextank.plugins.ConsoleOnline import \
+    ConsoleOnlinePlugin, AbstractInfoWidget
 from PhantomUtils import PhantomConfig
 from yandextank.core import AbstractPlugin
 import yandextank.core as tankcore

yandextank/plugins/TipsAndTricks/__init__.py

@@ -0,0 +1 @@
+from plugin import *

yandextank/plugins/TipsAndTricks/plugin.py

@@ -2,7 +2,9 @@
 Plugin showing tool learning hints in console
 '''
 
-from ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget
+from pkg_resources import resource_stream
+from yandextank.plugins.ConsoleOnline import \
+    ConsoleOnlinePlugin, AbstractInfoWidget
 from yandextank.core import AbstractPlugin
 import random
 import textwrap
@@ -17,7 +19,7 @@ class TipsAndTricksPlugin(AbstractPlugin, AbstractInfoWidget):
     def __init__(self, core):
         AbstractPlugin.__init__(self, core)
         AbstractInfoWidget.__init__(self)
-        lines = open('/etc/yandex-tank/TipsAndTricks/tips.txt').readlines()
+        lines = resource_stream(__name__, "config/tips.txt").readlines()
         line = random.choice(lines)
         self.section = line[:line.index(':')]
         self.tip = line[line.index(':') + 1:].strip()