valitydev/yandex-tank
14
0
Fork 0
mirror of https://github.com/valitydev/yandex-tank.git synced 2024-11-06 10:25:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge utils into core

Browse Source
This commit is contained in:
Andrey Pohilko 2012-10-03 20:35:52 +04:00
parent 9b1b60a027
commit 8d5c710d56
16 changed files with 205 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Tank/ConsoleWorker.py
View File

@ -2,7 +2,7 @@
Provides class to run TankCore from console environment
'''
from Tank.Core import TankCore
from Tank import Utils
from Tank import Core
import ConfigParser
import fnmatch
import logging
@ -81,6 +81,7 @@ class ConsoleTank:
self.signal_count = 0
self.scheduled_start = None
self.lock_file = None
def set_baseconfigs_dir(self, directory):
'''
@ -179,7 +180,7 @@ class ConsoleTank:
info = ConfigParser.ConfigParser()
info.read(full_name)
pid = info.get(TankCore.SECTION, self.PID_OPTION)
if not Utils.pid_exists(int(pid)):
if not Core.pid_exists(int(pid)):
self.log.debug("Lock PID %s not exists, ignoring and trying to remove", pid)
try:
os.remove(full_name)
@ -220,7 +221,8 @@ class ConsoleTank:
else:
self.log.warn("Lock files ignored. This is highly unrecommended practice!")
self.core.config.set_out_file(tempfile.mkstemp('.lock', 'lunapark_', self.LOCK_DIR)[1])
self.lock_file = tempfile.mkstemp('.lock', 'lunapark_', self.LOCK_DIR)[1]
self.core.config.set_out_file(self.lock_file)
configs = []

140
Tank/Core.py
View File

@ -3,14 +3,152 @@ The central part of the tool: Core
'''
from ConfigParser import NoSectionError
import ConfigParser
import datetime
import errno
import itertools
import logging
import os
import re
import select
import shutil
import subprocess
import sys
import tempfile
import time
import traceback
import datetime
def log_stdout_stderr(log, stdout, stderr, comment=""):
'''
This function polls stdout and stderr streams and writes their contents to log
'''
readable = select.select([stdout], [], [] , 0)[0]
if stderr:
exceptional = select.select([stderr], [], [] , 0)[0]
else:
exceptional = []
log.debug("Selected: %s, %s", readable, exceptional)
for handle in readable:
line = handle.read()
readable.remove(handle)
if line:
log.debug("%s stdout: %s", comment, line.strip())
for handle in exceptional:
line = handle.read()
exceptional.remove(handle)
if line:
log.warn("%s stderr: %s", comment, line.strip())
def expand_to_milliseconds(str_time):
'''
converts 1d2s into milliseconds
'''
return expand_time(str_time, 'ms', 1000)
def expand_to_seconds(str_time):
'''
converts 1d2s into seconds
'''
return expand_time(str_time, 's', 1)
def expand_time(str_time, default_unit='s', multiplier=1):
'''
helper for above functions
'''
parser = re.compile('(\d+)([a-zA-Z]*)')
parts = parser.findall(str_time)
result = 0.0
for value, unit in parts:
value = int(value)
unit = unit.lower()
if unit == '':
unit = default_unit
if unit == 'ms':
result += value * 0.001
continue
elif unit == 's':
result += value
continue
elif unit == 'm':
result += value * 60
continue
elif unit == 'h':
result += value * 60 * 60
continue
elif unit == 'd':
result += value * 60 * 60 * 24
continue
elif unit == 'w':
result += value * 60 * 60 * 24 * 7
continue
else:
raise ValueError("String contains unsupported unit %s: %s", unit, str_time)
return int(result * multiplier)
def pid_exists(pid):
"""Check whether pid exists in the current process table."""
if pid < 0:
return False
try:
os.kill(pid, 0)
except OSError, exc:
return exc.errno == errno.EPERM
else:
return True
def execute(cmd, shell=False, poll_period=1, catch_out=False):
'''
Wrapper for Popen
'''
log = logging.getLogger(__name__)
log.debug("Starting: %s", cmd)
if catch_out:
process = subprocess.Popen(cmd, shell=shell, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(cmd, shell=shell)
while process.poll() == None:
log.debug("Waiting for process to finish: %s", process)
time.sleep(poll_period)
if catch_out:
for line in process.stderr.readlines():
log.warn(line.strip())
for line in process.stdout.readlines():
log.debug(line.strip())
retcode = process.poll()
log.debug("Process exit code: %s", retcode)
return retcode
def splitstring(string):
"""
>>> string = 'apple orange "banana tree" green'
>>> splitstring(string)
['apple', 'orange', 'green', '"banana tree"']
"""
patt = re.compile(r'"[\w ]+"')
if patt.search(string):
quoted_item = patt.search(string).group()
newstring = patt.sub('', string)
return newstring.split() + [quoted_item]
else:
return string.split()
def pairs(lst):
'''
Iterate over pairs in the list
'''
return itertools.izip(lst[::2], lst[1::2])
class TankCore:
"""

6
Tank/Plugins/Aggregator.py
View File

@ -1,4 +1,4 @@
from Tank import Utils
from Tank import Core
from Tank.Core import AbstractPlugin
import copy
import datetime
@ -31,12 +31,12 @@ class AggregatorPlugin(AbstractPlugin):
self.preproc_out_filename = None
self.cumulative_data = SecondAggregateDataTotalItem()
self.reader = None
self.time_periods = [ Utils.expand_to_milliseconds(x) for x in self.default_time_periods.split(' ') ]
self.time_periods = [ Core.expand_to_milliseconds(x) for x in self.default_time_periods.split(' ') ]
self.last_sample_time = 0
def configure(self):
periods = self.get_option("time_periods", self.default_time_periods).split(" ")
self.time_periods = [ Utils.expand_to_milliseconds(x) for x in periods ]
self.time_periods = [ Core.expand_to_milliseconds(x) for x in periods ]
self.core.set_option(self.SECTION, "time_periods", " ".join([ str(x) for x in periods ]))
def start_test(self):

6
Tank/Plugins/ApacheBenchmark.py
View File

@ -1,4 +1,4 @@
from Tank import Utils
from Tank import Core
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin, AbstractReader
import os
@ -69,7 +69,7 @@ class ApacheBenchmarkPlugin(AbstractPlugin):
return -1
if self.process:
Utils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
Core.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
def end_test(self, retcode):
@ -80,7 +80,7 @@ class ApacheBenchmarkPlugin(AbstractPlugin):
self.log.info("Seems ab finished OK")
if self.process:
Utils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
Core.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
return retcode

14
Tank/Plugins/Autostop.py
View File

@ -1,4 +1,4 @@
from Tank import Utils
from Tank import Core
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener
from Tank.Plugins.ConsoleOnline import AbstractInfoWidget, ConsoleOnlinePlugin
@ -150,8 +150,8 @@ class AvgTimeCriteria(AbstractCriteria):
def __init__(self, autostop, param_str):
AbstractCriteria.__init__(self)
self.seconds_count = 0
self.rt_limit = Utils.expand_to_milliseconds(param_str.split(',')[0])
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[1])
self.rt_limit = Core.expand_to_milliseconds(param_str.split(',')[0])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[1])
self.autostop = autostop
def notify(self, aggregate_second):
@ -201,7 +201,7 @@ class HTTPCodesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[2])
def notify(self, aggregate_second):
@ -266,7 +266,7 @@ class NetCodesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[2])
def notify(self, aggregate_second):
@ -323,8 +323,8 @@ class QuantileCriteria(AbstractCriteria):
AbstractCriteria.__init__(self)
self.seconds_count = 0
self.quantile = float(param_str.split(',')[0])
self.rt_limit = Utils.expand_to_milliseconds(param_str.split(',')[1])
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
self.rt_limit = Core.expand_to_milliseconds(param_str.split(',')[1])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[2])
self.autostop = autostop
def notify(self, aggregate_second):

12
Tank/Plugins/JMeter.py
View File

@ -1,12 +1,12 @@
from Tank import Core
from Tank.Core import AbstractPlugin
import os
import subprocess
import signal
from Tank import Utils
from Tank.Plugins.Aggregator import AbstractReader, AggregatorPlugin, \
AggregateResultListener
import tempfile
from Tank.Plugins.ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget
import os
import signal
import subprocess
import tempfile
import time
@ -38,7 +38,7 @@ class JMeterPlugin(AbstractPlugin):
def prepare_test(self):
self.args = [self.jmeter_path, "-n", "-t", self.jmx, '-j', self.jmeter_log, '-Jjmeter.save.saveservice.default_delimiter=\\t']
self.args += Utils.splitstring(self.user_args)
self.args += Core.splitstring(self.user_args)
aggregator = None
try:

14
Tank/Plugins/Phantom.py
View File

@ -1,7 +1,7 @@
'''
Contains Phantom Plugin, Console widgets, result reader classes
'''
from Tank import Utils
from Tank import Core
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener, \
AbstractReader
@ -116,7 +116,7 @@ class PhantomPlugin(AbstractPlugin):
if reverse_name.startswith(self.address):
self.address = ip_addr
else:
raise ValueError("Address %s reverse-resolved to %s, but must match", self.address, reverse_name)
raise ValueError("Address %s reverse-resolved to %s, but must match" % (self.address, reverse_name))
def __read_phantom_options(self):
@ -286,7 +286,7 @@ class PhantomPlugin(AbstractPlugin):
self.config = self.__compose_config()
args = [self.phantom_path, 'check', self.config]
retcode = Utils.execute(args, catch_out=True)
retcode = Core.execute(args, catch_out=True)
if retcode:
raise RuntimeError("Subprocess returned %s",)
@ -322,7 +322,7 @@ class PhantomPlugin(AbstractPlugin):
def is_test_finished(self):
if not self.phout_import_mode:
Utils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
Core.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
retcode = self.process.poll()
if retcode != None:
@ -391,7 +391,7 @@ class PhantomPlugin(AbstractPlugin):
Get total test duration
'''
duration = 0
for rps, dur in Utils.pairs(steps):
for rps, dur in Core.pairs(steps):
duration += dur
self.core.set_option(self.SECTION, self.OPTION_TEST_DURATION, str(duration))
@ -578,7 +578,7 @@ class PhantomReader(AbstractReader):
self.log.debug("Opening phout file: %s", self.phantom.phout_file)
self.phout = open(self.phantom.phout_file, 'r')
# strange decision to place it here, but no better idea yet
for item in Utils.pairs(self.phantom.steps):
for item in Core.pairs(self.phantom.steps):
self.steps.append([item[0], item[1]])
if not self.stat and self.phantom.stat_log and os.path.exists(self.phantom.stat_log):
@ -717,7 +717,7 @@ class UsedInstancesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[1])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[1])
try:
phantom = autostop.core.get_plugin_of_type(PhantomPlugin)

4
Tank/Plugins/ShellExec.py
View File

@ -2,7 +2,7 @@
Contains shellexec plugin
'''
from Tank.Core import AbstractPlugin
from Tank import Utils
from Tank import Core
class ShellExecPlugin(AbstractPlugin):
'''
@ -58,6 +58,6 @@ class ShellExecPlugin(AbstractPlugin):
Execute and check exit code
'''
self.log.debug("Executing: %s", cmd)
retcode = Utils.execute(cmd, shell=True, poll_period=0.1)
retcode = Core.execute(cmd, shell=True, poll_period=0.1)
if retcode:
raise RuntimeError("Subprocess returned %s",)

10
Tank/Plugins/Stepper.py
View File

@ -8,7 +8,7 @@ import operator
import os
import re
import tempfile
from Tank import Utils
from Tank import Core
from progressbar import ProgressBar, Percentage, Bar, ETA
# TODO: 3 add stpd file size estimation
@ -130,7 +130,7 @@ def load_const(req, duration):
'''
Constant load pattern
'''
dur = Utils.expand_to_seconds(duration)
dur = Core.expand_to_seconds(duration)
steps = []
steps.append([req, dur])
ls = '%s,const,%s,%s,(%s,%s)\n' % (dur, req, req, req, duration)
@ -144,7 +144,7 @@ def load_line(start, end, duration):
'''
Linear load pattern
'''
dur = Utils.expand_to_seconds(duration)
dur = Core.expand_to_seconds(duration)
diff_k = float((end - start) / float(dur - 1))
(cnt, last_x, total) = (1, start, 0)
ls = '%s,line,%s,%s,(%s,%s,%s)\n' % (
@ -171,7 +171,7 @@ def load_step(start, end, step, duration,):
'''
Stepping load pattern
'''
dur = Utils.expand_to_seconds(duration)
dur = Core.expand_to_seconds(duration)
(steps, ls, total) = ([], '', 0)
if end > start:
for rps_level in range(start, end + 1, step):
@ -331,7 +331,7 @@ def constf(req, duration):
pattern = re.match("(\d+)\/(\d+)", req)
if pattern:
(a, b) = (int(pattern.group(1)), int(pattern.group(2)))
(dur, e) = (Utils.expand_to_seconds(duration), int(a / b))
(dur, e) = (Core.expand_to_seconds(duration), int(a / b))
fr = '%.3f' % (float(a) / b)
ls = '%s,const,%s,%s,(%s,%s)\n' % (dur, fr, fr, req, duration)

23
Tank/Plugins/TotalAutostop.py
View File

@ -1,16 +1,11 @@
from Tank import Utils
from Tank import Core
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener
from Tank.Plugins.ConsoleOnline import AbstractInfoWidget, ConsoleOnlinePlugin
from Tank.Plugins.Phantom import PhantomPlugin
from Tank.Plugins.Autostop import AbstractCriteria
from Tank.Plugins.Autostop import AutostopPlugin
from Tank.Plugins.Autostop import AutostopWidget
from Tank.Plugins.Aggregator import AggregateResultListener
from Tank.Plugins.Autostop import AbstractCriteria, AutostopPlugin
from collections import deque
import logging
import re
class TotalAutostopPlugin(AbstractPlugin, AggregateResultListener):
SECTION='autostop'
@staticmethod
@ -42,9 +37,9 @@ class TotalFracTimeCriteria(AbstractCriteria):
AbstractCriteria.__init__(self)
param = param_str.split(',')
self.seconds_count = 0
self.rt_limit = Utils.expand_to_milliseconds(param[0])
self.rt_limit = Core.expand_to_milliseconds(param[0])
self.frac = param[1][:-1]
self.seconds_limit = Utils.expand_to_seconds(param[2])
self.seconds_limit = Core.expand_to_seconds(param[2])
self.autostop = autostop
self.data = deque()
@ -98,7 +93,7 @@ class TotalHTTPCodesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[2])
def notify(self, aggregate_second):
matched_responses = self.count_matched_codes(self.codes_regex, aggregate_second.overall.http_codes)
@ -173,7 +168,7 @@ class TotalNetCodesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[2])
def notify(self, aggregate_second):
@ -244,7 +239,7 @@ class TotalNegativeHTTPCodesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
self.seconds_limit = Core.expand_to_seconds(param_str.split(',')[2])
def notify(self, aggregate_second):
matched_responses = self.count_matched_codes(self.codes_regex, aggregate_second.overall.http_codes)

6
Tank/Plugins/WebOnline.py
View File

@ -1,13 +1,13 @@
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from Tank import Core
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener
from threading import Thread
import json
import logging
import os.path
import socket
import time
import json
from Tank import Utils
class WebOnlinePlugin(AbstractPlugin, Thread, AggregateResultListener):
SECTION = "web"
@ -29,7 +29,7 @@ class WebOnlinePlugin(AbstractPlugin, Thread, AggregateResultListener):
def configure(self):
self.port = int(self.get_option("port", self.port))
self.interval = int(Utils.expand_to_seconds(self.get_option("interval", '1m')))
self.interval = int(Core.expand_to_seconds(self.get_option("interval", '1m')))
self.redirect = self.get_option("redirect", self.redirect)
def prepare_test(self):

143
Tank/Utils.py
View File

@ -1,143 +0,0 @@
'''
Commonly used utilities contained here
'''
import errno
import itertools
import logging
import os
import re
import select
import subprocess
import time
def log_stdout_stderr(log, stdout, stderr, comment=""):
'''
This function polls stdout and stderr streams and writes their contents to log
'''
readable = select.select([stdout], [], [] , 0)[0]
if stderr:
exceptional = select.select([stderr], [], [] , 0)[0]
else:
exceptional = []
log.debug("Selected: %s, %s", readable, exceptional)
for handle in readable:
line = handle.read()
readable.remove(handle)
if line:
log.debug("%s stdout: %s", comment, line.strip())
for handle in exceptional:
line = handle.read()
exceptional.remove(handle)
if line:
log.warn("%s stderr: %s", comment, line.strip())
def expand_to_milliseconds(str_time):
'''
converts 1d2s into milliseconds
'''
return expand_time(str_time, 'ms', 1000)
def expand_to_seconds(str_time):
'''
converts 1d2s into seconds
'''
return expand_time(str_time, 's', 1)
def expand_time(str_time, default_unit='s', multiplier=1):
'''
helper for above functions
'''
parser = re.compile('(\d+)([a-zA-Z]*)')
parts = parser.findall(str_time)
result = 0.0
for value, unit in parts:
value = int(value)
unit = unit.lower()
if unit == '':
unit = default_unit
if unit == 'ms':
result += value * 0.001
continue
elif unit == 's':
result += value
continue
elif unit == 'm':
result += value * 60
continue
elif unit == 'h':
result += value * 60 * 60
continue
elif unit == 'd':
result += value * 60 * 60 * 24
continue
elif unit == 'w':
result += value * 60 * 60 * 24 * 7
continue
else:
raise ValueError("String contains unsupported unit %s: %s", unit, str_time)
return int(result * multiplier)
def pid_exists(pid):
"""Check whether pid exists in the current process table."""
if pid < 0:
return False
try:
os.kill(pid, 0)
except OSError, exc:
return exc.errno == errno.EPERM
else:
return True
def execute(cmd, shell=False, poll_period=1, catch_out=False):
'''
Wrapper for Popen
'''
log = logging.getLogger(__name__)
log.debug("Starting: %s", cmd)
if catch_out:
process = subprocess.Popen(cmd, shell=shell, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(cmd, shell=shell)
while process.poll() == None:
log.debug("Waiting for process to finish: %s", process)
time.sleep(poll_period)
if catch_out:
for line in process.stderr.readlines():
log.warn(line.strip())
for line in process.stdout.readlines():
log.debug(line.strip())
retcode = process.poll()
log.debug("Process exit code: %s", retcode)
return retcode
def splitstring(string):
"""
>>> string = 'apple orange "banana tree" green'
>>> splitstring(string)
['apple', 'orange', 'green', '"banana tree"']
"""
patt = re.compile(r'"[\w ]+"')
if patt.search(string):
quoted_item = patt.search(string).group()
newstring = patt.sub('', string)
return newstring.split() + [quoted_item]
else:
return string.split()
def pairs(lst):
'''
Iterate over pairs in the list
'''
return itertools.izip(lst[::2], lst[1::2])

10
Tests/CommonUtilsTest.py
View File

@ -1,4 +1,4 @@
from Tank import Utils
from Tank import Core
from Tests.TankTests import TankTestCase
import unittest
@ -15,11 +15,11 @@ class CommonUtilsTest(TankTestCase):
def test_expand_to_seconds(self):
for i in self.stest:
self.assertEqual(Utils.expand_to_seconds(i[0]), i[1])
self.assertEqual(Core.expand_to_seconds(i[0]), i[1])
def test_expand_to_seconds_fail(self):
try:
Utils.expand_to_seconds("100n")
Core.expand_to_seconds("100n")
raise RuntimeError("Exception expected")
except ValueError, ex:
# it's ok, we have excpected exception
@ -28,12 +28,12 @@ class CommonUtilsTest(TankTestCase):
def test_expand_to_milliseconds(self):
for i in self.mstest:
self.assertEqual(Utils.expand_to_milliseconds(i[0]), i[1])
self.assertEqual(Core.expand_to_milliseconds(i[0]), i[1])
def test_expand_to_milliseconds_fail(self):
try:
Utils.expand_to_milliseconds("100n")
Core.expand_to_milliseconds("100n")
raise RuntimeError("Exception expected")
except ValueError, ex:
# it's ok, we have excpected exception

6
debian/install vendored
View File

@ -1,4 +1,8 @@
Tank usr/lib/yandex-tank
Tank/__init__.py usr/lib/yandex-tank/Tank
Tank/ConsoleWorker.py usr/lib/yandex-tank/Tank
Tank/Plugins usr/lib/yandex-tank/Tank
Tank/MonCollector usr/lib/yandex-tank/Tank
tank.py usr/lib/yandex-tank
*.sh usr/lib/yandex-tank
yandex-tank.ini etc/yandex-tank

3
debian/rules vendored
View File

@ -81,6 +81,9 @@ binary-indep: install
dh_md5sums
dh_builddeb
override_dh_pysupport:
dh_python2
# Build architecture-dependent files here.
binary-arch: install

3
debian/yandex-load-tank-base.pyinstall vendored Normal file
View File

@ -0,0 +1,3 @@
Tank/__init__.py 2.6-
Tank/Core.py 2.6-
Tank/Utils.py 2.6-