valitydev/yandex-tank
14
0
Fork 0
mirror of https://github.com/valitydev/yandex-tank.git synced 2024-11-06 10:25:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

remove unnecessary class

Browse Source
This commit is contained in:
Andrey Pohilko 2012-09-13 14:45:15 +04:00
parent 8c9589afbc
commit f06b9ccba2
11 changed files with 151 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Tank/ConsoleWorker.py
View File

@ -1,5 +1,5 @@
from Tank.Core import TankCore
from Tank.Utils import CommonUtils
from Tank import Utils
import ConfigParser
import fnmatch
import logging
@ -151,7 +151,7 @@ class ConsoleTank:
info = ConfigParser.ConfigParser()
info.read(full_name)
pid = info.get(TankCore.SECTION, self.PID_OPTION)
if not CommonUtils.pid_exists(int(pid)):
if not Utils.pid_exists(int(pid)):
self.log.debug("Lock PID %s not exists, ignoring and trying to remove", pid)
try:
os.remove(full_name)

4
Tank/Plugins/Aggregator.py
View File

@ -1,5 +1,5 @@
from Tank import Utils
from Tank.Core import AbstractPlugin
from Tank.Utils import CommonUtils
import datetime
import logging
import os
@ -46,7 +46,7 @@ class AggregatorPlugin(AbstractPlugin):
self.phout_file = self.get_option(self.OPTION_SOURCE_FILE, '')
self.threads_file = self.get_option(self.OPTION_STAT_FILE, '')
periods = self.get_option("time_periods", self.default_time_periods).split(" ")
self.time_periods = " ".join([ str(CommonUtils.expand_to_milliseconds(x)) for x in periods ])
self.time_periods = " ".join([ str(Utils.expand_to_milliseconds(x)) for x in periods ])
self.core.set_option(self.SECTION, "time_periods", self.time_periods)
# self.tank_type = self.core.get_option(self.SECTION, "tank_type")
self.preproc_out_filename = self.get_option("preproc_log_name", tempfile.mkstemp(".log", "preproc_")[1])

10
Tank/Plugins/ApacheBenchmark.py
View File

@ -1,6 +1,6 @@
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin
from Tank.Utils import CommonUtils
from Tank import Utils
import subprocess
import tempfile
@ -38,8 +38,8 @@ class ApacheBenchmarkPlugin(AbstractPlugin):
aggregator.set_source_files(self.out_file, None)
def start_test(self):
args = ['ab', '-r', '-g', self.out_file,
'-n', self.requests,
args = ['ab', '-r', '-g', self.out_file,
'-n', self.requests,
'-c', self.concurrency, self.url]
self.log.debug("Starting ab with arguments: %s", args)
self.process = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
@ -53,7 +53,7 @@ class ApacheBenchmarkPlugin(AbstractPlugin):
return -1
if self.process:
CommonUtils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
Utils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
def end_test(self, retcode):
@ -64,7 +64,7 @@ class ApacheBenchmarkPlugin(AbstractPlugin):
self.log.debug("Seems ab finished OK")
if self.process:
CommonUtils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
Utils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
return retcode

16
Tank/Plugins/Autostop.py
View File

@ -1,10 +1,10 @@
from Tank import Utils
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener
from Tank.Plugins.ConsoleOnline import AbstractInfoWidget, ConsoleOnlinePlugin
from Tank.Plugins.Phantom import PhantomPlugin
from Tank.Utils import CommonUtils
import logging
import re
from Tank.Plugins.ConsoleOnline import AbstractInfoWidget, ConsoleOnlinePlugin
class AutostopPlugin(AbstractPlugin, AggregateResultListener):
SECTION = 'autostop'
@ -130,8 +130,8 @@ class AvgTimeCriteria(AbstractCriteria):
def __init__(self, autostop, param_str):
AbstractCriteria.__init__(self)
self.seconds_count = 0
self.rt_limit = CommonUtils.expand_to_milliseconds(param_str.split(',')[0])
self.seconds_limit = CommonUtils.expand_to_seconds(param_str.split(',')[1])
self.rt_limit = Utils.expand_to_milliseconds(param_str.split(',')[0])
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[1])
self.autostop = autostop
def notify(self, aggregate_second):
@ -181,7 +181,7 @@ class HTTPCodesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = CommonUtils.expand_to_seconds(param_str.split(',')[2])
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
def notify(self, aggregate_second):
@ -246,7 +246,7 @@ class NetCodesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = CommonUtils.expand_to_seconds(param_str.split(',')[2])
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[2])
def notify(self, aggregate_second):
@ -311,7 +311,7 @@ class UsedInstancesCriteria(AbstractCriteria):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = CommonUtils.expand_to_seconds(param_str.split(',')[1])
self.seconds_limit = Utils.expand_to_seconds(param_str.split(',')[1])
phantom = autostop.core.get_plugin_of_type(PhantomPlugin)
self.threads_limit = phantom.instances
@ -341,8 +341,6 @@ class UsedInstancesCriteria(AbstractCriteria):
def get_rc(self):
return self.RC_INST
def get_level_str(self):
if self.is_relative:
level_str = str(100 * self.level) + "%"

49
Tank/Plugins/JMeter.py
View File

@ -1,8 +1,8 @@
from Tank.Core import AbstractPlugin
import logging
import os
import subprocess
from Tank.Utils import CommonUtils
import signal
from Tank import Utils
# TODO: make it work with all other plugins
@ -27,29 +27,13 @@ class JMeterPlugin(AbstractPlugin):
def prepare_test(self):
self.args = [self.jmeter_path, "-n", "-t", self.jmx, '-j', self.jmeter_log]
self.args += self.splitstring(self.user_args)
self.args += Utils.splitstring(self.user_args)
def start_test(self):
self.log.debug("Starting %s with arguments: %s", self.jmeter_path, self.args)
self.jmeter_process = subprocess.Popen(self.args, executable=self.jmeter_path, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
self.jmeter_process = subprocess.Popen(self.args, executable=self.jmeter_path, stderr=subprocess.PIPE, stdout=subprocess.PIPE, preexec_fn=os.setsid)
def end_test(self, retcode):
if self.jmeter_process.poll() == None:
self.log.warn("Terminating jmeter process with PID %s", self.jmeter_process.pid)
self.jmeter_process.terminate()
# FIXME: we have a problem here, the JMeter process isn't dead
#import signal
#os.killpg(self.jmeter_process.pid, signal.SIGTERM)
#pkill -P
else:
self.log.debug("Seems JMeter finished OK")
self.core.add_artifact_file(self.jmeter_log)
return retcode
def is_test_finished(self):
CommonUtils.log_stdout_stderr(self.log, self.jmeter_process.stdout, self.jmeter_process.stderr, "jmeter")
rc = self.jmeter_process.poll()
if rc != None:
self.log.debug("JMeter RC %s", rc)
@ -57,17 +41,16 @@ class JMeterPlugin(AbstractPlugin):
else:
return -1
def splitstring(self, string):
"""
>>> string = 'apple orange "banana tree" green'
>>> splitstring(string)
['apple', 'orange', 'green', '"banana tree"']
"""
import re
p = re.compile(r'"[\w ]+"')
if p.search(string):
quoted_item = p.search(string).group()
newstring = p.sub('', string)
return newstring.split() + [quoted_item]
def end_test(self, retcode):
if self.jmeter_process.poll() == None:
self.log.warn("Terminating jmeter process with PID %s", self.jmeter_process.pid)
self.jmeter_process.terminate()
os.killpg(self.jmeter_process.pid, signal.SIGTERM)
else:
return string.split()
self.log.debug("Seems JMeter finished OK")
Utils.log_stdout_stderr(self.log, self.jmeter_process.stdout, self.jmeter_process.stderr, "jmeter")
self.core.add_artifact_file(self.jmeter_log)
return retcode

7
Tank/Plugins/Monitoring.py
View File

@ -1,8 +1,7 @@
from Tank import Utils
from Tank.Core import AbstractPlugin
from Tank.Plugins.DataUploader import DataUploaderPlugin
from Tank.Plugins.Phantom import PhantomPlugin
from Tank.Utils import CommonUtils
import logging
import os
import signal
import subprocess
@ -91,8 +90,8 @@ class MonitoringPlugin(AbstractPlugin):
self.log.warn("Seems the monitoring has been finished")
if self.process:
CommonUtils.log_stdout_stderr(self.log, self.process.stdout, None, self.SECTION)
CommonUtils.log_stdout_stderr(self.log, self.process.stderr, None, self.SECTION + " err")
Utils.log_stdout_stderr(self.log, self.process.stdout, None, self.SECTION)
Utils.log_stdout_stderr(self.log, self.process.stderr, None, self.SECTION + " err")
return retcode

11
Tank/Plugins/Phantom.py
View File

@ -1,12 +1,12 @@
from Tank import Utils
from Tank.Core import AbstractPlugin
from Tank.Plugins.Aggregator import AggregatorPlugin, AggregateResultListener
from Tank.Plugins.ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget
from Tank.Utils import CommonUtils
from ipaddr import AddressValueError
import ConfigParser
import datetime
import hashlib
import ipaddr
import logging
import multiprocessing
import os
import shutil
@ -15,7 +15,6 @@ import string
import subprocess
import tempfile
import time
import datetime
# TODO: req/answ sizes in widget - last sec and curRPS
# TODO: implement phout import
@ -188,7 +187,7 @@ class PhantomPlugin(AbstractPlugin):
self.config = self.compose_config()
args = [self.phantom_path, 'check', self.config]
rc = CommonUtils.execute(args, catch_out=True)
rc = Utils.execute(args, catch_out=True)
if rc:
raise RuntimeError("Subprocess returned %s",)
@ -218,7 +217,7 @@ class PhantomPlugin(AbstractPlugin):
def is_test_finished(self):
CommonUtils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
Utils.log_stdout_stderr(self.log, self.process.stdout, self.process.stderr, self.SECTION)
rc = self.process.poll()
if rc != None:
@ -303,7 +302,7 @@ class PhantomPlugin(AbstractPlugin):
self.args = self.tools_path + "/stepper.py -a " + self.ammo_file + " -c " + stepper_config
self.log.info("Yet calling old external stepper.py")
rc = CommonUtils.execute(self.args, shell=True)
rc = Utils.execute(self.args, shell=True)
if rc:
raise RuntimeError("Subprocess returned %s",)
old_stepper_out_options = "lp.conf"

13
Tank/Plugins/ShellExec.py
View File

@ -2,14 +2,13 @@
Contains shellexec plugin
'''
from Tank.Core import AbstractPlugin
from Tank.Utils import CommonUtils
import logging
from Tank import Utils
'''
ShellExec plugin
allows executing shell scripts before/after test
'''
class ShellExecPlugin(AbstractPlugin):
'''
ShellExec plugin
allows executing shell scripts before/after test
'''
SECTION = 'shellexec'
def __init__(self, core):
@ -56,6 +55,6 @@ class ShellExecPlugin(AbstractPlugin):
def execute(self, cmd):
self.log.debug("Executing: %s", cmd)
retcode = CommonUtils.execute(cmd, shell=True, poll_period=0.1)
retcode = Utils.execute(cmd, shell=True, poll_period=0.1)
if retcode:
raise RuntimeError("Subprocess returned %s",)

195
Tank/Utils.py
View File

@ -10,105 +10,108 @@ import select
import subprocess
import time
'''
Trash code collection class
'''
class CommonUtils(object):
@staticmethod
def log_stdout_stderr(log, stdout, stderr, comment=""):
'''
This function polls stdout and stderr streams and writes their contents to log
'''
readable = select.select([stdout], [], [] , 0)[0]
if stderr:
exceptional = select.select([stderr], [], [] , 0)[0]
else:
exceptional = []
log.debug("Selected: %s, %s", readable, exceptional)
def log_stdout_stderr(log, stdout, stderr, comment=""):
'''
This function polls stdout and stderr streams and writes their contents to log
'''
readable = select.select([stdout], [], [] , 0)[0]
if stderr:
exceptional = select.select([stderr], [], [] , 0)[0]
else:
exceptional = []
log.debug("Selected: %s, %s", readable, exceptional)
for handle in readable:
line = handle.read()
readable.remove(handle)
if line:
log.debug("%s stdout: %s", comment, line.strip())
for handle in exceptional:
line = handle.read()
exceptional.remove(handle)
if line:
log.warn("%s stderr: %s", comment, line.strip())
for handle in readable:
line = handle.read()
readable.remove(handle)
if line:
log.debug("%s stdout: %s", comment, line.strip())
@staticmethod
def expand_to_milliseconds(str_time):
'''
converts 1d2s into milliseconds
'''
return CommonUtils.expand_time(str_time, 'ms', 1000)
@staticmethod
def expand_to_seconds(str_time):
'''
converts 1d2s into seconds
'''
return CommonUtils.expand_time(str_time, 's', 1)
@staticmethod
def expand_time(str_time, default_unit='s', multiplier=1):
'''
helper for above functions
'''
parser = re.compile('(\d+)([a-zA-Z]*)')
parts = parser.findall(str_time)
result = 0.0
for value, unit in parts:
value = int(value)
unit = unit.lower()
if unit == '': unit = default_unit
if unit == 'ms': result += value * 0.001; continue
elif unit == 's': result += value; continue
elif unit == 'm': result += value * 60; continue
elif unit == 'h': result += value * 60 * 60; continue
elif unit == 'd': result += value * 60 * 60 * 24; continue
elif unit == 'w': result += value * 60 * 60 * 24 * 7; continue
else: raise ValueError("String contains unsupported unit %s: %s", unit, str_time)
return int(result * multiplier)
@staticmethod
def pid_exists(pid):
"""Check whether pid exists in the current process table."""
if pid < 0:
return False
try:
os.kill(pid, 0)
except OSError, e:
return e.errno == errno.EPERM
else:
return True
@staticmethod
def execute(cmd, shell=False, poll_period=1, catch_out=False):
log = logging.getLogger(__name__)
log.debug("Starting: %s", cmd)
for handle in exceptional:
line = handle.read()
exceptional.remove(handle)
if line:
log.warn("%s stderr: %s", comment, line.strip())
if catch_out:
process = subprocess.Popen(cmd, shell=shell, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(cmd, shell=shell)
while process.poll() == None:
log.debug("Waiting for process to finish: %s", process)
time.sleep(poll_period)
def expand_to_milliseconds(str_time):
'''
converts 1d2s into milliseconds
'''
return expand_time(str_time, 'ms', 1000)
def expand_to_seconds(str_time):
'''
converts 1d2s into seconds
'''
return expand_time(str_time, 's', 1)
def expand_time(str_time, default_unit='s', multiplier=1):
'''
helper for above functions
'''
parser = re.compile('(\d+)([a-zA-Z]*)')
parts = parser.findall(str_time)
result = 0.0
for value, unit in parts:
value = int(value)
unit = unit.lower()
if unit == '': unit = default_unit
if catch_out:
for line in process.stderr.readlines():
log.warn(line.strip())
for line in process.stdout.readlines():
log.debug(line.strip())
if unit == 'ms': result += value * 0.001; continue
elif unit == 's': result += value; continue
elif unit == 'm': result += value * 60; continue
elif unit == 'h': result += value * 60 * 60; continue
elif unit == 'd': result += value * 60 * 60 * 24; continue
elif unit == 'w': result += value * 60 * 60 * 24 * 7; continue
else: raise ValueError("String contains unsupported unit %s: %s", unit, str_time)
return int(result * multiplier)
def pid_exists(pid):
"""Check whether pid exists in the current process table."""
if pid < 0:
return False
try:
os.kill(pid, 0)
except OSError, e:
return e.errno == errno.EPERM
else:
return True
def execute(cmd, shell=False, poll_period=1, catch_out=False):
log = logging.getLogger(__name__)
log.debug("Starting: %s", cmd)
if catch_out:
process = subprocess.Popen(cmd, shell=shell, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(cmd, shell=shell)
rc = process.poll()
log.debug("Process exit code: %s", rc)
return rc
while process.poll() == None:
log.debug("Waiting for process to finish: %s", process)
time.sleep(poll_period)
if catch_out:
for line in process.stderr.readlines():
log.warn(line.strip())
for line in process.stdout.readlines():
log.debug(line.strip())
rc = process.poll()
log.debug("Process exit code: %s", rc)
return rc
def splitstring(string):
"""
>>> string = 'apple orange "banana tree" green'
>>> splitstring(string)
['apple', 'orange', 'green', '"banana tree"']
"""
p = re.compile(r'"[\w ]+"')
if p.search(string):
quoted_item = p.search(string).group()
newstring = p.sub('', string)
return newstring.split() + [quoted_item]
else:
return string.split()

10
Tests/CommonUtilsTest.py
View File

@ -1,4 +1,4 @@
from Tank.Utils import CommonUtils
from Tank import Utils
from Tests.TankTests import TankTestCase
import unittest
@ -15,11 +15,11 @@ class CommonUtilsTest(TankTestCase):
def test_expand_to_seconds(self):
for i in self.stest:
self.assertEqual(CommonUtils.expand_to_seconds(i[0]), i[1])
self.assertEqual(Utils.expand_to_seconds(i[0]), i[1])
def test_expand_to_seconds_fail(self):
try:
CommonUtils.expand_to_seconds("100n")
Utils.expand_to_seconds("100n")
raise RuntimeError("Exception expected")
except ValueError, ex:
# it's ok, we have excpected exception
@ -28,12 +28,12 @@ class CommonUtilsTest(TankTestCase):
def test_expand_to_milliseconds(self):
for i in self.mstest:
self.assertEqual(CommonUtils.expand_to_milliseconds(i[0]), i[1])
self.assertEqual(Utils.expand_to_milliseconds(i[0]), i[1])
def test_expand_to_milliseconds_fail(self):
try:
CommonUtils.expand_to_milliseconds("100n")
Utils.expand_to_milliseconds("100n")
raise RuntimeError("Exception expected")
except ValueError, ex:
# it's ok, we have excpected exception

4
Tests/Dummy.py
View File

@ -1,11 +1,9 @@
from Tank.Core import AbstractPlugin
import logging
import time
class DummyPlugin(AbstractPlugin):
def __init__(self, core):
#super(AbstractPlugin, self).__init__(core)
self.log = logging.getLogger(__name__)
AbstractPlugin.__init__(self, core)
self.count = 0
@staticmethod