valitydev/yandex-tank
14
0
Fork 0
mirror of https://github.com/valitydev/yandex-tank.git synced 2024-11-06 18:35:18 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

maven plugin

Browse Source
This commit is contained in:
Alexey Lavrenuke 2016-04-05 17:17:00 +03:00
parent 159e3b357e
commit 9d3bf49154
9 changed files with 241 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
yandextank/core/consoleworker.py
View File

@ -179,12 +179,11 @@ class ConsoleTank:
try:
self.core.get_lock(self.options.ignore_lock)
break
except Exception, exc:
except Exception as exc:
if self.options.lock_fail:
raise RuntimeError("Lock file present, cannot continue")
self.log.info(
"Couldn't get lock. Will retry in 5 seconds... (%s)",
str(exc))
self.log.exception(
"Couldn't get lock. Will retry in 5 seconds...")
time.sleep(5)
try:

144
yandextank/core/resource.py
View File

@ -11,12 +11,10 @@ import traceback
class FormatDetector(object):
""" Format Detector
"""
def __init__(self):
self.log = logging.getLogger(__name__)
self.formats = {
'gzip': (0, b'\x1f\x8b'),
'tar': (257, b'ustar\x0000')
}
self.formats = {'gzip': (0, b'\x1f\x8b'), 'tar': (257, b'ustar\x0000')}
def detect_format(self, header):
for fmt, signature in self.formats.iteritems():
@ -26,7 +24,7 @@ class FormatDetector(object):
class ResourceManager(object):
""" Resource opener manager.
Use resource_filename and resource_string methods.
Use resource_filename and resource_string methods.
"""
def __init__(self):
@ -60,9 +58,12 @@ class ResourceManager(object):
try:
size = os.path.getsize(filename)
if size > 50 * 1024 * 1024:
self.log.warning('Reading large resource to memory: %s. Size: %s bytes', filename, size)
self.log.warning(
'Reading large resource to memory: %s. Size: %s bytes',
filename, size)
except Exception as exc:
self.log.debug('Unable to check resource size %s. %s', filename, exc)
self.log.debug('Unable to check resource size %s. %s', filename,
exc)
with opener(filename, 'r') as resource:
content = resource.read()
return content
@ -84,9 +85,11 @@ class ResourceManager(object):
opener = FileOpener(self.path)
return opener
class FileOpener(object):
""" File opener.
"""
def __init__(self, f_path):
self.log = logging.getLogger(__name__)
self.f_path = f_path
@ -143,15 +146,17 @@ class HttpOpener(object):
return self.open(*args, **kwargs)
def open(self, *args, **kwargs):
with closing(requests.get(self.url, stream=True, verify=False, timeout=self.timeout)) as stream:
with closing(requests.get(self.url,
stream=True,
verify=False,
timeout=self.timeout)) as stream:
stream_iterator = stream.raw.stream(100, decode_content=True)
header = stream_iterator.next()
fmt = self.fmt_detector.detect_format(header)
self.log.debug('Resource %s format detected: %s.', self.url, fmt)
if not self.force_download and fmt != 'gzip' and self.data_length > 10 ** 8:
if not self.force_download and fmt != 'gzip' and self.data_length > 10**8:
self.log.info(
"Resource data is not gzipped and larger than 100MB. Reading from stream.."
)
"Resource data is not gzipped and larger than 100MB. Reading from stream..")
return HttpStreamWrapper(self.url)
else:
downloaded_f_path = self.download_file()
@ -160,64 +165,60 @@ class HttpOpener(object):
else:
return open(downloaded_f_path, 'rb')
def download_file(self):
def download_file(self):
hasher = hashlib.md5()
hasher.update(self.hash)
tmpfile_path = "/tmp/%s" % hasher.hexdigest()
if os.path.exists(tmpfile_path):
self.log.info(
"Resource %s has already been downloaded to %s . Using it..", self.url, tmpfile_path
)
"Resource %s has already been downloaded to %s . Using it..",
self.url, tmpfile_path)
else:
self.log.info("Downloading resource %s to %s", self.url, tmpfile_path)
self.log.info("Downloading resource %s to %s", self.url,
tmpfile_path)
try:
data = requests.get(self.url, verify=False, timeout=10)
except requests.exceptions.Timeout as exc:
raise RuntimeError(
'Connection timeout reached '
'trying to download resource: %s \n'
'via HttpOpener: %s' % (self.url, exc)
)
raise RuntimeError('Connection timeout reached '
'trying to download resource: %s \n'
'via HttpOpener: %s' % (self.url, exc))
f = open(tmpfile_path, "wb")
f.write(data.content)
f.close()
self.log.info("Successfully downloaded resource %s to %s", self.url, tmpfile_path)
self.log.info("Successfully downloaded resource %s to %s",
self.url, tmpfile_path)
return tmpfile_path
def get_request_info(self):
self.log.info('Trying to get info about resource %s', self.url)
req = requests.Request('HEAD', self.url, headers={'Accept-Encoding': 'identity'})
req = requests.Request('HEAD',
self.url,
headers={'Accept-Encoding': 'identity'})
session = requests.Session()
prepared = session.prepare_request(req)
try:
self.data_info = session.send(
prepared,
verify=False,
allow_redirects=True,
timeout=self.timeout
)
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exc:
self.data_info = session.send(prepared,
verify=False,
allow_redirects=True,
timeout=self.timeout)
except (requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as exc:
self.log.warning(
'Connection error trying to get info about resource %s \n'
'Exception: %s \n'
'Retrying...' % (self.url, exc)
)
'Retrying...' % (self.url, exc))
try:
self.data_info = session.send(
prepared,
verify=False,
allow_redirects=True,
timeout=self.timeout
)
self.data_info = session.send(prepared,
verify=False,
allow_redirects=True,
timeout=self.timeout)
except Exception as exc:
self.log.debug(
'Connection error trying to get info about resource %s \n'
'Traceback: %s' % (traceback.format_exc(exc), self.url)
)
'Connection error trying to get info about resource %s \n'
'Traceback: %s' % (traceback.format_exc(exc), self.url))
raise RuntimeError(
'Connection error trying to get info about resource %s.'
'Exception: %s' % (self.url, exc)
)
'Exception: %s' % (self.url, exc))
finally:
session.close()
try:
@ -228,11 +229,9 @@ class HttpOpener(object):
"Resource storage does not support HEAD method. Ignore proto error and force download file.")
self.force_download = True
else:
raise RuntimeError(
'Invalid HTTP response '
'trying to get info about resource: %s \n'
'via HttpOpener: %s' % (self.url, exc)
)
raise RuntimeError('Invalid HTTP response '
'trying to get info about resource: %s \n'
'via HttpOpener: %s' % (self.url, exc))
@property
def get_filename(self):
@ -261,21 +260,22 @@ class HttpStreamWrapper:
self.pointer = 0
self._content_consumed = False
try:
self.stream = requests.get(self.url, stream=True, verify=False, timeout=10)
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exc:
self.stream = requests.get(self.url,
stream=True,
verify=False,
timeout=10)
except (requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as exc:
raise RuntimeError(
'Connection errors or timeout reached '
'trying to make stream while downloading resource: %s \n'
'via HttpStreamWrapper: %s' % (self.url, exc)
)
'via HttpStreamWrapper: %s' % (self.url, exc))
try:
self.stream.raise_for_status()
except requests.exceptions.HTTPError as exc:
raise RuntimeError(
'Invalid HTTP response'
'trying to open stream for resource: %s\n'
'via HttpStreamWrapper: %s' % (self.url, exc)
)
raise RuntimeError('Invalid HTTP response'
'trying to open stream for resource: %s\n'
'via HttpStreamWrapper: %s' % (self.url, exc))
def __enter__(self):
return self
@ -290,24 +290,25 @@ class HttpStreamWrapper:
def _reopen_stream(self):
self.stream.connection.close()
try:
self.stream = requests.get(self.url, stream=True, verify=False, timeout=30)
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exc:
self.stream = requests.get(self.url,
stream=True,
verify=False,
timeout=30)
except (requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as exc:
raise RuntimeError(
'Connection errors or timeout reached '
'trying to reopen stream while downloading resource: %s \n'
'via HttpStreamWrapper: %s' % (self.url, exc)
)
'via HttpStreamWrapper: %s' % (self.url, exc))
try:
self.stream.raise_for_status()
except requests.exceptions.HTTPError as exc:
raise RuntimeError(
'Invalid HTTP response'
'trying to reopen stream for resource: %s\n'
'via HttpStreamWrapper: %s' % (self.url, exc)
)
raise RuntimeError('Invalid HTTP response'
'trying to reopen stream for resource: %s\n'
'via HttpStreamWrapper: %s' % (self.url, exc))
self._content_consumed = False
def _enhance_buffer(self, bytes=10 ** 3):
def _enhance_buffer(self, bytes=10**3):
self.buffer += self.stream.iter_content(bytes).next()
def tell(self):
@ -327,7 +328,8 @@ class HttpStreamWrapper:
while '\n' not in self.buffer:
try:
self._enhance_buffer()
except (StopIteration, TypeError, requests.exceptions.StreamConsumedError):
except (StopIteration, TypeError,
requests.exceptions.StreamConsumedError):
self._content_consumed = True
break
if not self._content_consumed or self.buffer:
@ -344,7 +346,8 @@ class HttpStreamWrapper:
while len(self.buffer) < chunk_size:
try:
self._enhance_buffer()
except (StopIteration, TypeError, requests.exceptions.StreamConsumedError):
except (StopIteration, TypeError,
requests.exceptions.StreamConsumedError):
break
if len(self.buffer) > chunk_size:
chunk = self.buffer[:chunk_size]
@ -358,12 +361,13 @@ class HttpStreamWrapper:
"""
requests iter_lines() uses splitlines() thus losing '\r\n'
we need a different behavior for AmmoFileReader
and we have to use our buffer because we have probably read a bunch into it already
and we have to use our buffer because we have probably read
a bunch into it already
"""
try:
return self.next()
except StopIteration:
return ''
manager = ResourceManager()
manager = ResourceManager()

5
yandextank/core/tankcore.py
View File

@ -635,9 +635,8 @@ class ConfigManager(object):
filename = self.file
if filename:
handle = open(filename, 'wb')
self.config.write(handle)
handle.close()
with open(filename, 'wb') as handle:
self.config.write(handle)
def get_options(self, section, prefix=''):
""" Get options list with requested prefix """

2
yandextank/plugins/Console/plugin.py
View File

@ -179,7 +179,7 @@ class NoConsoleMarkup(RealConsoleMarkup):
# ======================================================
class AbstractInfoWidget:
class AbstractInfoWidget(object):
''' parent class for all right panel widgets '''
def __init__(self):

1
yandextank/plugins/Maven/__init__.py Normal file
View File

@ -0,0 +1 @@
from plugin import MavenPlugin

40
yandextank/plugins/Maven/console.py Normal file
View File

@ -0,0 +1,40 @@
from __future__ import division, absolute_import
from builtins import super
import time
import datetime
from ..Console import AbstractInfoWidget
from ..Console import screen as ConsoleScreen
class MavenInfoWidget(AbstractInfoWidget):
''' Right panel widget '''
def __init__(self, owner):
super().__init__()
self.krutilka = ConsoleScreen.krutilka()
self.owner = owner
def get_index(self):
return 0
def on_aggregated_data(self, data, stats):
pass
def render(self, screen):
text = " Maven Test %s" % next(self.krutilka)
space = screen.right_panel_width - len(text) - 1
left_spaces = space // 2
right_spaces = space // 2
dur_seconds = int(time.time()) - int(self.owner.process_start_time)
duration = str(datetime.timedelta(seconds=dur_seconds))
template = screen.markup.BG_BROWN + '~' * left_spaces + \
text + ' ' + '~' * right_spaces + screen.markup.RESET + "\n"
template += "Command Line: %s\n"
template += " Duration: %s"
data = (self.owner.maven_cmd, duration)
return template % data

106
yandextank/plugins/Maven/plugin.py Normal file
View File

@ -0,0 +1,106 @@
"""
Run maven test as load test
"""
from __future__ import division, absolute_import
from builtins import super
import subprocess
import time
import re
from ..Aggregator import AggregatorPlugin
from ..Console import ConsolePlugin
from ...core import AbstractPlugin
from ...core.resource import manager as resource_manager
from .console import MavenInfoWidget
from .reader import MavenReader, MavenStatsReader
import logging
logger = logging.getLogger(__name__)
class MavenPlugin(AbstractPlugin):
SECTION = "maven"
def __init__(self, core):
super().__init__(core)
self.maven_cmd = "mvn"
self.process = None
self.process_stderr = None
self.process_start_time = None
@staticmethod
def get_key():
return __file__
def get_available_options(self):
opts = ["pom", "testcase", "mvn_args"]
return opts
def configure(self):
# plugin part
self.pom = resource_manager.resource_filename(self.get_option(
"pom", "pom.xml"))
self.testcase = self.get_option("testcase", "")
self.maven_args = self.get_option("mvn_args", '').split()
def prepare_test(self):
aggregator = None
try:
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
except KeyError as ex:
logger.warning("No aggregator found: %s", ex)
if aggregator:
aggregator.reader = MavenReader()
aggregator.stats_reader = MavenStatsReader()
try:
console = self.core.get_plugin_of_type(ConsolePlugin)
except KeyError as ex:
logger.debug("Console not found: %s", ex)
console = None
if console:
widget = MavenInfoWidget(self)
console.add_info_widget(widget)
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
aggregator.add_result_listener(widget)
def start_test(self):
args = [self.maven_cmd, "test", "-Dtest=%s" % self.testcase
] + self.maven_args + ["-f", self.pom]
logger.info("Starting: %s", args)
self.process_start_time = time.time()
process_stderr_file = self.core.mkstemp(".log", "maven_")
self.core.add_artifact_file(process_stderr_file)
self.process_stderr = open(process_stderr_file, 'w')
self.process = subprocess.Popen(args,
stderr=self.process_stderr,
stdout=self.process_stderr,
close_fds=True)
def is_test_finished(self):
retcode = self.process.poll()
if retcode is not None:
logger.info("Subprocess done its work with exit code: %s", retcode)
return abs(retcode)
else:
return -1
def end_test(self, retcode):
if self.process and self.process.poll() is None:
logger.warn("Terminating worker process with PID %s",
self.process.pid)
self.process.terminate()
if self.process_stderr:
self.process_stderr.close()
else:
logger.debug("Seems subprocess finished OK")
return retcode
def get_info(self):
return None

14
yandextank/plugins/Maven/reader.py Normal file
View File

@ -0,0 +1,14 @@
class MavenReader(object):
def close(self):
pass
def __iter__(self):
yield None
class MavenStatsReader(object):
def close(self):
pass
def __iter__(self):
yield None

1
yandextank/plugins/Pandora/plugin.py
View File

@ -1,4 +1,3 @@
''' Contains Universal Plugin for phout-compatible shooter '''
from yandextank.plugins.Aggregator import AggregatorPlugin
from yandextank.plugins.Phantom import PhantomReader
from yandextank.core import AbstractPlugin