valitydev/yandex-tank
14
0
Fork 0
mirror of https://github.com/valitydev/yandex-tank.git synced 2024-11-06 02:15:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

yapf autoformat

Browse Source
This commit is contained in:
Alexey Lavrenuke 2016-12-30 16:46:29 +03:00
parent 443bd5291d
commit 40706c55b8
75 changed files with 1701 additions and 1560 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.style.yapf Normal file
View File

@ -0,0 +1,7 @@
[style]
based_on_style = pep8
COALESCE_BRACKETS = True
COLUMN_LIMIT = 80
DEDENT_CLOSING_BRACKETS = False
SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET = True
SPLIT_BEFORE_FIRST_ARGUMENT = True

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='yandextank',
version='1.8.29-9',
version='1.8.29-10',
description='a performance measurement tool',
longer_description='''
Yandex.Tank is a performance measurement and load testing automatization tool.

21
yandextank/api/apiworker.py
View File

@ -27,7 +27,8 @@ class ApiWorker:
file_handler = logging.FileHandler(self.log_filename)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(logging.Formatter(
file_handler.setFormatter(
logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s %(message)s"))
logger.addHandler(file_handler)
console_handler = logging.StreamHandler(sys.stdout)
@ -65,8 +66,8 @@ class ApiWorker:
""" Make preparations before running Tank """
self.options = options
if self.options.get('lock_dir', None):
self.core.set_option(self.core.SECTION, "lock_dir",
self.options['lock_dir'])
self.core.set_option(
self.core.SECTION, "lock_dir", self.options['lock_dir'])
while True:
try:
@ -109,12 +110,13 @@ class ApiWorker:
except KeyboardInterrupt as ex:
self.log.info(
"Do not press Ctrl+C again, the test will be broken otherwise")
self.log.debug("Caught KeyboardInterrupt: %s",
traceback.format_exc(ex))
self.log.debug(
"Caught KeyboardInterrupt: %s", traceback.format_exc(ex))
try:
retcode = self.__graceful_shutdown()
except KeyboardInterrupt as ex:
self.log.debug("Caught KeyboardInterrupt again: %s",
self.log.debug(
"Caught KeyboardInterrupt again: %s",
traceback.format_exc(ex))
self.log.info(
"User insists on exiting, aborting graceful shutdown...")
@ -138,11 +140,12 @@ class ApiWorker:
for filename in conf_files:
if fnmatch.fnmatch(filename, '*.ini'):
configs += [
os.path.realpath(self.baseconfigs_location + os.sep +
filename)
os.path.realpath(
self.baseconfigs_location + os.sep + filename)
]
except OSError:
self.log.warn(self.baseconfigs_location +
self.log.warn(
self.baseconfigs_location +
' is not accessible to get configs list')
configs += [os.path.expanduser('~/.yandex-tank')]

21
yandextank/common/interfaces.py
View File

@ -76,8 +76,8 @@ class AbstractPlugin(object):
def publish(self, key, value):
"""publish value to status"""
self.log.debug("Publishing status: %s/%s: %s", self.__class__.__name__,
key, value)
self.log.debug(
"Publishing status: %s/%s: %s", self.__class__.__name__, key, value)
self.core.publish(self.__class__.__name__, key, value)
def close(self):
@ -175,25 +175,20 @@ class AbstractCriterion(object):
class GeneratorPlugin(object):
DEFAULT_INFO = {'address': '',
DEFAULT_INFO = {
'address': '',
'port': 80,
'instances': 1,
'ammo_file': '',
'rps_schedule': [],
'duration': 0,
'loop_count': 0}
'loop_count': 0
}
class Info(object):
def __init__(
self,
address,
port,
instances,
ammo_file,
rps_schedule,
duration,
loop_count):
self, address, port, instances, ammo_file, rps_schedule,
duration, loop_count):
self.address = address
self.port = port
self.instances = instances

77
yandextank/common/resource.py
View File

@ -62,8 +62,7 @@ class ResourceManager(object):
'Reading large resource to memory: %s. Size: %s bytes',
filename, size)
except Exception as exc:
logger.debug('Unable to check resource size %s. %s', filename,
exc)
logger.debug('Unable to check resource size %s. %s', filename, exc)
with opener(filename, 'r') as resource:
content = resource.read()
return content
@ -146,19 +145,16 @@ class HttpOpener(object):
def open(self, *args, **kwargs):
with closing(
requests.get(
self.url,
stream=True,
verify=False,
timeout=self.timeout
)
) as stream:
self.url, stream=True, verify=False,
timeout=self.timeout)) as stream:
stream_iterator = stream.raw.stream(100, decode_content=True)
header = stream_iterator.next()
fmt = self.fmt_detector.detect_format(header)
logger.debug('Resource %s format detected: %s.', self.url, fmt)
if not self.force_download and fmt != 'gzip' and self.data_length > 10**8:
logger.info(
"Resource data is not gzipped and larger than 100MB. Reading from stream..")
"Resource data is not gzipped and larger than 100MB. Reading from stream.."
)
return HttpStreamWrapper(self.url)
else:
downloaded_f_path = self.download_file()
@ -176,41 +172,44 @@ class HttpOpener(object):
"Resource %s has already been downloaded to %s . Using it..",
self.url, tmpfile_path)
else:
logger.info("Downloading resource %s to %s", self.url,
tmpfile_path)
logger.info("Downloading resource %s to %s", self.url, tmpfile_path)
try:
data = requests.get(self.url, verify=False, timeout=10)
except requests.exceptions.Timeout as exc:
raise RuntimeError('Connection timeout reached '
raise RuntimeError(
'Connection timeout reached '
'trying to download resource: %s \n'
'via HttpOpener: %s' % (self.url, exc))
f = open(tmpfile_path, "wb")
f.write(data.content)
f.close()
logger.info("Successfully downloaded resource %s to %s",
self.url, tmpfile_path)
logger.info(
"Successfully downloaded resource %s to %s", self.url,
tmpfile_path)
return tmpfile_path
def get_request_info(self):
logger.info('Trying to get info about resource %s', self.url)
req = requests.Request('HEAD',
self.url,
headers={'Accept-Encoding': 'identity'})
req = requests.Request(
'HEAD', self.url, headers={'Accept-Encoding': 'identity'})
session = requests.Session()
prepared = session.prepare_request(req)
try:
self.data_info = session.send(prepared,
self.data_info = session.send(
prepared,
verify=False,
allow_redirects=True,
timeout=self.timeout)
except (requests.exceptions.Timeout,
except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as exc:
logger.warning(
'Connection error trying to get info about resource %s \n'
'Exception: %s \n'
'Retrying...' % (self.url, exc))
try:
self.data_info = session.send(prepared,
self.data_info = session.send(
prepared,
verify=False,
allow_redirects=True,
timeout=self.timeout)
@ -228,10 +227,12 @@ class HttpOpener(object):
except requests.exceptions.HTTPError as exc:
if exc.response.status_code == 405:
logger.info(
"Resource storage does not support HEAD method. Ignore proto error and force download file.")
"Resource storage does not support HEAD method. Ignore proto error and force download file."
)
self.force_download = True
else:
raise RuntimeError('Invalid HTTP response '
raise RuntimeError(
'Invalid HTTP response '
'trying to get info about resource: %s \n'
'via HttpOpener: %s' % (self.url, exc))
@ -262,14 +263,13 @@ class HttpStreamWrapper:
self.pointer = 0
self.stream_iterator = None
self._content_consumed = False
self.chunk_size = 10 ** 3
self.chunk_size = 10**3
try:
self.stream = requests.get(self.url,
stream=True,
verify=False,
timeout=10)
self.stream = requests.get(
self.url, stream=True, verify=False, timeout=10)
self.stream_iterator = self.stream.iter_content(self.chunk_size)
except (requests.exceptions.Timeout,
except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as exc:
raise RuntimeError(
'Connection errors or timeout reached '
@ -278,7 +278,8 @@ class HttpStreamWrapper:
try:
self.stream.raise_for_status()
except requests.exceptions.HTTPError as exc:
raise RuntimeError('Invalid HTTP response'
raise RuntimeError(
'Invalid HTTP response'
'trying to open stream for resource: %s\n'
'via HttpStreamWrapper: %s' % (self.url, exc))
@ -295,12 +296,11 @@ class HttpStreamWrapper:
def _reopen_stream(self):
self.stream.connection.close()
try:
self.stream = requests.get(self.url,
stream=True,
verify=False,
timeout=30)
self.stream = requests.get(
self.url, stream=True, verify=False, timeout=30)
self.stream_iterator = self.stream.iter_content(self.chunk_size)
except (requests.exceptions.Timeout,
except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as exc:
raise RuntimeError(
'Connection errors or timeout reached '
@ -309,7 +309,8 @@ class HttpStreamWrapper:
try:
self.stream.raise_for_status()
except requests.exceptions.HTTPError as exc:
raise RuntimeError('Invalid HTTP response'
raise RuntimeError(
'Invalid HTTP response'
'trying to reopen stream for resource: %s\n'
'via HttpStreamWrapper: %s' % (self.url, exc))
self._content_consumed = False
@ -334,7 +335,8 @@ class HttpStreamWrapper:
while '\n' not in self.buffer:
try:
self._enhance_buffer()
except (StopIteration, TypeError,
except (
StopIteration, TypeError,
requests.exceptions.StreamConsumedError):
self._content_consumed = True
break
@ -352,7 +354,8 @@ class HttpStreamWrapper:
while len(self.buffer) < chunk_size:
try:
self._enhance_buffer()
except (StopIteration, TypeError,
except (
StopIteration, TypeError,
requests.exceptions.StreamConsumedError):
break
if len(self.buffer) > chunk_size:

2
yandextank/common/tests/test_util.py
View File

@ -3,7 +3,6 @@ from yandextank.common.util import Drain, Chopper
class TestDrain(object):
def test_run(self):
"""
Test drain's run function (in a same thread)
@ -38,7 +37,6 @@ class TestDrain(object):
class TestChopper(object):
def test_output(self):
source = (range(i) for i in range(5))
expected = [0, 0, 1, 0, 1, 2, 0, 1, 2, 3]

82
yandextank/common/util.py
View File

@ -43,7 +43,6 @@ class Drain(th.Thread):
class SecuredShell(object):
def __init__(self, host, port, username, timeout):
self.host = host
self.port = port
@ -52,20 +51,22 @@ class SecuredShell(object):
def connect(self):
logger.debug(
"Opening SSH connection to {host}:{port}".format(host=self.host,
port=self.port))
"Opening SSH connection to {host}:{port}".format(
host=self.host, port=self.port))
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
try:
client.connect(self.host,
client.connect(
self.host,
port=self.port,
username=self.username,
timeout=self.timeout, )
except ValueError as e:
logger.error(e)
logger.warning("""
logger.warning(
"""
Patching Crypto.Cipher.AES.new and making another attempt.
See here for the details:
@ -82,7 +83,8 @@ http://uucode.com/blog/2015/02/20/workaround-for-ctr-mode-needs-counter-paramete
return orig_new(key, *ls)
Crypto.Cipher.AES.new = fixed_AES_new
client.connect(self.host,
client.connect(
self.host,
port=self.port,
username=self.username,
timeout=self.timeout, )
@ -107,19 +109,17 @@ http://uucode.com/blog/2015/02/20/workaround-for-ctr-mode-needs-counter-paramete
return self.execute("mkdir -p %s" % path)
def send_file(self, local_path, remote_path):
logger.info("Sending [{local}] to {host}:[{remote}]".format(
local=local_path,
host=self.host,
remote=remote_path))
logger.info(
"Sending [{local}] to {host}:[{remote}]".format(
local=local_path, host=self.host, remote=remote_path))
with self.connect() as client, client.open_sftp() as sftp:
result = sftp.put(local_path, remote_path)
return result
def get_file(self, remote_path, local_path):
logger.info("Receiving from {host}:[{remote}] to [{local}]".format(
local=local_path,
host=self.host,
remote=remote_path))
logger.info(
"Receiving from {host}:[{remote}] to [{local}]".format(
local=local_path, host=self.host, remote=remote_path))
with self.connect() as client, client.open_sftp() as sftp:
result = sftp.get(remote_path, local_path)
return result
@ -129,39 +129,27 @@ http://uucode.com/blog/2015/02/20/workaround-for-ctr-mode-needs-counter-paramete
def check_ssh_connection():
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
logging.basicConfig(
level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s')
logging.getLogger("paramiko.transport").setLevel(logging.DEBUG)
parser = argparse.ArgumentParser(
description='Test SSH connection for monitoring.')
parser.add_argument(
'-e', '--endpoint',
default='example.org',
help='which host to try')
'-e', '--endpoint', default='example.org', help='which host to try')
parser.add_argument(
'-u', '--username',
default=os.getlogin(),
help='SSH username')
'-u', '--username', default=os.getlogin(), help='SSH username')
parser.add_argument(
'-p', '--port',
default=22,
type=int,
help='SSH port')
parser.add_argument('-p', '--port', default=22, type=int, help='SSH port')
args = parser.parse_args()
logging.info(
"Checking SSH to %s@%s:%d",
args.username,
args.endpoint,
args.port)
"Checking SSH to %s@%s:%d", args.username, args.endpoint, args.port)
ssh = SecuredShell(args.endpoint, args.port, args.username, 10)
print(ssh.execute("ls -l"))
class AsyncSession(object):
def __init__(self, ssh, cmd):
self.client = ssh.connect()
self.session = self.client.get_transport().open_session()
@ -427,8 +415,8 @@ def expand_time(str_time, default_unit='s', multiplier=1):
result += value * 60 * 60 * 24 * 7
continue
else:
raise ValueError("String contains unsupported unit %s: %s" %
(unit, str_time))
raise ValueError(
"String contains unsupported unit %s: %s" % (unit, str_time))
return int(result * multiplier)
@ -460,7 +448,8 @@ def execute(cmd, shell=False, poll_period=1.0, catch_out=False):
cmd = shlex.split(cmd)
if catch_out:
process = subprocess.Popen(cmd,
process = subprocess.Popen(
cmd,
shell=shell,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
@ -503,14 +492,12 @@ def pairs(lst):
def update_status(status, multi_key, value):
if len(multi_key) > 1:
update_status(
status.setdefault(multi_key[0], {}), multi_key[1:], value)
update_status(status.setdefault(multi_key[0], {}), multi_key[1:], value)
else:
status[multi_key[0]] = value
class AddressWizard:
def __init__(self):
self.lookup_fn = socket.getaddrinfo
self.socket_class = socket.socket
@ -554,8 +541,9 @@ class AddressWizard:
resolved = self.lookup_fn(address_str, port)
logger.debug("Lookup result: %s", resolved)
except Exception as exc:
logger.debug("Exception trying to resolve hostname %s : %s",
address_str, traceback.format_exc(exc))
logger.debug(
"Exception trying to resolve hostname %s : %s", address_str,
traceback.format_exc(exc))
msg = "Failed to resolve hostname: %s. Error: %s"
raise RuntimeError(msg % (address_str, exc))
@ -565,7 +553,8 @@ class AddressWizard:
if explicit_port:
logger.warn(
"Using phantom.port option is deprecated. Use phantom.address=[address]:port instead")
"Using phantom.port option is deprecated. Use phantom.address=[address]:port instead"
)
port = int(explicit_port)
elif not port:
port = 80
@ -574,8 +563,9 @@ class AddressWizard:
try:
self.__test(family, (parsed_ip, port))
except RuntimeError as exc:
logger.warn("Failed TCP connection test using [%s]:%s",
parsed_ip, port)
logger.warn(
"Failed TCP connection test using [%s]:%s", parsed_ip,
port)
continue
return is_v6, parsed_ip, int(port), address_str
@ -589,8 +579,9 @@ class AddressWizard:
test_sock.settimeout(5)
test_sock.connect(sa)
except Exception as exc:
logger.debug("Exception on connect attempt [%s]:%s : %s", sa[0],
sa[1], traceback.format_exc(exc))
logger.debug(
"Exception on connect attempt [%s]:%s : %s", sa[0], sa[1],
traceback.format_exc(exc))
msg = "TCP Connection test failed for [%s]:%s, use phantom.connection_test=0 to disable it"
raise RuntimeError(msg % (sa[0], sa[1]))
finally:
@ -598,7 +589,6 @@ class AddressWizard:
class Chopper(object):
def __init__(self, source):
self.source = source

30
yandextank/core/cli.py
View File

@ -12,8 +12,10 @@ def main():
'-c',
'--config',
action='append',
help="Path to INI file containing run options, multiple options accepted")
parser.add_option('-f',
help="Path to INI file containing run options, multiple options accepted"
)
parser.add_option(
'-f',
'--fail-lock',
action='store_true',
dest='lock_fail',
@ -23,19 +25,23 @@ def main():
'--ignore-lock',
action='store_true',
dest='ignore_lock',
help="Ignore lock files from concurrent instances, has precedence before --lock-fail")
parser.add_option('-k',
help="Ignore lock files from concurrent instances, has precedence before --lock-fail"
)
parser.add_option(
'-k',
'--lock-dir',
action='store',
dest='lock_dir',
type="string",
help="Directory for lock file")
parser.add_option('-l',
parser.add_option(
'-l',
'--log',
action='store',
default="tank.log",
help="Tank log file location")
parser.add_option('-m',
parser.add_option(
'-m',
'--manual-start',
action='store_true',
dest='manual_start',
@ -50,8 +56,10 @@ def main():
'-o',
'--option',
action='append',
help="Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'")
parser.add_option('-q',
help="Set config option, multiple options accepted, example: -o 'shellexec.start=pwd'"
)
parser.add_option(
'-q',
'--quiet',
action='store_true',
help="Less console output, only errors and warnings")
@ -60,8 +68,10 @@ def main():
'--scheduled-start',
action='store',
dest='scheduled_start',
help="Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional")
parser.add_option('-v',
help="Start test at specified time, format 'YYYY-MM-DD hh:mm:ss', date part is optional"
)
parser.add_option(
'-v',
'--verbose',
action='store_true',
help="More console output, +debug messages")

63
yandextank/core/consoleworker.py
View File

@ -37,9 +37,11 @@ class RealConsoleMarkup(object):
def clean_markup(self, orig_str):
''' clean markup from string '''
for val in [self.YELLOW, self.RED, self.RESET, self.CYAN,
self.BG_MAGENTA, self.WHITE, self.BG_GREEN, self.GREEN,
self.BG_BROWN, self.RED_DARK, self.MAGENTA, self.BG_CYAN]:
for val in [
self.YELLOW, self.RED, self.RESET, self.CYAN, self.BG_MAGENTA,
self.WHITE, self.BG_GREEN, self.GREEN, self.BG_BROWN,
self.RED_DARK, self.MAGENTA, self.BG_CYAN
]:
orig_str = orig_str.replace(val, '')
return orig_str
@ -101,8 +103,10 @@ class ConsoleTank:
if self.log_filename:
file_handler = logging.FileHandler(self.log_filename)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s"))
file_handler.setFormatter(
logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s"
))
logger.addHandler(file_handler)
# create console handler with a higher log level
@ -110,7 +114,8 @@ class ConsoleTank:
stderr_hdl = logging.StreamHandler(sys.stderr)
fmt_verbose = logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s")
"%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s"
)
fmt_regular = logging.Formatter(
"%(asctime)s [%(levelname)s] %(message)s", "%H:%M:%S")
@ -155,11 +160,12 @@ class ConsoleTank:
for filename in conf_files:
if fnmatch.fnmatch(filename, '*.ini'):
configs += [
os.path.realpath(self.baseconfigs_location + os.sep +
filename)
os.path.realpath(
self.baseconfigs_location + os.sep + filename)
]
except OSError:
self.log.warn(self.baseconfigs_location +
self.log.warn(
self.baseconfigs_location +
' is not accessible to get configs list')
configs += [os.path.expanduser('~/.yandex-tank')]
@ -172,8 +178,8 @@ class ConsoleTank:
"Lock files ignored. This is highly unrecommended practice!")
if self.options.lock_dir:
self.core.set_option(self.core.SECTION, "lock_dir",
self.options.lock_dir)
self.core.set_option(
self.core.SECTION, "lock_dir", self.options.lock_dir)
while True:
try:
@ -202,7 +208,8 @@ class ConsoleTank:
elif os.path.exists(os.path.realpath('load.conf')):
# just for old 'lunapark' compatibility
self.log.warn(
"Using 'load.conf' is unrecommended, please use 'load.ini' instead")
"Using 'load.conf' is unrecommended, please use 'load.ini' instead"
)
conf_file = os.path.realpath('load.conf')
configs += [conf_file]
self.core.add_artifact_file(conf_file, True)
@ -260,11 +267,12 @@ class ConsoleTank:
self.core.plugins_configure()
self.core.plugins_prepare_test()
if self.scheduled_start:
self.log.info("Waiting scheduled time: %s...",
self.scheduled_start)
self.log.info(
"Waiting scheduled time: %s...", self.scheduled_start)
while datetime.datetime.now() < self.scheduled_start:
self.log.debug("Not yet: %s < %s", datetime.datetime.now(),
self.scheduled_start)
self.log.debug(
"Not yet: %s < %s",
datetime.datetime.now(), self.scheduled_start)
time.sleep(1)
self.log.info("Time has come: %s", datetime.datetime.now())
@ -283,12 +291,13 @@ class ConsoleTank:
sys.stdout.write(RealConsoleMarkup.RESET)
sys.stdout.write(RealConsoleMarkup.TOTAL_RESET)
self.signal_count += 1
self.log.debug("Caught KeyboardInterrupt: %s",
traceback.format_exc(ex))
self.log.debug(
"Caught KeyboardInterrupt: %s", traceback.format_exc(ex))
try:
retcode = self.__graceful_shutdown()
except KeyboardInterrupt as ex:
self.log.debug("Caught KeyboardInterrupt again: %s",
self.log.debug(
"Caught KeyboardInterrupt again: %s",
traceback.format_exc(ex))
self.log.info(
"User insists on exiting, aborting graceful shutdown...")
@ -310,7 +319,6 @@ class ConsoleTank:
class DevNullOpts:
def __init__(self):
pass
@ -318,18 +326,20 @@ class DevNullOpts:
class CompletionHelperOptionParser(OptionParser):
def __init__(self):
OptionParser.__init__(self, add_help_option=False)
self.add_option('--bash-switches-list',
self.add_option(
'--bash-switches-list',
action='store_true',
dest="list_switches",
help="Options list")
self.add_option('--bash-options-prev',
self.add_option(
'--bash-options-prev',
action='store',
dest="list_options_prev",
help="Options list")
self.add_option('--bash-options-cur',
self.add_option(
'--bash-options-cur',
action='store',
dest="list_options_cur",
help="Options list")
@ -362,8 +372,9 @@ class CompletionHelperOptionParser(OptionParser):
plugin_keys = cmdtank.core.config.get_options(
cmdtank.core.SECTION, cmdtank.core.PLUGIN_PREFIX)
for (plugin_name, plugin_path) in plugin_keys:
opts.append(cmdtank.core.SECTION + '.' +
cmdtank.core.PLUGIN_PREFIX + plugin_name + '=')
opts.append(
cmdtank.core.SECTION + '.' + cmdtank.core.PLUGIN_PREFIX +
plugin_name + '=')
for plugin in cmdtank.core.plugins:
for option in plugin.get_available_options():

8
yandextank/core/expvar.py
View File

@ -52,11 +52,10 @@ class Var(object):
class Int(Var):
def __init__(self, value=0):
if not isinstance(value, int):
raise ValueError("Value should be an integer, but it is '%s'" %
type(value))
raise ValueError(
"Value should be an integer, but it is '%s'" % type(value))
super(Int, self).__init__(value)
def inc(self, delta=1):
@ -76,7 +75,8 @@ class Metric(object):
if timestamp is None:
timestamp = int(time.time())
elif not isinstance(timestamp, int):
raise ValueError("Timestamp should be an integer, but it is '%s'" %
raise ValueError(
"Timestamp should be an integer, but it is '%s'" %
type(timestamp))
self.metric.put((timestamp, value))

154
yandextank/core/tankcore.py
View File

@ -30,12 +30,10 @@ if sys.version_info[0] < 3:
else:
import configparser as ConfigParser
logger = logging.getLogger(__name__)
class Job(object):
def __init__(
self,
name,
@ -114,8 +112,10 @@ class TankCore(object):
return self.uuid
def get_available_options(self):
return ["artifacts_base_dir", "artifacts_dir", "flush_config_to",
"taskset_path", "affinity"]
return [
"artifacts_base_dir", "artifacts_dir", "flush_config_to",
"taskset_path", "affinity"
]
def load_configs(self, configs):
""" Tells core to load configs set into options storage """
@ -129,8 +129,8 @@ class TankCore(object):
self.config.flush()
self.add_artifact_file(self.config.file)
self.set_option(self.SECTION, self.PID_OPTION, str(os.getpid()))
self.flush_config_to = self.get_option(self.SECTION, "flush_config_to",
"")
self.flush_config_to = self.get_option(
self.SECTION, "flush_config_to", "")
if self.flush_config_to:
self.config.flush(self.flush_config_to)
@ -148,29 +148,26 @@ class TankCore(object):
self.artifacts_dir_name = self.get_option(
self.SECTION, "artifacts_dir", "")
self.taskset_path = self.get_option(self.SECTION, 'taskset_path',
'taskset')
self.taskset_path = self.get_option(
self.SECTION, 'taskset_path', 'taskset')
self.taskset_affinity = self.get_option(self.SECTION, 'affinity', '')
options = self.config.get_options(self.SECTION, self.PLUGIN_PREFIX)
for (plugin_name, plugin_path) in options:
if not plugin_path:
logger.debug("Seems the plugin '%s' was disabled",
plugin_name)
logger.debug("Seems the plugin '%s' was disabled", plugin_name)
continue
logger.debug("Loading plugin %s from %s", plugin_name,
plugin_path)
logger.debug("Loading plugin %s from %s", plugin_name, plugin_path)
# FIXME cleanup an old deprecated plugin path format
if '/' in plugin_path:
logger.warning("Deprecated plugin path format: %s\n"
logger.warning(
"Deprecated plugin path format: %s\n"
"Should be in pythonic format. Example:\n"
" plugin_jmeter=yandextank.plugins.JMeter",
plugin_path)
" plugin_jmeter=yandextank.plugins.JMeter", plugin_path)
if plugin_path.startswith("Tank/Plugins/"):
plugin_path = "yandextank.plugins." + \
plugin_path.split('/')[-1].split('.')[0]
logger.warning("Converted plugin path to %s",
plugin_path)
logger.warning("Converted plugin path to %s", plugin_path)
else:
raise ValueError(
"Couldn't convert plugin path to new format:\n %s" %
@ -183,7 +180,8 @@ class TankCore(object):
"Deprecated plugin path format: %s\n"
"Tank plugins are now orginized using"
" namespace packages. Example:\n"
" plugin_jmeter=yandextank.plugins.JMeter", plugin_path)
" plugin_jmeter=yandextank.plugins.JMeter",
plugin_path)
plugin_path = plugin_path.replace(
"yatank_internal_", "yandextank.plugins.")
if plugin_path.startswith("yatank_"):
@ -191,7 +189,8 @@ class TankCore(object):
"Deprecated plugin path format: %s\n"
"Tank plugins are now orginized using"
" namespace packages. Example:\n"
" plugin_jmeter=yandextank.plugins.JMeter", plugin_path)
" plugin_jmeter=yandextank.plugins.JMeter",
plugin_path)
plugin_path = plugin_path.replace(
"yatank_", "yandextank.plugins.")
@ -201,9 +200,10 @@ class TankCore(object):
instance = getattr(plugin, 'Plugin')(self)
except:
logger.warning(
"Deprecated plugin classname: %s. Should be 'Plugin'", plugin)
instance = getattr(plugin, plugin_path.split(
'.')[-1] + 'Plugin')(self)
"Deprecated plugin classname: %s. Should be 'Plugin'",
plugin)
instance = getattr(
plugin, plugin_path.split('.')[-1] + 'Plugin')(self)
self.plugins.append(instance)
@ -247,26 +247,16 @@ class TankCore(object):
gen = None
self.job = Job(
name=self.get_option(
self.SECTION_META,
"job_name",
name=self.get_option(self.SECTION_META, "job_name",
'none').decode('utf8'),
description=self.get_option(
self.SECTION_META,
"job_dsc",
description=self.get_option(self.SECTION_META, "job_dsc",
'').decode('utf8'),
task=self.get_option(
self.SECTION_META,
'task',
task=self.get_option(self.SECTION_META, 'task',
'dir').decode('utf8'),
version=self.get_option(
self.SECTION_META,
'ver',
version=self.get_option(self.SECTION_META, 'ver',
'').decode('utf8'),
config_copy=self.get_option(
self.SECTION_META,
'copy_config_to',
'config_copy'),
self.SECTION_META, 'copy_config_to', 'config_copy'),
monitoring_plugin=mon,
aggregator_plugin=aggregator,
generator_plugin=gen,
@ -319,9 +309,7 @@ class TankCore(object):
end_time = time.time()
diff = end_time - begin_time
logger.debug("Polling took %s", diff)
logger.debug("Tank status:\n%s",
json.dumps(self.status,
indent=2))
logger.debug("Tank status:\n%s", json.dumps(self.status, indent=2))
# screen refresh every 0.5 s
if diff < 0.5:
time.sleep(0.5 - diff)
@ -340,8 +328,8 @@ class TankCore(object):
logger.debug("RC after: %s", retcode)
except Exception as ex:
logger.error("Failed finishing plugin %s: %s", plugin, ex)
logger.debug("Failed finishing plugin: %s",
traceback.format_exc(ex))
logger.debug(
"Failed finishing plugin: %s", traceback.format_exc(ex))
if not retcode:
retcode = 1
@ -363,9 +351,9 @@ class TankCore(object):
retcode = plugin.post_process(retcode)
logger.debug("RC after: %s", retcode)
except Exception as ex:
logger.error("Failed post-processing plugin %s: %s", plugin,
ex)
logger.debug("Failed post-processing plugin: %s",
logger.error("Failed post-processing plugin %s: %s", plugin, ex)
logger.debug(
"Failed post-processing plugin: %s",
traceback.format_exc(ex))
if not retcode:
retcode = 1
@ -380,15 +368,14 @@ class TankCore(object):
def taskset(self, pid, path, affinity):
if affinity:
args = "%s -pc %s %s" % (path, affinity, pid)
retcode, stdout, stderr = execute(args,
shell=True,
poll_period=0.1,
catch_out=True)
retcode, stdout, stderr = execute(
args, shell=True, poll_period=0.1, catch_out=True)
logger.debug('taskset stdout: %s', stdout)
if retcode != 0:
raise KeyError(stderr)
else:
logger.info("Enabled taskset for pid %s with affinity %s",
logger.info(
"Enabled taskset for pid %s with affinity %s",
str(pid), affinity)
def __collect_artifacts(self):
@ -427,8 +414,8 @@ class TankCore(object):
logger.debug("Expanding shell option %s", value)
retcode, stdout, stderr = execute(value[1:-1], True, 0.1, True)
if retcode or stderr:
raise ValueError("Error expanding option %s, RC: %s" %
(value, retcode))
raise ValueError(
"Error expanding option %s, RC: %s" % (value, retcode))
value = stdout.strip()
return value
@ -447,9 +434,10 @@ class TankCore(object):
Retrieve a plugin of desired class, KeyError raised otherwise
"""
logger.debug("Searching for plugin: %s", plugin_class)
matches = [plugin
for plugin in self.plugins
if isinstance(plugin, plugin_class)]
matches = [
plugin for plugin in self.plugins
if isinstance(plugin, plugin_class)
]
if len(matches) > 0:
if len(matches) > 1:
logger.debug(
@ -457,8 +445,7 @@ class TankCore(object):
plugin_class)
return matches[-1]
else:
raise KeyError("Requested plugin type not found: %s" %
plugin_class)
raise KeyError("Requested plugin type not found: %s" % plugin_class)
def __collect_file(self, filename, keep_original=False):
"""
@ -487,28 +474,30 @@ class TankCore(object):
Add file to be stored as result artifact on post-process phase
"""
if filename:
logger.debug("Adding artifact file to collect (keep=%s): %s",
keep_original, filename)
logger.debug(
"Adding artifact file to collect (keep=%s): %s", keep_original,
filename)
self.artifact_files[filename] = keep_original
def apply_shorthand_options(self, options, default_section='DEFAULT'):
for option_str in options:
try:
section = option_str[:option_str.index('.')]
option = option_str[
option_str.index('.') + 1:option_str.index('=')]
option = option_str[option_str.index('.') + 1:option_str.index(
'=')]
except ValueError:
section = default_section
option = option_str[:option_str.index('=')]
value = option_str[option_str.index('=') + 1:]
logger.debug("Override option: %s => [%s] %s=%s", option_str,
section, option, value)
logger.debug(
"Override option: %s => [%s] %s=%s", option_str, section,
option, value)
self.set_option(section, option, value)
def get_lock_dir(self):
if not self.lock_dir:
self.lock_dir = self.get_option(self.SECTION, "lock_dir",
self.LOCK_DIR)
self.lock_dir = self.get_option(
self.SECTION, "lock_dir", self.LOCK_DIR)
return os.path.expanduser(self.lock_dir)
@ -516,8 +505,8 @@ class TankCore(object):
if not force and self.__there_is_locks():
raise RuntimeError("There is lock files")
fh, self.lock_file = tempfile.mkstemp('.lock', 'lunapark_',
self.get_lock_dir())
fh, self.lock_file = tempfile.mkstemp(
'.lock', 'lunapark_', self.get_lock_dir())
os.close(fh)
os.chmod(self.lock_file, 0o644)
self.config.file = self.lock_file
@ -542,18 +531,19 @@ class TankCore(object):
info.read(full_name)
pid = info.get(TankCore.SECTION, self.PID_OPTION)
if not pid_exists(int(pid)):
logger.debug("Lock PID %s not exists, ignoring and "
logger.debug(
"Lock PID %s not exists, ignoring and "
"trying to remove", pid)
try:
os.remove(full_name)
except Exception as exc:
logger.debug("Failed to delete lock %s: %s",
full_name, exc)
logger.debug(
"Failed to delete lock %s: %s", full_name, exc)
else:
retcode = True
except Exception as exc:
logger.warn("Failed to load info from lock %s: %s",
full_name, exc)
logger.warn(
"Failed to load info from lock %s: %s", full_name, exc)
retcode = True
return retcode
@ -584,14 +574,15 @@ class TankCore(object):
plugin.close()
except Exception as ex:
logger.error("Failed closing plugin %s: %s", plugin, ex)
logger.debug("Failed closing plugin: %s",
traceback.format_exc(ex))
logger.debug(
"Failed closing plugin: %s", traceback.format_exc(ex))
@property
def artifacts_dir(self):
if not self._artifacts_dir:
if not self.artifacts_dir_name:
date_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S.")
date_str = datetime.datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S.")
self.artifacts_dir_name = tempfile.mkdtemp(
"", date_str, self.artifacts_base_dir)
elif not os.path.isdir(self.artifacts_dir_name):
@ -621,8 +612,9 @@ class ConfigManager(object):
def load_files(self, configs):
""" Read configs set into storage """
logger.debug("Reading configs: %s", configs)
config_filenames = [resource.resource_filename(config)
for config in configs]
config_filenames = [
resource.resource_filename(config) for config in configs
]
try:
self.config.read(config_filenames)
except Exception as ex:
@ -644,13 +636,13 @@ class ConfigManager(object):
try:
for option in self.config.options(section):
if not prefix or option.find(prefix) == 0:
res += [(option[len(prefix):],
self.config.get(section, option))]
res += [(
option[len(prefix):], self.config.get(section, option))]
except ConfigParser.NoSectionError as ex:
logger.warning("No section: %s", ex)
logger.debug("Section: [%s] prefix: '%s' options:\n%s", section,
prefix, res)
logger.debug(
"Section: [%s] prefix: '%s' options:\n%s", section, prefix, res)
return res
def find_sections(self, prefix):

17
yandextank/plugins/Aggregator/aggregator.py
View File

@ -38,8 +38,8 @@ class Worker(object):
np.linspace(5000, 9900, 50)) # 100µs accuracy
bins = np.append(bins,
np.linspace(10, 499, 490) * 1000) # 1ms accuracy
bins = np.append(bins, np.linspace(500, 2995, 500) *
1000) # 5ms accuracy
bins = np.append(bins,
np.linspace(500, 2995, 500) * 1000) # 5ms accuracy
bins = np.append(bins, np.linspace(3000, 9990, 700) *
1000) # 10ms accuracy
bins = np.append(bins, np.linspace(10000, 29950, 400) *
@ -119,7 +119,6 @@ class Worker(object):
class DataPoller(object):
def __init__(self, source, poll_period):
self.poll_period = poll_period
self.source = source
@ -132,7 +131,6 @@ class DataPoller(object):
class Aggregator(object):
def __init__(self, source, config, verbose_histogram):
self.worker = Worker(config, verbose_histogram)
self.source = source
@ -144,12 +142,11 @@ class Aggregator(object):
start_time = time.time()
result = {
"ts": ts,
"tagged": {
tag: self.worker.aggregate(data)
for tag, data in by_tag
},
"tagged":
{tag: self.worker.aggregate(data)
for tag, data in by_tag},
"overall": self.worker.aggregate(chunk),
}
logger.debug("Aggregation time: %.2fms",
(time.time() - start_time) * 1000)
logger.debug(
"Aggregation time: %.2fms", (time.time() - start_time) * 1000)
yield result

5
yandextank/plugins/Aggregator/chopper.py
View File

@ -23,9 +23,8 @@ class TimeChopper(object):
grouped = chunk.groupby(level=0)
for group_key, group_data in list(grouped):
if group_key in self.cache:
self.cache[group_key] = pd.concat([
self.cache[group_key], group_data
])
self.cache[group_key] = pd.concat(
[self.cache[group_key], group_data])
else:
self.cache[group_key] = group_data
while len(self.cache) > self.cache_size:

13
yandextank/plugins/Aggregator/plugin.py
View File

@ -59,8 +59,8 @@ class Plugin(AbstractPlugin):
return ["verbose_histogram"]
def configure(self):
self.aggregator_config = json.loads(resource_string(
__name__, 'config/phout.json').decode('utf8'))
self.aggregator_config = json.loads(
resource_string(__name__, 'config/phout.json').decode('utf8'))
verbose_histogram_option = self.get_option("verbose_histogram", "0")
self.verbose_histogram = (
verbose_histogram_option.lower() == "true") or (
@ -72,16 +72,15 @@ class Plugin(AbstractPlugin):
if self.reader and self.stats_reader:
pipeline = Aggregator(
TimeChopper(
DataPoller(source=self.reader,
poll_period=1),
cache_size=3),
DataPoller(
source=self.reader, poll_period=1), cache_size=3),
self.aggregator_config,
self.verbose_histogram)
self.drain = Drain(pipeline, self.results)
self.drain.start()
self.stats_drain = Drain(
Chopper(DataPoller(source=self.stats_reader,
poll_period=1)),
Chopper(DataPoller(
source=self.stats_reader, poll_period=1)),
self.stats)
self.stats_drain.start()
else:

8
yandextank/plugins/Aggregator/tests/test_chopper.py
View File

@ -7,7 +7,6 @@ from conftest import MAX_TS, random_split
class TestChopper(object):
def test_one_chunk(self, data):
chopper = TimeChopper([data], 5)
result = list(chopper)
@ -29,8 +28,9 @@ class TestChopper(object):
chopper = TimeChopper(chunks, 5)
result = list(chopper)
assert len(
result) == MAX_TS, "DataFrame is splitted into proper number of chunks"
result
) == MAX_TS, "DataFrame is splitted into proper number of chunks"
concatinated = pd.concat(r[1] for r in result)
assert len(data) == len(concatinated), "We did not lose anything"
assert np.allclose(concatinated.values,
data.values), "We did not corrupt the data"
assert np.allclose(
concatinated.values, data.values), "We did not corrupt the data"

16
yandextank/plugins/Aggregator/tests/test_pipeline.py
View File

@ -9,12 +9,12 @@ from yandextank.plugins.Aggregator.aggregator import Aggregator
from yandextank.plugins.Aggregator.chopper import TimeChopper
from yandextank.plugins.Aggregator.plugin import DataPoller
AGGR_CONFIG = json.loads(resource_string("yandextank.plugins.Aggregator",
'config/phout.json').decode('utf-8'))
AGGR_CONFIG = json.loads(
resource_string("yandextank.plugins.Aggregator", 'config/phout.json')
.decode('utf-8'))
class TestPipeline(object):
def test_partially_reversed_data(self, data):
results_queue = Queue()
chunks = list(random_split(data))
@ -22,9 +22,8 @@ class TestPipeline(object):
pipeline = Aggregator(
TimeChopper(
DataPoller(source=chunks,
poll_period=0.1),
cache_size=3),
DataPoller(
source=chunks, poll_period=0.1), cache_size=3),
AGGR_CONFIG,
False)
drain = Drain(pipeline, results_queue)
@ -44,9 +43,8 @@ class TestPipeline(object):
pipeline = Aggregator(
TimeChopper(
DataPoller(source=producer(),
poll_period=0.1),
cache_size=3),
DataPoller(
source=producer(), poll_period=0.1), cache_size=3),
AGGR_CONFIG,
False)
drain = Drain(pipeline, results_queue)

4
yandextank/plugins/Aggregator/tests/test_test.py
View File

@ -8,5 +8,5 @@ def test_random_split(data):
assert len(dataframes) > 1
concatinated = pd.concat(dataframes)
assert len(concatinated) == len(data), "We did not lose anything"
assert np.allclose(concatinated.values,
data.values), "We did not corrupt the data"
assert np.allclose(
concatinated.values, data.values), "We did not corrupt the data"

7
yandextank/plugins/Appium/plugin.py
View File

@ -44,7 +44,8 @@ class Plugin(AbstractPlugin):
process_stdout_file = self.core.mkstemp(".log", "appium_stdout_")
self.core.add_artifact_file(process_stdout_file)
self.process_stdout = open(process_stdout_file, 'w')
self.process = subprocess.Popen(args,
self.process = subprocess.Popen(
args,
stderr=self.process_stdout,
stdout=self.process_stdout,
close_fds=True)
@ -61,8 +62,8 @@ class Plugin(AbstractPlugin):
def end_test(self, retcode):
if self.process and self.process.poll() is None:
logger.info("Terminating appium process with PID %s",
self.process.pid)
logger.info(
"Terminating appium process with PID %s", self.process.pid)
self.process.terminate()
if self.process_stdout:
self.process_stdout.close()

58
yandextank/plugins/Autostop/criterions.py
View File

@ -20,14 +20,13 @@ class AvgTimeCriterion(AbstractCriterion):
def __init__(self, autostop, param_str):
AbstractCriterion.__init__(self)
self.seconds_count = 0
self.rt_limit = expand_to_milliseconds(param_str.split(',')[
0])
self.seconds_limit = expand_to_seconds(param_str.split(',')[
1])
self.rt_limit = expand_to_milliseconds(param_str.split(',')[0])
self.seconds_limit = expand_to_seconds(param_str.split(',')[1])
self.autostop = autostop
def notify(self, data, stat):
if (data["overall"]["interval_real"]["total"] / 1000.0 /
if (
data["overall"]["interval_real"]["total"] / 1000.0 /
data["overall"]["interval_real"]["len"]) > self.rt_limit:
if not self.seconds_count:
self.cause_second = (data, stat)
@ -47,10 +46,10 @@ class AvgTimeCriterion(AbstractCriterion):
return self.RC_TIME
def explain(self):
explanation = ("Average response time higher"
explanation = (
"Average response time higher"
" than %sms for %ss, since %s" %
(self.rt_limit, self.seconds_count,
self.cause_second[0]["ts"]))
(self.rt_limit, self.seconds_count, self.cause_second[0]["ts"]))
return explanation
def widget_explain(self):
@ -80,8 +79,7 @@ class HTTPCodesCriterion(AbstractCriterion):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = expand_to_seconds(param_str.split(',')[
2])
self.seconds_limit = expand_to_seconds(param_str.split(',')[2])
def notify(self, data, stat):
matched_responses = self.count_matched_codes(
@ -92,7 +90,8 @@ class HTTPCodesCriterion(AbstractCriterion):
"interval_real"]["len"]
else:
matched_responses = 0
logger.debug("HTTP codes matching mask %s: %s/%s", self.codes_mask,
logger.debug(
"HTTP codes matching mask %s: %s/%s", self.codes_mask,
matched_responses, self.level)
if matched_responses >= self.level:
@ -122,12 +121,14 @@ class HTTPCodesCriterion(AbstractCriterion):
return level_str
def explain(self):
items = (self.codes_mask, self.get_level_str(), self.seconds_count,
items = (
self.codes_mask, self.get_level_str(), self.seconds_count,
self.cause_second[0].get('ts'))
return "%s codes count higher than %s for %ss, since %s" % items
def widget_explain(self):
items = (self.codes_mask, self.get_level_str(), self.seconds_count,
items = (
self.codes_mask, self.get_level_str(), self.seconds_count,
self.seconds_limit)
return "HTTP %s>%s for %s/%ss" % items, float(
self.seconds_count) / self.seconds_limit
@ -154,8 +155,7 @@ class NetCodesCriterion(AbstractCriterion):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = expand_to_seconds(param_str.split(',')[
2])
self.seconds_limit = expand_to_seconds(param_str.split(',')[2])
def notify(self, data, stat):
codes = copy.deepcopy(data["overall"]["net_code"]["count"])
@ -168,7 +168,8 @@ class NetCodesCriterion(AbstractCriterion):
"interval_real"]["len"]
else:
matched_responses = 0
logger.debug("Net codes matching mask %s: %s/%s", self.codes_mask,
logger.debug(
"Net codes matching mask %s: %s/%s", self.codes_mask,
matched_responses, self.level)
if matched_responses >= self.level:
@ -198,12 +199,14 @@ class NetCodesCriterion(AbstractCriterion):
return level_str
def explain(self):
items = (self.codes_mask, self.get_level_str(), self.seconds_count,
items = (
self.codes_mask, self.get_level_str(), self.seconds_count,
self.cause_second[0].get("ts"))
return "%s net codes count higher than %s for %ss, since %s" % items
def widget_explain(self):
items = (self.codes_mask, self.get_level_str(), self.seconds_count,
items = (
self.codes_mask, self.get_level_str(), self.seconds_count,
self.seconds_limit)
return "Net %s>%s for %s/%ss" % items, float(
self.seconds_count) / self.seconds_limit
@ -225,8 +228,10 @@ class QuantileCriterion(AbstractCriterion):
self.autostop = autostop
def notify(self, data, stat):
quantiles = dict(zip(data["overall"]["interval_real"]["q"]["q"], data[
"overall"]["interval_real"]["q"]["value"]))
quantiles = dict(
zip(
data["overall"]["interval_real"]["q"]["q"], data["overall"][
"interval_real"]["q"]["value"]))
if self.quantile not in quantiles.keys():
logger.warning("No quantile %s in %s", self.quantile, quantiles)
if self.quantile in quantiles.keys() \
@ -249,12 +254,14 @@ class QuantileCriterion(AbstractCriterion):
return self.RC_TIME
def explain(self):
items = (self.quantile, self.rt_limit, self.seconds_count,
items = (
self.quantile, self.rt_limit, self.seconds_count,
self.cause_second[0].get("ts"))
return "Percentile %s higher than %sms for %ss, since %s" % items
def widget_explain(self):
items = (self.quantile, self.rt_limit, self.seconds_count,
items = (
self.quantile, self.rt_limit, self.seconds_count,
self.seconds_limit)
return "%s%% >%sms for %s/%ss" % items, float(
self.seconds_count) / self.seconds_limit
@ -272,13 +279,12 @@ class SteadyCumulativeQuantilesCriterion(AbstractCriterion):
AbstractCriterion.__init__(self)
self.seconds_count = 0
self.quantile_hash = ""
self.seconds_limit = expand_to_seconds(param_str.split(',')[
0])
self.seconds_limit = expand_to_seconds(param_str.split(',')[0])
self.autostop = autostop
def notify(self, data, stat):
quantiles = dict(zip(data["overall"]["q"]["q"], data["overall"]["q"][
"values"]))
quantiles = dict(
zip(data["overall"]["q"]["q"], data["overall"]["q"]["values"]))
quantile_hash = json.dumps(quantiles)
logging.debug("Cumulative quantiles hash: %s", quantile_hash)
if self.quantile_hash == quantile_hash:

140
yandextank/plugins/Autostop/cumulative_criterions.py
View File

@ -13,7 +13,6 @@ logger = logging.getLogger(__name__)
class WindowCounter(object):
def __init__(self, window_size):
self.window_size = window_size
self.value = 0.0
@ -60,8 +59,8 @@ class TotalFracTimeCriterion(AbstractCriterion):
def __fail_count(self, data):
ecdf = np.cumsum(data["overall"]["interval_real"]["hist"]["data"])
idx = np.searchsorted(data["overall"]["interval_real"]["hist"]["bins"],
self.rt_limit)
idx = np.searchsorted(
data["overall"]["interval_real"]["hist"]["bins"], self.rt_limit)
if idx == 0:
return ecdf[-1]
elif idx == len(ecdf):
@ -73,8 +72,8 @@ class TotalFracTimeCriterion(AbstractCriterion):
self.seconds.append((data, stat))
self.fail_counter.push(self.__fail_count(data))
self.total_counter.push(data["overall"]["interval_real"]["len"])
self.total_fail_ratio = (self.fail_counter.value /
self.total_counter.value)
self.total_fail_ratio = (
self.fail_counter.value / self.total_counter.value)
if self.total_fail_ratio >= self.fail_ratio_limit and len(
self.fail_counter) >= self.window_size:
self.cause_second = self.seconds[0]
@ -88,14 +87,16 @@ class TotalFracTimeCriterion(AbstractCriterion):
return 25
def explain(self):
return ("%.2f%% responses times higher "
"than %sms for %ss since: %s" %
(self.total_fail_ratio * 100, self.rt_limit / 1000,
return (
"%.2f%% responses times higher "
"than %sms for %ss since: %s" % (
self.total_fail_ratio * 100, self.rt_limit / 1000,
self.window_size, self.cause_second[0]["ts"]))
def widget_explain(self):
return ("%.2f%% times >%sms for %ss" %
(self.total_fail_ratio * 100, self.rt_limit / 1000,
return (
"%.2f%% times >%sms for %ss" % (
self.total_fail_ratio * 100, self.rt_limit / 1000,
self.window_size), self.total_fail_ratio)
@ -122,8 +123,7 @@ class TotalHTTPCodesCriterion(AbstractCriterion):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = expand_to_seconds(param_str.split(',')[
2])
self.seconds_limit = expand_to_seconds(param_str.split(',')[2])
def notify(self, data, stat):
matched_responses = self.count_matched_codes(
@ -134,7 +134,8 @@ class TotalHTTPCodesCriterion(AbstractCriterion):
"interval_real"]["len"] * 100
else:
matched_responses = 1
logger.debug("HTTP codes matching mask %s: %s/%s", self.codes_mask,
logger.debug(
"HTTP codes matching mask %s: %s/%s", self.codes_mask,
matched_responses, self.level)
self.data.append(matched_responses)
self.second_window.append((data, stat))
@ -144,11 +145,10 @@ class TotalHTTPCodesCriterion(AbstractCriterion):
queue_len = 1
if self.is_relative:
queue_len = len(self.data)
if (sum(self.data) / queue_len) >= self.level and len(
self.data) >= self.seconds_limit:
if (sum(self.data) / queue_len) >= self.level\
and len(self.data) >= self.seconds_limit: # yapf:disable
self.cause_second = self.second_window[0]
logger.debug(self.explain())
# self.autostop.add_counting(self)
return True
return False
@ -165,11 +165,14 @@ class TotalHTTPCodesCriterion(AbstractCriterion):
def explain(self):
if self.is_relative:
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return ("%s codes count higher "
return (
"%s codes count higher "
"than %s for %ss, ended at: %s" % items)
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return "%s codes count higher than %s for %ss, since %s" % items
@ -204,8 +207,7 @@ class TotalNetCodesCriterion(AbstractCriterion):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = expand_to_seconds(param_str.split(',')[
2])
self.seconds_limit = expand_to_seconds(param_str.split(',')[2])
def notify(self, data, stat):
codes = data["overall"]["net_code"]["count"].copy()
@ -216,13 +218,14 @@ class TotalNetCodesCriterion(AbstractCriterion):
if data["overall"]["interval_real"]["len"]:
matched_responses = float(matched_responses) / data["overall"][
"interval_real"]["len"] * 100
logger.debug("Net codes matching mask %s: %s%%/%s",
self.codes_mask, round(matched_responses,
2), self.get_level_str())
logger.debug(
"Net codes matching mask %s: %s%%/%s", self.codes_mask,
round(matched_responses, 2), self.get_level_str())
else:
matched_responses = 1
else:
logger.debug("Net codes matching mask %s: %s/%s", self.codes_mask,
logger.debug(
"Net codes matching mask %s: %s/%s", self.codes_mask,
matched_responses, self.get_level_str())
self.data.append(matched_responses)
@ -234,11 +237,10 @@ class TotalNetCodesCriterion(AbstractCriterion):
queue_len = 1
if self.is_relative:
queue_len = len(self.data)
if (sum(self.data) / queue_len) >= self.level and len(
self.data) >= self.seconds_limit:
if (sum(self.data) / queue_len) >= self.level\
and len(self.data) >= self.seconds_limit: # yapf:disable
self.cause_second = self.second_window[0]
logger.debug(self.explain())
# self.autostop.add_counting(self)
return True
return False
@ -255,11 +257,14 @@ class TotalNetCodesCriterion(AbstractCriterion):
def explain(self):
if self.is_relative:
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return ("%s net codes count higher "
return (
"%s net codes count higher "
"than %s for %ss, since %s" % items)
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return "%s net codes count higher than %s for %ss, since %s" % items
@ -294,8 +299,7 @@ class TotalNegativeHTTPCodesCriterion(AbstractCriterion):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = expand_to_seconds(param_str.split(',')[
2])
self.seconds_limit = expand_to_seconds(param_str.split(',')[2])
def notify(self, data, stat):
matched_responses = self.count_matched_codes(
@ -307,14 +311,15 @@ class TotalNegativeHTTPCodesCriterion(AbstractCriterion):
matched_responses = 100 - matched_responses
else:
matched_responses = 1
logger.debug("HTTP codes matching mask not %s: %s/%s",
self.codes_mask, round(matched_responses,
1), self.level)
logger.debug(
"HTTP codes matching mask not %s: %s/%s", self.codes_mask,
round(matched_responses, 1), self.level)
else:
matched_responses = (
data["overall"]["interval_real"]["len"] - matched_responses)
logger.debug("HTTP codes matching mask not %s: %s/%s",
self.codes_mask, matched_responses, self.level)
logger.debug(
"HTTP codes matching mask not %s: %s/%s", self.codes_mask,
matched_responses, self.level)
self.data.append(matched_responses)
self.second_window.append((data, stat))
if len(self.data) > self.seconds_limit:
@ -324,11 +329,10 @@ class TotalNegativeHTTPCodesCriterion(AbstractCriterion):
queue_len = 1
if self.is_relative:
queue_len = len(self.data)
if (sum(self.data) / queue_len) >= self.level and len(
self.data) >= self.seconds_limit:
if (sum(self.data) / queue_len) >= self.level\
and len(self.data) >= self.seconds_limit: # yapf:disable
self.cause_second = self.second_window[0]
logger.debug(self.explain())
# self.autostop.add_counting(self)
return True
return False
@ -345,11 +349,14 @@ class TotalNegativeHTTPCodesCriterion(AbstractCriterion):
def explain(self):
if self.is_relative:
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return ("Not %s codes count higher "
return (
"Not %s codes count higher "
"than %s for %ss, since %s" % items)
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return "Not %s codes count higher than %s for %ss, since %s" % items
@ -384,8 +391,7 @@ class TotalNegativeNetCodesCriterion(AbstractCriterion):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = expand_to_seconds(param_str.split(',')[
2])
self.seconds_limit = expand_to_seconds(param_str.split(',')[2])
def notify(self, data, stat):
codes = data["overall"]["net_code"]["count"].copy()
@ -399,14 +405,15 @@ class TotalNegativeNetCodesCriterion(AbstractCriterion):
matched_responses = 100 - matched_responses
else:
matched_responses = 1
logger.debug("Net codes matching mask not %s: %s/%s",
self.codes_mask, round(matched_responses,
1), self.level)
logger.debug(
"Net codes matching mask not %s: %s/%s", self.codes_mask,
round(matched_responses, 1), self.level)
else:
matched_responses = (
data["overall"]["interval_real"]["len"] - matched_responses)
logger.debug("Net codes matching mask not %s: %s/%s",
self.codes_mask, matched_responses, self.level)
logger.debug(
"Net codes matching mask not %s: %s/%s", self.codes_mask,
matched_responses, self.level)
self.data.append(matched_responses)
self.second_window.append((data, stat))
if len(self.data) > self.seconds_limit:
@ -416,8 +423,8 @@ class TotalNegativeNetCodesCriterion(AbstractCriterion):
queue_len = 1
if self.is_relative:
queue_len = len(self.data)
if (sum(self.data) / queue_len) >= self.level and len(
self.data) >= self.seconds_limit:
if (sum(self.data) / queue_len) >= self.level \
and len(self.data) >= self.seconds_limit: # yapf:disable
self.cause_second = self.second_window[0]
logger.debug(self.explain())
return True
@ -436,11 +443,14 @@ class TotalNegativeNetCodesCriterion(AbstractCriterion):
def explain(self):
if self.is_relative:
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return ("Not %s codes count higher "
return (
"Not %s codes count higher "
"than %s for %ss, since %s" % items)
items = (self.codes_mask, self.get_level_str(), self.seconds_limit,
items = (
self.codes_mask, self.get_level_str(), self.seconds_limit,
self.cause_second[0]["ts"])
return "Not %s codes count higher than %s for %ss, since %s" % items
@ -471,8 +481,7 @@ class TotalHTTPTrendCriterion(AbstractCriterion):
self.tangents.append(0)
self.last = 0
self.seconds_limit = expand_to_seconds(param_str.split(',')[
1])
self.seconds_limit = expand_to_seconds(param_str.split(',')[1])
self.measurement_error = float()
def notify(self, data, stat):
@ -491,8 +500,9 @@ class TotalHTTPTrendCriterion(AbstractCriterion):
self.measurement_error = self.calc_measurement_error(self.tangents)
self.total_tan = float(sum(self.tangents) / len(self.tangents))
logger.debug("Last trend for http codes %s: %.2f +/- %.2f",
self.codes_mask, self.total_tan, self.measurement_error)
logger.debug(
"Last trend for http codes %s: %.2f +/- %.2f", self.codes_mask,
self.total_tan, self.measurement_error)
if self.total_tan + self.measurement_error < 0:
self.cause_second = self.second_window[0]
@ -521,13 +531,16 @@ class TotalHTTPTrendCriterion(AbstractCriterion):
return 30
def explain(self):
items = (self.codes_mask, self.total_tan, self.measurement_error,
items = (
self.codes_mask, self.total_tan, self.measurement_error,
self.seconds_limit, self.cause_second[0]["ts"])
return ("Last trend for %s http codes "
return (
"Last trend for %s http codes "
"is %.2f +/- %.2f for %ss, since %s" % items)
def widget_explain(self):
items = (self.codes_mask, self.total_tan, self.measurement_error,
items = (
self.codes_mask, self.total_tan, self.measurement_error,
self.seconds_limit)
return ("HTTP(%s) trend is %.2f +/- %.2f < 0 for %ss" % items, 1.0)
@ -543,6 +556,7 @@ class QuantileOfSaturationCriterion(AbstractCriterion):
def __init__(self, autostop, param_str):
AbstractCriterion.__init__(self)
raise NotImplementedError
# self.autostop = autostop
# self.data = deque()
# self.second_window = deque()

14
yandextank/plugins/Autostop/plugin.py
View File

@ -51,8 +51,8 @@ class Plugin(AbstractPlugin, AggregateResultListener):
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
aggregator.add_result_listener(self)
self.criterion_str = " ".join(self.get_option("autostop", '').split(
"\n"))
self.criterion_str = " ".join(
self.get_option("autostop", '').split("\n"))
self._stop_report_path = os.path.join(
self.core.artifacts_dir,
self.get_option("report_file", 'autostop_report.txt'))
@ -92,7 +92,8 @@ class Plugin(AbstractPlugin, AggregateResultListener):
def is_test_finished(self):
if self.cause_criterion:
self.log.info("Autostop criterion requested test stop: %s",
self.log.info(
"Autostop criterion requested test stop: %s",
self.cause_criterion.explain())
return self.cause_criterion.get_rc()
else:
@ -107,8 +108,8 @@ class Plugin(AbstractPlugin, AggregateResultListener):
for criterion_class in self.custom_criterions:
if criterion_class.get_type_string() == type_str:
return criterion_class(self, parsed[1])
raise ValueError("Unsupported autostop criterion type: %s" %
criterion_str)
raise ValueError(
"Unsupported autostop criterion type: %s" % criterion_str)
def on_aggregated_data(self, data, stat):
self.counting = []
@ -116,8 +117,7 @@ class Plugin(AbstractPlugin, AggregateResultListener):
for criterion_text, criterion in self._criterions.iteritems():
if criterion.notify(data, stat):
self.log.debug(
"Autostop criterion requested test stop: %s",
criterion)
"Autostop criterion requested test stop: %s", criterion)
self.cause_criterion = criterion
open(self._stop_report_path, 'w').write(criterion_text)
self.core.add_artifact_file(self._stop_report_path)

12
yandextank/plugins/BatteryHistorian/plugin.py
View File

@ -22,10 +22,13 @@ class Plugin(AbstractPlugin):
self.default_target = None
self.device_id = None
self.cmds = {
"enable_full_log": "adb %s shell dumpsys batterystats --enable full-wake-history",
"disable_full_log": "adb %s shell dumpsys batterystats --disable full-wake-history",
"enable_full_log":
"adb %s shell dumpsys batterystats --enable full-wake-history",
"disable_full_log":
"adb %s shell dumpsys batterystats --disable full-wake-history",
"reset": "adb %s shell dumpsys batterystats --reset",
"dump": "adb %s shell dumpsys batterystats"}
"dump": "adb %s shell dumpsys batterystats"
}
def get_available_options(self):
return ["device_id"]
@ -57,8 +60,7 @@ class Plugin(AbstractPlugin):
try:
logger.debug('dumping battery stats')
dump = subprocess.Popen(
self.cmds['dump'],
stdout=subprocess.PIPE,
self.cmds['dump'], stdout=subprocess.PIPE,
shell=True).communicate()[0]
out = subprocess.check_output(
self.cmds['disable_full_log'], shell=True)

5
yandextank/plugins/Bfg/example/scenario_gun.py
View File

@ -40,4 +40,7 @@ def scenario_2(missile, marker, measure):
SCENARIOS module variable is used by Tank to choose the scenario to
shoot with. For each missile Tank will look up missile marker in this dict.
"""
SCENARIOS = {"scenario_1": scenario_1, "scenario_2": scenario_1, }
SCENARIOS = {
"scenario_1": scenario_1,
"scenario_2": scenario_1,
}

1
yandextank/plugins/Bfg/example/ultimate_gun.py
View File

@ -3,7 +3,6 @@ log = logging.getLogger(__name__)
class LoadTest(object):
def __init__(self, gun):
self.gun = gun

17
yandextank/plugins/Bfg/guns.py
View File

@ -16,7 +16,6 @@ requests.packages.urllib3.disable_warnings()
class AbstractGun(AbstractPlugin):
def __init__(self, core):
super(AbstractGun, self).__init__(core)
self.results = None
@ -49,8 +48,8 @@ class AbstractGun(AbstractPlugin):
raise
finally:
if data_item.get("interval_real") is None:
data_item["interval_real"] = int((time.time() - start_time) *
1e6)
data_item["interval_real"] = int(
(time.time() - start_time) * 1e6)
self.results.put(data_item, timeout=1)
@ -152,8 +151,8 @@ class CustomGun(AbstractGun):
module_name = self.get_option("module_name")
fp, pathname, description = imp.find_module(module_name, module_path)
try:
self.module = imp.load_module(module_name, fp, pathname,
description)
self.module = imp.load_module(
module_name, fp, pathname, description)
finally:
if fp:
fp.close()
@ -186,8 +185,8 @@ class ScenarioGun(AbstractGun):
module_name = self.get_option("module_name")
fp, pathname, description = imp.find_module(module_name, module_path)
try:
self.module = imp.load_module(module_name, fp, pathname,
description)
self.module = imp.load_module(
module_name, fp, pathname, description)
finally:
if fp:
fp.close()
@ -232,8 +231,8 @@ class UltimateGun(AbstractGun):
# it is imported to be sure Python won't be able to cache it
#
try:
self.module = imp.load_module("%s_%d" % (module_name, time.time()),
fp, pathname, description)
self.module = imp.load_module(
"%s_%d" % (module_name, time.time()), fp, pathname, description)
finally:
if fp:
fp.close()

7
yandextank/plugins/Bfg/plugin.py
View File

@ -62,14 +62,15 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
if gun_type in self.gun_classes:
self.gun = self.gun_classes[gun_type](self.core)
else:
raise NotImplementedError('No such gun type implemented: "%s"' %
gun_type)
raise NotImplementedError(
'No such gun type implemented: "%s"' % gun_type)
cached_stpd_option = self.get_option("cached_stpd", '0')
if cached_stpd_option == '1':
cached_stpd = True
else:
cached_stpd = False
self.bfg = BFG(gun=self.gun,
self.bfg = BFG(
gun=self.gun,
instances=self.stepper_wrapper.instances,
stpd_filename=self.stepper_wrapper.stpd,
cached_stpd=cached_stpd)

14
yandextank/plugins/Bfg/reader.py
View File

@ -14,11 +14,10 @@ def records_to_df(records):
def _expand_steps(steps):
return list(itt.chain(*[[rps] * int(duration) for rps, duration in steps]))
return list(itt.chain(* [[rps] * int(duration) for rps, duration in steps]))
class BfgReader(object):
def __init__(self, results):
self.buffer = ""
self.stat_buffer = ""
@ -43,7 +42,6 @@ class BfgReader(object):
class BfgStatsReader(object):
def __init__(self, instance_counter, steps):
self.closed = False
self.last_ts = 0
@ -59,9 +57,13 @@ class BfgStatsReader(object):
reqps = 0
if offset >= 0 and offset < len(self.steps):
reqps = self.steps[offset]
yield [{'ts': cur_ts,
'metrics': {'instances': self.instance_counter.value,
'reqps': reqps}}]
yield [{
'ts': cur_ts,
'metrics': {
'instances': self.instance_counter.value,
'reqps': reqps
}
}]
self.last_ts = cur_ts
else:
yield []

3
yandextank/plugins/Bfg/widgets.py
View File

@ -33,8 +33,7 @@ class BfgInfoWidget(AbstractInfoWidget):
res += str(self.instances)
res += "\nPlanned requests: %s for %s\nActual responses: " % (
self.planned,
datetime.timedelta(seconds=self.planned_rps_duration))
self.planned, datetime.timedelta(seconds=self.planned_rps_duration))
if not self.planned == self.RPS:
res += screen.markup.YELLOW + str(self.RPS) + screen.markup.RESET
else:

31
yandextank/plugins/Bfg/worker.py
View File

@ -16,11 +16,13 @@ class BFG(object):
"""
def __init__(self, gun, instances, stpd_filename, cached_stpd=False):
logger.info("""
logger.info(
"""
BFG using stpd from {stpd_filename}
Instances: {instances}
Gun: {gun.__class__.__name__}
""".format(stpd_filename=stpd_filename,
""".format(
stpd_filename=stpd_filename,
instances=instances,
gun=gun, ))
self.instances = int(instances)
@ -60,9 +62,12 @@ Gun: {gun.__class__.__name__}
Say the workers to finish their jobs and quit.
"""
self.quit.set()
while sorted([self.pool[i].is_alive()
# yapf:disable
while sorted([
self.pool[i].is_alive()
for i in xrange(len(self.pool))])[-1]:
time.sleep(1)
# yapf:enable
try:
while not self.task_queue.empty():
self.task_queue.get(timeout=0.1)
@ -94,19 +99,20 @@ Gun: {gun.__class__.__name__}
else:
continue
workers_count = self.instances
logger.info("Feeded all data. Publishing %d killer tasks" %
(workers_count))
logger.info(
"Feeded all data. Publishing %d killer tasks" % (workers_count))
retry_delay = 1
for _ in range(5):
try:
[self.task_queue.put(None,
timeout=1)
for _ in xrange(0, workers_count)]
[
self.task_queue.put(None, timeout=1)
for _ in xrange(0, workers_count)
]
break
except Full:
logger.debug("Couldn't post killer tasks"
" because queue is full. Retrying in %ss",
retry_delay)
logger.debug(
"Couldn't post killer tasks"
" because queue is full. Retrying in %ss", retry_delay)
time.sleep(retry_delay)
retry_delay *= 2
@ -160,8 +166,7 @@ Gun: {gun.__class__.__name__}
logger.debug("Empty queue. Exiting process")
return
except Full:
logger.warning(
"Couldn't put to result queue because it's full")
logger.warning("Couldn't put to result queue because it's full")
except Exception:
logger.exception("Bfg shoot exception")

25
yandextank/plugins/Console/plugin.py
View File

@ -40,8 +40,8 @@ class Plugin(AbstractPlugin, AggregateResultListener):
]
def configure(self):
self.info_panel_width = self.get_option("info_panel_width",
self.info_panel_width)
self.info_panel_width = self.get_option(
"info_panel_width", self.info_panel_width)
self.short_only = int(self.get_option("short_only", '0'))
if not int(self.get_option("disable_all_colors", '0')):
self.console_markup = RealConsoleMarkup()
@ -97,15 +97,17 @@ class Plugin(AbstractPlugin, AggregateResultListener):
if self.short_only:
overall = data.get('overall')
quantiles = dict(zip(overall['interval_real']['q']['q'], overall[
'interval_real']['q']['value']))
quantiles = dict(
zip(
overall['interval_real']['q']['q'], overall['interval_real']
['q']['value']))
info = (
"ts:{ts}\tRPS:{rps}\tavg:{avg_rt:.2f}\t"
"min:{min:.2f}\tmax:{q100:.2f}\tq95:{q95:.2f}\t").format(
ts=data.get('ts'),
rps=overall['interval_real']['len'],
avg_rt=float(overall['interval_real']['total']) / overall[
'interval_real']['len'] / 1000.0,
avg_rt=float(overall['interval_real']['total']) /
overall['interval_real']['len'] / 1000.0,
min=overall['interval_real']['min'] / 1000.0,
q100=quantiles[100] / 1000,
q95=quantiles[95] / 1000)
@ -120,6 +122,7 @@ class Plugin(AbstractPlugin, AggregateResultListener):
else:
self.screen.add_info_widget(widget)
# ======================================================
@ -147,12 +150,15 @@ class RealConsoleMarkup(object):
def clean_markup(self, orig_str):
''' clean markup from string '''
for val in [self.YELLOW, self.RED, self.RESET, self.CYAN,
self.BG_MAGENTA, self.WHITE, self.BG_GREEN, self.GREEN,
self.BG_BROWN, self.RED_DARK, self.MAGENTA, self.BG_CYAN]:
for val in [
self.YELLOW, self.RED, self.RESET, self.CYAN, self.BG_MAGENTA,
self.WHITE, self.BG_GREEN, self.GREEN, self.BG_BROWN,
self.RED_DARK, self.MAGENTA, self.BG_CYAN
]:
orig_str = orig_str.replace(val, '')
return orig_str
# ======================================================
# FIXME: 3 better way to have it?
@ -177,4 +183,5 @@ class NoConsoleMarkup(RealConsoleMarkup):
BG_BROWN = ''
BG_CYAN = ''
# ======================================================

154
yandextank/plugins/Console/screen.py
View File

@ -22,8 +22,8 @@ def get_terminal_size():
Helper to get console size
'''
try:
sizes = struct.unpack('hh', fcntl.ioctl(file_d, termios.TIOCGWINSZ,
'1234'))
sizes = struct.unpack(
'hh', fcntl.ioctl(file_d, termios.TIOCGWINSZ, '1234'))
except Exception:
sizes = default_size
return sizes
@ -85,8 +85,8 @@ class Screen(object):
if len(right_line) > self.right_panel_width:
right_line_plain = self.markup.clean_markup(right_line)
if len(right_line_plain) > self.right_panel_width:
right_line = right_line[
: self.right_panel_width] + self.markup.RESET
right_line = right_line[:self.
right_panel_width] + self.markup.RESET
return right_line
def __render_left_panel(self):
@ -124,8 +124,8 @@ class Screen(object):
def render_screen(self):
''' Main method to render screen view '''
self.term_width, self.term_height = get_terminal_size()
self.log.debug("Terminal size: %sx%s", self.term_width,
self.term_height)
self.log.debug(
"Terminal size: %sx%s", self.term_width, self.term_height)
self.right_panel_width = int(
(self.term_width - len(self.RIGHT_PANEL_SEPARATOR)) *
(float(self.info_panel_percent) / 100)) - 1
@ -135,14 +135,14 @@ class Screen(object):
else:
self.right_panel_width = 0
self.left_panel_width = self.term_width - 1
self.log.debug("Left/right panels width: %s/%s", self.left_panel_width,
self.log.debug(
"Left/right panels width: %s/%s", self.left_panel_width,
self.right_panel_width)
widget_output = []
if self.right_panel_width:
widget_output = []
self.log.debug("There are %d info widgets" %
len(self.info_widgets))
self.log.debug("There are %d info widgets" % len(self.info_widgets))
for index, widget in sorted(
self.info_widgets.iteritems(),
key=lambda item: (item[1].get_index(), item[0])):
@ -164,11 +164,11 @@ class Screen(object):
left_line_plain = self.markup.clean_markup(left_line)
if len(left_line) > self.left_panel_width:
if len(left_line_plain) > self.left_panel_width:
left_line = left_line[
: self.left_panel_width] + self.markup.RESET
left_line = left_line[:self.
left_panel_width] + self.markup.RESET
left_line += (' ' *
(self.left_panel_width - len(left_line_plain)))
left_line += (
' ' * (self.left_panel_width - len(left_line_plain)))
line += left_line
else:
line += ' ' * self.left_panel_width
@ -223,6 +223,7 @@ class AbstractBlock:
'''
raise RuntimeError("Abstract method needs to be overridden")
# ======================================================
@ -270,7 +271,8 @@ class CurrentTimesDistBlock(AbstractBlock):
def add_second(self, data):
self.current_rps = data["overall"]["interval_real"]["len"]
self.hist = zip(data["overall"]["interval_real"]["hist"]["bins"],
self.hist = zip(
data["overall"]["interval_real"]["hist"]["bins"],
data["overall"]["interval_real"]["hist"]["data"], )
def render(self):
@ -291,6 +293,7 @@ class CurrentTimesDistBlock(AbstractBlock):
self.width = max(self.width, len(self.lines[0]))
# ======================================================
@ -323,8 +326,8 @@ class CurrentHTTPBlock(AbstractBlock):
]
for code, count in sorted(self.times_dist.iteritems()):
line = self.format_line(code, count)
self.width = max(self.width,
len(self.screen.markup.clean_markup(line)))
self.width = max(
self.width, len(self.screen.markup.clean_markup(line)))
self.lines.append(line)
def format_line(self, code, count):
@ -358,6 +361,7 @@ class CurrentHTTPBlock(AbstractBlock):
return left_line
# ======================================================
@ -412,10 +416,11 @@ class CurrentNetBlock(AbstractBlock):
]
for code, count in sorted(self.times_dist.iteritems()):
line = self.format_line(code, count)
self.width = max(self.width,
len(self.screen.markup.clean_markup(line)))
self.width = max(
self.width, len(self.screen.markup.clean_markup(line)))
self.lines.append(line)
# ======================================================
@ -429,10 +434,12 @@ class CurrentQuantilesBlock(AbstractBlock):
self.quantiles = {}
def add_second(self, data):
self.quantiles = {k: v
for k, v in zip(data["overall"]["interval_real"][
"q"]["q"], data["overall"]["interval_real"]["q"][
"value"])}
self.quantiles = {
k: v
for k, v in zip(
data["overall"]["interval_real"]["q"]["q"], data["overall"][
"interval_real"]["q"]["value"])
}
def render(self):
self.lines = []
@ -442,10 +449,12 @@ class CurrentQuantilesBlock(AbstractBlock):
self.lines.append(line)
self.lines.reverse()
self.lines = [self.screen.markup.WHITE + 'Current Percentiles:' +
self.screen.markup.RESET] + self.lines
self.width = max(self.width,
len(self.screen.markup.clean_markup(self.lines[0])))
self.lines = [
self.screen.markup.WHITE + 'Current Percentiles:' +
self.screen.markup.RESET
] + self.lines
self.width = max(
self.width, len(self.screen.markup.clean_markup(self.lines[0])))
def __format_line(self, quan, timing):
''' Format line '''
@ -455,6 +464,7 @@ class CurrentQuantilesBlock(AbstractBlock):
left_line = tpl % data
return left_line
# ======================================================
@ -474,22 +484,24 @@ class AnswSizesBlock(AbstractBlock):
def render(self):
self.lines = [self.header]
if self.count:
self.lines.append(" Avg Request: %d bytes" %
(self.sum_out / self.count))
self.lines.append(" Avg Response: %d bytes" %
(self.sum_in / self.count))
self.lines.append(
" Avg Request: %d bytes" % (self.sum_out / self.count))
self.lines.append(
" Avg Response: %d bytes" % (self.sum_in / self.count))
self.lines.append("")
if self.cur_count:
self.lines.append(" Last Avg Request: %d bytes" %
self.lines.append(
" Last Avg Request: %d bytes" %
(self.cur_out / self.cur_count))
self.lines.append(" Last Avg Response: %d bytes" %
self.lines.append(
" Last Avg Response: %d bytes" %
(self.cur_in / self.cur_count))
else:
self.lines.append("")
self.lines.append("")
for line in self.lines:
self.width = max(self.width,
len(self.screen.markup.clean_markup(line)))
self.width = max(
self.width, len(self.screen.markup.clean_markup(line)))
def add_second(self, data):
@ -501,6 +513,7 @@ class AnswSizesBlock(AbstractBlock):
self.sum_in += self.cur_in
self.sum_out += self.cur_out
# ======================================================
@ -547,28 +560,41 @@ class AvgTimesBlock(AbstractBlock):
self.screen.markup.WHITE + self.header + self.screen.markup.RESET
]
if self.last_count:
len_all = str(len(str(max(
[self.all_connect, self.all_latency, self.all_overall,
self.all_receive, self.all_send]))))
len_last = str(len(str(max(
[self.last_connect, self.last_latency, self.last_overall,
self.last_receive, self.last_send]))))
len_all = str(
len(
str(
max([
self.all_connect, self.all_latency,
self.all_overall, self.all_receive, self.all_send
]))))
len_last = str(
len(
str(
max([
self.last_connect, self.last_latency,
self.last_overall, self.last_receive, self.last_send
]))))
tpl = "%" + len_all + "d / %" + len_last + "d"
self.lines.append(" Overall: " + tpl % (float(
self.all_overall) / self.all_count, float(self.last_overall) /
self.last_count))
self.lines.append(" Connect: " + tpl % (float(
self.all_connect) / self.all_count, float(self.last_connect) /
self.last_count))
self.lines.append(" Send: " + tpl % (float(
self.all_send) / self.all_count, float(self.last_send) /
self.last_count))
self.lines.append(" Latency: " + tpl % (float(
self.all_latency) / self.all_count, float(self.last_latency) /
self.last_count))
self.lines.append(" Receive: " + tpl % (float(
self.all_receive) / self.all_count, float(self.last_receive) /
self.last_count))
self.lines.append(
" Overall: " + tpl % (
float(self.all_overall) / self.all_count, float(
self.last_overall) / self.last_count))
self.lines.append(
" Connect: " + tpl % (
float(self.all_connect) / self.all_count, float(
self.last_connect) / self.last_count))
self.lines.append(
" Send: " + tpl % (
float(self.all_send) / self.all_count, float(
self.last_send) / self.last_count))
self.lines.append(
" Latency: " + tpl % (
float(self.all_latency) / self.all_count, float(
self.last_latency) / self.last_count))
self.lines.append(
" Receive: " + tpl % (
float(self.all_receive) / self.all_count, float(
self.last_receive) / self.last_count))
else:
self.lines.append("")
self.lines.append("")
@ -576,8 +602,9 @@ class AvgTimesBlock(AbstractBlock):
self.lines.append("")
self.lines.append("")
for line in self.lines:
self.width = max(self.width,
len(self.screen.markup.clean_markup(line)))
self.width = max(
self.width, len(self.screen.markup.clean_markup(line)))
# ======================================================
@ -610,14 +637,13 @@ class CasesBlock(AbstractBlock):
self.screen.markup.WHITE + self.header + self.screen.markup.RESET
]
total_count = sum(case[0] for case in self.cases.values())
tpl = " %s: %" + str(len(str(
total_count))) + "d %5.2f%% / avg %.1f ms"
tpl = " %s: %" + str(len(str(total_count))) + "d %5.2f%% / avg %.1f ms"
for name, (count, resp_time) in sorted(self.cases.iteritems()):
line = tpl % (" " * (self.max_case_len - len(name)) + name, count,
100 * float(count) / total_count,
float(resp_time) / count)
line = tpl % (
" " * (self.max_case_len - len(name)) + name, count,
100 * float(count) / total_count, float(resp_time) / count)
self.lines.append(line)
for line in self.lines:
self.width = max(self.width,
len(self.screen.markup.clean_markup(line)))
self.width = max(
self.width, len(self.screen.markup.clean_markup(line)))

52
yandextank/plugins/JMeter/plugin.py
View File

@ -44,8 +44,10 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
return __file__
def get_available_options(self):
return ["jmx", "args", "jmeter_path", "buffer_size",
"buffered_seconds", "exclude_markers"]
return [
"jmx", "args", "jmeter_path", "buffer_size", "buffered_seconds",
"exclude_markers"
]
def configure(self):
self.original_jmx = self.get_option("jmx")
@ -57,19 +59,19 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
self.jmeter_log = self.core.mkstemp('.log', 'jmeter_')
self.jmeter_ver = float(self.get_option('jmeter_ver', '3.0'))
self.ext_log = self.get_option(
'extended_log', self.get_option(
'ext_log', 'none'))
'extended_log', self.get_option('ext_log', 'none'))
if self.ext_log not in self.ext_levels:
self.ext_log = 'none'
if self.ext_log != 'none':
self.ext_log_file = self.core.mkstemp('.jtl', 'jmeter_ext_')
self.core.add_artifact_file(self.ext_log_file)
self.jmeter_buffer_size = int(self.get_option(
self.jmeter_buffer_size = int(
self.get_option(
'buffer_size', self.get_option('buffered_seconds', '3')))
self.core.add_artifact_file(self.jmeter_log, True)
self.exclude_markers = set(filter(
(lambda marker: marker != ''), self.get_option('exclude_markers',
[]).split(' ')))
self.exclude_markers = set(
filter((lambda marker: marker != ''),
self.get_option('exclude_markers', []).split(' ')))
self.jmx = self.__add_jmeter_components(
self.original_jmx, self.jtl_file, self._get_variables())
self.core.add_artifact_file(self.jmx)
@ -79,10 +81,11 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
self.jmeter_stderr = open(jmeter_stderr_file, 'w')
def prepare_test(self):
self.args = [self.jmeter_path, "-n", "-t", self.jmx, '-j',
self.jmeter_log,
self.args = [
self.jmeter_path, "-n", "-t", self.jmx, '-j', self.jmeter_log,
'-Jjmeter.save.saveservice.default_delimiter=\\t',
'-Jjmeter.save.saveservice.connect_time=true']
'-Jjmeter.save.saveservice.connect_time=true'
]
self.args += splitstring(self.user_args)
aggregator = None
@ -108,8 +111,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
aggregator.add_result_listener(widget)
def start_test(self):
logger.info("Starting %s with arguments: %s", self.jmeter_path,
self.args)
logger.info(
"Starting %s with arguments: %s", self.jmeter_path, self.args)
try:
self.jmeter_process = subprocess.Popen(
self.args,
@ -117,8 +120,7 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
preexec_fn=os.setsid,
close_fds=True,
stdout=self.jmeter_stderr,
stderr=self.jmeter_stderr
)
stderr=self.jmeter_stderr)
except OSError:
logger.debug(
"Unable to start JMeter process. Args: %s, Executable: %s",
@ -126,8 +128,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
self.jmeter_path,
exc_info=True)
raise RuntimeError(
"Unable to access to JMeter executable file or it does not exist: %s" %
self.jmeter_path)
"Unable to access to JMeter executable file or it does not exist: %s"
% self.jmeter_path)
self.start_time = time.time()
def is_test_finished(self):
@ -151,7 +153,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
def end_test(self, retcode):
if self.jmeter_process:
logger.info("Terminating jmeter process group with PID %s",
logger.info(
"Terminating jmeter process group with PID %s",
self.jmeter_process.pid)
try:
os.killpg(self.jmeter_process.pid, signal.SIGTERM)
@ -191,16 +194,20 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
if self.ext_log in ['errors', 'all']:
level_map = {'errors': 'true', 'all': 'false'}
tpl_resource = 'jmeter_writer_ext.xml'
tpl_args = {'jtl': self.jtl_file, 'udv': udv,
tpl_args = {
'jtl': self.jtl_file,
'udv': udv,
'ext_log': self.ext_log_file,
'ext_level': level_map[self.ext_log],
'save_connect': save_connect}
'save_connect': save_connect
}
else:
tpl_resource = 'jmeter_writer.xml'
tpl_args = {
'jtl': self.jtl_file,
'udv': udv,
'save_connect': save_connect}
'save_connect': save_connect
}
tpl = resource_string(__name__, 'config/' + tpl_resource)
@ -258,7 +265,8 @@ class JMeterInfoWidget(AbstractInfoWidget, AggregateResultListener):
template += " Duration: %s\n"
template += "Active Threads: %s\n"
template += " Responses/s: %s"
data = (os.path.basename(self.jmeter.original_jmx), duration,
data = (
os.path.basename(self.jmeter.original_jmx), duration,
self.active_threads, self.RPS)
return template % data

30
yandextank/plugins/JMeter/reader.py
View File

@ -59,8 +59,7 @@ def _exc_to_http(param1):
int(param1)
except:
logger.error(
"JMeter wrote some strange data into codes column: %s",
param1)
"JMeter wrote some strange data into codes column: %s", param1)
else:
return int(param1)
@ -113,19 +112,14 @@ def fix_latency(row):
# timeStamp,elapsed,label,responseCode,success,bytes,grpThreads,allThreads,Latency
def string_to_df(data):
chunk = pd.read_csv(
StringIO(data),
sep='\t',
names=jtl_columns,
dtype=jtl_types)
StringIO(data), sep='\t', names=jtl_columns, dtype=jtl_types)
chunk["receive_ts"] = (chunk["send_ts"] + chunk['interval_real']) / 1000.0
chunk['receive_sec'] = chunk["receive_ts"].astype(np.int64)
chunk['interval_real'] = chunk["interval_real"] * 1000 # convert to µs
chunk.set_index(['receive_sec'], inplace=True)
l = len(chunk)
chunk['connect_time'] = (
chunk['connect_time'].fillna(0) *
1000).astype(
np.int64)
chunk['connect_time'] = (chunk['connect_time'].fillna(0) *
1000).astype(np.int64)
chunk['latency'] = chunk['latency'] * 1000
chunk['latency'] = chunk.apply(fix_latency, axis=1)
chunk['send_time'] = np.zeros(l)
@ -139,7 +133,6 @@ def string_to_df(data):
class JMeterStatAggregator(object):
def __init__(self, source):
self.worker = agg.Worker({"allThreads": ["mean"]}, False)
self.source = source
@ -147,16 +140,19 @@ class JMeterStatAggregator(object):
def __iter__(self):
for ts, chunk in self.source:
stats = self.worker.aggregate(chunk)
yield [{'ts': ts,
'metrics': {'instances': stats['allThreads']['mean'],
'reqps': 0}}]
yield [{
'ts': ts,
'metrics': {
'instances': stats['allThreads']['mean'],
'reqps': 0
}
}]
def close(self):
pass
class JMeterReader(object):
def __init__(self, filename):
self.buffer = ""
self.stat_buffer = ""
@ -165,8 +161,8 @@ class JMeterReader(object):
self.agg_finished = False
self.closed = False
self.stat_queue = q.Queue()
self.stats_reader = JMeterStatAggregator(TimeChopper(
self._read_stat_queue(), 3))
self.stats_reader = JMeterStatAggregator(
TimeChopper(self._read_stat_queue(), 3))
def _read_stat_queue(self):
while not self.closed:

22
yandextank/plugins/JsonReport/plugin.py
View File

@ -24,20 +24,17 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
def configure(self):
self.monitoring_logger = self.create_file_logger(
'monitoring', self.get_option(
'monitoring_log', 'monitoring.log'))
'monitoring', self.get_option('monitoring_log', 'monitoring.log'))
self.aggregator_data_logger = self.create_file_logger(
'aggregator_data', self.get_option('test_data_log', 'test_data.log'))
'aggregator_data',
self.get_option('test_data_log', 'test_data.log'))
self.core.job.subscribe_plugin(self)
def create_file_logger(self, logger_name, file_name, formatter=None):
loggr = logging.getLogger(logger_name)
loggr.setLevel(logging.INFO)
handler = logging.FileHandler(
os.path.join(
self.core.artifacts_dir,
file_name),
mode='w')
os.path.join(self.core.artifacts_dir, file_name), mode='w')
handler.setLevel(logging.INFO)
if formatter:
handler.setFormatter(formatter)
@ -51,14 +48,19 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
@stats: stats about gun
"""
self.aggregator_data_logger.info(
json.dumps({'data': data, 'stats': stats}))
json.dumps({
'data': data,
'stats': stats
}))
def monitoring_data(self, data_list):
if self.is_telegraf:
self.monitoring_logger.info(json.dumps(data_list))
else:
[self.monitoring_logger.info(data.strip())
for data in data_list if data]
[
self.monitoring_logger.info(data.strip()) for data in data_list
if data
]
@property
def is_telegraf(self):

11
yandextank/plugins/Maven/plugin.py
View File

@ -41,8 +41,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
def configure(self):
# plugin part
self.pom = resource_manager.resource_filename(self.get_option(
"pom", "pom.xml"))
self.pom = resource_manager.resource_filename(
self.get_option("pom", "pom.xml"))
self.testcase = self.get_option("testcase", "")
self.maven_args = self.get_option("mvn_args", '').split()
@ -73,7 +73,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
process_stderr_file = self.core.mkstemp(".log", "maven_")
self.core.add_artifact_file(process_stderr_file)
self.process_stderr = open(process_stderr_file, 'w')
self.process = subprocess.Popen(args,
self.process = subprocess.Popen(
args,
stderr=self.process_stderr,
stdout=self.process_stderr,
close_fds=True)
@ -88,8 +89,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
def end_test(self, retcode):
if self.process and self.process.poll() is None:
logger.warn("Terminating worker process with PID %s",
self.process.pid)
logger.warn(
"Terminating worker process with PID %s", self.process.pid)
self.process.terminate()
if self.process_stderr:
self.process_stderr.close()

2
yandextank/plugins/Maven/reader.py
View File

@ -1,5 +1,4 @@
class MavenReader(object):
def close(self):
pass
@ -8,7 +7,6 @@ class MavenReader(object):
class MavenStatsReader(object):
def close(self):
pass

113
yandextank/plugins/Monitoring/agent/agent.py
View File

@ -16,7 +16,6 @@ import signal
from threading import Thread
from optparse import OptionParser
logger = logging.getLogger(__name__)
@ -28,8 +27,10 @@ def signal_handler(sig, frame):
def set_sig_handler():
uncatchable = ['SIG_DFL', 'SIGSTOP', 'SIGKILL']
for sig_name in [s for s in dir(signal) if (
s.startswith("SIG") and s not in uncatchable)]:
for sig_name in [
s for s in dir(signal)
if (s.startswith("SIG") and s not in uncatchable)
]:
try:
sig_num = getattr(signal, sig_name)
signal.signal(sig_num, signal_handler)
@ -53,7 +54,6 @@ class AbstractMetric:
class CpuLa(AbstractMetric):
def columns(self, ):
return ['System_la1', 'System_la5', 'System_la15']
@ -76,9 +76,11 @@ class CpuStat(AbstractMetric):
self.current_check = {}
def columns(self, ):
columns = ['System_csw', 'System_int', 'CPU_user', 'CPU_nice',
'CPU_system', 'CPU_idle', 'CPU_iowait', 'CPU_irq',
'CPU_softirq', 'System_numproc', 'System_numthreads']
columns = [
'System_csw', 'System_int', 'CPU_user', 'CPU_nice', 'CPU_system',
'CPU_idle', 'CPU_iowait', 'CPU_irq', 'CPU_softirq',
'System_numproc', 'System_numthreads'
]
return columns
def check(self, ):
@ -90,7 +92,8 @@ class CpuStat(AbstractMetric):
proc_stat_all = proc_stat_file.readlines()
proc_stat_file.close()
except Exception as exc:
logger.error('Error opening /proc/stat. Traceback: %s',
logger.error(
'Error opening /proc/stat. Traceback: %s',
traceback.format_exc(exc))
result.append([''] * 9)
else:
@ -105,7 +108,8 @@ class CpuStat(AbstractMetric):
if stat.startswith('intr '):
self.current_check['intr'] = float(stat.split()[1])
except Exception as exc:
logger.error('Error parsing /proc/stat data. Traceback: %s',
logger.error(
'Error parsing /proc/stat data. Traceback: %s',
traceback.format_exc(exc))
# Context switches and interrups delta
@ -116,10 +120,10 @@ class CpuStat(AbstractMetric):
self.prev_check['intr'] = self.current_check['intr']
result.extend([''] * 2)
else:
delta_csw = str(self.current_check['csw'] -
self.prev_check['csw'])
delta_intr = str(self.current_check['intr'] -
self.prev_check['intr'])
delta_csw = str(
self.current_check['csw'] - self.prev_check['csw'])
delta_intr = str(
self.current_check['intr'] - self.prev_check['intr'])
self.prev_check['csw'] = self.current_check['csw']
self.prev_check['intr'] = self.current_check['intr']
result.append(delta_csw)
@ -168,7 +172,8 @@ class CpuStat(AbstractMetric):
else:
pids.append(element)
except Exception as exc:
logger.error('Error trying to count numprocs. Traceback: %s',
logger.error(
'Error trying to count numprocs. Traceback: %s',
traceback.format_exc(exc))
result.append([''])
else:
@ -223,17 +228,15 @@ class Custom(AbstractMetric):
for el in self.tail:
cmnd = base64.b64decode(el.split(':')[1])
logger.debug("Run custom check: tail -n 1 %s", cmnd)
output = subprocess.Popen(
['tail', '-n', '1', cmnd],
output = subprocess.Popen(['tail', '-n', '1', cmnd],
stdout=subprocess.PIPE).communicate()[0]
res.append(self.diff_value(el, output.strip()))
for el in self.call:
cmnd = base64.b64decode(el.split(':')[1])
logger.debug("Run custom check: %s", cmnd)
output = subprocess.Popen(cmnd,
shell=True,
stdout=subprocess.PIPE).stdout.read()
output = subprocess.Popen(
cmnd, shell=True, stdout=subprocess.PIPE).stdout.read()
res.append(self.diff_value(el, output.strip()))
logger.debug("Collected:\n%s", res)
return res
@ -253,7 +256,6 @@ class Custom(AbstractMetric):
class Disk(AbstractMetric):
def __init__(self):
AbstractMetric.__init__(self)
self.read = 0
@ -281,8 +283,10 @@ class Disk(AbstractMetric):
writed += int(data[9])
if self.read or self.write:
result = [str(size * (read - self.read)),
str(size * (writed - self.write))]
result = [
str(size * (read - self.read)),
str(size * (writed - self.write))
]
else:
result = ['', '']
@ -328,8 +332,8 @@ class Disk(AbstractMetric):
devs.append(dsk_name)
break
except Exception as exc:
logger.info("Failed: %s",
traceback.format_exc(exc))
logger.info(
"Failed: %s", traceback.format_exc(exc))
except Exception as exc:
logger.info(
"Failed to get block device name via /sys/devices/: %s",
@ -347,14 +351,16 @@ class Mem(AbstractMetric):
def __init__(self):
AbstractMetric.__init__(self)
self.name = 'advanced memory usage'
self.vars = ('MemUsed', 'Buffers', 'Cached', 'MemFree', 'Dirty',
'MemTotal')
self.vars = (
'MemUsed', 'Buffers', 'Cached', 'MemFree', 'Dirty', 'MemTotal')
# self.open('/proc/meminfo')
def columns(self):
columns = ['Memory_total', 'Memory_used', 'Memory_free',
'Memory_shared', 'Memory_buff', 'Memory_cached']
columns = [
'Memory_total', 'Memory_used', 'Memory_free', 'Memory_shared',
'Memory_buff', 'Memory_cached'
]
logger.info("Start. Columns: %s" % columns)
return columns
@ -373,8 +379,10 @@ class Mem(AbstractMetric):
data.update({name: long(raw_value.split()[0]) / 1024.0})
data['MemUsed'] = data['MemTotal'] - data['MemFree'] - data[
'Buffers'] - data['Cached']
result = [data['MemTotal'], data['MemUsed'], data['MemFree'],
0, data['Buffers'], data['Cached']]
result = [
data['MemTotal'], data['MemUsed'], data['MemFree'], 0,
data['Buffers'], data['Cached']
]
except Exception as e:
logger.error("Can't get meminfo, %s", e, exc_info=True)
result.append([self.empty] * 9)
@ -396,8 +404,7 @@ class NetRetrans(AbstractMetric):
return ['Net_retransmit', ]
def check(self, ):
self.fetch = lambda: int(commands.getoutput(
'netstat -s | grep "segments retransmited" | awk \'{print $1}\''))
self.fetch = lambda: int(commands.getoutput('netstat -s | grep "segments retransmited" | awk \'{print $1}\''))
if self.retr_second is not None:
self.retr_first = self.fetch()
self.delta = []
@ -415,8 +422,16 @@ class NetTcp(AbstractMetric):
def __init__(self):
AbstractMetric.__init__(self)
self.fields = ['Net_closewait', 'Net_estab', 'Net_timewait', ]
self.keys = ['closed', 'estab', 'timewait', ]
self.fields = [
'Net_closewait',
'Net_estab',
'Net_timewait',
]
self.keys = [
'closed',
'estab',
'timewait',
]
def columns(self, ):
return self.fields
@ -427,6 +442,7 @@ class NetTcp(AbstractMetric):
if note set it to 0.
* make output ordered as "fields" list
"""
def fetch():
return commands.getoutput("ss -s | sed -ne '/^TCP:/p'")
@ -457,7 +473,10 @@ class NetTxRx(AbstractMetric):
self.prev_tx = 0
def columns(self, ):
return ['Net_tx', 'Net_rx', ]
return [
'Net_tx',
'Net_rx',
]
def check(self, ):
"""
@ -477,6 +496,7 @@ class NetTxRx(AbstractMetric):
def position(sample):
return lines[0].split().index(sample)
rx_pos = position('RX-OK')
tx_pos = position('TX-OK')
@ -504,7 +524,6 @@ class NetTxRx(AbstractMetric):
class Net(AbstractMetric):
def __init__(self):
AbstractMetric.__init__(self)
self.recv = 0
@ -553,6 +572,7 @@ class Net(AbstractMetric):
logger.debug("Network recieved/sent bytes: %s", result)
return result
# ===========================
@ -588,7 +608,8 @@ class AgentWorker(Thread):
@staticmethod
def popen(cmnd):
return subprocess.Popen(cmnd,
return subprocess.Popen(
cmnd,
bufsize=0,
preexec_fn=os.setsid,
close_fds=True,
@ -628,8 +649,8 @@ class AgentWorker(Thread):
while not self.finished:
logger.debug('Start check')
line = []
sync_time = str(self.c_start + (int(time.time()) -
self.c_local_start))
sync_time = str(
self.c_start + (int(time.time()) - self.c_local_start))
line.extend([self.c_host, sync_time])
# known metrics
@ -638,8 +659,8 @@ class AgentWorker(Thread):
continue
try:
data = self.known_metrics[metric_name].check()
if len(data) != len(self.known_metrics[
metric_name].columns()):
if len(data) != len(
self.known_metrics[metric_name].columns()):
raise RuntimeError(
"Data len not matched columns count: %s" % data)
except Exception as e:
@ -659,8 +680,7 @@ class AgentWorker(Thread):
sys.stdout.write(row + '\n')
sys.stdout.flush()
except IOError as e:
logger.error(
"Can't send data to collector, terminating, %s", e)
logger.error("Can't send data to collector, terminating, %s", e)
self.finished = True
self.fixed_sleep(self.c_interval)
@ -690,7 +710,6 @@ class AgentWorker(Thread):
class AgentConfig:
def __init__(self, def_cfg_path):
self.c_interval = 1
self.c_host = socket.getfqdn()
@ -717,7 +736,8 @@ class AgentConfig:
help='Config file path, default is: ./' + def_cfg_path,
default=def_cfg_path)
parser.add_option('-t',
parser.add_option(
'-t',
'--timestamp',
dest='timestamp',
type='int',
@ -813,7 +833,8 @@ if __name__ == '__main__':
logger.debug("Join the worker thread, waiting for cleanup")
worker.join(10)
if worker.isAlive():
logger.error("Worker have not finished shutdown in "
logger.error(
"Worker have not finished shutdown in "
"10 seconds, going to exit anyway")
sys.exit(1)
except KeyboardInterrupt:

88
yandextank/plugins/Monitoring/collector.py
View File

@ -78,10 +78,11 @@ class AgentClient(object):
def start(self):
"""Start remote agent"""
logger.debug('Start monitoring: %s', self.host)
self.session = self.ssh.async_session(" ".join([
self.session = self.ssh.async_session(
" ".join([
"DEBUG=1", self.python, self.path['AGENT_REMOTE_FOLDER'] +
'/agent.py', '-c', self.path['AGENT_REMOTE_FOLDER'] + '/agent.cfg',
'-t', str(int(time.time()))
'/agent.py', '-c', self.path['AGENT_REMOTE_FOLDER'] +
'/agent.cfg', '-t', str(int(time.time()))
]))
return self.session
@ -103,8 +104,8 @@ class AgentClient(object):
try:
float(self.interval)
except:
raise ValueError("Monitoring interval should be a number: '%s'" %
self.interval)
raise ValueError(
"Monitoring interval should be a number: '%s'" % self.interval)
cfg = ConfigParser.ConfigParser()
cfg.add_section('main')
@ -136,8 +137,8 @@ class AgentClient(object):
def install(self, loglevel):
"""Create folder and copy agent and metrics scripts to remote host"""
logger.info("Installing monitoring agent at %s@%s...", self.username,
self.host)
logger.info(
"Installing monitoring agent at %s@%s...", self.username, self.host)
# create remote temp dir
cmd = self.python + ' -c "import tempfile; print tempfile.mkdtemp();"'
@ -145,7 +146,8 @@ class AgentClient(object):
try:
out, errors, err_code = self.ssh.execute(cmd)
except:
logger.error("Failed to install monitoring agent to %s",
logger.error(
"Failed to install monitoring agent to %s",
self.host,
exc_info=True)
return None
@ -154,28 +156,29 @@ class AgentClient(object):
return None
if err_code:
logging.error("Failed to create remote dir via SSH"
" at %s@%s, code %s: %s" % (self.username, self.host,
err_code, out.strip()))
logging.error(
"Failed to create remote dir via SSH"
" at %s@%s, code %s: %s" %
(self.username, self.host, err_code, out.strip()))
return None
remote_dir = out.strip()
if remote_dir:
self.path['AGENT_REMOTE_FOLDER'] = remote_dir
logger.debug("Remote dir at %s:%s", self.host,
self.path['AGENT_REMOTE_FOLDER'])
logger.debug(
"Remote dir at %s:%s", self.host, self.path['AGENT_REMOTE_FOLDER'])
# Copy agent and config
agent_config = self.create_agent_config(loglevel)
try:
self.ssh.send_file(self.path['AGENT_LOCAL_FOLDER'] + '/agent.py',
self.ssh.send_file(
self.path['AGENT_LOCAL_FOLDER'] + '/agent.py',
self.path['AGENT_REMOTE_FOLDER'] + '/agent.py')
self.ssh.send_file(agent_config,
self.path['AGENT_REMOTE_FOLDER'] + '/agent.cfg')
self.ssh.send_file(
agent_config, self.path['AGENT_REMOTE_FOLDER'] + '/agent.cfg')
except:
logger.error("Failed to install agent on %s",
self.host,
exc_info=True)
logger.error(
"Failed to install agent on %s", self.host, exc_info=True)
return None
return agent_config
@ -186,12 +189,12 @@ class AgentClient(object):
if self.session:
self.session.send("stop\n")
self.session.close()
fhandle, log_filename = tempfile.mkstemp('.log',
"agent_" + self.host + "_")
fhandle, log_filename = tempfile.mkstemp(
'.log', "agent_" + self.host + "_")
os.close(fhandle)
try:
self.ssh.get_file(self.path['AGENT_REMOTE_FOLDER'] + "_agent.log",
log_filename)
self.ssh.get_file(
self.path['AGENT_REMOTE_FOLDER'] + "_agent.log", log_filename)
self.ssh.rm_r(self.path['AGENT_REMOTE_FOLDER'])
except:
logger.error("Exception while uninstalling agent", exc_info=True)
@ -265,9 +268,7 @@ class MonitoringCollector(object):
logger.debug("Got data from agent: %s", data.strip())
self.send_data.append(
self.filter_unused_data(
self.filter_conf, self.filter_mask, data
)
)
self.filter_conf, self.filter_mask, data))
logger.debug("Data after filtering: %s", self.send_data)
if not self.first_data_received and self.send_data:
@ -286,8 +287,10 @@ class MonitoringCollector(object):
def send_collected_data(self):
"""sends pending data set to listeners"""
[listener.monitoring_data(self.send_data)
for listener in self.listeners]
[
listener.monitoring_data(self.send_data)
for listener in self.listeners
]
self.send_data = []
def get_host_config(self, host, target_hint):
@ -307,14 +310,18 @@ class MonitoringCollector(object):
hostname = host.get('address').lower()
if hostname == '[target]':
if not target_hint:
raise ValueError("Can't use [target] keyword with "
raise ValueError(
"Can't use [target] keyword with "
"no target parameter specified")
logger.debug("Using target hint: %s", target_hint)
hostname = target_hint.lower()
stats = []
startups = []
shutdowns = []
custom = {'tail': [], 'call': [], }
custom = {
'tail': [],
'call': [],
}
metrics_count = 0
for metric in host:
# known metrics
@ -337,7 +344,8 @@ class MonitoringCollector(object):
isdiff = metric.get('diff')
if not isdiff:
isdiff = 0
stat = "%s:%s:%s" % (base64.b64encode(metric.get('label')),
stat = "%s:%s:%s" % (
base64.b64encode(metric.get('label')),
base64.b64encode(metric.text), isdiff)
stats.append('Custom:' + stat)
custom[metric.get('measure', 'call')].append(stat)
@ -379,7 +387,9 @@ class MonitoringCollector(object):
'shutdowns': shutdowns,
# XXX: should be separate?
'stats': {hostname: stats},
'stats': {
hostname: stats
},
}
def getconfig(self, filename, target_hint):
@ -414,7 +424,8 @@ class MonitoringCollector(object):
try:
res.append(filter_list[key])
except IndexError:
logger.warn("Problems filtering data: %s with %s", mask,
logger.warn(
"Problems filtering data: %s with %s", mask,
len(filter_list))
return None
return ';'.join(res)
@ -527,8 +538,7 @@ class MonitoringDataDecoder(object):
self.metrics[host] = []
for metric in data:
if metric.startswith("Custom:"):
metric = base64.standard_b64decode(metric.split(':')[
1])
metric = base64.standard_b64decode(metric.split(':')[1])
self.metrics[host].append(metric)
data_dict[metric] = self.NA
is_initial = True
@ -537,11 +547,12 @@ class MonitoringDataDecoder(object):
timestamp = data.pop(0)
if host not in self.metrics.keys():
raise ValueError("Host %s not in started metrics: %s" %
(host, self.metrics))
raise ValueError(
"Host %s not in started metrics: %s" % (host, self.metrics))
if len(self.metrics[host]) != len(data):
raise ValueError("Metrics len and data len differs: %s vs %s" %
raise ValueError(
"Metrics len and data len differs: %s vs %s" %
(len(self.metrics[host]), len(data)))
for metric in self.metrics[host]:
@ -550,4 +561,5 @@ class MonitoringDataDecoder(object):
logger.debug("Decoded data %s: %s", host, data_dict)
return host, data_dict, is_initial, timestamp
# FIXME: 3 synchronize times between agent and collector better

38
yandextank/plugins/Monitoring/plugin.py
View File

@ -43,8 +43,8 @@ class Plugin(AbstractPlugin):
def start_test(self):
if self.monitoring:
self.monitoring.load_start_time = time.time()
logger.debug("load_start_time = %s" %
self.monitoring.load_start_time)
logger.debug(
"load_start_time = %s" % self.monitoring.load_start_time)
def get_available_options(self):
return ["config", "default_target", 'ssh_timeout']
@ -67,8 +67,8 @@ class Plugin(AbstractPlugin):
self.config = xmlfile
if not os.path.exists(self.config):
raise OSError("Monitoring config file not found: %s" %
self.config)
raise OSError(
"Monitoring config file not found: %s" % self.config)
if self.config == 'none':
self.monitoring = None
@ -99,8 +99,8 @@ class Plugin(AbstractPlugin):
info = phantom.get_info()
if info:
self.default_target = info.address
logger.debug("Changed monitoring target to %s",
self.default_target)
logger.debug(
"Changed monitoring target to %s", self.default_target)
except KeyError as ex:
logger.debug("Phantom plugin not found: %s", ex)
@ -192,8 +192,8 @@ class SaveMonToFile(MonitoringDataListener):
self.store.close()
class MonitoringWidget(AbstractInfoWidget, MonitoringDataListener,
MonitoringDataDecoder):
class MonitoringWidget(
AbstractInfoWidget, MonitoringDataListener, MonitoringDataDecoder):
"""
Screen widget
"""
@ -252,8 +252,8 @@ class MonitoringWidget(AbstractInfoWidget, MonitoringDataListener,
res = "Monitoring is " + screen.markup.GREEN + \
"online" + screen.markup.RESET + ":\n"
for hostname, metrics in self.data.items():
tm_stamp = datetime.datetime.fromtimestamp(float(self.time[
hostname])).strftime('%H:%M:%S')
tm_stamp = datetime.datetime.fromtimestamp(
float(self.time[hostname])).strftime('%H:%M:%S')
res += (
" " + screen.markup.CYAN + "%s" + screen.markup.RESET +
" at %s:\n") % (hostname, tm_stamp)
@ -269,8 +269,8 @@ class MonitoringWidget(AbstractInfoWidget, MonitoringDataListener,
return res.strip()
class AbstractMetricCriterion(AbstractCriterion, MonitoringDataListener,
MonitoringDataDecoder):
class AbstractMetricCriterion(
AbstractCriterion, MonitoringDataListener, MonitoringDataDecoder):
""" Parent class for metric criterion """
def __init__(self, autostop, param_str):
@ -290,8 +290,7 @@ class AbstractMetricCriterion(AbstractCriterion, MonitoringDataListener,
self.host = param_str.split(',')[0].strip()
self.metric = param_str.split(',')[1].strip()
self.value_limit = float(param_str.split(',')[2])
self.seconds_limit = expand_to_seconds(param_str.split(',')[
3])
self.seconds_limit = expand_to_seconds(param_str.split(',')[3])
self.last_second = None
self.seconds_count = 0
@ -311,8 +310,9 @@ class AbstractMetricCriterion(AbstractCriterion, MonitoringDataListener,
self.metric] == self.NA:
data[self.metric] = 0
logger.debug("Compare %s %s/%s=%s to %s", self.get_type_string(),
host, self.metric, data[self.metric],
logger.debug(
"Compare %s %s/%s=%s to %s",
self.get_type_string(), host, self.metric, data[self.metric],
self.value_limit)
if self.comparison_fn(float(data[self.metric]), self.value_limit):
if not self.seconds_count:
@ -358,7 +358,8 @@ class MetricHigherCriterion(AbstractMetricCriterion):
return "%s/%s metric value is higher than %s for %s seconds" % items
def widget_explain(self):
items = (self.host, self.metric, self.value_limit, self.seconds_count,
items = (
self.host, self.metric, self.value_limit, self.seconds_count,
self.seconds_limit)
return "%s/%s > %s for %s/%ss" % items, float(
self.seconds_count) / self.seconds_limit
@ -385,7 +386,8 @@ class MetricLowerCriterion(AbstractMetricCriterion):
return "%s/%s metric value is lower than %s for %s seconds" % items
def widget_explain(self):
items = (self.host, self.metric, self.value_limit, self.seconds_count,
items = (
self.host, self.metric, self.value_limit, self.seconds_count,
self.seconds_limit)
return "%s/%s < %s for %s/%ss" % items, float(
self.seconds_count) / self.seconds_limit

68
yandextank/plugins/Overload/client.py
View File

@ -10,7 +10,6 @@ logger = logging.getLogger(__name__) # pylint: disable=C0103
class OverloadClient(object):
def __init__(self):
self.address = None
self.token = None
@ -77,7 +76,8 @@ class OverloadClient(object):
def get_task_data(self, task):
return self.get("api/task/" + task + "/summary.json")
def new_job(self, task, person, tank, target_host, target_port, loadscheme,
def new_job(
self, task, person, tank, target_host, target_port, loadscheme,
detailed_time, notify_list):
data = {
'task': task,
@ -110,20 +110,24 @@ class OverloadClient(object):
def get_job_summary(self, jobno):
result = self.get(
'api/job/' + str(jobno) +
"/summary.json?api_token=" + self.api_token)
'api/job/' + str(jobno) + "/summary.json?api_token=" +
self.api_token)
return result[0]
def close_job(self, jobno, retcode):
params = {'exitcode': str(retcode), 'api_token': self.api_token, }
params = {
'exitcode': str(retcode),
'api_token': self.api_token,
}
result = self.get('api/job/' + str(jobno) + '/close.json?' +
urllib.urlencode(params))
result = self.get(
'api/job/' + str(jobno) + '/close.json?' + urllib.urlencode(params))
return result[0]['success']
def edit_job_metainfo(self, jobno, job_name, job_dsc, instances, ammo_path,
loop_count, version_tested, is_regression, component,
tank_type, cmdline, is_starred):
def edit_job_metainfo(
self, jobno, job_name, job_dsc, instances, ammo_path, loop_count,
version_tested, is_regression, component, tank_type, cmdline,
is_starred):
data = {
'name': job_name,
'description': job_dsc,
@ -151,11 +155,8 @@ class OverloadClient(object):
data['description'] = comment.strip()
response = self.post(
'api/job/' +
str(jobno) +
"/set_imbalance.json?api_token=" +
self.api_token,
data)
'api/job/' + str(jobno) + "/set_imbalance.json?api_token=" +
self.api_token, data)
return response
def second_data_to_push_item(self, data, stat, timestamp, overall, case):
@ -191,20 +192,22 @@ class OverloadClient(object):
}
}
for q, value in zip(data["interval_real"]["q"]["q"],
for q, value in zip(
data["interval_real"]["q"]["q"],
data["interval_real"]["q"]["value"]):
api_data['trail']['q' + str(q)] = value / 1000.0
for code, cnt in data["net_code"]["count"].iteritems():
api_data['net_codes'].append({'code': int(code),
'count': int(cnt)})
api_data['net_codes'].append({'code': int(code), 'count': int(cnt)})
for code, cnt in data["proto_code"]["count"].iteritems():
api_data['http_codes'].append({'code': int(code),
'count': int(cnt)})
api_data['http_codes'].append({
'code': int(code),
'count': int(cnt)
})
api_data['time_intervals'] = self.convert_hist(data["interval_real"][
"hist"])
api_data['time_intervals'] = self.convert_hist(
data["interval_real"]["hist"])
return api_data
def convert_hist(self, hist):
@ -228,11 +231,11 @@ class OverloadClient(object):
case_name = "__EMPTY__"
if (len(case_name)) > 128:
raise RuntimeError('tag (case) name is too long: ' + case_name)
push_item = self.second_data_to_push_item(case_data, stat_item, ts,
0, case_name)
push_item = self.second_data_to_push_item(
case_data, stat_item, ts, 0, case_name)
items.append(push_item)
overall = self.second_data_to_push_item(data_item["overall"],
stat_item, ts, 1, '')
overall = self.second_data_to_push_item(
data_item["overall"], stat_item, ts, 1, '')
items.append(overall)
while True:
@ -252,7 +255,8 @@ class OverloadClient(object):
"Retry in 10 sec: %s", ex)
time.sleep(10) # FIXME this makes all plugins freeze
except requests.exceptions.RequestException as ex:
logger.warn("Failed to push second data to API,"
logger.warn(
"Failed to push second data to API,"
" retry in 10 sec: %s", ex)
time.sleep(10) # FIXME this makes all plugins freeze
except Exception: # pylint: disable=W0703
@ -288,7 +292,8 @@ class OverloadClient(object):
' retry in 10s: %s', ex)
time.sleep(10) # FIXME this makes all plugins freeze
except requests.exceptions.RequestException as ex:
logger.warning('Problems sending monitoring data,'
logger.warning(
'Problems sending monitoring data,'
' retry in 10s: %s', ex)
time.sleep(10) # FIXME this makes all plugins freeze
except Exception: # pylint: disable=W0703
@ -298,13 +303,12 @@ class OverloadClient(object):
return
def send_console(self, jobno, console):
logger.debug("Sending console view [%s]: %s", len(console),
console[:64])
logger.debug(
"Sending console view [%s]: %s", len(console), console[:64])
addr = ("api/job/%s/console.txt?api_token=" % jobno) + self.api_token,
self.post_raw(addr, {"console": console, })
def send_config_snapshot(self, jobno, config):
logger.debug("Sending config snapshot")
addr = ("api/job/%s/configinfo.txt?api_token=" %
jobno) + self.api_token
addr = ("api/job/%s/configinfo.txt?api_token=" % jobno) + self.api_token
self.post_raw(addr, {"configinfo": config, })

86
yandextank/plugins/Overload/plugin.py
View File

@ -58,17 +58,21 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
return __file__
def get_available_options(self):
opts = ["api_address",
opts = [
"api_address",
"task",
"job_name",
"job_dsc",
"notify",
"ver", ]
opts += ["component",
"ver",
]
opts += [
"component",
"regress",
"operator",
"copy_config_to",
"jobno_file", ]
"jobno_file",
]
opts += ["token_file"]
return opts
@ -79,20 +83,22 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
try:
with open(filename, 'r') as handle:
data = handle.read().strip()
logger.info("Read authentication token from %s, "
"token length is %d bytes", filename,
len(str(data)))
except IOError:
logger.error("Failed to read Overload API token from %s",
filename)
logger.info(
"Get your Overload API token from https://overload.yandex.net and provide it via 'overload.token_file' parameter")
"Read authentication token from %s, "
"token length is %d bytes", filename, len(str(data)))
except IOError:
logger.error(
"Failed to read Overload API token from %s", filename)
logger.info(
"Get your Overload API token from https://overload.yandex.net and provide it via 'overload.token_file' parameter"
)
raise RuntimeError("API token error")
return data
else:
logger.error("Overload API token filename is not defined")
logger.info(
"Get your Overload API token from https://overload.yandex.net and provide it via 'overload.token_file' parameter")
"Get your Overload API token from https://overload.yandex.net and provide it via 'overload.token_file' parameter"
)
raise RuntimeError("API token error")
def configure(self):
@ -102,17 +108,13 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
self.api_client.set_api_address(self.get_option("api_address"))
self.api_client.set_api_timeout(self.get_option("api_timeout", 30))
self.api_client.set_api_token(
self.read_token(
self.get_option(
"token_file", "")))
self.read_token(self.get_option("token_file", "")))
self.task = self.get_option("task", "DEFAULT")
self.job_name = unicode(
self.get_option(
"job_name",
"none").decode("utf8"))
self.get_option("job_name", "none").decode("utf8"))
if self.job_name == "ask" and sys.stdin.isatty():
self.job_name = unicode(raw_input(
"Please, enter job_name: ").decode("utf8"))
self.job_name = unicode(
raw_input("Please, enter job_name: ").decode("utf8"))
self.job_dsc = unicode(self.get_option("job_dsc", "").decode("utf8"))
if self.job_dsc == "ask" and sys.stdin.isatty():
self.job_dsc = unicode(
@ -204,8 +206,9 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
logger.info("Detected target: %s", self.target)
self.jobno = self.api_client.new_job(
self.task, self.operator, socket.getfqdn(), self.target, port,
loadscheme, detailed_field, self.notify_list)
self.task, self.operator,
socket.getfqdn(), self.target, port, loadscheme, detailed_field,
self.notify_list)
web_link = "%s%s" % (self.api_client.address, self.jobno)
logger.info("Web link: %s", web_link)
self.publish("jobno", self.jobno)
@ -250,11 +253,11 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
if autostop and autostop.cause_criterion:
rps = 0
if autostop.cause_criterion.cause_second:
rps = autostop.cause_criterion.cause_second[
1]["metrics"]["reqps"]
rps = autostop.cause_criterion.cause_second[1]["metrics"][
"reqps"]
if not rps:
rps = autostop.cause_criterion.cause_second[
0]["overall"]["interval_real"]["len"]
rps = autostop.cause_criterion.cause_second[0][
"overall"]["interval_real"]["len"]
self.api_client.set_imbalance_and_dsc(
self.jobno, rps, autostop.cause_criterion.explain())
@ -289,8 +292,10 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
self.api_client.push_monitoring_data(
self.jobno, json.dumps(data_list))
elif "Monitoring" in self.core.job.monitoring_plugin.__module__:
[self.api_client.push_monitoring_data(
self.jobno, data) for data in data_list if data]
[
self.api_client.push_monitoring_data(self.jobno, data)
for data in data_list if data
]
else:
logger.warn("The test was stopped from Web interface")
@ -305,7 +310,8 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
config_filename = mon.config
if config_filename and config_filename not in ['none', 'auto']:
with open(config_filename) as config_file:
config.set(MonitoringPlugin.SECTION, "config_contents",
config.set(
MonitoringPlugin.SECTION, "config_contents",
config_file.read())
except Exception: # pylint: disable=W0703
logger.debug("Can't get monitoring config", exc_info=True)
@ -314,8 +320,8 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
config.write(output)
if self.jobno:
try:
self.api_client.send_config_snapshot(self.jobno,
output.getvalue())
self.api_client.send_config_snapshot(
self.jobno, output.getvalue())
except Exception: # pylint: disable=W0703
logger.debug("Can't send config snapshot: %s", exc_info=True)
@ -329,11 +335,7 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
PLUGIN_DIR = os.path.join(self.core.artifacts_base_dir, self.SECTION)
if not os.path.exists(PLUGIN_DIR):
os.makedirs(PLUGIN_DIR)
os.symlink(
self.core.artifacts_dir,
os.path.join(
PLUGIN_DIR,
str(name)))
os.symlink(self.core.artifacts_dir, os.path.join(PLUGIN_DIR, str(name)))
def _core_with_tank_api(self):
"""
@ -347,13 +349,12 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
else:
api_found = isinstance(self.core, yandex_tank_api.worker.TankCore)
logger.debug(
"We are%s running under API server",
"" if api_found else " likely not")
"We are%s running under API server", ""
if api_found else " likely not")
return api_found
class JobInfoWidget(AbstractInfoWidget):
def __init__(self, sender):
AbstractInfoWidget.__init__(self)
self.owner = sender
@ -365,8 +366,9 @@ class JobInfoWidget(AbstractInfoWidget):
template = "Author: " + screen.markup.RED + "%s" + \
screen.markup.RESET + \
"%s\n Job: %s %s\n Web: %s%s"
data = (self.owner.operator[:1], self.owner.operator[1:],
self.owner.jobno, self.owner.job_name,
self.owner.api_client.address, self.owner.jobno)
data = (
self.owner.operator[:1], self.owner.operator[1:], self.owner.jobno,
self.owner.job_name, self.owner.api_client.address,
self.owner.jobno)
return template % data

19
yandextank/plugins/Pandora/config.py
View File

@ -28,7 +28,10 @@ def linear_schedule(start_rps, end_rps, period):
def unlimited_schedule(*args):
return {"LimiterType": "unlimited", "Parameters": {}, }
return {
"LimiterType": "unlimited",
"Parameters": {},
}
step_producers = {
@ -40,8 +43,8 @@ step_producers = {
def parse_schedule(schedule):
steps = [
step.strip()
for step in " ".join(schedule.split("\n")).split(')') if step.strip()
step.strip() for step in " ".join(schedule.split("\n")).split(')')
if step.strip()
]
if len(steps) > 1:
raise NotImplementedError("Composite schedules not implemented yet")
@ -50,12 +53,11 @@ def parse_schedule(schedule):
if schedule_type in step_producers:
return step_producers[schedule_type](*params)
else:
raise NotImplementedError("Step of type %s is not implemented" %
schedule_type)
raise NotImplementedError(
"Step of type %s is not implemented" % schedule_type)
class PandoraConfig(object):
def __init__(self):
self.pools = []
@ -70,10 +72,9 @@ class PandoraConfig(object):
class PoolConfig(object):
def __init__(self):
self.config = json.loads(resource_string(
__name__, 'config/pandora_pool_default.json'))
self.config = json.loads(
resource_string(__name__, 'config/pandora_pool_default.json'))
def set_ammo(self, ammo):
self.config["AmmoProvider"]["AmmoSource"] = ammo

22
yandextank/plugins/Pandora/plugin.py
View File

@ -34,16 +34,17 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
return __file__
def get_available_options(self):
opts = ["pandora_cmd", "buffered_seconds", "ammo", "loop",
"sample_log", "config_file", "startup_schedule",
"user_schedule", "gun_type"]
opts = [
"pandora_cmd", "buffered_seconds", "ammo", "loop", "sample_log",
"config_file", "startup_schedule", "user_schedule", "gun_type"
]
return opts
def configure(self):
# plugin part
self.pandora_cmd = self.get_option("pandora_cmd", "pandora")
self.buffered_seconds = int(self.get_option("buffered_seconds",
self.buffered_seconds))
self.buffered_seconds = int(
self.get_option("buffered_seconds", self.buffered_seconds))
pool_config = PoolConfig()
@ -94,8 +95,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
self.pandora_config_file = self.get_option("config_file", "")
if not self.pandora_config_file:
self.pandora_config_file = self.core.mkstemp(".json",
"pandora_config_")
self.pandora_config_file = self.core.mkstemp(
".json", "pandora_config_")
self.core.add_artifact_file(self.pandora_config_file)
with open(self.pandora_config_file, 'w') as config_file:
config_file.write(self.pandora_config.json())
@ -133,7 +134,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
process_stderr_file = self.core.mkstemp(".log", "pandora_")
self.core.add_artifact_file(process_stderr_file)
self.process_stderr = open(process_stderr_file, 'w')
self.process = subprocess.Popen(args,
self.process = subprocess.Popen(
args,
stderr=self.process_stderr,
stdout=self.process_stderr,
close_fds=True)
@ -148,8 +150,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
def end_test(self, retcode):
if self.process and self.process.poll() is None:
logger.warn("Terminating worker process with PID %s",
self.process.pid)
logger.warn(
"Terminating worker process with PID %s", self.process.pid)
self.process.terminate()
if self.process_stderr:
self.process_stderr.close()

26
yandextank/plugins/Pandora/reader.py
View File

@ -13,25 +13,31 @@ class PandoraStatsReader(object):
pandora_response = requests.get("http://localhost:1234/debug/vars")
pandora_stat = pandora_response.json()
return [{'ts': int(time.time() - 1),
return [{
'ts': int(time.time() - 1),
'metrics': {
'instances':
pandora_stat.get("engine_ActiveRequests"),
'instances': pandora_stat.get("engine_ActiveRequests"),
'reqps': pandora_stat.get("engine_ReqPS"),
}}]
}
}]
except requests.ConnectionError:
logger.info("Pandora expvar http interface is unavailable")
except requests.HTTPError:
logger.warning("Pandora expvar http interface is unavailable",
exc_info=True)
logger.warning(
"Pandora expvar http interface is unavailable", exc_info=True)
except Exception:
logger.warning("Couldn't decode pandora stat:\n%s\n",
logger.warning(
"Couldn't decode pandora stat:\n%s\n",
pandora_response.text,
exc_info=True)
return [{'ts': int(time.time() - 1),
'metrics': {'instances': 0,
'reqps': 0}}]
return [{
'ts': int(time.time() - 1),
'metrics': {
'instances': 0,
'reqps': 0
}
}]
def close(self):
pass

82
yandextank/plugins/Phantom/plugin.py
View File

@ -57,8 +57,9 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
return __file__
def get_available_options(self):
opts = ["phantom_path", "buffered_seconds", "exclude_markers",
"affinity"]
opts = [
"phantom_path", "buffered_seconds", "exclude_markers", "affinity"
]
opts += [PhantomConfig.OPTION_PHOUT, self.OPTION_CONFIG]
opts += PhantomConfig.get_available_options()
return opts
@ -68,11 +69,11 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
self.config = self.get_option(self.OPTION_CONFIG, '')
self.phantom_path = self.get_option("phantom_path", 'phantom')
self.enum_ammo = self.get_option("enum_ammo", False)
self.buffered_seconds = int(self.get_option("buffered_seconds",
self.buffered_seconds))
self.exclude_markers = set(filter(
(lambda marker: marker != ''), self.get_option('exclude_markers',
[]).split(' ')))
self.buffered_seconds = int(
self.get_option("buffered_seconds", self.buffered_seconds))
self.exclude_markers = set(
filter((lambda marker: marker != ''),
self.get_option('exclude_markers', []).split(' ')))
self.taskset_affinity = self.get_option('affinity', '')
try:
@ -83,8 +84,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
"No autostop plugin found, not adding instances criterion")
self.predefined_phout = self.get_option(PhantomConfig.OPTION_PHOUT, '')
if not self.get_option(self.OPTION_CONFIG,
'') and self.predefined_phout:
if not self.get_option(
self.OPTION_CONFIG, '') and self.predefined_phout:
self.phout_import_mode = True
if not self.config and not self.phout_import_mode:
@ -111,23 +112,26 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
"Config check failed. Subprocess returned code %s" %
retcode)
if result[2]:
raise RuntimeError("Subprocess returned message: %s" %
result[2])
raise RuntimeError(
"Subprocess returned message: %s" % result[2])
reader = PhantomReader(self.phantom.phout_file)
logger.debug("Linking sample reader to aggregator."
logger.debug(
"Linking sample reader to aggregator."
" Reading samples from %s", self.phantom.phout_file)
logger.debug("Linking stats reader to aggregator."
logger.debug(
"Linking stats reader to aggregator."
" Reading stats from %s", self.phantom.stat_log)
else:
reader = PhantomReader(self.predefined_phout)
logger.debug("Linking sample reader to aggregator."
logger.debug(
"Linking sample reader to aggregator."
" Reading samples from %s", self.predefined_phout)
if aggregator:
aggregator.reader = reader
info = self.phantom.get_info()
aggregator.stats_reader = PhantomStatsReader(self.phantom.stat_log,
info)
aggregator.stats_reader = PhantomStatsReader(
self.phantom.stat_log, info)
aggregator.add_result_listener(self)
try:
@ -151,26 +155,29 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
def start_test(self):
if not self.phout_import_mode:
args = [self.phantom_path, 'run', self.config]
logger.debug("Starting %s with arguments: %s", self.phantom_path,
args)
logger.debug(
"Starting %s with arguments: %s", self.phantom_path, args)
if self.taskset_affinity != '':
args = [self.core.taskset_path, '-c', self.taskset_affinity
args = [
self.core.taskset_path, '-c', self.taskset_affinity
] + args
logger.debug("Enabling taskset for phantom with affinity: %s,"
" cores count: %d", self.taskset_affinity,
self.cpu_count)
logger.debug(
"Enabling taskset for phantom with affinity: %s,"
" cores count: %d", self.taskset_affinity, self.cpu_count)
self.phantom_start_time = time.time()
phantom_stderr_file = self.core.mkstemp(".log",
"phantom_stdout_stderr_")
phantom_stderr_file = self.core.mkstemp(
".log", "phantom_stdout_stderr_")
self.core.add_artifact_file(phantom_stderr_file)
self.phantom_stderr = open(phantom_stderr_file, 'w')
self.process = subprocess.Popen(args,
self.process = subprocess.Popen(
args,
stderr=self.phantom_stderr,
stdout=self.phantom_stderr,
close_fds=True)
else:
if not os.path.exists(self.predefined_phout):
raise RuntimeError("Phout file not exists for import: %s" %
raise RuntimeError(
"Phout file not exists for import: %s" %
self.predefined_phout)
logger.warn(
"Will import phout file instead of running phantom: %s",
@ -180,14 +187,13 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
if not self.phout_import_mode:
retcode = self.process.poll()
if retcode is not None:
logger.info("Phantom done its work with exit code: %s",
retcode)
logger.info("Phantom done its work with exit code: %s", retcode)
return abs(retcode)
else:
info = self.get_info()
if info:
eta = int(info.duration) - (int(time.time()) -
int(self.phantom_start_time))
eta = int(info.duration) - (
int(time.time()) - int(self.phantom_start_time))
self.publish('eta', eta)
return -1
else:
@ -199,8 +205,8 @@ class Plugin(AbstractPlugin, GeneratorPlugin):
def end_test(self, retcode):
if self.process and self.process.poll() is None:
logger.warn("Terminating phantom process with PID %s",
self.process.pid)
logger.warn(
"Terminating phantom process with PID %s", self.process.pid)
self.process.terminate()
if self.process:
self.process.communicate()
@ -255,8 +261,7 @@ class UsedInstancesCriterion(AbstractCriterion):
else:
self.level = int(level_str)
self.is_relative = False
self.seconds_limit = expand_to_seconds(param_str.split(',')[
1])
self.seconds_limit = expand_to_seconds(param_str.split(',')[1])
try:
phantom = autostop.core.get_plugin_of_type(Plugin)
@ -264,7 +269,8 @@ class UsedInstancesCriterion(AbstractCriterion):
if info:
self.threads_limit = info.instances
if not self.threads_limit:
raise ValueError("Cannot create 'instances' criterion"
raise ValueError(
"Cannot create 'instances' criterion"
" with zero instances limit")
except KeyError:
logger.warning("No phantom module, 'instances' autostop disabled")
@ -302,9 +308,11 @@ class UsedInstancesCriterion(AbstractCriterion):
return level_str
def explain(self):
items = (self.get_level_str(), self.seconds_count,
items = (
self.get_level_str(), self.seconds_count,
self.cause_second[0].get('ts'))
return ("Testing threads (instances) utilization"
return (
"Testing threads (instances) utilization"
" higher than %s for %ss, since %s" % items)
def widget_explain(self):

31
yandextank/plugins/Phantom/reader.py
View File

@ -10,7 +10,6 @@ import datetime
import itertools as itt
from StringIO import StringIO
logger = logging.getLogger(__name__)
phout_columns = [
@ -38,10 +37,7 @@ dtypes = {
def string_to_df(data):
start_time = time.time()
chunk = pd.read_csv(
StringIO(data),
sep='\t',
names=phout_columns,
dtype=dtypes)
StringIO(data), sep='\t', names=phout_columns, dtype=dtypes)
chunk['receive_ts'] = chunk.send_ts + chunk.interval_real / 1e6
chunk['receive_sec'] = chunk.receive_ts.astype(np.int64)
@ -49,13 +45,11 @@ def string_to_df(data):
chunk['tag'] = chunk.tag.str.rsplit('#', 1, expand=True)[0]
chunk.set_index(['receive_sec'], inplace=True)
logger.debug("Chunk decode time: %.2fms",
(time.time() - start_time) * 1000)
logger.debug("Chunk decode time: %.2fms", (time.time() - start_time) * 1000)
return chunk
class PhantomReader(object):
def __init__(self, filename, cache_size=1024 * 1024 * 50):
self.buffer = ""
self.phout = open(filename, 'r')
@ -87,7 +81,6 @@ class PhantomReader(object):
class PhantomStatsReader(object):
def __init__(self, filename, phantom_info):
self.phantom_info = phantom_info
self.buffer = ""
@ -116,9 +109,13 @@ class PhantomStatsReader(object):
reqps = 0
if offset >= 0 and offset < len(self.phantom_info.steps):
reqps = self.phantom_info.steps[offset][0]
yield {'ts': chunk_date - 1,
'metrics': {'instances': instances,
'reqps': reqps}}
yield {
'ts': chunk_date - 1,
'metrics': {
'instances': instances,
'reqps': reqps
}
}
def _read_stat_data(self, stat_file):
chunk = stat_file.read(1024 * 1024 * 50)
@ -128,10 +125,12 @@ class PhantomStatsReader(object):
if len(parts) > 1:
ready_chunk = parts[0]
self.stat_buffer = parts[1]
chunks = [json.loads('{%s}}' % s)
for s in ready_chunk.split('\n},')]
return list(itt.chain(*(self._decode_stat_data(chunk)
for chunk in chunks)))
chunks = [
json.loads('{%s}}' % s) for s in ready_chunk.split('\n},')
]
return list(
itt.chain(
*(self._decode_stat_data(chunk) for chunk in chunks)))
else:
self.stat_buffer += stat_file.readline()

8
yandextank/plugins/Phantom/tests/test_reader.py
View File

@ -4,16 +4,14 @@ from yandextank.plugins.Phantom.reader import PhantomReader
class TestPhantomReader(object):
def test_read_all(self):
reader = PhantomReader(
'yandextank/plugins/Phantom/tests/phout.dat',
cache_size=1024)
'yandextank/plugins/Phantom/tests/phout.dat', cache_size=1024)
df = pd.DataFrame()
for chunk in reader:
if chunk is None:
reader.close()
else:
df = df.append(chunk)
assert(len(df) == 200)
assert(df['interval_real'].mean() == 11000714.0)
assert (len(df) == 200)
assert (df['interval_real'].mean() == 11000714.0)

67
yandextank/plugins/Phantom/utils.py
View File

@ -42,12 +42,14 @@ class PhantomConfig:
@staticmethod
def get_available_options():
opts = ["threads",
opts = [
"threads",
"phantom_modules_path",
"additional_libs",
"writelog",
"enum_ammo",
"timeout", ]
"timeout",
]
opts += StreamConfig.get_available_options()
return opts
@ -55,8 +57,8 @@ class PhantomConfig:
""" Read phantom tool specific options """
self.threads = self.get_option(
"threads", str(int(multiprocessing.cpu_count() / 2) + 1))
self.phantom_modules_path = self.get_option("phantom_modules_path",
"/usr/lib/phantom")
self.phantom_modules_path = self.get_option(
"phantom_modules_path", "/usr/lib/phantom")
self.additional_libs = self.get_option("additional_libs", "")
self.answ_log_level = self.get_option("writelog", "none")
if self.answ_log_level == '0':
@ -65,16 +67,17 @@ class PhantomConfig:
self.answ_log_level = 'all'
self.timeout = parse_duration(self.get_option("timeout", "11s"))
if self.timeout > 120000:
logger.warning("You've set timeout over 2 minutes."
logger.warning(
"You've set timeout over 2 minutes."
" Are you a functional tester?")
self.answ_log = self.core.mkstemp(".log", "answ_")
self.core.add_artifact_file(self.answ_log)
self.phout_file = self.core.get_option(self.SECTION, self.OPTION_PHOUT,
'')
self.phout_file = self.core.get_option(
self.SECTION, self.OPTION_PHOUT, '')
if not self.phout_file:
self.phout_file = self.core.mkstemp(".log", "phout_")
self.core.set_option(self.SECTION, self.OPTION_PHOUT,
self.phout_file)
self.core.set_option(
self.SECTION, self.OPTION_PHOUT, self.phout_file)
self.core.add_artifact_file(self.phout_file)
self.stat_log = self.core.mkstemp(".log", "phantom_stat_")
self.core.add_artifact_file(self.stat_log)
@ -82,13 +85,16 @@ class PhantomConfig:
self.core.add_artifact_file(self.phantom_log)
main_stream = StreamConfig(
self.core, len(self.streams), self.phout_file, self.answ_log,
self.core,
len(self.streams), self.phout_file, self.answ_log,
self.answ_log_level, self.timeout, self.SECTION)
self.streams.append(main_stream)
for section in self.core.config.find_sections(self.SECTION + '-'):
self.streams.append(StreamConfig(
self.core, len(self.streams), self.phout_file, self.answ_log,
self.streams.append(
StreamConfig(
self.core,
len(self.streams), self.phout_file, self.answ_log,
self.answ_log_level, self.timeout, section))
for stream in self.streams:
@ -175,8 +181,8 @@ class PhantomConfig:
result.ammo_file += stream.stepper_wrapper.ammo_file + ' '
result.ammo_count += stream.stepper_wrapper.ammo_count
result.duration = max(result.duration,
stream.stepper_wrapper.duration)
result.duration = max(
result.duration, stream.stepper_wrapper.duration)
result.instances += stream.instances
if not result.ammo_count:
@ -189,8 +195,8 @@ class StreamConfig:
OPTION_INSTANCES_LIMIT = 'instances'
def __init__(self, core, sequence, phout, answ, answ_level, timeout,
section):
def __init__(
self, core, sequence, phout, answ, answ_level, timeout, section):
self.core = core
self.address_wizard = AddressWizard()
@ -229,10 +235,14 @@ class StreamConfig:
@staticmethod
def get_available_options():
opts = ["ssl", "tank_type", 'gatling_ip', "method_prefix",
"source_log_prefix"]
opts += ["phantom_http_line", "phantom_http_field_num",
"phantom_http_field", "phantom_http_entity"]
opts = [
"ssl", "tank_type", 'gatling_ip', "method_prefix",
"source_log_prefix"
]
opts += [
"phantom_http_line", "phantom_http_field_num", "phantom_http_field",
"phantom_http_entity"
]
opts += ['address', "port", StreamConfig.OPTION_INSTANCES_LIMIT]
opts += StepperWrapper.get_available_options()
opts += ["connection_test"]
@ -245,16 +255,16 @@ class StreamConfig:
self.tank_type = self.get_option("tank_type", 'http')
# TODO: refactor. Maybe we should decide how to interact with
# StepperWrapper here.
self.instances = int(self.get_option(self.OPTION_INSTANCES_LIMIT,
'1000'))
self.instances = int(
self.get_option(self.OPTION_INSTANCES_LIMIT, '1000'))
self.gatling = ' '.join(self.get_option('gatling_ip', '').split("\n"))
self.method_prefix = self.get_option("method_prefix", 'method_stream')
self.method_options = self.get_option("method_options", '')
self.source_log_prefix = self.get_option("source_log_prefix", '')
self.phantom_http_line = self.get_option("phantom_http_line", "")
self.phantom_http_field_num = self.get_option("phantom_http_field_num",
"")
self.phantom_http_field_num = self.get_option(
"phantom_http_field_num", "")
self.phantom_http_field = self.get_option("phantom_http_field", "")
self.phantom_http_entity = self.get_option("phantom_http_entity", "")
@ -264,8 +274,8 @@ class StreamConfig:
self.ipv6, self.resolved_ip, self.port, self.address = self.address_wizard.resolve(
self.address, do_test_connect, explicit_port)
logger.info("Resolved %s into %s:%s", self.address, self.resolved_ip,
self.port)
logger.info(
"Resolved %s into %s:%s", self.address, self.resolved_ip, self.port)
self.client_cipher_suites = self.get_option("client_cipher_suites", "")
self.client_certificate = self.get_option("client_certificate", "")
@ -345,11 +355,12 @@ class StreamConfig:
fname = 'phantom_benchmark_main.tpl'
else:
fname = 'phantom_benchmark_additional.tpl'
template_str = template_str = resource_string(__name__,
"config/" + fname)
template_str = template_str = resource_string(
__name__, "config/" + fname)
tpl = string.Template(template_str)
config = tpl.substitute(kwargs)
return config
# ========================================================================

34
yandextank/plugins/Phantom/widget.py
View File

@ -53,7 +53,8 @@ class PhantomProgressBarWidget(AbstractInfoWidget):
elif self.ammo_progress:
left_part = self.ammo_count - self.ammo_progress
if left_part > 0:
eta_secs = int(float(dur_seconds) / float(self.ammo_progress) *
eta_secs = int(
float(dur_seconds) / float(self.ammo_progress) *
float(left_part))
else:
eta_secs = 0
@ -77,8 +78,8 @@ class PhantomProgressBarWidget(AbstractInfoWidget):
progress_chars += self.krutilka.next()
res += color_bg + progress_chars + screen.markup.RESET + color_fg
res += '~' * (pb_width - int(pb_width *
progress)) + screen.markup.RESET + ' '
res += '~' * (pb_width - int(pb_width * progress)
) + screen.markup.RESET + ' '
res += str_perc + "\n"
eta = 'ETA: %s' % eta_time
@ -123,7 +124,8 @@ class PhantomInfoWidget(AbstractInfoWidget):
info = self.owner.get_info()
if self.owner.phantom:
template = "Hosts: %s => %s:%s\n Ammo: %s\nCount: %s\n Load: %s"
data = (socket.gethostname(), info.address, info.port,
data = (
socket.gethostname(), info.address, info.port,
os.path.basename(info.ammo_file), self.ammo_count,
' '.join(info.rps_schedule))
res = template % data
@ -132,8 +134,7 @@ class PhantomInfoWidget(AbstractInfoWidget):
res += "Active instances: "
if float(self.instances) / self.instances_limit > 0.8:
res += screen.markup.RED + str(
self.instances) + screen.markup.RESET
res += screen.markup.RED + str(self.instances) + screen.markup.RESET
elif float(self.instances) / self.instances_limit > 0.5:
res += screen.markup.YELLOW + str(
self.instances) + screen.markup.RESET
@ -141,8 +142,7 @@ class PhantomInfoWidget(AbstractInfoWidget):
res += str(self.instances)
res += "\nPlanned requests: %s for %s\nActual responses: " % (
self.planned,
datetime.timedelta(seconds=self.planned_rps_duration))
self.planned, datetime.timedelta(seconds=self.planned_rps_duration))
if not self.planned == self.RPS:
res += screen.markup.YELLOW + str(self.RPS) + screen.markup.RESET
else:
@ -150,22 +150,22 @@ class PhantomInfoWidget(AbstractInfoWidget):
res += "\n Accuracy: "
if self.selfload < 80:
res += screen.markup.RED + ('%.2f' %
self.selfload) + screen.markup.RESET
res += screen.markup.RED + (
'%.2f' % self.selfload) + screen.markup.RESET
elif self.selfload < 95:
res += screen.markup.YELLOW + ('%.2f' %
self.selfload) + screen.markup.RESET
res += screen.markup.YELLOW + (
'%.2f' % self.selfload) + screen.markup.RESET
else:
res += ('%.2f' % self.selfload)
res += "%\n Time lag: "
if self.time_lag > self.owner.buffered_seconds * 5:
logger.debug("Time lag: %s", self.time_lag)
res += screen.markup.RED + str(datetime.timedelta(
seconds=self.time_lag)) + screen.markup.RESET
res += screen.markup.RED + str(
datetime.timedelta(seconds=self.time_lag)) + screen.markup.RESET
elif self.time_lag > self.owner.buffered_seconds:
res += screen.markup.YELLOW + str(datetime.timedelta(
seconds=self.time_lag)) + screen.markup.RESET
res += screen.markup.YELLOW + str(
datetime.timedelta(seconds=self.time_lag)) + screen.markup.RESET
else:
res += str(datetime.timedelta(seconds=self.time_lag))
@ -175,6 +175,8 @@ class PhantomInfoWidget(AbstractInfoWidget):
self.RPS = data["overall"]["interval_real"]["len"]
self.planned = stats["metrics"]["reqps"]
self.instances = stats["metrics"]["instances"]
# TODO:
# self.selfload = second_aggregate_data.overall.selfload
# self.time_lag = int(time.time() - time.mktime(

7
yandextank/plugins/Platform/plugin.py
View File

@ -28,6 +28,7 @@ class Plugin(AbstractPlugin):
def _echo_wrapper(cmd):
return 'echo "====Executing: {cmd}"; {cmd}'.format(cmd=cmd)
cmds = {
"dpkg": "dpkg -l",
"uname": "uname -a",
@ -59,8 +60,7 @@ class Plugin(AbstractPlugin):
self.timeout = int(self.get_option("timeout", 3))
except:
logger.error(
'Exception trying to configure Platform plugin',
exc_info=True)
'Exception trying to configure Platform plugin', exc_info=True)
self.logfile = self.core.mkstemp(".log", "platform_")
self.core.add_artifact_file(self.logfile)
@ -84,8 +84,7 @@ class Plugin(AbstractPlugin):
out, errors, err_code = self.ssh.execute(self.cmd)
except Exception:
logger.warning(
"Failed to check remote system information at %s:%s",
host,
"Failed to check remote system information at %s:%s", host,
self.port)
logger.debug(
"Failed to check remote system information at %s:%s",

18
yandextank/plugins/ResourceCheck/plugin.py
View File

@ -28,8 +28,8 @@ class Plugin(AbstractPlugin):
return ["interval", "disk_limit", "mem_limit"]
def configure(self):
self.interval = expand_to_seconds(self.get_option(
"interval", self.interval))
self.interval = expand_to_seconds(
self.get_option("interval", self.interval))
self.disk_limit = int(self.get_option("disk_limit", self.disk_limit))
self.mem_limit = int(self.get_option("mem_limit", self.mem_limit))
@ -58,18 +58,20 @@ class Plugin(AbstractPlugin):
self.log.debug("No disk usage info: %s", res[2])
return
disk_free = res[1]
self.log.debug("Disk free space: %s/%s", disk_free.strip(),
self.disk_limit)
self.log.debug(
"Disk free space: %s/%s", disk_free.strip(), self.disk_limit)
if int(disk_free.strip()) < self.disk_limit:
raise RuntimeError(
"Not enough local resources: disk space less than %sMB in %s: %sMB" %
(self.disk_limit, self.core.artifacts_base_dir, int(
disk_free.strip())))
"Not enough local resources: disk space less than %sMB in %s: %sMB"
% (
self.disk_limit, self.core.artifacts_base_dir,
int(disk_free.strip())))
def __check_mem(self):
''' raise exception on RAM exceeded '''
mem_free = psutil.virtual_memory().available / 2**20
self.log.debug("Memory free: %s/%s", mem_free, self.mem_limit)
if mem_free < self.mem_limit:
raise RuntimeError("Not enough resources: free memory less "
raise RuntimeError(
"Not enough resources: free memory less "
"than %sMB: %sMB" % (self.mem_limit, mem_free))

13
yandextank/plugins/ShellExec/plugin.py
View File

@ -47,13 +47,14 @@ class Plugin(AbstractPlugin):
def is_test_finished(self):
if self.poll:
self.log.info("Executing: %s", self.poll)
retcode = util.execute(self.poll,
retcode = util.execute(
self.poll,
shell=True,
poll_period=0.1,
catch_out=self.catch_out)[0]
if retcode:
self.log.warn("Non-zero exit code, interrupting test: %s",
retcode)
self.log.warn(
"Non-zero exit code, interrupting test: %s", retcode)
return retcode
return -1
@ -72,9 +73,7 @@ class Plugin(AbstractPlugin):
Execute and check exit code
'''
self.log.info("Executing: %s", cmd)
retcode = util.execute(cmd,
shell=True,
poll_period=0.1,
catch_out=self.catch_out)[0]
retcode = util.execute(
cmd, shell=True, poll_period=0.1, catch_out=self.catch_out)[0]
if retcode:
raise RuntimeError("Subprocess returned %s" % retcode)

39
yandextank/plugins/SvgReport/plugin.py
View File

@ -7,12 +7,18 @@ import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa:E402
_ALL_ = "All"
_CHARTSETS = {
"cpu-cpu-": {"CPU": _ALL_},
"net-": {"Network": {"bytes_sent", "bytes_recv"}},
"diskio-": {"Disk IO": {"read_bytes", "write_bytes"}, "Disk latency": {"read_time", "write_time"}},
"cpu-cpu-": {
"CPU": _ALL_
},
"net-": {
"Network": {"bytes_sent", "bytes_recv"}
},
"diskio-": {
"Disk IO": {"read_bytes", "write_bytes"},
"Disk latency": {"read_time", "write_time"}
},
}
_CUSTOM_PREFIX = "custom:"
_REPORT_FILE_OPTION = "report_file"
@ -33,7 +39,8 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
def configure(self):
self.__report_path = self.get_option(_REPORT_FILE_OPTION, "report.svg")
if os.path.split(self.__report_path)[0] or os.path.splitdrive(self.__report_path)[0]:
if os.path.split(self.__report_path)[0] or os.path.splitdrive(
self.__report_path)[0]:
raise Exception("Only simple file names supported")
self.__shooting_data = []
@ -50,7 +57,8 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
def post_process(self, retcode):
monitoring_chartsets = self.__get_monitoring_chartsets()
min_x = self.__shooting_data[0]["ts"] # sync start of shooting and start of monitoring
min_x = self.__shooting_data[0][
"ts"] # sync start of shooting and start of monitoring
seaborn.set(style="whitegrid", palette="Set2")
seaborn.despine()
@ -67,7 +75,8 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
plt.gca().legend(fontsize="x-small")
# monitoring
for plot_num, chartset_data in enumerate(sorted(monitoring_chartsets.iteritems()), 1):
for plot_num, chartset_data in enumerate(
sorted(monitoring_chartsets.iteritems()), 1):
chartset_title, signals = chartset_data
plt.subplot(plot_count, 1, plot_num + 1)
@ -96,9 +105,12 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
for chartset_prefix, chartset_data in _CHARTSETS.iteritems():
if signal_prefix.startswith(chartset_prefix):
for chartset_title, chartset_signals in chartset_data.iteritems():
for chartset_title, chartset_signals in chartset_data.iteritems(
):
if chartset_signals is _ALL_ or signal_suffix in chartset_signals:
return "{} {}".format(chartset_title, signal_prefix[len(chartset_prefix):])
return "{} {}".format(
chartset_title,
signal_prefix[len(chartset_prefix):])
else:
return None
else:
@ -115,11 +127,13 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
continue
signal_prefix, signal_suffix = signal_name.split("_", 1)
chartset_title = self.__find_monitoring_chartset(signal_prefix, signal_suffix)
chartset_title = self.__find_monitoring_chartset(
signal_prefix, signal_suffix)
if not chartset_title:
continue
chartsets.setdefault((chartset_title), set()).add((signal_name, signal_suffix))
chartsets.setdefault((chartset_title), set()).add(
(signal_name, signal_suffix))
return chartsets
@ -142,7 +156,8 @@ class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
y = {}
for data in self.__shooting_data:
timestamp = data["ts"]
for variant, count in data["overall"][signal_name]["count"].iteritems():
for variant, count in data["overall"][signal_name][
"count"].iteritems():
x.setdefault(variant, []).append(timestamp - min_x)
y.setdefault(variant, []).append(count)
return x, y

83
yandextank/plugins/Telegraf/agent/agent.py
View File

@ -13,7 +13,6 @@ import time
from optparse import OptionParser
import Queue as q
logger = logging.getLogger("agent")
collector_logger = logging.getLogger("telegraf")
@ -82,34 +81,35 @@ class Consolidator(object):
if data['name'] == 'diskio':
data['name'] = "{metric_name}-{disk_id}".format(
metric_name=data['name'],
disk_id=data['tags']['name']
)
disk_id=data['tags']['name'])
elif data['name'] == 'net':
data['name'] = "{metric_name}-{interface}".format(
metric_name=data['name'], interface=data['tags']['interface'])
data[
'name'] = "{metric_name}-{interface}".format(
metric_name=data['name'],
interface=data['tags']['interface'])
elif data['name'] == 'cpu':
data['name'] = "{metric_name}-{cpu_id}".format(
metric_name=data['name'],
cpu_id=data['tags']['cpu']
)
cpu_id=data['tags']['cpu'])
key = data['name'] + "_" + key
if key.endswith('_exec_value'):
key = key.replace('_exec_value', '')
self.results[ts][key] = value
except KeyError:
logger.error(
'Malformed json from source: %s', chunk, exc_info=True)
'Malformed json from source: %s',
chunk,
exc_info=True)
except:
logger.error(
'Something nasty happend in consolidator work',
exc_info=True)
if len(self.results) > 5:
ready_to_go_index = min(self.results)
yield json.dumps(
{
ready_to_go_index: self.results.pop(ready_to_go_index, None)
}
)
yield json.dumps({
ready_to_go_index:
self.results.pop(ready_to_go_index, None)
})
class Drain(threading.Thread):
@ -139,7 +139,6 @@ class Drain(threading.Thread):
class AgentWorker(threading.Thread):
def __init__(self, telegraf_path):
super(AgentWorker, self).__init__()
self.working_dir = os.path.dirname(__file__)
@ -167,8 +166,7 @@ class AgentWorker(threading.Thread):
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
stdin=subprocess.PIPE, )
def read_startup_config(self, cfg_file='agent_startup.cfg'):
try:
@ -188,12 +186,10 @@ class AgentWorker(threading.Thread):
logger.info(
'Successfully loaded startup config.\n'
'Startups: %s\n'
'Shutdowns: %s\n', self.startups, self.shutdowns
)
'Shutdowns: %s\n', self.startups, self.shutdowns)
except:
logger.error(
'Error trying to read agent startup config',
exc_info=True)
'Error trying to read agent startup config', exc_info=True)
def run(self):
logger.info("Running startup commands")
@ -204,9 +200,7 @@ class AgentWorker(threading.Thread):
logger.info('Starting metrics collector..')
cmnd = "{telegraf} -config {working_dir}/agent.cfg".format(
telegraf=self.telegraf_path,
working_dir=self.working_dir
)
telegraf=self.telegraf_path, working_dir=self.working_dir)
self.collector = self.popen(cmnd)
telegraf_output = self.working_dir + '/monitoring.rawdata'
@ -218,23 +212,17 @@ class AgentWorker(threading.Thread):
time.sleep(1)
self.drain = Drain(
Consolidator(
DataReader(telegraf_output)
),
self.results
)
Consolidator(DataReader(telegraf_output)), self.results)
self.drain.start()
self.drain_stdout = Drain(
DataReader(self.collector.stdout, pipe=True),
self.results_stdout
)
DataReader(
self.collector.stdout, pipe=True), self.results_stdout)
self.drain_stdout.start()
self.drain_err = Drain(
DataReader(self.collector.stderr, pipe=True),
self.results_err
)
DataReader(
self.collector.stderr, pipe=True), self.results_err)
self.drain_err.start()
while not self.finished:
@ -243,9 +231,7 @@ class AgentWorker(threading.Thread):
data = self.results.get_nowait()
logger.debug(
'send %s bytes of data to collector', len(data))
sys.stdout.write(
str(data) + '\n'
)
sys.stdout.write(str(data) + '\n')
except q.Empty:
break
except:
@ -297,16 +283,21 @@ class AgentWorker(threading.Thread):
def main():
fname = os.path.dirname(__file__) + "/_agent.log"
logging.basicConfig(
level=logging.DEBUG, filename=fname,
level=logging.DEBUG,
filename=fname,
format='%(asctime)s [%(levelname)s] %(name)s:%(lineno)d %(message)s')
parser = OptionParser()
parser.add_option(
"", "--telegraf", dest="telegraf_path",
"",
"--telegraf",
dest="telegraf_path",
help="telegraf_path",
default="/tmp/telegraf")
parser.add_option(
"", "--host", dest="hostname_path",
"",
"--host",
dest="hostname_path",
help="telegraf_path",
default="/usr/bin/telegraf")
(options, args) = parser.parse_args()
@ -315,24 +306,24 @@ def main():
customs_script = os.path.dirname(__file__) + '/agent_customs.sh'
try:
logger.info(
'Trying to make telegraf executable: %s',
options.telegraf_path)
'Trying to make telegraf executable: %s', options.telegraf_path)
# 0o755 compatible with old python versions. 744 is NOT enough
os.chmod(options.telegraf_path, 493)
except OSError:
logger.warning(
'Unable to set %s access rights to execute.',
options.telegraf_path, exc_info=True)
options.telegraf_path,
exc_info=True)
try:
logger.info(
'Trying to make customs script executable: %s',
customs_script)
'Trying to make customs script executable: %s', customs_script)
# 0o755 compatible with old python versions. 744 is NOT enough
os.chmod(customs_script, 493)
except OSError:
logger.warning(
'Unable to set %s access rights to execute.',
customs_script, exc_info=True)
customs_script,
exc_info=True)
worker = AgentWorker(options.telegraf_path)
worker.read_startup_config()

63
yandextank/plugins/Telegraf/client.py
View File

@ -58,10 +58,8 @@ class LocalhostClient(object):
customs_script = self.config.create_custom_exec_script()
try:
copyfile(
self.path['AGENT_LOCAL_FOLDER'] +
'/agent.py',
self.workdir +
'/agent.py')
self.path['AGENT_LOCAL_FOLDER'] + '/agent.py',
self.workdir + '/agent.py')
copyfile(agent_config, self.workdir + '/agent.cfg')
copyfile(startup_config, self.workdir + '/agent_startup.cfg')
copyfile(customs_script, self.workdir + '/agent_customs.sh')
@ -90,8 +88,7 @@ class LocalhostClient(object):
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
stdin=subprocess.PIPE, )
def start(self):
"""Start local agent"""
@ -102,9 +99,7 @@ class LocalhostClient(object):
telegraf_path=self.path['TELEGRAF_LOCAL_PATH'],
host=self.host)
self.session = self.popen(command)
self.reader_thread = threading.Thread(
target=self.read_buffer
)
self.reader_thread = threading.Thread(target=self.read_buffer)
self.reader_thread.setDaemon(True)
return self.session
@ -123,8 +118,8 @@ class LocalhostClient(object):
except ValueError:
logger.debug(
'this exc most likely raised during interpreter shutdown\n'
'otherwise something really nasty happend', exc_info=True
)
'otherwise something really nasty happend',
exc_info=True)
def uninstall(self):
"""
@ -182,8 +177,8 @@ class SSHClient(object):
def install(self):
"""Create folder and copy agent and metrics scripts to remote host"""
logger.info("Installing monitoring agent at %s@%s...", self.username,
self.host)
logger.info(
"Installing monitoring agent at %s@%s...", self.username, self.host)
# create remote temp dir
cmd = self.python + ' -c "import tempfile; print tempfile.mkdtemp();"'
@ -210,9 +205,7 @@ class SSHClient(object):
if remote_dir:
self.path['AGENT_REMOTE_FOLDER'] = remote_dir
logger.debug(
"Remote dir at %s:%s",
self.host,
self.path['AGENT_REMOTE_FOLDER'])
"Remote dir at %s:%s", self.host, self.path['AGENT_REMOTE_FOLDER'])
# create collector config
agent_config = self.config.create_collector_config(
@ -225,9 +218,7 @@ class SSHClient(object):
# support string formatting without indices
remote_cmd = 'import os; print os.path.isfile("' + self.path[
'TELEGRAF_REMOTE_PATH'] + '")'
cmd = self.python + ' -c \'{cmd}\''.format(
cmd=remote_cmd
)
cmd = self.python + ' -c \'{cmd}\''.format(cmd=remote_cmd)
remote_telegraf_exists = "False"
try:
out, err, err_code = self.ssh.execute(cmd)
@ -255,37 +246,30 @@ class SSHClient(object):
self.path['TELEGRAF_REMOTE_PATH'])
elif os.path.isfile("/usr/bin/telegraf"):
self.ssh.send_file(
'/usr/bin/telegraf', self.path['TELEGRAF_REMOTE_PATH']
)
'/usr/bin/telegraf', self.path['TELEGRAF_REMOTE_PATH'])
else:
logger.error(
'Telegraf binary not found neither on %s nor on localhost at specified path: %s\n'
'You can download telegraf binaries here: https://github.com/influxdata/telegraf\n'
'or install debian package: `telegraf`', self.host, self.path['TELEGRAF_LOCAL_PATH'])
'or install debian package: `telegraf`', self.host,
self.path['TELEGRAF_LOCAL_PATH'])
return None, None, None
self.ssh.send_file(
self.path['AGENT_LOCAL_FOLDER'] + '/agent.py',
self.path['AGENT_REMOTE_FOLDER'] + '/agent.py'
)
self.path['AGENT_REMOTE_FOLDER'] + '/agent.py')
self.ssh.send_file(
agent_config,
self.path['AGENT_REMOTE_FOLDER'] +
'/agent.cfg')
agent_config, self.path['AGENT_REMOTE_FOLDER'] + '/agent.cfg')
self.ssh.send_file(
startup_config,
self.path['AGENT_REMOTE_FOLDER'] +
'/agent_startup.cfg')
self.path['AGENT_REMOTE_FOLDER'] + '/agent_startup.cfg')
self.ssh.send_file(
customs_script,
self.path['AGENT_REMOTE_FOLDER'] +
'/agent_customs.sh')
self.path['AGENT_REMOTE_FOLDER'] + '/agent_customs.sh')
except Exception:
logger.error(
"Failed to install agent on %s",
self.host,
exc_info=True)
"Failed to install agent on %s", self.host, exc_info=True)
return None, None, None
return agent_config, startup_config, customs_script
@ -300,9 +284,7 @@ class SSHClient(object):
host=self.host)
logging.debug('Command to start agent: %s', command)
self.session = self.ssh.async_session(command)
self.reader_thread = threading.Thread(
target=self.read_buffer
)
self.reader_thread = threading.Thread(target=self.read_buffer)
self.reader_thread.setDaemon(True)
return self.session
@ -339,12 +321,9 @@ class SSHClient(object):
exc_info=True)
try:
self.ssh.get_file(
self.path['AGENT_REMOTE_FOLDER'] +
"/_agent.log",
log_filename)
self.path['AGENT_REMOTE_FOLDER'] + "/_agent.log", log_filename)
self.ssh.get_file(
self.path['AGENT_REMOTE_FOLDER'] +
"/monitoring.rawdata",
self.path['AGENT_REMOTE_FOLDER'] + "/monitoring.rawdata",
data_filename)
self.ssh.rm_r(self.path['AGENT_REMOTE_FOLDER'])
except Exception:

14
yandextank/plugins/Telegraf/collector.py
View File

@ -38,10 +38,7 @@ class MonitoringCollector(object):
self.load_start_time = None
self.config_manager = ConfigManager()
self.old_style_configs = False
self.clients = {
'localhost': LocalhostClient,
'ssh': SSHClient
}
self.clients = {'localhost': LocalhostClient, 'ssh': SSHClient}
def add_listener(self, obj):
self.listeners.append(obj)
@ -103,7 +100,8 @@ class MonitoringCollector(object):
}
self.send_data.append(ready_to_send)
logger.debug('Polling/decoding agents data took: %.2fms',
logger.debug(
'Polling/decoding agents data took: %.2fms',
(time.time() - start_time) * 1000)
collected_data_length = len(self.send_data)
@ -132,8 +130,10 @@ class MonitoringCollector(object):
def send_collected_data(self):
"""sends pending data set to listeners"""
[listener.monitoring_data(self.send_data)
for listener in self.listeners]
[
listener.monitoring_data(self.send_data)
for listener in self.listeners
]
self.send_data = []

83
yandextank/plugins/Telegraf/config.py
View File

@ -50,20 +50,22 @@ class ConfigManager(object):
},
"Memory": {
"name": '[inputs.mem]',
"fielddrop": '["active", "inactive", "total", "used_per*", "avail*"]',
"fielddrop":
'["active", "inactive", "total", "used_per*", "avail*"]',
},
"Disk": {
"name": '[inputs.diskio]',
"devices": '[{devices}]'.format(
devices=",".join(['"vda%s","sda%s"' % (num, num) for num in range(6)])
),
devices=",".join(
['"vda%s","sda%s"' % (num, num) for num in range(6)])),
},
"Net": {
"name": '[inputs.net]',
"interfaces": '[{interfaces}]'.format(
interfaces=",".join(['"eth%s"' % (num) for num in range(6)])
),
"fielddrop": '["icmp*", "ip*", "udplite*", "tcp*", "udp*", "drop*", "err*"]',
interfaces=",".join(
['"eth%s"' % (num) for num in range(6)])),
"fielddrop":
'["icmp*", "ip*", "udplite*", "tcp*", "udp*", "drop*", "err*"]',
},
"Nstat": {
"name": '[inputs.nstat]',
@ -89,17 +91,15 @@ class ConfigManager(object):
}
defaults_enabled = ['CPU', 'Memory', 'Disk', 'Net', 'System', 'Kernel']
defaults_boolean = [
'percpu',
'round_interval',
'fielddrop',
'fieldpass',
'interfaces',
'devices']
'percpu', 'round_interval', 'fielddrop', 'fieldpass', 'interfaces',
'devices'
]
hostname = host.get('address').lower()
if hostname == '[target]':
if not target_hint:
raise ValueError(
"Can't use `[target]` keyword with no target parameter specified")
"Can't use `[target]` keyword with no target parameter specified"
)
logger.debug("Using target hint: %s", target_hint)
hostname = target_hint.lower()
custom = []
@ -113,14 +113,12 @@ class ConfigManager(object):
if key != 'name' and key not in defaults_boolean:
value = metric.get(key, None)
if value:
defaults[
metric.tag][key] = "'{value}'".format(
defaults[metric.tag][key] = "'{value}'".format(
value=value)
elif key in defaults_boolean:
value = metric.get(key, None)
if value:
defaults[
metric.tag][key] = "{value}".format(
defaults[metric.tag][key] = "{value}".format(
value=value)
host_config[metric.tag] = defaults[metric.tag]
# custom metrics
@ -186,11 +184,15 @@ class AgentConfig(object):
# FIXME incinerate such a string formatting inside a method call
# T_T
config.add_section('startup')
[config.set('startup', "cmd%s" % idx, cmd)
for idx, cmd in enumerate(self.startups)]
[
config.set('startup', "cmd%s" % idx, cmd)
for idx, cmd in enumerate(self.startups)
]
config.add_section('shutdown')
[config.set('shutdown', "cmd%s" % idx, cmd)
for idx, cmd in enumerate(self.shutdowns)]
[
config.set('shutdown', "cmd%s" % idx, cmd)
for idx, cmd in enumerate(self.shutdowns)
]
with open(cfg_path, 'w') as fds:
config.write(fds)
@ -210,16 +212,14 @@ class AgentConfig(object):
if os.path.isfile(cfg_path):
logger.info(
'Found agent custom execs config file in working directory with the same name as created for host %s.\n'
'Creating new one via tempfile. This will affect predictable filenames for agent artefacts', self.host)
'Creating new one via tempfile. This will affect predictable filenames for agent artefacts',
self.host)
handle, cfg_path = tempfile.mkstemp('.sh', 'agent_customs_')
os.close(handle)
cmds = ""
for idx, cmd in enumerate(self.custom):
cmds += "-{idx}) {cmd};;\n".format(
idx=idx,
cmd=cmd['cmd']
)
cmds += "-{idx}) {cmd};;\n".format(idx=idx, cmd=cmd['cmd'])
customs_script = """
#!/bin/sh
while :
@ -263,8 +263,7 @@ class AgentConfig(object):
config.set(
"agent",
"interval",
"'{interval}s'".format(
interval=self.interval))
"'{interval}s'".format(interval=self.interval))
config.set("agent", "round_interval", "true")
config.set("agent", "flush_interval", "'1s'")
config.set("agent", "collection_jitter", "'0s'")
@ -280,9 +279,10 @@ class AgentConfig(object):
if key != 'name':
config.set(
"{section_name}".format(
section_name=self.host_config[section]['name']), "{key}".format(
key=key), "{value}".format(
value=value))
section_name=self.host_config[section][
'name']),
"{key}".format(key=key),
"{value}".format(value=value))
# monitoring-style config
else:
if section in defaults_old_enabled:
@ -291,23 +291,22 @@ class AgentConfig(object):
section_name=self.host_config[section]['name']))
for key, value in iteritems(self.host_config[section]):
if key in [
'fielddrop',
'fieldpass',
'percpu',
'devices',
'interfaces']:
'fielddrop', 'fieldpass', 'percpu',
'devices', 'interfaces'
]:
config.set(
"{section_name}".format(
section_name=self.host_config[section]['name']), "{key}".format(
key=key), "{value}".format(
value=value))
section_name=self.host_config[section][
'name']),
"{key}".format(key=key),
"{value}".format(value=value))
# outputs
config.add_section("[outputs.file]")
config.set("[outputs.file]",
config.set(
"[outputs.file]",
"files",
"['{config}']".format(
config=self.monitoring_data_output))
"['{config}']".format(config=self.monitoring_data_output))
config.set("[outputs.file]", "data_format", "'json'")
with open(cfg_path, 'w') as fds:

27
yandextank/plugins/Telegraf/decoder.py
View File

@ -6,7 +6,6 @@ logger = logging.getLogger(__name__)
class MetricsDecoder(object):
def __init__(self):
"""
translates telegraf metric names into common Monitoring metric names
@ -49,32 +48,22 @@ class MetricsDecoder(object):
self.diff_metrics = {
'cpu': [],
'mem': [],
'net': [
'packets_recv',
'packets_sent',
'bytes_recv',
'bytes_sent'],
'net': ['packets_recv', 'packets_sent', 'bytes_recv', 'bytes_sent'],
'nstat': ['TcpRetransSegs'],
'net_response': [],
'kernel': [
'context_switches',
'interrupts',
'processes_forked'],
'kernel': ['context_switches', 'interrupts', 'processes_forked'],
'diskio': [
'read_bytes',
'write_bytes',
'io_time',
'read_time',
'reads',
'write_time',
'writes'],
'custom': []}
'read_bytes', 'write_bytes', 'io_time', 'read_time', 'reads',
'write_time', 'writes'
],
'custom': []
}
def find_common_names(self, key):
if key in self.known_metrics:
return self.known_metrics[key]
else:
return 'custom:{}'. format(key)
return 'custom:{}'.format(key)
decoder = MetricsDecoder()

43
yandextank/plugins/Telegraf/plugin.py
View File

@ -24,7 +24,6 @@ if sys.version_info[0] < 3:
else:
from configparser import NoOptionError
logger = logging.getLogger(__name__)
@ -54,8 +53,7 @@ class Plugin(AbstractPlugin):
if self.monitoring:
self.monitoring.load_start_time = time.time()
logger.debug(
"load_start_time = %s",
self.monitoring.load_start_time)
"load_start_time = %s", self.monitoring.load_start_time)
def get_available_options(self):
return ["config", "default_target", "ssh_timeout"]
@ -77,7 +75,8 @@ class Plugin(AbstractPlugin):
is_monitoring = None
if is_telegraf and is_monitoring:
raise ValueError('Both telegraf and monitoring configs specified. '
raise ValueError(
'Both telegraf and monitoring configs specified. '
'Clean up your config and delete one of them')
if is_telegraf and not is_monitoring:
return 'telegraf'
@ -111,8 +110,7 @@ class Plugin(AbstractPlugin):
self.detected_conf = self.__detect_configuration()
if self.detected_conf:
logging.info(
'Detected monitoring configuration: %s',
self.detected_conf)
'Detected monitoring configuration: %s', self.detected_conf)
self.SECTION = self.detected_conf
self.config = self.get_option("config", "auto").strip()
self.default_target = self.get_option("default_target", "localhost")
@ -133,16 +131,21 @@ class Plugin(AbstractPlugin):
else:
if self.config.lower() == "auto":
self.die_on_fail = False
with open(resource.resource_filename(self.default_config), 'rb') as def_config:
with open(
resource.resource_filename(self.default_config),
'rb') as def_config:
config_contents = def_config.read()
else:
with open(resource.resource_filename(self.config), 'rb') as config:
with open(resource.resource_filename(self.config),
'rb') as config:
config_contents = config.read()
# dump config contents into a file
xmlfile = self.core.mkstemp(".xml", "monitoring_")
self.core.add_artifact_file(xmlfile)
with open(xmlfile, "wb") as f: # output file should be in binary mode to support py3
with open(
xmlfile, "wb"
) as f: # output file should be in binary mode to support py3
f.write(config_contents)
self.config = xmlfile
@ -169,8 +172,7 @@ class Plugin(AbstractPlugin):
if info:
self.default_target = info.address
logger.debug(
"Changed monitoring target to %s",
self.default_target)
"Changed monitoring target to %s", self.default_target)
self.monitoring.config = self.config
if self.default_target:
@ -325,11 +327,11 @@ class MonitoringWidget(AbstractInfoWidget, MonitoringDataListener):
res = "Monitoring is " + screen.markup.GREEN + \
"online" + screen.markup.RESET + ":\n"
for hostname, metrics in self.data.items():
tm_stamp = datetime.datetime.fromtimestamp(float(self.time[
hostname])).strftime('%H:%M:%S')
tm_stamp = datetime.datetime.fromtimestamp(
float(self.time[hostname])).strftime('%H:%M:%S')
res += (
" " + screen.markup.CYAN + "%s" +
screen.markup.RESET + " at %s:\n") % (hostname, tm_stamp)
" " + screen.markup.CYAN + "%s" + screen.markup.RESET +
" at %s:\n") % (hostname, tm_stamp)
for metric, value in sorted(metrics.iteritems()):
if self.sign[hostname][metric] > 0:
value = screen.markup.YELLOW + value + screen.markup.RESET
@ -383,8 +385,9 @@ class AbstractMetricCriterion(AbstractCriterion, MonitoringDataListener):
if self.metric not in data.keys() or not data[self.metric]:
data[self.metric] = 0
logger.debug("Compare %s %s/%s=%s to %s", self.get_type_string(),
host, self.metric, data[self.metric],
logger.debug(
"Compare %s %s/%s=%s to %s",
self.get_type_string(), host, self.metric, data[self.metric],
self.value_limit)
if self.comparison_fn(float(data[self.metric]), self.value_limit):
if not self.seconds_count:
@ -430,7 +433,8 @@ class MetricHigherCriterion(AbstractMetricCriterion):
return "%s/%s metric value is higher than %s for %s seconds" % items
def widget_explain(self):
items = (self.host, self.metric, self.value_limit, self.seconds_count,
items = (
self.host, self.metric, self.value_limit, self.seconds_count,
self.seconds_limit)
return "%s/%s > %s for %s/%ss" % items, float(
self.seconds_count) / self.seconds_limit
@ -457,7 +461,8 @@ class MetricLowerCriterion(AbstractMetricCriterion):
return "%s/%s metric value is lower than %s for %s seconds" % items
def widget_explain(self):
items = (self.host, self.metric, self.value_limit, self.seconds_count,
items = (
self.host, self.metric, self.value_limit, self.seconds_count,
self.seconds_limit)
return "%s/%s < %s for %s/%ss" % items, float(
self.seconds_count) / self.seconds_limit

22
yandextank/plugins/Telegraf/reader.py
View File

@ -7,12 +7,10 @@ import json
from ..Telegraf.decoder import decoder
logger = logging.getLogger(__name__)
class MonitoringReader(object):
def __init__(self, source):
self.buffer = []
self.source = source
@ -43,11 +41,12 @@ class MonitoringReader(object):
# key_group sample: diskio
# key_name sample: io_time
try:
key_group, key_name = key.split('_')[0].split(
'-')[0], '_'.join(key.split('_')[1:])
key_group, key_name = key.split('_')[
0].split('-')[0], '_'.join(
key.split('_')[1:])
except:
key_group, key_name = key.split(
'_')[0], '_'.join(key.split('_')[1:])
key_group, key_name = key.split('_')[
0], '_'.join(key.split('_')[1:])
if key_group in decoder.diff_metrics.keys():
if key_name in decoder.diff_metrics[
key_group]:
@ -60,7 +59,10 @@ class MonitoringReader(object):
except KeyError:
logger.debug(
'There is no diff value for metric %s.\n'
'Timestamp: %s. Is it initial data?', key, ts, exc_info=True)
'Timestamp: %s. Is it initial data?',
key,
ts,
exc_info=True)
value = 0
prepared_results[
decoded_key] = value
@ -69,8 +71,7 @@ class MonitoringReader(object):
key)
prepared_results[decoded_key] = value
else:
decoded_key = decoder.find_common_names(
key)
decoded_key = decoder.find_common_names(key)
prepared_results[decoded_key] = value
self.prev_check = jsn[ts]
collect.append((ts, prepared_results))
@ -78,8 +79,7 @@ class MonitoringReader(object):
logger.error(
'Telegraf agent send trash to output: %s', chunk)
logger.debug(
'Telegraf agent data block w/ trash: %s',
exc_info=True)
'Telegraf agent data block w/ trash: %s', exc_info=True)
return []
except:
logger.error(

13
yandextank/plugins/Telegraf/tests/test_config.py
View File

@ -7,7 +7,6 @@ else:
class TestConfigManager(object):
def test_rawxml_parse(self):
""" raw xml read from string """
manager = ConfigManager()
@ -29,8 +28,7 @@ class TestConfigManager(object):
'yandextank/plugins/Telegraf/tests/old_mon.xml', 'sometargethint')
assert (
configs[0]['host'] == 'somehost.yandex.tld' and
configs[0]['host_config']['CPU']['name'] == '[inputs.cpu]'
)
configs[0]['host_config']['CPU']['name'] == '[inputs.cpu]')
def test_xml_telegraf_parse(self):
""" telegraf-style monitoring xml parse """
@ -40,8 +38,7 @@ class TestConfigManager(object):
'sometargethint')
assert (
configs[0]['host'] == 'somehost.yandex.tld' and
configs[0]['host_config']['CPU']['name'] == '[inputs.cpu]'
)
configs[0]['host_config']['CPU']['name'] == '[inputs.cpu]')
def test_target_hint(self):
""" test target hint (special address=[target] option) """
@ -53,7 +50,6 @@ class TestConfigManager(object):
class TestAgentConfig(object):
def test_create_startup_configs(self):
""" test agent config creates startup config """
manager = ConfigManager()
@ -81,9 +77,8 @@ class TestAgentConfig(object):
cfg_parser.has_section('agent') and
cfg_parser.get('agent', 'interval') == "'1s'" and
cfg_parser.has_section('[outputs.file') and
cfg_parser.get(
'[outputs.file', 'files') == "['{rmt}/monitoring.rawdata']".format(rmt=remote_workdir)
)
cfg_parser.get('[outputs.file', 'files') ==
"['{rmt}/monitoring.rawdata']".format(rmt=remote_workdir))
def test_create_custom_exec_script(self):
""" test agent config creates custom_exec config """

13
yandextank/plugins/Telegraf/tests/test_plugin.py
View File

@ -3,14 +3,12 @@ from yandextank.plugins.Telegraf import Plugin as TelegrafPlugin
class TestTelegrafPlugin(object):
def test_plugin_configuration(self):
""" testing telegraf plugin configuration """
core = TankCore()
telegraf_plugin = TelegrafPlugin(core)
core.set_option(
'telegraf',
'config',
'telegraf', 'config',
'yandextank/plugins/Telegraf/tests/telegraf_mon.xml')
telegraf_plugin.configure()
assert telegraf_plugin.detected_conf == 'telegraf'
@ -20,8 +18,7 @@ class TestTelegrafPlugin(object):
core = TankCore()
telegraf_plugin = TelegrafPlugin(core)
core.set_option(
'monitoring',
'config',
'monitoring', 'config',
'yandextank/plugins/Telegraf/tests/old_mon.xml')
telegraf_plugin.configure()
assert telegraf_plugin.detected_conf == 'monitoring'
@ -31,12 +28,10 @@ class TestTelegrafPlugin(object):
core = TankCore()
telegraf_plugin = TelegrafPlugin(core)
core.set_option(
'monitoring',
'config',
'monitoring', 'config',
'yandextank/plugins/Telegraf/tests/old_mon.xml')
core.set_option(
'telegraf',
'config',
'telegraf', 'config',
'yandextank/plugins/Telegraf/tests/telegraf_mon.xml')
try:
telegraf_plugin.configure()

10
yandextank/plugins/TipsAndTricks/plugin.py
View File

@ -19,8 +19,10 @@ class Plugin(AbstractPlugin, AbstractInfoWidget):
def __init__(self, core):
AbstractPlugin.__init__(self, core)
AbstractInfoWidget.__init__(self)
self.lines = [l.decode(
'utf-8') for l in resource_stream(__name__, "config/tips.txt").readlines()]
self.lines = [
l.decode('utf-8')
for l in resource_stream(__name__, "config/tips.txt").readlines()
]
self.disable = 0
line = random.choice(self.lines)
@ -60,6 +62,6 @@ class Plugin(AbstractPlugin, AbstractInfoWidget):
self.probability += 1e-3
line = screen.markup.WHITE + "Tips & Tricks => " + \
self.section + screen.markup.RESET + ":\n "
line += "\n ".join(textwrap.wrap(self.tip, screen.right_panel_width -
2))
line += "\n ".join(
textwrap.wrap(self.tip, screen.right_panel_width - 2))
return line

26
yandextank/stepper/config.py
View File

@ -11,8 +11,8 @@ from .module_exceptions import StepperConfigurationError, AmmoFileError
class ComponentFactory():
def __init__(self,
def __init__(
self,
rps_schedule=None,
http_ver='1.1',
ammo_file=None,
@ -57,7 +57,8 @@ class ComponentFactory():
"""
if self.rps_schedule and self.instances_schedule:
raise StepperConfigurationError(
'Both rps and instances schedules specified. You must specify only one of them')
'Both rps and instances schedules specified. You must specify only one of them'
)
elif self.rps_schedule:
info.status.publish('loadscheme', self.rps_schedule)
return lp.create(self.rps_schedule)
@ -84,11 +85,11 @@ class ComponentFactory():
}
if self.uris and self.ammo_file:
raise StepperConfigurationError(
'Both uris and ammo file specified. You must specify only one of them')
'Both uris and ammo file specified. You must specify only one of them'
)
elif self.uris:
ammo_gen = missile.UriStyleGenerator(self.uris,
self.headers,
http_ver=self.http_ver)
ammo_gen = missile.UriStyleGenerator(
self.uris, self.headers, http_ver=self.http_ver)
elif self.ammo_file:
if self.ammo_type in af_readers:
if self.ammo_type == 'phantom':
@ -98,10 +99,12 @@ class ComponentFactory():
if not ammo.next()[0].isdigit():
self.ammo_type = 'uri'
self.log.info(
"Setting ammo_type 'uri' because ammo is not started with digit and you did not specify ammo format")
"Setting ammo_type 'uri' because ammo is not started with digit and you did not specify ammo format"
)
else:
self.log.info(
"Default ammo type ('phantom') used, use 'phantom.ammo_type' option to override it")
"Default ammo type ('phantom') used, use 'phantom.ammo_type' option to override it"
)
except StopIteration:
self.log.exception(
"Couldn't read first line of ammo file")
@ -110,9 +113,8 @@ class ComponentFactory():
else:
raise NotImplementedError(
'No such ammo type implemented: "%s"' % self.ammo_type)
ammo_gen = af_readers[self.ammo_type](self.ammo_file,
headers=self.headers,
http_ver=self.http_ver)
ammo_gen = af_readers[self.ammo_type](
self.ammo_file, headers=self.headers, http_ver=self.http_ver)
else:
raise StepperConfigurationError(
'Ammo not found. Specify uris or ammo file')

12
yandextank/stepper/format.py
View File

@ -15,7 +15,8 @@ class Stpd(object):
self.af = ammo_factory
def __iter__(self):
return ("%s %s %s\n%s\n" % (len(missile), timestamp, marker, missile)
return (
"%s %s %s\n%s\n" % (len(missile), timestamp, marker, missile)
for timestamp, marker, missile in self.af)
@ -36,6 +37,7 @@ class StpdReader(object):
return line # EOF
chunk_header = line.strip('\r\n')
return chunk_header
with open(self.filename, 'rb') as ammo_file:
chunk_header = read_chunk_header(ammo_file)
while chunk_header != '':
@ -47,12 +49,12 @@ class StpdReader(object):
missile = ammo_file.read(chunk_size)
if len(missile) < chunk_size:
raise StpdFileError(
"Unexpected end of file: read %s bytes instead of %s" %
(len(missile), chunk_size))
"Unexpected end of file: read %s bytes instead of %s"
% (len(missile), chunk_size))
yield (timestamp, missile, marker)
except (IndexError, ValueError) as e:
raise StpdFileError(
"Error while reading ammo file. Position: %s, header: '%s', original exception: %s" %
(ammo_file.tell(), chunk_header, e))
"Error while reading ammo file. Position: %s, header: '%s', original exception: %s"
% (ammo_file.tell(), chunk_header, e))
chunk_header = read_chunk_header(ammo_file)
self.log.info("Reached the end of stpd file")

19
yandextank/stepper/info.py
View File

@ -38,8 +38,8 @@ class StepperStatus(object):
def publish(self, key, value):
if key not in self.info:
raise RuntimeError("Tried to publish to a non-existent key: %s" %
key)
raise RuntimeError(
"Tried to publish to a non-existent key: %s" % key)
log.debug('Published %s to %s', value, key)
self.info[key] = value
@ -88,8 +88,8 @@ class StepperStatus(object):
self.info['loop_count'] = self._loop_count
for key in self.info:
if self.info[key] is None:
raise RuntimeError("Information for %s is not published yet." %
key)
raise RuntimeError(
"Information for %s is not published yet." % key)
return StepperInfo(**self.info)
def update_view(self):
@ -100,15 +100,16 @@ class StepperStatus(object):
self._timer = cur_time
if time_delta > 0:
stdout.write(
"AF: %3s%%, LP: %3s%%, loops: %10s, speed: %5s Krps\r" %
(self.af_progress, self.lp_progress, self.loop_count, int(
ammo_generated / time_delta / 1000.0)))
"AF: %3s%%, LP: %3s%%, loops: %10s, speed: %5s Krps\r" % (
self.af_progress, self.lp_progress, self.loop_count,
int(ammo_generated / time_delta / 1000.0)))
stdout.flush()
if self.core:
self.core.publish("stepper", "progress", self.lp_progress)
self.core.publish("stepper", "loop_count", self.loop_count)
self.core.publish("stepper", "speed", "%s Krps" %
int(ammo_generated / time_delta / 1000.0))
self.core.publish(
"stepper", "speed",
"%s Krps" % int(ammo_generated / time_delta / 1000.0))
def update_af_progress(self):
if self.af_size and self.loop_limit and self.af_position is not None:

31
yandextank/stepper/instance_plan.py
View File

@ -9,7 +9,6 @@ from builtins import range
class LoadPlanBuilder(object):
def __init__(self):
self.generators = []
self.steps = []
@ -33,15 +32,16 @@ class LoadPlanBuilder(object):
return self
def ramp(self, count, duration):
self.log.debug("Ramp %s instances in %sms from %sms" %
self.log.debug(
"Ramp %s instances in %sms from %sms" %
(count, duration, self.duration))
if count < 0:
raise StepperConfigurationError(
"Can not stop instances in instances_schedule.")
interval = float(duration) / (count - 1)
start_time = self.duration
self.generators.append(int(start_time + i * interval)
for i in range(0, count))
self.generators.append(
int(start_time + i * interval) for i in range(0, count))
self.steps += [(self.instances + i + 1, int(interval / 1000.0))
for i in range(0, count)]
self.instances += count
@ -58,8 +58,8 @@ class LoadPlanBuilder(object):
self.ramp(final_instances - initial_instances + 1, duration)
return self
def stairway(self, initial_instances, final_instances, step_size,
step_duration):
def stairway(
self, initial_instances, final_instances, step_size, step_duration):
step_count = (final_instances - initial_instances) // step_size
self.log.debug("Making a stairway: %s steps" % step_count)
self.start(initial_instances - self.instances)
@ -79,7 +79,8 @@ class LoadPlanBuilder(object):
self.ramp(int(instances), parse_duration(interval))
else:
self.log.info(
"Ramp step format: 'ramp(<instances_to_start>, <step_duration>)'")
"Ramp step format: 'ramp(<instances_to_start>, <step_duration>)'"
)
raise StepperConfigurationError(
"Error in step configuration: 'ramp(%s'" % params)
@ -91,7 +92,8 @@ class LoadPlanBuilder(object):
self.const(int(instances), parse_duration(interval))
else:
self.log.info(
"Const step format: 'const(<instances_count>, <step_duration>)'")
"Const step format: 'const(<instances_count>, <step_duration>)'"
)
raise StepperConfigurationError(
"Error in step configuration: 'const(%s'" % params)
@ -112,11 +114,12 @@ class LoadPlanBuilder(object):
if s_res:
initial_instances, final_instances, interval = s_res.groups()
self.line(
int(initial_instances), int(final_instances),
parse_duration(interval))
int(initial_instances),
int(final_instances), parse_duration(interval))
else:
self.log.info(
"Line step format: 'line(<initial_instances>, <final_instances>, <step_duration>)'")
"Line step format: 'line(<initial_instances>, <final_instances>, <step_duration>)'"
)
raise StepperConfigurationError(
"Error in step configuration: 'line(%s'" % params)
@ -139,11 +142,13 @@ class LoadPlanBuilder(object):
initial_instances, final_instances, step_size, step_duration = s_res.groups(
)
self.stairway(
int(initial_instances), int(final_instances),
int(initial_instances),
int(final_instances),
int(step_size), parse_duration(step_duration))
else:
self.log.info(
"Stairway step format: 'step(<initial_instances>, <final_instances>, <step_size>, <step_duration>)'")
"Stairway step format: 'step(<initial_instances>, <final_instances>, <step_size>, <step_duration>)'"
)
raise StepperConfigurationError(
"Error in step configuration: 'step(%s'" % params)

25
yandextank/stepper/load_plan.py
View File

@ -21,7 +21,8 @@ class Const(object):
if self.rps == 0:
return iter([])
interval = 1000.0 / self.rps
return (int(i * interval)
return (
int(i * interval)
for i in range(0, int(self.rps * self.duration / 1000)))
def rps_at(self, t):
@ -112,8 +113,8 @@ class Line(object):
:rtype: list
"""
seconds = range(0, int(self.duration) + 1)
rps_groups = groupby(
[proper_round(self.rps_at(t)) for t in seconds], lambda x: x)
rps_groups = groupby([proper_round(self.rps_at(t)) for t in seconds],
lambda x: x)
rps_list = [(rps, len(list(rpl))) for rps, rpl in rps_groups]
return rps_list
@ -140,12 +141,11 @@ class Composite(object):
return int(sum(step.__len__() for step in self.steps))
def get_rps_list(self):
return list(chain.from_iterable(step.get_rps_list()
for step in self.steps))
return list(
chain.from_iterable(step.get_rps_list() for step in self.steps))
class Stairway(Composite):
def __init__(self, minrps, maxrps, increment, step_duration):
if maxrps < minrps:
increment = -increment
@ -164,7 +164,6 @@ class Stairway(Composite):
class StepFactory(object):
@staticmethod
def line(params):
template = re.compile('([0-9.]+),\s*([0-9.]+),\s*([0-9.]+[dhms]?)+\)')
@ -183,8 +182,8 @@ class StepFactory(object):
'([0-9.]+),\s*([0-9.]+),\s*([0-9.]+),\s*([0-9.]+[dhms]?)+\)')
minrps, maxrps, increment, duration = template.search(params).groups()
return Stairway(
float(minrps), float(maxrps), float(increment),
parse_duration(duration))
float(minrps),
float(maxrps), float(increment), parse_duration(duration))
@staticmethod
def produce(step_config):
@ -198,8 +197,8 @@ class StepFactory(object):
if load_type in _plans:
return _plans[load_type](params)
else:
raise NotImplementedError('No such load type implemented: "%s"' %
load_type)
raise NotImplementedError(
'No such load type implemented: "%s"' % load_type)
def create(rps_schedule):
@ -207,8 +206,8 @@ def create(rps_schedule):
Create Load Plan as defined in schedule. Publish info about its duration.
"""
if len(rps_schedule) > 1:
lp = Composite([StepFactory.produce(step_config)
for step_config in rps_schedule])
lp = Composite(
[StepFactory.produce(step_config) for step_config in rps_schedule])
else:
lp = StepFactory.produce(rps_schedule[0])
info.status.publish('duration', lp.get_duration() / 1000)

52
yandextank/stepper/main.py
View File

@ -39,18 +39,18 @@ class AmmoFactory(object):
configured ComponentFactory, passed as a parameter to the
__init__ method of this class.
'''
ammo_stream = (ammo
ammo_stream = (
ammo
for ammo in ((missile, marker or self.marker(missile))
for missile, marker in self.ammo_generator)
if self.filter(ammo))
return ((timestamp, marker or self.marker(missile), missile)
for timestamp, (missile, marker) in zip(self.load_plan,
ammo_stream))
for timestamp, (missile, marker
) in zip(self.load_plan, ammo_stream))
class Stepper(object):
def __init__(self, core, **kwargs):
info.status = info.StepperStatus()
info.status.core = core
@ -118,19 +118,25 @@ class StepperWrapper(object):
def get_option(self, option_ammofile, param2=None):
''' get_option wrapper'''
result = self.core.get_option(self.section, option_ammofile, param2)
self.log.debug("Option %s.%s = %s", self.section, option_ammofile,
result)
self.log.debug(
"Option %s.%s = %s", self.section, option_ammofile, result)
return result
@staticmethod
def get_available_options():
opts = [StepperWrapper.OPTION_AMMOFILE, StepperWrapper.OPTION_LOOP,
opts = [
StepperWrapper.OPTION_AMMOFILE, StepperWrapper.OPTION_LOOP,
StepperWrapper.OPTION_SCHEDULE, StepperWrapper.OPTION_STPD,
StepperWrapper.OPTION_INSTANCES_LIMIT]
opts += ["instances_schedule", "uris", "headers", "header_http",
"autocases", "enum_ammo", "ammo_type", "ammo_limit"]
opts += ["use_caching", "cache_dir", "force_stepping", "file_cache",
"chosen_cases"]
StepperWrapper.OPTION_INSTANCES_LIMIT
]
opts += [
"instances_schedule", "uris", "headers", "header_http", "autocases",
"enum_ammo", "ammo_type", "ammo_limit"
]
opts += [
"use_caching", "cache_dir", "force_stepping", "file_cache",
"chosen_cases"
]
return opts
def read_config(self):
@ -150,12 +156,12 @@ class StepperWrapper(object):
steps.append(step.strip() + ')')
return steps
self.rps_schedule = make_steps(self.get_option(self.OPTION_SCHEDULE,
''))
self.instances_schedule = make_steps(self.get_option(
"instances_schedule", ''))
self.instances = int(self.get_option(self.OPTION_INSTANCES_LIMIT,
'1000'))
self.rps_schedule = make_steps(
self.get_option(self.OPTION_SCHEDULE, ''))
self.instances_schedule = make_steps(
self.get_option("instances_schedule", ''))
self.instances = int(
self.get_option(self.OPTION_INSTANCES_LIMIT, '1000'))
self.uris = self.get_option("uris", '').strip().split("\n")
while '' in self.uris:
self.uris.remove('')
@ -167,8 +173,8 @@ class StepperWrapper(object):
self.use_caching = int(self.get_option("use_caching", '1'))
self.file_cache = int(self.get_option('file_cache', '8192'))
cache_dir = self.core.get_option(self.section, "cache_dir",
self.core.artifacts_base_dir)
cache_dir = self.core.get_option(
self.section, "cache_dir", self.core.artifacts_base_dir)
self.cache_dir = os.path.expanduser(cache_dir)
self.force_stepping = int(self.get_option("force_stepping", '0'))
self.stpd = self.get_option(self.OPTION_STPD, "")
@ -209,7 +215,8 @@ class StepperWrapper(object):
instances=self.instances)
publish_info(stepper_info)
else:
if (self.force_stepping and
if (
self.force_stepping and
os.path.exists(self.__si_filename())):
os.remove(self.__si_filename())
self.__make_stpd_file()
@ -240,7 +247,8 @@ class StepperWrapper(object):
hashed_str += sep + str(self.ammo_limit) + sep + ';'.join(
self.rps_schedule) + sep + str(self.autocases)
hashed_str += sep + ";".join(self.uris) + sep + ";".join(
self.headers) + sep + self.http_ver + sep + ";".join(self.chosen_cases)
self.headers) + sep + self.http_ver + sep + ";".join(
self.chosen_cases)
hashed_str += sep + str(self.enum_ammo) + sep + str(self.ammo_type)
if self.instances_schedule:
hashed_str += sep + str(self.instances)

16
yandextank/stepper/mark.py
View File

@ -12,8 +12,8 @@ param1=50&param2=0&param3=hello
def __mark_by_uri(missile):
return '_'.join(missile.split('\n', 1)[0].split(' ', 2)[1].split('?')[
0].split('/'))
return '_'.join(
missile.split('\n', 1)[0].split(' ', 2)[1].split('?')[0].split('/'))
class __UriMarker(object):
@ -29,15 +29,18 @@ class __UriMarker(object):
self.limit = limit
def __call__(self, missile):
return '_'.join(missile.split('\n', 1)[0].split(' ', 2)[1].split('?')[
0].split('/')[0:self.limit + 1])
return '_'.join(
missile.split('\n', 1)[0].split(' ', 2)[1].split('?')[0].split('/')[
0:self.limit + 1])
__markers = {'uniq': lambda m: uuid4().hex, 'uri': __mark_by_uri, }
__markers = {
'uniq': lambda m: uuid4().hex,
'uri': __mark_by_uri,
}
class __Enumerator(object):
def __init__(self, marker):
self.marker = marker
self.number = int(0)
@ -80,6 +83,7 @@ def get_marker(marker_type, enum_ammo=False):
if limit:
marker = __UriMarker(limit)
else:
def marker(m):
return ''
except ValueError:

62
yandextank/stepper/missile.py
View File

@ -46,8 +46,8 @@ class HttpAmmo(object):
headers = '\r\n'.join(self.headers) + '\r\n'
else:
headers = ''
return "%s %s %s\r\n%s\r\n%s" % (self.method, self.uri, self.proto,
headers, self.body)
return "%s %s %s\r\n%s\r\n%s" % (
self.method, self.uri, self.proto, headers, self.body)
class SimpleGenerator(object):
@ -78,10 +78,9 @@ class UriStyleGenerator(object):
uris - a list of URIs as strings.
'''
self.uri_count = len(uris)
self.missiles = cycle([(HttpAmmo(uri,
headers,
http_ver=http_ver).to_s(), None)
for uri in uris])
self.missiles = cycle([(
HttpAmmo(
uri, headers, http_ver=http_ver).to_s(), None) for uri in uris])
def __iter__(self):
for m in self.missiles:
@ -120,8 +119,8 @@ class AmmoFileReader(object):
if chunk_size == 0:
if info.status.loop_count == 0:
self.log.info(
'Zero-sized chunk in ammo file at %s. Starting over.' %
ammo_file.tell())
'Zero-sized chunk in ammo file at %s. Starting over.'
% ammo_file.tell())
ammo_file.seek(0)
info.status.inc_loop_count()
chunk_header = read_chunk_header(ammo_file)
@ -130,13 +129,13 @@ class AmmoFileReader(object):
missile = ammo_file.read(chunk_size)
if len(missile) < chunk_size:
raise AmmoFileError(
"Unexpected end of file: read %s bytes instead of %s" %
(len(missile), chunk_size))
"Unexpected end of file: read %s bytes instead of %s"
% (len(missile), chunk_size))
yield (missile, marker)
except (IndexError, ValueError) as e:
raise AmmoFileError(
"Error while reading ammo file. Position: %s, header: '%s', original exception: %s" %
(ammo_file.tell(), chunk_header, e))
"Error while reading ammo file. Position: %s, header: '%s', original exception: %s"
% (ammo_file.tell(), chunk_header, e))
chunk_header = read_chunk_header(ammo_file)
if chunk_header == '':
ammo_file.seek(0)
@ -242,12 +241,14 @@ class AccessLogReader(object):
method, uri, proto = request.split()
http_ver = proto.split('/')[1]
if method == "GET":
yield (HttpAmmo(uri,
yield (
HttpAmmo(
uri,
headers=self.headers,
http_ver=http_ver, ).to_s(), None)
else:
self.warn("Skipped line: %s (unsupported method)" %
line)
self.warn(
"Skipped line: %s (unsupported method)" % line)
except (ValueError, IndexError) as e:
self.warn("Skipped line: %s (%s)" % (line, e))
ammo_file.seek(0)
@ -260,7 +261,6 @@ def _parse_header(header):
class UriReader(object):
def __init__(self, filename, headers=[], http_ver='1.1', **kwargs):
self.filename = filename
self.headers = {}
@ -278,8 +278,8 @@ class UriReader(object):
for line in ammo_file:
info.status.af_position = ammo_file.tell()
if line.startswith('['):
self.headers.update(_parse_header(line.strip(
'\r\n[]\t ')))
self.headers.update(
_parse_header(line.strip('\r\n[]\t ')))
elif len(line.rstrip('\r\n')):
fields = line.split()
uri = fields[0]
@ -287,13 +287,14 @@ class UriReader(object):
marker = fields[1]
else:
marker = None
yield (HttpAmmo(uri,
yield (
HttpAmmo(
uri,
headers=[
': '.join(header)
for header in self.headers.items()
],
http_ver=self.http_ver, ).to_s(),
marker)
http_ver=self.http_ver, ).to_s(), marker)
if info.status.ammo_count == 0:
self.log.error("No ammo in uri-style file")
raise AmmoFileError("No ammo! Cover me!")
@ -339,8 +340,8 @@ class UriPostReader(object):
chunk_size = int(fields[0])
if chunk_size == 0:
self.log.debug(
'Zero-sized chunk in ammo file at %s. Starting over.' %
ammo_file.tell())
'Zero-sized chunk in ammo file at %s. Starting over.'
% ammo_file.tell())
ammo_file.seek(0)
info.status.inc_loop_count()
chunk_header = read_chunk_header(ammo_file)
@ -350,21 +351,22 @@ class UriPostReader(object):
missile = ammo_file.read(chunk_size)
if len(missile) < chunk_size:
raise AmmoFileError(
"Unexpected end of file: read %s bytes instead of %s" %
(len(missile), chunk_size))
yield (HttpAmmo(uri=uri,
"Unexpected end of file: read %s bytes instead of %s"
% (len(missile), chunk_size))
yield (
HttpAmmo(
uri=uri,
headers=[
': '.join(header)
for header in self.headers.items()
],
method='POST',
body=missile,
http_ver=self.http_ver, ).to_s(),
marker)
http_ver=self.http_ver, ).to_s(), marker)
except (IndexError, ValueError) as e:
raise AmmoFileError(
"Error while reading ammo file. Position: %s, header: '%s', original exception: %s" %
(ammo_file.tell(), chunk_header, e))
"Error while reading ammo file. Position: %s, header: '%s', original exception: %s"
% (ammo_file.tell(), chunk_header, e))
chunk_header = read_chunk_header(ammo_file)
if chunk_header == '':
self.log.debug(

55
yandextank/stepper/tests/test_instance_plan.py
View File

@ -5,25 +5,48 @@ from yandextank.stepper.util import take
class TestCreate(object):
@pytest.mark.parametrize('n, loadplan, expected', [
(7, LoadPlanBuilder().ramp(5, 4000).create(), [0, 1000, 2000, 3000, 4000, 0, 0]),
(7, create(['ramp(5, 4s)']), [0, 1000, 2000, 3000, 4000, 0, 0]),
(12, create(['ramp(5, 4s)', 'wait(5s)', 'ramp(5,4s)']), [0, 1000, 2000, 3000, 4000, 9000, 10000, 11000, 12000, 13000, 0, 0]),
(7, create(['wait(5s)', 'ramp(5, 0)']), [5000, 5000, 5000, 5000, 5000, 0, 0]),
(7, create([]), [0, 0, 0, 0, 0, 0, 0]),
(12, create(['line(1, 9, 4s)']), [0, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 0, 0, 0]),
(12, create(['const(3, 5s)', 'line(7, 11, 2s)']), [0, 0, 0, 5000, 5000, 5000, 5000, 5500, 6000, 6500, 7000, 0]),
(12, create(['step(2, 10, 2, 3s)']), [0, 0, 3000, 3000, 6000, 6000, 9000, 9000, 12000, 12000, 0, 0]),
(12, LoadPlanBuilder().const(3, 1000).line(5, 10, 5000).steps, [(3, 1), (5, 1), (6, 1), (7, 1), (8, 1), (9, 1), (10, 1)]),
(12, LoadPlanBuilder().stairway(100, 950, 100, 30000).steps, [(100, 30), (200, 30), (300, 30), (400, 30), (500, 30), (600, 30), (700, 30), (800, 30), (900, 30), (950, 30)])])
@pytest.mark.parametrize(
'n, loadplan, expected',
[(
7, LoadPlanBuilder().ramp(5, 4000).create(),
[0, 1000, 2000, 3000, 4000, 0, 0]
), (
7, create(['ramp(5, 4s)']),
[0, 1000, 2000, 3000, 4000, 0, 0]
), (
12, create(['ramp(5, 4s)', 'wait(5s)', 'ramp(5,4s)']),
[0, 1000, 2000, 3000, 4000, 9000, 10000, 11000, 12000, 13000, 0, 0]
), (
7, create(['wait(5s)', 'ramp(5, 0)']),
[5000, 5000, 5000, 5000, 5000, 0, 0]
), (
7, create([]),
[0, 0, 0, 0, 0, 0, 0]
), (
12, create(['line(1, 9, 4s)']),
[0, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 0, 0, 0]
), (
12, create(['const(3, 5s)', 'line(7, 11, 2s)']),
[0, 0, 0, 5000, 5000, 5000, 5000, 5500, 6000, 6500, 7000, 0]
), (
12, create(['step(2, 10, 2, 3s)']),
[0, 0, 3000, 3000, 6000, 6000, 9000, 9000, 12000, 12000, 0, 0]
), (
12, LoadPlanBuilder().const(3, 1000).line(5, 10, 5000).steps,
[(3, 1), (5, 1), (6, 1), (7, 1), (8, 1), (9, 1), (10, 1)]
), (
12, LoadPlanBuilder().stairway(100, 950, 100, 30000).steps,
[
(100, 30), (200, 30), (300, 30), (400, 30), (500, 30),
(600, 30), (700, 30), (800, 30), (900, 30), (950, 30)]
)]) # yapf:disable
def test_steps(self, n, loadplan, expected):
assert take(n, loadplan) == expected
@pytest.mark.parametrize('loadplan, expected', [
(LoadPlanBuilder().stairway(100, 950, 100, 30000), 950),
@pytest.mark.parametrize(
'loadplan, expected',
[(LoadPlanBuilder().stairway(100, 950, 100, 30000), 950),
(LoadPlanBuilder().const(3, 1000).line(5, 10, 5000), 10),
(LoadPlanBuilder().line(1, 100, 60000), 100)
])
(LoadPlanBuilder().line(1, 100, 60000), 100)])
def test_instances(self, loadplan, expected):
assert loadplan.instances == expected

137
yandextank/stepper/tests/test_load_plan.py
View File

@ -4,7 +4,6 @@ from yandextank.stepper.util import take
class TestLine(object):
def test_get_rps_list(self):
lp = create(["line(1, 100, 10s)"])
rps_list = lp.get_rps_list()
@ -12,23 +11,19 @@ class TestLine(object):
assert rps_list[-1][0] == 100
@pytest.mark.parametrize("rps, duration, rps_list", [
(100, 3000, [(100, 3)]),
(0, 3000, [(0, 3)]),
(100, 0, [(100, 0)])
])
@pytest.mark.parametrize(
"rps, duration, rps_list",
[(100, 3000, [(100, 3)]), (0, 3000, [(0, 3)]), (100, 0, [(100, 0)])])
class TestConst(object):
@pytest.mark.parametrize("check_point, expected", [
(lambda duration: 0, lambda rps: rps),
@pytest.mark.parametrize(
"check_point, expected",
[(lambda duration: 0, lambda rps: rps),
(lambda duration: duration / 2, lambda rps: rps),
(lambda duration: duration + 1, lambda rps: 0),
(lambda duration: -1, lambda rps: 0)
])
(lambda duration: -1, lambda rps: 0)])
def test_rps_at(self, rps, duration, rps_list, check_point, expected):
assert Const(
rps, duration).rps_at(
check_point(duration)) == expected(rps)
assert Const(rps,
duration).rps_at(check_point(duration)) == expected(rps)
def test_get_rps_list(self, rps, duration, rps_list):
assert Const(rps, duration).get_rps_list() == rps_list
@ -36,22 +31,18 @@ class TestConst(object):
class TestLineNew(object):
@pytest.mark.parametrize("min_rps, max_rps, duration, check_point, expected", [
(0, 10, 30 * 1000, 0, 0),
(0, 10, 30 * 1000, 10, 3),
(0, 10, 30 * 1000, 29, 10),
(9, 10, 30 * 1000, 1, 9),
(9, 10, 30 * 1000, 20, 10)
])
@pytest.mark.parametrize(
"min_rps, max_rps, duration, check_point, expected",
[(0, 10, 30 * 1000, 0, 0), (0, 10, 30 * 1000, 10, 3),
(0, 10, 30 * 1000, 29, 10), (9, 10, 30 * 1000, 1, 9),
(9, 10, 30 * 1000, 20, 10)])
def test_rps_at(self, min_rps, max_rps, duration, check_point, expected):
assert round(
Line(
min_rps,
max_rps,
duration).rps_at(check_point)) == expected
assert round(Line(min_rps, max_rps, duration).rps_at(
check_point)) == expected
@pytest.mark.parametrize("min_rps, max_rps, duration, check_point, expected", [
@pytest.mark.parametrize(
"min_rps, max_rps, duration, check_point, expected",
[
(0, 10, 20 * 1000, 9, (9, 2)),
(0, 10, 30 * 1000, 0, (0, 2)),
(0, 10, 30 * 1000, 5, (5, 3)),
@ -69,54 +60,42 @@ class TestLineNew(object):
(10, 0, 30 * 1000, 10, (0, 2)),
])
def test_get_rps_list(
self,
min_rps,
max_rps,
duration,
check_point,
expected):
assert Line(min_rps, max_rps, duration).get_rps_list()[
check_point] == expected
self, min_rps, max_rps, duration, check_point, expected):
assert Line(min_rps, max_rps,
duration).get_rps_list()[check_point] == expected
@pytest.mark.parametrize("min_rps, max_rps, duration, expected_len, threshold, len_above_threshold", [
@pytest.mark.parametrize(
"min_rps, max_rps, duration, expected_len, threshold, len_above_threshold",
[
(2, 12, 25000, 175, 5000, 160),
(2, 12, 25000, 175, 10000, 135),
(2, 12, 25000, 175, 15000, 100),
(2, 12, 25000, 175, 20000, 55),
(0, 10, 25000, 125, 15000, 80),
(10, 12, 20000, 220, 10000, 115),
(10, 10, 20000, 200, 10000, 100),
(10, 0, 25000, 125, 10000, 45),
(10, 0, 25000, 125, 15000, 20),
])
def test_iter(
self,
min_rps,
max_rps,
duration,
expected_len,
threshold,
self, min_rps, max_rps, duration, expected_len, threshold,
len_above_threshold):
load_plan = Line(min_rps, max_rps, duration)
assert len(load_plan) == expected_len
assert len([ts for ts in load_plan if ts >= threshold]
) == len_above_threshold
assert len(
[ts for ts in load_plan if ts >= threshold]) == len_above_threshold
class TestComposite(object):
@pytest.mark.parametrize("steps, expected_len", [
([Line(0, 10, 20000), Const(10, 10000)], 200),
@pytest.mark.parametrize(
"steps, expected_len", [([Line(0, 10, 20000), Const(10, 10000)], 200),
([Line(0, 10, 20000), Line(10, 0, 20000)], 200),
([Const(5, 10000), Const(10, 5000)], 100)
])
([Const(5, 10000), Const(10, 5000)], 100)])
def test_iter(self, steps, expected_len):
assert len(Composite(steps)) == expected_len
@pytest.mark.parametrize("steps, check_point, expected", [
@pytest.mark.parametrize(
"steps, check_point, expected", [
([Line(0, 10, 20000), Const(10, 10000)], 9, (9, 2)),
([Line(0, 10, 20000), Const(10, 10000)], 10, (10, 2)),
([Line(0, 10, 20000), Const(10, 10000)], 11, (10, 10)),
@ -126,40 +105,42 @@ class TestComposite(object):
class TestStairway(object):
@pytest.mark.parametrize("min_rps, max_rps, increment, step_duration, expected_len, threshold, len_above_threshold",
[
(0, 1000, 50, 3000, 31500, 9000, 31050),
@pytest.mark.parametrize(
"min_rps, max_rps, increment, step_duration, expected_len, threshold, len_above_threshold",
[(0, 1000, 50, 3000, 31500, 9000, 31050),
(0, 1000, 50, 3000, 31500, 15000, 30000),
(0, 1000, 50, 3000, 31500, 45000, 15750)
])
(0, 1000, 50, 3000, 31500, 45000, 15750)])
def test_iter(
self,
min_rps,
max_rps,
increment,
step_duration,
expected_len,
threshold,
len_above_threshold):
self, min_rps, max_rps, increment, step_duration, expected_len,
threshold, len_above_threshold):
load_plan = Stairway(min_rps, max_rps, increment, step_duration)
assert len(load_plan) == expected_len
assert len([ts for ts in load_plan if ts >= threshold]
) == len_above_threshold
assert len(
[ts for ts in load_plan if ts >= threshold]) == len_above_threshold
class TestCreate(object):
@pytest.mark.parametrize('rps_schedule, check_point, expected', [
@pytest.mark.parametrize(
'rps_schedule, check_point, expected', [
(['line(1, 5, 2s)'], 100, [0, 618, 1000, 1302, 1561, 1791]),
(['line(1.1, 5.8, 2s)'], 100, [0, 566, 917, 1196, 1435, 1647]),
(['line(5, 1, 2s)'], 100, [0, 208, 438, 697, 1000, 1381]),
(['const(1, 10s)'], 100, [0, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000]),
(['const(200, 0.1s)'], 100, [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95]),
(['const(1, 2s)', 'const(2, 2s)'], 100, [0, 1000, 2000, 2500, 3000, 3500]),
(['const(1.5, 10s)'], 100, [0, 666, 1333, 2000, 2666, 3333, 4000, 4666, 5333, 6000, 6666, 7333, 8000, 8666, 9333]),
(['step(1, 5, 1, 5s)'], 10, [0, 1000, 2000, 3000, 4000, 5000, 5500, 6000, 6500, 7000]),
(['step(1.2, 5.7, 1.1, 5s)'], 10, [0, 833, 1666, 2500, 3333, 4166, 5000, 5434, 5869, 6304]),
(['const(1, 10s)'], 100,
[0, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000]),
(['const(200, 0.1s)'], 100, [
0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75,
80, 85, 90, 95
]),
(['const(1, 2s)', 'const(2, 2s)'], 100,
[0, 1000, 2000, 2500, 3000, 3500]),
(['const(1.5, 10s)'], 100, [
0, 666, 1333, 2000, 2666, 3333, 4000, 4666, 5333, 6000, 6666,
7333, 8000, 8666, 9333
]),
(['step(1, 5, 1, 5s)'], 10,
[0, 1000, 2000, 3000, 4000, 5000, 5500, 6000, 6500, 7000]),
(['step(1.2, 5.7, 1.1, 5s)'], 10,
[0, 833, 1666, 2500, 3333, 4166, 5000, 5434, 5869, 6304]),
(['const(1, 1)'], 10, [0]),
])
def test_create(self, rps_schedule, check_point, expected):

7
yandextank/stepper/util.py
View File

@ -31,7 +31,12 @@ def parse_duration(duration):
_re_token = re.compile("([0-9.]+)([dhms]?)")
def parse_token(time, multiplier):
multipliers = {'d': 86400, 'h': 3600, 'm': 60, 's': 1, }
multipliers = {
'd': 86400,
'h': 3600,
'm': 60,
's': 1,
}
if multiplier:
if multiplier in multipliers:
return int(float(time) * multipliers[multiplier] * 1000)