mirror of
https://github.com/valitydev/yandex-tank.git
synced 2024-11-07 02:45:21 +00:00
470 lines
16 KiB
Python
470 lines
16 KiB
Python
""" Autostop facility """
|
|
import copy
|
|
import logging
|
|
import re
|
|
import json
|
|
|
|
from Aggregator import AggregatorPlugin, AggregateResultListener
|
|
from ConsoleOnline import AbstractInfoWidget, ConsoleOnlinePlugin
|
|
from yandextank.core import AbstractPlugin
|
|
import yandextank.core as tankcore
|
|
import time
|
|
|
|
|
|
class AutostopPlugin(AbstractPlugin, AggregateResultListener):
|
|
""" Plugin that accepts criteria classes and triggers autostop """
|
|
SECTION = 'autostop'
|
|
|
|
def __init__(self, core):
|
|
AbstractPlugin.__init__(self, core)
|
|
self.cause_criteria = None
|
|
self.criterias = []
|
|
self.custom_criterias = []
|
|
self.counting = []
|
|
self.criteria_str = ''
|
|
|
|
@staticmethod
|
|
def get_key():
|
|
return __file__
|
|
|
|
def get_counting(self):
|
|
""" get criterias that are activated """
|
|
return self.counting
|
|
|
|
def add_counting(self, obj):
|
|
""" add criteria that activated """
|
|
self.counting += [obj]
|
|
|
|
def add_criteria_class(self, criteria_class):
|
|
""" add new criteria class """
|
|
self.custom_criterias += [criteria_class]
|
|
|
|
def get_available_options(self):
|
|
return ["autostop"]
|
|
|
|
def configure(self):
|
|
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
|
|
aggregator.add_result_listener(self)
|
|
self.criteria_str = " ".join(self.get_option("autostop", '').split("\n"))
|
|
self.add_criteria_class(AvgTimeCriteria)
|
|
self.add_criteria_class(NetCodesCriteria)
|
|
self.add_criteria_class(HTTPCodesCriteria)
|
|
self.add_criteria_class(QuantileCriteria)
|
|
self.add_criteria_class(SteadyCumulativeQuantilesCriteria)
|
|
self.add_criteria_class(TimeLimitCriteria)
|
|
|
|
def prepare_test(self):
|
|
for criteria_str in self.criteria_str.strip().split(")"):
|
|
if not criteria_str:
|
|
continue
|
|
self.log.debug("Criteria string: %s", criteria_str)
|
|
self.criterias.append(self.__create_criteria(criteria_str))
|
|
|
|
self.log.debug("Criteria object: %s", self.criterias)
|
|
|
|
try:
|
|
console = self.core.get_plugin_of_type(ConsoleOnlinePlugin)
|
|
except Exception, ex:
|
|
self.log.debug("Console not found: %s", ex)
|
|
console = None
|
|
|
|
if console:
|
|
console.add_info_widget(AutostopWidget(self))
|
|
|
|
def is_test_finished(self):
|
|
if self.cause_criteria:
|
|
self.log.info("Autostop criteria requested test stop: %s", self.cause_criteria.explain())
|
|
return self.cause_criteria.get_rc()
|
|
else:
|
|
return -1
|
|
|
|
def __create_criteria(self, criteria_str):
|
|
""" instantiate criteria from config string """
|
|
parsed = criteria_str.split("(")
|
|
type_str = parsed[0].strip().lower()
|
|
parsed[1] = parsed[1].split(")")[0].strip()
|
|
|
|
for criteria_class in self.custom_criterias:
|
|
if criteria_class.get_type_string() == type_str:
|
|
return criteria_class(self, parsed[1])
|
|
raise ValueError("Unsupported autostop criteria type: %s" % criteria_str)
|
|
|
|
def aggregate_second(self, second_aggregate_data):
|
|
self.counting = []
|
|
if not self.cause_criteria:
|
|
for criteria in self.criterias:
|
|
if criteria.notify(second_aggregate_data):
|
|
self.log.debug("Autostop criteria requested test stop: %s", criteria)
|
|
self.cause_criteria = criteria
|
|
|
|
|
|
class AutostopWidget(AbstractInfoWidget):
|
|
""" widget that displays counting criterias """
|
|
|
|
def __init__(self, sender):
|
|
AbstractInfoWidget.__init__(self)
|
|
self.owner = sender
|
|
|
|
def get_index(self):
|
|
return 25
|
|
|
|
def render(self, screen):
|
|
res = []
|
|
candidates = self.owner.get_counting()
|
|
for candidate in candidates:
|
|
text, perc = candidate.widget_explain()
|
|
if perc >= 0.95:
|
|
res += [screen.markup.RED_DARK + text + screen.markup.RESET]
|
|
elif perc >= 0.8:
|
|
res += [screen.markup.RED + text + screen.markup.RESET]
|
|
elif perc >= 0.5:
|
|
res += [screen.markup.YELLOW + text + screen.markup.RESET]
|
|
else:
|
|
res += [text]
|
|
|
|
if res:
|
|
return "Autostop:\n " + ("\n ".join(res))
|
|
else:
|
|
return ''
|
|
|
|
|
|
class AbstractCriteria:
|
|
""" parent class for all criterias """
|
|
RC_TIME = 21
|
|
RC_HTTP = 22
|
|
RC_NET = 23
|
|
RC_STEADY = 33
|
|
|
|
def __init__(self):
|
|
self.log = logging.getLogger(__name__)
|
|
self.cause_second = None
|
|
|
|
@staticmethod
|
|
def count_matched_codes(codes_regex, codes_dict):
|
|
""" helper to aggregate codes by mask """
|
|
total = 0
|
|
for code, count in codes_dict.items():
|
|
if codes_regex.match(str(code)):
|
|
total += count
|
|
return total
|
|
|
|
def notify(self, aggregate_second):
|
|
""" notification about aggregate data goes here """
|
|
raise NotImplementedError("Abstract methods requires overriding")
|
|
|
|
def get_rc(self):
|
|
""" get return code for test """
|
|
raise NotImplementedError("Abstract methods requires overriding")
|
|
|
|
def explain(self):
|
|
""" long explanation to show after test stop """
|
|
raise NotImplementedError("Abstract methods requires overriding")
|
|
|
|
def widget_explain(self):
|
|
""" short explanation to display in right panel """
|
|
return self.explain(), 0
|
|
|
|
@staticmethod
|
|
def get_type_string():
|
|
""" returns string that used as config name for criteria """
|
|
raise NotImplementedError("Abstract methods requires overriding")
|
|
|
|
|
|
class AvgTimeCriteria(AbstractCriteria):
|
|
""" average response time criteria """
|
|
|
|
@staticmethod
|
|
def get_type_string():
|
|
return 'time'
|
|
|
|
def __init__(self, autostop, param_str):
|
|
AbstractCriteria.__init__(self)
|
|
self.seconds_count = 0
|
|
self.rt_limit = tankcore.expand_to_milliseconds(param_str.split(',')[0])
|
|
self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[1])
|
|
self.autostop = autostop
|
|
|
|
def notify(self, aggregate_second):
|
|
if aggregate_second.overall.avg_response_time > self.rt_limit:
|
|
if not self.seconds_count:
|
|
self.cause_second = aggregate_second
|
|
|
|
self.log.debug(self.explain())
|
|
|
|
self.seconds_count += 1
|
|
self.autostop.add_counting(self)
|
|
if self.seconds_count >= self.seconds_limit:
|
|
return True
|
|
else:
|
|
self.seconds_count = 0
|
|
|
|
return False
|
|
|
|
def get_rc(self):
|
|
return self.RC_TIME
|
|
|
|
def explain(self):
|
|
items = (self.rt_limit, self.seconds_count, self.cause_second.time)
|
|
return "Average response time higher than %sms for %ss, since %s" % items
|
|
|
|
def widget_explain(self):
|
|
items = (self.rt_limit, self.seconds_count, self.seconds_limit)
|
|
return "Avg Time >%sms for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit
|
|
|
|
|
|
class HTTPCodesCriteria(AbstractCriteria):
|
|
""" HTTP codes criteria """
|
|
|
|
@staticmethod
|
|
def get_type_string():
|
|
return 'http'
|
|
|
|
def __init__(self, autostop, param_str):
|
|
AbstractCriteria.__init__(self)
|
|
self.seconds_count = 0
|
|
self.codes_mask = param_str.split(',')[0].lower()
|
|
self.codes_regex = re.compile(self.codes_mask.replace("x", '.'))
|
|
self.autostop = autostop
|
|
|
|
level_str = param_str.split(',')[1].strip()
|
|
if level_str[-1:] == '%':
|
|
self.level = float(level_str[:-1]) / 100
|
|
self.is_relative = True
|
|
else:
|
|
self.level = int(level_str)
|
|
self.is_relative = False
|
|
self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])
|
|
|
|
|
|
def notify(self, aggregate_second):
|
|
matched_responses = self.count_matched_codes(self.codes_regex, aggregate_second.overall.http_codes)
|
|
if self.is_relative:
|
|
if aggregate_second.overall.RPS:
|
|
matched_responses = float(matched_responses) / aggregate_second.overall.RPS
|
|
else:
|
|
matched_responses = 0
|
|
self.log.debug("HTTP codes matching mask %s: %s/%s", self.codes_mask, matched_responses, self.level)
|
|
|
|
if matched_responses >= self.level:
|
|
if not self.seconds_count:
|
|
self.cause_second = aggregate_second
|
|
|
|
self.log.debug(self.explain())
|
|
|
|
self.seconds_count += 1
|
|
self.autostop.add_counting(self)
|
|
if self.seconds_count >= self.seconds_limit:
|
|
return True
|
|
else:
|
|
self.seconds_count = 0
|
|
|
|
return False
|
|
|
|
def get_rc(self):
|
|
return self.RC_HTTP
|
|
|
|
def get_level_str(self):
|
|
""" format level str """
|
|
if self.is_relative:
|
|
level_str = str(100 * self.level) + "%"
|
|
else:
|
|
level_str = self.level
|
|
return level_str
|
|
|
|
def explain(self):
|
|
items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.cause_second.time)
|
|
return "%s codes count higher than %s for %ss, since %s" % items
|
|
|
|
def widget_explain(self):
|
|
items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.seconds_limit)
|
|
return "HTTP %s>%s for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit
|
|
|
|
|
|
class NetCodesCriteria(AbstractCriteria):
|
|
""" Net codes criteria """
|
|
|
|
@staticmethod
|
|
def get_type_string():
|
|
return 'net'
|
|
|
|
def __init__(self, autostop, param_str):
|
|
AbstractCriteria.__init__(self)
|
|
self.seconds_count = 0
|
|
self.codes_mask = param_str.split(',')[0].lower()
|
|
self.codes_regex = re.compile(self.codes_mask.replace("x", '.'))
|
|
self.autostop = autostop
|
|
|
|
level_str = param_str.split(',')[1].strip()
|
|
if level_str[-1:] == '%':
|
|
self.level = float(level_str[:-1]) / 100
|
|
self.is_relative = True
|
|
else:
|
|
self.level = int(level_str)
|
|
self.is_relative = False
|
|
self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])
|
|
|
|
|
|
def notify(self, aggregate_second):
|
|
codes = copy.deepcopy(aggregate_second.overall.net_codes)
|
|
if '0' in codes.keys():
|
|
codes.pop('0')
|
|
matched_responses = self.count_matched_codes(self.codes_regex, codes)
|
|
if self.is_relative:
|
|
if aggregate_second.overall.RPS:
|
|
matched_responses = float(matched_responses) / aggregate_second.overall.RPS
|
|
else:
|
|
matched_responses = 0
|
|
self.log.debug("Net codes matching mask %s: %s/%s", self.codes_mask, matched_responses, self.level)
|
|
|
|
if matched_responses >= self.level:
|
|
if not self.seconds_count:
|
|
self.cause_second = aggregate_second
|
|
|
|
self.log.debug(self.explain())
|
|
|
|
self.seconds_count += 1
|
|
self.autostop.add_counting(self)
|
|
if self.seconds_count >= self.seconds_limit:
|
|
return True
|
|
else:
|
|
self.seconds_count = 0
|
|
|
|
return False
|
|
|
|
def get_rc(self):
|
|
return self.RC_NET
|
|
|
|
def get_level_str(self):
|
|
""" format level str """
|
|
if self.is_relative:
|
|
level_str = str(100 * self.level) + "%"
|
|
else:
|
|
level_str = self.level
|
|
return level_str
|
|
|
|
def explain(self):
|
|
items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.cause_second.time)
|
|
return "%s net codes count higher than %s for %ss, since %s" % items
|
|
|
|
def widget_explain(self):
|
|
items = (self.codes_mask, self.get_level_str(), self.seconds_count, self.seconds_limit)
|
|
return "Net %s>%s for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit
|
|
|
|
|
|
class QuantileCriteria(AbstractCriteria):
|
|
""" quantile criteria """
|
|
|
|
@staticmethod
|
|
def get_type_string():
|
|
return 'quantile'
|
|
|
|
def __init__(self, autostop, param_str):
|
|
AbstractCriteria.__init__(self)
|
|
self.seconds_count = 0
|
|
self.quantile = float(param_str.split(',')[0])
|
|
self.rt_limit = tankcore.expand_to_milliseconds(param_str.split(',')[1])
|
|
self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[2])
|
|
self.autostop = autostop
|
|
|
|
def notify(self, aggregate_second):
|
|
if not (self.quantile in aggregate_second.overall.quantiles.keys()):
|
|
self.log.warning("No quantile %s in %s", self.quantile, aggregate_second.overall.quantiles)
|
|
if self.quantile in aggregate_second.overall.quantiles.keys() \
|
|
and aggregate_second.overall.quantiles[self.quantile] > self.rt_limit:
|
|
if not self.seconds_count:
|
|
self.cause_second = aggregate_second
|
|
|
|
self.log.debug(self.explain())
|
|
|
|
self.seconds_count += 1
|
|
self.autostop.add_counting(self)
|
|
if self.seconds_count >= self.seconds_limit:
|
|
return True
|
|
else:
|
|
self.seconds_count = 0
|
|
|
|
return False
|
|
|
|
def get_rc(self):
|
|
return self.RC_TIME
|
|
|
|
def explain(self):
|
|
items = (self.quantile, self.rt_limit, self.seconds_count, self.cause_second.time)
|
|
return "Percentile %s higher than %sms for %ss, since %s" % items
|
|
|
|
def widget_explain(self):
|
|
items = (self.quantile, self.rt_limit, self.seconds_count, self.seconds_limit)
|
|
return "%s%% >%sms for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit
|
|
|
|
|
|
class SteadyCumulativeQuantilesCriteria(AbstractCriteria):
|
|
""" quantile criteria """
|
|
|
|
@staticmethod
|
|
def get_type_string():
|
|
return 'steady_cumulative'
|
|
|
|
def __init__(self, autostop, param_str):
|
|
AbstractCriteria.__init__(self)
|
|
self.seconds_count = 0
|
|
self.hash = ""
|
|
self.seconds_limit = tankcore.expand_to_seconds(param_str.split(',')[0])
|
|
self.autostop = autostop
|
|
|
|
def notify(self, aggregate_second):
|
|
hash = json.dumps(aggregate_second.cumulative.quantiles)
|
|
logging.debug("Cumulative quantiles hash: %s", hash)
|
|
if self.hash == hash:
|
|
if not self.seconds_count:
|
|
self.cause_second = aggregate_second
|
|
|
|
self.log.debug(self.explain())
|
|
|
|
self.seconds_count += 1
|
|
self.autostop.add_counting(self)
|
|
if self.seconds_count >= self.seconds_limit:
|
|
return True
|
|
else:
|
|
self.seconds_count = 0
|
|
|
|
self.hash = hash
|
|
return False
|
|
|
|
def get_rc(self):
|
|
return self.RC_STEADY
|
|
|
|
def explain(self):
|
|
items = (self.seconds_count, self.cause_second.time)
|
|
return "Cumulative percentiles are steady for %ss, since %s" % items
|
|
|
|
def widget_explain(self):
|
|
items = (self.seconds_count, self.seconds_limit)
|
|
return "Steady for %s/%ss" % items, float(self.seconds_count) / self.seconds_limit
|
|
|
|
class TimeLimitCriteria(AbstractCriteria):
|
|
""" time limit criteria """
|
|
|
|
@staticmethod
|
|
def get_type_string():
|
|
return 'limit'
|
|
|
|
def __init__(self, autostop, param_str):
|
|
AbstractCriteria.__init__(self)
|
|
self.start_time = time.time()
|
|
self.end_time = time.time()
|
|
self.time_limit = tankcore.expand_to_seconds(param_str)
|
|
|
|
def notify(self, aggregate_second):
|
|
self.end_time = time.time()
|
|
return (self.end_time - self.start_time) > self.time_limit
|
|
|
|
|
|
def get_rc(self):
|
|
return self.RC_TIME
|
|
|
|
def explain(self):
|
|
return "Test time elapsed. Limit: %ss, actual time: %ss" % (self.time_limit, self.end_time - self.start_time)
|
|
|
|
def widget_explain(self):
|
|
return "Time limit: %ss, actual time: %ss" % (self.time_limit, self.end_time - self.start_time)
|