valitydev/yandex-tank
14
0
Fork 0
mirror of https://github.com/valitydev/yandex-tank.git synced 2024-11-06 02:15:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Initial import

Browse Source
This commit is contained in:
Andrey Pohilko 2012-07-23 15:22:09 +04:00
parent 5d7959538e
commit dac1618da9
21 changed files with 5723 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

570
Lunapark.pm Normal file
View File

@ -0,0 +1,570 @@
package Lunapark;
require Exporter;
@ISA = "Exporter";
@EXPORT = qw(formatDate formatFn lp_log sleep gettimeofday getconfig frps time2sec frps_cut const_f frps_real_fps move_logs MinSec getTankAliaces formTD curLPproc UserDialog UserDialogSymbol getlocks lp_warn lp_big_warn format_bytes terminal_size formatTS lp_conf save_conf read_conf update_conf update_value_conf parse_monitoring_config);
use warnings;
use strict;
use Term::ReadKey;
use Time::HiRes qw(sleep gettimeofday);
use Time::Local;
use Config::General;
use Term::ANSIColor;
use Data::Dumper;
use XML::Simple;
##################
### Date Formating
###### Format for logs
sub formatDate($) {
my $t = shift;
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($t);
return sprintf("%04d-%02d-%02d %02d:%02d:%02d",$year+1900,$mon+1,$mday,$hour,$min,$sec);
}
###### Format for filename
sub formatFn($) {
my $t = shift;
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($t);
return sprintf("%d%02d%02d-%02d%02d%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
}
sub formatTS($) {
my $t = shift;
my $time;
if ($t =~ /(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/) {
$time = timelocal($6, $5, $4,$3, $2-1, $1);
}
return $time;
}
###### Format for elapsed/remaining time
sub MinSec($$$) {
($_, my $fmsec, my $frmt)=@_;
my $pref = "";
$pref = "-" if $_ < 0;
$_ = abs($_);
my $res = '';
my $hour = int($_/3600000);
$_ -= $hour*3600000;
if ($hour != 0) {
$res .= $hour.'h';
}
my $min = int($_/60000);
$_ -= $min*60000;
if ($min!=0) {
$res .= $min.'m';
}
my $sec=int($_/1000);
if (not $fmsec) {
if ( ($_-$sec*1000) > 500) {
$sec++;
}
}
if ($sec != 0) {
$res .= $sec.'s';
}
my $msec = $_-$sec*1000;
if (($msec != 0)&&($fmsec)) {
if ($frmt != 0) {
$res .= ":";
}
$res .= substr('00'.$msec, length('00'.$msec)-3);
}
if ($res eq '') {
$res = '000';
}
if ($frmt !=0 ) {
$res = sprintf "%2.2d:%2.2d:%2.2d", $hour, $min, $sec;
if ($fmsec) {
$res .= sprintf ":%3.3d",$msec;
}
}
return $pref.$res;
}
####################
### Output Formating
sub formTD($$) {
my ($l, $d) = @_;
my $left = int(($l-length($d))/2);
my $right = $l - length($d) - $left;
return (" "x$left).$d.(" "x$right);
}
###################
#### Interactive
sub UserDialog($) {
my $f = shift;
print $f;
my $str = <STDIN>;
chomp($str);
return $str;
}
sub UserDialogSymbol($) {
my $f = shift;
print $f;
ReadMode 'cbreak';
my $key = ReadKey(0);
ReadMode 'normal';
print "\n";
return $key;
}
##################
#### System
sub getTankAliaces() {
my %al = ();
my $t = `hostname -f`;
chomp($t);
my $ping = `ping $t -c 1`;
if ($ping =~ /^PING $t \((.+?)\)/) {
$al{$1} = $t;
}
$t =~ s/\./-dummy\./;
$ping = `ping $t -c 1`;
if ($ping =~ /^PING $t \((.+?)\)/) {
$al{$1} = $t;
}
return \%al;
}
### List of all processes running by current Lunapark
sub curLPproc($); # prototype
sub curLPproc($) {
my $pid = shift;
my $lpdec = 0;
open(my $PS, "ps uh --ppid $pid |");
my @kill = ();
while(<$PS>) {
if ($_ =~ /^\S+\s+(\S+)\s+/) {
lp_log("Detected process to kill: ".$_);
unshift @kill, $1;
for my $cpid (@{curLPproc($1)}) {
unshift @kill, $cpid;
}
} else {
lp_log("Skipping line: ".$_);
}
}
close($PS);
return \@kill;
}
sub lp_conf($) {
my $ref = shift;
open(my $LP, ">>lp.conf");
print $LP $_." = ".$ref->{$_}."\n" for (keys %{$ref});
close($LP);
}
sub save_conf($) {
lp_log("Saving lp.conf");
my $ref = shift;
open(my $LP, ">lp.conf");
for (keys %{$ref}) {
next if $_ eq '[DEFAULT]';
unless(ref($ref->{$_})) {
print $LP $_." = ".$ref->{$_}."\n";
}
}
close($LP);
}
sub update_conf($$) {
my ($pri, $sec) = @_;
for (keys %{$sec}) {
$pri->{$_} = $sec->{$_} unless defined $pri->{$_};
}
return $pri;
}
sub update_value_conf($$) {
my ($pri, $sec) = @_;
for (keys %{$sec}) {
if ((not defined $pri->{$_}) || ($pri->{$_} ne $sec->{$_})) {
$pri->{$_} = $sec->{$_};
}
print $_."\n" if not defined $sec->{$_};
}
return $pri;
}
### Logging
sub lp_log($) {
my $str = shift;
chomp $str;
my ($sec, $microsec, $ts) = (gettimeofday, time);
my $mls = $microsec/1000000;
my $ms = sprintf("%03d", int(1000*$mls));
$0 =~ /^.+\/(.+?)$/;
my $script = $1;
if ($sec && $ms && $script && $str) {
open(my $DEBUG, ">>lunapark.log") or die "Cannot open log-file\n";
print $DEBUG "[".formatDate($sec).".$ms] [$script] ".$str."\n";
close($DEBUG);
} else {
print $str."\n";
print "Warning: Can't open log file [$sec / $ms / $script / $str]\n";
}
}
sub move_logs($) {
my $i = shift;
if (!$i->{jobno}) {
lp_log("No jobno, skip move logs");
return;
}
my $prefix = "$i->{jobno}_$i->{fn}";
my $lf = "logs/$i->{jobno}";
mkdir("logs") unless (-r 'logs');
mkdir($lf) unless (-r $lf);
lp_log("Moving logs to $lf");
`mv $i->{phantom_log_name} $lf/phout_$prefix.txt` if -r "$i->{phantom_log_name}";
`mv $i->{preproc_log_name} $lf/prepr_$prefix.txt` if -r "$i->{preproc_log_name}";
`mv $i->{answ_log_name} $lf/answ_$prefix.txt` if $i->{writelog} && -r $i->{answ_log_name};
# `mv $i->{script_log} $lf/script_$prefix.log` if $i->{script} && -r $i->{script_log};
`cp $i->{config} $lf/load.conf` if -r "$i->{config}";
`cp lp.conf $lf/lp.conf` if -r "lp.conf";
`mv lunapark_error.log $lf/lunapark_error_$i->{jobno}.log` if -r "lunapark_error.log";
`mv lunapark.log $lf/lunapark_$i->{jobno}.log` if -r "lunapark.log";
`mv phantom.conf $lf/phantom_$i->{jobno}.conf` if -r "phantom.conf";
`mv sql.log $lf/sql.log` if -r "sql.log";
`mv phantom.log $lf/phantom_$i->{jobno}.log` if -r "phantom.log";
`mv fantom-debug.log $lf/fantom-debug.log` if -r "fantom-debug.log";
`mv phantom_stat.log $lf/phantom_stat_$i->{jobno}.log` if -r "phantom_stat.log";
if (-r 'monitoring.log') {
`mv monitoring.log $lf/monitoring_$i->{jobno}.log`;
}
if (`ls monitoring_agent_*.log 2> /dev/null| wc -l`) {
`mv monitoring_agent_*.log $lf/ 2> /dev/null`;
}
if (-r "monitoring_$i->{jobno}.data") {
`mv monitoring_$i->{jobno}.data $lf/monitoring_$i->{jobno}.data`;
}
if (defined $i->{monitoring_tmp} and -r $i->{monitoring_tmp}) {
`mv $i->{monitoring_tmp} $lf/monitoring_$i->{jobno}.conf`;
}
}
### Config Parsing
sub getconfig($) {
return {} unless (-r $_[0]);
my $conf = new Config::General($_[0]);
my %conf = $conf->getall;
my @loads = ();
if (defined $conf{load}) {
if (ref($conf{load}) eq "ARRAY") {
for my $load (@{$conf{load}}) {
my @tmp = split("[)][ ]+", $load);
if (scalar(@tmp) == 1) {
push @loads, @tmp;
} else {
push @loads, $tmp[$_].")" for (0 .. $#tmp-1);
push @loads, $tmp[$#tmp];
}
}
} else {
my @tmp = split("[)][ ]+", $conf{load});
if (scalar(@tmp) == 1) {
push @loads, @tmp;
} else {
push @loads, $tmp[$_].")" for (0 .. $#tmp-1);
push @loads, $tmp[$#tmp];
}
}
@{$conf{loads}} = @loads;
$conf{loads_str} = join(";", @loads);
}
return \%conf;
}
sub read_conf($) {
die "Cannot open config '$_[0]'" unless (-r $_[0]);
my $conf = new Config::General($_[0]);
my %conf = $conf->getall;
return \%conf;
}
###
sub getlocks() {
my @ts = glob("/var/lock/lunapark*.lock");
return \@ts;
}
###### [stepper.pl] functions
sub time2sec ($) {
my $t = shift;
if ($t =~ /(\d+)h$/) {
return $1*3600;
} elsif ($t =~ /(\d+)(m|min)$/) {
return $1*60;
} elsif ($t =~ /(\d+)(s|sec|)$/) {
return $1;
}
die "Wrong time format\n";
}
sub countAmmo($) {
my $a = shift;
my $cnt = 0;
while($a =~ /(\d+)\s+(\d+)\n/g) {
$cnt += $1*$2;
}
return $cnt;
}
### const fractional scheme from load.conf
sub const_f ($$) {
my ($req, $dur_orig) = @_;
if ($req =~ /(\d+)\/(\d+)/ and $dur_orig) {
my ($a, $b, $dur, $e) = ($1, $2, time2sec($dur_orig), int($1/$2));
my $fr = sprintf("%.3f", $1/$2);
$a = $a % $b;
$req = "$a/$b";
my $ls = "$dur,const_f,$fr,$fr,($req,$dur_orig)\n";
my $out = "";
my $tail = $dur % $b;
for (my $i = 1; $i <= int($dur/$b); $i++) {
$out .= frps($req);
}
$out .= frps_cut($tail, $req) if $tail;
if ($e > 0) {
$out = frps_expand($out, $e);
}
return ($out, $ls, countAmmo($out));
} else {
die "error in 'const_f' function. rps:$req, duration:$dur_orig\n";
}
}
### fractional rps
sub frps_print($$) {
my ($s, $t) = @_;
my $out = "";
for (my $i = 1; $i<= $t; $i++) {
$out .= "$s 1\n";
}
return $out;
}
sub frps_vv($) {
return '0' if $_[0] eq '1';
return '1' if $_[0] eq '0';
}
sub frps_scheme($) {
my $c = shift;
my $out = "";
for (my $i = 1; $i <= $c->{chunks}; $i++) {
$out .= frps_print($c->{first}, $c->{per_chunk});
$c->{num1} -= $c->{per_chunk};
$out .= frps_print(frps_vv($c->{first}), 1);
$c->{num0} --;
}
$out .= frps_print($c->{first}, $c->{num1});
$out .= frps_print(frps_vv($c->{first}), $c->{num0});
}
sub frps($) {
my $f = shift;
if ($f =~ /(\d+)\/(\d+)/) {
my %c = ();
my ($num1, $num0) = ($1, $2-$1);
if ($num1 > $num0) {
($c{per_chunk}, $c{space}, $c{first}) = (int($num1/$num0), $num1%$num0, '1');
$c{chunks} = int($num0);
($c{num1}, $c{num0}) = ($num1, $num0);
} else {
($c{per_chunk}, $c{space}, $c{first}) = (int($num0/$num1), $num0%$num1, '0');
$c{chunks} = int($num1);
($c{num1}, $c{num0}) = ($num0, $num1);
}
return frps_scheme(\%c);
} else {
return "0";
}
}
sub frps_cut($$) {
my ($c, $r) = @_;
if ($r =~ /(\d+)\/(\d+)/) {
my ($a, $b) = ($1, $2);
if ($c < $2) {
my ($frps, $out, $cnt) = (frps($r), "", 0);
while ($frps =~ /(\d+) (\d+)\n/g) {
$cnt++;
$out .= "$1 $2\n";
last if $cnt == $c;
}
return $out;
} else {
die "Wrong cut:$c for rps $r\n";
}
} else {
die "Wrong rps format in 'frps_cut' function\n";
}
}
### Expand rps<1 to rps>1
sub frps_expand($$) {
my ($s, $e) = @_;
my $out = "";
while ($s =~ /(\d+) (\d+)\n/g) {
$out .= ($1+$e)." ".$2."\n";
}
return $out;
}
### Comparison of scheme rps and real rps
sub frps_real_fps($$) {
my ($s, $r) = @_;
my ($reqs, $dur) = 0;
while ($s =~ /(\d+) (\d+)\n/g) {
$reqs += $1;
$dur += $2;
}
my $real_rps = $reqs/$dur;
$r =~ /(\d+)\/(\d+)/;
my $rps = $1/$2;
return sprintf("%.6f", 100*abs($real_rps - $rps)/($rps));
}
sub lp_warn($) {
print color 'yellow';
print $_[0]."\n";
print color 'reset';
}
sub lp_big_warn {
lp_warn("########################");
lp_warn("####### Warning ########");
lp_warn("########################");
}
sub format_bytes($) {
my $b = shift;
my @suff = ("B", "K", "M", "G", "T", "P", "E", "Z", "Y");
my $prev = $b.$suff[0];
for (1 .. $#suff) {
my $a = sprintf("%.1f", $b/(1024**$_));
if ($a > 1) {
$prev = $a.$suff[$_];
next;
} else {
return $prev;
}
}
return $prev;
}
sub terminal_size {
my $w = `tput cols`;
chomp($w);
my $h = `tput lines`;
chomp($h);
return ($w, $h);
}
sub parse_monitoring_config($) {
my $file = shift;
my %default = (
'CPU' => 'idle,user,system,iowait',
'System' => 'csw,int',
'Memory' => 'free,used',
'Disk' => 'read,write',
'Net' => 'recv,send',
'interval' => 1,
'priority' => 0,
'comment' => '',
);
my @metrics = ('CPU', 'System', 'Memory', 'Disk', 'Net');
my @default_summary;
for my $m (@metrics) {
push @default_summary, map {$m."_".$_} split(",", $default{$m});
}
$default{metrics} = join(",", @default_summary);
my %targets = ();
my $conf = XML::Simple->new()->XMLin($file, ForceArray => 1);
for my $host (@{$conf->{Host}}) {
if ($host->{address}) {
my $base_count = 0;
my $adr = $host->{address};
my @summary;
# metrics
for my $m (@metrics) {
if (defined $host->{$m}) {
if (defined $host->{$m}->[0]->{measure}) {
$targets{$adr}{$m} = $host->{$m}->[0]->{measure};
} else {
$targets{$adr}{$m} = $default{$m};
}
push @summary, map {$m."_".$_} split(",", $targets{$adr}{$m});
} else {
$base_count ++;
}
}
$targets{$adr}{metrics} = join(",", @summary);
# custom
if (defined $host->{Custom}) {
if ($host->{Custom}) {
if (ref($host->{Custom}) eq 'ARRAY') {
for my $c (@{$host->{Custom}}) {
push @{$targets{$adr}{custom}}, $c;
}
} elsif (ref($host->{Custom}) eq 'HASH') {
push @{$targets{$adr}{custom}}, $host->{Custom};
}
}
} else {
if ($base_count == @metrics) {
%{$targets{$adr}} = %default;
}
}
# meta
for my $m ('interval', 'priority', 'comment') {
if ($host->{$m}) {
$targets{$adr}{$m} = $host->{$m};
} else {
$targets{$adr}{$m} = $default{$m};
}
}
}
}
return \%targets;
}
sub create_agent_config($) {
my $conf = shift;
my $o = "[main]\n";
$o .= "interval = ".$conf->{interval}."\n\n";
$o .= "[metric]\n";
$o .= "names = cpu-la,mem,cpu_stats\n";
return $o;
}
sub create_monitoring_summary($) {
my $conf = shift;
}
1;

3
db.conf Normal file
View File

@ -0,0 +1,3 @@
[DEFAULT]
# uncomment this setting if you have implemeted API that accepts data from console tank
#http_base=http://my-api-implementation-host/

36
debian/changelog vendored Normal file
View File

@ -0,0 +1,36 @@
yandex-load-tank-base (0.1.7) common; urgency=low
* remove API host
-- Andrey Pohilko (undera) <undera@yandex-team.ru> Thu, 07 Jun 2012 16:50:35 +0400
yandex-load-tank-base (0.1.6) common; urgency=low
* lunapark -c creates default config
* man pages shipped
-- Andrey Pohilko (undera) <undera@yandex-team.ru> Wed, 06 Jun 2012 19:06:05 +0400
yandex-load-tank-base (0.1.5) common; urgency=low
* fixed: phout import does not pass load scheme to API
-- Andrey Pohilko (undera) <undera@yandex-team.ru> Mon, 04 Jun 2012 15:34:09 +0400
yandex-load-tank-base (0.1.4) common; urgency=low
* reverted db.conf
-- Andrey Pohilko (undera) <undera@yandex-team.ru> Fri, 01 Jun 2012 20:50:46 +0400
yandex-load-tank-base (0.1.3) common; urgency=low
* rebuild for common
-- Andrey Pohilko (undera) <undera@yandex-team.ru> Fri, 01 Jun 2012 20:05:26 +0400
yandex-load-tank-base (0.1.0) hardy; urgency=low
* Initial release of internal Yandex code
-- Andrey Pohilko (undera) <undera@yandex-team.ru> Tue, 22 May 2012 15:32:02 +0400

1
debian/compat vendored Normal file
View File

@ -0,0 +1 @@
7

16
debian/control vendored Normal file
View File

@ -0,0 +1,16 @@
Source: yandex-load-tank-base
Section: yandex
Priority: extra
Maintainer: Yandex Load Testing Team <load@yandex-team.ru>
Build-Depends: debhelper (>= 7)
Standards-Version: 3.8.3
Package: yandex-load-tank-base
Architecture: all
Depends: perl, perl-base, perl-modules, libjson-perl, libconfig-general-perl, libxml-simple-perl, libnet-ip-perl, libfile-which-perl, libterm-readkey-perl,
phantom(>=0.14.0~pre29), phantom-ssl,
python (>=2.6), python-simplejson, python-progressbar,
Suggests: yandex-load-monitoring(>=0.1-30)
Description: Yandex.Lunapark (Load Testing) Tank part,
turns common Debian compatible computer into heavy machinegun

2
debian/dirs vendored Normal file
View File

@ -0,0 +1,2 @@
usr/lib/lunapark
etc/lunapark

1
debian/files vendored Normal file
View File

@ -0,0 +1 @@
yandex-load-tank-base_0.1.7_all.deb yandex extra

9
debian/install vendored Normal file
View File

@ -0,0 +1,9 @@
fantom.py usr/lib/lunapark
lunapark usr/lib/lunapark
prd.pl usr/lib/lunapark
preproc.pl usr/lib/lunapark
stepper.py usr/lib/lunapark
Lunapark.pm usr/share/perl5
db.conf /etc/lunapark
load.conf.example /etc/lunapark
yandex_load_lunapark usr/lib/lunapark

2
debian/manpages vendored Normal file
View File

@ -0,0 +1,2 @@
lunapark.1
lunapark.ru.1

8
debian/postinst vendored Normal file
View File

@ -0,0 +1,8 @@
#!/bin/sh
ln -sf /usr/lib/lunapark/lunapark /usr/local/bin/
ln -sf /usr/lib/lunapark/prd.pl /usr/local/bin/
ln -sf /usr/lib/lunapark/preproc.pl /usr/local/bin/
ln -sf /usr/lib/lunapark/stepper.py /usr/local/bin/
ln -sf /usr/lib/lunapark/fantom.py /usr/local/bin/

1148
fantom.py Executable file
View File

File diff suppressed because it is too large Load Diff

31
load.conf.example Normal file
View File

@ -0,0 +1,31 @@
#### Lunapark Config File ####
## Basic Section ##
# Адрес и порт тестируемой машинки
address=127.0.0.1:80
#Схема нагрузки
load = const (10,10m)
# Перечисление заголовков и GET запросов
header_http = 1.1
header_connection = close
header_host = target.yandex.net
uri = /
## Advanced Section ##
#ammofile=test.ammo
#ssl=1
#autostop = http(5xx,100%,1)
#instances=10
#writelog=1
#task=LOAD-999
#monitoring_config=<путь к файлу>
#inform = username
#time_periods = 10 45 50 100 150 300 500 1s 1500 2s 3s 10s # крайнее значение 10s является коннект таймаутом
#job_name = Краткое имя стрельбы
#job_dsc = Описание стрельбы
#ver = Версия пакета/коммита/whatever
## Expert Section ##
#instances_schedule = line (1,1000,10m)
#tank_type=2
#gatling_ip = 141.8.153.82 141.8.153.81

1447
lunapark Executable file
View File

File diff suppressed because it is too large Load Diff

79
lunapark.1 Normal file
View File

@ -0,0 +1,79 @@
.TH lunapark 1 "June 6, 2012" "" "LUNAPARK"
.SH NAME
lunapark \- an utility for measuring performance of web servers
.SH SYNOPSIS
.B lunapark ammo_file | {-c |--config} [config_file] ammo_file | {-s| --skip-step} ammo_file | {-o|--step-only} ammo_file | {-g | --gatling} ammo_file | ammo_file --script | --clear | --clearall | ammo_file {-p|--phantomlog} phout_file | --manual-start ammo_file | {-i|--instances} <NUM> ammo_file | --address <IP>:<PORT> ammo_file
.SH DESCRIPTION
This manual page explains the using lunapark utility
.B lunapark
program. This program is a high performance hit-based utility for web servers testing.
.SH OPTIONS
.PP
\fB-c\fP ,\fB --config exec\fP lunapark with given configuration file config_file. By default load.conf is used, located in your current directory. Execution without params creates config_file with default options.
.PP
\fB-s\fP, \fB--skip-step\fP run test without generation timestamped requests file, using already existed ammo.stpd from previous tests. Useful when you need exactly the same test but don't want to wait ammo.stpd generation. Needs ammo.stpd and lp.conf.
.PP
\fB-o\fP,\fB --step-only\fP prepare timestamped requests file for further tests execution with -s option.
.PP
\fB-g\fP,\fB --gatling\fP enable several local IP addresses usage in test. Effective in avoiding sockets/local ports depletion.
.PP
\fB--clear\fP clear current directory from tests artefacts but keep ./logs directory. Use --clearall for complete artefacts deletion, with ./logs.
.PP
\fB-p\fP ,\fB --phantomlog\fP load to lunapark custom tests results generated by any other utility. Test results file must be in phout_file format.
.PP
\fB--manual-start\fP lunapark generates all data for test execution and requests confirmation for start. Helpful in giving load at precisely chosen time.
.PP
\fB-i \fP, \fB--instances\fP set instances number. Overrides value in config_file.
.PP
\fB--address\fP set target's IP and port. Overrides values in config_file.
.B
\fB ammo_file\fP
file with requests in req-style or uri-style.
.br
.B
\fB phout_file\fP
file with non-aggregated per-request data receiving during the test.
.br
.B
\fB phout_file\fP
configuration file described
.br
.SH FILES
fantom.py
preproc.pl
stepper.py
Lunapark.pm
lunapark
load.conf.example
db.conf
prd.pl
yandex_load_lunapark
yandex_load_lunapark/stepper.py
yandex_load_lunapark/__init__.py
yandex_load_lunapark/status.py
.SH NOTES
Be careful with config_file tuning. Incorrect configuraton could lead to network devices or/and web servers overload.
.SH EXAMPLE
Simple configuration file making requests to / with constant rate 10 requests per second for 10 minutes. Server's IP and port are 127.0.0.1:80
#### BEGIN ####
address=127.0.0.1:80 #Target's address and port
load = const (10,10m) #Load scheme
header_http = 1.1
header_connection = close
header_host = target.yandex.net
uri = /
#### END ####
More complex example: ammo_file in req_style, web server has ipv6/ssl enabled, load profile is combined from different primitives.
Also there are enabled request/answer logging and jabber notification. Test results are uploading to Lunapark framework for detailed analysis
#### BEGIN ####
address=2a02:6b8:0:c1f::100:1:80 #Target's address and port
load = const (10,10m) line(10,100,10m) step(100,500,100,10m) #Load scheme
ssl = 1
inform = username
task = LOAD-999
#### END ####

79
lunapark.ru.1 Normal file
View File

@ -0,0 +1,79 @@
.TH lunapark 1 "June 6, 2012" "" "LUNAPARK"
.SH НАЗВАНИЕ
lunapark \- Утилита для тестирования производительности веб сервисов.
.SH СИНТАКСИС
.B lunapark ammo_file | {-c |--config} [config_file] ammo_file | {-s| --skip-step} ammo_file | {-o|--step-only} ammo_file | {-g | --gatling} ammo_file | ammo_file --script | --clear | --clearall | ammo_file {-p|--phantomlog} phout_file | --manual-start ammo_file | {-i|--instances} <NUM> ammo_file | --address <IP>:<PORT> ammo_file
.SH ОПИСАНИЕ
Документация по настройке и использованию высокопроизводительной hit-based утилиты для тестирования вебсервисов.
.SH OPTIONS
.PP
\fB-c\fP ,\fB --config exec\fPиспользуется для запуска lunapark в соответствии с конфигурационным файлом config_file. По-умолчанию используется load.conf, находящийся в локальной папке. Запуск без параметров приведет к созданию config_file с полями по-умолчанию.
.PP
\fB-s\fP, \fB--skip-step\fP запуск теста без генерации файла с запросами, используя уже имеющийся ammo.stpd от предыдущего теста. Полезно? если нужно запустить точно такой же тест, а ждать генерации ammo.stpd не хочется. Для запуска нужны файла ammo.stpd и lp.conf.
.PP
\fB-o\fP,\fB --step-only\fP только подготовка полного набора запросов для теста, сам тест откладывается.
.PP
\fB-g\fP,\fB --gatling\fP использование нескольких IP адресов при проведении теста. Может понадобиться при исчерпании сокетов/исходящих портов.
.PP
\fB--clear\fP очистка директории от служебных файлов, директория ./logs сохраняется. Для полной очистки используйте --clearall
.PP
\fB-p\fP ,\fB --phantomlog\fP загрузка данных тестирования, сгенерированных какой-либо сторонней утилитой в соответствии с форматом phout_file
.PP
\fB--manual-start\fP после генерации полного файла с запросами, стрельба становится на паузу - ожидается согласие на запуск. Удобно использовать для подачи нагрузки в точнозаданный момент.
.PP
\fB-i \fP, \fB--instances\fP принудительное задание числа потоков, перекрывает значения, указанные в config_file
.PP
\fB--address\fP указание IP и порта тестируемой машинки. Перекрывает значения в config_file
.br
.B
\fB ammo_file\fP
файл, содержащий в себе запросы к сервису в req-style или uri-style.
.br
.B
\fB phout_file\fP
файл, содержащий неагрегированные позапросные данные, получаемый в течении всего теста.
.br
.B
\fB phout_file\fP
файл, содержащий неагрегированные позапросные данные, получаемый в течении всего теста.
.br
.SH FILES
fantom.py
preproc.pl
stepper.py
Lunapark.pm
lunapark
load.conf.example
db.conf
prd.pl
yandex_load_lunapark
yandex_load_lunapark/stepper.py
yandex_load_lunapark/__init__.py
yandex_load_lunapark/status.py
.SH NOTES
Будьте осторожны при настройке config_file. При некорректной конфигурации вы можете перегрузить сетевые устройства и/или веб-сервисы.
.SH EXAMPLE
Простейший конфигурационный файл для подачи запросов к корневой странице с постоянной нагрузкой 10 запросов в секунду
в течении 10 минут, на сервис с адресом 127.0.0.1, порт 80:
#### BEGIN ####
address=127.0.0.1:80 #Адрес и порт тестируемой машинки
load = const (10,10m) #Схема нагрузки
header_http = 1.1
header_connection = close
header_host = target.yandex.net
uri = /
#### END ####
Конфигурационный файл для запуска сложного теста с поддержкой ammo_file с req-style запросами с ipv6/SSL.
Нагрузка задается разными профилями, с поддержкой логгирования, нотификацией и подключением платформы статистики Lunapark
#### BEGIN ####
address=2a02:6b8:0:c1f::100:1:80 #Адрес и порт тестируемой машинки
load = const (10,10m) line(10,100,10m) step(100,500,100,10m) #Схема нагрузки
ssl = 1
inform = username
task = LOAD-999
#### END ####

120
prd.pl Executable file
View File

@ -0,0 +1,120 @@
#!/usr/bin/perl
use strict;
use warnings;
use Time::HiRes qw(sleep);
use IO::Handle;
use Lunapark;
$| = 1; # disable buffering
sub checkReader() {
my %r;
open( my $SRC, "<step.conf" ) or die "Cannot open 'step.conf'.";
while (<$SRC>) {
$r{ammo_cnt} = $1 if ( $_ =~ /^ammo_cnt=(.+)$/ );
}
close($SRC);
if ( !$r{ammo_cnt} ) {
open( my $AM, "<ammo_cnt" );
$_ = <$AM>;
chomp;
$r{ammo_cnt} = $_;
close($AM);
}
return \%r;
}
### Return 1 if phantom.log contains 'phantom Exit' in last 5 lines. Otherwise - 0.
sub checkPhantomStop($) {
my $f = shift;
open( my $T, "tail -n 5 $f |" );
while (<$T>) {
return 1 if (/phantom Exit/);
return 1 if (/Test has ended/);
}
return 0;
}
### Return line count in file $f
sub getFlines($) {
my $f = shift;
open( my $L, "wc -l $f |" );
if ( <$L> =~ /(\d+).+$f/ ) {
return $1;
}
else {
return 0;
}
}
### Return array of last $c lines from file $f
sub getFtail($$) {
my ( $f, $c ) = @_;
open( my $T, "tail -n $c $f | " );
my @t;
while (<$T>) {
push @t, $_;
}
return \@t;
}
lp_log("Start");
#my %r = %{checkReader()};
my ( $phantom_stop, $total_lines ) = ( 0, 0 );
my $S;
my $L;
my $flag;
$flag = 1 if ($ARGV[1] && -r $ARGV[1] );
my $chunked_preproc=0;
open( $S, "<$ARGV[0]" ) or die("Have no phout file to read");
open( $L, "<$ARGV[1]" ) if $flag;
while ( 1 ) {
if ($flag) {
while (<$L>) {
print "phantom " . $_;
}
}
my $lines_read = 0;
$chunked_preproc=0;
read_S: while (<$S>) {
print $_;
# lp_log($_);
$total_lines++;
$lines_read++;
# 5000 rps is enough to have a break
if ( $lines_read > 5000 ) {
lp_log("Break each 5000 lines read");
$lines_read = 0;
$chunked_preproc=1;
last read_S;
}
}
sleep(0.01);
if (!$phantom_stop) {
$phantom_stop = checkPhantomStop("phantom.log");
if ( $phantom_stop == 1 ) {
lp_log("Phantom Stopped");
}
} else {
last if not $chunked_preproc;
}
}
close($S);
close($L) if $flag;
lp_log("Finish");

858
preproc.pl Executable file
View File

@ -0,0 +1,858 @@
#!/usr/bin/perl
use Lunapark;
use POSIX;
use List::Util qw(max);
use Data::Dumper;
use strict;
use warnings;
lp_log("Started");
our $debug = 1;
our $last_prined_sec = 0;
our @percentiles = ( 50, 75, 80, 85, 90, 95, 98, 99, 100 );
### Error codes
our %err;
$err{0} = "No error";
$err{1} = "phantom.conf not found";
$err{2} = "error in phantom.conf - timeout error";
$err{3} = "step.conf is empty";
$err{4} = "step.conf not found";
$err{5} = "can't write phout file";
$err{33} = "load-scheme in step.conf is empty";
$err{77} = "unnecessary case in log";
$err{99} = "unknown string";
our %run = ( debug => 1, );
sub dout($) {
print $_[0] if $run{debug};
}
sub percentile($$) {
my ( $y, $p ) = @_;
my @y = sort { $a <=> $b } @$y;
my $n = int( scalar(@y) * $p ) - 1;
return $y[$n];
}
sub checkStep($) {
my ( %cases, @load );
my $lp = read_conf("lp.conf");
lp_log( Dumper($lp) );
while ( $lp->{cases} =~ /'(.*?)'/g ) {
$cases{ ( $1 ? $1 : "sysempty" ) } = 1;
}
if ( !$lp->{steps} ) { lp_log("No steps!"); }
for ( split( " ", $lp->{steps} ) ) {
if ( $_ =~ /\((\d+);(\d+)\)/ ) {
for ( my $count = 0 ; $count < $2 ; $count++ ) {
push @load, $1;
}
}
}
while (@load) {
if ( $load[0] == '0' ) {
shift @load;
}
else {
last;
}
}
my $detailed = 'interval_real';
# print $lp->{detailed_time};
if ( defined $lp->{detailed_time} && $lp->{detailed_time} ) {
$detailed = $lp->{detailed_time};
}
return ( 0, \@load, \%cases, $lp->{ammo_count}, $detailed );
}
sub checkPhantom($) {
# print "Checking Phantom Conf\n";
my $path = shift;
my ( $timeout, $terr, $verr, $tverr1, $tverr2, $values ) =
( 0, 1, 1, 1, 1, "" );
open( my $SRC, "<$path/phantom.conf" ) or return 1;
while (<$SRC>) {
if ( $_ =~ /timeout = (\d+)(s|)/ ) {
$timeout = $1 . ( ( $2 eq 's' ) ? "000" : "" );
$terr = 0;
}
if ( $_ =~ /values = \{(.+)\}/ ) {
my @values = ( 0, split( " ", $1 ) );
$values = \@values;
for my $i ( 0 .. $#values ) {
if ( $values[$i] =~ /(\d+)s/ ) {
$values[$i] = $1 . "000";
}
if ( $values[$i] == $timeout ) {
$tverr1 = 0;
}
if ( $values[$i] > $timeout ) {
$tverr2 = 0;
}
}
$verr = 0;
}
}
close($SRC);
if ( $terr || $verr || $tverr1 || $tverr2 ) {
return 2;
}
else {
#print Dumper($values);
return ( 0, $values );
}
}
sub std($$) {
my ( $y, $m ) = @_;
my $sum = 0;
my $len = scalar(@$y);
for ( my $i = 0 ; $i < $len ; $i++ ) {
$sum += ( $y->[$i] - $m )**2;
}
return sprintf( "%.2f", sqrt( $sum / $len ) );
}
sub printAbout();
sub time_from_ts($) {
my $ts = shift;
my ( $sec, $min, $hour, $mday, $mon, $year ) = localtime($ts);
my $time = sprintf(
"%d%02d%02d%02d%02d%02d",
$year + 1900,
$mon + 1, $mday, $hour, $min, $sec
);
return $time;
}
sub check_case($$) {
my ( $case, $cases ) = @_;
if ( !$cases->{$case} ) {
lp_log ( Dumper( \@_ ) );
lp_log ( "errcode:77 (wrong case!)" );
exit 1;
}
}
if (@ARGV) {
if ( $ARGV[0] =~ /(--help|--about)/ ) {
print printAbout();
exit 0;
}
}
lp_log("Starting preproc...");
my $PATH = ".";
my ( $lastsec, $cur, $seconds, $count, $out, $tag, $ts, $confout, $start, $ts2 )
= ( 0, 0, 0, 0, "", "", 0, 1, 0, 0 );
my %agg;
my @errs;
my $overall;
my $tot_answ_tmp = 0;
lp_log("Checking preproc.conf... ");
#my $prconf = getconfig("preproc.conf");
my $prconf = read_conf("lp.conf");
our %prconf = %{$prconf};
lp_log("Checking preproc.conf... Done");
lp_log("Checking step.conf... ");
my ( $errstep, $load, $cases, $ammo_cnt, $detailed ) = checkStep($PATH);
lp_log("Checking step.conf... Done");
lp_log("Checking phantom.conf... ");
my ( $errph, $values ) = checkPhantom($PATH);
if ( !$values ) {
lp_log("Took time periods from lp.conf");
my @values = ( 0, split( " ", $prconf{time_periods} ) );
$values = \@values;
}
lp_log("Checking phantom.conf... Done");
if ( !$prconf{preproc_log_name} ) { die("No out file specified"); }
else { lp_log( "Out file:" . $prconf{preproc_log_name} ); }
open( my $DS, ">$prconf{preproc_log_name}" )
or die "Cannot create '$prconf{preproc_log_name}'.";
close($DS);
if ($errstep) { push @errs, $errstep; }
if ($errph) { push @errs, $errph; }
my $errs = join( "", sort { $a <=> $b } (@errs) );
my $errcode = ( $errs ? $errs : "0" );
if (@ARGV) {
if ( $ARGV[0] =~ /(--check|-ch)/ ) {
print "errcode: $errcode (" . $err{$errcode} . ")\n";
exit 0;
}
if ( $ARGV[0] =~ /(--bunny|-bn)/ ) {
print
"(\\_/)\n(O.o)\n(> <) <-- Help Bunny on his way to world domination!\n";
exit 0;
}
if ( $ARGV[0] =~ /--overall=(0|1)/ ) {
$overall = $1;
print "Overall: $overall\n";
}
}
sub printStack($) {
my $y = shift;
my @res = grep { $y->{$_} == 1 && $_ } keys %$y;
@res = sort { $a <=> $b } @res;
return \@res;
}
sub outputStack($$$$$$$$$$$) {
my (
$y, $d, $ag, $prconf, $cases, $values,
$start, $load, $errcode, $detailed, $task_data
) = @_;
my $out = '';
my @outed;
for ( sort { $a <=> $b } keys %$y ) {
# Output second.
if ( $_ > $last_prined_sec ) {
if ( $y->{$_} ) {
push @outed, $_;
my $reqps = (
( defined $load->[ $_ - $start + 1 ] )
? $load->[ $_ - $start + 1 ]
: 0
);
$reqps = ( ( $errcode == 33 ) ? '-1' : $reqps );
# Template for empty data. Content.
my $empty_out = "HTTPcode=200:0\nnetwcode=0:0\n";
$empty_out .=
"answ_time=" . $values->[0] . "-" . $values->[1] . ":0\n";
$empty_out .= "selfload=0\noutput=0\ninput=0\n";
for (
'interval_real', 'connect_time', 'send_time',
'latency', 'receive_time', 'interval_event'
)
{
$empty_out .= $_ . "_expect=0\n";
}
$empty_out .= $detailed . "_dispersion=0\n";
for (@percentiles) {
$empty_out .= $detailed . "_q" . $_ . "=0\n";
}
# Template for empty second. End.
my $out_end = "delta_plan=0\n===\n";
if ( defined $ag->{$_} ) {
# if ( defined %{ $ag->{$_} } ) {
# Output cases if exists
for my $key ( keys %{$cases} ) {
$out .= "overall=0\n";
$out .= 'time=' . time_from_ts($_) . "\n";
$out .=
"case=" . ( $key eq 'sysempty' ? '' : $key ) . "\n";
$out .= "reqps=$reqps\n";
$out .= "tasks=0\n";
if ( defined ${ $ag->{$_} }{$key} ) {
$out .= printSec( ${ $ag->{$_} }{$key},
$values, $prconf, $detailed );
}
else {
$out .= $empty_out;
}
$out .= $out_end;
}
# Output data for whole second.
$out .= "overall=1\n";
$out .= 'time=' . time_from_ts($_) . "\n";
$out .= "case=\n";
$out .= "reqps=$reqps\n";
if ( $task_data->{$_} ) {
$out .= "tasks=" . $task_data->{$_} . "\n";
}
else {
$out .= "tasks=0\n";
}
$out .= printSec( \%{ ${ $ag->{$_} }{overall} },
$values, $prconf, $detailed );
$out .= "===\n";
$last_prined_sec = $_;
}
else {
# Empty second overall = 0
for my $key ( keys %{$cases} ) {
$out .= "overall=0\n";
$out .= 'time=' . time_from_ts($_) . "\n";
$out .=
"case=" . ( $key eq 'sysempty' ? '' : $key ) . "\n";
$out .= "reqps=$reqps\n";
$out .= "tasks=0\n";
$out .= $empty_out . $out_end;
}
# Empty second overall = 1
$out .= "overall=1\n";
$out .= 'time=' . time_from_ts($_) . "\n";
$out .= "case=\n";
$out .= "reqps=$reqps\n";
if ( $task_data->{$_} ) {
$out .= "tasks=" . $task_data->{$_} . "\n";
}
else {
$out .= "tasks=0\n";
}
$out .= $empty_out . $out_end;
}
$d->{$_} = 1;
$last_prined_sec = $_;
}
}
}
return ( $d, $out, \@outed );
}
sub printSec($$$$) {
my ( $ref, $v, $prconf, $detailed ) = @_;
my $out = '';
# Calculating std, mean, self-load to %.2f format
my $out_expect = "";
for my $key ( sort keys %{$ref} ) {
if ( $key =~ /(.+?)_expect/ ) {
my $terme = $1 . "_expect";
$ref->{$terme} = 0 + sprintf( "%.2f", $ref->{$terme} );
$out_expect .= $terme . '=' . $ref->{$terme} . "\n";
if ( $1 eq $detailed ) {
my $termd = $1 . "_dispersion";
$ref->{$termd} =
0 + std( \@{ $ref->{ $1 . "_values" } }, $ref->{$terme} );
$out_expect .= $termd . '=' . $ref->{$termd} . "\n";
}
}
}
$ref->{selfload} = 0 + sprintf( "%.2f", $ref->{selfload} / $ref->{count} );
# Output http codes if tank type: http (1)
$out .= proutCodes( $ref->{http}, 'http' ) if $prconf->{tank_type} ne '2';
# Output net codes
$out .= proutCodes( $ref->{net}, 'net' );
# Output time intervals
if ( $ref->{diap} ) {
for ( my $i = 0 ; $i < scalar( @{ $ref->{diap} } ) ; $i++ ) {
if ( defined $ref->{diap}->[$i] ) {
$out .=
"answ_time="
. $values->[$i] . "-"
. $values->[ $i + 1 ] . ":"
. $ref->{diap}->[$i] . "\n";
}
}
}
else {
$out .= "answ_time=" . $values->[0] . "-" . $values->[1] . ":0\n";
}
# Output selfload, size in, size out
$out .= 'selfload=' . $ref->{selfload} . "%\n";
$out .= 'output=' . $ref->{sizeout} . "\n";
$out .= 'input=' . $ref->{sizein} . "\n";
$out .= $out_expect;
# Calculating percentiles
my $ref_detailed_values = $ref->{ $detailed . "_values" };
my @sorted_detailed_values = sort { $a <=> $b } @$ref_detailed_values;
for (@percentiles) {
my $q = 0 + sprintf( "%.2f",
$sorted_detailed_values[
int( scalar(@sorted_detailed_values) * $_ / 100 ) - 1 ] );
$out .= $detailed . "_q$_=" . $q . "\n";
}
return $out;
}
sub proutCodes($$) {
my ( $y, $t ) = @_;
my ( $tb, $out ) = ( 0, '' );
$t = 'netwcode' if $t eq 'net';
( $t, $tb ) = ( 'HTTPcode', '200' ) if ( $t eq 'http' );
if ($y) {
for ( keys %{$y} ) {
$out .= "$t=$_:" . $y->{$_} . "\n";
}
}
else {
$out .= "$t=$tb:0\n";
}
return $out;
}
sub show($) {
my $y = shift;
my $out = '';
for ( sort { $a <=> $b } keys %{$y} ) {
$out .= $_ . " " if ( $y->{$_} );
}
return $out;
}
#open(my $SRC, ">$ARGV[1]");
my ( $fsec, $cur_sec, $prev_sec, $new_sec, $WAITTIME, $add_prev ) =
( 0, 0, 0, 0, 100000, 0 );
my $jump = 0;
my %out_stack = ();
my %done = ();
my $last_agg = "";
my $prev_last_agg = "";
my $wait_sec = 0;
my $wait_time = 0;
# debug
my ( $wait, $previous ) = ( 0, 0 );
lp_log("Reading STDIN... ");
my ( $task_ts, %task_data );
# my $phout_parse_timings = '(\d+)\.(\d+)\s+(.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)';
# my $regexp_phout_parse_timings = '(\d+)\.(\d+)\s+(.*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)';
# my $regexp_time = 'time\s+(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})';
# my $regexp_phantom = '^phantom';
# my %dt;
#my %dt =
#(
# 'interval_real' => $4,
# 'connect_time' => $5,
# 'send_time' => $6,
# 'latency' => $7,
# 'receive_time' => $8,
# 'interval_event' => $9,
#);
#my %ot = (
# 'sizeout' => $10,
# 'sizein' => $11,
# 'net' => $12,
# 'http' => $13,
#);
if ($confout) {
lp_log("Print config information... ");
$out .= "tank_type=$prconf{tank_type}\n";
$out .= "job_n=$prconf{jobno}\n" if $prconf{jobno};
open( $DS, ">>$prconf{preproc_log_name}" );
print $DS $out;
close($DS);
$confout = 0;
}
lp_log (Dumper(\%ENV));
while (<STDIN>) {
if ($ENV{'DEBUG'}) { lp_log("Read line: $_"); }
#lp_log("Iteration: $tot_answ_tmp ... ");
# Parse phantom stdout
# if ( substr( $_, 0, 7 ) eq 'phantom' ) {
if ( $_ =~ /phantom/o ) {
if ( $_ =~ /time\s+(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})/o )
{
$task_ts = mktime( $6, $5, $4, $3, $2 - 1, $1 - 1900 );
}
elsif (/tasks\s+(\d+)/) {
if ($task_ts) {
$task_data{$task_ts} = $1;
$task_ts = '';
}
}
}
# Parse phantom phout log
else {
my @matched_vars;
#print $_;
unless ( @matched_vars = ($_ =~ /^(\d{10})\.(\d+)\s+(\S*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/o)
)
{
print " wrong line: $_\n";
lp_log( "ERROR: wrong line: " . $_ );
print $errcode = 99;
#print Dumper($cases);
$_ =
"0000000000.999\t"
. ( keys %$cases )[0]
. "\t1\t1\t1\t1\t1\t1\t1\t1\t999\t999";
@matched_vars = ($_ =~ /^(\d{10})\.(\d+)\s+(\S*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/o);
}
### else { # но по-моему, тут не нужен else
$tot_answ_tmp++;
# Detailed time values
# %dt = (
# 'interval_real' => $4,
# 'connect_time' => $5,
# 'send_time' => $6,
# 'latency' => $7,
# 'receive_time' => $8,
# 'interval_event' => $9
# );
my (
$dt_interval_real, $dt_connect_time, $dt_send_time,
$dt_latency, $dt_receive_time, $dt_interval_event,
$ot_sizeout, $ot_sizein, $ot_net,
$ot_http, $ms
)
= ( $matched_vars[3], $matched_vars[4], $matched_vars[5],
$matched_vars[6], $matched_vars[7], $matched_vars[8],
$matched_vars[9], $matched_vars[10], $matched_vars[11],
$matched_vars[12], 1000 * $matched_vars[1] + $matched_vars[4] );
# my $ms = 1000 * $2 + $5;
my ( $delta, $ts ) = ( $ms % 1000000, $matched_vars[0] + int( $ms / 1000000 ) );
$wait_time = 1000000 * ( $ts - $wait_sec ) + $delta;
if ( $ts != $cur_sec ) {
if ( $ts == $prev_sec ) {
$add_prev = 1;
}
if ( $ts > $cur_sec ) {
$new_sec = 1;
$wait_sec = $cur_sec;
$prev_sec = $cur_sec;
$cur_sec = $ts;
}
}
# my $cur_prevd = 0;
if ( ( $cur_sec - $prev_sec > 1 )
&& ( $prev_sec > 0 )
&& ( $cur_sec > 0 ) )
{
# $cur_prevd = $cur_sec - $prev_sec;
for ( my $i = $prev_sec + 1 ; $i < $cur_sec ; $i++ ) {
$out_stack{$i} = 1;
}
$jump = 1;
}
if ( $tot_answ_tmp == 1 ) {
( $new_sec, $start ) = ( 0, $ts );
}
if ( $new_sec == 1 && $ts == $cur_sec ) {
if ( $jump == 1 ) {
$out_stack{ $prev_sec - 1 } = 1;
$out_stack{$prev_sec} = 1;
}
if ( ( $wait_time > 1000000 + $WAITTIME ) ) {
$out_stack{$prev_sec} = 1;
$new_sec = 0;
}
else {
$new_sec = 1;
}
}
if ( $new_sec == 0 && $ts < $cur_sec ) {
$ts = $cur_sec;
}
$tag = ( $matched_vars[2] ? $matched_vars[2] : "sysempty" );
if ( scalar keys %$cases > 1 ) {
# check_case( $tag, $cases );
unless ( $cases->{$tag} ) {
print Dumper( \@_ );
print "errcode:77 (wrong case!)\n";
exit 1;
}
}
### Calculating non-time values
### for my $v ( $tag, 'overall' ) {
$agg{$ts}{$tag}{count}++;
$agg{$ts}{overall}{count}++;
my $ref_agg_ts_tag = $agg{$ts}{$tag};
my $ref_agg_ts_all = $agg{$ts}{overall};
# $ref_agg_ts_tag->{count}++;
# $ref_agg_ts_all->{count}++;
# Calculating input/output
$ref_agg_ts_tag->{sizein} += $ot_sizein;
$ref_agg_ts_tag->{sizeout} += $ot_sizeout;
$ref_agg_ts_all->{sizein} += $ot_sizein;
$ref_agg_ts_all->{sizeout} += $ot_sizeout;
# Calculating httpq codes
if ($ot_http) {
$ref_agg_ts_tag->{http}{$ot_http}++;
$ref_agg_ts_all->{http}{$ot_http}++;
}
# Calculating net codes
$ref_agg_ts_tag->{net}{$ot_net}++;
$ref_agg_ts_all->{net}{$ot_net}++;
# Calculating selfload
if ($dt_interval_real) {
my $tmp_self_load =
( ( $dt_interval_real - $dt_interval_event ) /
$dt_interval_real ) * 100;
$ref_agg_ts_tag->{selfload} += $tmp_self_load;
$ref_agg_ts_all->{selfload} += $tmp_self_load;
}
else {
$ref_agg_ts_tag->{selfload} += 100;
$ref_agg_ts_all->{selfload} += 100;
$ref_agg_ts_tag->{ie_null}++;
$ref_agg_ts_all->{ie_null}++;
}
# Time periods distribution
my $is_interval_found = 0;
for ( my $i = 0 ; $i < scalar(@$values) - 1 ; $i++ ) {
if ( $dt_interval_real < 1000 * $values->[ $i + 1 ] ) {
$ref_agg_ts_tag->{diap}->[$i]++;
$ref_agg_ts_all->{diap}->[$i]++;
$is_interval_found = 1;
last;
}
}
unless ($is_interval_found) {
lp_log("WARNING: interval_real > timeout, assigning to last interval. interval_real = $dt_interval_real " );
$agg{$ts}{$tag}{diap}->[ scalar(@$values) - 2 ]++;
$ref_agg_ts_all->{diap}->[ scalar(@$values) - 2 ]++;
}
### }
### Calculating mean time values
# $t = interval_real connect_time send_time latency receive_time interval_event
# $v = $tag 'overall'
### for my $t ( keys %dt ) {
### for my $v ( $tag, 'overall' ) {
# $ref_agg_ts_tag = $agg{$ts}{$tag};
my $cnt = $ref_agg_ts_tag->{count};
# First value
if ( $cnt == 1 ) {
$ref_agg_ts_tag->{interval_real_expect} = $dt_interval_real;
$ref_agg_ts_tag->{connect_time_expect} = $dt_connect_time;
$ref_agg_ts_tag->{send_time_expect} = $dt_send_time;
$ref_agg_ts_tag->{latency_expect} = $dt_latency;
$ref_agg_ts_tag->{receive_time_expect} = $dt_receive_time;
$ref_agg_ts_tag->{interval_event_expect} = $dt_interval_event;
# Recursive mean for other values
}
else {
my $count_count1 = $cnt - 1;
$ref_agg_ts_tag->{interval_real_expect} =
( $count_count1 * $ref_agg_ts_tag->{interval_real_expect} +
$dt_interval_real ) / $cnt;
$ref_agg_ts_tag->{connect_time_expect} =
( $count_count1 * $ref_agg_ts_tag->{connect_time_expect} +
$dt_connect_time ) / $cnt;
$ref_agg_ts_tag->{send_time_expect} =
( $count_count1 * $ref_agg_ts_tag->{send_time_expect} +
$dt_send_time ) / $cnt;
$ref_agg_ts_tag->{latency_expect} =
( $count_count1 * $ref_agg_ts_tag->{latency_expect} +
$dt_latency ) / $cnt;
$ref_agg_ts_tag->{receive_time_expect} =
( $count_count1 * $ref_agg_ts_tag->{receive_time_expect} +
$dt_receive_time ) / $cnt;
$ref_agg_ts_tag->{interval_event_expect} =
( $count_count1 * $ref_agg_ts_tag->{interval_event_expect} +
$dt_interval_event ) / $cnt;
}
# Save all exact values (for calculating std)
push @{ $ref_agg_ts_tag->{interval_real_values} },
$dt_interval_real;
push @{ $ref_agg_ts_tag->{connect_time_values} }, $dt_connect_time;
push @{ $ref_agg_ts_tag->{send_time_values} }, $dt_send_time;
push @{ $ref_agg_ts_tag->{latency_values} }, $dt_latency;
push @{ $ref_agg_ts_tag->{receive_time_values} }, $dt_receive_time;
push @{ $ref_agg_ts_tag->{interval_event_values} },
$dt_interval_event;
# $ref_agg_ts_tag = $agg{$ts}{overall};
$cnt = $ref_agg_ts_all->{count};
if ( $cnt == 1 ) {
$ref_agg_ts_all->{interval_real_expect} = $dt_interval_real;
$ref_agg_ts_all->{connect_time_expect} = $dt_connect_time;
$ref_agg_ts_all->{send_time_expect} = $dt_send_time;
$ref_agg_ts_all->{latency_expect} = $dt_latency;
$ref_agg_ts_all->{receive_time_expect} = $dt_receive_time;
$ref_agg_ts_all->{interval_event_expect} = $dt_interval_event;
# Recursive mean for other values
}
else {
my $count_count1 = $cnt - 1;
$ref_agg_ts_all->{interval_real_expect} =
( $count_count1 * $agg{$ts}{overall}{interval_real_expect} +
$dt_interval_real ) / $cnt;
$ref_agg_ts_all->{connect_time_expect} =
( $count_count1 * $agg{$ts}{overall}{connect_time_expect} +
$dt_connect_time ) / $cnt;
$ref_agg_ts_all->{send_time_expect} =
( $count_count1 * $agg{$ts}{overall}{send_time_expect} +
$dt_send_time ) / $cnt;
$ref_agg_ts_all->{latency_expect} =
( $count_count1 * $agg{$ts}{overall}{latency_expect} +
$dt_latency ) / $cnt;
$ref_agg_ts_all->{receive_time_expect} =
( $count_count1 * $agg{$ts}{overall}{receive_time_expect} +
$dt_receive_time ) / $cnt;
$ref_agg_ts_all->{interval_event_expect} =
( $count_count1 * $agg{$ts}{overall}{interval_event_expect} +
$dt_interval_event ) / $cnt;
}
push @{ $ref_agg_ts_all->{interval_real_values} },
$dt_interval_real;
push @{ $ref_agg_ts_all->{connect_time_values} }, $dt_connect_time;
push @{ $ref_agg_ts_all->{send_time_values} }, $dt_send_time;
push @{ $ref_agg_ts_all->{latency_values} }, $dt_latency;
push @{ $ref_agg_ts_all->{receive_time_values} }, $dt_receive_time;
push @{ $ref_agg_ts_all->{interval_event_values} },
$dt_interval_event;
### }
### }
### }
}
my ( $status, $s, $d, $out, $outed ) = (0);
if (%out_stack) {
$s = printStack( \%out_stack );
( $d, $out, $outed ) = outputStack(
\%out_stack, \%done, \%agg, \%prconf,
$cases, $values, $start, $load,
$errcode, $detailed, \%task_data
);
$status = 1;
open( $DS, ">>$prconf{preproc_log_name}" );
print $DS $out;
close($DS);
}
if ($status) {
for my $ot (@$outed) {
delete $agg{$ot};
delete $out_stack{$ot};
delete $done{$ot};
}
for my $os ( keys %out_stack ) {
if ( $os < $last_prined_sec ) {
delete $out_stack{$os};
}
}
}
$add_prev = 0;
}
if ( $new_sec == 1 ) {
$out_stack{$prev_sec} = 1;
}
$out_stack{$cur_sec} = 1;
my ( $d, $out1, $outed ) = outputStack(
\%out_stack, \%done, \%agg, \%prconf, $cases, $values,
$start, $load, $errcode, $detailed, \%task_data
);
if ($out1) {
sleep(0.1);
}
open( $DS, ">>$prconf{preproc_log_name}" );
print $DS $out1;
close($DS);
#print $out1;
sub printAbout() {
my ( @about, @pr );
$pr[0] = " #############################";
$pr[1] = " ######## Preprocessor #######";
push @about, "_____________________\$\$\$\n",
"____________________\$___\$\n", "_____________________\$\$\$\n",
"_____________________\$_\$\n", "_____________________\$_\$\n",
"___________________\$\$\$_\$\$\$\n",
"_________________\$\$__\$\$\$__\$\$\$\n",
"_______________\$\$__\$\$\$\$\$\$\$___\$\n",
"______________\$_______________\$\n",
"_____________\$_________________\$\n",
"_____________\$_________________\$\n",
"_____________\$_____\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\n",
"_____________\$____\$_______________\$\n",
"_____________\$____\$___\$\$\$\$\$\$\$\$\$\$\$\$\$\n",
"_____________\$___\$___\$___________\$\$\$\n",
"_____________\$___\$___\$_\$\$\$___\$\$\$__\$\$\n",
"_____________\$___\$___\$_\$\$\$___\$\$\$__\$\$\n",
"_____________\$___\$___\$___________\$\$\$\n",
"_____________\$____\$___\$\$\$\$\$\$\$\$\$\$\$\$\$\n",
"_____________\$_____\$\$\$\$\$\$\$\$\$\$\$\$\$\$\n",
"_____________\$_________________\$\n",
"_____________\$____\$\$\$\$\$\$\$\$\$\$\$\$\$\$\n",
"_____________\$___\$__\$__\$__\$__\$$pr[0]\n",
"_____________\$__\$\$\$\$\$\$\$\$\$\$\$\$\$\$$pr[1]\n",
"_____________\$__\$___\$__\$__\$__\$$pr[0]\n",
"_____________\$___\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\n",
"____________\$\$\$_________________\$\$\$\n",
"__________\$\$___\$\$\$_________\$\$\$\$\$___\$\$\n",
"________\$\$________\$\$\$\$\$\$\$\$\$__________\$\$\$\n",
"_______\$__\$\$_____________________\$\$\$\$___\$\$\n",
"____\$\$\$\$\$___\$\$\$\$\$\$\$\$______\$\$\$\$\$\$\$_______\$_\$\n",
"__\$______\$\$_________\$\$\$\$\$\$______________\$_\$\$\n",
"_\$____\$____\$____________________________\$_\$_\$\n",
"_\$_____\$___\$______________\$\$\$\$\$\$\$\$\$\$\$___\$_\$_\$\$\n",
"_\$\$\$____\$___\$__\$\$\$\$\$\$\$\$\$\$\$\$__________\$___\$_\$_\$\$\n",
"\$___\$\$\$\$____\$__\$_____________________\$___\$_\$\$_\$\n",
"\$\$\$____\$___\$\$__\$_____________________\$\$__\$_\$__\$\n",
"\$___\$__\$__\$\$___\$______________________\$__\$\$\$__\$\n",
"\$_____\$\$_\$\$____\$_______________\$\$\$____\$__\$_\$__\$\n";
for (@about) {
print $_;
sleep(0.1);
}
}

516
stepper.py Executable file
View File

@ -0,0 +1,516 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import ConfigParser
from collections import defaultdict
from optparse import OptionParser
import os
import re
# FIXME: remove this hack
import sys
sys.path.append("/usr/lib/python2.5")
from progressbar import Bar
from progressbar import ETA
from progressbar import Percentage
from progressbar import ProgressBar
from yandex_load_lunapark import stepper
print
print "==== Stepper ===="
### Command line argumentd
parser = OptionParser()
parser.add_option("-c", "--config", dest="config",
help="use custom config FILE, insted of load.conf", metavar="FILE", default='load.conf')
parser.add_option("-a", "--ammo", dest="ammo",
help="FILE with requests", metavar="FILE")
parser.add_option("-s", "--stats", dest="stats", action="store_true",
help="only ammo stats, no generating")
parser.add_option("-l", "--loadscheme", dest="loadscheme", action="store_true",
help="only loadscheme creating, no generating")
parser.add_option("--common-header", dest="header", action="store_true",
help="show common headers from ammo/config")
parser.add_option("--autocases", dest="autocases", action="store_true",
help="show autocases for given file")
(options, args) = parser.parse_args()
## Pattern for autocase 3rd level
pattern = re.compile('/(.*?)(/(.*?))?(/(.*?))?(\.|/|\?|$|\s)')
pattern = re.compile('^(GET|POST|PUT|HEAD|OPTIONS|PATCH|DELETE|TRACE|LINK|UNLINK|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK\s+)?\s*\/(.*?)(/(.*?))?(/(.*?))?(\.|/|\?|$|\s)')
### Defaults params for config file
default = {}
default["header_http"] = "1.0"
default["autocases"] = "1"
default["tank_type"] = "1"
### Parse config
### using FakeSecHead class for creating fake section [default] in config-file
configuration_file = ConfigParser.SafeConfigParser(default)
configuration_file.optionxform = str
configuration_file.readfp(stepper.FakeSecHead(open(options.config)))
### Tank type: 1 - HTTP requests, 2 - RAW requests (no ammo count and progress bar)
tank_type = configuration_file.getint('DEFAULT', 'tank_type')
### MultiValues parametres
# load - list of elements (step, line, const) for load scheme
# uri - list of uri for requests
load_multi = stepper.get_config_multiple(options.config)
(load_steps, load_scheme, load_count) = stepper.make_steps(options.config)
# handle instances schedule
try:
instances_schedule_count = 0
instances_schedule = []
instances_chunk_cnt = 0
instances = configuration_file.get('DEFAULT', 'instances_schedule')
sched_parts=instances.split(" ")
for sched_part in sched_parts:
[expanded_sched, skip, skip, max_val]=stepper.expand_load_spec(sched_part)
instances_schedule += expanded_sched
if max_val > instances_schedule_count:
instances_schedule_count = max_val
instances_schedule = stepper.collapse_schedule(instances_schedule)
except ConfigParser.NoOptionError:
pass
### Output '--loadscheme' argument
if options.loadscheme:
print load_scheme
exit(0)
### Use ammo defined ammo file or creating temp ammo file from config
ammo_file = options.ammo
if not ammo_file:
ammo_file=configuration_file.get('DEFAULT', 'ammofile')
ammo_type = ""
ammo_delete = ""
if tank_type == 1:
if load_multi['uri']:
#print "Creating tmp ammo file"
ammo_file = stepper.make_load_ammo(options.config)
ammo_type = "uri"
ammo_delete = ammo_file
else:
# Detect type of ammo file: 'uri', 'request' or 'unknown'
ammo_type = stepper.get_ammo_type(ammo_file)
if ammo_type == 'unknown':
print "[Error] Unknown type of ammo file"
exit(1)
elif ammo_type == 'request':
pass
#print "[Error] Type of ammo file: 'request'. You have to use stepper.pl instead of stepper.py"
#exit(1)
elif ammo_type == 'uri':
pass
#print "OK. Type of ammo file: 'uri'"
elif tank_type == 2:
ammo_type = "request"
### Make common headers from ammo file
if ammo_type == 'uri':
header_common = stepper.get_common_header(ammo_file)
### Make common headers from config
header_config = {}
if load_multi['header']:
for line in load_multi['header']:
header_config.update(stepper.get_headers_list(line))
### Output '--common-header' argument
if options.header:
if header_common:
print "==== ammo header begin ===="
print stepper.header_print(header_common),
print "===== ammo header end ====="
print
if header_config:
print "==== config header begin ===="
print stepper.header_print(header_config),
print "===== config header end ====="
print
### Autocases (work only for HTTP request - tank_type = 1)
cases_done, cases_output, ammo_count = {}, "", 0
if tank_type == 1:
if configuration_file.getint('DEFAULT', 'autocases') == 0:
ammo_count = stepper.get_ammo_count(ammo_file, load_count)
else:
if ammo_type == 'request' and stepper.detect_case_file(ammo_file) == True:
print "Ammo file has cases. Do not create autocases."
ammo_count = stepper.get_ammo_count(ammo_file, load_count)
else:
(l1, l2, l3, cases_tree, ammo_count) = stepper.get_autocases_tree(ammo_file)
if configuration_file.getint('DEFAULT', 'autocases'):
(cases_done, cases_output) = stepper.make_autocases_top(l1, l2, l3, ammo_count, cases_tree)
### Output autocases levels for '--autocases' argument
if options.autocases:
print cases_output
### Output some ammo stats for '--stats' argument
if options.stats:
print "ammo file: %s" % ammo_file
print "ammo type: %s" % ammo_type
print "ammo count: %s" % ammo_count
print "load count: %s" % load_count
print
exit(0)
http = configuration_file.get('DEFAULT', 'header_http')
### max value for ProgressBar
max_progress = 0
### cur progress value
cur_progress = 0
### Case of operation
case = 0
loop = 0
### Case 0. Neither 'load' nor 'loop' presents at 'load.conf'
# loop = 1
#print "load: %s" % load_count
#print load.has_option('default', 'loop')
if not configuration_file.has_option('DEFAULT', 'loop'):
if load_count == 0:
print "loop and load2"
loop = 1
else:
loop = -1
else:
loop = configuration_file.getint('DEFAULT', 'loop')
#print "loop: %s" % loop
base_loop = loop
stop_loop_count = 0
### Case 1. No generating.
# loop = 0 and ammo count not enough for load scheme
if loop == 0 and ammo_count < load_count:
case = 1
print "Not enough ammo (%s) in '%s' for load scheme (%s)" % (ammo_count, ammo_file, load_count)
print
exit(1)
### Case 2. Only looping.
# loop > 0 and loop*ammo_count < load scheme.
if loop > 0:
if load_count == 0:
case = 2
print "Load scheme is empty"
### Case 3. Load scheme generating.
if load_count > 0:
if loop == 0 and load_count <= ammo_count:
case = 3
if loop > 0 and load_count <= loop*ammo_count:
case = 3
if loop > 0 and loop*ammo_count < load_count:
print "Looped ammo count (%s * %s = %s) is less than load scheme (%s). Using loop count." % (loop, ammo_count, loop*ammo_count, load_count)
stop_loop_count = loop*ammo_count
case = 3
if loop == -1:
case = 3
print "Case: %s. Ammo type: %s" % (case, ammo_type)
already_cases = defaultdict(int)
### Ammo Generating
if ammo_type == 'request':
if case == 2:
max_progress = loop*ammo_count
load_steps = [[0,0]]
elif case == 3:
if stop_loop_count:
max_progress = stop_loop_count
else:
max_progress = load_count
#print "max progress = %s" % max_progress
base_time = 0
widgets = ['Ammo Generating: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ' ]
pbar = ProgressBar(widgets=widgets, maxval=max_progress).start()
pattern_request = re.compile("^(\d+)\s*\d*\s*(\w*)\s*$")
pattern_uri = re.compile("^(GET|POST|PUT|HEAD|OPTIONS|PATCH|DELETE|TRACE|LINK|UNLINK|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK\s+)?\s*(\/(.*?))($|\s)")
pattern_null = re.compile("^0")
ammo_len, line_num, chunk_begin, ammo = 0, 0, 0, ""
chunk_len, chunk_case, chunk_num, chunk_line_start = 0, "", 0, 1
last_added_chunk = ""
input_ammo = open(ammo_file, "rb")
stepped_ammo = open("ammo.stpd", "wb")
for step in load_steps:
if case == 3:
if stop_loop_count > 0 and cur_progress == stop_loop_count:
break
step_ammo_num, looping = 0, 1
count, duration = step[0], step[1]
if case == 3:
if int(count) == 0:
base_time += duration*1000
continue
marked = stepper.mark_sec(count, duration)
while looping:
chunk_start = input_ammo.readline()
# print chunk_start
if not chunk_start:
if not cur_progress:
raise RuntimeError("Empty ammo file, can't use it")
input_ammo.seek(0)
continue
m = re.match("^\s*(\d+)\s*\d*\s*(\w*)\s*$", chunk_start)
# meta information of new chunk - TRUE
if m:
chunk_size, chunk_case = int(m.group(1)), m.group(2)
#print chunk_case
already_cases[chunk_case] += 1
chunk = input_ammo.read(chunk_size)
if chunk_size > len(chunk):
print "\n\n[Error] Unexpected end of file"
print "Expected chunk size: %s" % chunk_size
print "Readed chunk size: %s" % len(chunk)
print
if chunk:
print "Readed chunk:\n----\n%s----\n" % chunk
if last_added_chunk:
print "Last written chunk:\n----\n%s----\n" % last_added_chunk
exit(1)
if chunk:
if not chunk_case:
request = pattern_uri.match(chunk)
if request:
chunk_case = stepper.get_prepared_case(request.group(2), cases_done, pattern)
else:
chunk_case = 'other'
time = 1000
#if tank_type == 2:
#chunk_case = ''
if case == 3:
time = base_time + marked[step_ammo_num]
elif case == 2:
# no load scheme
if step_ammo_num < instances_schedule_count:
if not instances_chunk_cnt:
[instances_chunk_cnt, instances_chunk_time] = instances_schedule.pop(0)
base_time += instances_chunk_time * 1000
instances_chunk_cnt -= 1
time=base_time
else:
time = 1000
# write chunk to file
if chunk_case:
stepped_ammo.write("%s %s %s\n" % (chunk_size, time, chunk_case))
else :
stepped_ammo.write("%s %s\n" % (chunk_size, time))
stepped_ammo.write(chunk)
stepped_ammo.write("\n")
step_ammo_num += 1
cur_progress += 1
last_added_chunk = chunk
pbar.update(cur_progress)
if int(cur_progress) == int(max_progress):
looping = 0
if case == 3:
if stop_loop_count > 0 and cur_progress == stop_loop_count:
break
if step_ammo_num == count*duration:
break
# meta information of new chunk - FALSE
else:
# pass empty strings between requests
if re.match("^\s*$", chunk_start):
pass
# wrong case format (not \w*)
else:
m = re.match("^\s*(\d+)\s*\d*\s*(.*)\s*$", chunk_start)
if m:
if m.group(2):
c = re.match("^\w+$", m.group(2))
if not c:
print "Wrong case format for '%s'" % m.group(2)
exit(1)
print "[Error] Wrong chunk size"
print
print "Chunk start:\n----\n%s----\n" % chunk_start
if chunk:
print "Readed chunk:\n----\n%s----\n" % chunk
if last_added_chunk:
print "Last writed chunk:\n----\n%s----\n" % last_added_chunk
exit(1)
if int(cur_progress) == int(max_progress):
looping = 0
break
if case == 3:
if stop_loop_count > 0 and cur_progress == stop_loop_count:
break
if step_ammo_num == count*duration:
break
if not step_ammo_num == count*duration:
pass
if step_ammo_num == load_count:
looping = 0
base_time += duration*1000
stepped_ammo.write("0\n")
pbar.finish()
elif (ammo_type == "uri"):
# Only looping.
if case == 2:
print "Looping '%s' for %s time(s):" % (ammo_file, loop)
print
max_progress = loop*ammo_count
widgets = ['Ammo Generating: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ' ]
pbar = ProgressBar(widgets=widgets, maxval=max_progress).start()
stepped_ammo = open("ammo.stpd", "w")
input_ammo = open(ammo_file, "r")
for l in range(1, loop + 1):
for line in input_ammo:
m = re.match("^\/", line)
if m:
real_case = stepper.get_prepared_case(line.rstrip(), cases_done, pattern)
chunk = stepper.chunk_by_uri(line.rstrip(), http, 1000, real_case, header_common, header_config)
stepped_ammo.write(chunk)
cur_progress += 1
pbar.update(cur_progress)
#sleep(1)
input_ammo.seek(0)
stepped_ammo.write("0\n")
pbar.finish()
print
# Steps generating
elif case == 3:
max_progress = load_count
if stop_loop_count:
max_progress = stop_loop_count
else:
max_progress = load_count
base_time = 0
widgets = ['Ammo Generating: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ' ]
pbar = ProgressBar(widgets=widgets, maxval=max_progress).start()
stepped_ammo = open("ammo.stpd", "w")
input_ammo = open(ammo_file, "r")
for step in load_steps:
# print step
if stop_loop_count > 0 and cur_progress == stop_loop_count:
looping = 0
break
step_ammo_num, looping = 0, 1
count, duration = step[0], step[1]
if count == 0:
base_time += duration * 1000
continue
marked = stepper.mark_sec(count, duration)
# print "marked: %s" % marked
while looping:
for line in input_ammo:
m = re.match("^\/", line)
if m:
time = marked[step_ammo_num]
real_case = stepper.get_prepared_case(line.rstrip(), cases_done, pattern)
chunk = stepper.chunk_by_uri(line.rstrip(), http, base_time + time, real_case, header_common, header_config)
stepped_ammo.write(chunk)
cur_progress += 1
step_ammo_num += 1
pbar.update(cur_progress)
if stop_loop_count > 0 and cur_progress == stop_loop_count:
looping = 0
break
if step_ammo_num == count*duration:
looping = 0
break
if not step_ammo_num == count*duration:
input_ammo.seek(0)
base_time += duration*1000
stepped_ammo.write("0\n")
pbar.finish()
print
### Save shared data to 'lp.conf' for using by 'preproc', 'fantom' and '-s' argument
if not os.path.exists("lp.conf"):
lp_conf = open("lp.conf", "w")
lp_conf.close()
lp = ConfigParser.SafeConfigParser(default)
lp.readfp(stepper.FakeSecHead(open("lp.conf")))
lp.set('DEFAULT', 'ammo_count', str(cur_progress))
if ammo_count > 0:
lp.set('DEFAULT', 'loop_count', "%.2f" % (float(cur_progress) / ammo_count))
else:
lp.set('DEFAULT', 'loop_count', "0")
if instances_schedule_count:
lp.set('DEFAULT', 'instances_schedule', "%s" % instances)
lp_loadscheme = re.sub("\n", ";", load_scheme);
lp.set('DEFAULT', 'loadscheme', lp_loadscheme)
lp_cases = ''
if cases_done:
for s in cases_done:
if cases_done[s] > 0:
lp_cases += "'" + s + "' "
elif already_cases :
for s in already_cases:
if s and already_cases[s] > 0:
lp_cases += "'" + s + "' "
if s == "" and len(already_cases) == 1:
lp_cases = "''"
else:
lp_cases = "''"
lp.set('DEFAULT', 'cases', lp_cases)
lp_steps = ''
for s in load_steps:
lp_steps += "(%s;%s) " % (s[0], s[1])
lp.set('DEFAULT', 'steps', lp_steps)
configfile = open('lp.conf', 'wb')
lp.write(configfile)
### Delete tmp load ammo file
if ammo_delete:
os.unlink(ammo_delete)
exit(0)

0
yandex_load_lunapark/__init__.py Normal file
View File

174
yandex_load_lunapark/status.py Normal file
View File

@ -0,0 +1,174 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import httplib
### HTTP codes
http = httplib.responses
### Extended list of HTTP status codes(WEBdav etc.)
### http://en.wikipedia.org/wiki/List_of_HTTP_status_codes
webdav = {
102: 'Processing',
103: 'Checkpoint',
122: 'Request-URI too long',
207: 'Multi-Status',
226: 'IM Used',
308: 'Resume Incomplete',
418: 'I\'m a teapot',
422: 'Unprocessable Entity',
423: 'Locked',
424: 'Failed Dependency',
425: 'Unordered Collection',
426: 'Upgrade Required',
444: 'No Response',
449: 'Retry With',
450: 'Blocked by Windows Parental Controls',
499: 'Client Closed Request',
506: 'Variant Also Negotiates',
507: 'Insufficient Storage',
509: 'Bandwidth Limit Exceeded',
510: 'Not Extended',
598: 'network read timeout error',
599: 'network connect timeout error',
999: 'lunapark internal error',
}
http.update(webdav)
### NET codes
net = {
0: "Success",
1: "Operation not permitted",
2: "No such file or directory",
3: "No such process",
4: "Interrupted system call",
5: "Input/output error",
6: "No such device or address",
7: "Argument list too long",
8: "Exec format error",
9: "Bad file descriptor",
10: "No child processes",
11: "Resource temporarily unavailable",
12: "Cannot allocate memory",
13: "Permission denied",
14: "Bad address",
15: "Block device required",
16: "Device or resource busy",
17: "File exists",
18: "Invalid cross-device link",
19: "No such device",
20: "Not a directory",
21: "Is a directory",
22: "Invalid argument",
23: "Too many open files in system",
24: "Too many open files",
25: "Inappropriate ioctl for device",
26: "Text file busy",
27: "File too large",
28: "No space left on device",
29: "Illegal seek",
30: "Read-only file system",
31: "Too many links",
32: "Broken pipe",
33: "Numerical argument out of domain",
34: "Numerical result out of range",
35: "Resource deadlock avoided",
36: "File name too long",
37: "No locks available",
38: "Function not implemented",
39: "Directory not empty",
40: "Too many levels of symbolic links",
41: "Unknown error 41",
42: "No message of desired type",
43: "Identifier removed",
44: "Channel number out of range",
45: "Level 2 not synchronized",
46: "Level 3 halted",
47: "Level 3 reset",
48: "Link number out of range",
49: "Protocol driver not attached",
50: "No CSI structure available",
51: "Level 2 halted",
52: "Invalid exchange",
53: "Invalid request descriptor",
54: "Exchange full",
55: "No anode",
56: "Invalid request code",
57: "Invalid slot",
58: "Unknown error 58",
59: "Bad font file format",
60: "Device not a stream",
61: "No data available",
62: "Timer expired",
63: "Out of streams resources",
64: "Machine is not on the network",
65: "Package not installed",
66: "Object is remote",
67: "Link has been severed",
68: "Advertise error",
69: "Srmount error",
70: "Communication error on send",
71: "Protocol error",
72: "Multihop attempted",
73: "RFS specific error",
74: "Bad message",
75: "Value too large for defined data type",
76: "Name not unique on network",
77: "File descriptor in bad state",
78: "Remote address changed",
79: "Can not access a needed shared library",
80: "Accessing a corrupted shared library",
81: ".lib section in a.out corrupted",
82: "Attempting to link in too many shared libraries",
83: "Cannot exec a shared library directly",
84: "Invalid or incomplete multibyte or wide character",
85: "Interrupted system call should be restarted",
86: "Streams pipe error",
87: "Too many users",
88: "Socket operation on non-socket",
89: "Destination address required",
90: "Message too long",
91: "Protocol wrong type for socket",
92: "Protocol not available",
93: "Protocol not supported",
94: "Socket type not supported",
95: "Operation not supported",
96: "Protocol family not supported",
97: "Address family not supported by protocol",
98: "Address already in use",
99: "Cannot assign requested address",
100: "Network is down",
101: "Network is unreachable",
102: "Network dropped connection on reset",
103: "Software caused connection abort",
104: "Connection reset by peer",
105: "No buffer space available",
106: "Transport endpoint is already connected",
107: "Transport endpoint is not connected",
108: "Cannot send after transport endpoint shutdown",
109: "Too many references: cannot splice",
110: "Connection timed out",
111: "Connection refused",
112: "Host is down",
113: "No route to host",
114: "Operation already in progress",
115: "Operation now in progress",
116: "Stale NFS file handle",
117: "Structure needs cleaning",
118: "Not a XENIX named type file",
119: "No XENIX semaphores available",
120: "Is a named type file",
121: "Remote I/O error",
122: "Disk quota exceeded",
123: "No medium found",
124: "Wrong medium type",
125: "Operation canceled",
126: "Required key not available",
127: "Key has expired",
128: "Key has been revoked",
129: "Key was rejected by service",
130: "Owner died",
131: "State not recoverable",
999: 'lunapark internal error',
}

623
yandex_load_lunapark/stepper.py Normal file
View File

@ -0,0 +1,623 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from collections import defaultdict
import datetime
import operator
import os
import re
import time
# FIXME: remove this hack
import sys
sys.path.append("/usr/lib/python2.5")
from progressbar import Bar
from progressbar import ETA
from progressbar import Percentage
from progressbar import ProgressBar
### Ammo file ###
###
def make_load_ammo(file):
pattern = re.compile("^\s*(header|uri)\s*=\s*(.+)")
dt = datetime.datetime.now()
filename = "ammo_load_%s" % int(time.mktime(dt.timetuple()))
conf = open(file, "r")
tmp_ammo = open(filename, "w")
for line in conf:
m = pattern.match(line)
if m:
el = m.groups()
tmp_ammo.write(el[1] + "\n")
return filename
def get_ammo_type(file):
ammo = open(file, "r")
first_line = ammo.readline()
if re.match("^(\/|\[)", first_line):
return "uri"
elif re.match("^\d", first_line):
return "request"
else:
return "unknown"
def detect_case_file(file):
ammo = open(file, "r")
first_line = ammo.readline()
m = re.match("^(\d+)\s*\d*\s*(\w*)\s*", first_line)
if m:
if m.group(2):
return True
return False
def get_ammo_count(file, stop_count):
ammo_type = get_ammo_type(file)
ammo_cnt = 0
pattern = re.compile("^(GET|POST|PUT|HEAD|OPTIONS|PATCH|DELETE|TRACE|LINK|UNLINK)\s")
if (ammo_type == "uri"):
input_ammo = open(file, "r")
for line in input_ammo:
if re.match("^\/", line):
ammo_cnt += 1
return ammo_cnt
elif (ammo_type == "request"):
max_progress = file_len(file)
widgets = ['Checking ammo count: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ' ]
pbar = ProgressBar(widgets=widgets, maxval=max_progress).start()
byte_cnt = 0
input_ammo = open(file, "r")
for line in input_ammo:
byte_cnt += len(line)
pbar.update(byte_cnt)
if pattern.match(line):
ammo_cnt += 1
if ammo_cnt == stop_count & stop_count > 0:
break
pbar.finish()
print
return ammo_cnt
def get_header(line):
header = ""
list = get_headers_list(line)
for h, v in list.iteritems():
header += "%s: %s\r\n" % (h, v)
return header
def header_print(list):
header = ""
for h, v in list.iteritems():
header += "%s: %s\r\n" % (h, v)
return header
def get_headers_list(line):
'''return a dict of {header: value}, parsed from string'''
list = defaultdict(str)
h = re.match("^\[", line)
if h:
m = re.match("^\[(.+)\]", line)
if m:
cases = re.split('\]\s*\[', line)
for case in cases:
case = re.sub("\[|\]", "", case)
c = re.match("^\s*(\S+)\s*:\s*(.+?)\s*$", case)
if c:
list[c.group(1)] = c.group(2)
return list
def get_common_header(file):
input_ammo = open(file, "r")
list = {}
for line in input_ammo:
list.update(get_headers_list(line))
return list
### Config file ###
###
class FakeSecHead(object):
def __init__(self, fp):
self.fp = fp
self.sechead = '[DEFAULT]\n'
def readline(self):
if self.sechead:
try:
return self.sechead
finally:
self.sechead = None
else: return self.fp.readline()
def get_config_multiple(file):
pattern = re.compile('^\s*(header|load|uri)\s*=\s*(.+)')
params = defaultdict(list)
conf = open(file, 'r')
for line in conf:
m = pattern.match(line)
if m:
el = m.groups()
if el[0] == 'load':
for k in re.findall(r"(line|step|const|const_f)\s*(\(.+?\))\s*", el[1]):
params[el[0]].append(k[0] + k[1])
elif el[0] == 'uri':
params[el[0]].append(el[1].rstrip())
elif el[0] == 'header':
params[el[0]].append(el[1].rstrip())
return params
### Loadscheme ###
###
def load_const(req, duration):
dur = str2sec(duration)
steps = []
steps.append([req, dur])
ls = "%s,const,%s,%s,(%s,%s)\n" % (dur,req,req,req,duration)
time = dur*int(req)
if int(req) == 0:
time = dur
return [steps, ls, time]
def load_line(start, end, duration):
dur = str2sec(duration)
k = float((end-start)/float(dur-1))
cnt, last_x, total = 1, start, 0
ls = "%s,line,%s,%s,(%s,%s,%s)\n" % (dur,start,end,start,end,duration)
steps = []
for i in range(1,dur+1):
cur_x = int(start + k*i)
if (cur_x == last_x):
cnt += 1
else:
steps.append([last_x, cnt])
total += cnt*last_x
cnt, last_x = 1, cur_x
return [steps, ls, total]
def load_step(start, end, step, duration):
dur = str2sec(duration)
steps, ls, total = [], "", 0
if end > start:
for x in range(start, end+1, step):
steps.append([x, dur])
ls += "%s,step,%s,%s,(%s,%s,%s,%s)\n" % (dur,x,x,start,end,step,duration)
total += x*dur
else:
for x in range(start, end-1, -step):
steps.append([x, dur])
ls += "%s,step,%s,%s,(%s,%s,%s,%s)\n" % (dur,x,x,start,end,step,duration)
total += x*dur
return [steps, ls, total]
# for instances
def collapse_schedule(schedule):
res=[]
prev_item = []
rolling_count = 0
base_time = 0
for item in schedule:
if base_time == 0:
base_time = item[1]
item[1] = 0
if prev_item:
if prev_item[0] == item[0]:
prev_item[1] += item[1]
continue
res += [[prev_item[0] - rolling_count, prev_item[1]]]
rolling_count = prev_item[0]
prev_item=item
if prev_item:
res += [prev_item]
return res
def make_steps_element(l):
steps, loadscheme, load_ammo = [], "", 0
params = []
params.append(l)
for l in params:
st, ls, cnt = [], "", 0
m = re.match("^const\s*\(\s*(\d+)\s*,\s*((\d+[hms]?)+)\s*", l)
if m:
st, ls, cnt = load_const(int(m.group(1)), m.group(2))
else:
m = re.match("^line\s*\(\s*(\d+)\s*,\s*(\d+)\s*,\s*((\d+[hms]?)+)\s*", l)
if m:
[st, ls, cnt] = load_line(int(m.group(1)), int(m.group(2)), m.group(3))
else:
m = re.match("^step\s*\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*((\d+[hms]?)+)\s*\)\s*", l)
if m:
[st, ls, cnt] = load_step(int(m.group(1)), int(m.group(2)), int(m.group(3)), m.group(4))
else:
m = re.match("^const\s*\(\s*(\d+\/\d+)\s*,\s*((\d+[hms]?)+)\s*", l)
if m:
[st, ls, cnt] = constf(m.group(1), m.group(2))
else:
print "[Warning] Wrong load format: '%s'. Ignoring." % (l)
if ls:
steps += st
loadscheme += ls
load_ammo += cnt
return (steps, loadscheme, load_ammo)
def expand_load_spec(l):
st, ls, cnt, max = [], "", 0, 0
m = re.match("^const\s*\(\s*(\d+)\s*,\s*((\d+[hms]?)+)\s*", l)
if m:
val=int(m.group(1))
st, ls, cnt = load_const(val, m.group(2))
if val>max:
max=val
else:
m = re.match("^line\s*\(\s*(\d+)\s*,\s*(\d+)\s*,\s*((\d+[hms]?)+)\s*", l)
if m:
val=int(m.group(2))
[st, ls, cnt] = load_line(int(m.group(1)), val, m.group(3))
if val>max:
max=val
else:
m = re.match("^step\s*\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*((\d+[hms]?)+)\s*\)\s*", l)
if m:
val=int(m.group(2))
[st, ls, cnt] = load_step(int(m.group(1)), val, int(m.group(3)), m.group(4))
if val>max:
max=val
else:
m = re.match("^const\s*\(\s*(\d+\/\d+)\s*,\s*((\d+[hms]?)+)\s*", l)
if m:
[st, ls, cnt] = constf(m.group(1), m.group(2))
else:
print "[Warning] Wrong load format: '%s'. Ignoring." % (l)
return st, ls, cnt, max
def make_steps(file):
params = get_config_multiple(file)
steps, loadscheme, load_ammo = [], "", 0
for l in params['load']:
st, ls, cnt, skip = expand_load_spec(l)
if ls:
steps += st
loadscheme += ls
load_ammo += cnt
return (steps, loadscheme, load_ammo)
def mark_sec(cnt, dur):
k = 1000/float(cnt)
times = [0]
for i in range(1, cnt*dur):
times.append(int(i*k))
return times
### Fractional rps ###
###
def constf(req, duration):
m = re.match("(\d+)\/(\d+)", req)
if m:
a, b = int(m.group(1)), int(m.group(2))
dur, e = str2sec(duration), int(a/b)
fr = "%.3f" % (float(a)/b)
ls = "%s,const,%s,%s,(%s,%s)\n" % (dur, fr, fr, req, duration)
a = a % b
req = "%s/%s" % (a, b)
out = []
tail = dur % b
for x in range(1, int(dur/b) + 1):
out += frps(req)
if tail:
out += frps_cut(tail, req)
if e > 0:
out = frps_expand(out, e)
return (out, ls, frps_ammo(out))
else :
print "Error in 'const_f' function. rps: %s, duration: %s" % (req, duration)
exit(1)
def frps(req):
m = re.match("(\d+)\/(\d+)", req)
if m:
c = defaultdict(str)
num1 = int(m.group(1))
num0 = int(m.group(2)) - num1
if num1 == 0:
out = []
for x in range(0, num0):
out.append([0,1])
return out
if num1 > num0:
c["per_chunk"], c["space"], c["first"] = int(num1/num0), num1 % num0, '1'
c["chunks"] = int(num0)
c["num1"], c["num0"] = num1, num0
else :
c["per_chunk"], c["space"], c["first"] = int(num0/num1), num0 % num1, '0'
c["chunks"] = int(num1)
c["num1"], c["num0"] = num0, num1
return frps_scheme(c)
else :
return 0
def frps_print(s,t):
out = []
for x in range(0, t):
out.append([s, 1])
return out
def frps_vv(a):
return (int(a) + 1) % 2
def frps_scheme(c):
out = []
for x in range(0, c["chunks"]):
out += frps_print(c["first"], c["per_chunk"])
c["num1"] -= c["per_chunk"]
out += frps_print(frps_vv(c["first"]), 1)
c["num0"] -= 1
out += frps_print(c["first"], c["num1"])
out += frps_print(frps_vv(c["first"]), c["num0"])
return out
def frps_cut(c, r):
m = re.match("(\d+)\/(\d+)", r)
if m:
a, b = int(m.group(1)), int(m.group(2))
if c < b:
frs = frps(r)
out = []
cnt = 0
for x, y in frs:
cnt += 1
out.append([x, y])
if cnt == c:
break
return out
else:
print "Wrong cut: $s for rps %s" % (c, r)
exit(1)
else :
print "Wrong rps format in 'frps_cut' function"
exit(1)
def frps_expand(s, e):
out = []
for x, y in s:
out.append([int(x) + e, y])
return out
def frps_ammo(s):
cnt = 0
for x, y in s:
cnt += int(x)
return cnt
### Autocases ###
###
def auto_case(url, pattern):
m = pattern.search(url)
res = ['', '', '']
if m:
el = m.groups()
res[0] = "front_page"
if el[1]:
res[0] = el[1]
if el[1] and el[3]:
res[1] = el[1] + "_" + el[3]
if el[1] and el[3] and el[5]:
res[2] = el[1] + "_" + el[3] + "_" + el[5]
return res
def get_prepared_case(uri, cases_done, pattern):
cases = auto_case(uri, pattern)
cases.reverse()
if cases_done:
for case in cases:
if cases_done[case]:
return case
return 'other'
else:
return ''
def get_autocases_tree(file):
## Pattern for autocase 3rd level
pattern = re.compile('/(.*?)(/(.*?))?(/(.*?))?(\.|/|\?|$|\s)')
pattern = re.compile('^(GET|POST|PUT|HEAD|OPTIONS|PATCH|DELETE|TRACE|LINK|UNLINK\s+)?\s*\/(.*?)(/(.*?))?(/(.*?))?(\.|/|\?|$|\s)')
max_progress = file_len(file)
widgets = ['Creating cases tree: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ' ]
pbar = ProgressBar(widgets=widgets, maxval=max_progress).start()
tree = {'other':'none'}
level1, level2, level3 = defaultdict(int), defaultdict(int), defaultdict(int)
total_cnt, line_cnt = 0, 0
input_ammo = open(file, 'r')
for line in input_ammo:
line_cnt += 1
pbar.update(line_cnt)
if re.match("^(\/|GET|POST|PUT|HEAD|OPTIONS|PATCH|DELETE|TRACE|LINK|UNLINK)", line):
total_cnt += 1
cases = auto_case(line, pattern)
if cases[0]:
level1[cases[0]] += 1
tree[cases[0]] = cases[0]
if cases[1]:
level2[cases[1]] += 1
tree[cases[1]] = cases[0]
if cases[2]:
level3[cases[2]] += 1
tree[cases[2]] = cases[1]
pbar.finish()
print
return (level1, level2, level3, tree, total_cnt)
def get_autocases_tree_access(file):
## Pattern for autocase 3rd level
pattern = re.compile('(GET|POST|PUT|HEAD|OPTIONS|PATCH|DELETE|TRACE|LINK|UNLINK\s+)\s*\/(.*?)(/(.*?))?(/(.*?))?(\.|/|\?|$|\s)')
max_progress = file_len(file)
widgets = ['Creating cases tree: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ' ]
pbar = ProgressBar(widgets=widgets, maxval=max_progress).start()
tree = {'other':'none'}
level1, level2, level3 = defaultdict(int), defaultdict(int), defaultdict(int)
total_cnt, byte_cnt = 0, 0
input_ammo = open(file, 'r')
for line in input_ammo:
byte_cnt += len(line)
pbar.update(byte_cnt)
if re.search("(\/|GET|POST|PUT|HEAD|OPTIONS|PATCH|DELETE|TRACE|LINK|UNLINK)", line):
total_cnt += 1
cases = auto_case(line, pattern)
if cases[0]:
level1[cases[0]] += 1
tree[cases[0]] = cases[0]
if cases[1]:
level2[cases[1]] += 1
tree[cases[1]] = cases[0]
if cases[2]:
level3[cases[2]] += 1
tree[cases[2]] = cases[1]
pbar.finish()
print
return (level1, level2, level3, tree, total_cnt)
def accesslog2phout(access_file, phout_file):
max_progress = file_len(access_file)
widgets = ['Access log to phout: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ' ]
pbar = ProgressBar(widgets=widgets, maxval=max_progress).start()
pattern = re.compile("\[(.+)\s(.+)\].+?\"(.+?)\"\s(\d+)\s.+?(\d+)$")
access = open(access_file, "r")
phout = open(phout_file, "w")
error = open("access.error", "w")
byte_cnt = 0
chunk = ""
for line in access:
byte_cnt += len(line)
pbar.update(byte_cnt)
m = pattern.search(line)
if m:
t = str(time.mktime(time.strptime(m.group(1),'%d/%b/%Y:%H:%M:%S')))+"00"
s = "%s\t\t%s\t%s\t0\t0\t0\t%s\n" % (t, m.group(5), m.group(5), m.group(4))
phout.write(s)
else :
error.write(line)
phout.write(chunk)
phout.close()
error.close()
access.close()
pbar.finish()
def make_autocases_top(l1, l2, l3, total_cnt, tree):
# max cases count & min percentage
N, alpha = 9, 1
total_done, n = 0, 0
output = ""
cases_done = defaultdict(int)
sl1 = sorted(l1.iteritems(), key=operator.itemgetter(1), reverse = True)
sl2 = sorted(l2.iteritems(), key=operator.itemgetter(1), reverse = True)
sl3 = sorted(l3.iteritems(), key=operator.itemgetter(1), reverse = True)
n = 0
output += "level 1\n"
for s in sl1:
t = float(100*s[1])/float(total_cnt)
if t >= alpha and n < N:
cases_done[s[0]] = s[1]
total_done += s[1]
n += 1
output += "\t%s. [level 1]\t%s\t%.2f%s\n" % (n,s[0],t,'%')
n = 0
output += "level 2:\n"
for s in sl2:
t = float(100*s[1])/float(total_cnt)
if t >= alpha and n < N:
if len(cases_done) < N:
cases_done[s[0]] = s[1]
total_done += s[1]
if cases_done[tree[s[0]]]:
cases_done[tree[s[0]]] -= s[1]
total_done -= s[1]
if cases_done[tree[s[0]]] == 0:
del cases_done[tree[s[0]]]
n -= 1
n += 1
output += "\t%s. [level 2]\t%s\t%.2f%s [%s]\n" % (n,s[0],t,'%',tree[s[0]])
output += "level 3:\n"
n = 0
for s in sl3:
t = float(100*s[1])/float(total_cnt)
if t >= alpha and n < N:
if len(cases_done) < N:
cases_done[s[0]] = s[1]
total_done += s[1]
if cases_done[tree[s[0]]]:
cases_done[tree[s[0]]] -= s[1]
total_done -= s[1]
if cases_done[tree[s[0]]] == 0:
del cases_done[tree[s[0]]]
n -= 1
n += 1
output += "\t%s. [level 3]\t%s\t%.2f%s [%s]\n" % (n,s[0],t,'%',tree[s[0]])
n = 0
if total_done < total_cnt:
cases_done['other'] = total_cnt - total_done
csds = sorted(cases_done.iteritems(), key=operator.itemgetter(1), reverse = True)
output += "cases done:\n"
for s in csds:
n += 1
t = float(100*s[1])/float(total_cnt)
output += "\t%s. [done]\t%s\t%s\t%.2f%s [%s]\n" % (n, s[1], s[0], t, '%', tree[s[0]])
return (cases_done, output)
### Utilities ###
###
def str2sec(st):
pattern = re.compile('(\d+)(h|m|s|)')
sec = 0
m = pattern.match(str(st))
if m:
el = m.groups()
if el[1] == 'h':
return int(el[0])*3600
elif el[1] == 'm':
return int(el[0])*60
elif el[1] == 's':
return int(el[0])
else:
return int(el[0])
return ''
def parse_uri(line):
uri, header_list = "", {}
pattern = re.compile('^(\/\S*?)\s+(\[.+)')
h = re.match("^\/", line)
if h:
m = pattern.match(line)
if m:
el = m.groups()
uri = el[0]
header_list = get_headers_list(el[1])
else:
uri = line
return (uri, header_list)
def chunk_by_uri(line, http, time, case, common_header, config_header):
'''Create request for uri and header'''
chunk = ""
header = {}
header.update(common_header)
(uri, header_custom) = parse_uri(line)
if header_custom:
header.update(header_custom)
header.update(config_header)
req = "GET %s HTTP/%s\r\n" % (uri, http) + header_print(header) + "\r\n"
chunk = "%s %s %s\n%s\n" % (len(req), time, case, req)
return chunk
def file_len(fname):
return os.path.getsize(fname)
def loop_ammo_file(file, loop, common_header):
print "1"
os.system("ls -lah")
pass