Merge pull request #1 from rbkmoney/test

Minigun load generator implementation

.gitignore

@@ -17,3 +17,6 @@ build/
 dist/
 yandextank.egg-info/
 setuptools-*
+logs/
+Dockerfile
+mg

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "build_utils"]
+	path = build_utils
+	url = git@github.com:rbkmoney/build_utils.git
+	branch = master
+[submodule "minigun"]
+	path = minigun
+	url = git@github.com:manikdv/minigun.git

Dockerfile.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+cat <<EOF
+FROM $BASE_IMAGE
+MAINTAINER Dmitry Manik <d.manik@rbkmoney.com>
+ADD . /tmp/yandex-tank
+RUN pip install -e /tmp/yandex-tank/ && \
+    pip install yatank-online && \
+    cp /tmp/yandex-tank/minigun/_build/default/bin/minigun /tmp/yandex-tank/mg
+
+EXPOSE 8022
+# A bit of magic below to get a proper branch name
+# even when the HEAD is detached (Hey Jenkins!
+# BRANCH_NAME is available in Jenkins env).
+LABEL com.rbkmoney.$SERVICE_NAME.parent=$BASE_IMAGE_NAME \
+      com.rbkmoney.$SERVICE_NAME.parent_tag=$BASE_IMAGE_TAG \
+      com.rbkmoney.$SERVICE_NAME.build_img=build \
+      com.rbkmoney.$SERVICE_NAME.build_img_tag=$BUILD_IMAGE_TAG \
+      com.rbkmoney.$SERVICE_NAME.commit_id=$(git rev-parse HEAD) \
+      com.rbkmoney.$SERVICE_NAME.commit_number=$(git rev-list --count HEAD) \
+      com.rbkmoney.$SERVICE_NAME.branch=$( \
+        if [ "HEAD" != $(git rev-parse --abbrev-ref HEAD) ]; then \
+          echo $(git rev-parse --abbrev-ref HEAD); \
+        elif [ -n "$BRANCH_NAME" ]; then \
+          echo $BRANCH_NAME; \
+        else \
+          echo $(git name-rev --name-only HEAD); \
+        fi)
+WORKDIR /tmp/yandex-tank
+ENTRYPOINT ["yandex-tank"]
+CMD ["--lock-dir", "/tmp", "-o", "phantom.connection_test=0"]
+EOF

Makefile

@@ -0,0 +1,43 @@
+REBAR := $(shell which rebar3 2>/dev/null || which ./rebar3)
+SUBMODULES = build_utils minigun
+SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES))
+
+UTILS_PATH := build_utils
+TEMPLATES_PATH := .
+
+# Name of the service
+SERVICE_NAME := yandex-tank
+# Service image default tag
+SERVICE_IMAGE_TAG ?= $(shell git rev-parse HEAD)
+# The tag for service image to be pushed with
+SERVICE_IMAGE_PUSH_TAG ?= $(SERVICE_IMAGE_TAG)
+
+# Base image for the service
+BASE_IMAGE_NAME := service-python
+BASE_IMAGE_TAG := ea0d48df85c141e6a625539d09fb7b46493e079f
+BUILD_IMAGE_TAG := 55e987e74e9457191a5b4a7c5dc9e3838ae82d2b
+CALL_ANYWHERE := all submodules install start
+
+# Hint: 'test' might be a candidate for CALL_W_CONTAINER-only target
+CALL_W_CONTAINER := $(CALL_ANYWHERE) build_minigun
+
+.PHONY: $(CALL_W_CONTAINER) build_minigun
+
+all: install minigun test
+
+-include $(UTILS_PATH)/make_lib/utils_container.mk
+-include $(UTILS_PATH)/make_lib/utils_image.mk
+
+$(SUBTARGETS): %/.git: %
+	git submodule update --init $<
+	touch $@
+
+submodules: $(SUBTARGETS)
+
+build_minigun: submodules
+	cd minigun; \
+	$(REBAR) escriptize; \
+	cp _build/default/bin/minigun ../mg
+
+test: submodules build_image
+	docker run -v $(PWD):$(PWD) --workdir $(PWD) $(SERVICE_IMAGE_NAME):$(SERVICE_IMAGE_TAG)

README.md

@@ -1,38 +1,22 @@
-# Yandex Tank [![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/yandex/yandex-tank?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+С помощью танка можно провести нагрузочное тестирование макросервиса.
 
-[![Build Status](https://secure.travis-ci.org/yandex/yandex-tank.png?branch=master)](http://travis-ci.org/yandex/yandex-tank)
-
-![Quantiles chart example](/logos/screen.png)
-
-## Description
-Yandex.Tank is an extendable open source load testing tool for advanced linux users which is especially good as a part of automated load testing suit.
-
-## Main features
-* different load generators supported:
-  * Evgeniy Mamchits' [phantom](https://github.com/yandex-load/phantom) is a very fast (100 000+ RPS) shooter written in C++ (default)
-  * [JMeter](http://jmeter.apache.org/) is an extendable and widely known one
-  * BFG is an experimental Python-based generator that allows you to write your own shooter function (included)
-* customizable reports in .html with pretty interactive charts based on [highcharts](http://www.highcharts.com/) library
-* [graphite](https://graphite.readthedocs.org/en/latest/overview.html) support
-* several ammo formats supported like plain url list or access.log
-* test autostop plugin
-* customizable and extendable monitoring that works over SSH
-
-## Installation and configuration
-Installation at [ReadTheDocs](http://yandextank.readthedocs.org/en/latest/install.html)
-
-## Get help
-Documentation at [ReadTheDocs](https://yandextank.readthedocs.org/en/latest/)
-
-Ask your questions at [Stackoverflow](https://stackoverflow.com/), use "load-testing" + "yandex" tags.
-
-## See also
-Evgeniy Mamchits' [phantom](https://github.com/yandex-load/phantom) - Phantom scalable IO Engine
-
-BlazeMeter's [Sense](https://sense.blazemeter.com/) - service for storing and analysing performance test results
-
-[Jenkins](https://jenkins-ci.org/) - an extendable open source continuous integration server that may be used to automate test execution.
-
-[Graphite](https://graphite.readthedocs.org/en/latest/overview.html) - an enterprise-scale monitoring tool, use it to store your test results and render graphs.
-
-![Yandex.Metrics counter](https://mc.yandex.ru/watch/17743264)
+Шаги:
+1. Поменять в load.ini настройки
+    ; Minigun config section:
+    [minigun]
+    phout=phout.log
+    minigun_cmd=./mg
+    worker_mod=minigun_worker_api_impl
+    ccw=10 ;Concurrent workers
+    aps=3 ;Actions per second
+    wps=5 ;Workers launching speed (Workers per second)
+    session_duration=5000
+    total_duration=50000
+    auth_url=auth.rbk.test
+    auth_port=8080
+    auth_name=demo_merchant
+    auth_pass=test
+    api_url=api.rbk.test
+    api_port=8080
+2. make wc_build_minigun
+3. make test

build_utils

@@ -0,0 +1 @@
+Subproject commit 877ae1c8299675fff8c4d445657b60064176b077

minigun

@@ -0,0 +1 @@
+Subproject commit 7f1f852f92792f1df730ab8f95225cd1c1742893

yandextank/plugins/Minigun.py

@@ -0,0 +1,173 @@
+from Aggregator import AggregatorPlugin, AggregateResultListener
+from yandextank.core import AbstractPlugin
+from Phantom import PhantomReader
+from ConsoleOnline import ConsoleOnlinePlugin, AbstractInfoWidget
+import subprocess
+import time
+import shlex
+import ConsoleScreen
+import datetime
+
+
+class MinigunPlugin(AbstractPlugin, AggregateResultListener):
+    SECTION = 'minigun'
+
+    def __init__(self, core):
+        AbstractPlugin.__init__(self, core)
+        self.process = None
+        self.process_stderr = None
+        self.process_start_time = None
+        self.enum_ammo = False
+        self.buffered_seconds = 2
+
+    @staticmethod
+    def get_key():
+        return __file__
+
+    def get_available_options(self):
+        opts = [
+            "minigun_cmd",
+            "worker_mod",
+            "ccw",
+            "aps",
+            "wps",
+            "session_duration",
+            "total_duration",
+            "auth_url",
+            "auth_port",
+            "auth_name",
+            "auth_pass",
+            "api_url",
+            "api_port"
+        ]
+        return opts
+
+    def configure(self):
+        ''' Read options from config '''
+        self.minigun_cmd = self.get_option("minigun_cmd")
+        self.worker_mod = self.get_option("worker_mod")
+        self.ccw = self.get_option("ccw")
+        self.aps = self.get_option("aps")
+        self.wps = self.get_option("wps")
+        self.session_duration = self.get_option("session_duration")
+        self.total_duration = self.get_option("total_duration")
+        self.phout = self.get_option("phout", "")
+        self.auth_url = self.get_option("auth_url")
+        self.auth_port = self.get_option("auth_port")
+        self.auth_name = self.get_option("auth_name")
+        self.auth_pass = self.get_option("auth_pass")
+        self.api_url = self.get_option("api_url")
+        self.api_port = self.get_option("api_port")
+
+        if self.phout:
+            self.core.add_artifact_file(self.phout)
+
+    def prepare_test(self):
+        aggregator = None
+        try:
+            aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
+        except Exception, ex:
+            self.log.warning("No aggregator found: %s", ex)
+
+        if aggregator:
+            aggregator.reader = PhantomReader(aggregator, self)
+            aggregator.reader.buffered_seconds = self.buffered_seconds
+            aggregator.add_result_listener(self)
+            aggregator.reader.phout_file = self.phout
+
+        try:
+            console = self.core.get_plugin_of_type(ConsoleOnlinePlugin)
+        except Exception, ex:
+            self.log.debug("Console not found: %s", ex)
+            console = None
+
+        if console:
+            widget = MinigunInfoWidget(self)
+            console.add_info_widget(widget)
+            aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
+            aggregator.add_result_listener(widget)
+
+    def start_test(self):
+        args = [
+            self.minigun_cmd,
+            self.worker_mod,
+            self.ccw,
+            self.aps,
+            self.wps,
+            self.session_duration,
+            self.total_duration,
+            self.auth_url,
+            self.auth_port,
+            self.auth_name,
+            self.auth_pass,
+            self.api_url,
+            self.api_port
+        ]
+        self.log.info("Starting: %s", args)
+        self.process_start_time = time.time()
+        process_stderr_file = self.core.mkstemp(".log", "minigun_")
+        self.core.add_artifact_file(process_stderr_file)
+        self.process_stderr = open(process_stderr_file, 'w')
+        self.process = subprocess.Popen(
+            args,
+            stderr=self.process_stderr,
+            stdout=self.process_stderr,
+            close_fds=True)
+        self.process = subprocess.Popen(args, stderr=self.process_stderr, stdout=self.process_stderr, close_fds=True)
+
+    def is_test_finished(self):
+        retcode = self.process.poll()
+        if retcode is not None:
+            self.log.info("Subprocess done its work with exit code: %s", retcode)
+            return abs(retcode)
+        else:
+            return -1
+
+    def end_test(self, retcode):
+        if self.process and self.process.poll() is None:
+            self.log.warn("Terminating worker process with PID %s", self.process.pid)
+            self.process.terminate()
+            if self.process_stderr:
+                self.process_stderr.close()
+        else:
+            self.log.debug("Seems subprocess finished OK")
+        return retcode
+
+    def get_info(self):
+        pass
+
+    def aggregate_second(self, second_aggregate_data):
+        pass
+
+
+class MinigunInfoWidget(AbstractInfoWidget):
+    ''' Right panel widget '''
+    def __init__(self, uniphout):
+        AbstractInfoWidget.__init__(self)
+        self.krutilka = ConsoleScreen.krutilka()
+        self.owner = uniphout
+        self.rps = 0
+
+    def get_index(self):
+        return 0
+
+    def aggregate_second(self, second_aggregate_data):
+        self.active_threads = second_aggregate_data.overall.active_threads
+        self.rps = second_aggregate_data.overall.RPS
+
+    def render(self, screen):
+        text = " Uniphout Test %s" % self.krutilka.next()
+        space = screen.right_panel_width - len(text) - 1
+        left_spaces = space / 2
+        right_spaces = space / 2
+
+        dur_seconds = int(time.time()) - int(self.owner.process_start_time)
+        duration = str(datetime.timedelta(seconds=dur_seconds))
+
+        template = screen.markup.BG_BROWN + '~' * left_spaces + text + ' ' + '~' * right_spaces + screen.markup.RESET + "\n"
+        template += "Command Line: %s\n"
+        template += "    Duration: %s\n"
+        template += " Responses/s: %s"
+        data = (self.owner.minigun_cmd, duration, self.rps)
+
+        return template % data