CAPI-344: add initial implementation (#1)

This commit is contained in:
Sergei 2019-04-05 14:02:11 +03:00 committed by GitHub
parent 9929ce0c02
commit 9202542588
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 1538 additions and 0 deletions

15
.gitignore vendored Normal file
View File

@ -0,0 +1,15 @@
# general
log
/_build/
/_checkouts/
*~
erl_crash.dump
.tags*
*.sublime-workspace
.edts
.DS_Store
Dockerfile
docker-compose.yml
/.idea/
*.beam
rebar3.crashdump

4
.gitmodules vendored Normal file
View File

@ -0,0 +1,4 @@
[submodule "build_utils"]
path = build_utils
url = git@github.com:rbkmoney/build_utils.git
branch = master

24
Dockerfile.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
cat <<EOF
FROM $BASE_IMAGE
MAINTAINER Sergei Shuvatov <s.shuvatov@rbkmoney.com>
COPY ./_build/prod/rel/bender /opt/bender
CMD /opt/bender/bin/bender foreground
EXPOSE 8022
LABEL base_image_tag=$BASE_IMAGE_TAG
LABEL build_image_tag=$BUILD_IMAGE_TAG
# A bit of magic to get a proper branch name
# even when the HEAD is detached (Hey Jenkins!
# BRANCH_NAME is available in Jenkins env).
LABEL branch=$( \
if [ "HEAD" != $(git rev-parse --abbrev-ref HEAD) ]; then \
echo $(git rev-parse --abbrev-ref HEAD); \
elif [ -n "$BRANCH_NAME" ]; then \
echo $BRANCH_NAME; \
else \
echo $(git name-rev --name-only HEAD); \
fi)
LABEL commit=$(git rev-parse HEAD)
LABEL commit_number=$(git rev-list --count HEAD)
WORKDIR /opt/bender
EOF

65
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,65 @@
#!groovy
// -*- mode: groovy -*-
def finalHook = {
runStage('store CT logs') {
archive '_build/test/logs/'
}
}
build('bender', 'docker-host', finalHook) {
checkoutRepo()
loadBuildUtils()
def pipeDefault
def withWsCache
runStage('load pipeline') {
env.JENKINS_LIB = "build_utils/jenkins_lib"
pipeDefault = load("${env.JENKINS_LIB}/pipeDefault.groovy")
withWsCache = load("${env.JENKINS_LIB}/withWsCache.groovy")
}
pipeDefault() {
if (env.BRANCH_NAME != 'master') {
runStage('compile') {
withGithubPrivkey {
sh 'make wc_compile'
}
}
runStage('lint') {
sh 'make wc_lint'
}
runStage('xref') {
sh 'make wc_xref'
}
runStage('dialyze') {
withWsCache("_build/default/rebar3_21.1.1_plt") {
sh 'make wc_dialyze'
}
}
runStage('test') {
sh "make wdeps_test"
}
}
runStage('make release') {
withGithubPrivkey {
sh "make wc_release"
}
}
runStage('build image') {
sh "make build_image"
}
try {
if (masterlikeBranch()) {
runStage('push image') {
sh "make push_image"
}
}
} finally {
runStage('rm local image') {
sh 'make rm_local_image'
}
}
}
}

75
Makefile Normal file
View File

@ -0,0 +1,75 @@
REBAR := $(shell which rebar3 2>/dev/null || which ./rebar3)
SUBMODULES = build_utils
SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES))
UTILS_PATH := build_utils
TEMPLATES_PATH := .
# Name of the service
SERVICE_NAME := bender
# Service image default tag
SERVICE_IMAGE_TAG ?= $(shell git rev-parse HEAD)
# The tag for service image to be pushed with
SERVICE_IMAGE_PUSH_TAG ?= $(SERVICE_IMAGE_TAG)
# Base image for the service
BASE_IMAGE_NAME := service-erlang
BASE_IMAGE_TAG := bdb3e60ddc70044bae1aa581d260d3a9803a2477
# Build image tag to be used
BUILD_IMAGE_TAG := f3732d29a5e622aabf80542b5138b3631a726adb
CALL_ANYWHERE := all submodules rebar-update compile xref lint dialyze start devrel release clean distclean check
CALL_W_CONTAINER := $(CALL_ANYWHERE) test
all: compile
-include $(UTILS_PATH)/make_lib/utils_container.mk
-include $(UTILS_PATH)/make_lib/utils_image.mk
.PHONY: $(CALL_W_CONTAINER)
# CALL_ANYWHERE
$(SUBTARGETS): %/.git: %
git submodule update --init $<
touch $@
submodules: $(SUBTARGETS)
rebar-update:
$(REBAR) update
compile: submodules rebar-update
$(REBAR) compile
xref: submodules
$(REBAR) xref
lint:
elvis rock
dialyze:
$(REBAR) dialyzer
check: xref lint dialyze
start: submodules
$(REBAR) run
devrel: submodules
$(REBAR) release
release: distclean
$(REBAR) as prod release
clean:
$(REBAR) clean
distclean:
$(REBAR) clean
rm -rf _build
# CALL_W_CONTAINER
test: submodules
$(REBAR) ct

3
apps/bender/rebar.config Normal file
View File

@ -0,0 +1,3 @@
{erl_opts, [
{parse_transform, lager_transform}
]}.

View File

@ -0,0 +1,18 @@
{application, bender, [
{description, "Bender service"},
{vsn, "1.0.0"},
{registered, []},
{applications, [
kernel,
stdlib,
lager,
woody,
scoper,
bender_proto,
machinery,
erl_health,
snowflake
]},
{mod, {bender, []}},
{env, []}
]}.

133
apps/bender/src/bender.erl Normal file
View File

@ -0,0 +1,133 @@
-module(bender).
-behaviour(supervisor).
-behaviour(application).
%% API
-export([start/0]).
-export([stop/0]).
%% Application callbacks
-export([start/2]).
-export([stop/1]).
%% Supervisor callbacks
-export([init/1]).
-include("bender_internal.hrl").
-type schema() :: snowflake | #constant{} | #sequence{}.
-export_type([schema/0]).
%% API
-spec start() ->
{ok, [atom()]} | {error, {atom(), term()}}.
start() ->
application:ensure_all_started(?MODULE).
-spec stop() ->
ok | {error, term()}.
stop() ->
application:stop(?MODULE).
%% Application callbacks
-spec start(normal, any()) ->
{ok, pid()} | {error, any()}.
start(_StartType, _StartArgs) ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-spec stop(any()) ->
ok.
stop(_State) ->
ok.
%% Supervisor callbacks
-spec init([]) ->
{ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}.
init([]) ->
ChildSpec = woody_server:child_spec(
?MODULE,
#{
ip => get_ip_address(),
port => get_port(),
protocol_opts => get_protocol_opts(),
transport_opts => get_transport_opts(),
event_handler => scoper_woody_event_handler,
handlers => [get_handler_spec()],
additional_routes => get_routes()
}
),
{ok, {
#{strategy => one_for_all, intensity => 6, period => 30},
[ChildSpec]
}}.
-spec get_ip_address() ->
inet:ip_address().
get_ip_address() ->
{ok, Address} = inet:parse_address(genlib_app:env(?MODULE, ip, "::")),
Address.
-spec get_port() ->
inet:port_number().
get_port() ->
genlib_app:env(?MODULE, port, 8022).
-spec get_protocol_opts() ->
woody_server_thrift_http_handler:protocol_opts().
get_protocol_opts() ->
genlib_app:env(?MODULE, protocol_opts, #{}).
-spec get_transport_opts() ->
woody_server_thrift_http_handler:transport_opts().
get_transport_opts() ->
genlib_app:env(?MODULE, transport_opts, #{}).
-spec get_handler_spec() ->
woody:http_handler(woody:th_handler()).
get_handler_spec() ->
Opts = genlib_app:env(?MODULE, service, #{}),
Path = maps:get(path, Opts, <<"/v1/bender">>),
{Path, {
{bender_thrift, 'Bender'},
bender_handler
}}.
-spec get_routes() ->
[woody_server_thrift_http_handler:route(_)].
get_routes() ->
RouteOptsEnv = genlib_app:env(?MODULE, route_opts, #{}),
RouteOpts = RouteOptsEnv#{event_handler => scoper_woody_event_handler},
Generator = genlib_app:env(bender, generator, #{}),
Sequence = genlib_app:env(bender, sequence, #{}),
Handlers = [
{bender_generator, #{
path => maps:get(path, Generator, <<"/v1/stateproc/bender_generator">>),
backend_config => #{
schema => maps:get(schema, Generator, machinery_mg_schema_generic)
}
}},
{bender_sequence, #{
path => maps:get(path, Sequence, <<"/v1/stateproc/bender_sequence">>),
backend_config => #{
schema => maps:get(schema, Sequence, machinery_mg_schema_generic)
}
}}
],
HealthCheckers = genlib_app:env(?MODULE, health_checkers, []),
[erl_health_handle:get_route(HealthCheckers) |
machinery_mg_backend:get_routes(Handlers, RouteOpts)].

View File

@ -0,0 +1,135 @@
-module(bender_generator).
%% API
-export([bind/4]).
%% Machinery callbacks
-behaviour(machinery).
-export([init/4]).
-export([process_call/4]).
-export([process_timeout/3]).
-export([process_repair/4]).
-type external_id() :: binary().
-type internal_id() :: binary().
-type schema() :: bender:schema().
-type user_context() :: msgpack_thrift:'Value'() | undefined.
-type state() :: #{
internal_id := internal_id(),
user_context := user_context()
}.
-type woody_context() :: woody_context:ctx().
-type args(T) :: machinery:args(T).
-type machine() :: machinery:machine(_, state()).
-type handler_args() :: machinery:handler_args(_).
-type handler_opts() :: machinery:handler_opts(_).
-type result(A) :: machinery:result(none(), A).
-include("bender_internal.hrl").
-define(NS, bender_generator).
%%% API
-spec bind(external_id(), schema(), user_context(), woody_context()) ->
{ok, internal_id(), user_context()} | no_return().
bind(ExternalID, Schema, UserCtx, WoodyCtx) ->
case start(ExternalID, Schema, UserCtx, WoodyCtx) of
ok ->
{ok, InternalID, _} = get(ExternalID, WoodyCtx),
{ok, InternalID, undefined};
{error, exists} ->
get(ExternalID, WoodyCtx)
end.
%%% Machinery callbacks
-spec init(args({schema(), user_context()}), machine(), handler_args(), handler_opts()) ->
result(state()).
init({Schema, UserCtx}, _Machine, _HandlerArgs, HandlerOpts) ->
InternalID = generate(Schema, HandlerOpts),
#{
aux_state => #{
internal_id => InternalID,
user_context => UserCtx
}
}.
-spec process_call(args(_), machine(), handler_args(), handler_opts()) ->
no_return().
process_call(_Args, _Machine, _HandlerArgs, _HandlerOpts) ->
not_implemented(call).
-spec process_timeout(machine(), handler_args(), handler_opts()) ->
no_return().
process_timeout(_Machine, _HandlerArgs, _HandlerOpts) ->
not_implemented(timeout).
-spec process_repair(args(_), machine(), handler_args(), handler_opts()) ->
no_return().
process_repair(_Args, _Machine, _HandlerArgs, _HandlerOpts) ->
not_implemented(repair).
%%% Internal functions
-spec start(external_id(), schema(), user_context(), woody_context()) ->
ok | {error, exists}.
start(ExternalID, Schema, UserCtx, WoodyCtx) ->
machinery:start(?NS, ExternalID, {Schema, UserCtx}, get_backend(WoodyCtx)).
-spec get(external_id(), woody_context()) ->
{ok, internal_id(), user_context()} | no_return().
get(ExternalID, WoodyCtx) ->
case machinery:get(?NS, ExternalID, get_backend(WoodyCtx)) of
{ok, Machine} ->
#{
internal_id := InternalID,
user_context := UserCtx
} = get_machine_state(Machine),
{ok, InternalID, UserCtx};
{error, notfound} ->
throw({not_found, ExternalID})
end.
-spec get_machine_state(machine()) ->
state().
get_machine_state(#{aux_state := State}) ->
State.
-spec get_backend(woody_context()) ->
machinery_mg_backend:backend().
get_backend(WoodyCtx) ->
bender_utils:get_backend(generator, WoodyCtx).
-spec not_implemented(any()) ->
no_return().
not_implemented(What) ->
erlang:error({not_implemented, What}).
-spec generate(schema(), handler_opts()) ->
internal_id().
generate(snowflake, _HandlerOpts) ->
bender_utils:unique_id();
generate(#constant{internal_id = InternalID}, _HandlerOpts) ->
InternalID;
generate(#sequence{id = SequenceID}, #{woody_ctx := WoodyCtx}) ->
{ok, Value} = bender_sequence:get_next(SequenceID, WoodyCtx),
integer_to_binary(Value).

View File

@ -0,0 +1,65 @@
-module(bender_handler).
%% Woody handler
-behaviour(woody_server_thrift_handler).
-export([handle_function/4]).
-include_lib("bender_proto/include/bender_thrift.hrl").
-include("bender_internal.hrl").
-type woody_context() :: woody_context:ctx().
-type external_id() :: bender_thrift:'ExternalID'().
-type schema() :: bender:schema().
-type user_context() :: msgpack_thrift:'Value'().
-type result() :: bender_thrift:'GenerationResult'().
-spec handle_function(woody:func(), woody:args(), woody_context(), woody:options()) ->
{ok, woody:result()}.
handle_function(Func, Args, WoodyCtx, Opts) ->
scoper:scope(bender,
fun() -> handle_function_(Func, Args, WoodyCtx, Opts) end
).
-spec handle_function_(woody:func(), woody:args(), woody_context(), woody:options()) ->
{ok, woody:result()}.
handle_function_('GenerateID', [ExternalID, Schema, UserCtx], WoodyCtx, _Opts) ->
scoper:add_meta(#{
external_id => ExternalID
}),
generate_id(ExternalID, Schema, UserCtx, WoodyCtx).
-spec generate_id(external_id(), bender_thrift:'GenerationSchema'(), user_context(), woody_context()) ->
{ok, result()} | no_return().
generate_id(ExternalID, {constant, #bender_ConstantSchema{} = Schema}, UserCtx, WoodyCtx) ->
NewInternalID = Schema#bender_ConstantSchema.internal_id,
Constant = #constant{internal_id = NewInternalID},
bind(ExternalID, Constant, UserCtx, WoodyCtx);
generate_id(ExternalID, {sequence, #bender_SequenceSchema{} = Schema}, UserCtx, WoodyCtx) ->
SequenceID = Schema#bender_SequenceSchema.sequence_id,
Sequence = #sequence{id = SequenceID},
bind(ExternalID, Sequence, UserCtx, WoodyCtx);
generate_id(ExternalID, {snowflake, #bender_SnowflakeSchema{}}, UserCtx, WoodyCtx) ->
bind(ExternalID, snowflake, UserCtx, WoodyCtx);
generate_id(_ExternalID, Schema, _UserCtx, _WoodyCtx) ->
erlang:error({unknown_schema, Schema}).
-spec bind(external_id(), schema(), user_context(), woody_context()) ->
{ok, result()} | no_return().
bind(ExternalID, Schema, UserCtx, WoodyCtx) ->
{ok, InternalID, PrevUserCtx} = bender_generator:bind(ExternalID, Schema, UserCtx, WoodyCtx),
Result = #bender_GenerationResult{
internal_id = InternalID,
context = PrevUserCtx
},
{ok, Result}.

View File

@ -0,0 +1,12 @@
-ifndef(__BENDER_INTERNAL_HRL__).
-define(__BENDER_INTERNAL_HRL__, included).
-record(constant, {
internal_id :: bender_thrift:'InternalID'()
}).
-record(sequence, {
id :: binary()
}).
-endif.

View File

@ -0,0 +1,161 @@
-module(bender_sequence).
%% API
-export([get_current/2]).
-export([get_next/2]).
%% Machinery callbacks
-behaviour(machinery).
-export([init/4]).
-export([process_call/4]).
-export([process_timeout/3]).
-export([process_repair/4]).
-type id() :: binary().
-export_types([id/0]).
-type woody_context() :: woody_context:ctx().
-type args(T) :: machinery:args(T).
-type machine() :: machinery:machine(_, state()).
-type handler_args() :: machinery:handler_args(_).
-type handler_opts() :: machinery:handler_opts(_).
-type result(A) :: machinery:result(none(), A).
-type response(T) :: machinery:response(T).
-type value() :: non_neg_integer().
-type state() :: #{
value := value()
}.
-define(NS, bender_sequence).
-define(initial_value, 0).
%%% API
-spec get_current(id(), woody_context()) ->
{ok, value()} | {error, notfound}.
get_current(SequenceID, WoodyCtx) ->
case get_state(SequenceID, WoodyCtx) of
{ok, State} ->
{ok, get_value(State)};
_ ->
{error, notfound}
end.
-spec get_next(id(), woody_context()) ->
{ok, value()}.
get_next(SequenceID, WoodyCtx) ->
ok = ensure_started(SequenceID, WoodyCtx),
call(SequenceID, get_next, WoodyCtx).
%%% Machinery callbacks
-spec init(args([]), machine(), handler_args(), handler_opts()) ->
result(state()).
init([], _Machine, _HandlerArgs, _HandlerOpts) ->
#{
aux_state => #{
value => ?initial_value
}
}.
-spec process_call(args(get_next | any()), machine(), handler_args(), handler_opts()) ->
{response(value()), result(state())} | no_return().
process_call(get_next, Machine, _HandlerArgs, _HandlerOpts) ->
State = get_machine_state(Machine),
Value = get_value(State),
NewValue = Value + 1,
NewState = set_value(State, NewValue),
{NewValue, #{aux_state => NewState}};
process_call(_Args, _Machine, _HandlerArgs, _HandlerOpts) ->
not_implemented(call).
-spec process_timeout(machine(), handler_args(), handler_opts()) ->
no_return().
process_timeout(_Machine, _HandlerArgs, _HandlerOpts) ->
not_implemented(timeout).
-spec process_repair(args(_), machine(), handler_args(), handler_opts()) ->
no_return().
process_repair(_Args, _Machine, _HandlerArgs, _HandlerOpts) ->
not_implemented(repair).
%%% Internal functions
-spec start(id(), woody_context()) ->
ok | {error, exists}.
start(SequenceID, WoodyCtx) ->
machinery:start(?NS, SequenceID, [], get_backend(WoodyCtx)).
-spec ensure_started(id(), woody_context()) ->
ok | no_return().
ensure_started(SequenceID, WoodyCtx) ->
case start(SequenceID, WoodyCtx) of
ok ->
ok;
{error, exists} ->
ok
end.
-spec call(id(), args(_), woody_context()) ->
{ok, response(_)} | {error, notfound}.
call(SequenceID, Msg, WoodyCtx) ->
machinery:call(?NS, SequenceID, Msg, get_backend(WoodyCtx)).
-spec get_state(id(), woody_context()) ->
{ok, state()} | {error, notfound}.
get_state(SequenceID, WoodyCtx) ->
case machinery:get(?NS, SequenceID, get_backend(WoodyCtx)) of
{ok, Machine} ->
State = get_machine_state(Machine),
{ok, State};
{error, notfound} = Error ->
Error
end.
-spec get_machine_state(machine()) ->
state().
get_machine_state(#{aux_state := State}) ->
State.
-spec get_backend(woody_context()) ->
machinery_mg_backend:backend().
get_backend(WoodyCtx) ->
bender_utils:get_backend(sequence, WoodyCtx).
-spec not_implemented(any()) ->
no_return().
not_implemented(What) ->
erlang:error({not_implemented, What}).
-spec get_value(state()) ->
value().
get_value(#{value := Value}) ->
Value.
-spec set_value(state(), value()) ->
state().
set_value(State, Value) ->
State#{value => Value}.

View File

@ -0,0 +1,48 @@
-module(bender_utils).
-export([unique_id/0]).
-export([get_backend/2]).
-type woody_context() :: woody_context:ctx().
-type schema() :: machinery_mg_schema_generic | atom().
-type event_handler() :: scoper_woody_event_handler | atom().
-type automaton() :: #{
url := binary(), % machinegun's automaton url
event_handler := event_handler(),
path => binary(), % state processor path
schema => schema(),
transport_opts => woody_client_thrift_http_transport:transport_options()
}.
%%% API
-spec unique_id() ->
binary().
unique_id() ->
<<ID:64>> = snowflake:new(),
genlib_format:format_int_base(ID, 62).
-spec get_backend(atom(), woody_context()) ->
machinery_mg_backend:backend().
get_backend(Service, WoodyCtx) ->
Automaton = genlib_app:env(bender, Service, #{}),
machinery_mg_backend:new(WoodyCtx, #{
client => get_woody_client(Automaton),
schema => maps:get(schema, Automaton, machinery_mg_schema_generic)
}).
%%% Internal functions
-spec get_woody_client(automaton()) ->
machinery_mg_client:woody_client().
get_woody_client(#{url := Url, event_handler := Handler} = Automaton) ->
genlib_map:compact(#{
url => Url,
event_handler => Handler,
transport_opts => maps:get(transport_opts, Automaton, undefined)
}).

View File

@ -0,0 +1,69 @@
-module(bender_client).
-export([new/0]).
-export([generate_id/4]).
-type client() :: woody_context:ctx().
-type external_id() :: bender_thrift:'ExternalID'().
-type schema() :: bender_thrift:'GenerationSchema'().
-type user_context() :: msgpack_thrift:'Value'().
-define(retry_stategy, {linear, 5, 1000}).
%%% API
-spec new() ->
client().
new() ->
woody_context:new().
-spec generate_id(external_id(), schema(), user_context(), client()) ->
woody:result() | no_return().
generate_id(ExternalID, Schema, UserCtx, Client) ->
call('GenerateID', [ExternalID, Schema, UserCtx], Client).
%%% Internal functions
-spec call(atom(), list(), client()) ->
woody:result() | no_return().
call(Function, Args, Client) ->
Call = {{bender_thrift, 'Bender'}, Function, Args},
Opts = #{
url => <<"http://bender:8022/v1/bender">>,
event_handler => scoper_woody_event_handler,
transport_opts => #{
max_connections => 10000
}
},
call(Call, Opts, Client, ?retry_stategy).
call(Call, Opts, Client, Retry) ->
try
do_call(Call, Opts, Client)
catch
error:{woody_error, {_Source, Class, _Details}} = Error
when Class =:= resource_unavailable orelse Class =:= result_unknown ->
NextRetry = next_retry(Retry, Error),
call(Call, Opts, Client, NextRetry)
end.
do_call(Call, Opts, Client) ->
case woody_client:call(Call, Opts, Client) of
{ok, Response} ->
Response;
{exception, Exception} ->
throw(Exception)
end.
next_retry(Retry, Error) ->
retry_step(genlib_retry:next_step(Retry), Error).
retry_step(finish, Error) ->
erlang:error(Error);
retry_step({wait, Timeout, Retry}, _) ->
ok = timer:sleep(Timeout),
Retry.

View File

@ -0,0 +1,275 @@
-module(bender_tests_SUITE).
-include_lib("common_test/include/ct.hrl").
-export([all/0]).
-export([groups/0]).
-export([init_per_suite/1]).
-export([end_per_suite/1]).
-export([init_per_testcase/2]).
-export([end_per_testcase/2]).
-export([constant/1]).
-export([sequence/1]).
-export([snowflake/1]).
-export([different_schemas/1]).
-export([contention/1]).
-export([generator_init/1]).
-include_lib("bender_proto/include/bender_thrift.hrl").
-type config() :: [{atom(), term()}].
-type group_name() :: atom().
-type test_case_name() :: atom().
-define(config(Key, C), (element(2, lists:keyfind(Key, 1, C)))).
-spec all() ->
[atom()].
all() ->
[
{group, main},
{group, contention}
].
-define(parallel_workers, 100).
-define(contention_test_workers, 100).
-spec groups() -> [{group_name(), list(), [test_case_name()]}].
groups() ->
[
{main, [parallel], [
{group, constant},
{group, sequence},
{group, snowflake},
{group, different_schemas},
{group, generator_init}
]},
{constant, [parallel], [ constant || _ <- lists:seq(1, ?parallel_workers) ]},
{sequence, [parallel], [ sequence || _ <- lists:seq(1, ?parallel_workers) ]},
{snowflake, [parallel], [ snowflake || _ <- lists:seq(1, ?parallel_workers) ]},
{different_schemas, [parallel], [ different_schemas || _ <- lists:seq(1, ?parallel_workers) ]},
{generator_init, [parallel], [ generator_init || _ <- lists:seq(1, ?parallel_workers) ]},
{contention, [{repeat_until_all_ok, 10}], [
contention
]}
].
-spec init_per_suite(config()) ->
config().
init_per_suite(C) ->
Apps = genlib_app:start_application_with(lager, [
{async_threshold, 1},
{async_threshold_window, 0},
{error_logger_hwm, 600},
{suppress_application_start_stop, true},
{handlers, [
{lager_common_test_backend, [error, {lager_logstash_formatter, []}]}
]}
]) ++ genlib_app:start_application_with(scoper, [
{storage, scoper_storage_lager}
]) ++ genlib_app:start_application_with(bender, [
{generator, #{
path => <<"/v1/stateproc/bender_generator">>,
schema => machinery_mg_schema_generic,
url => <<"http://machinegun:8022/v1/automaton">>,
event_handler => scoper_woody_event_handler,
transport_opts => #{
max_connections => 1000
}
}},
{sequence, #{
path => <<"/v1/stateproc/bender_sequence">>,
schema => machinery_mg_schema_generic,
url => <<"http://machinegun:8022/v1/automaton">>,
event_handler => scoper_woody_event_handler,
transport_opts => #{
max_connections => 1000
}
}},
{protocol_opts, #{
timeout => 60000
}},
{transport_opts, #{
max_connections => 10000,
num_acceptors => 100
}}
]),
[{suite_apps, Apps} | C].
-spec end_per_suite(config()) ->
ok.
end_per_suite(C) ->
genlib_app:stop_unload_applications(?config(suite_apps, C)).
-spec init_per_testcase(atom(), config()) ->
config().
init_per_testcase(_Name, C) ->
Client = bender_client:new(),
[{client, Client} | C].
-spec end_per_testcase(atom(), config()) ->
config().
end_per_testcase(_Name, _C) ->
ok.
-spec constant(config()) ->
ok.
constant(C) ->
Client = get_client(C),
ExternalID = bender_utils:unique_id(),
InternalID = bender_utils:unique_id(),
Schema = {constant, #bender_ConstantSchema{internal_id = InternalID}},
UserCtx = {bin, <<"spiel mit mir">>},
InternalID = generate_weak(ExternalID, Schema, UserCtx, Client),
InternalID = generate_strict(ExternalID, Schema, UserCtx, Client),
ok.
-spec sequence(config()) ->
ok.
sequence(C) ->
Client = get_client(C),
SequenceID = bender_utils:unique_id(),
ExternalID = bender_utils:unique_id(),
Schema = {sequence, #bender_SequenceSchema{sequence_id = SequenceID}},
UserCtx = {bin, <<"come to daddy">>},
<<"1">> = generate_weak(ExternalID, Schema, UserCtx, Client),
OtherID = bender_utils:unique_id(),
<<"2">> = generate_weak(OtherID, Schema, UserCtx, Client),
<<"1">> = generate_strict(ExternalID, Schema, UserCtx, Client),
ok.
-spec snowflake(config()) ->
ok.
snowflake(C) ->
Client = get_client(C),
ExternalID = bender_utils:unique_id(),
Schema = {snowflake, #bender_SnowflakeSchema{}},
UserCtx = {bin, <<"breaking nudes">>},
InternalID = generate_weak(ExternalID, Schema, UserCtx, Client),
InternalID = generate_strict(ExternalID, Schema, UserCtx, Client),
ok.
-spec different_schemas(config()) ->
ok.
different_schemas(C) ->
Client = get_client(C),
ExternalID = bender_utils:unique_id(),
Schema1 = {sequence, #bender_SequenceSchema{sequence_id = bender_utils:unique_id()}},
UserCtx = {bin, <<"wo bist do">>},
InternalID = generate_weak(ExternalID, Schema1, UserCtx, Client),
Schema2 = {snowflake, #bender_SnowflakeSchema{}},
InternalID = generate_strict(ExternalID, Schema2, UserCtx, Client),
Schema3 = {constant, #bender_ConstantSchema{internal_id = bender_utils:unique_id()}},
InternalID = generate_strict(ExternalID, Schema3, UserCtx, Client),
ok.
-spec contention(config()) ->
ok.
contention(C) ->
ExternalID = bender_utils:unique_id(),
SequenceID = bender_utils:unique_id(),
Data = [
{{snowflake, #bender_SnowflakeSchema{}},
bender_utils:unique_id()} ||
_ <- lists:seq(1, ?contention_test_workers)
] ++ [
{{constant, #bender_ConstantSchema{internal_id = bender_utils:unique_id()}},
bender_utils:unique_id()} ||
_ <- lists:seq(1, ?contention_test_workers)
] ++ [
{{sequence, #bender_SequenceSchema{sequence_id = SequenceID}},
bender_utils:unique_id()} ||
_ <- lists:seq(1, ?contention_test_workers)
],
Generate =
fun({Schema, UserCtx}) ->
Client = get_client(C),
UserCtx1 = {bin, term_to_binary({Schema, UserCtx})},
{InternalID, PrevUserCtx} = generate(ExternalID, Schema, UserCtx1, Client),
{{ExternalID, InternalID, PrevUserCtx}, {Schema, UserCtx}}
end,
Result = genlib_pmap:map(Generate, shuffle(Data)),
[
% There is a case possible when a winner receives transient error such as timeout
% but record is actually stored in machinegun, winner retries it's request and
% receives response with user context already stored before, not undefined.
% So we just repeat this test until ok or maximum number of retries reached
{{ExternalID, InternalID, undefined}, UserCtxOfWinner},
{{ExternalID, InternalID, {bin, BinaryCtx}}, _OtherUserCtx}
] = lists:ukeysort(1, Result),
UserCtxOfWinner = binary_to_term(BinaryCtx),
ok.
-include_lib("mg_proto/include/mg_proto_state_processing_thrift.hrl").
-spec generator_init(config()) ->
ok.
generator_init(_C) ->
Request = #mg_stateproc_SignalArgs{
signal = {
init,
#mg_stateproc_InitSignal{
arg = {arr, [{str, <<"tup">>}, {str, <<"snowflake">>}, {bin, <<"user context">>}]}
}
},
machine = #mg_stateproc_Machine{
ns = <<"bender_generator">>,
id = <<"42">>,
history = [],
history_range = #mg_stateproc_HistoryRange{},
aux_state = {bin, <<>>},
timer = undefined
}
},
Call = {{mg_proto_state_processing_thrift, 'Processor'}, 'ProcessSignal', [Request]},
Options = #{
url => <<"http://localhost:8022/v1/stateproc/bender_generator">>,
event_handler => scoper_woody_event_handler,
transport_opts => #{
checkout_timeout => 1000,
max_connections => 10000
}
},
{ok, _Result} = woody_client:call(Call, Options).
%%%
get_client(C) ->
?config(client, C).
generate(ExternalID, Schema, UserCtx, Client) ->
#bender_GenerationResult{
internal_id = InternalID,
context = PrevUserCtx
} = bender_client:generate_id(ExternalID, Schema, UserCtx, Client),
{InternalID, PrevUserCtx}.
generate_strict(ExternalID, Schema, UserCtx, Client) ->
{InternalID, UserCtx} = generate(ExternalID, Schema, UserCtx, Client),
InternalID.
generate_weak(ExternalID, Schema, UserCtx, Client) ->
case generate(ExternalID, Schema, UserCtx, Client) of
{InternalID, undefined} ->
InternalID;
{InternalID, UserCtx} ->
InternalID
end.
shuffle(L) ->
[ T || {_, T} <- lists:sort([ {rand:uniform(), E} || E <- L ]) ].

1
build_utils Submodule

@ -0,0 +1 @@
Subproject commit 870b70a63af18fc7a02c9ff26b06132d2b1993cb

67
config/sys.config Normal file
View File

@ -0,0 +1,67 @@
[
{bender, [
{service, #{
path => <<"/v1/bender">>
}},
{generator, #{
path => <<"/v1/stateproc/bender_generator">>,
schema => machinery_mg_schema_generic,
url => <<"http://machinegun:8022/v1/automaton">>, % mandatory
event_handler => scoper_woody_event_handler, % mandatory
transport_opts => #{
max_connections => 1000
}
}},
{sequence, #{
path => <<"/v1/stateproc/bender_sequence">>,
schema => machinery_mg_schema_generic,
url => <<"http://machinegun:8022/v1/automaton">>, % mandatory
event_handler => scoper_woody_event_handler, % mandatory
transport_opts => #{
max_connections => 1000
}
}},
{route_opts, #{
% handler_limits => #{}
}},
{ip, "::"},
{port, 8022},
{protocol_opts, #{
timeout => 60000
}},
{transport_opts, #{
handshake_timeout => 5000, % timeout() | infinity, default is 5000
max_connections => 10000, % maximum number of incoming connections, default is 1024
num_acceptors => 100 % size of acceptors pool, default is 10
}},
{health_checkers, [
{erl_health, disk , ["/", 99]},
{erl_health, cg_memory, [99]},
{erl_health, service , [<<"bender">>]}
]}
]},
{lager, [
{error_logger_redirect, true},
{log_root, "/var/log/bender"},
{handlers, [
{lager_console_backend, debug},
{lager_file_backend, [
{file, "log.json"},
{level, debug},
{formatter, lager_logstash_formatter}
]}
]}
]},
{scoper, [
{storage, scoper_storage_lager}
]}
].

6
config/vm.args Normal file
View File

@ -0,0 +1,6 @@
-sname bender
-setcookie bender_cookie
+K true
+A 10

29
docker-compose.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
cat <<EOF
version: '2.1'
services:
${SERVICE_NAME}:
image: ${BUILD_IMAGE}
volumes:
- .:$PWD
- $HOME/.cache:/home/$UNAME/.cache
working_dir: $PWD
command: /sbin/init
depends_on:
machinegun:
condition: service_healthy
machinegun:
image: dr2.rbkmoney.com/rbkmoney/machinegun:f89b294ac584d212264be60cc4db242aeaecce89
command: /opt/machinegun/bin/machinegun foreground
volumes:
- ./test/machinegun/config.yaml:/opt/machinegun/etc/config.yaml
healthcheck:
test: "curl http://localhost:8022/"
interval: 5s
timeout: 1s
retries: 12
EOF

89
elvis.config Normal file
View File

@ -0,0 +1,89 @@
[
{elvis, [
{config, [
#{
dirs => ["apps/*/src"],
filter => "*.erl",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace},
{elvis_style, macro_module_names},
{elvis_style, operator_spaces, #{rules => [{right, ","}, {right, "++"}, {left, "++"}]}},
{elvis_style, nesting_level, #{level => 3}},
{elvis_style, god_modules, #{limit => 25}},
{elvis_style, no_if_expression},
{elvis_style, invalid_dynamic_call, #{ignore => [elvis]}},
{elvis_style, used_ignored_variable},
{elvis_style, no_behavior_info},
{elvis_style, module_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*(_SUITE)?$"}},
{elvis_style, function_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*$"}},
{elvis_style, state_record_and_type},
{elvis_style, no_spec_with_records},
{elvis_style, dont_repeat_yourself, #{min_complexity => 10}},
{elvis_style, no_debug_call, #{ignore => [elvis, elvis_utils]}}
]
},
#{
dirs => ["apps/*/test"],
filter => "*.erl",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace},
{elvis_style, macro_module_names},
{elvis_style, operator_spaces, #{rules => [{right, ","}, {right, "++"}, {left, "++"}]}},
{elvis_style, nesting_level, #{level => 3}},
{elvis_style, god_modules, #{limit => 25}},
{elvis_style, no_if_expression},
{elvis_style, invalid_dynamic_call, #{ignore => [elvis]}},
{elvis_style, used_ignored_variable},
{elvis_style, no_behavior_info},
{elvis_style, module_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*(_SUITE)?$"}},
{elvis_style, function_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*$"}},
{elvis_style, state_record_and_type},
{elvis_style, no_spec_with_records},
{elvis_style, dont_repeat_yourself, #{min_complexity => 10}},
{elvis_style, no_debug_call, #{ignore => [elvis, elvis_utils]}}
]
},
#{
dirs => ["."],
filter => "Makefile",
ruleset => makefiles
},
#{
dirs => ["."],
filter => "elvis.config",
ruleset => elvis_config
},
#{
dirs => ["."],
filter => "rebar.config",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace}
]
},
#{
dirs => ["."],
filter => "rebar.config",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace}
]
},
#{
dirs => ["src"],
filter => "*.app.src",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace}
]
}
]}
]}
].

142
rebar.config Normal file
View File

@ -0,0 +1,142 @@
%% Common project erlang options.
{erl_opts, [
% mandatory
debug_info,
warnings_as_errors,
warn_export_all,
warn_missing_spec,
warn_untyped_record,
warn_export_vars,
% by default
warn_unused_record,
warn_bif_clash,
warn_obsolete_guard,
warn_unused_vars,
warn_shadow_vars,
warn_unused_import,
warn_unused_function,
warn_deprecated_function
% at will
% bin_opt_info
% no_auto_import
% warn_missing_spec_all
]}.
%% Common project dependencies.
{deps, [
{bender_proto,
{git, "git@github.com:rbkmoney/bender-proto.git",
{branch, "master"}}
},
{erl_health,
{git, "https://github.com/rbkmoney/erlang-health.git",
{branch, "master"}}
},
{genlib ,
{git, "https://github.com/rbkmoney/genlib.git",
{branch, "master"}}
},
{lager,
"3.6.4"
},
{lager_logstash_formatter,
{git, "git@github.com:rbkmoney/lager_logstash_formatter.git",
{branch, master}}
},
{machinery,
{git, "git@github.com:rbkmoney/machinery.git",
{branch, "master"}}
},
{scoper,
{git, "git@github.com:rbkmoney/scoper.git",
{branch, "master"}}
},
{snowflake,
{git, "https://github.com/rbkmoney/snowflake.git",
{branch, "master"}}
},
{woody,
{git, "git@github.com:rbkmoney/woody_erlang.git",
{branch, "master"}}
}
]}.
{plugins, [
{rebar3_thrift_compiler,
{git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {branch, "master"}}}
]}.
%% XRef checks
{xref_checks, [
undefined_function_calls,
undefined_functions,
deprecated_functions_calls,
deprecated_functions
]}.
% at will
{xref_warnings, true}.
%% Tests
{cover_enabled, true}.
%% Relx configuration
{relx, [
{release, {bender, "1.0.0"}, [
{lager_logstash_formatter, load}, % log formatter
bender
]},
{sys_config, "./config/sys.config"},
{vm_args, "./config/vm.args"},
{dev_mode, true},
{include_erts, false},
{extended_start_script, true}
]}.
%% Dialyzer static analyzing
{dialyzer, [
{warnings, [
% mandatory
unmatched_returns,
error_handling,
race_conditions,
unknown
% hardcore mode
% overspecs,
% underspecs
]},
{plt_apps, all_deps}
]}.
{profiles, [
{test, [
{deps, [
]}
]},
{prod, [
{deps, [
% for introspection on production
{recon, "2.3.2"}
]},
{relx, [
{release, {bender, "1.0.0"}, [
{recon, load}, % tools for introspection
{runtime_tools, load}, % debugger
{tools, load}, % profiler
{lager_logstash_formatter, load}, % log formatter
sasl,
bender
]},
{dev_mode, false},
{include_erts, true}
]}
]}
]}.
{plugins, [
rebar3_run
]}.

86
rebar.lock Normal file
View File

@ -0,0 +1,86 @@
{"1.1.0",
[{<<"bender_proto">>,
{git,"git@github.com:rbkmoney/bender-proto.git",
{ref,"b7f00305a365c4d0efdc1451082b5e9ec1e631b9"}},
0},
{<<"cache">>,{pkg,<<"cache">>,<<"2.2.0">>},1},
{<<"certifi">>,{pkg,<<"certifi">>,<<"2.4.2">>},2},
{<<"cg_mon">>,
{git,"https://github.com/rbkmoney/cg_mon.git",
{ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}},
1},
{<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.5.0">>},1},
{<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.6.0">>},2},
{<<"erl_health">>,
{git,"https://github.com/rbkmoney/erlang-health.git",
{ref,"2575c7b63d82a92de54d2d27e504413675e64811"}},
0},
{<<"genlib">>,
{git,"https://github.com/rbkmoney/genlib.git",
{ref,"41920d7774d119c294f3aaba4043ced12da2a815"}},
0},
{<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1},
{<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1},
{<<"hackney">>,{pkg,<<"hackney">>,<<"1.15.0">>},1},
{<<"idna">>,{pkg,<<"idna">>,<<"6.0.0">>},2},
{<<"jsx">>,{pkg,<<"jsx">>,<<"2.8.0">>},1},
{<<"lager">>,{pkg,<<"lager">>,<<"3.6.4">>},0},
{<<"lager_logstash_formatter">>,
{git,"git@github.com:rbkmoney/lager_logstash_formatter.git",
{ref,"24527c15c47749866f2d427b333fa1333a46b8af"}},
0},
{<<"machinery">>,
{git,"git@github.com:rbkmoney/machinery.git",
{ref,"81b0cb3ffee1f6f08c5d751fdf748d29b1c3a700"}},
0},
{<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2},
{<<"mg_proto">>,
{git,"git@github.com:rbkmoney/machinegun_proto.git",
{ref,"5c07c579014f9900357f7a72f9d10a03008b9da1"}},
1},
{<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.0.2">>},2},
{<<"msgpack_proto">>,
{git,"git@github.com:rbkmoney/msgpack-proto.git",
{ref,"946343842ee740a19701df087edd1f1641eff769"}},
1},
{<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3},
{<<"ranch">>,{pkg,<<"ranch">>,<<"1.6.2">>},2},
{<<"rfc3339">>,{pkg,<<"rfc3339">>,<<"0.2.2">>},1},
{<<"scoper">>,
{git,"git@github.com:rbkmoney/scoper.git",
{ref,"206f76e006207f75828c1df3dde0deaa8554f332"}},
0},
{<<"snowflake">>,
{git,"https://github.com/rbkmoney/snowflake.git",
{ref,"0a598108f6582affe3b4ae550fc5b9f2062e318a"}},
0},
{<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.4">>},2},
{<<"thrift">>,
{git,"https://github.com/rbkmoney/thrift_erlang.git",
{ref,"7843146f22a9d9d63be4ae1276b5fa03938f2e9c"}},
1},
{<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.4.1">>},3},
{<<"woody">>,
{git,"git@github.com:rbkmoney/woody_erlang.git",
{ref,"862358ee62a95bf46926be1815f70a72a8679d28"}},
0}]}.
[
{pkg_hash,[
{<<"cache">>, <<"3C11DBF4CD8FCD5787C95A5FB2A04038E3729CFCA0386016EEA8C953AB48A5AB">>},
{<<"certifi">>, <<"75424FF0F3BAACCFD34B1214184B6EF616D89E420B258BB0A5EA7D7BC628F7F0">>},
{<<"cowboy">>, <<"4EF3AE066EE10FE01EA3272EDC8F024347A0D3EB95F6FBB9AED556DACBFC1337">>},
{<<"cowlib">>, <<"8AA629F81A0FC189F261DC98A42243FA842625FEEA3C7EC56C48F4CCDB55490F">>},
{<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>},
{<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>},
{<<"hackney">>, <<"287A5D2304D516F63E56C469511C42B016423BCB167E61B611F6BAD47E3CA60E">>},
{<<"idna">>, <<"689C46CBCDF3524C44D5F3DDE8001F364CD7608A99556D8FBD8239A5798D4C10">>},
{<<"jsx">>, <<"749BEC6D205C694AE1786D62CEA6CC45A390437E24835FD16D12D74F07097727">>},
{<<"lager">>, <<"CED6E98070FB4E58EE93174D006D46479C79844DF7FC17FA4FEFC1049A320D88">>},
{<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>},
{<<"mimerl">>, <<"993F9B0E084083405ED8252B99460C4F0563E41729AB42D9074FD5E52439BE88">>},
{<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>},
{<<"ranch">>, <<"6DB93C78F411EE033DBB18BA8234C5574883ACB9A75AF0FB90A9B82EA46AFA00">>},
{<<"rfc3339">>, <<"1552DF616ACA368D982E9F085A0E933B6688A3F4938A671798978EC2C0C58730">>},
{<<"ssl_verify_fun">>, <<"F0EAFFF810D2041E93F915EF59899C923F4568F4585904D010387ED74988E77B">>},
{<<"unicode_util_compat">>, <<"D869E4C68901DD9531385BB0C8C40444EBF624E60B6962D95952775CAC5E90CD">>}]}
].

View File

@ -0,0 +1,16 @@
service_name: machinegun
snowflake_machine_id: 1
namespaces:
bender_generator:
processor:
url: http://bender:8022/v1/stateproc/bender_generator
pool_size: 500
bender_sequence:
processor:
url: http://bender:8022/v1/stateproc/bender_sequence
pool_size: 500
storage:
type: memory