[HG-27] new concept (#2)

* new concept

* add tests

* refactor things

* Add dmt_storage_mgun

* update damsel

* remove dmt_server

* update damsel

* Migrate to jenkins and make tests api-involved

* add dmt_poller

* drop wercker

* pr fixes

* remove obsolete case

* drop woody dependencie for dmt

* Revert "drop woody dependencie for dmt"

This reverts commit 99db9bc77904ab77237cd9356cf04c82b6e77bd1.
This commit is contained in:
Igor Savchuk 2016-08-24 14:27:31 +03:00 committed by GitHub
parent 11cb1934ca
commit ba644bcce2
37 changed files with 1395 additions and 5 deletions

15
.gitignore vendored Normal file
View File

@ -0,0 +1,15 @@
# general
log
/_build/
*~
erl_crash.dump
.tags*
*.sublime-workspace
.DS_Store
# wercker
/_builds/
/_cache/
/_projects/
/_steps/
/_temp/

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "apps/dmt_proto/damsel"]
path = apps/dmt_proto/damsel
url = git@github.com:rbkmoney/damsel.git

6
Dockerfile Normal file
View File

@ -0,0 +1,6 @@
FROM rbkmoney/service_erlang:latest
MAINTAINER Igor Savchuk <i.savchuk@rbkmoney.com>
COPY _build/prod/rel/dmt /opt/dominant
CMD ["/opt/dominant/bin/dmt", "foreground"]
LABEL service_version="semver"
WORKDIR /opt/dominant

34
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,34 @@
#!groovy
// Args:
// GitHub repo name
// Jenkins agent label
// Tracing artifacts to be stored alongside build logs
// Optional: artifacts to cache between the builds
pipeline("dmt", 'docker-host', "_build/") {
// ToDo: Uncomment the stage as soon as Elvis is in the build image!
// runStage('lint') {
// sh 'make w_container_lint'
// }
runStage('compile') {
sh 'make w_container_compile'
}
runStage('xref') {
sh 'make w_container_xref'
}
runStage('test_api') {
sh "make w_container_test_api"
}
runStage('test_client') {
sh "make w_container_test_client"
}
runStage('dialyze') {
sh 'make w_container_dialyze'
}
}

83
Makefile Normal file
View File

@ -0,0 +1,83 @@
REBAR := $(shell which rebar3 2>/dev/null || which ./rebar3)
SUBMODULES = apps/dmt_proto/damsel
SUBTARGETS = $(patsubst %,%/.git,$(SUBMODULES))
ORG_NAME := rbkmoney
BASE_IMAGE := "$(ORG_NAME)/build:latest"
RELNAME := dominant
TAG = latest
IMAGE_NAME = "$(ORG_NAME)/$(RELNAME):$(TAG)"
CALL_ANYWHERE := submodules rebar-update compile xref lint dialyze start devrel release clean distclean
CALL_W_CONTAINER := $(CALL_ANYWHERE) test_api
include utils.mk
.PHONY: $(CALL_W_CONTAINER) all containerize push $(UTIL_TARGETS)
# CALL_ANYWHERE
$(SUBTARGETS): %/.git: %
git submodule update --init $<
touch $@
submodules: $(SUBTARGETS)
rebar-update:
$(REBAR) update
compile: submodules rebar-update
$(REBAR) compile
xref: submodules
$(REBAR) xref
lint: compile
elvis rock
dialyze:
$(REBAR) dialyzer
start: submodules
$(REBAR) run
devrel: submodules
$(REBAR) release
release: distclean
$(REBAR) as prod release
clean:
$(REBAR) clean
distclean:
$(REBAR) clean -a
rm -rfv _build _builds _cache _steps _temp
# CALL_W_CONTAINER
test_api: submodules
$(REBAR) ct --suite apps/dmt_api/test/dmt_api_tests_SUITE.erl
test_client: submodules
$(REBAR) ct --suite apps/dmt_api/test/dmt_client_tests_SUITE.erl
w_container_test_client: submodules
{ \
$(DOCKER_COMPOSE) up -d ; \
$(DOCKER_COMPOSE) exec -T -d dominant make start ; \
$(DOCKER_COMPOSE) exec -T dmt_client make test_client ; \
res=$$? ; \
$(DOCKER_COMPOSE) down ; \
exit $$res ; \
}
# OTHER
all: compile
containerize: w_container_release
$(DOCKER) build --force-rm --tag $(IMAGE_NAME) .
push: containerize
$(DOCKER) push "$(IMAGE_NAME)"

View File

@ -14,17 +14,12 @@
> _Хозяйке на заметку._ При этом используется стандартный Erlang релиз, собранный при помощи [relx][3] в режиме разработчика. > _Хозяйке на заметку._ При этом используется стандартный Erlang релиз, собранный при помощи [relx][3] в режиме разработчика.
Рекомендуется вести разработку и сборку проекта в рамках локальной виртуальной среды, предоставляемой [wercker][1]. Настоятельно рекомендуется прогоны тестовых сценариев проводить только в этой среде.
$ wercker dev
> _Хозяйке на заметку._ В зависимости от вашего окружения и операционной системы вам может понадобиться [Docker Machine][4]. > _Хозяйке на заметку._ В зависимости от вашего окружения и операционной системы вам может понадобиться [Docker Machine][4].
## Документация ## Документация
Дальнейшую документацию можно почерпнуть, пройдясь по ссылкам в [соответствующем документе](doc/index.md). Дальнейшую документацию можно почерпнуть, пройдясь по ссылкам в [соответствующем документе](doc/index.md).
[1]: http://devcenter.wercker.com/learn/basics/the-wercker-cli.html
[2]: http://erlang.org/doc/man/shell.html [2]: http://erlang.org/doc/man/shell.html
[3]: https://github.com/erlware/relx [3]: https://github.com/erlware/relx
[4]: https://docs.docker.com/machine/install-machine/ [4]: https://docs.docker.com/machine/install-machine/

View File

@ -0,0 +1 @@
-define(MG_TAG, <<"dmt_storage">>).

23
apps/dmt/src/dmt.app.src Normal file
View File

@ -0,0 +1,23 @@
{application, dmt, [
{description, "Domain config service"},
{vsn, "0"},
{registered, []},
{mod, {dmt, []}},
{applications, [
kernel,
stdlib,
lager,
genlib,
woody,
dmt_proto
]},
{env, [
{mgun_automaton_url, "http://machinegun:8022/v1/automaton_service"}
]},
{modules, []},
{maintainers, [
"Andrey Mayorov <a.mayorov@rbkmoney.com>"
]},
{licenses, []},
{links, ["https://github.com/rbkmoney/dominant"]}
]}.

110
apps/dmt/src/dmt.erl Normal file
View File

@ -0,0 +1,110 @@
%%% @doc Public API, supervisor and application startup.
%%% @end
-module(dmt).
-behaviour(supervisor).
-behaviour(application).
%% API
-export([checkout/1]).
-export([checkout_object/2]).
-export([pull/1]).
-export([commit/2]).
-export([validate_commit/3]).
%% Supervisor callbacks
-export([init/1]).
%% Application callbacks
-export([start/2]).
-export([stop/1]).
%% Type shortcuts
-export_type([version/0]).
-export_type([head/0]).
-export_type([ref/0]).
-export_type([snapshot/0]).
-export_type([commit/0]).
-export_type([operation/0]).
-export_type([history/0]).
-export_type([object_ref/0]).
-export_type([domain/0]).
-export_type([domain_object/0]).
-type version() :: dmt_domain_config_thrift:'Version'().
-type head() :: dmt_domain_config_thrift:'Head'().
-type ref() :: dmt_domain_config_thrift:'Reference'().
-type snapshot() :: dmt_domain_config_thrift:'Snapshot'().
-type commit() :: dmt_domain_config_thrift:'Commit'().
-type operation() :: dmt_domain_config_thrift:'Operation'().
-type history() :: dmt_domain_config_thrift:'History'().
-type object_ref() :: dmt_domain_thrift:'Reference'().
-type domain() :: dmt_domain_thrift:'Domain'().
-type domain_object() :: dmt_domain_thrift:'DomainObject'().
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
%% API
-spec checkout(ref()) -> snapshot().
checkout(Reference) ->
dmt_cache:checkout(Reference).
-spec checkout_object(ref(), object_ref()) ->
dmt_domain_config_thrift:'VersionedObject'().
checkout_object(Reference, ObjectReference) ->
#'Snapshot'{version = Version, domain = Domain} = checkout(Reference),
Object = dmt_domain:get_object(ObjectReference, Domain),
#'VersionedObject'{version = Version, object = Object}.
-spec pull(version()) -> history().
pull(Version) ->
dmt_mg:get_history(Version).
-spec commit(version(), commit()) -> version().
commit(Version, Commit) ->
ok = dmt_mg:commit(Version, Commit),
Version + 1.
-spec validate_commit(version(), commit(), history()) -> ok.
validate_commit(Version, _Commit, History) ->
%%TODO: actually validate commit
LastVersion = case map_size(History) of
0 ->
0;
_Size ->
lists:max(maps:keys(History))
end,
case Version =:= LastVersion of
true ->
ok;
false ->
throw(bad_version)
end.
%% Supervisor callbacks
-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}.
init([]) ->
Cache = #{id => dmt_cache, start => {dmt_cache, start_link, []}, restart => permanent},
Poller = #{id => dmt_poller, start => {dmt_poller, start_link, []}, restart => permanent},
Children = case application:get_env(dmt, slave_mode) of
{ok, true} ->
[Cache, Poller];
_ ->
[Cache]
end,
{ok, {#{strategy => rest_for_one, intensity => 10, period => 60}, Children}}.
%% Application callbacks
-spec start(normal, any()) -> {ok, pid()} | {error, any()}.
start(_StartType, _StartArgs) ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-spec stop(any()) -> ok.
stop(_State) ->
ok.

148
apps/dmt/src/dmt_cache.erl Normal file
View File

@ -0,0 +1,148 @@
-module(dmt_cache).
-behaviour(gen_server).
%%
-export([start_link/0]).
-export([checkout_head/0]).
-export([checkout/1]).
-export([cache/1]).
-export([cache_snapshot/1]).
-export([commit/1]).
%%
-export([init/1]).
-export([handle_call/3]).
-export([handle_cast/2]).
-export([handle_info/2]).
-export([terminate/2]).
-export([code_change/3]).
-define(TABLE, ?MODULE).
-define(SERVER, ?MODULE).
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
%%
-spec start_link() -> {ok, pid()} | {error, term()}. % FIXME
start_link() ->
gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
-spec checkout_head() -> dmt:snapshot().
checkout_head() ->
checkout({head, #'Head'{}}).
-spec checkout(dmt:ref()) -> dmt:snapshot().
checkout({head, #'Head'{}}) ->
checkout({version, head()});
checkout({version, Version}) ->
case ets:lookup(?TABLE, Version) of
[Snapshot] ->
Snapshot;
[] ->
gen_server:call(?SERVER, {cache, Version})
end.
-spec cache(dmt:version()) -> dmt:snapshot().
cache(Version) ->
gen_server:call(?SERVER, {cache, Version}).
-spec cache_snapshot(dmt:snapshot()) -> ok.
cache_snapshot(Snapshot) ->
gen_server:call(?SERVER, {cache_snapshot, Snapshot}).
-spec commit(dmt:commit()) -> ok.
commit(Commit) ->
gen_server:call(?SERVER, {commit, Commit}).
%%
-record(state, {
}).
-type state() :: #state{}.
-spec init(_) -> {ok, state()}.
init(_) ->
EtsOpts = [
named_table,
ordered_set,
protected,
{read_concurrency, true},
{keypos, #'Snapshot'.version}
],
?TABLE = ets:new(?TABLE, EtsOpts),
{ok, #state{}}.
-spec handle_call(term(), {pid(), term()}, state()) -> {reply, term(), state()}.
handle_call({cache, Version}, _From, State) ->
Closest = closest_snapshot(Version),
Snapshot = dmt_history:travel(Version, dmt_mg:get_history(), Closest),
true = ets:insert(?TABLE, Snapshot),
{reply, Snapshot, State};
handle_call({cache_snapshot, Snapshot}, _From, State) ->
true = ets:insert(?TABLE, Snapshot),
{reply, ok, State};
handle_call({commit, #'Commit'{ops = Ops}}, _From, State) ->
#'Snapshot'{version = Version, domain = Domain} = checkout({head, #'Head'{}}),
NewSnapshot = #'Snapshot'{
version = Version + 1,
domain = dmt_domain:apply_operations(Ops, Domain)
},
true = ets:insert(?TABLE, NewSnapshot),
{reply, NewSnapshot, State};
handle_call(_Msg, _From, State) ->
{noreply, State}.
-spec handle_cast(term(), state()) -> {noreply, state()}.
handle_cast(_Msg, State) ->
{noreply, State}.
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info(_Msg, State) ->
{noreply, State}.
-spec terminate(term(), state()) -> ok.
terminate(_Reason, _State) ->
ok.
-spec code_change(term(), state(), term()) -> {error, noimpl}.
code_change(_OldVsn, _State, _Extra) ->
{error, noimpl}.
%% internal
-spec head() -> dmt:version().
head() ->
case ets:last(?TABLE) of
'$end_of_table' ->
% #'Snapshot'{version = Version} = dmt_history:head(dmt_mg:get_history()),
% Version;
0;
Version ->
Version
end.
-spec closest_snapshot(dmt:version()) -> dmt:snapshot().
closest_snapshot(Version) ->
CachedVersions = ets:select(?TABLE, ets:fun2ms(fun (#'Snapshot'{version = V}) -> V end)),
Closest = lists:foldl(fun (V, Acc) ->
case abs(V - Version) =< abs(Acc - Version) of
true ->
V;
false ->
Acc
end
end, 0, CachedVersions),
case Closest of
0 ->
#'Snapshot'{version = 0, domain = dmt_domain:new()};
Closest ->
ets:lookup(?TABLE, Closest)
end.

View File

@ -0,0 +1,85 @@
-module(dmt_domain).
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
%%
-export([new/0]).
-export([get_object/2]).
-export([apply_operations/2]).
-export([revert_operations/2]).
%%
-spec new() ->
dmt:domain().
new() ->
#{}.
-spec get_object(dmt:object_ref(), dmt:domain()) ->
dmt:domain_object().
get_object(ObjectReference, Domain) ->
case maps:find(ObjectReference, Domain) of
{ok, Object} ->
Object;
error ->
throw(object_not_found)
end.
-spec apply_operations([dmt:operation()], dmt:domain()) -> dmt:domain().
apply_operations([], Domain) ->
Domain;
apply_operations([{insert, #'InsertOp'{object = Object}} | Rest], Domain) ->
apply_operations(Rest, insert(Object, Domain));
apply_operations([{update, #'UpdateOp'{old_object = OldObject, new_object = NewObject}} | Rest], Domain) ->
apply_operations(Rest, update(OldObject, NewObject, Domain));
apply_operations([{remove, #'RemoveOp'{object = Object}} | Rest], Domain) ->
apply_operations(Rest, delete(Object, Domain)).
-spec revert_operations([dmt:operation()], dmt:domain()) -> dmt:domain().
revert_operations([], Domain) ->
Domain;
revert_operations([{insert, #'InsertOp'{object = Object}} | Rest], Domain) ->
revert_operations(Rest, delete(Object, Domain));
revert_operations([{update, #'UpdateOp'{old_object = OldObject, new_object = NewObject}} | Rest], Domain) ->
revert_operations(Rest, update(NewObject, OldObject, Domain));
revert_operations([{remove, #'RemoveOp'{object = Object}} | Rest], Domain) ->
revert_operations(Rest, insert(Object, Domain)).
-spec insert(dmt:domain_object(), dmt:domain()) -> dmt:domain().
insert(Object, Domain) ->
ObjectReference = get_ref(Object),
case maps:is_key(ObjectReference, Domain) of
false ->
maps:put(ObjectReference, Object, Domain);
true ->
throw(object_already_exists)
end.
-spec update(dmt:domain_object(), dmt:domain_object(), dmt:domain()) -> dmt:domain().
update(OldObject, NewObject, Domain) ->
ObjectReference = get_ref(OldObject),
ObjectReference = get_ref(NewObject),
case maps:find(ObjectReference, Domain) of
{ok, OldObject} ->
maps:put(ObjectReference, NewObject, Domain);
error ->
throw(object_not_found)
end.
-spec delete(dmt:domain_object(), dmt:domain()) -> dmt:domain().
delete(Object, Domain) ->
ObjectReference = get_ref(Object),
case maps:find(ObjectReference, Domain) of
{ok, Object} ->
maps:remove(ObjectReference, Domain);
error ->
throw(object_not_found)
end.
%%TODO:elaborate
-spec get_ref(dmt:domain_object()) -> dmt:object_ref().
get_ref({Tag, {_Type, Ref, _Data}}) ->
{Tag, Ref}.

View File

@ -0,0 +1,39 @@
-module(dmt_history).
-export([head/1]).
-export([head/2]).
-export([travel/3]).
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
-spec head(dmt:history()) -> dmt:snapshot().
head(History) when map_size(History) =:= 0 ->
#'Snapshot'{version = 0, domain = dmt_domain:new()};
head(History) ->
head(History, #'Snapshot'{version = 0, domain = dmt_domain:new()}).
-spec head(dmt:history(), dmt:snapshot()) -> dmt:snapshot().
head(History, Snapshot) ->
Head = lists:max(maps:keys(History)),
travel(Head, History, Snapshot).
-spec travel(dmt:version(), dmt:history(), dmt:snapshot()) -> dmt:snapshot().
travel(To, _History, #'Snapshot'{version = From} = Snapshot)
when To =:= From ->
Snapshot;
travel(To, History, #'Snapshot'{version = From, domain = Domain})
when To > From ->
#'Commit'{ops = Ops} = maps:get(From + 1, History),
NextSnapshot = #'Snapshot'{
version = From + 1,
domain = dmt_domain:apply_operations(Ops, Domain)
},
travel(To, History, NextSnapshot);
travel(To, History, #'Snapshot'{version = From, domain = Domain})
when To < From ->
#'Commit'{ops = Ops} = maps:get(From, History),
PreviousSnapshot = #'Snapshot'{
version = From - 1,
domain = dmt_domain:revert_operations(Ops, Domain)
},
travel(To, History, PreviousSnapshot).

62
apps/dmt/src/dmt_mg.erl Normal file
View File

@ -0,0 +1,62 @@
-module(dmt_mg).
-export([call/2]).
-export([start/0]).
-export([get_commit/1]).
-export([get_history/0]).
-export([get_history/1]).
-export([commit/2]).
-export([read_history/1]).
-include_lib("dmt_proto/include/dmt_state_processing_thrift.hrl").
-include_lib("dmt/include/dmt_mg.hrl").
-spec call(atom(), list(term())) ->
{ok, term()} | ok | no_return().
call(Method, Args) ->
Request = {{dmt_state_processing_thrift, 'Automaton'}, Method, Args},
Context = woody_client:new_context(
woody_client:make_id(<<"dmt">>),
dmt_api_woody_event_handler
),
{ok, MgunAutomatonUrl} = application:get_env(dmt, mgun_automaton_url),
woody_client:call(Context, Request, #{url => MgunAutomatonUrl}).
-spec start() -> ok.
start() ->
{{ok, {_, _}}, _} = call(start, {'Args', <<>>}),
ok.
-spec get_commit(dmt:version()) -> dmt:commit().
get_commit(Id) ->
#{Id := Commit} = get_history(),
Commit.
%% TODO: add range requests after they are fixed in mg
-spec get_history() -> dmt:history().
get_history() ->
get_history(undefined).
-spec get_history(dmt:version() | undefined) -> dmt:history().
get_history(After) ->
{{ok, History}, _Context} = call(getHistory, [{tag, ?MG_TAG}, #'HistoryRange'{'after' = After}]),
read_history(History).
-spec commit(dmt:version(), dmt:commit()) -> ok.
commit(Version, Commit) ->
Call = <<"commit", (term_to_binary({Version, Commit}))/binary>>,
{{ok, <<"ok">>}, _Context} = call(call, [{tag, ?MG_TAG}, Call]),
ok.
%% utils
-spec read_history([dmt_state_processing_thrift:'Event'()]) -> dmt:history().
read_history(Events) ->
read_history(Events, #{}).
-spec read_history([dmt_state_processing_thrift:'Event'()], dmt:history()) ->
dmt:history().
read_history([], History) ->
History;
read_history([#'Event'{id = Id, event_payload = BinaryPayload} | Rest], History) ->
read_history(Rest, History#{Id => binary_to_term(BinaryPayload)}).

View File

@ -0,0 +1,74 @@
-module(dmt_poller).
-behaviour(gen_server).
-export([start_link/0]).
-export([poll/0]).
-export([init/1]).
-export([handle_call/3]).
-export([handle_cast/2]).
-export([handle_info/2]).
-export([terminate/2]).
-export([code_change/3]).
-define(SERVER, ?MODULE).
-define(INTERVAL, 5000).
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
-spec start_link() -> {ok, pid()} | {error, term()}.
start_link() ->
gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
-spec poll() -> ok.
poll() ->
gen_server:call(?SERVER, poll).
-record(state, {
timer :: reference(),
last_version = 0 :: dmt:version()
}).
-type state() :: #state{}.
-spec init(_) -> {ok, state()}.
init(_) ->
Timer = erlang:send_after(?INTERVAL, self(), poll),
{ok, #state{timer = Timer}}.
-spec handle_call(term(), {pid(), term()}, state()) -> {reply, term(), state()}.
handle_call(poll, _From, #state{last_version = LastVersion, timer = Timer} = State) ->
_ = erlang:cancel_timer(Timer),
NewLastVersion = pull(LastVersion),
NewTimer = erlang:send_after(?INTERVAL, self(), poll),
{reply, ok, State#state{timer = NewTimer, last_version = NewLastVersion}}.
-spec handle_cast(term(), state()) -> {noreply, state()}.
handle_cast(_Msg, State) ->
{noreply, State}.
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info(timer, #state{last_version = LastVersion} = State) ->
NewLastVersion = pull(LastVersion),
Timer = erlang:send_after(?INTERVAL, self(), poll),
{noreply, State#state{last_version = NewLastVersion, timer = Timer}};
handle_info(_Msg, State) ->
{noreply, State}.
-spec terminate(term(), state()) -> ok.
terminate(_Reason, _State) ->
ok.
-spec code_change(term(), state(), term()) -> {error, noimpl}.
code_change(_OldVsn, _State, _Extra) ->
{error, noimpl}.
%% Internal
-spec pull(dmt:version()) -> dmt:version().
pull(LastVersion) ->
FreshHistory = dmt_api_client:pull(LastVersion),
OldHead = dmt_cache:checkout_head(),
#'Snapshot'{version = NewLastVersion} = NewHead = dmt_history:head(FreshHistory, OldHead),
ok = dmt_cache:cache_snapshot(NewHead),
NewLastVersion.

View File

@ -0,0 +1,3 @@
{erl_opts, [
{parse_transform, lager_transform}
]}.

View File

@ -0,0 +1,14 @@
{application, dmt_api, [
{description, "Domain config service interfaces"},
{vsn, "0"},
{registered, []},
{applications, [
kernel,
stdlib,
lager,
dmt,
woody
]},
{mod, {dmt_api, []}},
{env, []}
]}.

View File

@ -0,0 +1,65 @@
-module(dmt_api).
-behaviour(application).
-behaviour(supervisor).
-export([get_handler_spec/1]).
-export([start/2]).
-export([stop/1]).
-export([init/1]).
%%
-spec start(application:start_type(), term()) -> {ok, pid()} | {error, term()}.
start(_StartType, _Args) ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-spec stop(term()) -> ok.
stop(_State) ->
ok.
%%
-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec(), ...]}}.
init([]) ->
WoodyChildSpec = woody_server:child_spec(
?MODULE,
#{
ip => dmt_api_utils:get_hostname_ip(genlib_app:env(?MODULE, host, "dominant")),
port => genlib_app:env(?MODULE, port, 8800),
net_opts => [],
event_handler => dmt_api_woody_event_handler,
handlers => [
get_handler_spec(repository),
get_handler_spec(repository_client),
get_handler_spec(mg_processor)
]
}
),
{ok, {#{strategy => one_for_one, intensity => 0, period => 1}, [WoodyChildSpec]}}.
-spec get_handler_spec(Which) -> {Path, {woody_t:service(), module(), term()}} when
Which :: repository | repository_client | mg_processor,
Path :: iodata().
get_handler_spec(repository) ->
{"/v1/domain/repository", {
{dmt_domain_config_thrift, 'Repository'},
dmt_api_repository_handler,
[]
}};
get_handler_spec(repository_client) ->
{"/v1/domain/repository_client", {
{dmt_domain_config_thrift, 'RepositoryClient'},
dmt_api_repository_client_handler,
[]
}};
get_handler_spec(mg_processor) ->
{"/v1/domain/mgun_processor", {
{dmt_state_processing_thrift, 'Processor'},
dmt_api_mgun_handler,
[]
}}.

View File

@ -0,0 +1,42 @@
-module(dmt_api_client).
-export([commit/2]).
-export([checkout/1]).
-export([pull/1]).
-export([checkout_object/2]).
-spec commit(dmt:version(), dmt:commit()) -> dmt:version().
commit(Version, Commit) ->
call(repository, 'Commit', [Version, Commit]).
-spec checkout(dmt:ref()) -> dmt:snapshot().
checkout(Reference) ->
call(repository, 'Checkout', [Reference]).
-spec pull(dmt:version()) -> dmt:history().
pull(Version) ->
call(repository, 'Pull', [Version]).
-spec checkout_object(dmt:ref(), dmt:object_ref()) -> dmt:domain_object().
checkout_object(Reference, ObjectReference) ->
call(repository_client, 'checkoutObject', [Reference, ObjectReference]).
call(ServiceName, Function, Args) ->
Host = application:get_env(dmt, client_host, "dominant"),
Port = integer_to_list(application:get_env(dmt, client_port, 8800)),
{Path, {Service, _Handler, _Opts}} = dmt_api:get_handler_spec(ServiceName),
Call = {Service, Function, Args},
Server = #{url => Host ++ ":" ++ Port ++ Path},
Context = woody_client:new_context(woody_client:make_id(<<"dmt_client">>), dmt_api_woody_event_handler),
case woody_client:call_safe(Context, Call, Server) of
{ok, _Context} ->
ok;
{{ok, Response}, _Context} ->
Response;
{{exception, Exception}, _Context} ->
throw(Exception);
{{error, Error}, _Context} ->
error(Error)
end.

View File

@ -0,0 +1,32 @@
-module(dmt_api_mgun_handler).
-behaviour(woody_server_thrift_handler).
-export([handle_function/4]).
%%
-include_lib("dmt_proto/include/dmt_state_processing_thrift.hrl").
-include_lib("dmt/include/dmt_mg.hrl").
-spec handle_function(
woody_t:func(),
woody_server_thrift_handler:args(),
woody_client:context(),
woody_server_thrift_handler:handler_opts()
) -> {ok | {ok, woody_server_thrift_handler:result()}, woody_client:context()} | no_return().
handle_function(processCall, {#'CallArgs'{call = <<"commit", Data/binary>>, history = History}}, Context, _Opts) ->
{Version, Commit} = binary_to_term(Data),
ok = dmt:validate_commit(Version, Commit, dmt_mg:read_history(History)),
_Snapshot = dmt_cache:commit(Commit),
{
{ok, #'CallResult'{
events = [term_to_binary(Commit)],
action = #'ComplexAction'{},
response = <<"ok">>
}},
Context
};
handle_function(processSignal, {#'SignalArgs'{signal = {init, #'InitSignal'{}}}}, Context, _Opts) ->
CA = #'ComplexAction'{tag = #'TagAction'{tag = ?MG_TAG}},
{{ok, #'SignalResult'{events = [], action = CA}}, Context}.

View File

@ -0,0 +1,25 @@
-module(dmt_api_repository_client_handler).
-behaviour(woody_server_thrift_handler).
-export([handle_function/4]).
%%
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
-spec handle_function(
woody_t:func(),
woody_server_thrift_handler:args(),
woody_client:context(),
woody_server_thrift_handler:handler_opts()
) -> {ok | {ok, woody_server_thrift_handler:result()}, woody_client:context()} | no_return().
handle_function('checkoutObject', {Reference, ObjectReference}, Context, _Opts) ->
try
Object = dmt:checkout_object(Reference, ObjectReference),
{{ok, Object}, Context}
catch
object_not_found ->
throw({#'ObjectNotFound'{}, Context});
version_not_found ->
throw({#'VersionNotFound'{}, Context})
end.

View File

@ -0,0 +1,41 @@
-module(dmt_api_repository_handler).
-behaviour(woody_server_thrift_handler).
-export([handle_function/4]).
%%
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
-spec handle_function(
woody_t:func(),
woody_server_thrift_handler:args(),
woody_client:context(),
woody_server_thrift_handler:handler_opts()
) -> {ok | {ok, woody_server_thrift_handler:result()}, woody_client:context()} | no_return().
handle_function('Commit', {Version, Commit}, Context, _Opts) ->
try
NewVersion = dmt:commit(Version, Commit),
{{ok, NewVersion}, Context}
catch
operation_conflict ->
throw({#'OperationConflict'{}, Context});
version_not_found ->
throw({#'VersionNotFound'{}, Context})
end;
handle_function('Checkout', {Reference}, Context, _Opts) ->
try
Snapshot = dmt:checkout(Reference),
{{ok, Snapshot}, Context}
catch
version_not_found ->
throw({#'VersionNotFound'{}, Context})
end;
handle_function('Pull', {Version}, Context, _Opts) ->
try
History = dmt:pull(Version),
{{ok, History}, Context}
catch
version_not_found ->
throw({#'VersionNotFound'{}, Context})
end.

View File

@ -0,0 +1,22 @@
-module(dmt_api_utils).
-export([get_hostname_ip/1]).
%%
-include_lib("kernel/include/inet.hrl").
-spec get_hostname_ip(Hostname | IP) -> IP when
Hostname :: string(),
IP :: inet:ip_address().
get_hostname_ip(IP) when tuple_size(IP) == 4 orelse tuple_size(IP) == 8 ->
IP;
get_hostname_ip(Host) ->
case inet:gethostbyname(Host) of
{ok, #hostent{h_addr_list = [IP | _]}} ->
IP;
{error, Error} ->
exit(Error)
end.

View File

@ -0,0 +1,21 @@
-module(dmt_api_woody_event_handler).
-behaviour(woody_event_handler).
-export([handle_event/3]).
-spec handle_event(EventType, RpcID, EventMeta)
-> _ when
EventType :: woody_event_handler:event_type(),
RpcID :: woody_t:rpc_id(),
EventMeta :: woody_event_handler:event_meta_type().
handle_event(EventType, RpcID, #{status := error, class := Class, reason := Reason, stack := Stack}) ->
lager:error(
maps:to_list(RpcID),
"[server] ~s with ~s:~p at ~s",
[EventType, Class, Reason, genlib_format:format_stacktrace(Stack, [newlines])]
);
handle_event(EventType, RpcID, EventMeta) ->
lager:debug(maps:to_list(RpcID), "[server] ~s: ~p", [EventType, EventMeta]).

View File

@ -0,0 +1,94 @@
-module(dmt_api_tests_SUITE).
-include_lib("common_test/include/ct.hrl").
-export([all/0]).
-export([groups/0]).
-export([init_per_suite/1]).
-export([end_per_suite/1]).
-export([application_stop/1]).
-export([insert/1]).
-export([update/1]).
-export([delete/1]).
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
%%
%% tests descriptions
%%
-spec all() -> [term()].
all() ->
[
{group, basic_lifecycle}
].
-spec groups() -> [term()].
groups() ->
[
{basic_lifecycle, [sequence], [
insert,
update,
delete
]}
].
%%
%% starting/stopping
-spec init_per_suite(term()) -> term().
init_per_suite(C) ->
{ok, Apps} = application:ensure_all_started(dmt_api),
ok = dmt_mg:start(),
[{apps, Apps}|C].
-spec end_per_suite(term()) -> term().
end_per_suite(C) ->
[application_stop(App) || App <- proplists:get_value(apps, C)].
-spec application_stop(term()) -> term().
application_stop(App) ->
application:stop(App).
%%
%% tests
-spec insert(term()) -> term().
insert(_C) ->
Object = fixture_domain_object(1, <<"InsertFixture">>),
Ref = fixture_object_ref(1),
#'ObjectNotFound'{} = (catch dmt_api_client:checkout_object({head, #'Head'{}}, Ref)),
#'Snapshot'{version = Version1} = dmt_api_client:checkout({head, #'Head'{}}),
Version2 = dmt_api_client:commit(Version1, #'Commit'{ops = [{insert, #'InsertOp'{object = Object}}]}),
#'VersionedObject'{object = Object} = dmt_api_client:checkout_object({head, #'Head'{}}, Ref),
#'ObjectNotFound'{} = (catch dmt_api_client:checkout_object({version, Version1}, Ref)),
#'VersionedObject'{object = Object} = dmt_api_client:checkout_object({version, Version2}, Ref).
-spec update(term()) -> term().
update(_C) ->
Object1 = fixture_domain_object(2, <<"UpdateFixture1">>),
Object2 = fixture_domain_object(2, <<"UpdateFixture2">>),
Ref = fixture_object_ref(2),
#'Snapshot'{version = Version0} = dmt_api_client:checkout({head, #'Head'{}}),
Version1 = dmt_api_client:commit(Version0, #'Commit'{ops = [{insert, #'InsertOp'{object = Object1}}]}),
Version2 = dmt_api_client:commit(
Version1,
#'Commit'{ops = [{update, #'UpdateOp'{old_object = Object1, new_object = Object2}}]}
),
#'VersionedObject'{object = Object1} = dmt_api_client:checkout_object({version, Version1}, Ref),
#'VersionedObject'{object = Object2} = dmt_api_client:checkout_object({version, Version2}, Ref).
-spec delete(term()) -> term().
delete(_C) ->
Object = fixture_domain_object(3, <<"DeleteFixture">>),
Ref = fixture_object_ref(3),
#'Snapshot'{version = Version0} = dmt_api_client:checkout({head, #'Head'{}}),
Version1 = dmt_api_client:commit(Version0, #'Commit'{ops = [{insert, #'InsertOp'{object = Object}}]}),
Version2 = dmt_api_client:commit(Version1, #'Commit'{ops = [{remove, #'RemoveOp'{object = Object}}]}),
#'VersionedObject'{object = Object} = dmt_api_client:checkout_object({version, Version1}, Ref),
#'ObjectNotFound'{} = (catch dmt_api_client:checkout_object({version, Version2}, Ref)).
fixture_domain_object(Ref, Data) ->
{category, #'CategoryObject'{
ref = #'CategoryRef'{id = Ref},
data = #'Category'{name = Data, description = Data}
}}.
fixture_object_ref(Ref) ->
{category, #'CategoryRef'{id = Ref}}.

View File

@ -0,0 +1,71 @@
-module(dmt_client_tests_SUITE).
-include_lib("common_test/include/ct.hrl").
-export([all/0]).
-export([groups/0]).
-export([init_per_suite/1]).
-export([end_per_suite/1]).
-export([application_stop/1]).
-export([poll/1]).
-include_lib("dmt_proto/include/dmt_domain_config_thrift.hrl").
%%
%% tests descriptions
%%
-spec all() -> [term()].
all() ->
[
{group, basic_lifecycle}
].
-spec groups() -> [term()].
groups() ->
[
{basic_lifecycle, [sequence], [
poll
]}
].
%%
%% starting/stopping
-spec init_per_suite(term()) -> term().
init_per_suite(C) ->
{ok, Apps} = application:ensure_all_started(dmt),
{ok, _PollerPid} = supervisor:start_child(dmt, #
{id => dmt_poller, start => {dmt_poller, start_link, []}, restart => permanent}
),
ok = dmt_mg:start(),
[{apps, Apps}|C].
-spec end_per_suite(term()) -> term().
end_per_suite(C) ->
[application_stop(App) || App <- proplists:get_value(apps, C)].
-spec application_stop(term()) -> term().
application_stop(App) ->
application:stop(App).
%%
%% tests
-spec poll(term()) -> term().
poll(_C) ->
Object = fixture_domain_object(1, <<"InsertFixture">>),
Ref = fixture_object_ref(1),
{'ObjectNotFound'} = (catch dmt_api_client:checkout_object({head, #'Head'{}}, Ref)),
#'Snapshot'{version = Version1} = dmt_api_client:checkout({head, #'Head'{}}),
Version2 = dmt_api_client:commit(Version1, #'Commit'{ops = [{insert, #'InsertOp'{object = Object}}]}),
#'Snapshot'{version = Version1} = dmt:checkout({head, #'Head'{}}),
object_not_found = (catch dmt:checkout_object({head, #'Head'{}}, Ref)),
ok = dmt_poller:poll(),
#'Snapshot'{version = Version2} = dmt:checkout({head, #'Head'{}}),
#'VersionedObject'{object = Object} = dmt:checkout_object({head, #'Head'{}}, Ref).
fixture_domain_object(Ref, Data) ->
{category, #'CategoryObject'{
ref = #'CategoryRef'{id = Ref},
data = #'Category'{name = Data, description = Data}
}}.
fixture_object_ref(Ref) ->
{category, #'CategoryRef'{id = Ref}}.

2
apps/dmt_proto/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
/include/dmt_*_thrift.hrl
/src/dmt_*_thrift.erl

1
apps/dmt_proto/damsel Submodule

@ -0,0 +1 @@
Subproject commit 7eda08c2a54c617d7fad2b14e6c185ba9f3fdc27

View File

@ -0,0 +1,17 @@
{plugins, [
{rebar3_thrift_compiler,
{git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {tag, "0.2"}}}
]}.
{provider_hooks, [
{pre, [
{compile, {thrift, compile}},
{clean, {thrift, clean}}
]}
]}.
{thrift_compiler_opts, [
{in_dir, "damsel/proto"},
{in_files, ["domain_config.thrift", "state_processing.thrift"]},
{gen, "erlang:app_prefix=dmt"}
]}.

View File

@ -0,0 +1,9 @@
{application, dmt_proto, [
{description, "Domain config protocol definitions"},
{vsn, "0"},
{registered, []},
{applications, [
kernel,
stdlib
]}
]}.

4
config/sys.config Normal file
View File

@ -0,0 +1,4 @@
[
{dmt, [
]}
].

6
config/vm.args Normal file
View File

@ -0,0 +1,6 @@
-sname dominant
-setcookie dominant_cookie
+K true
+A 10

5
doc/index.md Normal file
View File

@ -0,0 +1,5 @@
# Документация
1. [Общее описание](overview.md)
1. [Установка](install.md)
1. [Первоначалная настройка](configuration.md)

21
docker-compose.yml Normal file
View File

@ -0,0 +1,21 @@
version: '2'
services:
dominant:
image: rbkmoney/build:latest
volumes:
- .:/code
working_dir: /code
command: /sbin/init
links:
- machinegun
dmt_client:
image: rbkmoney/build:latest
volumes:
- .:/code
working_dir: /code
command: /sbin/init
links:
- dominant
machinegun:
image: rbkmoney/mg:dmt
command: /opt/mgun/bin/mgun foreground

68
elvis.config Normal file
View File

@ -0,0 +1,68 @@
[
{elvis, [
{config, [
#{
dirs => ["apps/*/src"],
filter => "*.erl",
ignore => ["_thrift.erl$"],
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace},
{elvis_style, macro_names},
{elvis_style, macro_module_names},
{elvis_style, operator_spaces, #{rules => [{right, ","}, {right, "++"}, {left, "++"}]}},
{elvis_style, nesting_level, #{level => 3}},
{elvis_style, god_modules, #{limit => 25}},
{elvis_style, no_if_expression},
{elvis_style, invalid_dynamic_call, #{ignore => [elvis]}},
{elvis_style, used_ignored_variable},
{elvis_style, no_behavior_info},
{elvis_style, module_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*(_SUITE)?$"}},
{elvis_style, function_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)*$"}},
{elvis_style, state_record_and_type},
{elvis_style, no_spec_with_records},
{elvis_style, dont_repeat_yourself, #{min_complexity => 10}},
{elvis_style, no_debug_call, #{ignore => [elvis, elvis_utils]}}
]
},
#{
dirs => ["."],
filter => "Makefile",
ruleset => makefiles
},
#{
dirs => ["."],
filter => "elvis.config",
ruleset => elvis_config
},
#{
dirs => ["apps", "apps/*"],
filter => "rebar.config",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace}
]
},
#{
dirs => ["."],
filter => "rebar.config",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace}
]
},
#{
dirs => ["apps/*/src"],
filter => "*.app.src",
rules => [
{elvis_style, line_length, #{limit => 120, skip_comments => false}},
{elvis_style, no_tabs},
{elvis_style, no_trailing_whitespace}
]
}
]}
]}
].

85
rebar.config Normal file
View File

@ -0,0 +1,85 @@
%% Common project erlang options.
{erl_opts, [
% mandatory
debug_info,
warnings_as_errors,
warn_export_all,
warn_missing_spec,
warn_untyped_record,
warn_export_vars,
% by default
warn_unused_record,
warn_bif_clash,
warn_obsolete_guard,
warn_unused_vars,
warn_shadow_vars,
warn_unused_import,
warn_unused_function,
warn_deprecated_function
% at will
% bin_opt_info
% no_auto_import
% warn_missing_spec_all
]}.
%% Common project dependencies.
{deps, [
{genlib , {git, "https://github.com/rbkmoney/genlib.git", {branch, "master"}}},
{woody , {git, "git@github.com:rbkmoney/woody_erlang.git", {branch, "master"}}},
{lager , "3.0.2"}
]}.
%% XRef checks
{xref_checks, [
undefined_function_calls,
undefined_functions,
deprecated_functions_calls,
deprecated_functions
]}.
% at will
% {xref_warnings, true}.
%% Tests
{cover_enabled, true}.
%% Relx configuration
{relx, [
{release, {dominant, "0.1"}, [
dmt_api,
sasl
]},
{sys_config, "./config/sys.config"},
{vm_args, "./config/vm.args"},
{dev_mode, true},
{include_erts, false},
{extended_start_script, true}
]}.
%% Dialyzer static analyzing
{dialyzer, [
{warnings, [
% mandatory
unmatched_returns,
error_handling,
race_conditions,
unknown
]},
{plt_apps, all_deps}
]}.
{profiles, [
{prod, [
{relx, [
{dev_mode, false},
{include_erts, true}
]}
]}
]}.
{plugins, [
rebar3_run
]}.

27
rebar.lock Normal file
View File

@ -0,0 +1,27 @@
[{<<"certifi">>,{pkg,<<"certifi">>,<<"0.4.0">>},2},
{<<"cowboy">>,{pkg,<<"cowboy">>,<<"1.0.4">>},1},
{<<"cowlib">>,{pkg,<<"cowlib">>,<<"1.0.2">>},2},
{<<"genlib">>,
{git,"https://github.com/rbkmoney/genlib.git",
{ref,"66db7fe296465a875b6894eb5ac944c90f82f913"}},
0},
{<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.7">>},1},
{<<"hackney">>,{pkg,<<"hackney">>,<<"1.5.7">>},1},
{<<"idna">>,{pkg,<<"idna">>,<<"1.2.0">>},2},
{<<"lager">>,{pkg,<<"lager">>,<<"3.0.2">>},0},
{<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2},
{<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.0.2">>},2},
{<<"ranch">>,{pkg,<<"ranch">>,<<"1.2.1">>},2},
{<<"snowflake">>,
{git,"https://github.com/tel/snowflake.git",
{ref,"7a8eab0f12757133623b2151a7913b6d2707b629"}},
1},
{<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.0">>},2},
{<<"thrift">>,
{git,"https://github.com/rbkmoney/thrift_erlang.git",
{ref,"1978d5a1e350694bfa3d2dc3a762176bc60c6e64"}},
1},
{<<"woody">>,
{git,"git@github.com:rbkmoney/woody_erlang.git",
{ref,"dbe03ca171087f2649a7057a379cdb10c5dce90c"}},
0}].

37
utils.mk Normal file
View File

@ -0,0 +1,37 @@
SHELL := /bin/bash
which = $(if $(shell which $(1) 2>/dev/null),\
$(shell which $(1) 2>/dev/null),\
$(error "Error: could not locate $(1)!"))
DOCKER = $(call which,docker)
DOCKER_COMPOSE = $(call which,docker-compose)
UTIL_TARGETS := to_dev_container w_container_% run_w_container_% check_w_container_%
ifndef RELNAME
$(error RELNAME is not set)
endif
ifndef CALL_W_CONTAINER
$(error CALL_W_CONTAINER is not set)
endif
to_dev_container:
$(DOCKER) run -it --rm -v $$PWD:$$PWD --workdir $$PWD $(BASE_IMAGE) /bin/bash
w_container_%:
$(MAKE) -s run_w_container_$*
run_w_container_%: check_w_container_%
{ \
$(DOCKER_COMPOSE) up -d ; \
$(DOCKER_COMPOSE) exec -T $(RELNAME) make $* ; \
res=$$? ; \
$(DOCKER_COMPOSE) down ; \
exit $$res ; \
}
check_w_container_%:
$(if $(filter $*,$(CALL_W_CONTAINER)),,\
$(error "Error: target '$*' cannot be called w_container_"))