valitydev/dmt-client
14
0
Fork 0
mirror of https://github.com/valitydev/dmt-client.git synced 2024-11-06 01:15:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

DC-35: added cache and bumped some dependencies (#14)

Browse Source
This commit is contained in:
Evgeny Levenets 2017-05-12 12:29:13 +03:00 committed by GitHub
parent 601f7bfb22
commit d2be019477
12 changed files with 280 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile vendored
View File

@ -35,7 +35,7 @@ build('dmt_client', 'docker-host', finalHook) {
sh 'make wc_xref'
}
runStage('dialyze') {
withWsCache("_build/default/rebar3_18.3_plt") {
withWsCache("_build/default/rebar3_19.1_plt") {
sh 'make wc_dialyze'
}
}

2
Makefile
View File

@ -9,7 +9,7 @@ TEMPLATES_PATH := .
SERVICE_NAME := dmt_client
# Build image tag to be used
BUILD_IMAGE_TAG := 80c38dc638c0879687f6661f4e16e8de9fc0d2c6
BUILD_IMAGE_TAG := 3750c129119b83ea399dc4aa0ed923fb0e3bf0f0
CALL_ANYWHERE := all submodules rebar-update compile xref lint dialyze clean distclean
CALL_W_CONTAINER := $(CALL_ANYWHERE) test

9
config/sys.config Normal file
View File

@ -0,0 +1,9 @@
[
{dmt_client, [
{cache_update_interval, 5000}, % milliseconds
{service_urls, #{
'Repository' => <<"dominant:8022/v1/domain/repository">>,
'RepositoryClient' => <<"dominant:8022/v1/domain/repository_client">>
}}
]}
].

4
docker-compose.sh
View File

@ -12,12 +12,12 @@ services:
links:
- dominant
dominant:
image: dr.rbkmoney.com/rbkmoney/dominant:b79f1e6acf5fd07ac60a51f9551faca48115770f
image: dr.rbkmoney.com/rbkmoney/dominant:e6af73a005779d5714a1d3b9e310a12f69f6fb0c
command: /opt/dominant/bin/dominant foreground
links:
- machinegun
machinegun:
image: dr.rbkmoney.com/rbkmoney/machinegun:2c956c1172cf8f7b4a09512cd1571bdd4c57f1c1
image: dr.rbkmoney.com/rbkmoney/machinegun:e04e529f4c5682b527d12d73a13a3cf9eb296d4d
command: /opt/machinegun/bin/machinegun foreground
volumes:
- ./test/machinegun/sys.config:/opt/machinegun/releases/0.1.0/sys.config

2
rebar.config
View File

@ -29,7 +29,7 @@
{deps, [
{genlib , {git, "https://github.com/rbkmoney/genlib.git", {branch, "master"}}},
{woody , {git, "git@github.com:rbkmoney/woody_erlang.git", {branch, "master"}}},
{dmt , {git, "git@github.com:rbkmoney/dmt_core.git", {branch, "master"}}},
{dmt_core, {git, "git@github.com:rbkmoney/dmt_core.git", {branch, "master"}}},
{dmsl , {git, "git@github.com:rbkmoney/damsel_erlang.git", {branch, "master"}}},
{lager , "3.2.1"}
]}.

33
rebar.lock
View File

@ -1,17 +1,18 @@
{"1.1.0",
[{<<"certifi">>,{pkg,<<"certifi">>,<<"0.7.0">>},2},
{<<"cowboy">>,{pkg,<<"cowboy">>,<<"1.0.4">>},1},
{<<"cowlib">>,{pkg,<<"cowlib">>,<<"1.0.2">>},2},
{<<"dmsl">>,
{git,"git@github.com:rbkmoney/damsel_erlang.git",
{ref,"33d87167d5bc0827f9b6ea009eb049e85bf7cc22"}},
{ref,"56fb48e6a23471e49a88c0d646680fba14e74d35"}},
0},
{<<"dmt">>,
{<<"dmt_core">>,
{git,"git@github.com:rbkmoney/dmt_core.git",
{ref,"cbcc1d24b8e50afc50a884a829c808d30da1a521"}},
{ref,"fdc4c1a3b7c22c148e04bbbdbbb83aeba9f99ea3"}},
0},
{<<"genlib">>,
{git,"https://github.com/rbkmoney/genlib.git",
{ref,"02998eb643c9c6b49969fc77054c4925b883bf26"}},
{ref,"82ff16f4314fc406dd90752467a08fe401b009ef"}},
0},
{<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.8">>},1},
{<<"hackney">>,{pkg,<<"hackney">>,<<"1.6.2">>},1},
@ -19,17 +20,31 @@
{<<"lager">>,{pkg,<<"lager">>,<<"3.2.1">>},0},
{<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2},
{<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.0.2">>},2},
{<<"ranch">>,{pkg,<<"ranch">>,<<"1.2.1">>},2},
{<<"ranch">>,{pkg,<<"ranch">>,<<"1.3.2">>},2},
{<<"snowflake">>,
{git,"https://github.com/rbkmoney/snowflake.git",
{ref,"36b978a3ad711c9d9349b799a24c5499a95ae29a"}},
{ref,"0a598108f6582affe3b4ae550fc5b9f2062e318a"}},
1},
{<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.1">>},2},
{<<"thrift">>,
{git,"https://github.com/rbkmoney/thrift_erlang.git",
{ref,"aca7fca9f1a7161a1324bf5b92f8402c90d0519e"}},
{ref,"240bbc842f6e9b90d01bd07838778cf48752b510"}},
1},
{<<"woody">>,
{git,"git@github.com:rbkmoney/woody_erlang.git",
{ref,"b5ae9ae3abc8da470f68d2e2ca15ccc606801225"}},
0}].
{ref,"992c279466e7eb1c24a9d9c6e7e8d66c597bc7e1"}},
0}]}.
[
{pkg_hash,[
{<<"certifi">>, <<"861A57F3808F7EB0C2D1802AFEAAE0FA5DE813B0DF0979153CBAFCD853ABABAF">>},
{<<"cowboy">>, <<"A324A8DF9F2316C833A470D918AAF73AE894278B8AA6226CE7A9BF699388F878">>},
{<<"cowlib">>, <<"9D769A1D062C9C3AC753096F868CA121E2730B9A377DE23DEC0F7E08B1DF84EE">>},
{<<"goldrush">>, <<"2024BA375CEEA47E27EA70E14D2C483B2D8610101B4E852EF7F89163CDB6E649">>},
{<<"hackney">>, <<"96A0A5E7E65B7ACAD8031D231965718CC70A9B4131A8B033B7543BBD673B8210">>},
{<<"idna">>, <<"AC62EE99DA068F43C50DC69ACF700E03A62A348360126260E87F2B54ECED86B2">>},
{<<"lager">>, <<"EEF4E18B39E4195D37606D9088EA05BF1B745986CF8EC84F01D332456FE88D17">>},
{<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>},
{<<"mimerl">>, <<"993F9B0E084083405ED8252B99460C4F0563E41729AB42D9074FD5E52439BE88">>},
{<<"ranch">>, <<"E4965A144DC9FBE70E5C077C65E73C57165416A901BD02EA899CFD95AA890986">>},
{<<"ssl_verify_fun">>, <<"28A4D65B7F59893BC2C7DE786DEC1E1555BD742D336043FE644AE956C3497FBE">>}]}
].

2
src/dmt_client.app.src
View File

@ -7,7 +7,7 @@
stdlib,
lager,
woody,
dmt
dmt_core
]},
{mod, {dmt_client, []}},
{env, []}

44
src/dmt_client.erl
View File

@ -17,21 +17,42 @@
-export([start/2]).
-export([stop/1]).
-export_type([ref/0]).
-export_type([version/0]).
-export_type([snapshot/0]).
-export_type([commit/0]).
-export_type([object_ref/0]).
-export_type([history/0]).
-include_lib("dmsl/include/dmsl_domain_config_thrift.hrl").
-type ref() :: dmsl_domain_config_thrift:'Reference'().
-type version() :: dmsl_domain_config_thrift:'Version'().
-type snapshot() :: dmsl_domain_config_thrift:'Snapshot'().
-type commit() :: dmsl_domain_config_thrift:'Commit'().
-type object_ref() :: dmsl_domain_thrift:'Reference'().
-type history() :: dmsl_domain_config_thrift:'History'().
%% API
-spec checkout(dmt:ref()) -> dmt:snapshot().
-spec checkout(ref()) -> snapshot() | no_return().
checkout(Reference) ->
try
dmt_cache:checkout(Reference)
catch
version_not_found ->
dmt_cache:cache_snapshot(dmt_client_api:checkout(Reference))
CacheResult = case Reference of
{head, #'Head'{}} ->
dmt_client_cache:get_latest();
{version, Version} ->
dmt_client_cache:get(Version)
end,
case CacheResult of
{ok, Snapshot} ->
Snapshot;
{error, version_not_found} ->
dmt_client_cache:put(dmt_client_api:checkout(Reference))
end.
-spec checkout_object(dmt:ref(), dmt:object_ref()) ->
dmsl_domain_config_thrift:'VersionedObject'() | no_return().
-spec checkout_object(ref(), object_ref()) -> dmsl_domain_config_thrift:'VersionedObject'() | no_return().
checkout_object(Reference, ObjectReference) ->
#'Snapshot'{version = Version, domain = Domain} = checkout(Reference),
case dmt_domain:get_object(ObjectReference, Domain) of
@ -41,7 +62,8 @@ checkout_object(Reference, ObjectReference) ->
throw(object_not_found)
end.
-spec commit(dmt:version(), dmt:commit()) -> dmt:version().
-spec commit(version(), commit()) -> version() | no_return().
commit(Version, Commit) ->
dmt_client_api:commit(Version, Commit).
@ -50,8 +72,8 @@ commit(Version, Commit) ->
-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}.
init([]) ->
Poller = #{id => dmt_client_poller, start => {dmt_client_poller, start_link, []}, restart => permanent},
{ok, {#{strategy => one_for_one, intensity => 10, period => 60}, [Poller]}}.
Cache = #{id => dmt_client_cache, start => {dmt_client_cache, start_link, []}, restart => permanent},
{ok, {#{strategy => one_for_one, intensity => 10, period => 60}, [Cache]}}.
%% Application callbacks

44
src/dmt_client_api.erl
View File

@ -5,33 +5,35 @@
-export([pull/1]).
-export([checkout_object/2]).
-spec commit(dmt_client:version(), dmt_client:commit()) -> dmt_client:version() | no_return().
-spec commit(dmt:version(), dmt:commit()) -> dmt:version().
commit(Version, Commit) ->
call(repository, 'Commit', [Version, Commit]).
call('Repository', 'Commit', [Version, Commit]).
-spec checkout(dmt_client:ref()) -> dmt_client:snapshot() | no_return().
-spec checkout(dmt:ref()) -> dmt:snapshot().
checkout(Reference) ->
call(repository, 'Checkout', [Reference]).
call('Repository', 'Checkout', [Reference]).
-spec pull(dmt_client:version()) -> dmt_client:history() | no_return().
-spec pull(dmt:version()) -> dmt:history().
pull(Version) ->
call(repository, 'Pull', [Version]).
call('Repository', 'Pull', [Version]).
-spec checkout_object(dmt_client:ref(), dmt_client:object_ref()) -> dmsl_domain_thrift:'DomainObject'() | no_return().
-spec checkout_object(dmt:ref(), dmt:object_ref()) -> dmt:domain_object().
checkout_object(Reference, ObjectReference) ->
call(repository_client, 'checkoutObject', [Reference, ObjectReference]).
call('RepositoryClient', 'checkoutObject', [Reference, ObjectReference]).
call(ServiceName, Function, Args) ->
Host = application:get_env(dmt, client_host, "dominant"),
Port = integer_to_list(application:get_env(dmt, client_port, 8022)),
{Path, Service} = get_handler_spec(ServiceName),
Url = get_service_url(ServiceName),
Service = get_service_modname(ServiceName),
Call = {Service, Function, Args},
Opts = #{
url => Host ++ ":" ++ Port ++ Path,
url => Url,
event_handler => {dmt_client_woody_event_handler, undefined}
},
},
Context = woody_context:new(),
case woody_client:call(Call, Opts, Context) of
{ok, Response} ->
@ -40,9 +42,13 @@ call(ServiceName, Function, Args) ->
throw(Exception)
end.
get_handler_spec(repository) ->
{"/v1/domain/repository",
{dmsl_domain_config_thrift, 'Repository'}};
get_handler_spec(repository_client) ->
{"/v1/domain/repository_client",
{dmsl_domain_config_thrift, 'RepositoryClient'}}.
get_service_url(ServiceName) ->
maps:get(ServiceName, genlib_app:env(dmt_client, service_urls)).
get_service_modname(ServiceName) ->
{get_service_module(ServiceName), ServiceName}.
get_service_module('Repository') ->
dmsl_domain_config_thrift;
get_service_module('RepositoryClient') ->
dmsl_domain_config_thrift.

174
src/dmt_client_cache.erl Normal file
View File

@ -0,0 +1,174 @@
-module(dmt_client_cache).
-behaviour(gen_server).
%%
-export([start_link/0]).
-export([put/1]).
-export([get/1]).
-export([get_latest/0]).
-export([update/0]).
%%
-export([init/1]).
-export([handle_call/3]).
-export([handle_cast/2]).
-export([handle_info/2]).
-export([terminate/2]).
-export([code_change/3]).
-define(TABLE, ?MODULE).
-define(SERVER, ?MODULE).
-define(DEFAULT_INTERVAL, 5000).
-include_lib("dmsl/include/dmsl_domain_config_thrift.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
%%
-spec start_link() -> {ok, pid()} | {error, term()}. % FIXME
start_link() ->
gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
-spec put(dmt_client:snapshot()) -> dmt_client:snapshot().
put(Snapshot) ->
ok = gen_server:call(?SERVER, {put, Snapshot}),
Snapshot.
-spec get(dmt_client:version()) -> {ok, dmt_client:snapshot()} | {error, version_not_found}.
get(Version) ->
get_snapshot(Version).
-spec get_latest() -> {ok, dmt_client:snapshot()} | {error, version_not_found}.
get_latest() ->
latest_snapshot().
-spec update() -> {ok, dmt_client:version()} | {error, term()}.
update() ->
gen_server:call(?SERVER, update).
%%
-record(state, {
timer = undefined :: undefined | reference()
}).
-type state() :: #state{}.
-spec init(_) -> {ok, state(), 0}.
init(_) ->
EtsOpts = [
named_table,
ordered_set,
protected,
{read_concurrency, true},
{keypos, #'Snapshot'.version}
],
?TABLE = ets:new(?TABLE, EtsOpts),
{ok, #state{}, 0}.
-spec handle_call(term(), {pid(), term()}, state()) -> {reply, term(), state()}.
handle_call({put, Snapshot}, _From, State) ->
{reply, put_snapshot(Snapshot), State};
handle_call(update, _From, State) ->
{reply, update_cache(), restart_timer(State)};
handle_call(_Msg, _From, State) ->
{noreply, State}.
-spec handle_cast(term(), state()) -> {noreply, state()}.
handle_cast(_Msg, State) ->
{noreply, State}.
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info(timeout, State) ->
_Result = update_cache(),
{noreply, restart_timer(State)};
handle_info(_Msg, State) ->
{noreply, State}.
-spec terminate(term(), state()) -> ok.
terminate(_Reason, _State) ->
ok.
-spec code_change(term(), state(), term()) -> {error, noimpl}.
code_change(_OldVsn, _State, _Extra) ->
{error, noimpl}.
%% internal
-spec put_snapshot(dmt_client:snapshot()) -> ok.
put_snapshot(Snapshot) ->
true = ets:insert(?TABLE, Snapshot),
ok.
-spec get_snapshot(dmt_client:version()) -> {ok, dmt_client:snapshot()} | {error, version_not_found}.
get_snapshot(Version) ->
case ets:lookup(?TABLE, Version) of
[Snapshot] ->
{ok, Snapshot};
[] ->
{error, version_not_found}
end.
-spec latest_snapshot() -> {ok, dmt_client:snapshot()} | {error, version_not_found}.
latest_snapshot() ->
case ets:last(?TABLE) of
'$end_of_table' ->
{error, version_not_found};
Version ->
get_snapshot(Version)
end.
-spec restart_timer(state()) -> state().
restart_timer(State = #state{timer = undefined}) ->
start_timer(State);
restart_timer(State = #state{timer = TimerRef}) ->
_ = erlang:cancel_timer(TimerRef),
start_timer(State#state{timer = undefined}).
-spec start_timer(state()) -> state().
start_timer(State = #state{timer = undefined}) ->
Interval = genlib_app:env(dmt_client, cache_update_interval, ?DEFAULT_INTERVAL),
State#state{timer = erlang:send_after(Interval, self(), timeout)}.
-spec update_cache() -> {ok, dmt_client:version()} | {error, term()}.
update_cache() ->
try
NewHead = case latest_snapshot() of
{ok, OldHead} ->
FreshHistory = dmt_client_api:pull(OldHead#'Snapshot'.version),
dmt_history:head(FreshHistory, OldHead);
{error, version_not_found} ->
dmt_client_api:checkout({head, #'Head'{}})
end,
ok = put_snapshot(NewHead),
{ok, NewHead#'Snapshot'.version}
catch
error:{woody_error, {_Source, Class, _Details}} = Error when
Class == resource_unavailable;
Class == result_unknown
->
{error, Error}
end.

99
src/dmt_client_poller.erl
View File

@ -1,99 +0,0 @@
-module(dmt_client_poller).
-behaviour(gen_server).
-export([start_link/0]).
-export([poll/0]).
-export([init/1]).
-export([handle_call/3]).
-export([handle_cast/2]).
-export([handle_info/2]).
-export([terminate/2]).
-export([code_change/3]).
-define(SERVER, ?MODULE).
-include_lib("dmsl/include/dmsl_domain_config_thrift.hrl").
-spec start_link() -> {ok, pid()} | {error, term()}.
start_link() ->
gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
-spec poll() -> {ok, dmt:version()} | {error, term()}.
poll() ->
gen_server:call(?SERVER, poll).
-record(state, {
timer :: reference()
}).
-type state() :: #state{}.
-spec init(_) -> {ok, state()}.
init(_) ->
{ok, start_timer(#state{})}.
-spec handle_call(poll, {pid(), term()}, state()) -> {reply, term(), state()}.
handle_call(poll, _From, State) ->
{reply, pull_safe(), restart_timer(State)}.
-spec handle_cast(term(), state()) -> {noreply, state()}.
handle_cast(_Msg, State) ->
{noreply, State}.
-spec handle_info(poll, state()) -> {noreply, state()}.
handle_info(poll, State) ->
_Result = pull_safe(),
{noreply, restart_timer(State)}.
-spec terminate(term(), state()) -> ok.
terminate(_Reason, _State) ->
ok.
-spec code_change(term(), state(), term()) -> {error, noimpl}.
code_change(_OldVsn, _State, _Extra) ->
{error, noimpl}.
%% Internal
-define(DEFAULT_INTERVAL, 5000).
-spec restart_timer(#state{}) -> #state{}.
restart_timer(State = #state{timer = undefined}) ->
start_timer(State);
restart_timer(State = #state{timer = TimerRef}) ->
_ = erlang:cancel_timer(TimerRef),
start_timer(State#state{timer = undefined}).
-spec start_timer(#state{}) -> #state{}.
start_timer(State = #state{timer = undefined}) ->
Interval = genlib_app:env(dmt_client, poll_interval, ?DEFAULT_INTERVAL),
State#state{timer = erlang:send_after(Interval, self(), poll)}.
-spec pull() -> dmt:version().
pull() ->
OldHead = try
dmt_cache:checkout_head()
catch
version_not_found ->
#'Snapshot'{version = 0, domain = dmt_domain:new()}
end,
FreshHistory = dmt_client_api:pull(OldHead#'Snapshot'.version),
#'Snapshot'{version = NewLastVersion} = NewHead = dmt_history:head(FreshHistory, OldHead),
_ = dmt_cache:cache_snapshot(NewHead),
NewLastVersion.
-spec pull_safe() -> {ok, dmt:version()} | {error, term()}.
pull_safe() ->
try
NewLastVersion = pull(),
{ok, NewLastVersion}
catch
error:{woody_error, {_Source, Class, _Details}} = Error when
Class == resource_unavailable;
Class == result_unknown
->
{error, Error}
end.

18
test/dmt_client_tests_SUITE.erl
View File

@ -5,7 +5,6 @@
-export([groups/0]).
-export([init_per_suite/1]).
-export([end_per_suite/1]).
-export([application_stop/1]).
-export([poll/1]).
-include_lib("dmsl/include/dmsl_domain_config_thrift.hrl").
@ -31,16 +30,18 @@ groups() ->
%% starting/stopping
-spec init_per_suite(term()) -> term().
init_per_suite(C) ->
{ok, Apps} = application:ensure_all_started(dmt_client),
Apps = genlib_app:start_application_with(dmt_client, [
{cache_update_interval, 5000}, % milliseconds
{service_urls, #{
'Repository' => <<"dominant:8022/v1/domain/repository">>,
'RepositoryClient' => <<"dominant:8022/v1/domain/repository_client">>
}}
]),
[{apps, Apps}|C].
-spec end_per_suite(term()) -> term().
end_per_suite(C) ->
[application_stop(App) || App <- proplists:get_value(apps, C)].
-spec application_stop(term()) -> term().
application_stop(App) ->
application:stop(App).
genlib_app:stop_unload_applications(proplists:get_value(apps, C)).
%%
%% tests
@ -52,8 +53,7 @@ poll(_C) ->
#'Snapshot'{version = Version1} = dmt_client:checkout({head, #'Head'{}}),
Version2 = dmt_client_api:commit(Version1, #'Commit'{ops = [{insert, #'InsertOp'{object = Object}}]}),
true = Version1 < Version2,
%% TODO: replace with sleep(genlib_app:env(dmt_client, poll_interval))
{ok, _} = dmt_client_poller:poll(),
_ = dmt_client_cache:update(),
#'Snapshot'{version = Version2} = dmt_client:checkout({head, #'Head'{}}),
#'VersionedObject'{object = Object} = dmt_client:checkout_object({head, #'Head'{}}, Ref).