FIN-31: Bumps valitydev/damsel@ab292d9, adds support for commit creation timestamp (#37)

* FIN-31: Bumps valitydev/damsel@ab292d9, adds support for commit creation timestamp

* Bumps workflow, prometheus and dmt-client

* Fixes formatting

* Refactors commits history to respect events occurrence timestamps
This commit is contained in:
Aleksey Kashapov 2024-05-15 16:40:12 +03:00 committed by GitHub
parent f17373b997
commit f95b1607fb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 88 additions and 51 deletions

View File

@ -30,10 +30,11 @@ jobs:
run:
name: Run checks
needs: setup
uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.10
uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.14
with:
otp-version: ${{ needs.setup.outputs.otp-version }}
rebar-version: ${{ needs.setup.outputs.rebar-version }}
use-thrift: true
thrift-version: ${{ needs.setup.outputs.thrift-version }}
run-ct-with-compose: true
upload-coverage: false

View File

@ -70,7 +70,8 @@
]},
{snowflake, [
{max_backward_clock_moving, 1000}, % 1 second
% 1 second
{max_backward_clock_moving, 1000},
{machine_id, hostname_hash}
]},
@ -78,13 +79,7 @@
{collectors, [default]}
]},
{how_are_you, [
{metrics_publishers, [
% {hay_statsd_publisher, #{
% key_prefix => <<"dominant.">>,
% host => "localhost",
% port => 8125
% }}
]}
]}
{hackney, [
{mod_metrics, woody_hackney_prometheus}
]}
].

View File

@ -32,6 +32,8 @@
{machinery, {git, "https://github.com/valitydev/machinery-erlang.git", {branch, "master"}}},
{scoper, {git, "https://github.com/valitydev/scoper.git", {branch, "master"}}},
{erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, master}}},
{prometheus, "4.8.1"},
{prometheus_cowboy, "0.1.8"},
{opentelemetry_api, "1.2.1"},
{opentelemetry, "1.3.0"},
{opentelemetry_exporter, "1.3.0"}
@ -67,7 +69,7 @@
{test, [
{deps, [
{meck, "0.9.2"},
{dmt_client, {git, "https://github.com/valitydev/dmt-client.git", {ref, "b8bc028"}}}
{dmt_client, {git, "https://github.com/valitydev/dmt-client.git", {ref, "master"}}}
]},
{dialyzer, [
{plt_extra_apps, [meck, dmt_client]}
@ -75,10 +77,6 @@
]},
{prod, [
{deps, [
{prometheus, "4.6.0"},
{prometheus_cowboy, "0.1.8"},
{woody_api_hay, {git, "https://github.com/valitydev/woody_api_hay.git", {ref, "4c39134cd"}}},
{how_are_you, {git, "https://github.com/valitydev/how_are_you.git", {ref, "2fd80134"}}},
{logger_logstash_formatter,
{git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}},
% for introspection on production
@ -98,8 +96,6 @@
% profiler
{tools, load},
{opentelemetry, temporary},
woody_api_hay,
how_are_you,
prometheus,
prometheus_cowboy,
sasl,
@ -122,5 +118,13 @@
{erlfmt, [
{print_width, 120},
{files, ["{src,include,test}/*.{hrl,erl,app.src}", "rebar.config"]}
{files, ["{src,include,test}/*.{hrl,erl,app.src}", "rebar.config", "config/sys.config"]}
]}.
%% NOTE
%% It is needed to use rebar3 lint plugin
{overrides, [
{del, accept, [{plugins, [{rebar3_archive_plugin, "0.0.2"}]}]},
{del, prometheus_cowboy, [{plugins, [{rebar3_archive_plugin, "0.0.1"}]}]},
{del, prometheus_httpd, [{plugins, [{rebar3_archive_plugin, "0.0.1"}]}]}
]}.

View File

@ -1,6 +1,6 @@
{"1.2.0",
[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},2},
{<<"bear">>,{pkg,<<"bear">>,<<"0.9.0">>},2},
[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2},
{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},2},
{<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1},
{<<"certifi">>,{pkg,<<"certifi">>,<<"2.8.0">>},2},
{<<"cg_mon">>,
@ -13,20 +13,16 @@
{<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},2},
{<<"damsel">>,
{git,"https://github.com/valitydev/damsel.git",
{ref,"b04aba83100a4d0adc19b5797372970fd632f911"}},
{ref,"ab292d91f5265237351342675c8f69de17add673"}},
0},
{<<"dmt_core">>,
{git,"https://github.com/valitydev/dmt-core.git",
{ref,"75841332fe0b40a77da0c12ea8d5dbb994da8e82"}},
{ref,"19d8f57198f2cbe5b64aa4a923ba32774e505503"}},
0},
{<<"erl_health">>,
{git,"https://github.com/valitydev/erlang-health.git",
{ref,"49716470d0e8dab5e37db55d52dea78001735a3d"}},
0},
{<<"folsom">>,
{git,"https://github.com/folsom-project/folsom.git",
{ref,"62fd0714e6f0b4e7833880afe371a9c882ea0fc2"}},
1},
{<<"genlib">>,
{git,"https://github.com/valitydev/genlib.git",
{ref,"f6074551d6586998e91a97ea20acb47241254ff3"}},
@ -34,10 +30,6 @@
{<<"gproc">>,{pkg,<<"gproc">>,<<"0.9.0">>},1},
{<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.16.0">>},1},
{<<"hackney">>,{pkg,<<"hackney">>,<<"1.18.0">>},1},
{<<"how_are_you">>,
{git,"https://github.com/rbkmoney/how_are_you.git",
{ref,"2fd8013420328464c2c84302af2781b86577b39f"}},
0},
{<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},3},
{<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},2},
{<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},1},
@ -50,7 +42,7 @@
{git,"https://github.com/valitydev/machinegun-proto",
{ref,"96f7f11b184c29d8b7e83cd7646f3f2c13662bda"}},
1},
{<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},2},
{<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.3.0">>},2},
{<<"opentelemetry">>,{pkg,<<"opentelemetry">>,<<"1.3.0">>},0},
{<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.2.1">>},0},
{<<"opentelemetry_exporter">>,
@ -60,8 +52,10 @@
{pkg,<<"opentelemetry_semantic_conventions">>,<<"0.2.0">>},
1},
{<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2},
{<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.1">>},1},
{<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},2},
{<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.1">>},0},
{<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0},
{<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.11">>},1},
{<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1},
{<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},2},
{<<"scoper">>,
{git,"https://github.com/valitydev/scoper.git",
@ -82,12 +76,12 @@
{<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2},
{<<"woody">>,
{git,"https://github.com/valitydev/woody_erlang.git",
{ref,"3e2337a818086f33f0a1ede5d204aee7744c7c36"}},
{ref,"81219ba5408e1c67f5eaed3c7e566ede42da88d4"}},
0}]}.
[
{pkg_hash,[
{<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>},
{<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>},
{<<"bear">>, <<"A31CCF5361791DD5E708F4789D67E2FEF496C4F05935FC59ADC11622F834D128">>},
{<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>},
{<<"certifi">>, <<"D4FB0A6BB20B7C9C3643E22507E42F356AC090A1DCEA9AB99E27E0376D695EBA">>},
{<<"chatterbox">>, <<"6F059D97BCAA758B8EA6FFFE2B3B81362BD06B639D3EA2BB088335511D691EBF">>},
@ -101,21 +95,23 @@
{<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>},
{<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>},
{<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>},
{<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>},
{<<"mimerl">>, <<"D0CD9FC04B9061F82490F6581E0128379830E78535E017F7780F37FEA7545726">>},
{<<"opentelemetry">>, <<"988AC3C26ACAC9720A1D4FB8D9DC52E95B45ECFEC2D5B5583276A09E8936BC5E">>},
{<<"opentelemetry_api">>, <<"7B69ED4F40025C005DE0B74FCE8C0549625D59CB4DF12D15C32FE6DC5076FF42">>},
{<<"opentelemetry_exporter">>, <<"1D8809C0D4F4ACF986405F7700ED11992BCBDB6A4915DD11921E80777FFA7167">>},
{<<"opentelemetry_semantic_conventions">>, <<"B67FE459C2938FCAB341CB0951C44860C62347C005ACE1B50F8402576F241435">>},
{<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>},
{<<"prometheus">>, <<"FA76B152555273739C14B06F09F485CF6D5D301FE4E9D31B7FF803D26025D7A0">>},
{<<"prometheus_cowboy">>, <<"CFCE0BC7B668C5096639084FCD873826E6220EA714BF60A716F5BD080EF2A99C">>},
{<<"prometheus_httpd">>, <<"F616ED9B85B536B195D94104063025A91F904A4CFC20255363F49A197D96C896">>},
{<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>},
{<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>},
{<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>},
{<<"tls_certificate_check">>, <<"C76C4C5D79EE79A2B11C84F910C825D6F024A78427C854F515748E9BD025E987">>},
{<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]},
{pkg_hash_ext,[
{<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>},
{<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>},
{<<"bear">>, <<"47F71F098F2E3CD05E124A896C5EC2F155967A2B6FF6731E0D627312CCAB7E28">>},
{<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>},
{<<"certifi">>, <<"6AC7EFC1C6F8600B08D625292D4BBF584E14847CE1B6B5C44D983D273E1097EA">>},
{<<"chatterbox">>, <<"B93D19104D86AF0B3F2566C4CBA2A57D2E06D103728246BA1AC6C3C0FF010AA7">>},
@ -129,13 +125,15 @@
{<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>},
{<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>},
{<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>},
{<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>},
{<<"mimerl">>, <<"A1E15A50D1887217DE95F0B9B0793E32853F7C258A5CD227650889B38839FE9D">>},
{<<"opentelemetry">>, <<"8E09EDC26AAD11161509D7ECAD854A3285D88580F93B63B0B1CF0BAC332BFCC0">>},
{<<"opentelemetry_api">>, <<"6D7A27B7CAD2AD69A09CABF6670514CAFCEC717C8441BEB5C96322BAC3D05350">>},
{<<"opentelemetry_exporter">>, <<"2B40007F509D38361744882FD060A8841AF772AB83BB542AA5350908B303AD65">>},
{<<"opentelemetry_semantic_conventions">>, <<"D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895">>},
{<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>},
{<<"prometheus">>, <<"6EDFBE928D271C7F657A6F2C46258738086584BD6CAE4A000B8B9A6009BA23A5">>},
{<<"prometheus_cowboy">>, <<"BA286BECA9302618418892D37BCD5DC669A6CC001F4EB6D6AF85FF81F3F4F34C">>},
{<<"prometheus_httpd">>, <<"0BBE831452CFDF9588538EB2F570B26F30C348ADAE5E95A7D87F35A5910BCF92">>},
{<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>},
{<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>},
{<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>},

View File

@ -6,6 +6,9 @@
kernel,
stdlib,
scoper,
cowboy,
prometheus,
prometheus_cowboy,
woody,
damsel,
dmt_core,

View File

@ -11,6 +11,7 @@
-spec start(application:start_type(), term()) -> {ok, pid()} | {error, term()}.
start(_StartType, _Args) ->
ok = setup_metrics(),
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-spec stop(term()) -> ok.
@ -97,3 +98,7 @@ enable_health_logging(Check) ->
-spec get_prometheus_route() -> {iodata(), module(), _Opts :: any()}.
get_prometheus_route() ->
{"/metrics/[:registry]", prometheus_cowboy2_handler, []}.
setup_metrics() ->
ok = woody_ranch_prometheus_collector:setup(),
ok = woody_hackney_prometheus_collector:setup().

View File

@ -188,19 +188,27 @@ apply_commit(
#domain_conf_Snapshot{version = VersionWas, domain = DomainWas},
#domain_conf_Commit{ops = Ops} = Commit
) ->
%% NOTE Actual timestamp of a produced event may differ since it
%% is set by MG, but we require it now to construct latest
%% snapshot. Thus, we must bear in mind that snapshot saved in
%% event metadata always have timestamp originated by this service
%% and not the actual commit event from the according machine's
%% history.
CreatedAt = current_timestamp(),
case dmt_domain:apply_operations(Ops, DomainWas) of
{ok, Domain} ->
Snapshot = #domain_conf_Snapshot{version = VersionWas + 1, domain = Domain},
Event = make_event(Snapshot, Commit),
Snapshot = #domain_conf_Snapshot{version = VersionWas + 1, domain = Domain, created_at = CreatedAt},
Event = make_event(Snapshot, Commit#domain_conf_Commit{created_at = CreatedAt}),
{{ok, Snapshot}, #{events => [Event]}};
{error, Reason} ->
{{error, {operation_error, Reason}}, #{}}
end.
check_commit(Version, Commit, #st{snapshot = BaseSnapshot, history = History}) ->
check_commit(Version, #domain_conf_Commit{ops = Ops}, #st{snapshot = BaseSnapshot, history = History}) ->
%% NOTE Match only 'ops' because timestamp will differ
case maps:get(Version + 1, History) of
Commit ->
% it's ok, commit alredy applied, lets return this snapshot
#domain_conf_Commit{ops = Ops} ->
% it's ok, commit already applied, lets return this snapshot
{dmt_history:travel(Version + 1, History, BaseSnapshot), #{}};
_ ->
{{error, head_mismatch}, #{}}
@ -211,8 +219,9 @@ read_history(#{history := Events}) ->
lists:foldl(fun apply_event/2, #st{}, Events).
-spec apply_event(machinery:event(event()), st()) -> st().
apply_event({ID, _CreatedAt, {commit, Commit, Meta}}, #st{history = History} = St) ->
StNext = St#st{history = History#{ID => Commit}},
apply_event({ID, CreatedAt, {commit, Commit0, Meta}}, #st{history = History} = St) ->
Commit1 = Commit0#domain_conf_Commit{created_at = historical_timestamp(CreatedAt)},
StNext = St#st{history = History#{ID => Commit1}},
case Meta of
#{snapshot := Snapshot} ->
StNext#st{snapshot = Snapshot};
@ -220,6 +229,14 @@ apply_event({ID, _CreatedAt, {commit, Commit, Meta}}, #st{history = History} = S
StNext
end.
current_timestamp() ->
genlib_rfc3339:format(erlang:system_time(microsecond), microsecond).
historical_timestamp(MachineEventCreatedAt) ->
%% NOTE Reuse generic marshaling since timestamp is still same
%% rfc3339 binary.
machinery_mg_codec:marshal(timestamp, MachineEventCreatedAt).
squash_state(#st{snapshot = BaseSnapshot, history = History}) ->
case dmt_history:head(History, BaseSnapshot) of
{ok, Snapshot} ->

View File

@ -170,7 +170,7 @@ insert(_C) ->
Object = fixture_domain_object(ID, <<"InsertFixture">>),
Ref = fixture_object_ref(ID),
#domain_conf_ObjectNotFound{} = (catch dmt_client:checkout_object(Ref)),
#domain_conf_Snapshot{version = Version1} = dmt_client:checkout(latest),
#domain_conf_Snapshot{version = Version1, created_at = _} = dmt_client:checkout(latest),
Version2 = dmt_client:insert(Version1, Object),
_ = dmt_client_cache:update(),
Object = dmt_client:checkout_object(Ref),
@ -188,7 +188,8 @@ update(_C) ->
Version2 = dmt_client:commit(Version1, dmt_ct_helper:mk_update_commit(Object1, Object2)),
_ = dmt_client_cache:update(),
Object1 = dmt_client:checkout_object(Version1, Ref),
Object2 = dmt_client:checkout_object(Version2, Ref).
Object2 = dmt_client:checkout_object(Version2, Ref),
#domain_conf_Snapshot{version = Version2} = dmt_client:checkout(latest).
-spec delete(term()) -> term().
delete(_C) ->
@ -199,7 +200,8 @@ delete(_C) ->
Version1 = dmt_client:commit(Version0, dmt_ct_helper:mk_insert_commit(Object)),
Version2 = dmt_client:commit(Version1, dmt_ct_helper:mk_remove_commit(Object)),
Object = dmt_client:checkout_object(Version1, Ref),
#domain_conf_ObjectNotFound{} = (catch dmt_client:checkout_object(Version2, Ref)).
#domain_conf_ObjectNotFound{} = (catch dmt_client:checkout_object(Version2, Ref)),
#domain_conf_Snapshot{version = Version2} = dmt_client:checkout(latest).
-spec pull_commit(term()) -> term().
pull_commit(_C) ->
@ -207,9 +209,21 @@ pull_commit(_C) ->
History1 = #{} = dmt_client:pull_range(0, ?DEFAULT_LIMIT),
Version1 = lists:max([0 | maps:keys(History1)]),
Object = fixture_domain_object(ID, <<"PullFixture">>),
Commit = dmt_ct_helper:mk_insert_commit(Object),
Timestamp = <<"2024-05-14T10:00:00+03:00">>,
Commit = (dmt_ct_helper:mk_insert_commit(Object))#domain_conf_Commit{created_at = Timestamp},
#domain_conf_Commit{ops = CommitOps} = Commit,
Version2 = dmt_client:commit(Version1, Commit),
#{Version2 := Commit} = dmt_client:pull_range(Version1, ?DEFAULT_LIMIT).
PulledCommits = dmt_client:pull_range(Version1, ?DEFAULT_LIMIT),
%% Commit matches ops but not given timestamp
?assertMatch(
#{Version2 := #domain_conf_Commit{ops = CommitOps, created_at = CreatedAt}} when CreatedAt =/= Timestamp,
PulledCommits
),
%% All pulled commits must have historical timestamps
_ = [
?assertMatch(#domain_conf_Commit{created_at = CreatedAt} when is_binary(CreatedAt), C)
|| C <- maps:values(PulledCommits)
].
-spec retry_commit(term()) -> term().
retry_commit(_C) ->