From d8a4f490d49c038d96f1cbc2a279164c6f4039f9 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Wed, 15 May 2024 10:10:57 +0300 Subject: [PATCH] FIN-31: Bumps valitydev/damsel@ab292d9 and valitydev/dmt-core@19d8f57 (#17) * FIN-31: Bumps valitydev/damsel@ab292d9 and valitydev/dmt-core@19d8f57 * Adds creation timestamp support for cached snapshot building * Temporarily disables codecov report upload; fixes protocol and dialyzer warnings * Fixes linting after including prometheus as a dependency --- .github/workflows/erlang-checks.yml | 3 ++- compose.yaml | 2 +- config/sys.config | 9 ++++++--- rebar.config | 12 ++++++++++-- rebar.lock | 9 ++++++--- src/dmt_client.erl | 4 ++++ src/dmt_client_cache.erl | 18 +++++++++++------- 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 8386179..72ca808 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,10 +30,11 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.10 + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1.0.14 with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} use-thrift: true thrift-version: ${{ needs.setup.outputs.thrift-version }} run-ct-with-compose: true + upload-coverage: false diff --git a/compose.yaml b/compose.yaml index a75f97e..45d31fd 100755 --- a/compose.yaml +++ b/compose.yaml @@ -30,7 +30,7 @@ services: retries: 12 machinegun: - image: ghcr.io/valitydev/machinegun:sha-51546f7 + image: ghcr.io/valitydev/machinegun:sha-5c0db56 command: /opt/machinegun/bin/machinegun foreground volumes: - ./test/machinegun/config.yaml:/opt/machinegun/etc/config.yaml diff --git a/config/sys.config b/config/sys.config index 58b375c..12be2ba 100644 --- a/config/sys.config +++ b/config/sys.config @@ -13,12 +13,15 @@ } }} ]}, - {cache_update_interval, 5000}, % milliseconds - {cache_update_pull_limit, 10}, % event batch size + % milliseconds + {cache_update_interval, 5000}, + % event batch size + {cache_update_pull_limit, 10}, {use_cached_last_version, false}, {max_cache_size, #{ elements => 20, - memory => 52428800 % 50Mb + % 50Mb + memory => 52428800 }}, {cache_server_call_timeout, 10000}, {service_urls, #{ diff --git a/rebar.config b/rebar.config index 8466d25..05b4776 100644 --- a/rebar.config +++ b/rebar.config @@ -27,6 +27,7 @@ %% Common project dependencies. {deps, [ {genlib, {git, "https://github.com/valitydev/genlib.git", {branch, "master"}}}, + {prometheus, "4.6.0"}, {woody, {git, "https://github.com/valitydev/woody_erlang.git", {branch, "master"}}}, {dmt_core, {git, "https://github.com/valitydev/dmt-core.git", {branch, "master"}}}, {damsel, {git, "https://github.com/valitydev/damsel.git", {branch, "master"}}} @@ -48,7 +49,7 @@ % mandatory unmatched_returns, error_handling, - race_conditions, + % NOTE 'race_conditions' was removed https://www.erlang.org/doc/apps/dialyzer/notes#dialyzer-5.0 unknown ]}, {plt_apps, all_deps} @@ -77,7 +78,7 @@ {erlfmt, [ {print_width, 120}, - {files, ["{src,include,test}/*.{hrl,erl}", "rebar.config", "elvis.config"]} + {files, ["{src,include,test}/*.{hrl,erl,app.src}", "rebar.config", "config/sys.config", "elvis.config"]} ]}. {covertool, [ @@ -86,3 +87,10 @@ "ct.coverdata" ]} ]}. + +%% NOTE +%% It is needed to use rebar3 lint plugin +{overrides, [ + {del, accept, [{plugins, [{rebar3_archive_plugin, "0.0.2"}]}]}, + {del, prometheus, [{plugins, [{rebar3_archive_plugin, "0.0.1"}]}]} +]}. diff --git a/rebar.lock b/rebar.lock index c9cb989..1ee2b56 100644 --- a/rebar.lock +++ b/rebar.lock @@ -5,11 +5,11 @@ {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},2}, {<<"damsel">>, {git,"https://github.com/valitydev/damsel.git", - {ref,"f718741970470474efcd32800daf885cb8d75584"}}, + {ref,"ab292d91f5265237351342675c8f69de17add673"}}, 0}, {<<"dmt_core">>, {git,"https://github.com/valitydev/dmt-core.git", - {ref,"75841332fe0b40a77da0c12ea8d5dbb994da8e82"}}, + {ref,"19d8f57198f2cbe5b64aa4a923ba32774e505503"}}, 0}, {<<"genlib">>, {git,"https://github.com/valitydev/genlib.git", @@ -25,6 +25,7 @@ {pkg,<<"opentelemetry_semantic_conventions">>,<<"0.2.0">>}, 2}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2}, + {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.6.0">>},0}, {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},2}, {<<"snowflake">>, {git,"https://github.com/valitydev/snowflake.git", @@ -38,7 +39,7 @@ {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2}, {<<"woody">>, {git,"https://github.com/valitydev/woody_erlang.git", - {ref,"5d46291a6bfcee0bae2a9346a7d927603a909249"}}, + {ref,"81219ba5408e1c67f5eaed3c7e566ede42da88d4"}}, 0}]}. [ {pkg_hash,[ @@ -54,6 +55,7 @@ {<<"opentelemetry_api">>, <<"7B69ED4F40025C005DE0B74FCE8C0549625D59CB4DF12D15C32FE6DC5076FF42">>}, {<<"opentelemetry_semantic_conventions">>, <<"B67FE459C2938FCAB341CB0951C44860C62347C005ACE1B50F8402576F241435">>}, {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>}, + {<<"prometheus">>, <<"20510F381DB1CCAB818B4CF2FAC5FA6AB5CC91BC364A154399901C001465F46F">>}, {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>}, {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>}, {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]}, @@ -70,6 +72,7 @@ {<<"opentelemetry_api">>, <<"6D7A27B7CAD2AD69A09CABF6670514CAFCEC717C8441BEB5C96322BAC3D05350">>}, {<<"opentelemetry_semantic_conventions">>, <<"D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895">>}, {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>}, + {<<"prometheus">>, <<"4905FD2992F8038ECCD7AA0CD22F40637ED618C0BED1F75C05AACEC15B7545DE">>}, {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>}, {<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>}, {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]} diff --git a/src/dmt_client.erl b/src/dmt_client.erl index 308990c..4d5aceb 100644 --- a/src/dmt_client.erl +++ b/src/dmt_client.erl @@ -59,6 +59,7 @@ -export_type([ref/0]). -export_type([vsn/0]). +-export_type([vsn_created_at/0]). -export_type([version/0]). -export_type([limit/0]). -export_type([snapshot/0]). @@ -77,6 +78,7 @@ -type ref() :: dmsl_domain_conf_thrift:'Reference'(). -type vsn() :: dmsl_domain_conf_thrift:'Version'(). +-type vsn_created_at() :: dmsl_base_thrift:'Timestamp'() | undefined. -type version() :: vsn() | latest. -type limit() :: dmsl_domain_conf_thrift:'Limit'(). -type snapshot() :: dmsl_domain_conf_thrift:'Snapshot'(). @@ -254,6 +256,8 @@ pull_range(Reference, Limit, Opts) -> %% - and all of them do not let combine operations (insert X + update Ys + remove Z), %% so the changes will be split via N (3 in this case) commits and won't be transactional +%% TODO Allow to set creation timestamp + -spec insert(domain_object() | [domain_object()]) -> vsn() | no_return(). insert(ObjectOrObjects) -> insert(latest, ObjectOrObjects). diff --git a/src/dmt_client_cache.erl b/src/dmt_client_cache.erl index 110005e..e5444f7 100644 --- a/src/dmt_client_cache.erl +++ b/src/dmt_client_cache.erl @@ -55,6 +55,7 @@ -record(snap, { vsn :: dmt_client:vsn(), + created_at :: dmt_client:vsn_created_at(), tid :: ets:tid(), last_access :: timestamp() }). @@ -305,7 +306,7 @@ do_fold_objects(Version, Folder, Acc) -> end. -spec put_snapshot(dmt_client:snapshot()) -> ok. -put_snapshot(#domain_conf_Snapshot{version = Version, domain = Domain}) -> +put_snapshot(#domain_conf_Snapshot{version = Version, domain = Domain, created_at = CreatedAt}) -> case fetch_snap(Version) of {ok, _Snap} -> ok; @@ -314,6 +315,7 @@ put_snapshot(#domain_conf_Snapshot{version = Version, domain = Domain}) -> true = put_domain_to_table(TID, Domain), Snap = #snap{ vsn = Version, + created_at = CreatedAt, tid = TID, last_access = timestamp() }, @@ -448,7 +450,7 @@ dispatch_reply(From, Response) -> gen_server:reply(From, Response). -spec build_snapshot(snap()) -> {ok, dmt_client:snapshot()} | {error, version_not_found}. -build_snapshot(#snap{vsn = Version, tid = TID}) -> +build_snapshot(#snap{vsn = Version, created_at = CreatedAt, tid = TID}) -> try Domain = ets:foldl( fun(#object{obj = Object}, Domain) -> @@ -458,7 +460,7 @@ build_snapshot(#snap{vsn = Version, tid = TID}) -> dmt_domain:new(), TID ), - {ok, #domain_conf_Snapshot{version = Version, domain = Domain}} + {ok, #domain_conf_Snapshot{version = Version, domain = Domain, created_at = CreatedAt}} catch % table was deleted due to cleanup process or crash error:badarg -> @@ -588,21 +590,23 @@ cleanup() -> -spec test_cleanup() -> _. test_cleanup() -> set_cache_limits(2), - ok = put_snapshot(#domain_conf_Snapshot{version = 4, domain = dmt_domain:new()}), + CreatedAt = <<"2024-05-14T10:00:00+03:00">>, + ok = put_snapshot(#domain_conf_Snapshot{version = 4, domain = dmt_domain:new(), created_at = CreatedAt}), ok = put_snapshot(#domain_conf_Snapshot{version = 3, domain = dmt_domain:new()}), ok = put_snapshot(#domain_conf_Snapshot{version = 2, domain = dmt_domain:new()}), ok = put_snapshot(#domain_conf_Snapshot{version = 1, domain = dmt_domain:new()}), cleanup(), [ #snap{vsn = 1, _ = _}, - #snap{vsn = 4, _ = _} + #snap{vsn = 4, created_at = CreatedAt, _ = _} ] = get_all_snaps(). -spec test_last_access() -> _. test_last_access() -> set_cache_limits(3), + CreatedAt = <<"2024-05-14T10:00:00+03:00">>, % Tables already created in test_cleanup/0 - ok = put_snapshot(#domain_conf_Snapshot{version = 4, domain = dmt_domain:new()}), + ok = put_snapshot(#domain_conf_Snapshot{version = 4, domain = dmt_domain:new(), created_at = CreatedAt}), ok = put_snapshot(#domain_conf_Snapshot{version = 3, domain = dmt_domain:new()}), ok = put_snapshot(#domain_conf_Snapshot{version = 2, domain = dmt_domain:new()}), Ref = {category, #domain_CategoryRef{id = 1}}, @@ -612,7 +616,7 @@ test_last_access() -> [ #snap{vsn = 1, _ = _}, #snap{vsn = 3, _ = _}, - #snap{vsn = 4, _ = _} + #snap{vsn = 4, created_at = CreatedAt, _ = _} ] = get_all_snaps(). -spec test_get_object() -> _.