Upgrade to Erlang 21 (with lager) (#322)

* MSFP-433: Erlang 21 with lager

Jenkinsfile

@@ -33,7 +33,7 @@ build('hellgate', 'docker-host', finalHook) {
         sh 'make wc_xref'
       }
       runStage('dialyze') {
-        withWsCache("_build/default/rebar3_19.1_plt") {
+        withWsCache("_build/default/rebar3_21.1.1_plt") {
           sh 'make wc_dialyze'
         }
       }

Makefile

@@ -13,11 +13,11 @@ SERVICE_IMAGE_TAG ?= $(shell git rev-parse HEAD)
 SERVICE_IMAGE_PUSH_TAG ?= $(SERVICE_IMAGE_TAG)
 
 # Base image for the service
-BASE_IMAGE_NAME := service_erlang
-BASE_IMAGE_TAG := 16e2b3ef17e5fdefac8554ced9c2c74e5c6e9e11
+BASE_IMAGE_NAME := service-erlang
+BASE_IMAGE_TAG := bdb3e60ddc70044bae1aa581d260d3a9803a2477
 
 # Build image tag to be used
-BUILD_IMAGE_TAG := 4fa802d2f534208b9dc2ae203e2a5f07affbf385
+BUILD_IMAGE_TAG := f3732d29a5e622aabf80542b5138b3631a726adb
 
 CALL_ANYWHERE := all submodules rebar-update compile xref lint dialyze start devrel release clean distclean
 

apps/hellgate/rebar.config

@@ -1,3 +0,0 @@
-{erl_opts, [
-    {parse_transform, lager_transform}
-]}.

apps/hellgate/src/hellgate.erl

@@ -67,8 +67,8 @@ get_api_child_spec(MachineHandlers, Opts) ->
         #{
             ip            => Ip,
             port          => genlib_app:env(?MODULE, port, 8022),
-            transport_opts => genlib_app:env(?MODULE, transport_opts, []),
-            protocol_opts => genlib_app:env(?MODULE, protocol_opts, []),
+            transport_opts => genlib_app:env(?MODULE, transport_opts, #{}),
+            protocol_opts => genlib_app:env(?MODULE, protocol_opts, #{}),
             event_handler => scoper_woody_event_handler,
             handlers      => hg_machine:get_service_handlers(MachineHandlers, Opts) ++ [
                 construct_service_handler(party_management             , hg_party_woody_handler, Opts),
@@ -80,7 +80,8 @@ get_api_child_spec(MachineHandlers, Opts) ->
                 construct_service_handler(proxy_host_provider          , hg_proxy_host_provider, Opts),
                 construct_service_handler(payment_processing_eventsink , hg_event_sink_handler , Opts)
             ],
-            additional_routes => [erl_health_handle:get_route(HealthCheckers)]
+            additional_routes => [erl_health_handle:get_route(HealthCheckers)],
+            shutdown_timeout => genlib_app:env(?MODULE, shutdown_timeout, 0)
         }
     ).
 

apps/hellgate/src/hg_invoice.erl

@@ -1306,4 +1306,4 @@ wrap_event_payload(Payload) ->
     #{
         format_version => 1,
         data => {bin, Bin}
-    }.
+    }.

apps/hellgate/src/hg_proxy.erl

@@ -23,6 +23,6 @@ construct_transport_options(Opts) ->
 
 construct_transport_options(Opts, #{transport_opts := TransportOpts = #{}}) ->
     Fields = [connect_timeout, recv_timeout, pool, max_connections],
-    Opts#{transport_opts => maps:to_list(maps:with(Fields, TransportOpts))};
+    Opts#{transport_opts => maps:with(Fields, TransportOpts)};
 construct_transport_options(Opts, #{}) ->
     Opts.

apps/hellgate/src/hg_routing.erl

@@ -41,7 +41,7 @@
     hg_selector:varset(),
     hg_domain:revision()
 ) ->
-    {ok, route()} | {error, {no_route_found | risk_score_is_too_high, reject_context()}}.
+    {ok, route()} | {error, {no_route_found, {unknown | risk_score_is_too_high, reject_context()}}}.
 
 choose(Predestination, PaymentInstitution, VS, Revision) ->
     % TODO not the optimal strategy

apps/hellgate/src/hg_utils.erl

@@ -31,7 +31,6 @@ uid(ID) when is_binary(ID) ->
 -spec logtag_process(atom(), any()) -> ok.
 
 logtag_process(Key, Value) when is_atom(Key) ->
-    % TODO preformat into binary?
     lager:md(orddict:store(Key, Value, lager:md())).
 
 %%
@@ -77,4 +76,4 @@ unwrap_result({error, E}) ->
 -spec format_reason(atom()) -> binary().
 %% TODO: fix this dirty hack
 format_reason(V) ->
-    genlib:to_binary(V).
+    genlib:to_binary(V).

apps/hellgate/test/hg_ct_helper.erl

@@ -111,8 +111,8 @@ start_app(dmt_client = AppName) ->
             memory => 52428800 % 50Mb
         }},
         {service_urls, #{
-            'Repository' => <<"dominant:8022/v1/domain/repository">>,
-            'RepositoryClient' => <<"dominant:8022/v1/domain/repository_client">>
+            'Repository' => <<"http://dominant:8022/v1/domain/repository">>,
+            'RepositoryClient' => <<"http://dominant:8022/v1/domain/repository_client">>
         }}
     ]), #{}};
 
@@ -170,7 +170,7 @@ start_app(cowboy = AppName, Env) ->
         transport_opts := TransOpt,
         proto_opts := ProtoOpt
     } = Env,
-    cowboy:start_http(Ref, Count, TransOpt, ProtoOpt),
+    cowboy:start_clear(Ref, [{num_acceptors, Count} | TransOpt], ProtoOpt),
     [AppName];
 
 start_app(AppName, Env) ->

apps/hellgate/test/hg_customer_tests_SUITE.erl

@@ -472,7 +472,7 @@ start_service_handler(Name, Module, C, HandlerOpts) ->
     Port = get_random_port(),
     Opts = maps:merge(HandlerOpts, #{hellgate_root_url => cfg(root_url, C)}),
     ChildSpec = hg_test_proxy:get_child_spec(Name, Module, IP, Port, Opts),
-    {ok, _} = supervisor:start_child(cfg(test_sup, C), ChildSpec),
+    {ok, _} = supervisor:start_child(cfg(test_sup, C), ChildSpec#{id => Name}),
     hg_test_proxy:get_url(Module, IP, Port).
 
 get_random_port() ->

apps/hellgate/test/hg_dummy_provider.erl

@@ -17,8 +17,7 @@
 -export([make_payment_tool/1]).
 
 %% cowboy http callbacks
--export([init/3]).
--export([handle/2]).
+-export([init/2]).
 -export([terminate/3]).
 %%
 
@@ -53,7 +52,7 @@ get_http_cowboy_spec() ->
         listener_ref => ?MODULE,
         acceptors_count => 10,
         transport_opts => [{port, ?COWBOY_PORT}],
-        proto_opts => [{env, [{dispatch, Dispatch}]}]
+        proto_opts => #{env => #{dispatch => Dispatch}}
     }.
 
 %%
@@ -560,17 +559,12 @@ decode_failure_scenario_step($e) ->
 
 %%
 
--spec init(atom(), cowboy_req:req(), list()) -> {ok, cowboy_req:req(), state}.
+-spec init(cowboy_req:req(), list()) -> {ok, cowboy_req:req(), list()}.
 
-init(_Transport, Req, []) ->
-    {ok, Req, undefined}.
-
--spec handle(cowboy_req:req(), state) -> {ok, cowboy_req:req(), state}.
-
-handle(Req, State) ->
-    {Method, Req2} = cowboy_req:method(Req),
-    {ok, Req3} = handle_user_interaction_response(Method, Req2),
-    {ok, Req3, State}.
+init(Req, Opts) ->
+    Method = cowboy_req:method(Req),
+    Req2 = handle_user_interaction_response(Method, Req),
+    {ok, Req2, Opts}.
 
 -spec terminate(term(), cowboy_req:req(), state) -> ok.
 
@@ -583,7 +577,7 @@ get_callback_url() ->
     genlib:to_binary("http://127.0.0.1:" ++ integer_to_list(?COWBOY_PORT)).
 
 handle_user_interaction_response(<<"POST">>, Req) ->
-    {ok, Body, Req2} = cowboy_req:body(Req),
+    {ok, Body, Req2} = cowboy_req:read_body(Req),
     Form = maps:from_list(cow_qs:parse_qs(Body)),
     RespCode = case maps:get(<<"tag">>, Form, undefined) of
         %% sleep intent
@@ -597,7 +591,7 @@ handle_user_interaction_response(<<"POST">>, Req) ->
             Payload = maps:get(<<"payload">>, Form, Tag),
             callback_to_hell(Tag, Payload)
     end,
-    cowboy_req:reply(RespCode, [{<<"content-type">>, <<"text/plain; charset=utf-8">>}], <<>>, Req2);
+    cowboy_req:reply(RespCode, #{<<"content-type">> => <<"text/plain; charset=utf-8">>}, <<>>, Req2);
 handle_user_interaction_response(_, Req) ->
     %% Method not allowed.
     cowboy_req:reply(405, Req).

apps/hg_client/rebar.config

@@ -1,3 +0,0 @@
-{erl_opts, [
-    {parse_transform, lager_transform}
-]}.

apps/hg_client/src/hg_client_api.erl

@@ -40,7 +40,7 @@ call(ServiceName, Function, Args, {RootUrl, Context}) ->
             Context
         )
     catch
-        error:Error ->
-            {error, {Error, erlang:get_stacktrace()}}
+        error:Error:ST ->
+            {error, {Error, ST}}
     end,
     {Result, {RootUrl, Context}}.

build_utils

@@ -1 +1 @@
-Subproject commit 0a57c5f10795d77ecf121d509fde7c654175c3c1
+Subproject commit ea4aa042f482551d624fd49a570d28488f479e93

config/sys.config

@@ -20,10 +20,15 @@
         {ip, "::"},
         {port, 8022},
         {default_woody_handling_timeout, 30000},
-        {protocol_opts, [
+        %% 1 sec above cowboy's request_timeout
+        {shutdown_timeout, 7000},
+        {protocol_opts, #{
             % Bump keepalive timeout up to a minute
-            {timeout, 60000}
-        ]},
+            request_timeout => 6000,
+            % Should be greater than any other timeouts
+            idle_timeout => infinity
+            }
+        },
         {services, #{
             automaton           => "http://machinegun:8022/v1/automaton",
             eventsink           => "http://machinegun:8022/v1/event_sink",
@@ -58,8 +63,8 @@
             memory => 52428800 % 50Mb
         }},
         {service_urls, #{
-            'Repository' => <<"dominant:8022/v1/domain/repository">>,
-            'RepositoryClient' => <<"dominant:8022/v1/domain/repository_client">>
+            'Repository' => <<"http://dominant:8022/v1/domain/repository">>,
+            'RepositoryClient' => <<"http://dominant:8022/v1/domain/repository_client">>
         }}
     ]},
 

config/vm.args

@@ -2,5 +2,4 @@
 
 -setcookie hellgate_cookie
 
-+K true
-+A 10
++K true

docker-compose.sh

@@ -31,7 +31,7 @@ services:
         condition: service_healthy
 
   machinegun:
-    image: dr.rbkmoney.com/rbkmoney/machinegun:5e26162266a3bcf857852cb7844e5626fb0ebf7a
+    image: dr2.rbkmoney.com/rbkmoney/machinegun:aec434f47029dbd81762e10de04c9422e3c93e5e
     command: /opt/machinegun/bin/machinegun foreground
     volumes:
       - ./test/machinegun/config.yaml:/opt/machinegun/etc/config.yaml

rebar.config

@@ -1,5 +1,6 @@
 % Common project erlang options.
 {erl_opts, [
+    {parse_transform, lager_transform},
 
     % mandatory
     debug_info,
@@ -28,10 +29,10 @@
 
 % Common project dependencies.
 {deps, [
-    {lager  , "3.2.1"},
+    {lager, "3.6.10"},
     {lager_logstash_formatter, {git, "git@github.com:rbkmoney/lager_logstash_formatter.git", {branch, "master"}}},
     {rfc3339, "0.2.2"},
-    {gproc  , "0.6.1"},
+    {gproc  , "0.8.0"},
     {genlib , {git, "https://github.com/rbkmoney/genlib.git", {branch, "master"}}},
     {woody  , {git, "git@github.com:rbkmoney/woody_erlang.git", {branch, "master"}}},
     {woody_user_identity,

rebar.lock

@@ -1,20 +1,20 @@
 {"1.1.0",
 [{<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},2},
  {<<"cache">>,{pkg,<<"cache">>,<<"2.2.0">>},1},
- {<<"certifi">>,{pkg,<<"certifi">>,<<"2.3.1">>},2},
+ {<<"certifi">>,{pkg,<<"certifi">>,<<"2.5.1">>},2},
  {<<"cg_mon">>,
   {git,"https://github.com/rbkmoney/cg_mon.git",
        {ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}},
   1},
- {<<"cowboy">>,{pkg,<<"cowboy">>,<<"1.0.4">>},1},
- {<<"cowlib">>,{pkg,<<"cowlib">>,<<"1.0.2">>},2},
+ {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.5.0">>},1},
+ {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.6.0">>},2},
  {<<"dmsl">>,
   {git,"git@github.com:rbkmoney/damsel.git",
        {ref,"240d8e6de35a23823b1d8e35f989262a4655124f"}},
   0},
  {<<"dmt_client">>,
   {git,"git@github.com:rbkmoney/dmt_client.git",
-       {ref,"ea5b1fd6d0812b7f8d7dbe95996cfdf9318ad830"}},
+       {ref,"6bb0b65a183910c2031b5b81eb84fee045b7de8a"}},
   0},
  {<<"dmt_core">>,
   {git,"git@github.com:rbkmoney/dmt_core.git",
@@ -22,7 +22,7 @@
   1},
  {<<"erl_health">>,
   {git,"https://github.com/rbkmoney/erlang-health.git",
-       {ref,"ab3ca1ccab6e77905810aa270eb936dbe70e02f8"}},
+       {ref,"2575c7b63d82a92de54d2d27e504413675e64811"}},
   0},
  {<<"folsom">>,
   {git,"git@github.com:folsom-project/folsom.git",
@@ -30,18 +30,18 @@
   1},
  {<<"genlib">>,
   {git,"https://github.com/rbkmoney/genlib.git",
-       {ref,"8501050c19e5a36063cf0ae0d181245662bdfa32"}},
+       {ref,"f805a11f6e73faffb05656c5192fbe199df36f27"}},
   0},
- {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.8">>},1},
- {<<"gproc">>,{pkg,<<"gproc">>,<<"0.6.1">>},0},
- {<<"hackney">>,{pkg,<<"hackney">>,<<"1.13.0">>},1},
+ {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1},
+ {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},0},
+ {<<"hackney">>,{pkg,<<"hackney">>,<<"1.15.1">>},1},
  {<<"how_are_you">>,
   {git,"https://github.com/rbkmoney/how_are_you.git",
-       {ref,"e960df58c1e8a764894206623eaed0ec57878b91"}},
+       {ref,"2bb46054e16aaba9357747cc72b7c42e1897a56d"}},
   0},
- {<<"idna">>,{pkg,<<"idna">>,<<"5.1.2">>},2},
- {<<"jsx">>,{pkg,<<"jsx">>,<<"2.8.2">>},1},
- {<<"lager">>,{pkg,<<"lager">>,<<"3.2.1">>},0},
+ {<<"idna">>,{pkg,<<"idna">>,<<"6.0.0">>},2},
+ {<<"jsx">>,{pkg,<<"jsx">>,<<"2.8.0">>},1},
+ {<<"lager">>,{pkg,<<"lager">>,<<"3.6.10">>},0},
  {<<"lager_logstash_formatter">>,
   {git,"git@github.com:rbkmoney/lager_logstash_formatter.git",
        {ref,"24527c15c47749866f2d427b333fa1333a46b8af"}},
@@ -51,17 +51,17 @@
   {git,"git@github.com:rbkmoney/machinegun_proto.git",
        {ref,"ebae56fe2b3e79e4eb34afc8cb55c9012ae989f8"}},
   0},
- {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.0.2">>},2},
- {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.2.0">>},3},
+ {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},2},
+ {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3},
  {<<"party_client">>,
   {git,"git@github.com:rbkmoney/party_client_erlang.git",
-       {ref,"d10247f876bf69b0ac28a4309159e1537cfd81b3"}},
+       {ref,"b71b071ee3fca4eada86732f8cca71e76048ea2b"}},
   0},
  {<<"payproc_errors">>,
   {git,"git@github.com:rbkmoney/payproc-errors-erlang.git",
        {ref,"9c720534eb88edc6ba47af084939efabceb9b2d6"}},
   0},
- {<<"ranch">>,{pkg,<<"ranch">>,<<"1.4.0">>},2},
+ {<<"ranch">>,{pkg,<<"ranch">>,<<"1.6.2">>},2},
  {<<"rfc3339">>,{pkg,<<"rfc3339">>,<<"0.2.2">>},0},
  {<<"scoper">>,
   {git,"git@github.com:rbkmoney/scoper.git",
@@ -75,38 +75,38 @@
   {git,"https://github.com/rbkmoney/snowflake.git",
        {ref,"0a598108f6582affe3b4ae550fc5b9f2062e318a"}},
   1},
- {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.1">>},2},
+ {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.4">>},2},
  {<<"thrift">>,
   {git,"https://github.com/rbkmoney/thrift_erlang.git",
-       {ref,"240bbc842f6e9b90d01bd07838778cf48752b510"}},
+       {ref,"7843146f22a9d9d63be4ae1276b5fa03938f2e9c"}},
   1},
- {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.3.1">>},3},
+ {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.4.1">>},3},
  {<<"woody">>,
   {git,"git@github.com:rbkmoney/woody_erlang.git",
-       {ref,"d8de03d34ac4b296f842e78f2a368c0ec2ff52ff"}},
+       {ref,"1f72c9b1e3f6cd1e2e5e671212984f34f87b3057"}},
   0},
  {<<"woody_user_identity">>,
   {git,"git@github.com:rbkmoney/woody_erlang_user_identity.git",
-       {ref,"9c7ad2b8beac9c88c54594b264743dec7b9cf696"}},
+       {ref,"fe973ab27fee9fc1cc39281e88816c7b6dce84c6"}},
   0}]}.
 [
 {pkg_hash,[
  {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>},
  {<<"cache">>, <<"3C11DBF4CD8FCD5787C95A5FB2A04038E3729CFCA0386016EEA8C953AB48A5AB">>},
- {<<"certifi">>, <<"D0F424232390BF47D82DA8478022301C561CF6445B5B5FB6A84D49A9E76D2639">>},
- {<<"cowboy">>, <<"A324A8DF9F2316C833A470D918AAF73AE894278B8AA6226CE7A9BF699388F878">>},
- {<<"cowlib">>, <<"9D769A1D062C9C3AC753096F868CA121E2730B9A377DE23DEC0F7E08B1DF84EE">>},
- {<<"goldrush">>, <<"2024BA375CEEA47E27EA70E14D2C483B2D8610101B4E852EF7F89163CDB6E649">>},
- {<<"gproc">>, <<"4579663E5677970758A05D8F65D13C3E9814EC707AD51D8DCEF7294EDA1A730C">>},
- {<<"hackney">>, <<"24EDC8CD2B28E1C652593833862435C80661834F6C9344E84B6A2255E7AEEF03">>},
- {<<"idna">>, <<"E21CB58A09F0228A9E0B95EAA1217F1BCFC31A1AAA6E1FDF2F53A33F7DBD9494">>},
- {<<"jsx">>, <<"7ACC7D785B5ABE8A6E9ADBDE926A24E481F29956DD8B4DF49E3E4E7BCC92A018">>},
- {<<"lager">>, <<"EEF4E18B39E4195D37606D9088EA05BF1B745986CF8EC84F01D332456FE88D17">>},
+ {<<"certifi">>, <<"867CE347F7C7D78563450A18A6A28A8090331E77FA02380B4A21962A65D36EE5">>},
+ {<<"cowboy">>, <<"4EF3AE066EE10FE01EA3272EDC8F024347A0D3EB95F6FBB9AED556DACBFC1337">>},
+ {<<"cowlib">>, <<"8AA629F81A0FC189F261DC98A42243FA842625FEEA3C7EC56C48F4CCDB55490F">>},
+ {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>},
+ {<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>},
+ {<<"hackney">>, <<"9F8F471C844B8CE395F7B6D8398139E26DDCA9EBC171A8B91342EE15A19963F4">>},
+ {<<"idna">>, <<"689C46CBCDF3524C44D5F3DDE8001F364CD7608A99556D8FBD8239A5798D4C10">>},
+ {<<"jsx">>, <<"749BEC6D205C694AE1786D62CEA6CC45A390437E24835FD16D12D74F07097727">>},
+ {<<"lager">>, <<"6172B43AB720AC33914CCD0AEB21FDBDF88213847707D4B91E6AF57B2AE5C4D2">>},
  {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>},
- {<<"mimerl">>, <<"993F9B0E084083405ED8252B99460C4F0563E41729AB42D9074FD5E52439BE88">>},
- {<<"parse_trans">>, <<"2ADFA4DAF80C14DC36F522CF190EB5C4EE3E28008FC6394397C16F62A26258C2">>},
- {<<"ranch">>, <<"10272F95DA79340FA7E8774BA7930B901713D272905D0012B06CA6D994F8826B">>},
+ {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>},
+ {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>},
+ {<<"ranch">>, <<"6DB93C78F411EE033DBB18BA8234C5574883ACB9A75AF0FB90A9B82EA46AFA00">>},
  {<<"rfc3339">>, <<"1552DF616ACA368D982E9F085A0E933B6688A3F4938A671798978EC2C0C58730">>},
- {<<"ssl_verify_fun">>, <<"28A4D65B7F59893BC2C7DE786DEC1E1555BD742D336043FE644AE956C3497FBE">>},
- {<<"unicode_util_compat">>, <<"A1F612A7B512638634A603C8F401892AFBF99B8CE93A45041F8AACA99CADB85E">>}]}
+ {<<"ssl_verify_fun">>, <<"F0EAFFF810D2041E93F915EF59899C923F4568F4585904D010387ED74988E77B">>},
+ {<<"unicode_util_compat">>, <<"D869E4C68901DD9531385BB0C8C40444EBF624E60B6962D95952775CAC5E90CD">>}]}
 ].