TD-428: More tests and routing light refactor (#49)

* added cascade with limits test * added machine test tool and limit complex test * refactored routing * fixed * fixed log * added requested changes * fixed * refactored ct machine * Revert "Auxiliary commit to revert individual files from a2e126e6aeb8c71cde2d2b2560270ab5d25bf49f" This reverts commit 58c9951ac2830dda4d17f2e920e1d66c72c4fd33. * refactored test tool * refactored test * fixed format * cleaned up * fixed * added minor * added one more test * changed to gen server * fixed * removed * fixed types * added patch * fixed
2024-11-06 02:35:18 +00:00 · 2022-12-02 14:47:28 +04:00 · 2022-12-02 14:47:28 +04:00 · 75592f2a54
commit 75592f2a54
parent 1e74eb0c73
13 changed files with 567 additions and 105 deletions
--- a/apps/ff_cth/include/ct_domain.hrl
+++ b/apps/ff_cth/include/ct_domain.hrl
@ -11,6 +11,8 @@
 -define(LIMIT_TURNOVER_NUM_PAYTOOL_ID2, <<"ID2">>).
 -define(LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID1, <<"ID3">>).
 -define(LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID2, <<"ID4">>).
+-define(LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID3, <<"ID5">>).
+-define(LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID4, <<"ID6">>).

 -define(glob(), #domain_GlobalsRef{}).
 -define(cur(ID), #domain_CurrencyRef{symbolic_code = ID}).
--- a/apps/ff_cth/src/ct_helper.erl
+++ b/apps/ff_cth/src/ct_helper.erl
@ -3,6 +3,7 @@
 -include_lib("common_test/include/ct.hrl").

 -export([cfg/2]).
+-export([cfg/3]).

 -export([start_apps/1]).
 -export([start_app/1]).
@ -99,6 +100,23 @@ start_app(dmt_client = AppName) ->
        ]),
        #{}
    };
+start_app(party_client = AppName) ->
+    {
+        start_app_with(AppName, [
+            {services, #{
+                party_management => "http://party-management:8022/v1/processing/partymgmt"
+            }},
+            {woody, #{
+                cache_mode => safe,
+                options => #{
+                    woody_client => #{
+                        event_handler => {scoper_woody_event_handler, #{}}
+                    }
+                }
+            }}
+        ]),
+        #{}
+    };
 start_app(ff_server = AppName) ->
    {
        start_app_with(AppName, [
--- a/apps/ff_cth/src/ct_payment_system.erl
+++ b/apps/ff_cth/src/ct_payment_system.erl
@ -71,6 +71,7 @@ start_processing_apps(Options) ->
        scoper,
        woody,
        dmt_client,
+        party_client,
        {fistful, [
            {services, services(Options)},
            {providers, identity_provider_config(Options)}
@ -460,6 +461,22 @@ domain_config(Options) ->
                    condition(cost_in, {903000, <<"RUB">>}),
                    ?ruleset(?PAYINST1_ROUTING_POLICIES + 19)
                ),
+                delegate(
+                    condition(cost_in, {904000, <<"RUB">>}),
+                    ?ruleset(?PAYINST1_ROUTING_POLICIES + 20)
+                ),
+                delegate(
+                    condition(cost_in, {3000000, <<"RUB">>}),
+                    ?ruleset(?PAYINST1_ROUTING_POLICIES + 21)
+                ),
+                delegate(
+                    condition(cost_in, {905000, <<"RUB">>}),
+                    ?ruleset(?PAYINST1_ROUTING_POLICIES + 22)
+                ),
+                delegate(
+                    condition(cost_in, {3001000, <<"RUB">>}),
+                    ?ruleset(?PAYINST1_ROUTING_POLICIES + 23)
+                ),
                delegate(
                    condition(cost_in, {910000, <<"RUB">>}),
                    ?ruleset(?PAYINST1_ROUTING_POLICIES + 30)
@ -603,6 +620,36 @@ domain_config(Options) ->
            ]}
        ),

+        routing_ruleset(
+            ?ruleset(?PAYINST1_ROUTING_POLICIES + 20),
+            {candidates, [
+                candidate({constant, true}, ?trm(2300), 4000),
+                candidate({constant, true}, ?trm(2400), 1000)
+            ]}
+        ),
+
+        routing_ruleset(
+            ?ruleset(?PAYINST1_ROUTING_POLICIES + 21),
+            {candidates, [
+                candidate({constant, true}, ?trm(2400))
+            ]}
+        ),
+
+        routing_ruleset(
+            ?ruleset(?PAYINST1_ROUTING_POLICIES + 22),
+            {candidates, [
+                candidate({constant, true}, ?trm(2500), 4000),
+                candidate({constant, true}, ?trm(2600), 1000)
+            ]}
+        ),
+
+        routing_ruleset(
+            ?ruleset(?PAYINST1_ROUTING_POLICIES + 23),
+            {candidates, [
+                candidate({constant, true}, ?trm(2700))
+            ]}
+        ),
+
        routing_ruleset(
            ?ruleset(?PAYINST1_ROUTING_POLICIES + 30),
            {candidates, [
@ -955,6 +1002,81 @@ domain_config(Options) ->
            }
        ),

+        ct_domain:withdrawal_terminal(
+            ?trm(2300),
+            ?prv(4),
+            #domain_ProvisionTermSet{
+                wallet = #domain_WalletProvisionTerms{
+                    withdrawals = #domain_WithdrawalProvisionTerms{
+                        turnover_limit =
+                            {value, [
+                                ?trnvrlimit(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID2, 2000000)
+                            ]}
+                    }
+                }
+            }
+        ),
+
+        ct_domain:withdrawal_terminal(
+            ?trm(2400),
+            ?prv(5),
+            #domain_ProvisionTermSet{
+                wallet = #domain_WalletProvisionTerms{
+                    withdrawals = #domain_WithdrawalProvisionTerms{
+                        turnover_limit =
+                            {value, [
+                                ?trnvrlimit(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID2, 3000000)
+                            ]}
+                    }
+                }
+            }
+        ),
+
+        ct_domain:withdrawal_terminal(
+            ?trm(2500),
+            ?prv(4),
+            #domain_ProvisionTermSet{
+                wallet = #domain_WalletProvisionTerms{
+                    withdrawals = #domain_WithdrawalProvisionTerms{
+                        turnover_limit =
+                            {value, [
+                                ?trnvrlimit(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID3, 2000000)
+                            ]}
+                    }
+                }
+            }
+        ),
+
+        ct_domain:withdrawal_terminal(
+            ?trm(2600),
+            ?prv(5),
+            #domain_ProvisionTermSet{
+                wallet = #domain_WalletProvisionTerms{
+                    withdrawals = #domain_WithdrawalProvisionTerms{
+                        turnover_limit =
+                            {value, [
+                                ?trnvrlimit(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID4, 3000000)
+                            ]}
+                    }
+                }
+            }
+        ),
+
+        ct_domain:withdrawal_terminal(
+            ?trm(2700),
+            ?prv(5),
+            #domain_ProvisionTermSet{
+                wallet = #domain_WalletProvisionTerms{
+                    withdrawals = #domain_WithdrawalProvisionTerms{
+                        turnover_limit =
+                            {value, [
+                                ?trnvrlimit(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID3, 4000000)
+                            ]}
+                    }
+                }
+            }
+        ),
+
        ct_domain:withdrawal_terminal(
            ?trm(3000),
            ?prv(1),
--- a/apps/ff_transfer/src/ff_withdrawal.erl
+++ b/apps/ff_transfer/src/ff_withdrawal.erl
@ -18,7 +18,6 @@
    created_at => ff_time:timestamp_ms(),
    party_revision => party_revision(),
    domain_revision => domain_revision(),
-    iteration => non_neg_integer(),
    route => route(),
    attempts => attempts(),
    resource => destination_resource(),
@ -191,6 +190,7 @@
 -export_type([adjustment_params/0]).
 -export_type([start_adjustment_error/0]).
 -export_type([limit_check_details/0]).
+-export_type([activity/0]).

 %% Transfer logic callbacks

@ -217,6 +217,7 @@
 -export([destination_resource/1]).
 -export([metadata/1]).
 -export([params/1]).
+-export([activity/1]).

 %% API

@ -393,6 +394,10 @@ metadata(T) ->
 params(#{params := V}) ->
    V.

+-spec activity(withdrawal_state()) -> activity().
+activity(Withdrawal) ->
+    deduce_activity(Withdrawal).
+
 %% API

 -spec gen(gen_args()) -> withdrawal().
@ -744,7 +749,7 @@ do_process_transfer(routing, Withdrawal) ->
 do_process_transfer(p_transfer_start, Withdrawal) ->
    process_p_transfer_creation(Withdrawal);
 do_process_transfer(p_transfer_prepare, Withdrawal) ->
-    ok = do_rollback_routing(route(Withdrawal), Withdrawal),
+    ok = do_rollback_routing([route(Withdrawal)], Withdrawal),
    Tr = ff_withdrawal_route_attempt_utils:get_current_p_transfer(attempts(Withdrawal)),
    {ok, Events} = ff_postings_transfer:prepare(Tr),
    {continue, [{p_transfer, Ev} || Ev <- Events]};
@ -790,7 +795,7 @@ process_routing(Withdrawal) ->

 -spec process_rollback_routing(withdrawal_state()) -> process_result().
 process_rollback_routing(Withdrawal) ->
-    _ = do_rollback_routing(undefined, Withdrawal),
+    ok = do_rollback_routing([], Withdrawal),
    {undefined, []}.

 -spec do_process_routing(withdrawal_state()) -> {ok, [route()]} | {error, Reason} when
@ -799,7 +804,8 @@ process_rollback_routing(Withdrawal) ->
 do_process_routing(Withdrawal) ->
    do(fun() ->
        {Varset, Context} = make_routing_varset_and_context(Withdrawal),
-        GatherResult = ff_withdrawal_routing:gather_routes(Varset, Context),
+        ExcludeRoutes = ff_withdrawal_route_attempt_utils:get_terminals(attempts(Withdrawal)),
+        GatherResult = ff_withdrawal_routing:gather_routes(Varset, Context, ExcludeRoutes),
        FilterResult = ff_withdrawal_routing:filter_limit_overflow_routes(GatherResult, Varset, Context),
        ff_withdrawal_routing:log_reject_context(FilterResult),
        Routes = unwrap(ff_withdrawal_routing:routes(FilterResult)),
@ -813,46 +819,21 @@ do_process_routing(Withdrawal) ->
        end
    end).

-% filter_attempts(#{routes := Routes} = Result, Withdrawal) ->
-%     NextRoutesResult = ff_withdrawal_route_attempt_utils:next_routes(
-%         [
-%             ff_withdrawal_routing:make_route(ProviderID, TerminalID)
-%          || #{
-%                 provider_ref := #domain_ProviderRef{id = ProviderID},
-%                 terminal_ref := #domain_TerminalRef{id = TerminalID}
-%             } <- Routes
-%         ],
-%         attempts(Withdrawal),
-%         get_attempt_limit(Withdrawal)
-%     ),
-%     case NextRoutesResult of
-%         {ok, Left} ->
-%             {ok, Result#{
-%                 routes => [
-%                     Route
-%                  || Route = #{
-%                         provider_ref := #domain_ProviderRef{id = ProviderID},
-%                         terminal_ref := #domain_TerminalRef{id = TerminalID}
-%                     } <- Routes,
-%                     lists:member(ff_withdrawal_routing:make_route(ProviderID, TerminalID), Left)
-%                 ]
-%             }};
-%         {error, Reason} ->
-%             {error, Reason}
-%     end.
-
-do_rollback_routing(ExcludeRoute, Withdrawal) ->
-    do(fun() ->
+do_rollback_routing(ExcludeRoutes0, Withdrawal) ->
    {Varset, Context} = make_routing_varset_and_context(Withdrawal),
-        Routes = unwrap(ff_withdrawal_routing:routes(ff_withdrawal_routing:gather_routes(Varset, Context))),
-        RollbackRoutes = maybe_exclude_route(ExcludeRoute, Routes),
-        rollback_routes_limits(RollbackRoutes, Varset, Context)
-    end).
-
-maybe_exclude_route(#{terminal_id := TerminalID}, Routes) ->
-    lists:filter(fun(#{terminal_id := TID}) -> TerminalID =/= TID end, Routes);
-maybe_exclude_route(undefined, Routes) ->
-    Routes.
+    ExcludeUsedRoutes0 = ff_withdrawal_route_attempt_utils:get_terminals(attempts(Withdrawal)),
+    ExcludeUsedRoutes1 =
+        case ff_withdrawal_route_attempt_utils:get_current_p_transfer_status(attempts(Withdrawal)) of
+            cancelled ->
+                ExcludeUsedRoutes0;
+            _ ->
+                CurrentRoute = ff_withdrawal_route_attempt_utils:get_current_terminal(attempts(Withdrawal)),
+                lists:filter(fun(R) -> CurrentRoute =/= R end, ExcludeUsedRoutes0)
+        end,
+    ExcludeRoutes1 =
+        ExcludeUsedRoutes1 ++ lists:map(fun(R) -> ff_withdrawal_routing:get_terminal(R) end, ExcludeRoutes0),
+    Routes = ff_withdrawal_routing:get_routes(ff_withdrawal_routing:gather_routes(Varset, Context, ExcludeRoutes1)),
+    rollback_routes_limits(Routes, Varset, Context).

 rollback_routes_limits(Routes, Withdrawal) ->
    {Varset, Context} = make_routing_varset_and_context(Withdrawal),
@ -885,7 +866,7 @@ make_routing_varset_and_context(Withdrawal) ->
        domain_revision => DomainRevision,
        identity => Identity,
        withdrawal => Withdrawal,
-        iteration => maps:get(iteration, Withdrawal)
+        iteration => ff_withdrawal_route_attempt_utils:get_index(attempts(Withdrawal))
    },
    {build_party_varset(VarsetParams), Context}.

@ -1906,15 +1887,8 @@ apply_event_({limit_check, Details}, T) ->
    add_limit_check(Details, T);
 apply_event_({p_transfer, Ev}, T) ->
    Tr = ff_postings_transfer:apply_event(Ev, p_transfer(T)),
-    Iteration =
-        case maps:get(status, Tr, undefined) of
-            committed -> maps:get(iteration, T) + 1;
-            cancelled -> maps:get(iteration, T) + 1;
-            _ -> maps:get(iteration, T)
-        end,
-    Attempts = attempts(T),
-    R = ff_withdrawal_route_attempt_utils:update_current_p_transfer(Tr, Attempts),
-    update_attempts(R, T#{iteration => Iteration});
+    R = ff_withdrawal_route_attempt_utils:update_current_p_transfer(Tr, attempts(T)),
+    update_attempts(R, T);
 apply_event_({session_started, SessionID}, T) ->
    Session = #{id => SessionID},
    Attempts = attempts(T),
@ -1941,11 +1915,10 @@ apply_event_({adjustment, _Ev} = Event, T) ->
 make_state(#{route := Route} = T) ->
    Attempts = ff_withdrawal_route_attempt_utils:new(),
    T#{
-        iteration => 1,
        attempts => ff_withdrawal_route_attempt_utils:new_route(Route, Attempts)
    };
 make_state(T) when not is_map_key(route, T) ->
-    T#{iteration => 1}.
+    T.

 get_attempt_limit(Withdrawal) ->
    #{
--- a/apps/ff_transfer/src/ff_withdrawal_route_attempt_utils.erl
+++ b/apps/ff_transfer/src/ff_withdrawal_route_attempt_utils.erl
@ -20,8 +20,10 @@
 -export([new_route/2]).
 -export([next_route/3]).
 -export([next_routes/3]).
+-export([get_index/1]).
 -export([get_current_session/1]).
 -export([get_current_p_transfer/1]).
+-export([get_current_p_transfer_status/1]).
 -export([get_current_limit_checks/1]).
 -export([update_current_session/2]).
 -export([update_current_p_transfer/2]).
@ -29,21 +31,27 @@

 -export([get_sessions/1]).
 -export([get_attempt/1]).
+-export([get_terminals/1]).
+-export([get_current_terminal/1]).

 -opaque attempts() :: #{
    attempts := #{route_key() => attempt()},
    inversed_routes := [route_key()],
    attempt := non_neg_integer(),
-    current => route_key()
+    current => route_key(),
+    index := index()
 }.

+-type index() :: non_neg_integer().
+-define(DEFAULT_INDEX, 1).
+
 -export_type([attempts/0]).

 %% Iternal types

 -type p_transfer() :: ff_postings_transfer:transfer().
+-type p_transfer_status() :: ff_postings_transfer:status().
 -type limit_check_details() :: ff_withdrawal:limit_check_details().
-type account() :: ff_account:account().
 -type route() :: ff_withdrawal_routing:route().
 -type route_key() :: {ff_payouts_provider:id(), ff_payouts_terminal:id()} | unknown.
 -type session() :: ff_withdrawal:session().
@ -62,7 +70,8 @@ new() ->
    #{
        attempts => #{},
        inversed_routes => [],
-        attempt => 0
+        attempt => 0,
+        index => ?DEFAULT_INDEX
    }.

 -spec new_route(route(), attempts()) -> attempts().
@ -99,6 +108,12 @@ next_routes(Routes, #{attempts := Existing}, _AttemptLimit) ->
            Routes
        )}.

+-spec get_index(attempts() | undefined) -> index().
+get_index(undefined) ->
+    ?DEFAULT_INDEX;
+get_index(#{index := Index}) ->
+    Index.
+
 -spec get_current_session(attempts()) -> undefined | session().
 get_current_session(Attempts) ->
    Attempt = current(Attempts),
@ -109,6 +124,11 @@ get_current_p_transfer(Attempts) ->
    Attempt = current(Attempts),
    maps:get(p_transfer, Attempt, undefined).

+-spec get_current_p_transfer_status(attempts()) -> undefined | p_transfer_status().
+get_current_p_transfer_status(Attempts) ->
+    Attempt = current(Attempts),
+    maps:get(status, maps:get(p_transfer, Attempt, #{}), undefined).
+
 -spec get_current_limit_checks(attempts()) -> undefined | [limit_check_details()].
 get_current_limit_checks(Attempts) ->
    Attempt = current(Attempts),
@ -122,13 +142,19 @@ update_current_session(Session, Attempts) ->
    },
    update_current(Updated, Attempts).

-spec update_current_p_transfer(account(), attempts()) -> attempts().
-update_current_p_transfer(Account, Attempts) ->
+-spec update_current_p_transfer(p_transfer(), attempts()) -> attempts().
+update_current_p_transfer(PTransfer, Attempts = #{index := Index}) ->
    Attempt = current(Attempts),
    Updated = Attempt#{
-        p_transfer => Account
+        p_transfer => PTransfer
    },
-    update_current(Updated, Attempts).
+    NewIndex =
+        case maps:get(status, PTransfer, undefined) of
+            committed -> Index + 1;
+            cancelled -> Index + 1;
+            _ -> Index
+        end,
+    update_current(Updated, Attempts#{index => NewIndex}).

 -spec update_current_limit_checks([limit_check_details()], attempts()) -> attempts().
 update_current_limit_checks(LimitChecks, Routes) ->
@ -160,6 +186,18 @@ get_sessions(#{attempts := Attempts, inversed_routes := InvRoutes}) ->
 get_attempt(#{attempt := Attempt}) ->
    Attempt.

+-spec get_terminals(attempts()) -> [ff_payouts_terminal:id()].
+get_terminals(#{attempts := Attempts}) ->
+    lists:map(fun({_, TerminalID}) -> TerminalID end, maps:keys(Attempts));
+get_terminals(_) ->
+    [].
+
+-spec get_current_terminal(attempts()) -> undefined | ff_payouts_terminal:id().
+get_current_terminal(#{current := {_, TerminalID}}) ->
+    TerminalID;
+get_current_terminal(_) ->
+    undefined.
+
 %% Internal

 -spec route_key(route()) -> route_key().
@ -192,8 +230,4 @@ update_current(Attempt, #{current := Route, attempts := Attempts} = R) ->
        attempts => Attempts#{
            Route => Attempt
        }
-    };
-update_current(Attempt, R) when not is_map_key(current, R) ->
-    % There are some legacy operations without a route in storage
-    % It looks like we should save other data without route.
-    update_current(Attempt, add_route(unknown, R)).
+    }.
--- a/apps/ff_transfer/src/ff_withdrawal_routing.erl
+++ b/apps/ff_transfer/src/ff_withdrawal_routing.erl
@ -5,6 +5,7 @@
 -export([prepare_routes/2]).
 -export([prepare_routes/3]).
 -export([gather_routes/2]).
+-export([gather_routes/3]).
 -export([filter_limit_overflow_routes/3]).
 -export([rollback_routes_limits/3]).
 -export([commit_routes_limits/3]).
@ -12,6 +13,7 @@
 -export([get_provider/1]).
 -export([get_terminal/1]).
 -export([routes/1]).
+-export([get_routes/1]).
 -export([log_reject_context/1]).

 -import(ff_pipeline, [do/1, unwrap/1]).
@ -26,8 +28,8 @@
 -type routing_context() :: #{
    domain_revision := domain_revision(),
    identity := identity(),
-    withdrawal => withdrawal(),
-    iteration => pos_integer()
+    iteration := pos_integer(),
+    withdrawal => withdrawal()
 }.

 -type routing_state() :: #{
@ -66,7 +68,7 @@
 -spec prepare_routes(party_varset(), identity(), domain_revision()) ->
    {ok, [route()]} | {error, route_not_found}.
 prepare_routes(PartyVarset, Identity, DomainRevision) ->
-    prepare_routes(PartyVarset, #{identity => Identity, domain_revision => DomainRevision}).
+    prepare_routes(PartyVarset, #{identity => Identity, domain_revision => DomainRevision, iteration => 1}).

 -spec prepare_routes(party_varset(), routing_context()) ->
    {ok, [route()]} | {error, route_not_found}.
@ -77,7 +79,12 @@ prepare_routes(PartyVarset, Context) ->

 -spec gather_routes(party_varset(), routing_context()) ->
    routing_state().
-gather_routes(PartyVarset, Context = #{identity := Identity, domain_revision := DomainRevision}) ->
+gather_routes(PartyVarset, Context) ->
+    gather_routes(PartyVarset, Context, []).
+
+-spec gather_routes(party_varset(), routing_context(), [terminal_id()]) ->
+    routing_state().
+gather_routes(PartyVarset, Context = #{identity := Identity, domain_revision := DomainRevision}, ExcludeRoutes) ->
    {ok, PaymentInstitutionID} = ff_party:get_identity_payment_institution_id(Identity),
    {ok, PaymentInstitution} = ff_payment_institution:get(PaymentInstitutionID, PartyVarset, DomainRevision),
    {Routes, RejectContext} = ff_routing_rule:gather_routes(
@ -86,7 +93,8 @@ gather_routes(PartyVarset, Context = #{identity := Identity, domain_revision :=
        PartyVarset,
        DomainRevision
    ),
-    filter_valid_routes(#{routes => Routes, reject_context => RejectContext}, PartyVarset, Context).
+    State = exclude_routes(#{routes => Routes, reject_context => RejectContext}, ExcludeRoutes),
+    filter_valid_routes(State, PartyVarset, Context).

 -spec filter_limit_overflow_routes(routing_state(), party_varset(), routing_context()) ->
    routing_state().
@ -130,9 +138,9 @@ make_route(ProviderID, TerminalID) ->
 get_provider(#{provider_id := ProviderID}) ->
    ProviderID.

-spec get_terminal(route()) -> ff_maybe:maybe(terminal_id()).
-get_terminal(Route) ->
-    maps:get(terminal_id, Route, undefined).
+-spec get_terminal(route()) -> terminal_id().
+get_terminal(#{terminal_id := TerminalID}) ->
+    TerminalID.

 -spec routes(routing_state()) ->
    {ok, [route()]} | {error, route_not_found}.
@ -141,6 +149,17 @@ routes(#{routes := Routes = [_ | _]}) ->
 routes(_) ->
    {error, route_not_found}.

+-spec get_routes(routing_state()) ->
+    [route()].
+get_routes(#{routes := Routes}) ->
+    [
+        make_route(P, T)
+     || #{
+            provider_ref := #domain_ProviderRef{id = P},
+            terminal_ref := #domain_TerminalRef{id = T}
+        } <- Routes
+    ].
+
 -spec sort_routes([routing_rule_route()]) -> [route()].
 sort_routes(RoutingRuleRoutes) ->
    ProviderTerminalMap = lists:foldl(
@ -233,6 +252,26 @@ get_route_terms_and_process(
            {error, Error}
    end.

+exclude_routes(#{routes := Routes, reject_context := RejectContext}, ExcludeRoutes) ->
+    lists:foldl(
+        fun(Route, State = #{routes := ValidRoutes0, reject_context := RejectContext0}) ->
+            ProviderRef = maps:get(provider_ref, Route),
+            TerminalRef = maps:get(terminal_ref, Route),
+            case not lists:member(ff_routing_rule:terminal_id(Route), ExcludeRoutes) of
+                true ->
+                    ValidRoutes1 = [Route | ValidRoutes0],
+                    State#{routes => ValidRoutes1};
+                false ->
+                    RejectedRoutes0 = maps:get(rejected_routes, RejectContext0),
+                    RejectedRoutes1 = [{ProviderRef, TerminalRef, member_of_exlude_list} | RejectedRoutes0],
+                    RejectContext1 = maps:put(rejected_routes, RejectedRoutes1, RejectContext0),
+                    State#{reject_context => RejectContext1}
+            end
+        end,
+        #{routes => [], reject_context => RejectContext},
+        Routes
+    ).
+
 -spec do_rollback_limits(withdrawal_provision_terms(), party_varset(), route(), routing_context()) ->
    ok.
 do_rollback_limits(CombinedTerms, _PartyVarset, Route, #{withdrawal := Withdrawal, iteration := Iter}) ->
--- a/apps/ff_transfer/test/ff_ct_barrier.erl
+++ b/apps/ff_transfer/test/ff_ct_barrier.erl
@ -0,0 +1,57 @@
+-module(ff_ct_barrier).
+
+-export([start_link/0]).
+-export([stop/1]).
+-export([enter/2]).
+-export([release/1]).
+
+%% Gen Server
+
+-behaviour(gen_server).
+-export([init/1]).
+-export([handle_call/3]).
+-export([handle_cast/2]).
+
+-type caller() :: {pid(), reference()}.
+
+-type st() :: #{
+    blocked := [caller()]
+}.
+
+%%
+
+-spec enter(pid(), timeout()) -> ok.
+enter(ServerRef, Timeout) ->
+    gen_server:call(ServerRef, enter, Timeout).
+
+-spec release(pid()) -> ok.
+release(ServerRef) ->
+    gen_server:call(ServerRef, release).
+
+-spec start_link() -> {ok, pid()}.
+start_link() ->
+    gen_server:start_link(?MODULE, [], []).
+
+-spec stop(pid()) -> ok.
+stop(ServerRef) ->
+    proc_lib:stop(ServerRef, normal, 5000).
+
+-spec init(_) -> {ok, st()}.
+init(_Args) ->
+    {ok, #{blocked => []}}.
+
+-spec handle_call(enter | release, caller(), st()) ->
+    {noreply, st()}
+    | {reply, ok, st()}.
+handle_call(enter, From = {ClientPid, _}, St = #{blocked := Blocked}) ->
+    false = lists:any(fun({Pid, _}) -> Pid == ClientPid end, Blocked),
+    {noreply, St#{blocked => [From | Blocked]}};
+handle_call(release, _From, St = #{blocked := Blocked}) ->
+    ok = lists:foreach(fun(Caller) -> gen_server:reply(Caller, ok) end, Blocked),
+    {reply, ok, St#{blocked => []}};
+handle_call(Call, _From, _St) ->
+    error({badcall, Call}).
+
+-spec handle_cast(_Cast, st()) -> no_return().
+handle_cast(Cast, _St) ->
+    error({badcast, Cast}).
--- a/apps/ff_transfer/test/ff_ct_machine.erl
+++ b/apps/ff_transfer/test/ff_ct_machine.erl
@ -0,0 +1,53 @@
+%%%
+%%% Test machine
+%%%
+
+-module(ff_ct_machine).
+
+-dialyzer({nowarn_function, dispatch_signal/4}).
+
+-export([load_per_suite/0]).
+-export([unload_per_suite/0]).
+
+-export([set_hook/2]).
+-export([clear_hook/1]).
+
+-spec load_per_suite() -> ok.
+load_per_suite() ->
+    meck:new(machinery, [no_link, passthrough]),
+    meck:expect(machinery, dispatch_signal, fun dispatch_signal/4),
+    meck:expect(machinery, dispatch_call, fun dispatch_call/4).
+
+-spec unload_per_suite() -> ok.
+unload_per_suite() ->
+    meck:unload(machinery).
+
+-type hook() :: fun((machinery:machine(_, _), module(), _Args) -> _).
+
+-spec set_hook(timeout, hook()) -> ok.
+set_hook(On = timeout, Fun) when is_function(Fun, 3) ->
+    persistent_term:put({?MODULE, hook, On}, Fun).
+
+-spec clear_hook(timeout) -> ok.
+clear_hook(On = timeout) ->
+    _ = persistent_term:erase({?MODULE, hook, On}),
+    ok.
+
+dispatch_signal({init, Args}, Machine, {Handler, HandlerArgs}, Opts) ->
+    Handler:init(Args, Machine, HandlerArgs, Opts);
+dispatch_signal(timeout, Machine, {Handler, HandlerArgs}, Opts) when Handler =/= fistful ->
+    _ =
+        case persistent_term:get({?MODULE, hook, timeout}, undefined) of
+            Fun when is_function(Fun) ->
+                Fun(Machine, Handler, HandlerArgs);
+            undefined ->
+                ok
+        end,
+    Handler:process_timeout(Machine, HandlerArgs, Opts);
+dispatch_signal(timeout, Machine, {Handler, HandlerArgs}, Opts) ->
+    Handler:process_timeout(Machine, HandlerArgs, Opts);
+dispatch_signal({notification, Args}, Machine, {Handler, HandlerArgs}, Opts) ->
+    Handler:process_notification(Args, Machine, HandlerArgs, Opts).
+
+dispatch_call(Args, Machine, {Handler, HandlerArgs}, Opts) ->
+    Handler:process_call(Args, Machine, HandlerArgs, Opts).
--- a/apps/ff_transfer/test/ff_limiter_helper.erl
+++ b/apps/ff_transfer/test/ff_limiter_helper.erl
@ -33,6 +33,14 @@ init_per_suite(Config) ->
    {ok, #config_LimitConfig{}} = ff_ct_limiter_client:create_config(
        limiter_create_amount_params(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID2),
        ct_helper:get_woody_ctx(Config)
+    ),
+    {ok, #config_LimitConfig{}} = ff_ct_limiter_client:create_config(
+        limiter_create_amount_params(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID3),
+        ct_helper:get_woody_ctx(Config)
+    ),
+    {ok, #config_LimitConfig{}} = ff_ct_limiter_client:create_config(
+        limiter_create_amount_params(?LIMIT_TURNOVER_AMOUNT_PAYTOOL_ID4),
+        ct_helper:get_woody_ctx(Config)
    ).

 -spec get_limit_amount(id(), withdrawal(), config()) -> integer().
--- a/apps/ff_transfer/test/ff_withdrawal_limits_SUITE.erl
+++ b/apps/ff_transfer/test/ff_withdrawal_limits_SUITE.erl
@ -1,7 +1,6 @@
 -module(ff_withdrawal_limits_SUITE).

 -include_lib("stdlib/include/assert.hrl").
-include_lib("fistful_proto/include/fistful_fistful_base_thrift.hrl").
 -include_lib("ff_cth/include/ct_domain.hrl").
 -include_lib("damsel/include/dmsl_wthd_domain_thrift.hrl").

@ -21,6 +20,9 @@
 -export([limit_overflow/1]).
 -export([choose_provider_without_limit_overflow/1]).
 -export([provider_limits_exhaust_orderly/1]).
+-export([provider_retry/1]).
+-export([limit_exhaust_on_provider_retry/1]).
+-export([first_limit_exhaust_on_provider_retry/1]).

 %% Internal types

@ -44,12 +46,16 @@ groups() ->
            limit_success,
            limit_overflow,
            choose_provider_without_limit_overflow,
-            provider_limits_exhaust_orderly
+            provider_limits_exhaust_orderly,
+            provider_retry,
+            limit_exhaust_on_provider_retry,
+            first_limit_exhaust_on_provider_retry
        ]}
    ].

 -spec init_per_suite(config()) -> config().
 init_per_suite(C0) ->
+    ff_ct_machine:load_per_suite(),
    C1 = ct_helper:makeup_cfg(
        [
            ct_helper:test_case_name(init),
@ -62,6 +68,7 @@ init_per_suite(C0) ->

 -spec end_per_suite(config()) -> _.
 end_per_suite(C) ->
+    ff_ct_machine:unload_per_suite(),
    ok = ct_payment_system:shutdown(C).

 %%
@ -77,20 +84,43 @@ end_per_group(_, _) ->
 %%

 -spec init_per_testcase(test_case_name(), config()) -> config().
-init_per_testcase(Name, C) ->
+init_per_testcase(Name, C0) ->
    C1 = ct_helper:makeup_cfg(
        [
            ct_helper:test_case_name(Name),
            ct_helper:woody_ctx()
        ],
-        C
+        C0
    ),
    ok = ct_helper:set_context(C1),
-    C1.
+    PartyID = create_party(C1),
+    C2 = ct_helper:cfg('$party', PartyID, C1),
+    case Name of
+        Name when
+            Name =:= provider_retry orelse
+                Name =:= limit_exhaust_on_provider_retry orelse
+                Name =:= first_limit_exhaust_on_provider_retry
+        ->
+            _ = set_retryable_errors(PartyID, [<<"authorization_error">>]);
+        _ ->
+            ok
+    end,
+    C2.

 -spec end_per_testcase(test_case_name(), config()) -> _.
-end_per_testcase(_Name, _C) ->
-    ok = ct_helper:unset_context().
+end_per_testcase(Name, C) ->
+    case Name of
+        Name when
+            Name =:= provider_retry orelse
+                Name =:= limit_exhaust_on_provider_retry orelse
+                Name =:= first_limit_exhaust_on_provider_retry
+        ->
+            PartyID = ct_helper:cfg('$party', C),
+            _ = set_retryable_errors(PartyID, []);
+        _ ->
+            ok
+    end,
+    ct_helper:unset_context().

 %% Tests

@ -234,16 +264,105 @@ provider_limits_exhaust_orderly(C) ->
    Result = await_final_withdrawal_status(WithdrawalID),
    ?assertMatch({failed, #{code := <<"no_route_found">>}}, Result).

-%% Utils
+-spec provider_retry(config()) -> test_return().
+provider_retry(C) ->
+    Currency = <<"RUB">>,
+    Cash = {904000, Currency},
+    #{
+        wallet_id := WalletID,
+        destination_id := DestinationID
+    } = prepare_standard_environment(Cash, C),
+    WithdrawalID = generate_id(),
+    WithdrawalParams = #{
+        id => WithdrawalID,
+        destination_id => DestinationID,
+        wallet_id => WalletID,
+        body => Cash,
+        external_id => WithdrawalID
+    },
+    ok = ff_withdrawal_machine:create(WithdrawalParams, ff_entity_context:new()),
+    ?assertEqual(succeeded, await_final_withdrawal_status(WithdrawalID)),
+    Withdrawal = get_withdrawal(WithdrawalID),
+    ?assertEqual(WalletID, ff_withdrawal:wallet_id(Withdrawal)),
+    ?assertEqual(DestinationID, ff_withdrawal:destination_id(Withdrawal)),
+    ?assertEqual(Cash, ff_withdrawal:body(Withdrawal)),
+    ?assertEqual(WithdrawalID, ff_withdrawal:external_id(Withdrawal)).

-get_limit_amount(Cash, WalletID, DestinationID, LimitID, C) ->
+-spec limit_exhaust_on_provider_retry(config()) -> test_return().
+limit_exhaust_on_provider_retry(C) ->
+    ?assertEqual(
+        {failed, #{code => <<"authorization_error">>, sub => #{code => <<"insufficient_funds">>}}},
+        await_provider_retry(904000, 3000000, 4000000, C)
+    ).
+
+-spec first_limit_exhaust_on_provider_retry(config()) -> test_return().
+first_limit_exhaust_on_provider_retry(C) ->
+    ?assertEqual(succeeded, await_provider_retry(905000, 3001000, 4000000, C)).
+
+await_provider_retry(FirstAmount, SecondAmount, TotalAmount, C) ->
+    Currency = <<"RUB">>,
+    #{
+        wallet_id := WalletID,
+        destination_id := DestinationID
+    } = prepare_standard_environment({TotalAmount, Currency}, C),
+    WithdrawalID1 = generate_id(),
+    WithdrawalParams1 = #{
+        id => WithdrawalID1,
+        destination_id => DestinationID,
+        wallet_id => WalletID,
+        body => {FirstAmount, Currency},
+        external_id => WithdrawalID1
+    },
+    WithdrawalID2 = generate_id(),
+    WithdrawalParams2 = #{
+        id => WithdrawalID2,
+        destination_id => DestinationID,
+        wallet_id => WalletID,
+        body => {SecondAmount, Currency},
+        external_id => WithdrawalID2
+    },
+    Activity = {fail, session},
+    {ok, Barrier} = ff_ct_barrier:start_link(),
+    ok = ff_ct_machine:set_hook(
+        timeout,
+        fun
+            (Machine, ff_withdrawal_machine, _Args) ->
+                Withdrawal = ff_machine:model(ff_machine:collapse(ff_withdrawal, Machine)),
+                case {ff_withdrawal:id(Withdrawal), ff_withdrawal:activity(Withdrawal)} of
+                    {WithdrawalID1, Activity} ->
+                        ff_ct_barrier:enter(Barrier, _Timeout = 10000);
+                    _ ->
+                        ok
+                end;
+            (_Machine, _Handler, _Args) ->
+                false
+        end
+    ),
+    ok = ff_withdrawal_machine:create(WithdrawalParams1, ff_entity_context:new()),
+    _ = await_withdrawal_activity(Activity, WithdrawalID1),
+    ok = ff_withdrawal_machine:create(WithdrawalParams2, ff_entity_context:new()),
+    ?assertEqual(succeeded, await_final_withdrawal_status(WithdrawalID2)),
+    ok = ff_ct_barrier:release(Barrier),
+    Status = await_final_withdrawal_status(WithdrawalID1),
+    ok = ff_ct_machine:clear_hook(timeout),
+    ok = ff_ct_barrier:stop(Barrier),
+    Status.
+
+set_retryable_errors(PartyID, ErrorList) ->
+    application:set_env(ff_transfer, withdrawal, #{
+        party_transient_errors => #{
+            PartyID => ErrorList
+        }
+    }).
+
+get_limit_withdrawal(Cash, WalletID, DestinationID) ->
    {ok, WalletMachine} = ff_wallet_machine:get(WalletID),
    Wallet = ff_wallet_machine:wallet(WalletMachine),
    WalletAccount = ff_wallet:account(Wallet),
    {ok, SenderSt} = ff_identity_machine:get(ff_account:identity(WalletAccount)),
    SenderIdentity = ff_identity_machine:identity(SenderSt),

-    Withdrawal = #wthd_domain_Withdrawal{
+    #wthd_domain_Withdrawal{
        created_at = ff_codec:marshal(timestamp_ms, ff_time:now()),
        body = ff_dmsl_codec:marshal(cash, Cash),
        destination = ff_adapter_withdrawal_codec:marshal(resource, get_destination_resource(DestinationID)),
@ -251,7 +370,10 @@ get_limit_amount(Cash, WalletID, DestinationID, LimitID, C) ->
            id => ff_identity:id(SenderIdentity),
            owner_id => ff_identity:party(SenderIdentity)
        })
-    },
+    }.
+
+get_limit_amount(Cash, WalletID, DestinationID, LimitID, C) ->
+    Withdrawal = get_limit_withdrawal(Cash, WalletID, DestinationID),
    ff_limiter_helper:get_limit_amount(LimitID, Withdrawal, C).

 get_destination_resource(DestinationID) ->
@ -261,15 +383,15 @@ get_destination_resource(DestinationID) ->
    Resource.

 prepare_standard_environment({_Amount, Currency} = WithdrawalCash, C) ->
-    Party = create_party(C),
-    IdentityID = create_person_identity(Party, C),
+    PartyID = ct_helper:cfg('$party', C),
+    IdentityID = create_person_identity(PartyID, C),
    WalletID = create_wallet(IdentityID, <<"My wallet">>, Currency, C),
    ok = await_wallet_balance({0, Currency}, WalletID),
    DestinationID = create_destination(IdentityID, Currency, C),
    ok = set_wallet_balance(WithdrawalCash, WalletID),
    #{
        identity_id => IdentityID,
-        party_id => Party,
+        party_id => PartyID,
        wallet_id => WalletID,
        destination_id => DestinationID
    }.
@ -299,6 +421,16 @@ await_final_withdrawal_status(WithdrawalID) ->
    ),
    get_withdrawal_status(WithdrawalID).

+await_withdrawal_activity(Activity, WithdrawalID) ->
+    ct_helper:await(
+        Activity,
+        fun() ->
+            {ok, Machine} = ff_withdrawal_machine:get(WithdrawalID),
+            ff_withdrawal:activity(ff_withdrawal_machine:withdrawal(Machine))
+        end,
+        genlib_retry:linear(20, 1000)
+    ).
+
 create_party(_C) ->
    ID = genlib:bsuuid(),
    _ = ff_party:create(ID),
--- a/apps/fistful/src/ff_postings_transfer.erl
+++ b/apps/fistful/src/ff_postings_transfer.erl
@ -57,7 +57,6 @@
 %% Internal types

 -type id() :: ff_accounting:id().
-type account() :: ff_account:account().

 %%

@ -177,7 +176,7 @@ cancel(#{status := Status}) ->

 %%

-spec apply_event(event(), ff_maybe:maybe(account())) -> account().
+-spec apply_event(event(), ff_maybe:maybe(transfer())) -> transfer().
 apply_event({created, Transfer}, undefined) ->
    Transfer;
 apply_event({status_changed, S}, Transfer) ->
--- a/apps/fistful/src/ff_routing_rule.erl
+++ b/apps/fistful/src/ff_routing_rule.erl
@ -6,11 +6,18 @@
 -export([gather_routes/4]).
 -export([log_reject_context/1]).

+%% Accessors
+
+-export([provider_id/1]).
+-export([terminal_id/1]).
+
 -type payment_institution() :: ff_payment_institution:payment_institution().
 -type routing_ruleset_ref() :: dmsl_domain_thrift:'RoutingRulesetRef'().
 -type provider_ref() :: dmsl_domain_thrift:'ProviderRef'().
 -type provider() :: dmsl_domain_thrift:'Provider'().
 -type terminal_ref() :: dmsl_domain_thrift:'TerminalRef'().
+-type provider_id() :: dmsl_domain_thrift:'ObjectID'().
+-type terminal_id() :: dmsl_domain_thrift:'ObjectID'().
 -type priority() :: integer().
 -type weight() :: integer().
 -type varset() :: ff_varset:varset().
@ -33,6 +40,7 @@

 -type reject_context() :: #{
    varset := varset(),
+    accepted_routes := [route()],
    rejected_routes := [rejected_route()]
 }.

@ -42,12 +50,23 @@

 -import(ff_pipeline, [do/1, unwrap/1]).

+%% Accessors
+
+-spec provider_id(route()) -> provider_id().
+provider_id(#{provider_ref := #domain_ProviderRef{id = ProviderID}}) ->
+    ProviderID.
+
+-spec terminal_id(route()) -> terminal_id().
+terminal_id(#{terminal_ref := #domain_TerminalRef{id = TerminalID}}) ->
+    TerminalID.
+
 %%

 -spec new_reject_context(varset()) -> reject_context().
 new_reject_context(VS) ->
    #{
        varset => VS,
+        accepted_routes => [],
        rejected_routes => []
    }.

@ -56,7 +75,10 @@ gather_routes(PaymentInstitution, RoutingRuleTag, VS, Revision) ->
    RejectContext = new_reject_context(VS),
    case do_gather_routes(PaymentInstitution, RoutingRuleTag, VS, Revision) of
        {ok, {AcceptedRoutes, RejectedRoutes}} ->
-            {AcceptedRoutes, RejectContext#{rejected_routes => RejectedRoutes}};
+            {AcceptedRoutes, RejectContext#{
+                accepted_routes => AcceptedRoutes,
+                rejected_routes => RejectedRoutes
+            }};
        {error, misconfiguration} ->
            logger:warning("Routing rule misconfiguration. Varset:~n~p", [VS]),
            {[], RejectContext}
@ -155,18 +177,14 @@ make_route(Candidate, Revision) ->

 -spec log_reject_context(reject_context()) -> ok.
 log_reject_context(RejectContext) ->
-    Level = warning,
-    RejectReason = unknown,
    _ = logger:log(
-        Level,
-        "No route found, reason = ~p, varset: ~p",
-        [RejectReason, maps:get(varset, RejectContext)],
-        logger:get_process_metadata()
-    ),
-    _ = logger:log(
-        Level,
-        "No route found, reason = ~p, rejected routes: ~p",
-        [RejectReason, maps:get(rejected_routes, RejectContext)],
+        info,
+        "Routing reject context: rejected routes: ~p, accepted routes: ~p, varset: ~p",
+        [
+            maps:get(rejected_routes, RejectContext),
+            maps:get(accepted_routes, RejectContext),
+            maps:get(varset, RejectContext)
+        ],
        logger:get_process_metadata()
    ),
    ok.
--- a/elvis.config
+++ b/elvis.config
@ -39,12 +39,19 @@
                    {elvis_text_style, line_length, #{limit => 120}},
                    {elvis_style, nesting_level, #{level => 3}},
                    {elvis_style, no_if_expression, disable},
-                    {elvis_style, invalid_dynamic_call, #{ignore => [ff_ct_provider_handler]}},
+                    {elvis_style, invalid_dynamic_call, #{
+                        ignore => [
+                            ff_ct_provider_handler,
+                            ff_ct_barrier,
+                            ff_ct_machine
+                        ]
+                    }},
                    % We want to use `ct:pal/2` and friends in test code.
                    {elvis_style, no_debug_call, disable},
                    % Assert macros can trigger use of ignored binding, yet we want them for better
                    % readability.
                    {elvis_style, used_ignored_variable, disable},
+                    {elvis_style, state_record_and_type, disable},
                    % Tests are usually more comprehensible when a bit more verbose.
                    {elvis_style, dont_repeat_yourself, #{min_complexity => 50}},
                    {elvis_style, god_modules, disable}