ED-114: gunner metrics via hay (#26)

2024-11-06 02:15:18 +00:00 · 2021-07-21 14:17:57 +03:00 · 2021-07-21 14:17:57 +03:00 · e15446c301
commit e15446c301
parent 7ff60dc174
7 changed files with 590 additions and 46 deletions
--- a/config/sys.config
+++ b/config/sys.config
@ -29,6 +29,7 @@
            request_timeout => 1000,
            %% Pool options, see gunner_pool:pool_opts()
            pool_opts => #{
+                event_handler => {bouncer_gunner_metrics_event_h, #{}},
                cleanup_interval => 1000,
                max_connection_load => 1,
                max_connection_idle_age => 2000,
@ -81,15 +82,15 @@
        ]}
    ]},

-    % {how_are_you, [
-    %     {metrics_publishers, [
-    %         {hay_statsd_publisher, #{
-    %             key_prefix => <<"bender.">>,
-    %             host => "localhost",
-    %             port => 8125
-    %         }}
-    %     ]}
-    % ]},
+    {how_are_you, [
+        {metrics_publishers, [
+            {hay_statsd_publisher, #{
+                key_prefix => <<"bouncer.">>,
+                host => "localhost",
+                port => 8125
+            }}
+        ]}
+    ]},

    {scoper, [
        {storage, scoper_storage_logger}
--- a/rebar.config
+++ b/rebar.config
@ -77,7 +77,10 @@
                    % Implements parts of logger duties, including message formatting.
                    bouncer_audit_log
                ]
-            }}
+            }},
+            % Readable code for building bin keys
+            {elvis_style, no_if_expression, #{ignore => [bouncer_gunner_metrics_event_h]}},
+            {elvis_style, macro_names, #{ignore => [bouncer_gunner_metrics_event_h]}}
        ]
    },
    #{
--- a/rebar.lock
+++ b/rebar.lock
@ -2,10 +2,10 @@
 [{<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},2},
 {<<"bouncer_proto">>,
  {git,"git@github.com:rbkmoney/bouncer-proto.git",
-       {ref,"7dee26e77ad3cc64c7b09350ee54ee3ab1e1ca34"}},
+       {ref,"60cedce6c0147558e3ba322387151d23b001d68b"}},
  0},
- {<<"cache">>,{pkg,<<"cache">>,<<"2.2.0">>},1},
- {<<"certifi">>,{pkg,<<"certifi">>,<<"2.5.1">>},2},
+ {<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1},
+ {<<"certifi">>,{pkg,<<"certifi">>,<<"2.6.1">>},2},
 {<<"cg_mon">>,
  {git,"https://github.com/rbkmoney/cg_mon.git",
       {ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}},
@ -14,7 +14,7 @@
 {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.9.1">>},1},
 {<<"erl_health">>,
  {git,"https://github.com/rbkmoney/erlang-health.git",
-       {ref,"982af88738ca062eea451436d830eef8c1fbe3f9"}},
+       {ref,"5958e2f35cd4d09f40685762b82b82f89b4d9333"}},
  0},
 {<<"folsom">>,
  {git,"https://github.com/folsom-project/folsom.git",
@ -22,28 +22,28 @@
  1},
 {<<"genlib">>,
  {git,"https://github.com/rbkmoney/genlib.git",
-       {ref,"7637d915c4c769f7f45c99f8688b17922e801027"}},
+       {ref,"3e1776536802739d8819351b15d54ec70568aba7"}},
  0},
- {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1},
+ {<<"gproc">>,{pkg,<<"gproc">>,<<"0.9.0">>},1},
 {<<"gun">>,
  {git,"https://github.com/ninenines/gun.git",
-       {ref,"fe25965f3a2f1347529fec8c7afa981313378e31"}},
+       {ref,"f9175998687678e227bdd49669e2d83f0648fa57"}},
  0},
 {<<"gunner">>,
  {git,"git@github.com:rbkmoney/gunner.git",
-       {ref,"fe2a29fba781be199aee9d4c408dd4662f52d474"}},
+       {ref,"5d7acae56b53ca8c2f846dd0c9fc3f24f6e35c84"}},
  0},
- {<<"hackney">>,{pkg,<<"hackney">>,<<"1.15.2">>},1},
+ {<<"hackney">>,{pkg,<<"hackney">>,<<"1.17.4">>},1},
 {<<"how_are_you">>,
  {git,"https://github.com/rbkmoney/how_are_you.git",
-       {ref,"8f11d17eeb6eb74096da7363a9df272fd3099718"}},
+       {ref,"29f9d3d7c35f7a2d586c8571f572838df5ec91dd"}},
  0},
- {<<"idna">>,{pkg,<<"idna">>,<<"6.0.0">>},2},
+ {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},2},
 {<<"jesse">>,{pkg,<<"jesse">>,<<"1.5.5">>},0},
 {<<"jsx">>,{pkg,<<"jsx">>,<<"3.0.0">>},0},
 {<<"logger_logstash_formatter">>,
  {git,"https://github.com/rbkmoney/logger_logstash_formatter.git",
-       {ref,"013525713c0d2a1ff0a0daff5db872793f5e6012"}},
+       {ref,"87e52c755cf9e64d651e3ddddbfcd2ccd1db79db"}},
  0},
 {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2},
 {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},2},
@ -51,69 +51,69 @@
  {git,"git@github.com:rbkmoney/org-management-proto.git",
       {ref,"06c5c8430e445cb7874e54358e457cbb5697fc32"}},
  0},
- {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3},
+ {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2},
 {<<"ranch">>,{pkg,<<"ranch">>,<<"1.7.1">>},1},
 {<<"recon">>,{pkg,<<"recon">>,<<"2.5.1">>},0},
 {<<"rfc3339">>,{pkg,<<"rfc3339">>,<<"0.2.2">>},1},
 {<<"scoper">>,
  {git,"https://github.com/rbkmoney/scoper.git",
-       {ref,"23b1625bf2c6940a56cfc30389472a5e384a229f"}},
+       {ref,"89a973bf3cedc5a48c9fd89d719d25e79fe10027"}},
  0},
 {<<"snowflake">>,
  {git,"https://github.com/rbkmoney/snowflake.git",
-       {ref,"7f379ad5e389e1c96389a8d60bae8117965d6a6d"}},
+       {ref,"de159486ef40cec67074afe71882bdc7f7deab72"}},
  1},
- {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.5">>},2},
+ {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},2},
 {<<"thrift">>,
  {git,"https://github.com/rbkmoney/thrift_erlang.git",
       {ref,"846a0819d9b6d09d0c31f160e33a78dbad2067b4"}},
  0},
- {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.4.1">>},3},
+ {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2},
 {<<"woody">>,
  {git,"https://github.com/rbkmoney/woody_erlang.git",
-       {ref,"3b766220542ea49965e99a4c4352727d6e9e0cbf"}},
+       {ref,"d9fca6da55a46e39bdb7ad2c0dba0b7205a7e70b"}},
  0},
 {<<"woody_user_identity">>,
  {git,"https://github.com/rbkmoney/woody_erlang_user_identity.git",
-       {ref,"d6d8c570e6aaae7adfd2737e47007da43728c1b3"}},
+       {ref,"a480762fea8d7c08f105fb39ca809482b6cb042e"}},
  0}]}.
 [
 {pkg_hash,[
 {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>},
- {<<"cache">>, <<"3C11DBF4CD8FCD5787C95A5FB2A04038E3729CFCA0386016EEA8C953AB48A5AB">>},
- {<<"certifi">>, <<"867CE347F7C7D78563450A18A6A28A8090331E77FA02380B4A21962A65D36EE5">>},
+ {<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>},
+ {<<"certifi">>, <<"DBAB8E5E155A0763EEA978C913CA280A6B544BFA115633FA20249C3D396D9493">>},
 {<<"cowboy">>, <<"F3DC62E35797ECD9AC1B50DB74611193C29815401E53BAC9A5C0577BD7BC667D">>},
 {<<"cowlib">>, <<"61A6C7C50CF07FDD24B2F45B89500BB93B6686579B069A89F88CB211E1125C78">>},
- {<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>},
- {<<"hackney">>, <<"07E33C794F8F8964EE86CEBEC1A8ED88DB5070E52E904B8F12209773C1036085">>},
- {<<"idna">>, <<"689C46CBCDF3524C44D5F3DDE8001F364CD7608A99556D8FBD8239A5798D4C10">>},
+ {<<"gproc">>, <<"853CCB7805E9ADA25D227A157BA966F7B34508F386A3E7E21992B1B484230699">>},
+ {<<"hackney">>, <<"99DA4674592504D3FB0CFEF0DB84C3BA02B4508BAE2DFF8C0108BAA0D6E0977C">>},
+ {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>},
 {<<"jesse">>, <<"ECFD2C1634C49052CA907B4DFDE1D1F44B7FD7862D933F4590807E42759B8072">>},
 {<<"jsx">>, <<"20A170ABD4335FC6DB24D5FAD1E5D677C55DADF83D1B20A8A33B5FE159892A39">>},
 {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>},
 {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>},
- {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>},
+ {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>},
 {<<"ranch">>, <<"6B1FAB51B49196860B733A49C07604465A47BDB78AA10C1C16A3D199F7F8C881">>},
 {<<"recon">>, <<"430FFA60685AC1EFDFB1FE4C97B8767C92D0D92E6E7C3E8621559BA77598678A">>},
 {<<"rfc3339">>, <<"1552DF616ACA368D982E9F085A0E933B6688A3F4938A671798978EC2C0C58730">>},
- {<<"ssl_verify_fun">>, <<"6EAF7AD16CB568BB01753DBBD7A95FF8B91C7979482B95F38443FE2C8852A79B">>},
- {<<"unicode_util_compat">>, <<"D869E4C68901DD9531385BB0C8C40444EBF624E60B6962D95952775CAC5E90CD">>}]},
+ {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>},
+ {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]},
 {pkg_hash_ext,[
 {<<"bear">>, <<"534217DCE6A719D59E54FB0EB7A367900DBFC5F85757E8C1F94269DF383F6D9B">>},
- {<<"cache">>, <<"3E7D6706DE5DF76C4D71C895B4BE62B01C3DE6EDB63197035E465C3BCE63F19B">>},
- {<<"certifi">>, <<"805ABD97539CAF89EC6D4732C91E62BA9DA0CDA51AC462380BBD28EE697A8C42">>},
+ {<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>},
+ {<<"certifi">>, <<"524C97B4991B3849DD5C17A631223896272C6B0AF446778BA4675A1DFF53BB7E">>},
 {<<"cowboy">>, <<"4643E4FBA74AC96D4D152C75803DE6FAD0B3FA5DF354C71AFDD6CBEEB15FAC8A">>},
 {<<"cowlib">>, <<"E4175DC240A70D996156160891E1C62238EDE1729E45740BDD38064DAD476170">>},
- {<<"gproc">>, <<"580ADAFA56463B75263EF5A5DF4C86AF321F68694E7786CB057FD805D1E2A7DE">>},
- {<<"hackney">>, <<"E0100F8EF7D1124222C11AD362C857D3DF7CB5F4204054F9F0F4A728666591FC">>},
- {<<"idna">>, <<"4BDD305EB64E18B0273864920695CB18D7A2021F31A11B9C5FBCD9A253F936E2">>},
+ {<<"gproc">>, <<"587E8AF698CCD3504CF4BA8D90F893EDE2B0F58CABB8A916E2BF9321DE3CF10B">>},
+ {<<"hackney">>, <<"DE16FF4996556C8548D512F4DBE22DD58A587BF3332E7FD362430A7EF3986B16">>},
+ {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>},
 {<<"jesse">>, <<"38D9C4743F97F073D0486CF6626AB370C842D94EE207BB3574052845979A7C0D">>},
 {<<"jsx">>, <<"37BECA0435F5CA8A2F45F76A46211E76418FBEF80C36F0361C249FC75059DC6D">>},
 {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>},
 {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>},
- {<<"parse_trans">>, <<"17EF63ABDE837AD30680EA7F857DD9E7CED9476CDD7B0394432AF4BFC241B960">>},
+ {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>},
 {<<"ranch">>, <<"451D8527787DF716D99DC36162FCA05934915DB0B6141BBDAC2EA8D3C7AFC7D7">>},
 {<<"recon">>, <<"5721C6B6D50122D8F68CCCAC712CAA1231F97894BAB779EFF5FF0F886CB44648">>},
 {<<"rfc3339">>, <<"986D7F9BAC6891AA4D5051690058DE4E623245620BBEADA7F239F85C4DF8F23C">>},
- {<<"ssl_verify_fun">>, <<"13104D7897E38ED7F044C4DE953A6C28597D1C952075EB2E328BC6D6F2BFC496">>},
- {<<"unicode_util_compat">>, <<"1D1848C40487CDB0B30E8ED975E34E025860C02E419CB615D255849F3427439D">>}]}
+ {<<"ssl_verify_fun">>, <<"BDB0D2471F453C88FF3908E7686F86F9BE327D065CC1EC16FA4540197EA04680">>},
+ {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]}
 ].
--- a/src/bouncer.app.src
+++ b/src/bouncer.app.src
@ -15,7 +15,8 @@
        bouncer_proto,
        org_management_proto,
        erl_health,
-        gunner
+        gunner,
+        how_are_you
    ]},
    {env, []},
    {licenses, ["Apache 2.0"]}
--- a/src/bouncer_gunner_metrics_event_h.erl
+++ b/src/bouncer_gunner_metrics_event_h.erl
@ -0,0 +1,255 @@
+-module(bouncer_gunner_metrics_event_h).
+
+-include_lib("gunner/include/gunner_events.hrl").
+
+%% gunner_event_h behaviour
+
+-behaviour(gunner_event_h).
+
+-export([handle_event/2]).
+
+%% Internal types
+
+-type state() :: #{timer_id() => timer_start_ts()}.
+
+-type timer_id() :: term().
+-type timer_start_ts() :: non_neg_integer().
+-type metric_key() :: how_are_you:metric_key().
+
+%%
+%% gunner_event_h behaviour
+%%
+
+-spec handle_event(gunner_event_h:event(), state()) -> state().
+handle_event(Event, State) ->
+    try
+        ok = create_metric(Event),
+        process_timers(Event, State)
+    catch
+        throw:{stop_timer_failed, {no_timer, TimerKey}} ->
+            _ = logger:error("Tried to stop a non-existant timer: ~p", [TimerKey]),
+            State
+    end.
+
+%%
+%% Metrics
+%%
+
+-define(METRIC_KEY(Tag, Content), [gunner, Tag, Content]).
+-define(METRIC_KEY(Tag, Content, GroupID), [gunner, Tag, Content, group, encode_group(GroupID)]).
+
+-define(METRIC_DURATION(Key), ?METRIC_KEY(duration, Key)).
+-define(METRIC_ACQUIRE(Evt, GroupID), ?METRIC_KEY(acquire, Evt, GroupID)).
+-define(METRIC_FREE(Evt, GroupID), ?METRIC_KEY(free, Evt, GroupID)).
+-define(METRIC_CONNECTION_COUNT(Category), ?METRIC_KEY(connections, Category)).
+-define(METRIC_CONNECTION_COUNT(Category, GroupID), ?METRIC_KEY(connections, Category, GroupID)).
+-define(METRIC_CONNECTION(Evt, GroupID), ?METRIC_KEY(connection, Evt, GroupID)).
+
+-define(TIMER_KEY(Tag, Content), {Tag, Content}).
+-define(TIMER_CLEANUP, cleanup).
+-define(TIMER_ACQUIRE(GroupID, ClientID),
+    ?TIMER_KEY(acquire, {GroupID, ClientID})
+).
+-define(TIMER_FREE(ConnectionID, GroupID, ClientID),
+    ?TIMER_KEY(free, {ConnectionID, GroupID, ClientID})
+).
+-define(TIMER_CONNECTION_INIT(ConnectionID, GroupID),
+    ?TIMER_KEY(connection_init, {ConnectionID, GroupID})
+).
+
+process_timers(Event, State) ->
+    case is_timed_event(Event) of
+        {true, {start, TimerID}} ->
+            start_timer(TimerID, State);
+        {true, {finish, TimerID, MetricID}} ->
+            {Elapsed, State1} = stop_timer(TimerID, State),
+            ok = create_duration(MetricID, Elapsed),
+            State1;
+        false ->
+            State
+    end.
+
+is_timed_event(#gunner_acquire_started_event{group_id = GroupID, client = ClientID}) ->
+    {true, {start, ?TIMER_ACQUIRE(GroupID, ClientID)}};
+is_timed_event(#gunner_acquire_finished_event{group_id = GroupID, client = ClientID}) ->
+    {true, {finish, ?TIMER_ACQUIRE(GroupID, ClientID), ?METRIC_DURATION(acquire)}};
+is_timed_event(#gunner_free_started_event{
+    group_id = GroupID,
+    client = ClientID,
+    connection = ConnectionID
+}) ->
+    {true, {start, ?TIMER_FREE(ConnectionID, GroupID, ClientID)}};
+is_timed_event(#gunner_free_finished_event{
+    group_id = GroupID,
+    client = ClientID,
+    connection = ConnectionID
+}) ->
+    {true, {finish, ?TIMER_FREE(ConnectionID, GroupID, ClientID), ?METRIC_DURATION(free)}};
+is_timed_event(#gunner_cleanup_started_event{}) ->
+    {true, {start, ?TIMER_CLEANUP}};
+is_timed_event(#gunner_cleanup_finished_event{}) ->
+    {true, {finish, ?TIMER_CLEANUP, ?METRIC_DURATION(cleanup)}};
+is_timed_event(
+    #gunner_connection_init_started_event{
+        group_id = GroupID,
+        connection = ConnectionID
+    }
+) ->
+    {true, {start, ?TIMER_CONNECTION_INIT(GroupID, ConnectionID)}};
+is_timed_event(
+    #gunner_connection_init_finished_event{
+        group_id = GroupID,
+        connection = ConnectionID
+    }
+) ->
+    {true,
+        {finish, ?TIMER_CONNECTION_INIT(GroupID, ConnectionID), ?METRIC_DURATION(connection_init)}};
+is_timed_event(_) ->
+    false.
+
+create_metric(#gunner_pool_init_event{pool_opts = PoolOpts}) ->
+    ok = create_gauge(?METRIC_CONNECTION_COUNT([config, max]), maps:get(max_size, PoolOpts)),
+    create_gauge(?METRIC_CONNECTION_COUNT([config, min]), maps:get(min_size, PoolOpts));
+create_metric(#gunner_pool_terminate_event{}) ->
+    ok;
+%%
+create_metric(#gunner_acquire_started_event{group_id = GroupID}) ->
+    counter_inc(?METRIC_ACQUIRE(started, GroupID));
+create_metric(#gunner_acquire_finished_event{group_id = GroupID, result = Result}) ->
+    counter_inc(?METRIC_ACQUIRE([finished, encode_result(Result)], GroupID));
+%%
+create_metric(#gunner_connection_locked_event{group_id = GroupID}) ->
+    counter_inc(?METRIC_CONNECTION_COUNT(locked, GroupID));
+create_metric(#gunner_connection_unlocked_event{group_id = GroupID}) ->
+    counter_dec(?METRIC_CONNECTION_COUNT(locked, GroupID));
+%%
+create_metric(#gunner_free_started_event{group_id = GroupID}) ->
+    counter_inc(?METRIC_FREE(started, GroupID));
+create_metric(#gunner_free_finished_event{group_id = GroupID}) ->
+    counter_inc(?METRIC_FREE(finished, GroupID));
+create_metric(#gunner_free_error_event{}) ->
+    counter_inc([gunner, free, error]);
+%%
+create_metric(#gunner_cleanup_started_event{}) ->
+    ok;
+create_metric(#gunner_cleanup_finished_event{active_connections = Active}) ->
+    create_gauge(?METRIC_CONNECTION_COUNT(active), Active);
+%%
+create_metric(#gunner_client_down_event{}) ->
+    counter_inc([gunner, client, down]);
+%%
+create_metric(#gunner_connection_init_started_event{group_id = GroupID}) ->
+    counter_inc(?METRIC_CONNECTION([init, started], GroupID));
+%%
+create_metric(#gunner_connection_init_finished_event{group_id = GroupID, result = ok}) ->
+    ok = counter_inc(?METRIC_CONNECTION([init, finished, ok], GroupID)),
+    counter_inc(?METRIC_CONNECTION_COUNT(total, GroupID));
+create_metric(#gunner_connection_init_finished_event{group_id = GroupID, result = _}) ->
+    counter_inc(?METRIC_CONNECTION([init, finished, error], GroupID));
+%%
+create_metric(#gunner_connection_down_event{group_id = GroupID}) ->
+    ok = counter_inc(?METRIC_CONNECTION(down, GroupID)),
+    counter_dec(?METRIC_CONNECTION_COUNT(total, GroupID)).
+
+%%
+%% Internal
+%%
+
+encode_group({IP, Port}) when is_tuple(IP) ->
+    encode_group({inet:ntoa(IP), Port});
+encode_group({Host, Port}) when is_list(Host) ->
+    encode_group(list_to_binary(Host), integer_to_binary(Port)).
+
+encode_group(Host, Port) ->
+    <<Host/binary, ":", Port/binary>>.
+
+encode_result(ok) ->
+    ok;
+encode_result({error, pool_unavailable}) ->
+    pool_unavailable;
+encode_result({error, {connection_failed, _Reason}}) ->
+    connection_failed.
+
+start_timer(TimerKey, State) ->
+    Time = erlang:monotonic_time(microsecond),
+    State#{TimerKey => Time}.
+
+stop_timer(TimerKey, State) ->
+    Time = erlang:monotonic_time(microsecond),
+    case maps:get(TimerKey, State, undefined) of
+        TimeStarted when TimeStarted =/= undefined ->
+            {Time - TimeStarted, maps:remove(TimerKey, State)};
+        undefined ->
+            throw({stop_timer_failed, {no_timer, TimerKey}})
+    end.
+
+%%
+%% Hay utils
+%%
+
+-spec counter_inc(metric_key()) -> ok.
+counter_inc(Key) ->
+    create_counter(Key, 1).
+
+-spec counter_dec(metric_key()) -> ok.
+counter_dec(Key) ->
+    create_counter(Key, -1).
+
+-spec create_counter(metric_key(), integer()) -> ok.
+create_counter(Key, Number) ->
+    create_metric(counter, Key, Number).
+
+-spec create_gauge(metric_key(), non_neg_integer()) -> ok.
+create_gauge(Key, Number) ->
+    create_metric(gauge, Key, Number).
+
+-spec create_duration(metric_key(), non_neg_integer()) -> ok.
+create_duration(KeyPrefix, Duration) ->
+    BinKey = build_bin_key(Duration),
+    create_metric(counter, [KeyPrefix, BinKey], 1).
+
+-spec create_metric(atom(), metric_key(), integer()) -> ok.
+create_metric(Type, Key, Value) ->
+    Metric = how_are_you:metric_construct(Type, Key, Value),
+    how_are_you:metric_push(Metric).
+
+%%
+
+-define(_10US, "10μs").
+-define(_50US, "50μs").
+-define(_100US, "100μs").
+-define(_500US, "500μs").
+-define(_1MS, "1ms").
+-define(_5MS, "5ms").
+-define(_10MS, "10ms").
+-define(_25MS, "25ms").
+-define(_50MS, "50ms").
+-define(_100MS, "100ms").
+-define(_250MS, "250ms").
+-define(_500MS, "500ms").
+-define(_1S, "1s").
+-define(_5S, "5s").
+
+-define(BETWEEN(Bin1, Bin2), <<"from_", Bin1, "_to_", Bin2>>).
+-define(LT(Bin), <<"less_than_", Bin>>).
+-define(GT(Bin), <<"greater_than_", Bin>>).
+
+-spec build_bin_key(Value :: number()) -> metric_key().
+build_bin_key(Value) ->
+    if
+        Value < 10 -> ?LT(?_10US);
+        Value < 50 -> ?BETWEEN(?_10US, ?_50US);
+        Value < 100 -> ?BETWEEN(?_50US, ?_100US);
+        Value < 500 -> ?BETWEEN(?_100US, ?_100US);
+        Value < 1000 -> ?BETWEEN(?_500US, ?_1MS);
+        Value < 5 * 1000 -> ?BETWEEN(?_1MS, ?_5MS);
+        Value < 10 * 1000 -> ?BETWEEN(?_5MS, ?_10MS);
+        Value < 25 * 1000 -> ?BETWEEN(?_10MS, ?_25MS);
+        Value < 50 * 1000 -> ?BETWEEN(?_25MS, ?_50MS);
+        Value < 100 * 1000 -> ?BETWEEN(?_50MS, ?_100MS);
+        Value < 250 * 1000 -> ?BETWEEN(?_100MS, ?_250MS);
+        Value < 500 * 1000 -> ?BETWEEN(?_250MS, ?_500MS);
+        Value < 1000 * 1000 -> ?BETWEEN(?_500MS, ?_1S);
+        Value < 5 * 1000 * 1000 -> ?BETWEEN(?_1S, ?_5S);
+        true -> ?GT(?_5S)
+    end.
--- a/test/bouncer_gunner_metrics_SUITE.erl
+++ b/test/bouncer_gunner_metrics_SUITE.erl
@ -0,0 +1,215 @@
+-module(bouncer_gunner_metrics_SUITE).
+
+-include_lib("stdlib/include/assert.hrl").
+
+-export([all/0]).
+-export([groups/0]).
+-export([init_per_suite/1]).
+-export([end_per_suite/1]).
+-export([init_per_group/2]).
+-export([end_per_group/2]).
+-export([init_per_testcase/2]).
+-export([end_per_testcase/2]).
+
+-type config() :: ct_helper:config().
+-type group_name() :: atom().
+-type test_case_name() :: atom().
+
+-export([basic_metrics_test/1]).
+
+%%
+
+-include_lib("bouncer_proto/include/bouncer_decisions_thrift.hrl").
+
+-define(CONFIG(Key, C), (element(2, lists:keyfind(Key, 1, C)))).
+-define(OPA_HOST, "opa").
+-define(OPA_ENDPOINT, {?OPA_HOST, 8181}).
+
+-spec all() -> [atom()].
+all() ->
+    [
+        basic_metrics_test
+    ].
+
+-spec groups() -> [{group_name(), list(), [test_case_name()]}].
+groups() ->
+    [].
+
+-spec init_per_suite(config()) -> config().
+init_per_suite(C) ->
+    Apps =
+        genlib_app:start_application(woody) ++
+            genlib_app:start_application_with(scoper, [
+                {storage, scoper_storage_logger}
+            ]),
+    [{suite_apps, Apps} | C].
+
+-spec end_per_suite(config()) -> ok.
+end_per_suite(C) ->
+    genlib_app:stop_unload_applications(?CONFIG(suite_apps, C)).
+
+-spec init_per_group(group_name(), config()) -> config().
+init_per_group(Name, C) ->
+    [{groupname, Name} | C].
+
+-spec end_per_group(group_name(), config()) -> _.
+end_per_group(_Name, _C) ->
+    ok.
+
+-spec init_per_testcase(atom(), config()) -> config().
+init_per_testcase(Name, C) ->
+    start_bouncer([], [{testcase, Name} | C]).
+
+-spec end_per_testcase(atom(), config()) -> config().
+end_per_testcase(_Name, C) ->
+    stop_bouncer(C).
+
+%%
+
+-spec basic_metrics_test(config()) -> _.
+basic_metrics_test(C) ->
+    _ = call_judge("service/authz/api", #bdcs_Context{fragments = #{}}, mk_client(C)),
+    _ = timer:sleep(100),
+    ?assertEqual(25, ct_hay_publisher:get_metric([gunner, connections, config, max])),
+    ?assertEqual(5, ct_hay_publisher:get_metric([gunner, connections, config, min])),
+    ?assertEqual(
+        1,
+        ct_hay_publisher:get_metric([
+            gunner,
+            acquire,
+            started,
+            group,
+            encode_group(?OPA_ENDPOINT)
+        ])
+    ),
+    ?assertEqual(
+        1,
+        ct_hay_publisher:get_metric([
+            gunner,
+            connection,
+            init,
+            started,
+            group,
+            encode_group(?OPA_ENDPOINT)
+        ])
+    ),
+    ?assertEqual(
+        1,
+        ct_hay_publisher:get_metric([
+            gunner,
+            connection,
+            init,
+            finished,
+            ok,
+            group,
+            encode_group(?OPA_ENDPOINT)
+        ])
+    ),
+    ?assertEqual(
+        1,
+        ct_hay_publisher:get_metric([
+            gunner,
+            acquire,
+            finished,
+            ok,
+            group,
+            encode_group(?OPA_ENDPOINT)
+        ])
+    ),
+    ?assertEqual(
+        1,
+        ct_hay_publisher:get_metric([
+            gunner,
+            connections,
+            total,
+            group,
+            encode_group(?OPA_ENDPOINT)
+        ])
+    ).
+
+%%
+
+mk_client(C) ->
+    WoodyCtx = woody_context:new(genlib:to_binary(?CONFIG(testcase, C))),
+    ServiceURLs = ?CONFIG(service_urls, C),
+    {WoodyCtx, ServiceURLs}.
+
+call_judge(RulesetID, Context, Client) ->
+    call(arbiter, 'Judge', {genlib:to_binary(RulesetID), Context}, Client).
+
+call(ServiceName, Fn, Args, {WoodyCtx, ServiceURLs}) ->
+    Service = get_service_spec(ServiceName),
+    Opts = #{
+        url => maps:get(ServiceName, ServiceURLs),
+        event_handler => scoper_woody_event_handler
+    },
+    case woody_client:call({Service, Fn, Args}, Opts, WoodyCtx) of
+        {ok, Response} ->
+            Response;
+        {exception, Exception} ->
+            throw(Exception)
+    end.
+
+get_service_spec(arbiter) ->
+    {bouncer_decisions_thrift, 'Arbiter'}.
+
+%%
+
+start_bouncer(Env, C) ->
+    IP = "127.0.0.1",
+    Port = 8022,
+    ArbiterPath = <<"/v1/arbiter">>,
+    Apps0 = genlib_app:start_application_with(
+        how_are_you,
+        [
+            {metrics_publishers, [ct_hay_publisher]},
+            {metrics_handlers, []}
+        ]
+    ),
+    Apps1 = genlib_app:start_application_with(
+        bouncer,
+        [
+            {ip, IP},
+            {port, Port},
+            {services, #{
+                arbiter => #{path => ArbiterPath}
+            }},
+            {transport_opts, #{
+                max_connections => 1000,
+                num_acceptors => 4
+            }},
+            {opa, #{
+                endpoint => ?OPA_ENDPOINT,
+                pool_opts => #{
+                    event_handler => {bouncer_gunner_metrics_event_h, #{}},
+                    connection_opts => #{
+                        transport => tcp
+                    }
+                }
+            }}
+        ] ++ Env
+    ),
+    Services = #{
+        arbiter => mk_url(IP, Port, ArbiterPath)
+    },
+    [{testcase_apps, Apps0 ++ Apps1}, {service_urls, Services} | C].
+
+mk_url(IP, Port, Path) ->
+    iolist_to_binary(["http://", IP, ":", genlib:to_binary(Port), Path]).
+
+stop_bouncer(C) ->
+    ct_helper:with_config(
+        testcase_apps,
+        C,
+        fun(Apps) -> genlib_app:stop_unload_applications(Apps) end
+    ).
+
+%%
+
+encode_group({IP, Port}) when is_tuple(IP) ->
+    encode_group({inet:ntoa(IP), Port});
+encode_group({Host, Port}) when is_list(Host) ->
+    encode_group(list_to_binary(Host), integer_to_binary(Port)).
+
+encode_group(Host, Port) ->
+    <<Host/binary, ":", Port/binary>>.
--- a/test/ct_hay_publisher.erl
+++ b/test/ct_hay_publisher.erl
@ -0,0 +1,69 @@
+-module(ct_hay_publisher).
+
+-behaviour(hay_metrics_publisher).
+
+%% hay_metrics_publisher callbacks
+-export([init/1]).
+-export([get_interval/1]).
+-export([publish_metrics/2]).
+
+%% API
+-export([get_metric/1]).
+
+%% Types
+
+-type options() :: #{
+    interval => timeout()
+}.
+
+-export_type([options/0]).
+
+%% Internal types
+-define(ETS_NAME, ?MODULE).
+
+-record(state, {
+    interval :: timeout(),
+    ets :: ets:tid() | atom()
+}).
+
+-record(metric, {
+    key :: how_are_you:metric_key(),
+    value :: how_are_you:metric_value()
+}).
+
+-type state() :: #state{}.
+
+%% API
+
+-spec init(options()) -> {ok, state()}.
+init(Options) ->
+    {ok, #state{
+        interval = maps:get(interval, Options, 100),
+        ets = ets:new(?ETS_NAME, [named_table, set, {keypos, #metric.key}])
+    }}.
+
+-spec get_interval(state()) -> timeout().
+get_interval(#state{interval = Interval}) ->
+    Interval.
+
+-spec publish_metrics(hay_metrics_publisher:metric_fold(), state()) ->
+    {ok, state()} | {error, Reason :: term()}.
+publish_metrics(Fold, #state{ets = Ets} = State) ->
+    true = Fold(
+        fun(M, _) ->
+            ets:insert(Ets, #metric{key = hay_metrics:key(M), value = hay_metrics:value(M)})
+        end,
+        true
+    ),
+    {ok, State}.
+
+-spec get_metric(how_are_you:metric_key()) -> how_are_you:metric_value() | undefined.
+get_metric(Key) ->
+    % Convert key to hay internal format
+    EKey = hay_metrics:key(how_are_you:metric_construct(gauge, Key, 0)),
+    case ets:lookup(?ETS_NAME, EKey) of
+        [#metric{value = Value}] ->
+            Value;
+        [] ->
+            undefined
+    end.