TD-686: Instruments OTEL tracing (#39)

* TD-686: Instruments OTEL tracing * Adds missing application deps * Wraps swagger operation handler call * Bumps logger_logstash_formatter * Refactors single process span ctx relations * Refactors span relations in-process state * Bumps woody to use generic event handler for otel * Adds scoper meta as span attributes * Bumps deps * Clean up .app.src * Bumps genlib * Migrates back to scoper' woody event handler * Bumps scoper * Removes obsolete meta sync * Adds OTEL propagation handling upon JSON API request
2024-11-06 01:55:20 +00:00 · 2023-10-26 13:34:48 +03:00 · 2023-10-26 13:34:48 +03:00 · 2bed81a844
commit 2bed81a844
parent a1ef52f641
4 changed files with 79 additions and 20 deletions
--- a/apps/capi/src/capi.app.src
+++ b/apps/capi/src/capi.app.src
@ -29,7 +29,10 @@
        token_keeper_client,
        party_client,
        payout_manager_proto,
-        magista_proto
+        magista_proto,
+        opentelemetry_api,
+        opentelemetry_exporter,
+        opentelemetry
    ]},
    {env, []}
 ]}.
--- a/apps/capi/src/capi_handler.erl
+++ b/apps/capi/src/capi_handler.erl
@ -1,5 +1,8 @@
 -module(capi_handler).

+-include_lib("opentelemetry_api/include/otel_tracer.hrl").
+-include_lib("opentelemetry_api/include/opentelemetry.hrl").
+
 -behaviour(swag_server_logic_handler).

 -type error_type() :: swag_server_logic_handler:error_type().
@ -63,7 +66,8 @@

 -spec authorize_api_key(operation_id(), swag_server:api_key(), request_context(), handler_opts()) ->
    Result :: false | {true, capi_auth:preauth_context()}.
-authorize_api_key(OperationID, ApiKey, _Context, _HandlerOpts) ->
+authorize_api_key(OperationID, ApiKey, Context, _HandlerOpts) ->
+    ok = set_otel_context(Context),
    %% Since we require the request id field to create a woody context for our trip to token_keeper
    %% it seems it is no longer possible to perform any authorization in this method.
    %% To gain this ability back be would need to rewrite the swagger generator to perform its
@ -71,8 +75,8 @@ authorize_api_key(OperationID, ApiKey, _Context, _HandlerOpts) ->
    %% But since a decent chunk of authorization logic is already defined in the handler function
    %% it is probably easier to move it there in its entirety.
    case capi_auth:preauthorize_api_key(ApiKey) of
-        {ok, Context} ->
-            {true, Context};
+        {ok, Context1} ->
+            {true, Context1};
        {error, Error} ->
            _ = logger:info("API Key preauthorization failed for ~p due to ~p", [OperationID, Error]),
            false
@ -121,8 +125,11 @@ get_handlers() ->
    HandlerOpts :: handler_opts()
 ) -> {ok | error, response()}.
 handle_request(OperationID, Req, SwagContext, HandlerOpts) ->
+    SpanName = <<"server ", (atom_to_binary(OperationID))/binary>>,
+    ?with_span(SpanName, #{kind => ?SPAN_KIND_SERVER}, fun(_SpanCtx) ->
        scoper:scope(swagger, fun() ->
            handle_function_(OperationID, Req, SwagContext, HandlerOpts)
+        end)
    end).

 -spec handle_function_(
@ -293,6 +300,14 @@ process_general_error(Class, Reason, Stacktrace, Req, SwagContext) ->
    ),
    {error, server_error(500)}.

+set_otel_context(#{cowboy_req := Req}) ->
+    Headers = cowboy_req:headers(Req),
+    %% Implicitly puts OTEL context into process dictionary.
+    %% Since cowboy does not reuse process for other requests, we don't care
+    %% about cleaning it up.
+    _OtelCtx = otel_propagator_text_map:extract(maps:to_list(Headers)),
+    ok.
+
 -spec set_context_meta(processing_context()) -> ok.
 set_context_meta(Context) ->
    AuthContext = capi_handler_utils:get_auth_context(Context),
--- a/rebar.config
+++ b/rebar.config
@ -32,7 +32,7 @@
    {jose, {git, "https://github.com/potatosalad/erlang-jose.git", {tag, "1.11.2"}}},
    {genlib, {git, "https://github.com/valitydev/genlib.git", {branch, "master"}}},
    {cowboy_draining_server, {git, "https://github.com/valitydev/cowboy_draining_server.git", {branch, "master"}}},
-    {woody, {git, "https://github.com/valitydev/woody_erlang.git", {branch, "master"}}},
+    {woody, {git, "https://github.com/valitydev/woody_erlang.git", {branch, master}}},
    {woody_user_identity, {git, "https://github.com/valitydev/woody_erlang_user_identity.git", {branch, "master"}}},
    {damsel, {git, "https://github.com/valitydev/damsel.git", {branch, "master"}}},
    {bender_proto, {git, "https://github.com/valitydev/bender-proto.git", {branch, "master"}}},
@ -41,7 +41,7 @@
    {cowboy_cors, {git, "https://github.com/valitydev/cowboy_cors.git", {branch, "master"}}},
    {cowboy_access_log, {git, "https://github.com/valitydev/cowboy_access_log.git", {branch, "master"}}},
    {payproc_errors, {git, "https://github.com/valitydev/payproc-errors-erlang.git", {branch, "master"}}},
-    {scoper, {git, "https://github.com/valitydev/scoper.git", {branch, master}}},
+    {scoper, {git, "https://github.com/valitydev/scoper.git", {branch, "master"}}},
    {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, master}}},
    {lechiffre, {git, "https://github.com/valitydev/lechiffre.git", {branch, master}}},
    {bouncer_proto, {git, "https://github.com/valitydev/bouncer-proto.git", {branch, master}}},
@ -53,7 +53,11 @@
    {magista_proto, {git, "https://github.com/valitydev/magista-proto.git", {branch, master}}},
    %% Libraries generated with swagger-codegen-erlang from valitydev/swag-payments
    {swag_server, {git, "https://github.com/valitydev/swag-payments", {branch, "release/erlang/server/v2"}}},
-    {swag_client, {git, "https://github.com/valitydev/swag-payments", {branch, "release/erlang/client/v2"}}}
+    {swag_client, {git, "https://github.com/valitydev/swag-payments", {branch, "release/erlang/client/v2"}}},
+    %% OpenTelemetry deps
+    {opentelemetry_api, "1.2.1"},
+    {opentelemetry, "1.3.0"},
+    {opentelemetry_exporter, "1.3.0"}
 ]}.

 %% XRef checks
@ -93,7 +97,7 @@
            {prometheus_cowboy, "0.1.8"},
            {recon, "2.3.6"},
            {logger_logstash_formatter,
-                {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "87e52c7"}}},
+                {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}},
            {iosetopts, {git, "https://github.com/valitydev/iosetopts.git", {ref, "edb445c"}}}
        ]},
        {relx, [
@ -105,6 +109,7 @@
                {runtime_tools, load},
                % profiler
                {tools, load},
+                {opentelemetry, temporary},
                logger_logstash_formatter,
                prometheus,
                prometheus_cowboy,
--- a/rebar.lock
+++ b/rebar.lock
@ -1,7 +1,8 @@
 {"1.2.0",
-[{<<"bender_client">>,
+[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},2},
+ {<<"bender_client">>,
  {git,"https://github.com/valitydev/bender-client-erlang.git",
-       {ref,"4e15070a194ed2f3f033891eb2da935982a06c30"}},
+       {ref,"d8837617c8dc36216ce8c4ffc9a56a34e423ca5e"}},
  0},
 {<<"bender_proto">>,
  {git,"https://github.com/valitydev/bender-proto.git",
@ -9,7 +10,7 @@
  0},
 {<<"bouncer_client">>,
  {git,"https://github.com/valitydev/bouncer-client-erlang.git",
-       {ref,"2ce8d4aeeef490bf2a013e7c5a1aee498d5b6178"}},
+       {ref,"4f89634d9a009cac807cf86a0eff9e80d33318ea"}},
  0},
 {<<"bouncer_proto">>,
  {git,"https://github.com/valitydev/bouncer-proto.git",
@ -18,9 +19,10 @@
 {<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1},
 {<<"certifi">>,{pkg,<<"certifi">>,<<"2.6.1">>},2},
 {<<"cg_mon">>,
-  {git,"https://github.com/valitydev/cg_mon.git",
+  {git,"https://github.com/rbkmoney/cg_mon.git",
       {ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}},
  1},
+ {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.13.0">>},2},
 {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0},
 {<<"cowboy_access_log">>,
  {git,"https://github.com/valitydev/cowboy_access_log.git",
@ -35,13 +37,14 @@
       {ref,"186cf4d0722d4ad79afe73d371df6b1371e51905"}},
  0},
 {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},1},
+ {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},2},
 {<<"damsel">>,
  {git,"https://github.com/valitydev/damsel.git",
       {ref,"2374f52326fb9b10e79e0ec0a769af35529938e9"}},
  0},
 {<<"dmt_client">>,
  {git,"https://github.com/valitydev/dmt_client.git",
-       {ref,"ce6678af1499230fe13f8b34258aabe8b92ac722"}},
+       {ref,"b8bc0281dbf1e55a1a67ef6da861e0353ff14913"}},
  0},
 {<<"dmt_core">>,
  {git,"https://github.com/valitydev/dmt-core.git",
@ -50,7 +53,7 @@
 {<<"email_validator">>,{pkg,<<"email_validator">>,<<"1.1.0">>},1},
 {<<"erl_health">>,
  {git,"https://github.com/valitydev/erlang-health.git",
-       {ref,"5958e2f35cd4d09f40685762b82b82f89b4d9333"}},
+       {ref,"7ffbc855bdbe79e23efad1803b0b185c9ea8d2f1"}},
  0},
 {<<"feat">>,
  {git,"https://github.com/valitydev/feat.git",
@ -58,14 +61,16 @@
  0},
 {<<"genlib">>,
  {git,"https://github.com/valitydev/genlib.git",
-       {ref,"82c5ff3866e3019eb347c7f1d8f1f847bed28c10"}},
+       {ref,"f6074551d6586998e91a97ea20acb47241254ff3"}},
  0},
 {<<"gproc">>,{pkg,<<"gproc">>,<<"0.9.0">>},1},
+ {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.16.0">>},1},
 {<<"gun">>,
  {git,"https://github.com/ninenines/gun.git",
       {ref,"e7dd9f227e46979d8073e71c683395a809b78cb4"}},
  1},
 {<<"hackney">>,{pkg,<<"hackney">>,<<"1.17.4">>},1},
+ {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},3},
 {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},2},
 {<<"jesse">>,
  {git,"https://github.com/valitydev/jesse.git",
@ -90,6 +95,14 @@
  {git,"https://github.com/valitydev/msgpack-proto.git",
       {ref,"7e447496aa5df4a5f1ace7ef2e3c31248b2a3ed0"}},
  1},
+ {<<"opentelemetry">>,{pkg,<<"opentelemetry">>,<<"1.3.0">>},0},
+ {<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.2.1">>},0},
+ {<<"opentelemetry_exporter">>,
+  {pkg,<<"opentelemetry_exporter">>,<<"1.3.0">>},
+  0},
+ {<<"opentelemetry_semantic_conventions">>,
+  {pkg,<<"opentelemetry_semantic_conventions">>,<<"0.2.0">>},
+  1},
 {<<"org_management_proto">>,
  {git,"https://github.com/valitydev/org-management-proto",
       {ref,"03a269df4805fa604e8fd2d04241619a739e2ae3"}},
@ -110,7 +123,7 @@
 {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},1},
 {<<"scoper">>,
  {git,"https://github.com/valitydev/scoper.git",
-       {ref,"7f3183df279bc8181efe58dafd9cae164f495e6f"}},
+       {ref,"41a14a558667316998af9f49149ee087ffa8bef2"}},
  0},
 {<<"snowflake">>,
  {git,"https://github.com/valitydev/snowflake.git",
@ -129,9 +142,12 @@
  {git,"https://github.com/valitydev/thrift_erlang.git",
       {ref,"846a0819d9b6d09d0c31f160e33a78dbad2067b4"}},
  1},
+ {<<"tls_certificate_check">>,
+  {pkg,<<"tls_certificate_check">>,<<"1.19.0">>},
+  1},
 {<<"token_keeper_client">>,
  {git,"https://github.com/valitydev/token-keeper-client.git",
-       {ref,"4ef858ac5a226789d17d0a29fe40c7e38ed94193"}},
+       {ref,"417916aab09baf0cd7f1a2a174945446203ea0bf"}},
  0},
 {<<"token_keeper_proto">>,
  {git,"https://github.com/valitydev/token-keeper-proto.git",
@ -140,7 +156,7 @@
 {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2},
 {<<"woody">>,
  {git,"https://github.com/valitydev/woody_erlang.git",
-       {ref,"0c2e16dfc8a51f6f63fcd74df982178a9aeab322"}},
+       {ref,"5d46291a6bfcee0bae2a9346a7d927603a909249"}},
  0},
 {<<"woody_user_identity">>,
  {git,"https://github.com/valitydev/woody_erlang_user_identity.git",
@ -148,35 +164,55 @@
  0}]}.
 [
 {pkg_hash,[
+ {<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>},
 {<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>},
 {<<"certifi">>, <<"DBAB8E5E155A0763EEA978C913CA280A6B544BFA115633FA20249C3D396D9493">>},
+ {<<"chatterbox">>, <<"6F059D97BCAA758B8EA6FFFE2B3B81362BD06B639D3EA2BB088335511D691EBF">>},
 {<<"cowboy">>, <<"865DD8B6607E14CF03282E10E934023A1BD8BE6F6BACF921A7E2A96D800CD452">>},
 {<<"cowlib">>, <<"0B9FF9C346629256C42EBE1EEB769A83C6CB771A6EE5960BD110AB0B9B872063">>},
+ {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>},
 {<<"email_validator">>, <<"7E09A862E9AA99AE2CA6FD2A718D2B94360E32940A1339B53DFEE6B774BCDB03">>},
 {<<"gproc">>, <<"853CCB7805E9ADA25D227A157BA966F7B34508F386A3E7E21992B1B484230699">>},
+ {<<"grpcbox">>, <<"B83F37C62D6EECA347B77F9B1EC7E9F62231690CDFEB3A31BE07CD4002BA9C82">>},
 {<<"hackney">>, <<"99DA4674592504D3FB0CFEF0DB84C3BA02B4508BAE2DFF8C0108BAA0D6E0977C">>},
+ {<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>},
 {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>},
 {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>},
 {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>},
 {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>},
+ {<<"opentelemetry">>, <<"988AC3C26ACAC9720A1D4FB8D9DC52E95B45ECFEC2D5B5583276A09E8936BC5E">>},
+ {<<"opentelemetry_api">>, <<"7B69ED4F40025C005DE0B74FCE8C0549625D59CB4DF12D15C32FE6DC5076FF42">>},
+ {<<"opentelemetry_exporter">>, <<"1D8809C0D4F4ACF986405F7700ED11992BCBDB6A4915DD11921E80777FFA7167">>},
+ {<<"opentelemetry_semantic_conventions">>, <<"B67FE459C2938FCAB341CB0951C44860C62347C005ACE1B50F8402576F241435">>},
 {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>},
 {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>},
 {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>},
+ {<<"tls_certificate_check">>, <<"C76C4C5D79EE79A2B11C84F910C825D6F024A78427C854F515748E9BD025E987">>},
 {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]},
 {pkg_hash_ext,[
+ {<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>},
 {<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>},
 {<<"certifi">>, <<"524C97B4991B3849DD5C17A631223896272C6B0AF446778BA4675A1DFF53BB7E">>},
+ {<<"chatterbox">>, <<"B93D19104D86AF0B3F2566C4CBA2A57D2E06D103728246BA1AC6C3C0FF010AA7">>},
 {<<"cowboy">>, <<"2C729F934B4E1AA149AFF882F57C6372C15399A20D54F65C8D67BEF583021BDE">>},
 {<<"cowlib">>, <<"2B3E9DA0B21C4565751A6D4901C20D1B4CC25CBB7FD50D91D2AB6DD287BC86A9">>},
+ {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>},
 {<<"email_validator">>, <<"2B1E6DF7BB14155C8D7D131F1C95CF4676200BC056EEBA82123396833FF94DA2">>},
 {<<"gproc">>, <<"587E8AF698CCD3504CF4BA8D90F893EDE2B0F58CABB8A916E2BF9321DE3CF10B">>},
+ {<<"grpcbox">>, <<"294DF743AE20A7E030889F00644001370A4F7CE0121F3BBDAF13CF3169C62913">>},
 {<<"hackney">>, <<"DE16FF4996556C8548D512F4DBE22DD58A587BF3332E7FD362430A7EF3986B16">>},
+ {<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>},
 {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>},
 {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>},
 {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>},
 {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>},
+ {<<"opentelemetry">>, <<"8E09EDC26AAD11161509D7ECAD854A3285D88580F93B63B0B1CF0BAC332BFCC0">>},
+ {<<"opentelemetry_api">>, <<"6D7A27B7CAD2AD69A09CABF6670514CAFCEC717C8441BEB5C96322BAC3D05350">>},
+ {<<"opentelemetry_exporter">>, <<"2B40007F509D38361744882FD060A8841AF772AB83BB542AA5350908B303AD65">>},
+ {<<"opentelemetry_semantic_conventions">>, <<"D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895">>},
 {<<"parse_trans">>, <<"620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A">>},
 {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>},
 {<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>},
+ {<<"tls_certificate_check">>, <<"4083B4A298ADD534C96125337CB01161C358BB32DD870D5A893AAE685FD91D70">>},
 {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]}
 ].