From 2b05c52f601335344edda29f610ed73259c4dd0d Mon Sep 17 00:00:00 2001 From: Sergey Yelin Date: Tue, 19 May 2020 21:55:01 +0300 Subject: [PATCH] MSPF-532: Remove rfc3339 (#217) * MSPF-532: Remove rfc3339 * Review fixes * Fix error reporting --- apps/ff_core/src/ff_time.erl | 6 +- apps/ff_cth/src/ct_domain.erl | 3 +- apps/ff_server/src/ff_codec.erl | 51 ++++--- apps/ff_server/src/ff_deposit_codec.erl | 2 +- apps/ff_server/src/ff_w2w_transfer_codec.erl | 2 +- apps/ff_server/src/ff_withdrawal_codec.erl | 2 +- .../src/machinery_mg_eventsink.erl | 20 ++- apps/wapi/rebar.config | 126 ------------------ apps/wapi/src/wapi.app.src | 1 - apps/wapi/src/wapi_utils.erl | 26 +--- apps/wapi/src/wapi_wallet_ff_backend.erl | 6 +- apps/wapi/src/wapi_wallet_handler.erl | 2 +- rebar.config | 30 ++++- rebar.lock | 6 +- 14 files changed, 80 insertions(+), 203 deletions(-) delete mode 100644 apps/wapi/rebar.config diff --git a/apps/ff_core/src/ff_time.erl b/apps/ff_core/src/ff_time.erl index 2f48b9e..a0feac9 100644 --- a/apps/ff_core/src/ff_time.erl +++ b/apps/ff_core/src/ff_time.erl @@ -29,13 +29,11 @@ now() -> -spec to_rfc3339(timestamp_ms()) -> binary(). to_rfc3339(Timestamp) -> - {ok, BTimestamp} = rfc3339:format(Timestamp, millisecond), - BTimestamp. + genlib_rfc3339:format_relaxed(Timestamp, millisecond). -spec from_rfc3339(binary()) -> timestamp_ms(). from_rfc3339(BTimestamp) -> - {ok, Timestamp} = rfc3339:to_time(BTimestamp, millisecond), - Timestamp. + genlib_rfc3339:parse(BTimestamp, millisecond). -spec add_interval(timestamp_ms(), datetime_interval()) -> timestamp_ms(). diff --git a/apps/ff_cth/src/ct_domain.erl b/apps/ff_cth/src/ct_domain.erl index 3d42828..99bb706 100644 --- a/apps/ff_cth/src/ct_domain.erl +++ b/apps/ff_cth/src/ct_domain.erl @@ -390,5 +390,4 @@ account(SymCode, C) -> end. timestamp() -> - {ok, Now} = rfc3339:format(calendar:universal_time()), - Now. + genlib_rfc3339:format(genlib_time:daytime_to_unixtime(calendar:universal_time()), second). diff --git a/apps/ff_server/src/ff_codec.erl b/apps/ff_server/src/ff_codec.erl index 8fc73bc..795db49 100644 --- a/apps/ff_server/src/ff_codec.erl +++ b/apps/ff_server/src/ff_codec.erl @@ -229,13 +229,17 @@ marshal(sub_failure, Failure) -> sub = maybe_marshal(sub_failure, ff_failure:sub_failure(Failure)) }; -marshal(timestamp, {{Date, Time}, USec} = V) -> - case rfc3339:format({Date, Time, USec, 0}) of - {ok, R} when is_binary(R) -> - R; - Error -> - error({bad_timestamp, Error}, [timestamp, V]) - end; +marshal(timestamp, {DateTime, USec}) -> + DateTimeinSeconds = genlib_time:daytime_to_unixtime(DateTime), + {TimeinUnit, Unit} = + case USec of + 0 -> + {DateTimeinSeconds, second}; + USec -> + MicroSec = erlang:convert_time_unit(DateTimeinSeconds, second, microsecond), + {MicroSec + USec, microsecond} + end, + genlib_rfc3339:format_relaxed(TimeinUnit, Unit); marshal(timestamp_ms, V) -> ff_time:to_rfc3339(V); marshal(domain_revision, V) when is_integer(V) -> @@ -513,31 +517,24 @@ maybe_marshal(_Type, undefined) -> maybe_marshal(Type, Value) -> marshal(Type, Value). -%% Suppress dialyzer warning until rfc3339 spec will be fixed. -%% see https://github.com/talentdeficit/rfc3339/pull/5 --dialyzer([{nowarn_function, [parse_timestamp/1]}, no_match]). -spec parse_timestamp(binary()) -> machinery:timestamp(). parse_timestamp(Bin) -> - case rfc3339:parse(Bin) of - {ok, {_Date, _Time, _Usec, TZ}} when TZ =/= 0 andalso TZ =/= undefined -> - erlang:error({bad_deadline, not_utc}, [Bin]); - {ok, {Date, Time, undefined, _TZ}} -> - {to_calendar_datetime(Date, Time), 0}; - {ok, {Date, Time, Usec, _TZ}} -> - {to_calendar_datetime(Date, Time), Usec div 1000}; - {error, Error} -> - erlang:error({bad_timestamp, Error}, [Bin]) + try + MicroSeconds = genlib_rfc3339:parse(Bin, microsecond), + case genlib_rfc3339:is_utc(Bin) of + false -> + erlang:error({bad_timestamp, not_utc}, [Bin]); + true -> + USec = MicroSeconds rem 1000000, + DateTime = calendar:system_time_to_universal_time(MicroSeconds, microsecond), + {DateTime, USec} + end + catch + error:Error:St -> + erlang:raise(error, {bad_timestamp, Bin, Error}, St) end. -to_calendar_datetime(Date, Time = {H, _, S}) when H =:= 24 orelse S =:= 60 -> - %% Type specifications for hours and seconds differ in calendar and rfc3339, - %% so make a proper calendar:datetime() here. - Sec = calendar:datetime_to_gregorian_seconds({Date, Time}), - calendar:gregorian_seconds_to_datetime(Sec); -to_calendar_datetime(Date, Time) -> - {Date, Time}. - marshal_msgpack(nil) -> {nl, #msgp_Nil{}}; marshal_msgpack(V) when is_boolean(V) -> {b, V}; marshal_msgpack(V) when is_integer(V) -> {i, V}; diff --git a/apps/ff_server/src/ff_deposit_codec.erl b/apps/ff_server/src/ff_deposit_codec.erl index 3d7ae36..beb6913 100644 --- a/apps/ff_server/src/ff_deposit_codec.erl +++ b/apps/ff_server/src/ff_deposit_codec.erl @@ -182,7 +182,7 @@ deposit_symmetry_test() -> id = genlib:unique(), domain_revision = 24500062, party_revision = 140028, - created_at = <<"2025-01-01T00:00:00.001000Z">> + created_at = <<"2025-01-01T00:00:00.001Z">> }, ?assertEqual(Encoded, marshal(deposit, unmarshal(deposit, Encoded))). diff --git a/apps/ff_server/src/ff_w2w_transfer_codec.erl b/apps/ff_server/src/ff_w2w_transfer_codec.erl index 6388b69..021ea4a 100644 --- a/apps/ff_server/src/ff_w2w_transfer_codec.erl +++ b/apps/ff_server/src/ff_w2w_transfer_codec.erl @@ -138,7 +138,7 @@ w2w_transfer_symmetry_test() -> id = genlib:unique(), domain_revision = 24500062, party_revision = 140028, - created_at = <<"2025-01-01T00:00:00.001000Z">> + created_at = <<"2025-01-01T00:00:00.001Z">> }, ?assertEqual(Encoded, marshal(w2w_transfer, unmarshal(w2w_transfer, Encoded))). diff --git a/apps/ff_server/src/ff_withdrawal_codec.erl b/apps/ff_server/src/ff_withdrawal_codec.erl index 78de0f9..a26d275 100644 --- a/apps/ff_server/src/ff_withdrawal_codec.erl +++ b/apps/ff_server/src/ff_withdrawal_codec.erl @@ -265,7 +265,7 @@ withdrawal_symmetry_test() -> }, domain_revision = 1, party_revision = 3, - created_at = <<"2099-01-01T00:00:00.123000Z">> + created_at = <<"2099-01-01T00:00:00.123Z">> }, ?assertEqual(In, marshal_withdrawal(unmarshal_withdrawal(In))). diff --git a/apps/machinery_extra/src/machinery_mg_eventsink.erl b/apps/machinery_extra/src/machinery_mg_eventsink.erl index 7ac5d2a..2f78747 100644 --- a/apps/machinery_extra/src/machinery_mg_eventsink.erl +++ b/apps/machinery_extra/src/machinery_mg_eventsink.erl @@ -75,13 +75,19 @@ unmarshal(namespace, V) -> unmarshal(event_id, V) -> unmarshal(integer, V); unmarshal(timestamp, V) when is_binary(V) -> - case rfc3339:parse(V) of - {ok, {Date, Time, USec, TZOffset}} when TZOffset == undefined orelse TZOffset == 0 -> - {{Date, Time}, USec}; - {ok, _} -> - error(badarg, {timestamp, V, badoffset}); - {error, Reason} -> - error(badarg, {timestamp, V, Reason}) + try + MilliSec = genlib_rfc3339:parse(V, millisecond), + case genlib_rfc3339:is_utc(V) of + false -> + erlang:error(badarg, [timestamp, V, badoffset]); + true -> + USec = MilliSec rem 1000, + DateTime = calendar:system_time_to_universal_time(MilliSec, millisecond), + {DateTime, USec} + end + catch + error:Reason:St -> + erlang:raise(error, {timestamp, V, Reason}, St) end; unmarshal( {evsink_event, Schema}, diff --git a/apps/wapi/rebar.config b/apps/wapi/rebar.config deleted file mode 100644 index efa9d4b..0000000 --- a/apps/wapi/rebar.config +++ /dev/null @@ -1,126 +0,0 @@ -%% Common project erlang options. -{erl_opts, [ - -%% % mandatory -%% debug_info, -%% warnings_as_errors, -%% warn_export_all, -%% warn_missing_spec, -%% warn_untyped_record, -%% warn_export_vars, - -%% % by default -%% warn_unused_record, -%% warn_bif_clash, -%% warn_obsolete_guard, -%% warn_unused_vars, -%% warn_shadow_vars, -%% warn_unused_import, -%% warn_unused_function, -%% warn_deprecated_function, - -%% % at will -%% % bin_opt_info -%% % no_auto_import -%% % warn_missing_spec_all -]}. - -%% Common project dependencies. -{deps, [ - {cowboy, "2.7.0"}, - %% {rfc3339, "0.2.2"}, - {jose, "1.9.0"}, - %% {lager, "3.6.1"}, - {base64url, "0.0.1"}, - {jsx, "2.9.0"}, - %% {genlib, - %% {git, "https://github.com/rbkmoney/genlib.git", {branch, "master"}} - %% }, - %% {woody, - %% {git, "git@github.com:rbkmoney/woody_erlang.git", {branch, "master"}} - %% }, - %% {woody_user_identity, - %% {git, "git@github.com:rbkmoney/woody_erlang_user_identity.git", {branch, "master"}} - %% }, - %% {dmsl, - %% {git, "git@github.com:rbkmoney/damsel.git", {branch, "release/erlang/master"}} - %% }, - {cowboy_cors, - {git, "https://github.com/rbkmoney/cowboy_cors.git", {branch, master}} - }, - {cowboy_access_log, - {git, "git@github.com:rbkmoney/cowboy_access_log.git", {branch, "master"}} - }, - {payproc_errors, - {git, "git@github.com:rbkmoney/payproc-errors-erlang.git", {branch, "master"}} - }, - {erl_health, - {git, "https://github.com/rbkmoney/erlang-health.git", {branch, "master"}} - } -]}. - -%% XRef checks -%% {xref_checks, [ -%% undefined_function_calls, -%% undefined_functions, -%% deprecated_functions_calls, -%% deprecated_functions -%% ]}. -% at will -% {xref_warnings, true}. - -%% Tests -%% {cover_enabled, true}. - -%% Relx configuration -%% {relx, [ -%% {release, { capi , "0.1.0"}, [ -%% {recon , load }, % tools for introspection -%% {runtime_tools, load }, % debugger -%% {tools , load }, % profiler -%% capi, -%% sasl -%% ]}, -%% {sys_config, "./config/sys.config"}, -%% {vm_args, "./config/vm.args"}, -%% {dev_mode, true}, -%% {include_erts, false}, -%% {extended_start_script, true} -%% ]}. - -%% Dialyzer static analyzing -%% {dialyzer, [ -%% {warnings, [ -%% % mandatory -%% unmatched_returns, -%% error_handling, -%% race_conditions, -%% unknown -%% ]}, -%% {plt_apps, all_deps} -%% ]}. - -%% {profiles, [ -%% {prod, [ -%% {deps, [ -%% % for introspection on production -%% {recon, "2.3.2"} -%% ]}, -%% {relx, [ -%% {dev_mode, false}, -%% {include_erts, true}, -%% {overlay, [ -%% {mkdir , "var/keys/capi" }, -%% {copy , "var/keys/capi/private.pem" , "var/keys/capi/private.pem" } -%% ]} -%% ]} -%% ]}, -%% {test, [ -%% {cover_enabled, true}, -%% {deps, []} -%% ]} -%% ]}. - -%% {pre_hooks, [ -%% {thrift, "git submodule update --init"} -%% ]}. diff --git a/apps/wapi/src/wapi.app.src b/apps/wapi/src/wapi.app.src index 8ad2cbe..3c0e3d8 100644 --- a/apps/wapi/src/wapi.app.src +++ b/apps/wapi/src/wapi.app.src @@ -24,7 +24,6 @@ jsx, cowboy_cors, cowboy_access_log, - rfc3339, base64url, snowflake, woody_user_identity, diff --git a/apps/wapi/src/wapi_utils.erl b/apps/wapi/src/wapi_utils.erl index 4505fbe..b1c91b3 100644 --- a/apps/wapi/src/wapi_utils.erl +++ b/apps/wapi/src/wapi_utils.erl @@ -102,20 +102,8 @@ mask(leading, MaskLen, MaskChar, Chardata) -> -spec to_universal_time(Timestamp :: binary()) -> TimestampUTC :: binary(). to_universal_time(Timestamp) -> - {ok, {Date, Time, Usec, TZOffset}} = rfc3339:parse(Timestamp), - Seconds = calendar:datetime_to_gregorian_seconds({Date, Time}), - %% The following crappy code is a dialyzer workaround - %% for the wrong rfc3339:parse/1 spec. - {DateUTC, TimeUTC} = calendar:gregorian_seconds_to_datetime( - case TZOffset of - _ when is_integer(TZOffset) -> - Seconds - (60 * TZOffset); - _ -> - Seconds - end - ), - {ok, TimestampUTC} = rfc3339:format({DateUTC, TimeUTC, Usec, 0}), - TimestampUTC. + TimestampMS = genlib_rfc3339:parse(Timestamp, microsecond), + genlib_rfc3339:format_relaxed(TimestampMS, microsecond). -spec unwrap(ok | {ok, Value} | {error, _Error}) -> Value | no_return(). @@ -206,7 +194,7 @@ try_parse_woody_default(DeadlineStr) -> catch error:{bad_deadline, _Reason} -> {error, bad_deadline}; - error:{badmatch, {error, baddate}} -> + error:{badmatch, _} -> {error, bad_deadline}; error:deadline_reached -> {error, bad_deadline} @@ -260,10 +248,10 @@ get_unique_id() -> -spec to_universal_time_test() -> _. to_universal_time_test() -> - ?assertEqual(<<"2017-04-19T13:56:07Z">>, to_universal_time(<<"2017-04-19T13:56:07Z">>)), - ?assertEqual(<<"2017-04-19T13:56:07.530000Z">>, to_universal_time(<<"2017-04-19T13:56:07.53Z">>)), - ?assertEqual(<<"2017-04-19T10:36:07.530000Z">>, to_universal_time(<<"2017-04-19T13:56:07.53+03:20">>)), - ?assertEqual(<<"2017-04-19T17:16:07.530000Z">>, to_universal_time(<<"2017-04-19T13:56:07.53-03:20">>)). + ?assertEqual(<<"2017-04-19T13:56:07Z">>, to_universal_time(<<"2017-04-19T13:56:07Z">>)), + ?assertEqual(<<"2017-04-19T13:56:07.530Z">>, to_universal_time(<<"2017-04-19T13:56:07.53Z">>)), + ?assertEqual(<<"2017-04-19T10:36:07.530Z">>, to_universal_time(<<"2017-04-19T13:56:07.53+03:20">>)), + ?assertEqual(<<"2017-04-19T17:16:07.530Z">>, to_universal_time(<<"2017-04-19T13:56:07.53-03:20">>)). -spec redact_test() -> _. redact_test() -> diff --git a/apps/wapi/src/wapi_wallet_ff_backend.erl b/apps/wapi/src/wapi_wallet_ff_backend.erl index ecf48b4..f267e05 100644 --- a/apps/wapi/src/wapi_wallet_ff_backend.erl +++ b/apps/wapi/src/wapi_wallet_ff_backend.erl @@ -1982,9 +1982,9 @@ to_swag(withdrawal_event, {EventId, Ts, {status_changed, Status}}) -> )] }); -to_swag(timestamp, {{Date, Time}, Usec}) -> - {ok, Timestamp} = rfc3339:format({Date, Time, Usec, undefined}), - Timestamp; +to_swag(timestamp, {DateTime, USec}) -> + DateTimeSeconds = genlib_time:daytime_to_unixtime(DateTime), + genlib_rfc3339:format_relaxed(DateTimeSeconds + USec, microsecond); to_swag(timestamp_ms, Timestamp) -> ff_time:to_rfc3339(Timestamp); to_swag(currency, Currency) -> diff --git a/apps/wapi/src/wapi_wallet_handler.erl b/apps/wapi/src/wapi_wallet_handler.erl index 50edfb6..aeec8ec 100644 --- a/apps/wapi/src/wapi_wallet_handler.erl +++ b/apps/wapi/src/wapi_wallet_handler.erl @@ -723,4 +723,4 @@ get_expiration_deadline(Expiration) -> get_default_url_lifetime() -> Now = erlang:system_time(second), Lifetime = application:get_env(wapi, file_storage_url_lifetime, ?DEFAULT_URL_LIFETIME), - wapi_utils:unwrap(rfc3339:format(Now + Lifetime, second)). + genlib_rfc3339:format(Now + Lifetime, second). diff --git a/rebar.config b/rebar.config index 261a571..af42fa9 100644 --- a/rebar.config +++ b/rebar.config @@ -31,9 +31,6 @@ {genlib, {git, "https://github.com/rbkmoney/genlib.git", {branch, "master"}} }, - {rfc3339, - "0.2.2" - }, {uuid, {git, "https://github.com/okeuday/uuid.git", {branch, "master"}} }, @@ -58,9 +55,18 @@ {hackney, "1.15.1" }, - % {erlang_localtime, - % {git, "https://github.com/kpy3/erlang_localtime", {branch, "master"}} - % }, + {cowboy, + "2.7.0" + }, + {jose, + "1.9.0" + }, + {base64url, + "0.0.1" + }, + {jsx, + "2.9.0" + }, {cds_proto, {git, "git@github.com:rbkmoney/cds-proto.git", {branch, "master"}} }, @@ -102,6 +108,18 @@ }, {lechiffre, {git, "git@github.com:rbkmoney/lechiffre.git", {branch, "master"}} + }, + {cowboy_cors, + {git, "https://github.com/rbkmoney/cowboy_cors.git", {branch, master}} + }, + {cowboy_access_log, + {git, "git@github.com:rbkmoney/cowboy_access_log.git", {branch, "master"}} + }, + {payproc_errors, + {git, "git@github.com:rbkmoney/payproc-errors-erlang.git", {branch, "master"}} + }, + {erl_health, + {git, "https://github.com/rbkmoney/erlang-health.git", {branch, "master"}} } ]}. diff --git a/rebar.lock b/rebar.lock index b389436..87a1bb3 100644 --- a/rebar.lock +++ b/rebar.lock @@ -75,7 +75,7 @@ 2}, {<<"genlib">>, {git,"https://github.com/rbkmoney/genlib.git", - {ref,"6601a9f1cfd4bce566f0bd0016fdee5a26c8818c"}}, + {ref,"54920e768a71f121304a5eda547ee60295398f3c"}}, 0}, {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},0}, {<<"gun">>, @@ -98,7 +98,7 @@ {<<"idna">>,{pkg,<<"idna">>,<<"6.0.0">>},1}, {<<"jesse">>, {git,"https://github.com/rbkmoney/jesse.git", - {ref,"723e835708a022bbce9e57807ecf220b00fb771a"}}, + {ref,"a21da0609e446f328c01b1a72191cda26a8969a4"}}, 0}, {<<"jiffy">>, {git,"git@github.com:davisp/jiffy.git", @@ -142,7 +142,6 @@ 0}, {<<"quickrand">>,{pkg,<<"quickrand">>,<<"1.7.3">>},1}, {<<"ranch">>,{pkg,<<"ranch">>,<<"1.7.1">>},1}, - {<<"rfc3339">>,{pkg,<<"rfc3339">>,<<"0.2.2">>},0}, {<<"scoper">>, {git,"git@github.com:rbkmoney/scoper.git", {ref,"f2ac9c0b4e98a49a569631c3763c0585ec76abe5"}}, @@ -191,7 +190,6 @@ {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, {<<"quickrand">>, <<"0E4FB48FAC904FE0C6E21D7E8C31A288A0700E1E81A35B38B649FC119079755D">>}, {<<"ranch">>, <<"6B1FAB51B49196860B733A49C07604465A47BDB78AA10C1C16A3D199F7F8C881">>}, - {<<"rfc3339">>, <<"1552DF616ACA368D982E9F085A0E933B6688A3F4938A671798978EC2C0C58730">>}, {<<"ssl_verify_fun">>, <<"F0EAFFF810D2041E93F915EF59899C923F4568F4585904D010387ED74988E77B">>}, {<<"unicode_util_compat">>, <<"D869E4C68901DD9531385BB0C8C40444EBF624E60B6962D95952775CAC5E90CD">>}, {<<"uuid">>, <<"C5DF97D1A3D626235C2415E74053C47B2138BB863C5CD802AB5CAECB8ECC019F">>}]}