From 2c720088b4e47dc50af7174b1b673fe974007919 Mon Sep 17 00:00:00 2001 From: ggmaleva Date: Mon, 6 Sep 2021 14:18:35 +0300 Subject: [PATCH 1/6] add notificator --- docker-compose.yml | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index fdc7396..bd97428 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -213,6 +213,7 @@ services: management.metrics.binders.jvm.enabled: 'false' service.payment.url: 'http://fraudbusters:8022/fraud_payment/v1/' service.historical.url: 'http://fraudbusters:8022/historical_data/v1/' + service.fb-notificator.url: 'http://fb-notificator:8080/' spring.profiles.active: 'debug' keycloak.enabled: "true" keycloak.realm: "fraudbusters-realm" @@ -227,9 +228,61 @@ services: - wblist - postgres-fb - kafka-setup + - fb-notificator volumes: - ./log-java:/opt/fraudbusters-management/tmp + fb-notificator: + image: dr2.rbkmoney.com/rbkmoney/fraudbusters-notificator:313a917f75424417393158527c257626ddb5e890 + hostname: fb-notificator + container_name: fb-notificator + environment: + spring.datasource.url: "jdbc:postgresql://postgres-notificator:5432/fb_notificator" + logging.config: "./tmp/logback-test.xml" + management.metrics.binders.jvm.enabled: 'false' + warehouse.url: 'http://fb-warehouse:8022/query/v1/' + spring.profiles.active: 'debug' + ports: + - 8086:8080 + depends_on: + - postgres-notificator + - fb-warehouse + volumes: + - ./log-java:/opt/fraudbusters-notificator/tmp + + postgres-notificator: + container_name: postgres_notificator + hostname: postgres-notificator + image: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: fb_notificator + PGDATA: /data/postgres + ports: + - 6433:5432 + volumes: + - ./postgres/data/4:/data/postgres + restart: unless-stopped + + fb-warehouse: + image: dr2.rbkmoney.com/rbkmoney/fraudbusters-warehouse:f129351e97cc2a8c4e49d9a20f1e3415a68ccee9 + hostname: fb-warehouse + container_name: fb-warehouse + environment: + clickhouse.db.url: "jdbc:clickhouse://clickhouse:8123/default" + clickhouse.db.user: 'user' + clickhouse.db.password: 'password' + logging.config: "./tmp/logback-test.xml" + management.metrics.binders.jvm.enabled: 'false' + spring.profiles.active: 'debug' + ports: + - 8088:8080 + depends_on: + - clickhouse + volumes: + - ./log-java:/opt/fraudbusters-warehouse/tmp + fraudbusters-ui: image: dr2.rbkmoney.com/rbkmoney/fraudbusters-ui:052ef663e0c86d1521ef213b04355da58b0e5cc9 hostname: fraudbusters-ui From 52c58063ec4a3f31979609b05a5381abd0029061 Mon Sep 17 00:00:00 2001 From: ggmaleva Date: Mon, 6 Sep 2021 16:46:18 +0300 Subject: [PATCH 2/6] add analytic schema in clickhouse --- .../docker-entrypoint-initdb.d/init-db.sh | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) mode change 100644 => 100755 clickhouse/docker-entrypoint-initdb.d/init-db.sh diff --git a/clickhouse/docker-entrypoint-initdb.d/init-db.sh b/clickhouse/docker-entrypoint-initdb.d/init-db.sh old mode 100644 new mode 100755 index b863f72..d414d5c --- a/clickhouse/docker-entrypoint-initdb.d/init-db.sh +++ b/clickhouse/docker-entrypoint-initdb.d/init-db.sh @@ -602,4 +602,168 @@ clickhouse client -n <<-EOSQL VALUES ('2019-12-05', 1587761208, 1587759200000, 'group_1', '2035728', 'email', '4bef59146f8e4640ab34915f84ddac8b', 5000, 'RUB', 'failed', '666', '1DkratTHbpg.1', '204.26.61.110', '666', '3125', 'bank_card', '477bba133c182267fe5f086924abdc5db71f77bfc27f01f2843f2cdc69d89f05', 'VISA', '123', '1', 'RUS', 'Error', '1111vbtt'); + + CREATE DATABASE IF NOT EXISTS analytic; + + DROP TABLE IF EXISTS analytic.events_sink; + + create table analytic.events_sink + ( + timestamp Date, + eventTime UInt64, + eventTimeHour UInt64, + + partyId String, + shopId String, + + email String, + providerName String, + + amount UInt64, + guaranteeDeposit UInt64, + systemFee UInt64, + providerFee UInt64, + externalFee UInt64, + currency String, + + status Enum8('pending' = 1, 'processed' = 2, 'captured' = 3, 'cancelled' = 4, 'failed' = 5), + errorReason String, + errorCode String, + + invoiceId String, + paymentId String, + sequenceId UInt64, + + ip String, + bin String, + maskedPan String, + paymentTool String, + fingerprint String, + cardToken String, + paymentSystem String, + digitalWalletProvider String, + digitalWalletToken String, + cryptoCurrency String, + mobileOperator String, + + paymentCountry String, + bankCountry String, + + paymentTime UInt64, + providerId String, + terminal String + ) ENGINE = ReplacingMergeTree() + PARTITION BY toYYYYMM (timestamp) + ORDER BY (eventTimeHour, partyId, shopId, paymentTool, status, currency, providerName, fingerprint, cardToken, invoiceId, paymentId, sequenceId); + + DROP TABLE IF EXISTS analytic.events_sink_refund; + + create table analytic.events_sink_refund + ( + timestamp Date, + eventTime UInt64, + eventTimeHour UInt64, + + partyId String, + shopId String, + + email String, + providerName String, + + amount UInt64, + guaranteeDeposit UInt64, + systemFee UInt64, + providerFee UInt64, + externalFee UInt64, + currency String, + + reason String, + + status Enum8('pending' = 1, 'succeeded' = 2, 'failed' = 3), + errorReason String, + errorCode String, + + invoiceId String, + refundId String, + paymentId String, + sequenceId UInt64, + + ip String, + fingerprint String, + cardToken String, + paymentSystem String, + digitalWalletProvider String, + digitalWalletToken String, + cryptoCurrency String, + mobileOperator String, + + paymentCountry String, + bankCountry String, + + paymentTime UInt64, + providerId String, + terminal String + ) ENGINE = ReplacingMergeTree() + PARTITION BY toYYYYMM (timestamp) + ORDER BY (eventTimeHour, partyId, shopId, status, currency, providerName, fingerprint, cardToken, invoiceId, paymentId, refundId, sequenceId); + + DROP TABLE IF EXISTS analytic.events_sink_adjustment; + + create table analytic.events_sink_adjustment + ( + timestamp Date, + eventTime UInt64, + eventTimeHour UInt64, + + partyId String, + shopId String, + + email String, + providerName String, + + amount UInt64, + guaranteeDeposit UInt64, + systemFee UInt64, + providerFee UInt64, + externalFee UInt64, + + oldAmount UInt64, + oldGuaranteeDeposit UInt64, + oldSystemFee UInt64, + oldProviderFee UInt64, + oldExternalFee UInt64, + + currency String, + + reason String, + + status Enum8('captured' = 1, 'cancelled' = 2), + errorCode String, + errorReason String, + + invoiceId String, + adjustmentId String, + paymentId String, + sequenceId UInt64, + + ip String, + fingerprint String, + cardToken String, + paymentSystem String, + digitalWalletProvider String, + digitalWalletToken String, + cryptoCurrency String, + mobileOperator String, + + paymentCountry String, + bankCountry String, + + paymentTime UInt64, + providerId String, + terminal String + + ) ENGINE = ReplacingMergeTree() + PARTITION BY toYYYYMM (timestamp) + ORDER BY (eventTimeHour, partyId, shopId, status, currency, providerName, fingerprint, cardToken, invoiceId, paymentId, adjustmentId, sequenceId); + EOSQL From 73a1379aa5620b7630f3fb0c3ff3e3066cf008e7 Mon Sep 17 00:00:00 2001 From: ggmaleva Date: Mon, 6 Sep 2021 17:29:41 +0300 Subject: [PATCH 3/6] revert schema --- .../docker-entrypoint-initdb.d/init-db.sh | 164 ------------------ 1 file changed, 164 deletions(-) diff --git a/clickhouse/docker-entrypoint-initdb.d/init-db.sh b/clickhouse/docker-entrypoint-initdb.d/init-db.sh index d414d5c..b863f72 100755 --- a/clickhouse/docker-entrypoint-initdb.d/init-db.sh +++ b/clickhouse/docker-entrypoint-initdb.d/init-db.sh @@ -602,168 +602,4 @@ clickhouse client -n <<-EOSQL VALUES ('2019-12-05', 1587761208, 1587759200000, 'group_1', '2035728', 'email', '4bef59146f8e4640ab34915f84ddac8b', 5000, 'RUB', 'failed', '666', '1DkratTHbpg.1', '204.26.61.110', '666', '3125', 'bank_card', '477bba133c182267fe5f086924abdc5db71f77bfc27f01f2843f2cdc69d89f05', 'VISA', '123', '1', 'RUS', 'Error', '1111vbtt'); - - CREATE DATABASE IF NOT EXISTS analytic; - - DROP TABLE IF EXISTS analytic.events_sink; - - create table analytic.events_sink - ( - timestamp Date, - eventTime UInt64, - eventTimeHour UInt64, - - partyId String, - shopId String, - - email String, - providerName String, - - amount UInt64, - guaranteeDeposit UInt64, - systemFee UInt64, - providerFee UInt64, - externalFee UInt64, - currency String, - - status Enum8('pending' = 1, 'processed' = 2, 'captured' = 3, 'cancelled' = 4, 'failed' = 5), - errorReason String, - errorCode String, - - invoiceId String, - paymentId String, - sequenceId UInt64, - - ip String, - bin String, - maskedPan String, - paymentTool String, - fingerprint String, - cardToken String, - paymentSystem String, - digitalWalletProvider String, - digitalWalletToken String, - cryptoCurrency String, - mobileOperator String, - - paymentCountry String, - bankCountry String, - - paymentTime UInt64, - providerId String, - terminal String - ) ENGINE = ReplacingMergeTree() - PARTITION BY toYYYYMM (timestamp) - ORDER BY (eventTimeHour, partyId, shopId, paymentTool, status, currency, providerName, fingerprint, cardToken, invoiceId, paymentId, sequenceId); - - DROP TABLE IF EXISTS analytic.events_sink_refund; - - create table analytic.events_sink_refund - ( - timestamp Date, - eventTime UInt64, - eventTimeHour UInt64, - - partyId String, - shopId String, - - email String, - providerName String, - - amount UInt64, - guaranteeDeposit UInt64, - systemFee UInt64, - providerFee UInt64, - externalFee UInt64, - currency String, - - reason String, - - status Enum8('pending' = 1, 'succeeded' = 2, 'failed' = 3), - errorReason String, - errorCode String, - - invoiceId String, - refundId String, - paymentId String, - sequenceId UInt64, - - ip String, - fingerprint String, - cardToken String, - paymentSystem String, - digitalWalletProvider String, - digitalWalletToken String, - cryptoCurrency String, - mobileOperator String, - - paymentCountry String, - bankCountry String, - - paymentTime UInt64, - providerId String, - terminal String - ) ENGINE = ReplacingMergeTree() - PARTITION BY toYYYYMM (timestamp) - ORDER BY (eventTimeHour, partyId, shopId, status, currency, providerName, fingerprint, cardToken, invoiceId, paymentId, refundId, sequenceId); - - DROP TABLE IF EXISTS analytic.events_sink_adjustment; - - create table analytic.events_sink_adjustment - ( - timestamp Date, - eventTime UInt64, - eventTimeHour UInt64, - - partyId String, - shopId String, - - email String, - providerName String, - - amount UInt64, - guaranteeDeposit UInt64, - systemFee UInt64, - providerFee UInt64, - externalFee UInt64, - - oldAmount UInt64, - oldGuaranteeDeposit UInt64, - oldSystemFee UInt64, - oldProviderFee UInt64, - oldExternalFee UInt64, - - currency String, - - reason String, - - status Enum8('captured' = 1, 'cancelled' = 2), - errorCode String, - errorReason String, - - invoiceId String, - adjustmentId String, - paymentId String, - sequenceId UInt64, - - ip String, - fingerprint String, - cardToken String, - paymentSystem String, - digitalWalletProvider String, - digitalWalletToken String, - cryptoCurrency String, - mobileOperator String, - - paymentCountry String, - bankCountry String, - - paymentTime UInt64, - providerId String, - terminal String - - ) ENGINE = ReplacingMergeTree() - PARTITION BY toYYYYMM (timestamp) - ORDER BY (eventTimeHour, partyId, shopId, status, currency, providerName, fingerprint, cardToken, invoiceId, paymentId, adjustmentId, sequenceId); - EOSQL From 50242c42be06e02115f04e330a736c976fe7a61a Mon Sep 17 00:00:00 2001 From: ggmaleva Date: Mon, 6 Sep 2021 17:30:41 +0300 Subject: [PATCH 4/6] revert schema --- clickhouse/docker-entrypoint-initdb.d/init-db.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/clickhouse/docker-entrypoint-initdb.d/init-db.sh b/clickhouse/docker-entrypoint-initdb.d/init-db.sh index b863f72..8b7c76b 100755 --- a/clickhouse/docker-entrypoint-initdb.d/init-db.sh +++ b/clickhouse/docker-entrypoint-initdb.d/init-db.sh @@ -601,5 +601,4 @@ clickhouse client -n <<-EOSQL ip, bin, maskedPan, paymentTool, cardToken, paymentSystem, terminal, providerId, bankCountry, errorCode, paymentId) VALUES ('2019-12-05', 1587761208, 1587759200000, 'group_1', '2035728', 'email', '4bef59146f8e4640ab34915f84ddac8b', 5000, 'RUB', 'failed', '666', '1DkratTHbpg.1', '204.26.61.110', '666', '3125', 'bank_card', '477bba133c182267fe5f086924abdc5db71f77bfc27f01f2843f2cdc69d89f05', 'VISA', '123', '1', 'RUS', 'Error', '1111vbtt'); - EOSQL From f2a43c189b5d401d752c828e6793033bef9db089 Mon Sep 17 00:00:00 2001 From: ggmaleva Date: Mon, 6 Sep 2021 17:32:22 +0300 Subject: [PATCH 5/6] revert schema --- clickhouse/docker-entrypoint-initdb.d/init-db.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse/docker-entrypoint-initdb.d/init-db.sh b/clickhouse/docker-entrypoint-initdb.d/init-db.sh index 8b7c76b..b863f72 100755 --- a/clickhouse/docker-entrypoint-initdb.d/init-db.sh +++ b/clickhouse/docker-entrypoint-initdb.d/init-db.sh @@ -601,4 +601,5 @@ clickhouse client -n <<-EOSQL ip, bin, maskedPan, paymentTool, cardToken, paymentSystem, terminal, providerId, bankCountry, errorCode, paymentId) VALUES ('2019-12-05', 1587761208, 1587759200000, 'group_1', '2035728', 'email', '4bef59146f8e4640ab34915f84ddac8b', 5000, 'RUB', 'failed', '666', '1DkratTHbpg.1', '204.26.61.110', '666', '3125', 'bank_card', '477bba133c182267fe5f086924abdc5db71f77bfc27f01f2843f2cdc69d89f05', 'VISA', '123', '1', 'RUS', 'Error', '1111vbtt'); + EOSQL From 68ff01e208119321ca9cb7eceb2b605806c62de5 Mon Sep 17 00:00:00 2001 From: ggmaleva Date: Tue, 7 Sep 2021 17:06:53 +0300 Subject: [PATCH 6/6] update notificator --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index bd97428..e2f10a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -233,7 +233,7 @@ services: - ./log-java:/opt/fraudbusters-management/tmp fb-notificator: - image: dr2.rbkmoney.com/rbkmoney/fraudbusters-notificator:313a917f75424417393158527c257626ddb5e890 + image: dr2.rbkmoney.com/rbkmoney/fraudbusters-notificator:01dac03e4b6da5f47cc7a0e981971969eeb40270 hostname: fb-notificator container_name: fb-notificator environment: