diff --git a/sink-connector-lightweight/README.md b/sink-connector-lightweight/README.md index 88f765a..f29c98f 100644 --- a/sink-connector-lightweight/README.md +++ b/sink-connector-lightweight/README.md @@ -1,5 +1,8 @@ +### Altinity Replicator for ClickHosue +### Getting started (MySQL) +Currently the configuration is passed to the application as environment variables. \ +`(sink-connector-lightweight/docker/docker.env)` -### Configuration **Configuration** ``` export database.hostname="mysql-master" @@ -7,6 +10,10 @@ export database.port="3306" export database.user="root" export database.password="root" export database.include.list=sbtest +export clickhouse.server.url="clickhouse" +export clickhouse.server.user="root" +export clickhouse.server.pass="root" +export clickhouse.server.port="8123" #export table.include.list=sbtest1 export database.allowPublicKeyRetrieval"="true" export snapshot.mode="schema_only" @@ -15,40 +22,31 @@ export clickhouse.server.database="test" export auto.create.tables=true export replacingmergetree.delete.column="_sign" export metrics.port=8083 -export clickhouse.server.url="clickhouse" -export clickhouse.server.user="root" -export clickhouse.server.pass="root" -export clickhouse.server.port="8123" - +``` +After the environment variables are set, start the docker container +A Sample docker-compose is provided , it starts the docker container \ +`registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest` +``` +cd sink-connector-lightweight +cd docker +docker-compose up ``` -In `deploy\docker.env`, Update **MySQL** and **ClickHouse** configuration. +### Getting Started (PostgreSQL) +`(sink-connector-lightweight/docker/docker_postgres.env)` -**MySQL** +**Configuration** ``` -export database.hostname="localhost" -export database.port="3306" -export database.user="root" -export database.password="root" -export database.whitelist="test" -``` -**ClickHouse** -``` -export clickhouse.server.url="clickhouse" -export clickhouse.server.user="root" -export clickhouse.server.pass="root" -export clickhouse.server.port="8123" -``` - -**PostgreSQL** -``` - export database.hostname="postgres" export database.port="5432" export database.user="root" export database.password="root" export snapshot.mode="schema_only" +export clickhouse.server.url="clickhouse" +export clickhouse.server.user="root" +export clickhouse.server.pass="root" +export clickhouse.server.port="8123" export connector.class="io.debezium.connector.postgresql.PostgresConnector" export plugin.name="pgoutput" export table.include.list="public.tm" @@ -56,19 +54,25 @@ export clickhouse.server.database="test" export auto.create.tables=true export replacingmergetree.delete.column="_sign" export metrics.port=8083 -export clickhouse.server.url="clickhouse" -export clickhouse.server.user="root" -export clickhouse.server.pass="root" -export clickhouse.server.port="8123" + ``` -PostgreSQL specific configuration: +After the environment variables are set, start the docker container ``` -export connector.class="io.debezium.connector.postgresql.PostgresConnector" -export database.dbname="public" -export plugin.name="pgoutput" +cd sink-connector-lightweight +cd docker +docker-compose up -f docker-compose-postgres.yml ``` +### Getting Started (MongoDB) +`(sink-connector-lightweight/docker/docker_mongo.env)` + +**Configuration** +``` + +``` + +### Getting Started (Helm Charts) #### Helm Charts Update the MySQL/PostgreSQL and ClickHouse configuration values ` @@ -119,19 +123,10 @@ Update the MySQL/PostgreSQL and ClickHouse configuration values value: "8083" ``` - ``` cd helm helm install clickhouse-debezium-embedded . -``` -### Getting started - - -**Start application** -``` -cd docker -docker-compose up ``` ### Building from sources diff --git a/sink-connector-lightweight/build_docker.sh b/sink-connector-lightweight/build_docker.sh index 546dfa3..f4b3f73 100755 --- a/sink-connector-lightweight/build_docker.sh +++ b/sink-connector-lightweight/build_docker.sh @@ -1,6 +1,8 @@ #!/bin/bash +today_date=$(date +%F) + docker login registry.gitlab.com -docker build . -t clickhouse_debezium_embedded -docker tag clickhouse_debezium_embedded registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded -docker push registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded +docker build . -t clickhouse_debezium_embedded:${today_date} +docker tag clickhouse_debezium_embedded:${today_date} registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:${today_date} +docker push registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:${today_date} diff --git a/sink-connector-lightweight/docker/docker-compose-mongo.yml b/sink-connector-lightweight/docker/docker-compose-mongo.yml new file mode 100644 index 0000000..995f467 --- /dev/null +++ b/sink-connector-lightweight/docker/docker-compose-mongo.yml @@ -0,0 +1,72 @@ +version: "3.4" + +# Ubuntu , set this for redpanda to start +# https://sort.veritas.com/public/documents/HSO/2.0/linux/productguides/html/hfo_admin_ubuntu/ch04s03.htm + +# Clickhouse Table Schema +# create table test(id int, message String) ENGINE=MergeTree() PRIMARY KEY id; + +services: + mongo: + image: mongo:latest + restart: always + ports: + - "27017:27017" + environment: + - MONGO_INITDB_DATABASE=project + volumes: + - ../sql/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js + + clickhouse: + # clickhouse-client --host=127.0.0.1 --port=9000 --user=root --password=root --database=test + container_name: clickhouse + image: clickhouse/clickhouse-server:latest + restart: "no" + ports: + - "8123:8123" + - "9000:9000" + environment: + - CLICKHOUSE_USER=root + - CLICKHOUSE_PASSWORD=root + - CLICKHOUSE_DB=test + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=0 + ulimits: + nofile: + soft: "262144" + hard: "262144" + volumes: + #- ../sql/init_ch.sql:/docker-entrypoint-initdb.d/init_clickhouse.sql + - ../clickhouse/users.xml:/etc/clickhouse-server/users.xml + depends_on: + zookeeper: + condition: service_healthy + + zookeeper: + image: zookeeper:3.6.2 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable + + debezium-embedded: + image: registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:2023-04-19 + # build: + # context: ../ + restart: "no" + ports: + - "8083:8083" + depends_on: + - clickhouse + env_file: + - docker_mongo.env + extra_hosts: + - "host.docker.internal:host-gateway" diff --git a/sink-connector-lightweight/docker/docker-compose-postgres-standalone.yml b/sink-connector-lightweight/docker/docker-compose-postgres-standalone.yml new file mode 100644 index 0000000..78f74d9 --- /dev/null +++ b/sink-connector-lightweight/docker/docker-compose-postgres-standalone.yml @@ -0,0 +1,16 @@ +services: + postgres: + image: debezium/postgres:15-alpine + restart: always + ports: + - "5432:5432" + environment: + - POSTGRES_PASSWORD=root + - POSTGRES_USER=root + - POSTGRES_DB=public + volumes: + - ../sql/init_postgres.sql:/docker-entrypoint-initdb.d/init_postgres.sql + command: + - "postgres" + - "-c" + - "wal_level=logical" \ No newline at end of file diff --git a/sink-connector-lightweight/docker/docker-compose-postgres.yml b/sink-connector-lightweight/docker/docker-compose-postgres.yml index 9bf0c76..31f44c3 100644 --- a/sink-connector-lightweight/docker/docker-compose-postgres.yml +++ b/sink-connector-lightweight/docker/docker-compose-postgres.yml @@ -65,14 +65,14 @@ services: - label:disable debezium-embedded: - image: registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest + image: registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:2023-04-19 # build: # context: ../ restart: "no" ports: - "8083:8083" - # depends_on: - # - mysql-master + depends_on: + - clickhouse env_file: - docker_postgres.env extra_hosts: diff --git a/sink-connector-lightweight/docker/docker_mongo.env b/sink-connector-lightweight/docker/docker_mongo.env new file mode 100644 index 0000000..affc899 --- /dev/null +++ b/sink-connector-lightweight/docker/docker_mongo.env @@ -0,0 +1,22 @@ +export mongodb.connection.string="mongodb://localhost:49252" +export mongodb.members.auto.discover="true" +export topic.prefix="mongo-ch" +export collection.include.list="project.items" +export snapshot.include.collection.list="project.items" +export database.include.list="project" +export database.dbname="project" +export database.user="project" +export database.password="project" +export snapshot.mode="initial" +export connector.class="io.debezium.connector.mongodb.MongoDbConnector" +export offset.storage="org.apache.kafka.connect.storage.FileOffsetBackingStore" +export offset.flush.interval.ms="60000" +export auto.create.tables="true" +export clickhouse.server.url="clickhouse" +export clickhouse.server.port="8123" +export clickhouse.server.user="root" +export clickhouse.server.pass="root" +export clickhouse.server.database="project" +export replacingmergetree.delete.column="_sign" +export metrics.port="8087" +export database.allowPublicKeyRetrieval="true" \ No newline at end of file diff --git a/sink-connector-lightweight/sql/init_postgres.sql b/sink-connector-lightweight/sql/init_postgres.sql index bc8394e..3aed879 100644 --- a/sink-connector-lightweight/sql/init_postgres.sql +++ b/sink-connector-lightweight/sql/init_postgres.sql @@ -21,6 +21,8 @@ vcreated timestamp with time zone, vbilling_currency character varying ); +ALTER TABLE public.tm REPLICA IDENTITY FULL; + INSERT INTO public.tm VALUES ( '9cb52b2a-8ef2-4987-8856-c79a1b2c2f71', @@ -66,4 +68,4 @@ NULL, 200000.00000, '2022-10-16 16:53:15.01957', 'IDR' -); \ No newline at end of file +); diff --git a/sink-connector-lightweight/sql/mongo-init.js b/sink-connector-lightweight/sql/mongo-init.js new file mode 100644 index 0000000..13c59e4 --- /dev/null +++ b/sink-connector-lightweight/sql/mongo-init.js @@ -0,0 +1,33 @@ +db.createUser({ + user: 'project', + pwd: 'project', + roles: [ + { + role: 'readWrite', + db: 'project', + }, + ], +}); + +db = new Mongo().getDB('project'); + +db.createCollection('users', { capped: false }); +db.createCollection('items', { capped: false }); + +//db.items.insert([ +// { +// uuid: '28105d81-dac5-48a4-b70d-a40b2882a719', +// price: '49', +// name: 'T-Shirt', +// }, +// { +// uuid: '49b63da2-aa02-4a04-a7f1-62d2da389897', +// price: '89', +// name: 'Jeans', +// }, +// { +// uuid: 'f0038e77-dc96-4236-a979-f06b993b0332', +// price: '120', +// name: 'Jacket', +// }, +//]); \ No newline at end of file diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java index 5ade6e6..3f20e10 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java @@ -292,6 +292,7 @@ public class DebeziumChangeEventCapture { } } + // db.items.insert({_id:ObjectId(), uuid:ObjectId(), price:22, name:"New record"}); private void trySomething(Configuration config){ PostgresConnectorConfig postgresConfig = new PostgresConnectorConfig(config); // ConfigValue hostnameValue = (ConfigValue) configValues.get(RelationalDatabaseConnectorConfig.HOSTNAME.name()); diff --git a/sink-connector-lightweight/src/test/resources/init_clickhouse.sql b/sink-connector-lightweight/src/test/resources/init_clickhouse.sql index 963353f..a4d5c93 100644 --- a/sink-connector-lightweight/src/test/resources/init_clickhouse.sql +++ b/sink-connector-lightweight/src/test/resources/init_clickhouse.sql @@ -1,4 +1,29 @@ CREATE database datatypes; CREATE database employees; CREATE database public; -CREATE database project; \ No newline at end of file +CREATE database project; + +CREATE TABLE project.items +( + `price` Int64, + `name` String, + `_id` String, + `uuid` String, + `_sign` Int8, + `_version` UInt64 +) +ENGINE = ReplacingMergeTree(_version) +ORDER BY _id; + + +CREATE TABLE public.protocol_test +( + `id` Int64, + `consultation_id` Int64, + `recomendation` Nullable(String), +`create_date` DateTime64(6), +`_sign` Int8, +`_version` UInt64 +) +ENGINE = ReplacingMergeTree(_version) +ORDER BY id diff --git a/sink-connector-lightweight/src/test/resources/init_postgres.sql b/sink-connector-lightweight/src/test/resources/init_postgres.sql index b876f5e..f66927c 100644 --- a/sink-connector-lightweight/src/test/resources/init_postgres.sql +++ b/sink-connector-lightweight/src/test/resources/init_postgres.sql @@ -66,4 +66,55 @@ NULL, 200000.00000, '2022-10-16 16:53:15.01957', 'IDR' -); \ No newline at end of file +); + +CREATE TABLE protocol_test +( + id bigserial NOT NULL, + consultation_id int8 NOT NULL, + recomendation text NULL, + create_date timestamptz(0) NOT NULL DEFAULT now(), +CONSTRAINT protocol_pkey PRIMARY KEY (id) +); + +INSERT INTO protocol_test VALUES ('1778392', '15563836', 'Henry I voyuer Ellen Holdings LLC 4510 Elsmore viking friendship', '2001-11-17T08:34:48.338Z'); +INSERT INTO protocol_test VALUES ('1778393', '17226989', 'Egbert modify Clothing Stores Inc 6497 Macclesfield Street touched journey', '2006-03-08T14:57:41.217Z'); +INSERT INTO protocol_test VALUES ('1778394', '10588494', 'Aethelbert incorrect Configuration Company 0652 Bellhouse Avenue update descending', '1998-06-17T19:56:22.373Z'); +INSERT INTO protocol_test VALUES ('1778395', '929854', 'William III roster Proxy Energy 3851 Tamerton Lane operated assault', '1976-04-27T09:57:28.520Z'); +INSERT INTO protocol_test VALUES ('1778396', '3802619', 'Edward VI phentermine Intent 2052 Mardale Road apparent negotiation', '1974-11-21T03:13:00.486Z'); +INSERT INTO protocol_test VALUES ('1778397', '14180264', 'Oliver Cromwell leaders Levitra Corporation 4607 Blackmore Street ex factor', '2019-01-19T17:07:00.548Z'); +INSERT INTO protocol_test VALUES ('1778398', '4694776', 'Edward IV disposition Casa A.G 1014 Windsor Circle noted nervous', '2001-09-04T16:41:35.782Z'); +INSERT INTO protocol_test VALUES ('1778399', '17504297', 'George VI personality Infrastructure Stores Pte. Ltd 5416 Isherwood Avenue beastality surgical', '1973-10-31T09:33:52.476Z'); +INSERT INTO protocol_test VALUES ('1778400', '1849158', 'Edward III indication Industrial International Inc 5362 Brow programs fountain', '1989-04-24T16:16:17.682Z'); +INSERT INTO protocol_test VALUES ('1778401', '11321075', 'Henry III valves People 7759 Ashdale Road pharmaceuticals eddie', '1984-06-24T04:37:33.883Z'); +INSERT INTO protocol_test VALUES ('1778402', '14017777', 'Anne matches Pro SIA 8300 Daisygate Lane calibration keno', '1990-05-22T12:08:53.159Z'); +INSERT INTO protocol_test VALUES ('1778403', '20704443', 'Offa msn Knee International S.A 4174 Bodmin Road however portions', '2004-06-13T09:51:21.807Z'); +INSERT INTO protocol_test VALUES ('1778404', '13219949', 'Edward III domestic Fragrances Mutual A.G 2683 Brewery Avenue business pen', '2013-03-13T19:38:56.313Z'); +INSERT INTO protocol_test VALUES ('1778405', '12850499', 'Harthacanut canvas Scoring Mutual 7526 Imperial Avenue broker absent', '2010-12-17T21:41:22.045Z'); +INSERT INTO protocol_test VALUES ('1778406', '9028650', 'Harold II jobs Gamespot A.G 5029 Maypool speaker shorter', '2005-03-22T09:04:38.300Z'); +INSERT INTO protocol_test VALUES ('1778407', '124999', 'Oliver Cromwell generating Loud Stores Corp 1309 Tennyson Road village lopez', '1971-01-31T00:13:14.868Z'); +INSERT INTO protocol_test VALUES ('1778408', '8208132', 'Charles I pat Requiring Ltd 5633 Fistral Street velvet keep', '1999-09-18T10:31:12.849Z'); +INSERT INTO protocol_test VALUES ('1778409', '11043995', 'Charles II objects Estimated Mutual S.A 3911 Ridge Lane distributors exciting', '1994-07-18T19:38:17.970Z'); +INSERT INTO protocol_test VALUES ('1778410', '18749480', 'Henry III trail Codes Software Inc 9875 Thomas Circle psi ion', '2006-12-09T21:56:40.567Z'); +INSERT INTO protocol_test VALUES ('1778411', '7187805', 'Edward VII kyle Rca Stores B.V 5530 Lesser Road contemporary nr', '2003-03-25T03:34:25.305Z'); +INSERT INTO protocol_test VALUES ('1778412', '15110317', 'William IV dishes Para Software GmbH 2986 Fitton Avenue defining turbo', '1981-03-10T07:11:28.227Z'); +INSERT INTO protocol_test VALUES ('1778413', '19393663', 'Stephen wax Suggested 8909 Torwood planning committed', '1993-05-22T20:58:55.054Z'); +INSERT INTO protocol_test VALUES ('1778414', '7891828', 'Aethelred I schedule Dicke SIA 5202 Hathaway Avenue sets requested', '2002-11-24T11:17:19.723Z'); +INSERT INTO protocol_test VALUES ('1778415', '10389394', 'Harthacanut incidence Locally International Pte. Ltd 2172 Ashwood Road end nato', '2017-03-12T01:23:42.442Z'); +INSERT INTO protocol_test VALUES ('1778416', '9613139', 'George I weblog Chi Mutual SIA 0947 Dowling Road satin courses', '2019-06-02T10:57:33.557Z'); +INSERT INTO protocol_test VALUES ('1778417', '3294112', 'Ethelred II the Unready diversity Adjust S.A 0876 Farmside Circle thumbzilla edge', '1982-04-12T06:21:46.045Z'); +INSERT INTO protocol_test VALUES ('1778418', '22868110', 'Aethelbald batch Twenty 4871 Heaton Lane even punch', '2002-07-24T10:39:09.013Z'); +INSERT INTO protocol_test VALUES ('1778419', '9784823', 'Charles I dinner Tracking Stores A.G 1793 Hurst Circle charming proposals', '1991-03-12T16:10:01.117Z'); +INSERT INTO protocol_test VALUES ('1778420', '17503042', 'Richard Cromwell journals Lease 7747 Back Road patrick periodically', '1992-12-17T08:25:26.611Z'); +INSERT INTO protocol_test VALUES ('1778421', '4105988', 'Egbert weak Qualifying Stores Corp 1025 Falmer Street plants engines', '1993-02-16T06:50:46.555Z'); +INSERT INTO protocol_test VALUES ('1778422', '20367835', 'Edward III layout Opening International Ltd 9873 Stansfield cocktail mails', '1978-09-25T18:43:38.845Z'); +INSERT INTO protocol_test VALUES ('1778423', '4133456', 'Richard Cromwell forced Gtk B.V 6057 Brownsville volvo encourages', '1979-08-10T06:13:07.451Z'); +INSERT INTO protocol_test VALUES ('1778424', '15763048', 'George III women Part Ltd 2830 Turf Lane ap diseases', '1978-04-24T19:44:06.966Z'); +INSERT INTO protocol_test VALUES ('1778425', '13783051', 'George II wagon Express Energy Pte. Ltd 7317 Adair refer producer', '1976-11-04T05:18:23.814Z'); +INSERT INTO protocol_test VALUES ('1778426', '5048883', 'Ethelred II the Unready retain Medline Mutual A.G 6841 Cudworth chester mg', '1971-05-23T22:24:10.661Z'); +INSERT INTO protocol_test VALUES ('1778427', '19987357', 'James I ireland Salary Industries 9869 Northholt Road vernon reactions', '2023-02-26T00:29:05.151Z'); +INSERT INTO protocol_test VALUES ('1778428', '9518612', 'Stephen poly Holmes LLC 9482 Honiton Road participants equations', '1979-08-04T04:31:07.812Z'); +INSERT INTO protocol_test VALUES ('1778429', '9288848', 'William IV printers Parker Industries Corporation 3335 Privet Lane null prostate', '1976-03-05T15:27:18.569Z'); +INSERT INTO protocol_test VALUES ('1778430', '12041969', 'Henry VII stocks Skin A.G 1983 Affetside Lane shanghai sleeve', '1970-09-28T17:34:52.197Z'); +INSERT INTO protocol_test VALUES ('1778431', '1228695', 'Mary I silver Visiting 6041 Caesar Street cameras judge', '1970-11-07T02:49:21.977Z'); +INSERT INTO protocol_test VALUES ('1778432', '21481203', 'Edward V prisoners Peterson Pte. Ltd 2577 Limebank Circle span religious', '1975-02-02T01:13:06.152Z'); \ No newline at end of file diff --git a/sink-connector-lightweight/src/test/resources/mongo-init.js b/sink-connector-lightweight/src/test/resources/mongo-init.js index 9886616..13c59e4 100644 --- a/sink-connector-lightweight/src/test/resources/mongo-init.js +++ b/sink-connector-lightweight/src/test/resources/mongo-init.js @@ -14,20 +14,20 @@ db = new Mongo().getDB('project'); db.createCollection('users', { capped: false }); db.createCollection('items', { capped: false }); -db.items.insert([ - { - uuid: '28105d81-dac5-48a4-b70d-a40b2882a719', - price: '49.00', - name: 'T-Shirt', - }, - { - uuid: '49b63da2-aa02-4a04-a7f1-62d2da389897', - price: '89.99', - name: 'Jeans', - }, - { - uuid: 'f0038e77-dc96-4236-a979-f06b993b0332', - price: '120.00', - name: 'Jacket', - }, -]); \ No newline at end of file +//db.items.insert([ +// { +// uuid: '28105d81-dac5-48a4-b70d-a40b2882a719', +// price: '49', +// name: 'T-Shirt', +// }, +// { +// uuid: '49b63da2-aa02-4a04-a7f1-62d2da389897', +// price: '89', +// name: 'Jeans', +// }, +// { +// uuid: 'f0038e77-dc96-4236-a979-f06b993b0332', +// price: '120', +// name: 'Jacket', +// }, +//]); \ No newline at end of file