mirror of
https://github.com/valitydev/clickhouse-sink-connector.git
synced 2024-11-06 10:35:21 +00:00
Updated README.md and postgres configuration files.
This commit is contained in:
Kanthi Subramanian
parent
8a95cbca17
commit
ceb80787be
64
README.md
64
README.md
@ -61,10 +61,72 @@ schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector
|
||||
enable.snapshot.ddl: "false"
|
||||
|
||||
```
|
||||
### PostgreSQL Config(docker/config_postgres.yml)
|
||||
```
|
||||
database.hostname: "postgres"
|
||||
database.port: "5432"
|
||||
database.user: "root"
|
||||
database.password: "root"
|
||||
database.server.name: "ER54"
|
||||
database.include.list: sbtest
|
||||
plugin.name: "pgoutput"
|
||||
table.include.list: "public.tm"
|
||||
clickhouse.server.url: "clickhouse"
|
||||
clickhouse.server.user: "root"
|
||||
clickhouse.server.pass: "root"
|
||||
clickhouse.server.port: "8123"
|
||||
clickhouse.server.database: "test"
|
||||
database.allowPublicKeyRetrieval: "true"
|
||||
snapshot.mode: "schema_only"
|
||||
offset.flush.interval.ms: 5000
|
||||
connector.class: "io.debezium.connector.postgresql.PostgresConnector"
|
||||
offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
|
||||
offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
|
||||
offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123"
|
||||
offset.storage.jdbc.user: "root"
|
||||
offset.storage.jdbc.password: "root"
|
||||
offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
|
||||
(
|
||||
`id` String,
|
||||
`offset_key` String,
|
||||
`offset_val` String,
|
||||
`record_insert_ts` DateTime,
|
||||
`record_insert_seq` UInt64,
|
||||
`_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(_version)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity = 8198"
|
||||
offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
|
||||
schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
|
||||
schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123"
|
||||
schema.history.internal.jdbc.user: "root"
|
||||
schema.history.internal.jdbc.password: "root"
|
||||
schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
|
||||
(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
|
||||
|
||||
schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
|
||||
enable.snapshot.ddl: "false"
|
||||
```
|
||||
|
||||
## Command Line(JAR)
|
||||
Start the application.
|
||||
`java -jar clickhouse-debezium-embedded-1.0-SNAPSHOT.jar mysql_config.yaml`
|
||||
|
||||
`java -jar clickhouse-debezium-embedded-1.0-SNAPSHOT.jar <yaml_config_file>`
|
||||
|
||||
### Docker compose
|
||||
|
||||
**MySQL**
|
||||
```
|
||||
cd docker
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
**PostgreSQL**
|
||||
```
|
||||
cd docker
|
||||
docker-compose -f docker-compose-postgres.yml up
|
||||
```
|
||||
#### Configuration
|
||||
Configuration | Description |
|
||||
|---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
name: "altinity_sink_connector"
|
||||
database.hostname: "mysql-master"
|
||||
database.port: "3306"
|
||||
database.user: "root"
|
||||
|
||||
0
sink-connector-lightweight/docker/config_mongo.yml
Normal file
0
sink-connector-lightweight/docker/config_mongo.yml
Normal file
44
sink-connector-lightweight/docker/config_postgres.yml
Normal file
44
sink-connector-lightweight/docker/config_postgres.yml
Normal file
@ -0,0 +1,44 @@
|
||||
database.hostname: "postgres"
|
||||
database.port: "5432"
|
||||
database.user: "root"
|
||||
database.password: "root"
|
||||
database.server.name: "ER54"
|
||||
database.include.list: sbtest
|
||||
plugin.name: "pgoutput"
|
||||
table.include.list: "public.tm"
|
||||
clickhouse.server.url: "clickhouse"
|
||||
clickhouse.server.user: "root"
|
||||
clickhouse.server.pass: "root"
|
||||
clickhouse.server.port: "8123"
|
||||
clickhouse.server.database: "test"
|
||||
database.allowPublicKeyRetrieval: "true"
|
||||
snapshot.mode: "schema_only"
|
||||
offset.flush.interval.ms: 5000
|
||||
connector.class: "io.debezium.connector.postgresql.PostgresConnector"
|
||||
offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
|
||||
offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
|
||||
offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123"
|
||||
offset.storage.jdbc.user: "root"
|
||||
offset.storage.jdbc.password: "root"
|
||||
offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
|
||||
(
|
||||
`id` String,
|
||||
`offset_key` String,
|
||||
`offset_val` String,
|
||||
`record_insert_ts` DateTime,
|
||||
`record_insert_seq` UInt64,
|
||||
`_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(_version)
|
||||
ORDER BY id
|
||||
SETTINGS index_granularity = 8198"
|
||||
offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
|
||||
schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
|
||||
schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123"
|
||||
schema.history.internal.jdbc.user: "root"
|
||||
schema.history.internal.jdbc.password: "root"
|
||||
schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
|
||||
(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
|
||||
|
||||
schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
|
||||
enable.snapshot.ddl: "false"
|
||||
@ -77,3 +77,6 @@ services:
|
||||
- docker_postgres.env
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
#- ./data:/data
|
||||
- ./config_postgres.yml:/config.yml
|
||||
|
||||
Loading…
Reference in New Issue
Block a user