mirror of
https://github.com/valitydev/clickhouse-sink-connector.git
synced 2024-11-06 02:25:18 +00:00
Updated README instructions.
This commit is contained in:
Kanthi Subramanian
parent
0f851a1996
commit
08f9806916
35
deploy/docker/docker-compose-confluent-schema-registry.yaml → deploy/docker/docker-compose-apicurio-schema-registry.yaml
Executable file → Normal file
35
deploy/docker/docker-compose-confluent-schema-registry.yaml → deploy/docker/docker-compose-apicurio-schema-registry.yaml
Executable file → Normal file
@ -79,18 +79,18 @@ services:
|
||||
|
||||
schemaregistry:
|
||||
container_name: schemaregistry
|
||||
#image: apicurio/apicurio-registry-mem:latest-release
|
||||
image: confluentinc/cp-schema-registry:latest
|
||||
image: apicurio/apicurio-registry-mem:latest-release
|
||||
#image: confluentinc/cp-schema-registry:latest
|
||||
restart: "no"
|
||||
ports:
|
||||
- "8081:8081"
|
||||
#environment:
|
||||
# - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
|
||||
- "8080:8080"
|
||||
environment:
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
|
||||
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
|
||||
- SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081
|
||||
- SCHEMA_REGISTRY_DEBUG=true
|
||||
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
|
||||
# environment:
|
||||
# - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
|
||||
# - SCHEMA_REGISTRY_HOST_NAME=localhost
|
||||
# - SCHEMA_REGISTRY_LISTENERS=http://localhost:8081
|
||||
# - SCHEMA_REGISTRY_DEBUG=true
|
||||
|
||||
depends_on:
|
||||
- kafka
|
||||
@ -101,7 +101,7 @@ services:
|
||||
build:
|
||||
context: ../../docker/debezium_jmx
|
||||
args:
|
||||
DEBEZIUM_VERSION: 1.9.5.Final
|
||||
DEBEZIUM_VERSION: 2.0
|
||||
JMX_AGENT_VERSION: 0.15.0
|
||||
restart: "no"
|
||||
ports:
|
||||
@ -120,8 +120,17 @@ services:
|
||||
- OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium
|
||||
- STATUS_STORAGE_TOPIC=status-storage-topic-debezium
|
||||
- LOG_LEVEL=INFO
|
||||
- KEY_CONVERTER=io.confluent.connect.avro.AvroConverter
|
||||
- VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter
|
||||
- ENABLE_APICURIO_CONVERTERS=true
|
||||
- KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- CONNECT_KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- CONNECT_KEY_CONVERTER_APICURIO.REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
|
||||
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
|
||||
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
|
||||
- CONNECT_VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
|
||||
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
|
||||
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
|
||||
- KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml
|
||||
- JMXHOST=localhost
|
||||
- JMXPORT=1976
|
||||
@ -274,7 +283,7 @@ services:
|
||||
build:
|
||||
context: ../../docker/grafana
|
||||
args:
|
||||
GRAFANA_VERSION: latest
|
||||
GRAFANA_VERSION: 7.5.5
|
||||
#container_name: grafana
|
||||
#image: grafana/grafana
|
||||
restart: "no"
|
||||
35
deploy/docker/docker-compose.yaml
Normal file → Executable file
35
deploy/docker/docker-compose.yaml
Normal file → Executable file
@ -79,18 +79,18 @@ services:
|
||||
|
||||
schemaregistry:
|
||||
container_name: schemaregistry
|
||||
image: apicurio/apicurio-registry-mem:latest-release
|
||||
#image: confluentinc/cp-schema-registry:latest
|
||||
#image: apicurio/apicurio-registry-mem:latest-release
|
||||
image: confluentinc/cp-schema-registry:latest
|
||||
restart: "no"
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "8081:8081"
|
||||
#environment:
|
||||
# - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
|
||||
environment:
|
||||
- KAFKA_BOOTSTRAP_SERVERS=kafka:9092
|
||||
# environment:
|
||||
# - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
|
||||
# - SCHEMA_REGISTRY_HOST_NAME=localhost
|
||||
# - SCHEMA_REGISTRY_LISTENERS=http://localhost:8081
|
||||
# - SCHEMA_REGISTRY_DEBUG=true
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092
|
||||
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
|
||||
- SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081
|
||||
- SCHEMA_REGISTRY_DEBUG=true
|
||||
|
||||
depends_on:
|
||||
- kafka
|
||||
@ -101,7 +101,7 @@ services:
|
||||
build:
|
||||
context: ../../docker/debezium_jmx
|
||||
args:
|
||||
DEBEZIUM_VERSION: 2.0
|
||||
DEBEZIUM_VERSION: 1.9.5.Final
|
||||
JMX_AGENT_VERSION: 0.15.0
|
||||
restart: "no"
|
||||
ports:
|
||||
@ -120,17 +120,8 @@ services:
|
||||
- OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium
|
||||
- STATUS_STORAGE_TOPIC=status-storage-topic-debezium
|
||||
- LOG_LEVEL=INFO
|
||||
- ENABLE_APICURIO_CONVERTERS=true
|
||||
- KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- CONNECT_KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- CONNECT_KEY_CONVERTER_APICURIO.REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
|
||||
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
|
||||
- CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
|
||||
- CONNECT_VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter
|
||||
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2
|
||||
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true
|
||||
- CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true
|
||||
- KEY_CONVERTER=io.confluent.connect.avro.AvroConverter
|
||||
- VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter
|
||||
- KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml
|
||||
- JMXHOST=localhost
|
||||
- JMXPORT=1976
|
||||
@ -283,7 +274,7 @@ services:
|
||||
build:
|
||||
context: ../../docker/grafana
|
||||
args:
|
||||
GRAFANA_VERSION: 7.5.5
|
||||
GRAFANA_VERSION: latest
|
||||
#container_name: grafana
|
||||
#image: grafana/grafana
|
||||
restart: "no"
|
||||
|
||||
34
doc/setup.md
34
doc/setup.md
@ -26,14 +26,23 @@ It will start:
|
||||
4. RedPanda
|
||||
5. clickhouse-kafka-sink-connector
|
||||
6. Clickhouse
|
||||
7. Confluent Schema registry or Apicurio Schema registry
|
||||
|
||||
The `start-docker-compose.sh` by default uses the `latest` tag, you could also pass the docker tag to the script.
|
||||
Altinity sink images are tagged on every successful build with the following format(yyyy-mm-dd) Example(2022-07-19)
|
||||
|
||||
### MySQL:
|
||||
```bash
|
||||
cd deploy/docker
|
||||
./start-docker-compose.sh
|
||||
```
|
||||
|
||||
### Postgres:
|
||||
```bash
|
||||
cd deploy/docker
|
||||
docker-compose -f docker-compose-postgresql.yaml up
|
||||
```
|
||||
|
||||
### Start Docker-compose with a specific docker tag
|
||||
```bash
|
||||
cd deploy/docker
|
||||
@ -48,14 +57,35 @@ cd deploy/docker
|
||||
# Source connector
|
||||
After all the docker containers are up and running, execute the following command
|
||||
to create the Debezium MySQL connector.
|
||||
Make sure MySQL master/slave is up and running before executing the following script.
|
||||
|
||||
Make sure MySQL master/slave is up and running before executing the following script.\
|
||||
|
||||
### MySQL:
|
||||
```bash
|
||||
../deploy/debezium-connector-setup-schema-registry.sh
|
||||
```
|
||||
[debezium-connector-setup-schema-registry.sh](../deploy/debezium-connector-setup-schema-registry.sh)
|
||||
|
||||
### Postgres(Using Apicurio):
|
||||
```bash
|
||||
../deploy/debezium-connector-setup-schema-registry.sh postgres apicurio
|
||||
```
|
||||
|
||||
# Sink Connector
|
||||
After the source connector is created,
|
||||
execute the script [sink-connector-setup-schema-registry.sh](../deploy/sink-connector-setup-schema-registry.sh)
|
||||
to create the Clickhouse Sink connector using Kafka connect REST API
|
||||
|
||||
### MySQL:
|
||||
```bash
|
||||
../deploy/sink-connector-setup-schema-registry.sh
|
||||
```
|
||||
### Postgres(Using Apicurio):
|
||||
```bash
|
||||
../deploy/sink-connector-setup-schema-registry.sh postgres apicurio
|
||||
```
|
||||
|
||||
|
||||
# Deleting connectors
|
||||
The source connector can be deleted using the following script
|
||||
[debezium-delete.sh](../deploy/debezium-delete.sh)
|
||||
@ -66,7 +96,7 @@ The sink connector can be deleted using the following script
|
||||
# References
|
||||
Kafka Connect REST API - (https://docs.confluent.io/platform/current/connect/references/restapi.html)
|
||||
|
||||
[docker-compose.yaml]: ../deploy/docker/docker-compose.yaml
|
||||
[docker-compose.yaml]: ../deploy/docker/docker-compose-apicurio-schema-registry.yaml
|
||||
[Dockerfile]: ../docker/Dockerfile-sink-on-debezium-base-image
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user