diff --git a/deploy/docker/docker-compose-confluent-schema-registry.yaml b/deploy/docker/docker-compose-apicurio-schema-registry.yaml old mode 100755 new mode 100644 similarity index 85% rename from deploy/docker/docker-compose-confluent-schema-registry.yaml rename to deploy/docker/docker-compose-apicurio-schema-registry.yaml index 3211ad5..a6848b2 --- a/deploy/docker/docker-compose-confluent-schema-registry.yaml +++ b/deploy/docker/docker-compose-apicurio-schema-registry.yaml @@ -79,18 +79,18 @@ services: schemaregistry: container_name: schemaregistry - #image: apicurio/apicurio-registry-mem:latest-release - image: confluentinc/cp-schema-registry:latest + image: apicurio/apicurio-registry-mem:latest-release + #image: confluentinc/cp-schema-registry:latest restart: "no" ports: - - "8081:8081" - #environment: - # - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + - "8080:8080" environment: - - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092 - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081 - - SCHEMA_REGISTRY_DEBUG=true + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + # environment: + # - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092 + # - SCHEMA_REGISTRY_HOST_NAME=localhost + # - SCHEMA_REGISTRY_LISTENERS=http://localhost:8081 + # - SCHEMA_REGISTRY_DEBUG=true depends_on: - kafka @@ -101,7 +101,7 @@ services: build: context: ../../docker/debezium_jmx args: - DEBEZIUM_VERSION: 1.9.5.Final + DEBEZIUM_VERSION: 2.0 JMX_AGENT_VERSION: 0.15.0 restart: "no" ports: @@ -120,8 +120,17 @@ services: - OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium - STATUS_STORAGE_TOPIC=status-storage-topic-debezium - LOG_LEVEL=INFO - - KEY_CONVERTER=io.confluent.connect.avro.AvroConverter - - VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter + - ENABLE_APICURIO_CONVERTERS=true + - KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter + - VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter + - CONNECT_KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter + - CONNECT_KEY_CONVERTER_APICURIO.REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2 + - CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true + - CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true + - CONNECT_VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter + - CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2 + - CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true + - CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true - KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml - JMXHOST=localhost - JMXPORT=1976 @@ -274,7 +283,7 @@ services: build: context: ../../docker/grafana args: - GRAFANA_VERSION: latest + GRAFANA_VERSION: 7.5.5 #container_name: grafana #image: grafana/grafana restart: "no" diff --git a/deploy/docker/docker-compose.yaml b/deploy/docker/docker-compose.yaml old mode 100644 new mode 100755 index a6848b2..3211ad5 --- a/deploy/docker/docker-compose.yaml +++ b/deploy/docker/docker-compose.yaml @@ -79,18 +79,18 @@ services: schemaregistry: container_name: schemaregistry - image: apicurio/apicurio-registry-mem:latest-release - #image: confluentinc/cp-schema-registry:latest + #image: apicurio/apicurio-registry-mem:latest-release + image: confluentinc/cp-schema-registry:latest restart: "no" ports: - - "8080:8080" + - "8081:8081" + #environment: + # - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 environment: - - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 - # environment: - # - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092 - # - SCHEMA_REGISTRY_HOST_NAME=localhost - # - SCHEMA_REGISTRY_LISTENERS=http://localhost:8081 - # - SCHEMA_REGISTRY_DEBUG=true + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092 + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081 + - SCHEMA_REGISTRY_DEBUG=true depends_on: - kafka @@ -101,7 +101,7 @@ services: build: context: ../../docker/debezium_jmx args: - DEBEZIUM_VERSION: 2.0 + DEBEZIUM_VERSION: 1.9.5.Final JMX_AGENT_VERSION: 0.15.0 restart: "no" ports: @@ -120,17 +120,8 @@ services: - OFFSET_STORAGE_TOPIC=offset-storage-topic-debezium - STATUS_STORAGE_TOPIC=status-storage-topic-debezium - LOG_LEVEL=INFO - - ENABLE_APICURIO_CONVERTERS=true - - KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter - - VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter - - CONNECT_KEY_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter - - CONNECT_KEY_CONVERTER_APICURIO.REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2 - - CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true - - CONNECT_KEY_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true - - CONNECT_VALUE_CONVERTER=io.apicurio.registry.utils.converter.AvroConverter - - CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_URL=http://schemaregistry:8080/apis/registry/v2 - - CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_AUTO-REGISTER=true - - CONNECT_VALUE_CONVERTER_APICURIO_REGISTRY_FIND-LATEST=true + - KEY_CONVERTER=io.confluent.connect.avro.AvroConverter + - VALUE_CONVERTER=io.confluent.connect.avro.AvroConverter - KAFKA_OPTS=-javaagent:/kafka/etc/jmx_prometheus_javaagent.jar=8080:/kafka/etc/config.yml - JMXHOST=localhost - JMXPORT=1976 @@ -283,7 +274,7 @@ services: build: context: ../../docker/grafana args: - GRAFANA_VERSION: 7.5.5 + GRAFANA_VERSION: latest #container_name: grafana #image: grafana/grafana restart: "no" diff --git a/doc/setup.md b/doc/setup.md index d4a8849..078666c 100644 --- a/doc/setup.md +++ b/doc/setup.md @@ -26,14 +26,23 @@ It will start: 4. RedPanda 5. clickhouse-kafka-sink-connector 6. Clickhouse +7. Confluent Schema registry or Apicurio Schema registry The `start-docker-compose.sh` by default uses the `latest` tag, you could also pass the docker tag to the script. Altinity sink images are tagged on every successful build with the following format(yyyy-mm-dd) Example(2022-07-19) + +### MySQL: ```bash cd deploy/docker ./start-docker-compose.sh ``` +### Postgres: +```bash +cd deploy/docker +docker-compose -f docker-compose-postgresql.yaml up +``` + ### Start Docker-compose with a specific docker tag ```bash cd deploy/docker @@ -48,14 +57,35 @@ cd deploy/docker # Source connector After all the docker containers are up and running, execute the following command to create the Debezium MySQL connector. -Make sure MySQL master/slave is up and running before executing the following script. + +Make sure MySQL master/slave is up and running before executing the following script.\ + +### MySQL: +```bash + ../deploy/debezium-connector-setup-schema-registry.sh +``` [debezium-connector-setup-schema-registry.sh](../deploy/debezium-connector-setup-schema-registry.sh) +### Postgres(Using Apicurio): +```bash +../deploy/debezium-connector-setup-schema-registry.sh postgres apicurio +``` + # Sink Connector After the source connector is created, execute the script [sink-connector-setup-schema-registry.sh](../deploy/sink-connector-setup-schema-registry.sh) to create the Clickhouse Sink connector using Kafka connect REST API +### MySQL: +```bash + ../deploy/sink-connector-setup-schema-registry.sh +``` +### Postgres(Using Apicurio): +```bash +../deploy/sink-connector-setup-schema-registry.sh postgres apicurio +``` + + # Deleting connectors The source connector can be deleted using the following script [debezium-delete.sh](../deploy/debezium-delete.sh) @@ -66,7 +96,7 @@ The sink connector can be deleted using the following script # References Kafka Connect REST API - (https://docs.confluent.io/platform/current/connect/references/restapi.html) -[docker-compose.yaml]: ../deploy/docker/docker-compose.yaml +[docker-compose.yaml]: ../deploy/docker/docker-compose-apicurio-schema-registry.yaml [Dockerfile]: ../docker/Dockerfile-sink-on-debezium-base-image