From 5ce2354b73db006bcb596045bfbb47896b6815a8 Mon Sep 17 00:00:00 2001 From: kanthi Subramanian Date: Mon, 23 Oct 2023 22:09:05 -0400 Subject: [PATCH] Added variable to include postgres values file in helm chart. --- .../helm/sink-connector-lightweight/README.md | 11 +- .../templates/configmap.yaml | 104 ++++++++++++++---- .../sink-connector-lightweight/values.yaml | 73 +----------- 3 files changed, 92 insertions(+), 96 deletions(-) diff --git a/sink-connector-lightweight/helm/sink-connector-lightweight/README.md b/sink-connector-lightweight/helm/sink-connector-lightweight/README.md index 5fe6789..9960375 100644 --- a/sink-connector-lightweight/helm/sink-connector-lightweight/README.md +++ b/sink-connector-lightweight/helm/sink-connector-lightweight/README.md @@ -1,5 +1,14 @@ To install Helm charts from this repository, add the repository to Helm: +The configuration is stored in templates/configmap.yaml. +#ToDo need to move the configuration to values. + +For postgres replication, set this variable to true in `values.yaml` +```#### Postgres connector +postgres: true ``` -helm install sink-connector-lightweight + +``` +cd sink-connector-lightweight/helm/sink-connector-lightweight +helm install sink-connector-lightweight . ``` \ No newline at end of file diff --git a/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml b/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml index 32480d7..828a2a6 100644 --- a/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml +++ b/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml @@ -1,50 +1,106 @@ +{{- if .Values.postgres -}} apiVersion: v1 kind: ConfigMap metadata: name: sink-connector-lightweight-config data: config.yml: |- - name: "company-1" - database.hostname: "mysql" - database.port: "3306" + name: "debezium-embedded-postgres" + database.hostname: "postgres" + database.port: "5432" database.user: "root" - database.password: "password" + database.password: "root" database.server.name: "ER54" - database.include.list: "test" - #table.include.list=sbtest1 - clickhouse.server.url: "chi-argocd-demo-0-0" + schema.include.list: public + plugin.name: "pgoutput" + table.include.list: "public.tm,public.tm2" + clickhouse.server.url: "clickhouse" clickhouse.server.user: "root" - clickhouse.server.pass: "secretsecret" + clickhouse.server.password: "root" clickhouse.server.port: "8123" - clickhouse.server.database: "default" + clickhouse.server.database: "test" database.allowPublicKeyRetrieval: "true" snapshot.mode: "initial" - offset.flush.interval.ms: "5000" - connector.class: "io.debezium.connector.mysql.MySqlConnector" + offset.flush.interval.ms: 5000 + connector.class: "io.debezium.connector.postgresql.PostgresConnector" offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore" offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info" - offset.storage.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector" + offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector" offset.storage.jdbc.user: "root" - offset.storage.jdbc.password: "secretsecret" + offset.storage.jdbc.password: "root" offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s - ( - `id` String, - `offset_key` String, - `offset_val` String, - `record_insert_ts` DateTime, - `record_insert_seq` UInt64, - `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9)) + ( + `id` String, + `offset_key` String, + `offset_val` String, + `record_insert_ts` DateTime, + `record_insert_seq` UInt64, + `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9)) ) ENGINE = ReplacingMergeTree(_version) ORDER BY id SETTINGS index_granularity = 8198" offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1" schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory" - schema.history.internal.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector" + schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector" schema.history.internal.jdbc.user: "root" - schema.history.internal.jdbc.password: "secretsecret" + schema.history.internal.jdbc.password: "root" schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id" - + schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history" - enable.snapshot.ddl: "true" \ No newline at end of file + enable.snapshot.ddl: "true" + auto.create.tables: "true" + database.dbname: "public" +{{- else -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: sink-connector-lightweight-config +data: + config.yml: |- + name: "company-1" + database.hostname: "mysql" + database.port: "3306" + database.user: "root" + database.password: "password" + database.server.name: "ER54" + database.include.list: "test" + #table.include.list=sbtest1 + clickhouse.server.url: "chi-argocd-demo-0-0" + clickhouse.server.user: "root" + clickhouse.server.pass: "secretsecret" + clickhouse.server.port: "8123" + clickhouse.server.database: "default" + database.allowPublicKeyRetrieval: "true" + snapshot.mode: "initial" + offset.flush.interval.ms: "5000" + connector.class: "io.debezium.connector.mysql.MySqlConnector" + offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore" + offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info" + offset.storage.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector" + offset.storage.jdbc.user: "root" + offset.storage.jdbc.password: "secretsecret" + offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s + ( + `id` String, + `offset_key` String, + `offset_val` String, + `record_insert_ts` DateTime, + `record_insert_seq` UInt64, + `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9)) + ) + ENGINE = ReplacingMergeTree(_version) + ORDER BY id + SETTINGS index_granularity = 8198" + offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1" + schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory" + schema.history.internal.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector" + schema.history.internal.jdbc.user: "root" + schema.history.internal.jdbc.password: "secretsecret" + schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s + (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id" + + schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history" + enable.snapshot.ddl: "true" +{{- end -}} \ No newline at end of file diff --git a/sink-connector-lightweight/helm/sink-connector-lightweight/values.yaml b/sink-connector-lightweight/helm/sink-connector-lightweight/values.yaml index b813239..ea26591 100644 --- a/sink-connector-lightweight/helm/sink-connector-lightweight/values.yaml +++ b/sink-connector-lightweight/helm/sink-connector-lightweight/values.yaml @@ -89,74 +89,5 @@ debezium: pod: mountPath: "/usr/share/debezium/storage" -env: - - name: database.hostname - value: "mysql" - - name: database.port - value: "3306" - - name: database.user - value: "root" - - name: database.password - value: "password" - - name: database.server.name - value: "ER54" - - name: database.include.list - value: "test" - - name: clickhouse.server.url - value: "clickhouse" - - name: clickhouse.server.port - value: "8123" - - name: clickhouse.server.user - value: "root" - - name: clickhouse.server.password - value: "secretsecret" - - name: clickhouse.server.database - value: "public" - - name: database.allowPublicKeyRetrieval - value: "true" - - name: snapshot.mode - value: "initial" - - name: offset.flush.interval.ms - value: "60000" - - name: connector.class - value: "io.debezium.connector.mysql.MySqlConnector" - - name: offset.storage - value: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore" - - name: offset.storage.offset.storage.jdbc.offset.table.name - value: "altinity_sink_connector.replica_source_info" - - name: offset.storage.jdbc.url - value: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector" - - name: offset.storage.jdbc.user - value: "root" - - name: offset.storage.jdbc.password - value: "secretsecret" - - name: offset.storage.offset.storage.jdbc.offset.table.ddl - value: "CREATE TABLE if not exists %s - ( - `id` String, - `offset_key` String, - `offset_val` String, - `record_insert_ts` DateTime, - `record_insert_seq` UInt64, - `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9)) - ) - ENGINE = ReplacingMergeTree(_version) - ORDER BY id - SETTINGS index_granularity = 8198" - - name: offset.storage.offset.storage.jdbc.offset.table.delete - value: "delete from %s where 1=1" - - name: schema.history.internal - value: "io.debezium.storage.jdbc.history.JdbcSchemaHistory" - - name: schema.history.internal.jdbc.url - value: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector" - - name: schema.history.internal.jdbc.user - value: "root" - - name: schema.history.internal.jdbc.password - value: "secretsecret" - - name: schema.history.internal.jdbc.schema.history.table.ddl - value: "CREATE TABLE if not exists %s - (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id" - - name: schema.history.internal.jdbc.schema.history.table.name - value: "altinity_sink_connector.replicate_schema_history" - - name: enable.snapshot.ddl - value: "true" +#### Postgres connector +postgres: true