Merge pull request #337 from Altinity/add_postgres_helm_chart

Added variable to include postgres values file in helm chart.
2024-11-06 02:25:18 +00:00 · 2023-10-23 22:12:02 -04:00 · 2023-10-23 22:12:02 -04:00 · 4465815639
commit 4465815639
parent f45406fa1e 5ce2354b73
3 changed files with 92 additions and 96 deletions
--- a/sink-connector-lightweight/helm/sink-connector-lightweight/README.md
+++ b/sink-connector-lightweight/helm/sink-connector-lightweight/README.md
@ -1,5 +1,14 @@
 To install Helm charts from this repository, add the repository to Helm:

+The configuration is stored in templates/configmap.yaml.
+#ToDo need to move the configuration to values.
+
+For postgres replication, set this variable to true in `values.yaml`
+```#### Postgres connector
+postgres: true
 ```
-helm install sink-connector-lightweight 
+
+```
+cd sink-connector-lightweight/helm/sink-connector-lightweight
+helm install sink-connector-lightweight . 
 ```
--- a/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml
+++ b/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml
@ -1,50 +1,106 @@
+{{- if .Values.postgres -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: sink-connector-lightweight-config
 data:
  config.yml: |-
-    name: "company-1"
-    database.hostname: "mysql"
-    database.port: "3306"
+    name: "debezium-embedded-postgres"
+    database.hostname: "postgres"
+    database.port: "5432"
    database.user: "root"
-    database.password: "password"
+    database.password: "root"
    database.server.name: "ER54"
-    database.include.list: "test"
-    #table.include.list=sbtest1
-    clickhouse.server.url: "chi-argocd-demo-0-0"
+    schema.include.list: public
+    plugin.name: "pgoutput"
+    table.include.list: "public.tm,public.tm2"
+    clickhouse.server.url: "clickhouse"
    clickhouse.server.user: "root"
-    clickhouse.server.pass: "secretsecret"
+    clickhouse.server.password: "root"
    clickhouse.server.port: "8123"
-    clickhouse.server.database: "default"
+    clickhouse.server.database: "test"
    database.allowPublicKeyRetrieval: "true"
    snapshot.mode: "initial"
-    offset.flush.interval.ms: "5000"
-    connector.class: "io.debezium.connector.mysql.MySqlConnector"
+    offset.flush.interval.ms: 5000
+    connector.class: "io.debezium.connector.postgresql.PostgresConnector"
    offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
    offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
-    offset.storage.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector"
+    offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
    offset.storage.jdbc.user: "root"
-    offset.storage.jdbc.password: "secretsecret"
+    offset.storage.jdbc.password: "root"
    offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
-      (
-        `id` String,
-        `offset_key` String,
-        `offset_val` String,
-        `record_insert_ts` DateTime,
-        `record_insert_seq` UInt64,
-        `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
+    (
+      `id` String,
+      `offset_key` String,
+      `offset_val` String,
+      `record_insert_ts` DateTime,
+      `record_insert_seq` UInt64,
+      `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
    )
    ENGINE = ReplacingMergeTree(_version)
    ORDER BY id
    SETTINGS index_granularity = 8198"
    offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
    schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
-    schema.history.internal.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector"
+    schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
    schema.history.internal.jdbc.user: "root"
-    schema.history.internal.jdbc.password: "secretsecret"
+    schema.history.internal.jdbc.password: "root"
    schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
    (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
-    
+
    schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
-    enable.snapshot.ddl: "true"
+    enable.snapshot.ddl: "true"
+    auto.create.tables: "true"
+    database.dbname: "public"
+{{- else -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sink-connector-lightweight-config
+data:
+    config.yml: |-
+      name: "company-1"
+      database.hostname: "mysql"
+      database.port: "3306"
+      database.user: "root"
+      database.password: "password"
+      database.server.name: "ER54"
+      database.include.list: "test"
+      #table.include.list=sbtest1
+      clickhouse.server.url: "chi-argocd-demo-0-0"
+      clickhouse.server.user: "root"
+      clickhouse.server.pass: "secretsecret"
+      clickhouse.server.port: "8123"
+      clickhouse.server.database: "default"
+      database.allowPublicKeyRetrieval: "true"
+      snapshot.mode: "initial"
+      offset.flush.interval.ms: "5000"
+      connector.class: "io.debezium.connector.mysql.MySqlConnector"
+      offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
+      offset.storage.offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info"
+      offset.storage.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector"
+      offset.storage.jdbc.user: "root"
+      offset.storage.jdbc.password: "secretsecret"
+      offset.storage.offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
+        (
+          `id` String,
+          `offset_key` String,
+          `offset_val` String,
+          `record_insert_ts` DateTime,
+          `record_insert_seq` UInt64,
+          `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
+      )
+      ENGINE = ReplacingMergeTree(_version)
+      ORDER BY id
+      SETTINGS index_granularity = 8198"
+      offset.storage.offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
+      schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
+      schema.history.internal.jdbc.url: "jdbc:clickhouse://chi-argocd-demo-0-0:8123/altinity_sink_connector"
+      schema.history.internal.jdbc.user: "root"
+      schema.history.internal.jdbc.password: "secretsecret"
+      schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
+      (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
+
+      schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
+      enable.snapshot.ddl: "true"
+{{- end -}}
--- a/sink-connector-lightweight/helm/sink-connector-lightweight/values.yaml
+++ b/sink-connector-lightweight/helm/sink-connector-lightweight/values.yaml
@ -89,74 +89,5 @@ debezium:
  pod:
    mountPath: "/usr/share/debezium/storage"

-env:
-  - name: database.hostname
-    value: "mysql"
-  - name: database.port
-    value: "3306"
-  - name: database.user
-    value: "root"
-  - name: database.password
-    value: "password"
-  - name: database.server.name
-    value: "ER54"
-  - name: database.include.list
-    value: "test"
-  - name: clickhouse.server.url
-    value: "clickhouse"
-  - name: clickhouse.server.port
-    value: "8123"
-  - name: clickhouse.server.user
-    value: "root"
-  - name: clickhouse.server.password
-    value: "secretsecret"
-  - name: clickhouse.server.database
-    value: "public"
-  - name: database.allowPublicKeyRetrieval
-    value: "true"
-  - name: snapshot.mode
-    value: "initial"
-  - name: offset.flush.interval.ms
-    value: "60000"
-  - name: connector.class
-    value: "io.debezium.connector.mysql.MySqlConnector"
-  - name: offset.storage
-    value: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
-  - name: offset.storage.offset.storage.jdbc.offset.table.name
-    value: "altinity_sink_connector.replica_source_info"
-  - name: offset.storage.jdbc.url
-    value: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
-  - name: offset.storage.jdbc.user
-    value: "root"
-  - name: offset.storage.jdbc.password
-    value: "secretsecret"
-  - name: offset.storage.offset.storage.jdbc.offset.table.ddl
-    value: "CREATE TABLE if not exists %s
-    (
-      `id` String,
-      `offset_key` String,
-      `offset_val` String,
-      `record_insert_ts` DateTime,
-      `record_insert_seq` UInt64,
-      `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
-    )
-    ENGINE = ReplacingMergeTree(_version)
-    ORDER BY id
-    SETTINGS index_granularity = 8198"
-  - name: offset.storage.offset.storage.jdbc.offset.table.delete
-    value: "delete from %s where 1=1"
-  - name: schema.history.internal
-    value: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
-  - name: schema.history.internal.jdbc.url
-    value: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector"
-  - name: schema.history.internal.jdbc.user
-    value: "root"
-  - name: schema.history.internal.jdbc.password
-    value: "secretsecret"
-  - name: schema.history.internal.jdbc.schema.history.table.ddl
-    value: "CREATE TABLE if not exists %s
-    (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
-  - name: schema.history.internal.jdbc.schema.history.table.name
-    value: "altinity_sink_connector.replicate_schema_history"
-  - name: enable.snapshot.ddl
-    value: "true"
+#### Postgres connector
+postgres: true