Added redpanda UI(console), Strimzi debezium image, Added confluent schema registry ks8 manifest.

2024-11-06 10:35:21 +00:00 · 2022-08-24 02:09:44 -04:00 · 2022-08-24 02:09:44 -04:00 · 1a0cabffcb
commit 1a0cabffcb
parent f2d4be6277
12 changed files with 328 additions and 90 deletions
--- a/deploy/k8s/debezium-apicurio-connect.yaml
+++ b/deploy/k8s/debezium-apicurio-connect.yaml
@ -0,0 +1,48 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaConnect
+metadata:
+  namespace: debezium
+  name: debezium
+  annotations:
+    strimzi.io/use-connector-resources: "true"
+spec:
+  version: 3.1.0
+  replicas: 1
+  image: localhost:5000/altinity/debezium-mysql-source-connector:latest
+  bootstrapServers: redpanda.redpanda:9092
+#  tls:
+#    trustedCertificates:
+#      - secretName: my-cluster-cluster-ca-cert
+#        certificate: ca.crt
+
+  # https://kafka.apache.org/documentation/#connectconfigs
+  #build:
+  #  output:
+  #    type: docker
+  #    image: altinity/debezium-mysql-source-connector:latest
+  #    pushSecret: docker-access-secret
+  #  plugins:
+  #    - name: debezium-mysql-connector
+  #      artifacts:
+  #        - type: tgz
+  #          url: https://repo1.maven.org/maven2/io/debezium/debezium-connector-mysql/1.9.2.Final/debezium-connector-mysql-1.9.2.Final-plugin.tar.gz
+  #        - type: tgz
+  #          url: https://repo1.maven.org/maven2/io/apicurio/apicurio-registry-distro-connect-converter/2.1.5.Final/apicurio-registry-distro-connect-converter-2.1.5.Final.tar.gz
+  config:
+    group.id: 1
+
+    # -1 means it will use the default replication factor configured in the broker
+    config.storage.topic: config-storage-topic-debezium
+    config.storage.replication.factor: 1
+    config.storage.partitions: 1
+
+    offset.storage.topic: offset-storage-topic-debezium
+    offset.storage.replication.factor: 1
+    offset.storage.partitions: 1
+
+    status.storage.topic: status-storage-topic-debezium
+    status.storage.replication.factor: 1
+    status.storage.partitions: 1
+
+    key.converter: "io.apicurio.registry.utils.converter.AvroConverter"
+    value.converter: "io.apicurio.registry.utils.converter.AvroConverter"
--- a/deploy/k8s/debezium-connect.yaml
+++ b/deploy/k8s/debezium-connect.yaml
@ -2,13 +2,13 @@ apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaConnect
 metadata:
  namespace: debezium
-  name: debezium
+  name: debezium-kafkaconnect
  annotations:
    strimzi.io/use-connector-resources: "true"
 spec:
  version: 3.1.0
  replicas: 1
-  image: localhost:5000/altinity/debezium-mysql-source-connector:latest
+  image: subkanthi/debezium-mysql-source-connector:latest
  bootstrapServers: redpanda.redpanda:9092
 #  tls:
 #    trustedCertificates:
@ -44,5 +44,5 @@ spec:
    status.storage.replication.factor: 1
    status.storage.partitions: 1

-    key.converter: "io.apicurio.registry.utils.converter.AvroConverter"
-    value.converter: "io.apicurio.registry.utils.converter.AvroConverter"
+    key.converter: "io.confluent.connect.avro.AvroConverter"
+    value.converter: "io.confluent.connect.avro.AvroConverter"
--- a/deploy/k8s/debezium-connector-apicurio-avro.yaml
+++ b/deploy/k8s/debezium-connector-apicurio-avro.yaml
@ -0,0 +1,52 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaConnector
+metadata:
+  namespace: debezium
+  name: debezium
+  labels:
+    strimzi.io/cluster: debezium
+spec:
+  class: io.debezium.connector.mysql.MySqlConnector
+  tasksMax: 1
+  config:
+    # List of mysql options:
+    # https://debezium.io/documentation/reference/stable/connectors/mysql.html#_required_debezium_mysql_connector_configuration_properties
+
+    snapshot.mode: "initial"
+    snapshot.locking.mode: "minimal"
+    snapshot.delay.ms: "10000"
+    include.schema.changes: "true"
+
+    database.hostname: "${MYSQL_HOST}"
+    database.port: "${MYSQL_PORT}"
+    database.user: "${MYSQL_USER}"
+    database.password: "${MYSQL_PASSWORD}"
+    database.server.id: "${DATABASE_SERVER_ID}"
+    database.server.name: "${DATABASE_SERVER_NAME}"
+    database.include.list: "${MYSQL_DBS}"
+    # In order to deal with exception
+    # "message": "Connector configuration is invalid and contains the following 1 error(s):\nUnable to connect: Public Key Retrieval is not allowed\nYou can also find the above list of errors at the endpoint `/{connectorType}/config/validate`"
+    database.allowPublicKeyRetrieval: "true"
+    database.useSSL: "false"
+
+    database.history.kafka.bootstrap.servers: "${KAFKA_BOOTSTRAP_SERVERS}"
+    database.history.kafka.topic: "${KAFKA_TOPIC}"
+
+    # List of topic options:
+    # https://debezium.io/documentation/reference/stable/configuration/topic-auto-create-config.html
+    topic.creation.default.replication.factor: "1"
+    topic.creation.default.partitions: "1"
+    # ?
+    topic.creation.$alias.partitions: "1"
+
+    # Avro serialization description:
+    # https://debezium.io/documentation/reference/stable/configuration/avro.html
+    key.converter: "io.apicurio.registry.utils.converter.AvroConverter"
+    key.converter.apicurio.registry.url: "${REGISTRY_URL}"
+    key.converter.apicurio.registry.auto-register: "true"
+    key.converter.apicurio.registry.find-latest: "true"
+
+    value.converter: "io.apicurio.registry.utils.converter.AvroConverter"
+    value.converter.apicurio.registry.url: "${REGISTRY_URL}"
+    value.converter.apicurio.registry.auto-register: "true"
+    value.converter.apicurio.registry.find-latest: "true"
--- a/deploy/k8s/debezium-connector-avro.yaml
+++ b/deploy/k8s/debezium-connector-avro.yaml
@ -2,16 +2,13 @@ apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaConnector
 metadata:
  namespace: debezium
-  name: debezium
+  name: debezium-kctr
  labels:
-    strimzi.io/cluster: debezium
+    strimzi.io/cluster: debezium-kafkaconnect
 spec:
  class: io.debezium.connector.mysql.MySqlConnector
  tasksMax: 1
  config:
-    # List of mysql options:
-    # https://debezium.io/documentation/reference/stable/connectors/mysql.html#_required_debezium_mysql_connector_configuration_properties
-
    snapshot.mode: "initial"
    snapshot.locking.mode: "minimal"
    snapshot.delay.ms: "10000"
@ -29,8 +26,8 @@ spec:
    database.allowPublicKeyRetrieval: "true"
    database.useSSL: "false"

-    database.history.kafka.bootstrap.servers: "${KAFKA_BOOTSTRAP_SERVERS}"
-    database.history.kafka.topic: "${KAFKA_TOPIC}"
+    database.history.kafka.bootstrap.servers: "redpanda.redpanda:9092"
+    database.history.kafka.topic: "schema-changes.dashboard"

    # List of topic options:
    # https://debezium.io/documentation/reference/stable/configuration/topic-auto-create-config.html
@ -41,12 +38,8 @@ spec:

    # Avro serialization description:
    # https://debezium.io/documentation/reference/stable/configuration/avro.html
-    key.converter: "io.apicurio.registry.utils.converter.AvroConverter"
-    key.converter.apicurio.registry.url: "${REGISTRY_URL}"
-    key.converter.apicurio.registry.auto-register: "true"
-    key.converter.apicurio.registry.find-latest: "true"
+    key.converter: "io.confluent.connect.avro.AvroConverter"
+    value.converter: "io.confluent.connect.avro.AvroConverter"

-    value.converter: "io.apicurio.registry.utils.converter.AvroConverter"
-    value.converter.apicurio.registry.url: "${REGISTRY_URL}"
-    value.converter.apicurio.registry.auto-register: "true"
-    value.converter.apicurio.registry.find-latest: "true"
+    key.converter.schema.registry.url : "http://schemaregistry.registry:8081"
+    value.converter.schema.registry.url: "http://schemaregistry.registry:8081"
--- a/deploy/k8s/mysql-client.yaml
+++ b/deploy/k8s/mysql-client.yaml
@ -0,0 +1,43 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+spec:
+  ports:
+    - port: 3306
+  selector:
+    app: mysql
+  clusterIP: None
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mysql
+spec:
+  selector:
+    matchLabels:
+      app: mysql
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      containers:
+        - image: mysql:5.6
+          name: mysql
+          env:
+            # Use secret in real usage
+            - name: MYSQL_ROOT_PASSWORD
+              value: password
+          ports:
+            - containerPort: 3306
+              name: mysql
+#          volumeMounts:
+#            - name: mysql-persistent-storage
+#              mountPath: /var/lib/mysql
+#      volumes:
+#        - name: mysql-persistent-storage
+#          persistentVolumeClaim:
+#            claimName: mysql-pv-claim
--- a/deploy/k8s/redpanda-external.yaml
+++ b/deploy/k8s/redpanda-external.yaml
@ -27,4 +27,4 @@ spec:
          enabled: true
    adminApi:
      - port: 9644
-    developerMode: true
+    developerMode: false
--- a/deploy/k8s/redpanda-ui.yaml
+++ b/deploy/k8s/redpanda-ui.yaml
@ -0,0 +1,49 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  namespace: redpanda
+  name: redpanda-ui
+  labels:
+    app: redpanda-ui
+spec:
+  selector:
+    matchLabels:
+      app: redpanda-ui
+  template:
+    metadata:
+      labels:
+        app: redpanda-ui
+    spec:
+      enableServiceLinks: false
+      containers:
+        - name: redpanda-ui
+          env:
+            - name: CONFIG_FILEPATH
+              value: /tmp/config.yml
+            - name: CONSOLE_CONFIG_FILE
+              value:  |
+                kafka:
+                  brokers: ["redpanda.redpanda:9092"]
+                connect:
+                  enabled: true
+                  clusters:
+                    - name: datagen
+                      url: http://debezium:8083
+          image: docker.redpanda.com/vectorized/console:master-217260f
+          command: [ "/bin/sh" ]
+          args: [ "-c", "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console" ]
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: redpanda-ui
+spec:
+  selector:
+    app: redpanda-ui
+  ports:
+    - protocol: TCP
+      port: 8089
+      targetPort: 8080
--- a/deploy/k8s/schema-registry-confluent.yaml
+++ b/deploy/k8s/schema-registry-confluent.yaml
@ -1,68 +0,0 @@
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: schema-registry
-data:
-  SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: redpanda.redpanda:9092
-  SCHEMA_REGISTRY_HOST_NAME: schema-registry
-  SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
-  SCHEMA_REGISTRY_DEBUG: true
-  #APPLICATION_ID: schema-registry
-  #QUARKUS_PROFILE: prod # dev/prod
---
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  name: schema-registry
-  labels:
-    app: schema-registry
-spec:
-  selector:
-    matchLabels:
-      app: schema-registry
-  template:
-    metadata:
-      labels:
-        app: schema-registry
-    spec:
-      containers:
-        - name: schema-registry
-          image: confluentinc/cp-schema-registry:latest
-          imagePullPolicy: IfNotPresent
-          envFrom:
-            - configMapRef:
-                name: schema-registry
-          ports:
-            - containerPort: 8080
-          livenessProbe:
-            httpGet:
-              path: /health/live
-              port: 8080
-              scheme: HTTP
-            initialDelaySeconds: 5
-            timeoutSeconds: 5
-            periodSeconds: 10
-            successThreshold: 1
-            failureThreshold: 3
-          readinessProbe:
-            httpGet:
-              path: /health/ready
-              port: 8080
-              scheme: HTTP
-            initialDelaySeconds: 5
-            timeoutSeconds: 5
-            periodSeconds: 10
-            successThreshold: 1
-            failureThreshold: 3
---
-kind: Service
-apiVersion: v1
-metadata:
-  name: schema-registry
-spec:
-  selector:
-    app: schema-registry
-  ports:
-    - protocol: TCP
-      port: 8080
-      targetPort: 8080
--- a/deploy/k8s/schema-registry.yaml
+++ b/deploy/k8s/schema-registry.yaml
@ -0,0 +1,90 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: schema-registry
+data:
+  SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: redpanda.redpanda:9092
+  SCHEMA_REGISTRY_HOST_NAME: avro-schema-registry
+  SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+  SCHEMA_REGISTRY_DEBUG: "true"
+  #APPLICATION_ID: schema-registry
+  #QUARKUS_PROFILE: prod # dev/prod
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: schema-registry
+  labels:
+    app: schema-registry
+spec:
+  selector:
+    matchLabels:
+      app: schema-registry
+  template:
+    metadata:
+      labels:
+        app: schema-registry
+    spec:
+      enableServiceLinks: false
+      containers:
+        - name: schema-registry
+          env:
+            - name: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS
+              value: redpanda.redpanda:9092
+            - name: SCHEMA_REGISTRY_HOST_NAME
+              value: schema-registry
+            - name: SCHEMA_REGISTRY_LISTENERS
+              value: http://0.0.0.0:8081
+            - name: SCHEMA_REGISTRY_DEBUG
+              value: "true"
+          image: confluentinc/cp-schema-registry:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8081
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 8081
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8081
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+#          livenessProbe:
+#            httpGet:
+#              path: /health/live
+#              port: 8081
+#              scheme: HTTP
+#            initialDelaySeconds: 5
+#            timeoutSeconds: 5
+#            periodSeconds: 10
+#            successThreshold: 1
+#            failureThreshold: 3
+#          readinessProbe:
+#            httpGet:
+#              path: /health/ready
+#              port: 8081
+#              scheme: HTTP
+#            initialDelaySeconds: 5
+#            timeoutSeconds: 5
+#            periodSeconds: 10
+#            successThreshold: 1
+#            failureThreshold: 3
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: schema-registry
+spec:
+  selector:
+    app: schema-registry
+  ports:
+    - protocol: TCP
+      port: 8081
+      targetPort: 8081
--- a/doc/k8s_pipeline_setup.md
+++ b/doc/k8s_pipeline_setup.md
@ -1,3 +1,7 @@
+## Start local Docker registry
+```bash
+docker run -d -p 5000:5000 --restart=always --name registry registry:2
+```

 ## RedPanda

@ -177,7 +181,7 @@ kubectl -n $NAMESPACE rollout status -w statefulset/chi-clickhouse-cluster-0-0
 kubectl -n $NAMESPACE get statefulset
 ```

-### schema registry
+### schema registry(Confluent)

 ```bash
 NAMESPACE="registry"
--- a/docker/Dockerfile-debezium-on-strimzi-image
+++ b/docker/Dockerfile-debezium-on-strimzi-image
@ -14,4 +14,20 @@ RUN 'mkdir' '-p' '/opt/kafka/plugins/debezium-mysql-connector/3542976d' \
      && 'tar' 'xvfz' '/opt/kafka/plugins/debezium-mysql-connector/3542976d.tgz' '-C' '/opt/kafka/plugins/debezium-mysql-connector/3542976d' \
      && 'rm' '-vf' '/opt/kafka/plugins/debezium-mysql-connector/3542976d.tgz'

-USER 1001
+
+# Copy confluent
+RUN 'curl' '-L' '--output' '/opt/kafka/libs/kafka-connect-avro-converter-3.1.0.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-converter/3.1.0/kafka-connect-avro-converter-3.1.0.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/libs/kafka-avro-serializer-3.1.0.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-avro-serializer/3.1.0/kafka-avro-serializer-3.1.0.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/libs/kafka-schema-registry-3.1.0.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-schema-registry/3.1.0/kafka-schema-registry-3.1.0.jar'
+
+
+RUN 'curl' '-L' '--output' '/opt/kafka/libs/kafka-schema-registry-client-3.1.0.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-schema-registry-client/3.1.0/kafka-schema-registry-client-3.1.0.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/libs/common-config-3.1.0.jar' 'https://packages.confluent.io/maven/io/confluent/common-config/3.1.0/common-config-3.1.0.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/libs/common-utils-3.1.0.jar' 'https://packages.confluent.io/maven/io/confluent/common-utils/3.1.0/common-utils-3.1.0.jar'
+
+RUN 'curl' '-L' '--output' '/opt/kafka/libs/avro-1.10.1.jar' 'https://repo1.maven.org/maven2/org/apache/avro/avro/1.10.1/avro-1.10.1.jar'
+
+#RUN 'curl' '-L' '--output' '/opt/kafka/plugins/debezium-mysql-connector/3542976d/kafka-serde-tools-package-3.1.0-tests.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-serde-tools-package/3.1.0/kafka-serde-tools-package-3.1.0-tests.jar'
+
+
+#USER 1001
--- a/docker/Dockerfile-sink-on-strimzi-image
+++ b/docker/Dockerfile-sink-on-strimzi-image
@ -16,4 +16,15 @@ RUN 'mkdir' '-p' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d' \
      && 'tar' 'xvfz' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d.tgz' '-C' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d' \
      && 'rm' '-vf' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d.tgz'

+# Copy confluent
+RUN 'curl' '-L' '--output' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d/kafka-connect-avro-converter-7.2.1.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-converter/7.2.1/kafka-connect-avro-converter-7.2.1.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d/kafka-avro-serializer-7.2.1.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-avro-serializer/7.2.1/kafka-avro-serializer-7.2.1.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d/kafka-schema-registry-7.2.1.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-schema-registry/7.2.1/kafka-schema-registry-7.2.1.jar'
+
+
+RUN 'curl' '-L' '--output' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d/kafka-schema-registry-client-7.2.1.jar' 'https://packages.confluent.io/maven/io/confluent/kafka-schema-registry-client/7.2.1/kafka-schema-registry-client-7.2.1.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d/common-config-7.2.1.jar' 'https://packages.confluent.io/maven/io/confluent/common-config/7.2.1/common-config-7.2.1.jar'
+RUN 'curl' '-L' '--output' '/opt/kafka/plugins/clickhouse-sink-connector/3542976d/common-utils-7.2.1.jar' 'https://packages.confluent.io/maven/io/confluent/common-utils/7.2.1/common-utils-7.2.1.jar'
+
+
 USER 1001