Add kafka-resources for core, connect & schema-registry

kafka-resources/base/kafka-connect/connect-configs-topic.yaml

@@ -0,0 +1,13 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: kafka-connect-cluster-configs
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: kafka-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    # http://kafka.apache.org/documentation/#topicconfigs
+    cleanup.policy: compact

kafka-resources/base/kafka-connect/connect-offsets-topic.yaml

@@ -0,0 +1,13 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: kafka-connect-cluster-offsets
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: kafka-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    # http://kafka.apache.org/documentation/#topicconfigs
+    cleanup.policy: compact

kafka-resources/base/kafka-connect/connect-status-topic.yaml

@@ -0,0 +1,13 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: kafka-connect-cluster-status
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: kafka-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    # http://kafka.apache.org/documentation/#topicconfigs
+    cleanup.policy: compact

kafka-resources/base/kafka-connect/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- connect-configs-topic.yaml
+- connect-offsets-topic.yaml
+- connect-status-topic.yaml
+- s3-kafka-connect.yaml
+- s3-kafka-connector.yaml

kafka-resources/base/kafka-connect/s3-kafka-connect.yaml

@@ -0,0 +1,69 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaConnect
+metadata:
+  name: kafka-connect-cluster
+  namespace: kafka
+  annotations:
+    strimzi.io/use-connector-resources: "true"
+spec:
+  replicas: 1
+  bootstrapServers: kafka-cluster-kafka-plainext-bootstrap:9092
+  # tls:
+  #   trustedCertificates:
+  #   - secretName: my-cluster-cluster-ca-cert
+  #     certificate: ca.crt
+  config:
+    group.id: kafka-connect-cluster
+    offset.storage.topic: kafka-connect-cluster-offsets
+    config.storage.topic: kafka-connect-cluster-configs
+    status.storage.topic: kafka-connect-cluster-status
+    key.converter: org.apache.kafka.connect.storage.StringConverter
+    value.converter: io.confluent.connect.avro.AvroConverter
+    key.converter.schema.registry.url: http://89.47.191.210:30081
+    value.converter.schema.registry.url: http://89.47.191.210:30081
+    config.storage.replication.factor: 1
+    offset.storage.replication.factor: 1
+    status.storage.replication.factor: 1
+  resources:
+    requests:
+      cpu: "1"
+      memory: 1Gi
+    limits:
+      cpu: "2"
+      memory: 1Gi
+  externalConfiguration:
+    env:
+    - name: AWS_ACCESS_KEY_ID
+      valueFrom:
+        secretKeyRef:
+          name: s3-minio-creds
+          key: AWS_ACCESS_KEY_ID
+    - name: AWS_SECRET_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: s3-minio-creds
+          key: AWS_SECRET_ACCESS_KEY
+  build:
+    output:
+      type: docker
+      image: platform.sunet.se/benedith/das-kafka-connect-cluster:latest
+      pushSecret: docker-platform-creds
+    plugins:
+    - name: kafka-connect-s3
+      artifacts:
+      - type: zip
+        url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/confluentinc-kafka-connect-s3-10.5.17.zip
+        sha512sum: 51dc4eb5e618a7743b3d29c7c5586f5bda00a254a9f105ee816cad7c8e9509a7c1a1ea43351e77dcf97847900c21895962716ed6a1bfb2de4a2b4695233d8804
+      - type: zip
+        url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/confluentinc-kafka-connect-avro-converter-7.7.1.zip
+        sha512sum: cebc6fece5c5551d3cff5f1cc8f4660e83da6292a9d695c1f8851af880661b2882e59ef0eeb3df395c3fc314e483cc26961d6a6df271237aab7ef2d8732af3f4
+      - type: zip
+        url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/jcustenborder-kafka-connect-transform-common-0.1.0.58.zip
+        sha512sum: 17b80dcf0944baf3024af28a81d61140dcad3e75af4b08807d31349ba0d51cb871dca5f123174973150dd8dace4cf72511c02e4f8c1ec94ac6206d7ccf9431a9
+      - type: jar
+        url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/kafka-clients-3.8.0.jar
+        sha512sum: e61c2ab5e7d3daa1e546904512815ee9b3e3ac1cd887aa44ce920bc1597f13cc5f097dfc0c011b4dea23650a5394ba8e060104f0064d1ad37b589b11dcb0fa3c
+  template:
+    pod:
+      imagePullSecrets:
+      - name: docker-platform-creds

kafka-resources/base/kafka-connect/s3-kafka-connector.yaml

@@ -0,0 +1,59 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaConnector
+metadata:
+  name: kafka-connect-s3-connector
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: kafka-connect-cluster
+spec:
+  class: io.confluent.connect.s3.S3SinkConnector
+  tasksMax: 2
+  config:
+    # Core Connector Configuration
+    topics: test-topic
+
+    # S3 Configuration
+    s3.region: us-east-1
+    s3.bucket.name: delta-lake-test
+    s3.part.size: '134217728' # Part size for upload (1 GB/ 128mb)
+
+    # Performance tuning
+    flush.size: 500
+
+    # MinIO (or S3) store URL (use environment variable for security)
+    store.url: https://s3.dco1.safedc.net
+
+    # Storage and Format Configuration
+    storage.class: io.confluent.connect.s3.storage.S3Storage
+    format.class: io.confluent.connect.s3.format.parquet.ParquetFormat
+    partitioner.class: io.confluent.connect.storage.partitioner.TimeBasedPartitioner
+    path.format: "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH" # Added explicit path format
+    partition.duration.ms: 3600000 # Ensures hourly partitions for manageability
+
+    # Behavior settings 
+    behavior.on.null.values: ignore
+
+    # Use Swedish locale
+    locale: sv_SE
+    timezone: Europe/Stockholm
+    # # Serialization
+    # # * Key
+    # key.converter: org.apache.kafka.connect.storage.StringConverter
+    # key.converter.schemas.enable: false # Avro schemas usually not needed for keys
+    # key.converter.schema.registry.url: http://89.47.191.210:30081
+    # # * Value
+    # value.converter: io.confluent.connect.avro.AvroConverter
+    # value.converter.schema.registry.url: http://89.47.191.210:30081
+    # value.converter.schemas.enable: true
+    # schema.compatibility: BACKWARD # Allow schema evolution
+
+    # # Rotation and Batch Handling
+    # rotate.interval.ms: 600000 # reduce overhead in high-throughput scenarios
+    # key.converter.use.latest.version: true
+    # value.converter.use.latest.version: true
+    # # Optional: 
+    # # s3.compression.type: gzip
+    # # store.kafka.keys: true
+    # # logging.level: debug
+
+

kafka-resources/base/kafka-core/kafka-das-topic.yaml

@@ -0,0 +1,14 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: kafka-das-topic
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: kafka-cluster
+    app.kubernetes.io/part-of: kafka
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    # http://kafka.apache.org/documentation/#topicconfigs
+    cleanup.policy: compact

kafka-resources/base/kafka-core/kafka-persistent-cluster.yaml

@@ -0,0 +1,59 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: kafka-cluster
+  namespace: kafka
+  labels:
+    app.kubernetes.io/part-of: kafka
+spec:
+  kafka:
+    version: 3.8.0
+    replicas: 1
+    listeners:
+    - name: plainext
+      port: 9092
+      type: nodeport
+      tls: false
+      configuration:
+        brokers:
+        - broker: 0
+          nodePort: 31542
+    - name: tls
+      port: 9093
+      type: internal
+      tls: true
+    - name: external
+      port: 9094
+      tls: false
+      type: nodeport
+      configuration:
+        bootstrap:
+          nodePort: 32100
+        brokers:
+        - broker: 0
+          nodePort: 32700
+    config:
+      offsets.topic.replication.factor: 1
+      transaction.state.log.replication.factor: 1
+      transaction.state.log.min.isr: 1
+      default.replication.factor: 1
+      min.insync.replicas: 1
+      inter.broker.protocol.version: "3.8"
+    storage:
+      type: jbod
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: 1Gi
+        deleteClaim: false
+        class: csi-sc-cinderplugin
+  zookeeper:
+    replicas: 1
+    storage:
+      type: persistent-claim
+      size: 1Gi
+      deleteClaim: false
+      class: csi-sc-cinderplugin
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}

kafka-resources/base/kafka-core/kafka-test-topic.yaml

@@ -0,0 +1,13 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: test-topic
+  labels:
+    strimzi.io/cluster: kafka-cluster
+    app.kubernetes.io/part-of: kafka
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    # http://kafka.apache.org/documentation/#topicconfigs
+    cleanup.policy: compact

kafka-resources/base/kafka-core/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kafka
+commonLabels:
+  app.kubernetes.io/name: core
+resources:
+- kafka-persistent-cluster.yaml
+- kafka-das-topic.yaml
+- kafka-test-topic.yaml

kafka-resources/base/kafka-schema-registry/kafka-registry-topic.yaml

@@ -0,0 +1,12 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: registry-schemas
+  labels:
+    strimzi.io/cluster: kafka-cluster
+spec:
+  partitions: 1
+  replicas: 1
+  config:
+    # http://kafka.apache.org/documentation/#topicconfigs
+    cleanup.policy: compact

kafka-resources/base/kafka-schema-registry/kafka-user-registry.yaml

@@ -0,0 +1,36 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+  name: confluent-schema-registry
+  labels:
+    strimzi.io/cluster: kafka-cluster
+spec:
+  authentication:
+    type: tls
+  authorization:
+    # Official docs on authorizations required for the Schema Registry:
+    # https://docs.confluent.io/current/schema-registry/security/index.html#authorizing-access-to-the-schemas-topic
+    type: simple
+    acls:
+    # Allow all operations on the registry-schemas topic
+    # Read, Write, and DescribeConfigs are known to be required
+    - resource:
+        type: topic
+        name: registry-schemas
+        patternType: literal
+      operation: All
+      type: allow
+    # Allow all operations on the schema-registry* group
+    - resource:
+        type: group
+        name: schema-registry
+        patternType: prefix
+      operation: All
+      type: allow
+    # Allow Describe on the __consumer_offsets topic
+    - resource:
+        type: topic
+        name: __consumer_offsets
+        patternType: literal
+      operation: Describe
+      type: allow

kafka-resources/base/kafka-schema-registry/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kafka
+resources:
+- schema-registry-deployment.yaml
+- schema-registry-service.yaml
+- kafka-registry-topic.yaml
+# - kafka-user-registry.yaml

kafka-resources/base/kafka-schema-registry/schema-registry-deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    kompose.cmd: kompose --file docker-compose.yml convert
+    kompose.version: 1.34.0 (cbf2835db)
+  labels:
+    app: schema-registry
+  name: schema-registry
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: schema-registry
+  template:
+    metadata:
+      annotations:
+        kompose.cmd: kompose --file docker-compose.yml convert
+        kompose.version: 1.34.0 (cbf2835db)
+      labels:
+        app: schema-registry
+    spec:
+      containers:
+      - name: schema-registry
+        image: confluentinc/cp-schema-registry:6.1.1
+        ports:
+        - containerPort: 8081
+          protocol: TCP
+        env:
+        - name: SCHEMA_REGISTRY_HOST_NAME
+          value: schema-registry
+        - name: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS
+          value: kafka-cluster-kafka-plainext-bootstrap:9092
+        - name: SCHEMA_REGISTRY_LISTENERS
+          value: http://0.0.0.0:8081
+      #hostname: schema-registry
+      #restartPolicy: Always

kafka-resources/base/kafka-schema-registry/schema-registry-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    kompose.cmd: kompose --file docker-compose.yml convert
+    kompose.version: 1.34.0 (cbf2835db)
+  labels:
+    app: schema-registry
+  name: schema-registry
+  namespace: kafka
+spec:
+  type: NodePort
+  ports:
+  - port: 8081
+    targetPort: 8081
+    nodePort: 30081
+  selector:
+    app: schema-registry

kafka-resources/overlay/dev/kafka-core/kafka-persistent-cluster.yaml

@@ -0,0 +1,31 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: kafka-cluster
+  namespace: kafka
+spec:
+  kafka:
+    replicas: 2
+    config:
+      offsets.topic.replication.factor: 1
+      transaction.state.log.replication.factor: 1
+      transaction.state.log.min.isr: 1
+      default.replication.factor: 1
+      min.insync.replicas: 1
+      inter.broker.protocol.version: "3.8"
+    storage:
+      type: jbod
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: 10Gi
+        deleteClaim: false
+        class: csi-sc-cinderplugin
+  zookeeper:
+    replicas: 1
+    storage:
+      type: persistent-claim
+      size: 10Gi
+      deleteClaim: false
+      class: csi-sc-cinderplugin
+

kafka-resources/overlay/dev/kafka-core/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../base/kafka-core/
+patchesStrategicMerge:
+  - kafka-persistent-cluster.yaml

kafka-resources/overlay/dev/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kafka
+commonLabels:
+  env: dev
+resources:
+  - kafka-core/

kafka-resources/overlay/prod/kafka-core/kafka-persistent-cluster.yaml

@@ -0,0 +1,31 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: kafka-cluster
+  namespace: kafka
+spec:
+  kafka:
+    replicas: 3
+    config:
+      offsets.topic.replication.factor: 3
+      transaction.state.log.replication.factor: 3
+      transaction.state.log.min.isr: 2
+      default.replication.factor: 3
+      min.insync.replicas: 2
+      inter.broker.protocol.version: "3.8"
+    storage:
+      type: jbod
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: 112Gi
+        deleteClaim: false
+        class: csi-sc-cinderplugin
+  zookeeper:
+    replicas: 3
+    storage:
+      type: persistent-claim
+      size: 112Gi
+      deleteClaim: false
+      class: csi-sc-cinderplugin
+

kafka-resources/overlay/prod/kafka-core/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../base/kafka-core/
+patchesStrategicMerge:
+  - kafka-persistent-cluster.yaml

kafka-resources/overlay/prod/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kafka
+commonLabels:
+  env: prod
+resources:
+  - kafka-core/

kafka-resources/overlay/test/kafka-core/kafka-persistent-cluster.yaml

@@ -0,0 +1,31 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: kafka-cluster
+  namespace: kafka
+spec:
+  kafka:
+    replicas: 2
+    config:
+      offsets.topic.replication.factor: 2
+      transaction.state.log.replication.factor: 2
+      transaction.state.log.min.isr: 1
+      default.replication.factor: 2
+      min.insync.replicas: 1
+      inter.broker.protocol.version: "3.8"
+    storage:
+      type: jbod
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: 20Gi
+        deleteClaim: false
+        class: csi-sc-cinderplugin
+  zookeeper:
+    replicas: 2
+    storage:
+      type: persistent-claim
+      size: 20Gi
+      deleteClaim: false
+      class: csi-sc-cinderplugin
+

kafka-resources/overlay/test/kafka-core/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../base/kafka-core/
+patchesStrategicMerge:
+  - kafka-persistent-cluster.yaml

kafka-resources/overlay/test/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kafka
+commonLabels:
+  env: test
+resources:
+  - kafka-core/

kafka-resources/readme.md

@@ -0,0 +1,66 @@
+# Kustomize and Kubernetes Commands
+
+## Generate and Apply Manifests
+
+To build a Kubernetes manifest using Kustomize and apply it to the `kafka` namespace:
+
+```bash
+kustomize build overlay/test/ > manifest.yaml | kubectl kustomize overlay/test/ > manifest.yaml
+kubectl -n kafka apply -f manifest.yaml
+```
+
+## Viewing the Kustomize Directory
+
+You can view the configuration for a Kustomize directory as follows:
+
+```bash
+kubectl kustomize argo-cd-work/base/kafka-core
+```
+
+---
+
+## Kustomize Documentation
+
+For detailed documentation on Kustomization, visit: [Kustomize Documentation](https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/)
+
+---
+
+# Applying Overlays
+
+To apply a specific overlay (e.g., `dev`), first build the configuration and apply it to the `kafka` namespace:
+
+```bash
+kustomize build overlay/dev/ > dev.yml
+kubectl -n kafka apply -f dev.yml
+```
+
+---
+
+# Preparing for Kafka Connect
+
+Before setting up Kafka Connect, ensure the following steps are completed:
+
+1. **Create a Docker Image**  
+   Build and push the Docker image required for Kafka Connect.
+
+2. **Create Docker Secrets for Authentication**  
+   Generate secrets for S3 login or other credentials.
+
+3. **Create Platform and S3 Credentials**  
+   Create the following Kubernetes secrets for authentication:
+   - `docker-platform-creds`
+   - `s3-minio-creds`
+
+   Verify the secrets are present in the `kafka` namespace:
+
+   ```bash
+   kubectl -n kafka get secrets
+   ```
+
+   Example output:
+
+   ```plaintext
+   NAME                    TYPE                              DATA   AGE
+   docker-platform-creds   kubernetes.io/dockerconfigjson    1     15d
+   s3-minio-creds          Opaque                            2     13d
+   ```