Update kafka-connector to use new topic & add argo-app

kafdrop/argocd-application.yaml

@@ -7,7 +7,7 @@ spec:
   project: default
   source:
     repoURL: https://platform.sunet.se/streams/streams-manifests.git
-    targetRevision: streams-manifests-2024-12-11-v06
+    targetRevision: streams-manifests-2024-12-11-v07
     path: kafdrop/overlays/test
   destination:
     server: https://kubernetes.default.svc

kafka-resources/argo-cd-application.yaml

@@ -0,0 +1,18 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kafka-resources
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://platform.sunet.se/streams/streams-manifests.git
+    targetRevision: streams-manifests-2024-12-09-v09
+    path: kafka-resources/overlays/prod
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kafka
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

kafka-resources/overlays/prod/kafka-connect/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
 - ../../../base/kafka-connect/
 patchesStrategicMerge:
 - s3-kafka-connect.yaml
+- s3-kafka-connector.yaml

kafka-resources/overlays/prod/kafka-connect/s3-kafka-connector.yaml

@@ -0,0 +1,38 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaConnector
+metadata:
+  name: kafka-connect-s3-connector
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: kafka-connect-cluster
+spec:
+  class: io.confluent.connect.s3.S3SinkConnector
+  tasksMax: 2
+  config:
+    # Core Connector Configuration
+    topics: kafka-das-topic
+
+    # S3 Configuration
+    s3.region: us-east-1
+    s3.bucket.name: delta-lake-test
+    s3.part.size: '134217728' # Part size for upload (1 GB/ 128mb)
+
+    # Performance tuning
+    flush.size: 500
+
+    # MinIO (or S3) store URL (use environment variable for security)
+    store.url: https://s3.dco1.safedc.net
+
+    # Storage and Format Configuration
+    storage.class: io.confluent.connect.s3.storage.S3Storage
+    format.class: io.confluent.connect.s3.format.parquet.ParquetFormat
+    partitioner.class: io.confluent.connect.storage.partitioner.TimeBasedPartitioner
+    path.format: "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH" # Added explicit path format
+    partition.duration.ms: 3600000 # Ensures hourly partitions for manageability
+
+    # Behavior settings 
+    behavior.on.null.values: ignore
+
+    # Use Swedish locale
+    locale: sv_SE
+    timezone: Europe/Stockholm