diff --git a/kafdrop/argocd-application.yaml b/kafdrop/argocd-application.yaml index 336c92c..c200f2a 100644 --- a/kafdrop/argocd-application.yaml +++ b/kafdrop/argocd-application.yaml @@ -7,7 +7,7 @@ spec: project: default source: repoURL: https://platform.sunet.se/streams/streams-manifests.git - targetRevision: streams-manifests-2024-12-11-v06 + targetRevision: streams-manifests-2024-12-11-v07 path: kafdrop/overlays/test destination: server: https://kubernetes.default.svc diff --git a/kafka-resources/argo-cd-application.yaml b/kafka-resources/argo-cd-application.yaml new file mode 100644 index 0000000..4bd09ed --- /dev/null +++ b/kafka-resources/argo-cd-application.yaml @@ -0,0 +1,18 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kafka-resources + namespace: argocd +spec: + project: default + source: + repoURL: https://platform.sunet.se/streams/streams-manifests.git + targetRevision: streams-manifests-2024-12-09-v09 + path: kafka-resources/overlays/prod + destination: + server: https://kubernetes.default.svc + namespace: kafka + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/kafka-resources/overlays/prod/kafka-connect/kustomization.yaml b/kafka-resources/overlays/prod/kafka-connect/kustomization.yaml index f38dfc1..252584f 100644 --- a/kafka-resources/overlays/prod/kafka-connect/kustomization.yaml +++ b/kafka-resources/overlays/prod/kafka-connect/kustomization.yaml @@ -6,3 +6,4 @@ resources: - ../../../base/kafka-connect/ patchesStrategicMerge: - s3-kafka-connect.yaml +- s3-kafka-connector.yaml diff --git a/kafka-resources/overlays/prod/kafka-connect/s3-kafka-connector.yaml b/kafka-resources/overlays/prod/kafka-connect/s3-kafka-connector.yaml new file mode 100644 index 0000000..3d1f0c6 --- /dev/null +++ b/kafka-resources/overlays/prod/kafka-connect/s3-kafka-connector.yaml @@ -0,0 +1,38 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: kafka-connect-s3-connector + namespace: kafka + labels: + strimzi.io/cluster: kafka-connect-cluster +spec: + class: io.confluent.connect.s3.S3SinkConnector + tasksMax: 2 + config: + # Core Connector Configuration + topics: kafka-das-topic + + # S3 Configuration + s3.region: us-east-1 + s3.bucket.name: delta-lake-test + s3.part.size: '134217728' # Part size for upload (1 GB/ 128mb) + + # Performance tuning + flush.size: 500 + + # MinIO (or S3) store URL (use environment variable for security) + store.url: https://s3.dco1.safedc.net + + # Storage and Format Configuration + storage.class: io.confluent.connect.s3.storage.S3Storage + format.class: io.confluent.connect.s3.format.parquet.ParquetFormat + partitioner.class: io.confluent.connect.storage.partitioner.TimeBasedPartitioner + path.format: "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH" # Added explicit path format + partition.duration.ms: 3600000 # Ensures hourly partitions for manageability + + # Behavior settings + behavior.on.null.values: ignore + + # Use Swedish locale + locale: sv_SE + timezone: Europe/Stockholm