Add kafka-resources for core, connect & schema-registry
This commit is contained in:
parent
a9ec608b3b
commit
bedfb56ba4
GPG key ID:
62D68B584B4B3EB3
25 changed files with 588 additions and 0 deletions
|
|
@ -0,0 +1,13 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaTopic
|
||||
metadata:
|
||||
name: kafka-connect-cluster-configs
|
||||
namespace: kafka
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-cluster
|
||||
spec:
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
config:
|
||||
# http://kafka.apache.org/documentation/#topicconfigs
|
||||
cleanup.policy: compact
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaTopic
|
||||
metadata:
|
||||
name: kafka-connect-cluster-offsets
|
||||
namespace: kafka
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-cluster
|
||||
spec:
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
config:
|
||||
# http://kafka.apache.org/documentation/#topicconfigs
|
||||
cleanup.policy: compact
|
||||
13
kafka-resources/base/kafka-connect/connect-status-topic.yaml
Normal file
13
kafka-resources/base/kafka-connect/connect-status-topic.yaml
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaTopic
|
||||
metadata:
|
||||
name: kafka-connect-cluster-status
|
||||
namespace: kafka
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-cluster
|
||||
spec:
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
config:
|
||||
# http://kafka.apache.org/documentation/#topicconfigs
|
||||
cleanup.policy: compact
|
||||
8
kafka-resources/base/kafka-connect/kustomization.yaml
Normal file
8
kafka-resources/base/kafka-connect/kustomization.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- connect-configs-topic.yaml
|
||||
- connect-offsets-topic.yaml
|
||||
- connect-status-topic.yaml
|
||||
- s3-kafka-connect.yaml
|
||||
- s3-kafka-connector.yaml
|
||||
69
kafka-resources/base/kafka-connect/s3-kafka-connect.yaml
Normal file
69
kafka-resources/base/kafka-connect/s3-kafka-connect.yaml
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaConnect
|
||||
metadata:
|
||||
name: kafka-connect-cluster
|
||||
namespace: kafka
|
||||
annotations:
|
||||
strimzi.io/use-connector-resources: "true"
|
||||
spec:
|
||||
replicas: 1
|
||||
bootstrapServers: kafka-cluster-kafka-plainext-bootstrap:9092
|
||||
# tls:
|
||||
# trustedCertificates:
|
||||
# - secretName: my-cluster-cluster-ca-cert
|
||||
# certificate: ca.crt
|
||||
config:
|
||||
group.id: kafka-connect-cluster
|
||||
offset.storage.topic: kafka-connect-cluster-offsets
|
||||
config.storage.topic: kafka-connect-cluster-configs
|
||||
status.storage.topic: kafka-connect-cluster-status
|
||||
key.converter: org.apache.kafka.connect.storage.StringConverter
|
||||
value.converter: io.confluent.connect.avro.AvroConverter
|
||||
key.converter.schema.registry.url: http://89.47.191.210:30081
|
||||
value.converter.schema.registry.url: http://89.47.191.210:30081
|
||||
config.storage.replication.factor: 1
|
||||
offset.storage.replication.factor: 1
|
||||
status.storage.replication.factor: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
externalConfiguration:
|
||||
env:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-minio-creds
|
||||
key: AWS_ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-minio-creds
|
||||
key: AWS_SECRET_ACCESS_KEY
|
||||
build:
|
||||
output:
|
||||
type: docker
|
||||
image: platform.sunet.se/benedith/das-kafka-connect-cluster:latest
|
||||
pushSecret: docker-platform-creds
|
||||
plugins:
|
||||
- name: kafka-connect-s3
|
||||
artifacts:
|
||||
- type: zip
|
||||
url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/confluentinc-kafka-connect-s3-10.5.17.zip
|
||||
sha512sum: 51dc4eb5e618a7743b3d29c7c5586f5bda00a254a9f105ee816cad7c8e9509a7c1a1ea43351e77dcf97847900c21895962716ed6a1bfb2de4a2b4695233d8804
|
||||
- type: zip
|
||||
url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/confluentinc-kafka-connect-avro-converter-7.7.1.zip
|
||||
sha512sum: cebc6fece5c5551d3cff5f1cc8f4660e83da6292a9d695c1f8851af880661b2882e59ef0eeb3df395c3fc314e483cc26961d6a6df271237aab7ef2d8732af3f4
|
||||
- type: zip
|
||||
url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/jcustenborder-kafka-connect-transform-common-0.1.0.58.zip
|
||||
sha512sum: 17b80dcf0944baf3024af28a81d61140dcad3e75af4b08807d31349ba0d51cb871dca5f123174973150dd8dace4cf72511c02e4f8c1ec94ac6206d7ccf9431a9
|
||||
- type: jar
|
||||
url: https://platform.sunet.se/benedith/strimzi-kafka-connectors/raw/branch/main/jars/kafka-clients-3.8.0.jar
|
||||
sha512sum: e61c2ab5e7d3daa1e546904512815ee9b3e3ac1cd887aa44ce920bc1597f13cc5f097dfc0c011b4dea23650a5394ba8e060104f0064d1ad37b589b11dcb0fa3c
|
||||
template:
|
||||
pod:
|
||||
imagePullSecrets:
|
||||
- name: docker-platform-creds
|
||||
59
kafka-resources/base/kafka-connect/s3-kafka-connector.yaml
Normal file
59
kafka-resources/base/kafka-connect/s3-kafka-connector.yaml
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaConnector
|
||||
metadata:
|
||||
name: kafka-connect-s3-connector
|
||||
namespace: kafka
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-connect-cluster
|
||||
spec:
|
||||
class: io.confluent.connect.s3.S3SinkConnector
|
||||
tasksMax: 2
|
||||
config:
|
||||
# Core Connector Configuration
|
||||
topics: test-topic
|
||||
|
||||
# S3 Configuration
|
||||
s3.region: us-east-1
|
||||
s3.bucket.name: delta-lake-test
|
||||
s3.part.size: '134217728' # Part size for upload (1 GB/ 128mb)
|
||||
|
||||
# Performance tuning
|
||||
flush.size: 500
|
||||
|
||||
# MinIO (or S3) store URL (use environment variable for security)
|
||||
store.url: https://s3.dco1.safedc.net
|
||||
|
||||
# Storage and Format Configuration
|
||||
storage.class: io.confluent.connect.s3.storage.S3Storage
|
||||
format.class: io.confluent.connect.s3.format.parquet.ParquetFormat
|
||||
partitioner.class: io.confluent.connect.storage.partitioner.TimeBasedPartitioner
|
||||
path.format: "'year'=YYYY/'month'=MM/'day'=dd/'hour'=HH" # Added explicit path format
|
||||
partition.duration.ms: 3600000 # Ensures hourly partitions for manageability
|
||||
|
||||
# Behavior settings
|
||||
behavior.on.null.values: ignore
|
||||
|
||||
# Use Swedish locale
|
||||
locale: sv_SE
|
||||
timezone: Europe/Stockholm
|
||||
# # Serialization
|
||||
# # * Key
|
||||
# key.converter: org.apache.kafka.connect.storage.StringConverter
|
||||
# key.converter.schemas.enable: false # Avro schemas usually not needed for keys
|
||||
# key.converter.schema.registry.url: http://89.47.191.210:30081
|
||||
# # * Value
|
||||
# value.converter: io.confluent.connect.avro.AvroConverter
|
||||
# value.converter.schema.registry.url: http://89.47.191.210:30081
|
||||
# value.converter.schemas.enable: true
|
||||
# schema.compatibility: BACKWARD # Allow schema evolution
|
||||
|
||||
# # Rotation and Batch Handling
|
||||
# rotate.interval.ms: 600000 # reduce overhead in high-throughput scenarios
|
||||
# key.converter.use.latest.version: true
|
||||
# value.converter.use.latest.version: true
|
||||
# # Optional:
|
||||
# # s3.compression.type: gzip
|
||||
# # store.kafka.keys: true
|
||||
# # logging.level: debug
|
||||
|
||||
|
||||
14
kafka-resources/base/kafka-core/kafka-das-topic.yaml
Normal file
14
kafka-resources/base/kafka-core/kafka-das-topic.yaml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaTopic
|
||||
metadata:
|
||||
name: kafka-das-topic
|
||||
namespace: kafka
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-cluster
|
||||
app.kubernetes.io/part-of: kafka
|
||||
spec:
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
config:
|
||||
# http://kafka.apache.org/documentation/#topicconfigs
|
||||
cleanup.policy: compact
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: Kafka
|
||||
metadata:
|
||||
name: kafka-cluster
|
||||
namespace: kafka
|
||||
labels:
|
||||
app.kubernetes.io/part-of: kafka
|
||||
spec:
|
||||
kafka:
|
||||
version: 3.8.0
|
||||
replicas: 1
|
||||
listeners:
|
||||
- name: plainext
|
||||
port: 9092
|
||||
type: nodeport
|
||||
tls: false
|
||||
configuration:
|
||||
brokers:
|
||||
- broker: 0
|
||||
nodePort: 31542
|
||||
- name: tls
|
||||
port: 9093
|
||||
type: internal
|
||||
tls: true
|
||||
- name: external
|
||||
port: 9094
|
||||
tls: false
|
||||
type: nodeport
|
||||
configuration:
|
||||
bootstrap:
|
||||
nodePort: 32100
|
||||
brokers:
|
||||
- broker: 0
|
||||
nodePort: 32700
|
||||
config:
|
||||
offsets.topic.replication.factor: 1
|
||||
transaction.state.log.replication.factor: 1
|
||||
transaction.state.log.min.isr: 1
|
||||
default.replication.factor: 1
|
||||
min.insync.replicas: 1
|
||||
inter.broker.protocol.version: "3.8"
|
||||
storage:
|
||||
type: jbod
|
||||
volumes:
|
||||
- id: 0
|
||||
type: persistent-claim
|
||||
size: 1Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
zookeeper:
|
||||
replicas: 1
|
||||
storage:
|
||||
type: persistent-claim
|
||||
size: 1Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
entityOperator:
|
||||
topicOperator: {}
|
||||
userOperator: {}
|
||||
13
kafka-resources/base/kafka-core/kafka-test-topic.yaml
Normal file
13
kafka-resources/base/kafka-core/kafka-test-topic.yaml
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaTopic
|
||||
metadata:
|
||||
name: test-topic
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-cluster
|
||||
app.kubernetes.io/part-of: kafka
|
||||
spec:
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
config:
|
||||
# http://kafka.apache.org/documentation/#topicconfigs
|
||||
cleanup.policy: compact
|
||||
10
kafka-resources/base/kafka-core/kustomization.yaml
Normal file
10
kafka-resources/base/kafka-core/kustomization.yaml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: kafka
|
||||
commonLabels:
|
||||
app.kubernetes.io/name: core
|
||||
resources:
|
||||
- kafka-persistent-cluster.yaml
|
||||
- kafka-das-topic.yaml
|
||||
- kafka-test-topic.yaml
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaTopic
|
||||
metadata:
|
||||
name: registry-schemas
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-cluster
|
||||
spec:
|
||||
partitions: 1
|
||||
replicas: 1
|
||||
config:
|
||||
# http://kafka.apache.org/documentation/#topicconfigs
|
||||
cleanup.policy: compact
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: KafkaUser
|
||||
metadata:
|
||||
name: confluent-schema-registry
|
||||
labels:
|
||||
strimzi.io/cluster: kafka-cluster
|
||||
spec:
|
||||
authentication:
|
||||
type: tls
|
||||
authorization:
|
||||
# Official docs on authorizations required for the Schema Registry:
|
||||
# https://docs.confluent.io/current/schema-registry/security/index.html#authorizing-access-to-the-schemas-topic
|
||||
type: simple
|
||||
acls:
|
||||
# Allow all operations on the registry-schemas topic
|
||||
# Read, Write, and DescribeConfigs are known to be required
|
||||
- resource:
|
||||
type: topic
|
||||
name: registry-schemas
|
||||
patternType: literal
|
||||
operation: All
|
||||
type: allow
|
||||
# Allow all operations on the schema-registry* group
|
||||
- resource:
|
||||
type: group
|
||||
name: schema-registry
|
||||
patternType: prefix
|
||||
operation: All
|
||||
type: allow
|
||||
# Allow Describe on the __consumer_offsets topic
|
||||
- resource:
|
||||
type: topic
|
||||
name: __consumer_offsets
|
||||
patternType: literal
|
||||
operation: Describe
|
||||
type: allow
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: kafka
|
||||
resources:
|
||||
- schema-registry-deployment.yaml
|
||||
- schema-registry-service.yaml
|
||||
- kafka-registry-topic.yaml
|
||||
# - kafka-user-registry.yaml
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose --file docker-compose.yml convert
|
||||
kompose.version: 1.34.0 (cbf2835db)
|
||||
labels:
|
||||
app: schema-registry
|
||||
name: schema-registry
|
||||
namespace: kafka
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: schema-registry
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose --file docker-compose.yml convert
|
||||
kompose.version: 1.34.0 (cbf2835db)
|
||||
labels:
|
||||
app: schema-registry
|
||||
spec:
|
||||
containers:
|
||||
- name: schema-registry
|
||||
image: confluentinc/cp-schema-registry:6.1.1
|
||||
ports:
|
||||
- containerPort: 8081
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: SCHEMA_REGISTRY_HOST_NAME
|
||||
value: schema-registry
|
||||
- name: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS
|
||||
value: kafka-cluster-kafka-plainext-bootstrap:9092
|
||||
- name: SCHEMA_REGISTRY_LISTENERS
|
||||
value: http://0.0.0.0:8081
|
||||
#hostname: schema-registry
|
||||
#restartPolicy: Always
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
kompose.cmd: kompose --file docker-compose.yml convert
|
||||
kompose.version: 1.34.0 (cbf2835db)
|
||||
labels:
|
||||
app: schema-registry
|
||||
name: schema-registry
|
||||
namespace: kafka
|
||||
spec:
|
||||
type: NodePort
|
||||
ports:
|
||||
- port: 8081
|
||||
targetPort: 8081
|
||||
nodePort: 30081
|
||||
selector:
|
||||
app: schema-registry
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: Kafka
|
||||
metadata:
|
||||
name: kafka-cluster
|
||||
namespace: kafka
|
||||
spec:
|
||||
kafka:
|
||||
replicas: 2
|
||||
config:
|
||||
offsets.topic.replication.factor: 1
|
||||
transaction.state.log.replication.factor: 1
|
||||
transaction.state.log.min.isr: 1
|
||||
default.replication.factor: 1
|
||||
min.insync.replicas: 1
|
||||
inter.broker.protocol.version: "3.8"
|
||||
storage:
|
||||
type: jbod
|
||||
volumes:
|
||||
- id: 0
|
||||
type: persistent-claim
|
||||
size: 10Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
zookeeper:
|
||||
replicas: 1
|
||||
storage:
|
||||
type: persistent-claim
|
||||
size: 10Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ../../../base/kafka-core/
|
||||
patchesStrategicMerge:
|
||||
- kafka-persistent-cluster.yaml
|
||||
8
kafka-resources/overlay/dev/kustomization.yaml
Normal file
8
kafka-resources/overlay/dev/kustomization.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kafka
|
||||
commonLabels:
|
||||
env: dev
|
||||
resources:
|
||||
- kafka-core/
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: Kafka
|
||||
metadata:
|
||||
name: kafka-cluster
|
||||
namespace: kafka
|
||||
spec:
|
||||
kafka:
|
||||
replicas: 3
|
||||
config:
|
||||
offsets.topic.replication.factor: 3
|
||||
transaction.state.log.replication.factor: 3
|
||||
transaction.state.log.min.isr: 2
|
||||
default.replication.factor: 3
|
||||
min.insync.replicas: 2
|
||||
inter.broker.protocol.version: "3.8"
|
||||
storage:
|
||||
type: jbod
|
||||
volumes:
|
||||
- id: 0
|
||||
type: persistent-claim
|
||||
size: 112Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
zookeeper:
|
||||
replicas: 3
|
||||
storage:
|
||||
type: persistent-claim
|
||||
size: 112Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ../../../base/kafka-core/
|
||||
patchesStrategicMerge:
|
||||
- kafka-persistent-cluster.yaml
|
||||
8
kafka-resources/overlay/prod/kustomization.yaml
Normal file
8
kafka-resources/overlay/prod/kustomization.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kafka
|
||||
commonLabels:
|
||||
env: prod
|
||||
resources:
|
||||
- kafka-core/
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
apiVersion: kafka.strimzi.io/v1beta2
|
||||
kind: Kafka
|
||||
metadata:
|
||||
name: kafka-cluster
|
||||
namespace: kafka
|
||||
spec:
|
||||
kafka:
|
||||
replicas: 2
|
||||
config:
|
||||
offsets.topic.replication.factor: 2
|
||||
transaction.state.log.replication.factor: 2
|
||||
transaction.state.log.min.isr: 1
|
||||
default.replication.factor: 2
|
||||
min.insync.replicas: 1
|
||||
inter.broker.protocol.version: "3.8"
|
||||
storage:
|
||||
type: jbod
|
||||
volumes:
|
||||
- id: 0
|
||||
type: persistent-claim
|
||||
size: 20Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
zookeeper:
|
||||
replicas: 2
|
||||
storage:
|
||||
type: persistent-claim
|
||||
size: 20Gi
|
||||
deleteClaim: false
|
||||
class: csi-sc-cinderplugin
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ../../../base/kafka-core/
|
||||
patchesStrategicMerge:
|
||||
- kafka-persistent-cluster.yaml
|
||||
8
kafka-resources/overlay/test/kustomization.yaml
Normal file
8
kafka-resources/overlay/test/kustomization.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kafka
|
||||
commonLabels:
|
||||
env: test
|
||||
resources:
|
||||
- kafka-core/
|
||||
66
kafka-resources/readme.md
Normal file
66
kafka-resources/readme.md
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
# Kustomize and Kubernetes Commands
|
||||
|
||||
## Generate and Apply Manifests
|
||||
|
||||
To build a Kubernetes manifest using Kustomize and apply it to the `kafka` namespace:
|
||||
|
||||
```bash
|
||||
kustomize build overlay/test/ > manifest.yaml | kubectl kustomize overlay/test/ > manifest.yaml
|
||||
kubectl -n kafka apply -f manifest.yaml
|
||||
```
|
||||
|
||||
## Viewing the Kustomize Directory
|
||||
|
||||
You can view the configuration for a Kustomize directory as follows:
|
||||
|
||||
```bash
|
||||
kubectl kustomize argo-cd-work/base/kafka-core
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Kustomize Documentation
|
||||
|
||||
For detailed documentation on Kustomization, visit: [Kustomize Documentation](https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/)
|
||||
|
||||
---
|
||||
|
||||
# Applying Overlays
|
||||
|
||||
To apply a specific overlay (e.g., `dev`), first build the configuration and apply it to the `kafka` namespace:
|
||||
|
||||
```bash
|
||||
kustomize build overlay/dev/ > dev.yml
|
||||
kubectl -n kafka apply -f dev.yml
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# Preparing for Kafka Connect
|
||||
|
||||
Before setting up Kafka Connect, ensure the following steps are completed:
|
||||
|
||||
1. **Create a Docker Image**
|
||||
Build and push the Docker image required for Kafka Connect.
|
||||
|
||||
2. **Create Docker Secrets for Authentication**
|
||||
Generate secrets for S3 login or other credentials.
|
||||
|
||||
3. **Create Platform and S3 Credentials**
|
||||
Create the following Kubernetes secrets for authentication:
|
||||
- `docker-platform-creds`
|
||||
- `s3-minio-creds`
|
||||
|
||||
Verify the secrets are present in the `kafka` namespace:
|
||||
|
||||
```bash
|
||||
kubectl -n kafka get secrets
|
||||
```
|
||||
|
||||
Example output:
|
||||
|
||||
```plaintext
|
||||
NAME TYPE DATA AGE
|
||||
docker-platform-creds kubernetes.io/dockerconfigjson 1 15d
|
||||
s3-minio-creds Opaque 2 13d
|
||||
```
|
||||
Loading…
Add table
Reference in a new issue