Make rook deployment fully multizone aware

This commit is contained in:
Magnus Andersson 2024-10-28 14:20:12 +01:00
parent 1384c2df90
commit ad52a3c054
Signed by: mandersson
GPG key ID: 1F7C896B34B28164
7 changed files with 140 additions and 5 deletions

View file

@ -0,0 +1,65 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: matrix-registry
namespace: matrix-registry
labels:
k8s-app: matrix-registry
kubernetes.io/cluster-service: "true"
spec:
replicas: 3
selector:
matchLabels:
k8s-app: matrix-registry
template:
metadata:
labels:
k8s-app: matrix-registry
kubernetes.io/cluster-service: "true"
spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
k8s-app: matrix-registry
containers:
- name: registry
image: registry:2
imagePullPolicy: Always
resources:
limits:
cpu: 200m
memory: 300Mi
env:
- name: REGISTRY_HTTP_ADDR
value: :5000
- name: REGISTRY_HTTP_SECRET
valueFrom:
secretKeyRef:
name: matrix-registry-secret
key: http-secret
- name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
value: /var/lib/registry
volumeMounts:
- name: image-store
mountPath: /var/lib/registry
ports:
- containerPort: 5000
name: registry
protocol: TCP
livenessProbe:
httpGet:
path: /
port: registry
readinessProbe:
httpGet:
path: /
port: registry
volumes:
- name: image-store
persistentVolumeClaim:
claimName: cephfs-pvc
readOnly: false

View file

@ -0,0 +1,7 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: matrix-registry
labels:
name: matrix-registry-namespace

View file

@ -0,0 +1,13 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cephfs-pvc
namespace: matrix-registry
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
storageClassName: rook-cephfs

3
k8s/rook/README.md Normal file
View file

@ -0,0 +1,3 @@
### Rook deployment
In the operator.yaml change ROOK_CSI_KUBELET_DIR_PATH to "/var/snap/microk8s/common/var/lib/kubelet"

View file

@ -18,21 +18,25 @@ spec:
mon:
count: 3
allowMultiplePerNode: false
failureDomainLabel: topology.kubernetes.io/zone
failureDomainLabel: topology.kubernetes.io/zone
zones:
- name: dco
- name: sto3
- name: sto4
mgr:
count: 2
allowMultiplePerNode: false
modules:
- name: rook
enabled: true
- name: pg_autoscaler
enabled: true
cephVersion:
image: quay.io/ceph/ceph:v18.2.4
allowUnsupported: true
allowUnsupported: false
skipUpgradeChecks: false
continueUpgradeAfterChecksEvenIfNotHealthy: false
waitTimeoutForHealthyOSDInMinutes: 10
dashboard:
enabled: true
ssl: true
@ -81,3 +85,46 @@ spec:
mgr: system-cluster-critical
disruptionManagement:
managePodBudgets: true
csi:
readAffinity:
# Enable read affinity to enable clients to optimize reads from an OSD in the same topology.
# Enabling the read affinity may cause the OSDs to consume some extra memory.
# For more details see this doc:
# https://rook.io/docs/rook/latest/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/#enable-read-affinity-for-rbd-volumes
enabled: false
# cephfs driver specific settings.
cephfs:
# Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options.
# kernelMountOptions: ""
# Set CephFS Fuse mount options to use https://docs.ceph.com/en/quincy/man/8/ceph-fuse/#options.
# fuseMountOptions: ""
# healthChecks
# Valid values for daemons are 'mon', 'osd', 'status'
healthCheck:
daemonHealth:
mon:
disabled: false
interval: 45s
osd:
disabled: false
interval: 60s
status:
disabled: false
interval: 60s
# Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons.
livenessProbe:
mon:
disabled: false
mgr:
disabled: false
osd:
disabled: false
# Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons.
startupProbe:
mon:
disabled: false
mgr:
disabled: false
osd:
disabled: false

View file

@ -6,12 +6,12 @@ metadata:
namespace: rook-ceph
spec:
metadataPool:
failureDomain: host
failureDomain: zone
replicated:
size: 3
dataPools:
- name: replicated
failureDomain: host
failureDomain: zone
replicated:
size: 3
preserveFilesystemOnDelete: true

View file

@ -161,7 +161,7 @@ data:
# CSI_NFS_PLUGIN_UPDATE_STRATEGY: "OnDelete"
# kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path.
# ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet"
ROOK_CSI_KUBELET_DIR_PATH: "/var/snap/microk8s/common/var/lib/kubelet"
# Labels to add to the CSI CephFS Deployments and DaemonSets Pods.
# ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2"