Make rook deployment fully multizone aware

k8s/examples/registry/matrix-registry-deployment.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: matrix-registry
+  namespace: matrix-registry
+  labels:
+    k8s-app: matrix-registry
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      k8s-app: matrix-registry
+  template:
+    metadata:
+      labels:
+        k8s-app: matrix-registry
+        kubernetes.io/cluster-service: "true"
+    spec:
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              k8s-app: matrix-registry
+      containers:
+        - name: registry
+          image: registry:2
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: 200m
+              memory: 300Mi
+          env:
+            - name: REGISTRY_HTTP_ADDR
+              value: :5000
+            - name: REGISTRY_HTTP_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: matrix-registry-secret
+                  key: http-secret
+            - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
+              value: /var/lib/registry
+          volumeMounts:
+            - name: image-store
+              mountPath: /var/lib/registry
+          ports:
+            - containerPort: 5000
+              name: registry
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: registry
+          readinessProbe:
+            httpGet:
+              path: /
+              port: registry
+      volumes:
+        - name: image-store
+          persistentVolumeClaim:
+            claimName: cephfs-pvc
+            readOnly: false

k8s/examples/registry/matrix-registry-namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: matrix-registry
+  labels:
+    name: matrix-registry-namespace

k8s/examples/registry/matrix-registry-pvc.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: cephfs-pvc
+  namespace: matrix-registry
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: rook-cephfs

k8s/rook/README.md

@@ -0,0 +1,3 @@
+### Rook deployment
+
+In the operator.yaml change ROOK_CSI_KUBELET_DIR_PATH to "/var/snap/microk8s/common/var/lib/kubelet"

k8s/rook/cluster-multizone.yaml

@@ -18,21 +18,25 @@ spec:
   mon:
     count: 3
     allowMultiplePerNode: false
-    failureDomainLabel: topology.kubernetes.io/zone 
+    failureDomainLabel: topology.kubernetes.io/zone
     zones:
       - name: dco
       - name: sto3
       - name: sto4
   mgr:
     count: 2
+    allowMultiplePerNode: false
     modules:
+      - name: rook
+        enabled: true
       - name: pg_autoscaler
         enabled: true
   cephVersion:
     image: quay.io/ceph/ceph:v18.2.4
-    allowUnsupported: true
+    allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
+  waitTimeoutForHealthyOSDInMinutes: 10
   dashboard:
     enabled: true
     ssl: true
@@ -81,3 +85,46 @@ spec:
     mgr: system-cluster-critical
   disruptionManagement:
     managePodBudgets: true
+  csi:
+    readAffinity:
+      # Enable read affinity to enable clients to optimize reads from an OSD in the same topology.
+      # Enabling the read affinity may cause the OSDs to consume some extra memory.
+      # For more details see this doc:
+      # https://rook.io/docs/rook/latest/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/#enable-read-affinity-for-rbd-volumes
+      enabled: false
+
+    # cephfs driver specific settings.
+    cephfs:
+      # Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options.
+      # kernelMountOptions: ""
+      # Set CephFS Fuse mount options to use https://docs.ceph.com/en/quincy/man/8/ceph-fuse/#options.
+      # fuseMountOptions: ""
+    # healthChecks
+  # Valid values for daemons are 'mon', 'osd', 'status'
+  healthCheck:
+    daemonHealth:
+      mon:
+        disabled: false
+        interval: 45s
+      osd:
+        disabled: false
+        interval: 60s
+      status:
+        disabled: false
+        interval: 60s
+    # Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons.
+    livenessProbe:
+      mon:
+        disabled: false
+      mgr:
+        disabled: false
+      osd:
+        disabled: false
+    # Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons.
+    startupProbe:
+      mon:
+        disabled: false
+      mgr:
+        disabled: false
+      osd:
+        disabled: false

k8s/rook/filesystem.yaml

@@ -6,12 +6,12 @@ metadata:
   namespace: rook-ceph
 spec:
   metadataPool:
-    failureDomain: host
+    failureDomain: zone
     replicated:
       size: 3
   dataPools:
     - name: replicated
-      failureDomain: host
+      failureDomain: zone
       replicated:
         size: 3
   preserveFilesystemOnDelete: true

k8s/rook/operator.yaml

@@ -161,7 +161,7 @@ data:
   # CSI_NFS_PLUGIN_UPDATE_STRATEGY: "OnDelete"
 
   # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path.
-  # ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet"
+  ROOK_CSI_KUBELET_DIR_PATH: "/var/snap/microk8s/common/var/lib/kubelet"
 
   # Labels to add to the CSI CephFS Deployments and DaemonSets Pods.
   # ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2"