diff --git a/k8s/examples/registry/matrix-registry-deployment.yaml b/k8s/examples/registry/matrix-registry-deployment.yaml new file mode 100644 index 0000000..992d29a --- /dev/null +++ b/k8s/examples/registry/matrix-registry-deployment.yaml @@ -0,0 +1,65 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matrix-registry + namespace: matrix-registry + labels: + k8s-app: matrix-registry + kubernetes.io/cluster-service: "true" +spec: + replicas: 3 + selector: + matchLabels: + k8s-app: matrix-registry + template: + metadata: + labels: + k8s-app: matrix-registry + kubernetes.io/cluster-service: "true" + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + k8s-app: matrix-registry + containers: + - name: registry + image: registry:2 + imagePullPolicy: Always + resources: + limits: + cpu: 200m + memory: 300Mi + env: + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_HTTP_SECRET + valueFrom: + secretKeyRef: + name: matrix-registry-secret + key: http-secret + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + volumeMounts: + - name: image-store + mountPath: /var/lib/registry + ports: + - containerPort: 5000 + name: registry + protocol: TCP + livenessProbe: + httpGet: + path: / + port: registry + readinessProbe: + httpGet: + path: / + port: registry + volumes: + - name: image-store + persistentVolumeClaim: + claimName: cephfs-pvc + readOnly: false diff --git a/k8s/examples/registry/matrix-registry-namespace.yaml b/k8s/examples/registry/matrix-registry-namespace.yaml new file mode 100644 index 0000000..70c3c3e --- /dev/null +++ b/k8s/examples/registry/matrix-registry-namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: matrix-registry + labels: + name: matrix-registry-namespace diff --git a/k8s/examples/registry/matrix-registry-pvc.yaml b/k8s/examples/registry/matrix-registry-pvc.yaml new file mode 100644 index 0000000..e012764 --- /dev/null +++ b/k8s/examples/registry/matrix-registry-pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cephfs-pvc + namespace: matrix-registry +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: rook-cephfs diff --git a/k8s/rook/README.md b/k8s/rook/README.md new file mode 100644 index 0000000..4bd9a24 --- /dev/null +++ b/k8s/rook/README.md @@ -0,0 +1,3 @@ +### Rook deployment + +In the operator.yaml change ROOK_CSI_KUBELET_DIR_PATH to "/var/snap/microk8s/common/var/lib/kubelet" diff --git a/k8s/rook/cluster-multizone.yaml b/k8s/rook/cluster-multizone.yaml index ba50479..c804078 100644 --- a/k8s/rook/cluster-multizone.yaml +++ b/k8s/rook/cluster-multizone.yaml @@ -18,21 +18,25 @@ spec: mon: count: 3 allowMultiplePerNode: false - failureDomainLabel: topology.kubernetes.io/zone + failureDomainLabel: topology.kubernetes.io/zone zones: - name: dco - name: sto3 - name: sto4 mgr: count: 2 + allowMultiplePerNode: false modules: + - name: rook + enabled: true - name: pg_autoscaler enabled: true cephVersion: image: quay.io/ceph/ceph:v18.2.4 - allowUnsupported: true + allowUnsupported: false skipUpgradeChecks: false continueUpgradeAfterChecksEvenIfNotHealthy: false + waitTimeoutForHealthyOSDInMinutes: 10 dashboard: enabled: true ssl: true @@ -81,3 +85,46 @@ spec: mgr: system-cluster-critical disruptionManagement: managePodBudgets: true + csi: + readAffinity: + # Enable read affinity to enable clients to optimize reads from an OSD in the same topology. + # Enabling the read affinity may cause the OSDs to consume some extra memory. + # For more details see this doc: + # https://rook.io/docs/rook/latest/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/#enable-read-affinity-for-rbd-volumes + enabled: false + + # cephfs driver specific settings. + cephfs: + # Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. + # kernelMountOptions: "" + # Set CephFS Fuse mount options to use https://docs.ceph.com/en/quincy/man/8/ceph-fuse/#options. + # fuseMountOptions: "" + # healthChecks + # Valid values for daemons are 'mon', 'osd', 'status' + healthCheck: + daemonHealth: + mon: + disabled: false + interval: 45s + osd: + disabled: false + interval: 60s + status: + disabled: false + interval: 60s + # Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons. + livenessProbe: + mon: + disabled: false + mgr: + disabled: false + osd: + disabled: false + # Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons. + startupProbe: + mon: + disabled: false + mgr: + disabled: false + osd: + disabled: false diff --git a/k8s/rook/filesystem.yaml b/k8s/rook/filesystem.yaml index 28e6e2f..0525ec3 100644 --- a/k8s/rook/filesystem.yaml +++ b/k8s/rook/filesystem.yaml @@ -6,12 +6,12 @@ metadata: namespace: rook-ceph spec: metadataPool: - failureDomain: host + failureDomain: zone replicated: size: 3 dataPools: - name: replicated - failureDomain: host + failureDomain: zone replicated: size: 3 preserveFilesystemOnDelete: true diff --git a/k8s/rook/operator.yaml b/k8s/rook/operator.yaml index 9c0a4dd..60ef806 100644 --- a/k8s/rook/operator.yaml +++ b/k8s/rook/operator.yaml @@ -161,7 +161,7 @@ data: # CSI_NFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. - # ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet" + ROOK_CSI_KUBELET_DIR_PATH: "/var/snap/microk8s/common/var/lib/kubelet" # Labels to add to the CSI CephFS Deployments and DaemonSets Pods. # ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2"