From 94279e41b076cb3fd00edea94eee9de1b4b1b45b Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 16 Oct 2024 09:17:47 +0200 Subject: [PATCH] Add cinder and fix ns for cert-manager --- argocd-nginx/base/cert-manager-issuer.yml | 2 +- cinder/cinder-csi-controllerplugin-rbac.yaml | 184 +++++++++++++++++++ cinder/cinder-csi-controllerplugin.yaml | 141 ++++++++++++++ cinder/cinder-csi-nodeplugin-rbac.yaml | 30 +++ cinder/cinder-csi-nodeplugin.yaml | 118 ++++++++++++ cinder/cinder-csi-storageclass.yaml | 5 + cinder/csi-cinder-driver.yaml | 10 + 7 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 cinder/cinder-csi-controllerplugin-rbac.yaml create mode 100644 cinder/cinder-csi-controllerplugin.yaml create mode 100644 cinder/cinder-csi-nodeplugin-rbac.yaml create mode 100644 cinder/cinder-csi-nodeplugin.yaml create mode 100644 cinder/cinder-csi-storageclass.yaml create mode 100644 cinder/csi-cinder-driver.yaml diff --git a/argocd-nginx/base/cert-manager-issuer.yml b/argocd-nginx/base/cert-manager-issuer.yml index a09c522..900df6a 100644 --- a/argocd-nginx/base/cert-manager-issuer.yml +++ b/argocd-nginx/base/cert-manager-issuer.yml @@ -2,7 +2,7 @@ apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: letsencrypt - namespace: cert-manager + namespace: argocd spec: acme: # The ACME server URL diff --git a/cinder/cinder-csi-controllerplugin-rbac.yaml b/cinder/cinder-csi-controllerplugin-rbac.yaml new file mode 100644 index 0000000..067e0f8 --- /dev/null +++ b/cinder/cinder-csi-controllerplugin-rbac.yaml @@ -0,0 +1,184 @@ +# This YAML file contains RBAC API objects, +# which are necessary to run csi controller plugin + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cinder-controller-sa + namespace: kube-system + +--- +# external attacher +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- +# external Provisioner +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +# external snapshotter +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # Secret permission is optional. + # Enable it if your driver needs secret. + # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. + # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +# External Resizer +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-resizer-role + apiGroup: rbac.authorization.k8s.io + diff --git a/cinder/cinder-csi-controllerplugin.yaml b/cinder/cinder-csi-controllerplugin.yaml new file mode 100644 index 0000000..45a76d8 --- /dev/null +++ b/cinder/cinder-csi-controllerplugin.yaml @@ -0,0 +1,141 @@ +# This YAML file contains CSI Controller Plugin Sidecars +# external-attacher, external-provisioner, external-snapshotter +# external-resize, liveness-probe + +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-cinder-controllerplugin + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: csi-cinder-controllerplugin + template: + metadata: + labels: + app: csi-cinder-controllerplugin + spec: + serviceAccount: csi-cinder-controller-sa + containers: + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--leader-election=true" + - "--default-fstype=ext4" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.1 + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--default-fstype=ext4" + - "--feature-gates=Topology=true" + - "--extra-create-metadata" + - "--leader-election=true" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1 + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--extra-create-metadata" + - "--leader-election=true" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--handle-volume-inuse-error=false" + - "--leader-election=true" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.9.0 + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - name: cinder-csi-plugin + image: registry.k8s.io/provider-os/cinder-csi-plugin:v1.28.2 + args: + - /bin/cinder-csi-plugin + - "--endpoint=$(CSI_ENDPOINT)" + - "--cloud-config=$(CLOUD_CONFIG)" + - "--cluster=$(CLUSTER_NAME)" + - "--v=1" + env: + - name: CSI_ENDPOINT + value: unix://csi/csi.sock + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + - name: CLUSTER_NAME + value: kubernetes + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + # The probe + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 60 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: secret-cinderplugin + mountPath: /etc/config + readOnly: true + # - name: cacert + # mountPath: /etc/cacert + # readOnly: true + volumes: + - name: socket-dir + emptyDir: + - name: secret-cinderplugin + secret: + secretName: cloud-config + # - name: cacert + # hostPath: + # path: /etc/cacert diff --git a/cinder/cinder-csi-nodeplugin-rbac.yaml b/cinder/cinder-csi-nodeplugin-rbac.yaml new file mode 100644 index 0000000..912923f --- /dev/null +++ b/cinder/cinder-csi-nodeplugin-rbac.yaml @@ -0,0 +1,30 @@ +# This YAML defines all API objects to create RBAC roles for csi node plugin. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cinder-node-sa + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-nodeplugin-role + apiGroup: rbac.authorization.k8s.io diff --git a/cinder/cinder-csi-nodeplugin.yaml b/cinder/cinder-csi-nodeplugin.yaml new file mode 100644 index 0000000..d114655 --- /dev/null +++ b/cinder/cinder-csi-nodeplugin.yaml @@ -0,0 +1,118 @@ +# This YAML file contains driver-registrar & csi driver nodeplugin API objects, +# which are necessary to run csi nodeplugin for cinder. + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-cinder-nodeplugin + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-cinder-nodeplugin + template: + metadata: + labels: + app: csi-cinder-nodeplugin + spec: + tolerations: + - operator: Exists + serviceAccount: csi-cinder-node-sa + hostNetwork: true + containers: + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.6.3 + args: + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/snap/microk8s/common/var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.9.0 + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: cinder-csi-plugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: registry.k8s.io/provider-os/cinder-csi-plugin:v1.27.1 + args: + - /bin/cinder-csi-plugin + - "--endpoint=$(CSI_ENDPOINT)" + - "--cloud-config=$(CLOUD_CONFIG)" + - "--v=1" + env: + - name: CSI_ENDPOINT + value: unix://csi/csi.sock + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + # The probe + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/snap/microk8s/common/var/lib/kubelet + mountPropagation: "Bidirectional" + - name: pods-probe-dir + mountPath: /dev + mountPropagation: "HostToContainer" + - name: secret-cinderplugin + mountPath: /etc/config + readOnly: true + # - name: cacert + # mountPath: /etc/cacert + # readOnly: true + volumes: + - name: socket-dir + hostPath: + path: /var/snap/microk8s/common/var/lib/kubelet/plugins/cinder.csi.openstack.org + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/snap/microk8s/common/var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet-dir + hostPath: + path: /var/snap/microk8s/common/var/lib/kubelet + type: Directory + - name: pods-probe-dir + hostPath: + path: /dev + type: Directory + - name: secret-cinderplugin + secret: + secretName: cloud-config + # - name: cacert + # hostPath: + # path: /etc/cacert diff --git a/cinder/cinder-csi-storageclass.yaml b/cinder/cinder-csi-storageclass.yaml new file mode 100644 index 0000000..d1c4999 --- /dev/null +++ b/cinder/cinder-csi-storageclass.yaml @@ -0,0 +1,5 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-sc-cinderplugin +provisioner: cinder.csi.openstack.org diff --git a/cinder/csi-cinder-driver.yaml b/cinder/csi-cinder-driver.yaml new file mode 100644 index 0000000..5b681e4 --- /dev/null +++ b/cinder/csi-cinder-driver.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: cinder.csi.openstack.org +spec: + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: + - Persistent + - Ephemeral