diff --git a/cc-deploy/csi-azuredisk-controller.yaml b/cc-deploy/csi-azuredisk-controller.yaml new file mode 100644 index 000000000000..312a33c10959 --- /dev/null +++ b/cc-deploy/csi-azuredisk-controller.yaml @@ -0,0 +1,202 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-cc-local-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-cc-local-controller + template: + metadata: + labels: + app: csi-cc-local-controller + spec: + hostNetwork: true + serviceAccountName: csi-cc-local-controller-sa + nodeSelector: + kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node + priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v4.0.0 + args: + - "--feature-gates=Topology=true,HonorPVReclaimPolicy=true" + - "--csi-address=$(ADDRESS)" + - "--v=2" + - "--timeout=30s" + - "--leader-election" + - "--leader-election-namespace=kube-system" + - "--worker-threads=100" + - "--extra-create-metadata=true" + - "--strict-topology=true" + - "--kube-api-qps=50" + - "--kube-api-burst=100" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-attacher + image: mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v4.5.0 + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "-timeout=1200s" + - "-leader-election" + - "--leader-election-namespace=kube-system" + - "-worker-threads=1000" + - "-kube-api-qps=200" + - "-kube-api-burst=400" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-snapshotter + image: mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v6.3.3 + args: + - "-csi-address=$(ADDRESS)" + - "-leader-election" + - "--leader-election-namespace=kube-system" + - "--v=2" + - "--timeout=1200s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-resizer + image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v1.9.3 + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "--leader-election-namespace=kube-system" + - '-handle-volume-inuse-error=false' + - '-feature-gates=RecoverVolumeExpansionFailure=true' + - "-timeout=240s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.12.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29802 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: cc-local + image: marinerconfpodsdev.azurecr.io/oss/kubernetes-sig/cc-local-csi:latest + imagePullPolicy: Always + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:29804" + - "--user-agent-suffix=OSS-kubectl" + - "--disable-avset-nodes=false" + - "--allow-empty-cloud-config=false" + ports: + - containerPort: 29802 + name: healthz + protocol: TCP + - containerPort: 29804 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: DISK_IMAGE_DIR + value: /images + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /images + name: image-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + - name: image-dir + hostPath: + path: /var/lib/kubelet/plugins/cc.local.csi.azure.com/images + type: DirectoryOrCreate diff --git a/cc-deploy/csi-azuredisk-driver.yaml b/cc-deploy/csi-azuredisk-driver.yaml new file mode 100644 index 000000000000..ba6ac2f83e9e --- /dev/null +++ b/cc-deploy/csi-azuredisk-driver.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: cc.local.csi.azure.com + annotations: + csiDriver: v1.30.0 + snapshot: v6.2.1 +spec: + attachRequired: true + podInfoOnMount: false + fsGroupPolicy: File diff --git a/cc-deploy/csi-azuredisk-node.yaml b/cc-deploy/csi-azuredisk-node.yaml new file mode 100644 index 000000000000..22fd4b1ba1df --- /dev/null +++ b/cc-deploy/csi-azuredisk-node.yaml @@ -0,0 +1,186 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-cc-local-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-cc-local-node + template: + metadata: + labels: + app: csi-cc-local-node + spec: + hostNetwork: true + dnsPolicy: Default + serviceAccountName: csi-cc-local-node-sa + nodeSelector: + kubernetes.io/os: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - operator: "Exists" + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.12.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29803 + - --v=2 + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.10.0 + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/cc.local.csi.azure.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: cc-local + image: marinerconfpodsdev.azurecr.io/oss/kubernetes-sig/cc-local-csi:latest + imagePullPolicy: Always + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--enable-perf-optimization=true" + - "--allow-empty-cloud-config=true" + - "--get-node-info-from-labels=false" + ports: + - containerPort: 29803 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /dev + name: device-dir + - mountPath: /sys/bus/scsi/devices + name: sys-devices-dir + - mountPath: /sys/class/ + name: sys-class + - mountPath: /images + name: image-dir + - mountPath: /run/kata-containers/shared/direct-volumes + name: kata-direct-volumes + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/cc.local.csi.azure.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /sys/bus/scsi/devices + type: Directory + name: sys-devices-dir + - hostPath: + path: /sys/class/ + type: Directory + name: sys-class + - hostPath: + path: /var/lib/kubelet/plugins/cc.local.csi.azure.com/images + type: DirectoryOrCreate + name: image-dir + - hostPath: + path: /run/kata-containers/shared/direct-volumes/ + type: DirectoryOrCreate + name: kata-direct-volumes +--- diff --git a/cc-deploy/rbac-csi-azuredisk-controller.yaml b/cc-deploy/rbac-csi-azuredisk-controller.yaml new file mode 100644 index 000000000000..03c8f921f782 --- /dev/null +++ b/cc-deploy/rbac-csi-azuredisk-controller.yaml @@ -0,0 +1,196 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cc-local-controller-sa + namespace: kube-system +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-cc-local-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: cc-local-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-cc-local-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: cc-local-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-cc-local-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: cc-local-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cc-local-csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-cc-local-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: cc-local-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cc-local-controller-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cc-local-controller-secret-binding +subjects: + - kind: ServiceAccount + name: csi-cc-local-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cc-local-controller-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/cc-deploy/rbac-csi-azuredisk-node.yaml b/cc-deploy/rbac-csi-azuredisk-node.yaml new file mode 100644 index 000000000000..c7edc2399810 --- /dev/null +++ b/cc-deploy/rbac-csi-azuredisk-node.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cc-local-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cc-local-node-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cc-local-node-secret-binding +subjects: + - kind: ServiceAccount + name: csi-cc-local-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-cc-local-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/cc-deploy/storageclass-cc-local-csi.yaml b/cc-deploy/storageclass-cc-local-csi.yaml new file mode 100644 index 000000000000..0b3d46b9d934 --- /dev/null +++ b/cc-deploy/storageclass-cc-local-csi.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: cc-local-csi +provisioner: cc.local.csi.azure.com +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true diff --git a/pkg/azuredisk/nodeserver.go b/pkg/azuredisk/nodeserver.go index dcdfa2e505c5..9e035982ac8f 100644 --- a/pkg/azuredisk/nodeserver.go +++ b/pkg/azuredisk/nodeserver.go @@ -254,8 +254,11 @@ func (d *Driver) NodePublishVolume(_ context.Context, req *csi.NodePublishVolume VolumeType: "blk", Device: device, FsType: fstype, - Metadata: nil, - Options: nil, + Metadata: map[string]string{ + "confidential": "true", + "ephemeral": "true", + }, + Options: nil, } rawMountInfo, err := json.Marshal(mountInfo)