From edcd09f685d1fcbc68dff8fd8d283b453c9d47a0 Mon Sep 17 00:00:00 2001 From: Amarnath Valluri Date: Sat, 14 Nov 2020 10:43:35 +0200 Subject: [PATCH] operator: API(v1beta1) changes to support per-container resources Moving the deployment API to `v1beta1` along with supporting per-container resources. In Kubernetes resource requiremets are per-container. Existing `ControllerResources` and `NodeResources` API fields are not suffice to configure resources for sidecar containers. Hence added new fields to deployment specification for per-container resources. Also chose the default cpu,memory resource requests/limits for the containers as suggested by the VPA (Verticall Pod Autoscaler). TODOS: - To support deprecated `v1alpha1` API, needs a conversion webhook in the cluster to handled conversions between different API versions. FIXES: #616 --- ...m-csi.intel.com_v1beta1_deployment_cr.yaml | 12 + .../crd/pmem-csi.intel.com_deployments.yaml | 287 ++++++++++++++++++ .../kustomize/olm-catalog/kustomization.yaml | 1 + docs/DEVELOPMENT.md | 9 + docs/design.md | 2 +- docs/install.md | 26 +- hack/stress-driver.sh | 27 ++ pkg/apis/addtoscheme_pmemcsi_v1beta1.go | 16 + pkg/apis/pmemcsi/base/defaults.go | 28 ++ pkg/apis/pmemcsi/v1alpha1/deployment_types.go | 8 + pkg/apis/pmemcsi/v1beta1/deployment_types.go | 152 ++++++++++ .../pmemcsi/v1beta1/deployment_types_test.go | 210 +++++++++++++ pkg/apis/pmemcsi/v1beta1/doc.go | 10 + pkg/apis/pmemcsi/v1beta1/register.go | 25 ++ .../pmemcsi/v1beta1/zz_generated.deepcopy.go | 105 +++++++ pkg/deployments/load.go | 45 ++- pkg/deployments/load_test.go | 2 +- .../deployment/controller_driver.go | 10 +- .../deployment/deployment_controller.go | 20 +- .../deployment/deployment_controller_test.go | 26 +- .../deployment/testcases/testcases.go | 58 +++- pkg/pmem-csi-operator/main.go | 2 +- test/e2e/deploy/deploy.go | 2 +- test/e2e/deploy/operator.go | 2 +- test/e2e/operator/deployment_api.go | 34 ++- test/e2e/operator/validate/validate.go | 2 +- 26 files changed, 1060 insertions(+), 61 deletions(-) create mode 100644 deploy/common/pmem-csi.intel.com_v1beta1_deployment_cr.yaml create mode 100755 hack/stress-driver.sh create mode 100644 pkg/apis/addtoscheme_pmemcsi_v1beta1.go create mode 100644 pkg/apis/pmemcsi/v1beta1/deployment_types.go create mode 100644 pkg/apis/pmemcsi/v1beta1/deployment_types_test.go create mode 100644 pkg/apis/pmemcsi/v1beta1/doc.go create mode 100644 pkg/apis/pmemcsi/v1beta1/register.go create mode 100644 pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go diff --git a/deploy/common/pmem-csi.intel.com_v1beta1_deployment_cr.yaml b/deploy/common/pmem-csi.intel.com_v1beta1_deployment_cr.yaml new file mode 100644 index 0000000000..bf1cfa2ebd --- /dev/null +++ b/deploy/common/pmem-csi.intel.com_v1beta1_deployment_cr.yaml @@ -0,0 +1,12 @@ +apiVersion: pmem-csi.intel.com/v1beta1 +kind: Deployment +metadata: + name: pmem-csi.intel.com +spec: + deviceMode: "lvm" + nodeSelector: + # When using Node Feature Discovery (NFD): + feature.node.kubernetes.io/memory-nv.dax: "true" + # When using manual node labeling with that label: + # storage: pmem + diff --git a/deploy/crd/pmem-csi.intel.com_deployments.yaml b/deploy/crd/pmem-csi.intel.com_deployments.yaml index 1f00430b31..f9ce969ca2 100644 --- a/deploy/crd/pmem-csi.intel.com_deployments.yaml +++ b/deploy/crd/pmem-csi.intel.com_deployments.yaml @@ -246,6 +246,293 @@ spec: type: object type: object served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.deviceMode + name: DeviceMode + type: string + - jsonPath: .spec.nodeSelector + name: NodeSelector + type: string + - jsonPath: .spec.image + name: Image + type: string + - jsonPath: .status.phase + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Deployment is the Schema for the deployments API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeploymentSpec defines the desired state of Deployment + properties: + caCert: + description: CACert encoded root certificate of the CA by which the + registry and node controller certificates are signed If not provided + operator uses a self-signed CA certificate + format: byte + type: string + controllerDriverResources: + description: ControllerDriverResources Compute resources required + by driver container running on master node + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + deviceMode: + description: DeviceMode to use to manage PMEM devices. One of lvm, + direct + type: string + image: + description: PMEM-CSI driver container image + type: string + imagePullPolicy: + description: PullPolicy image pull policy one of Always, Never, IfNotPresent + type: string + kubeletDir: + description: KubeletDir kubelet's root directory path + type: string + labels: + additionalProperties: + type: string + description: Labels contains additional labels for all objects created + by the operator. + type: object + logLevel: + description: LogLevel number for the log verbosity kubebuilder:default=3 + type: integer + nodeControllerCert: + description: NodeControllerCert encoded certificate signed by a CA + for node controller server authentication If not provided, provisioned + one by the operator using self-signed CA + format: byte + type: string + nodeControllerKey: + description: NodeControllerPrivateKey encoded private key used for + node controller server certificate If not provided, provisioned + one by the operator + format: byte + type: string + nodeDriverResources: + description: NodeDriverResources Compute resources required by driver + container running on worker nodes + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + nodeRegistrarImage: + description: NodeRegistrarImage CSI node driver registrar sidecar + image + type: string + nodeRegistrarResources: + description: NodeRegistrarResources Compute resources required by + node registrar sidecar container + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector node labels to use for selection of driver + node + type: object + pmemPercentage: + description: PMEMPercentage represents the percentage of space to + be used by the driver in each PMEM region on every node. This is + only valid for driver in LVM mode. -kubebuilder:validation:Minimum=1 + -kubebuilder:validation:Maximum=100 -kubebuilder:default=100 + type: integer + provisionerImage: + description: ProvisionerImage CSI provisioner sidecar image + type: string + provisionerResources: + description: ProvisionerResources Compute resources required by provisioner + sidecar container + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + registryCert: + description: RegistryCert encoded certificate signed by a CA for registry + server authentication If not provided, provisioned one by the operator + using self-signed CA + format: byte + type: string + registryKey: + description: RegistryPrivateKey encoded private key used for registry + server certificate If not provided, provisioned one by the operator + format: byte + type: string + type: object + status: + description: DeploymentStatus defines the observed state of Deployment + properties: + conditions: + description: Conditions + items: + description: DeploymentCondition type definition for driver deployment + status conditions + properties: + lastUpdateTime: + description: Last time the condition was probed. + format: date-time + type: string + reason: + description: Message human readable text that explain why this + condition is in this state + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + driverComponents: + items: + description: DriverStatus type definition for representing deployed + driver status + properties: + component: + description: 'DriverComponent represents type of the driver: + controller or node' + type: string + lastUpdated: + description: LastUpdated time of the driver status + format: date-time + type: string + reason: + description: Reason represents the human readable text that + explains why the driver is in this state. + type: string + status: + description: Status represents the state of the component; one + of `Ready` or `NotReady`. Component becomes `Ready` if all + the instances(Pods) of the driver component are in running + state. Otherwise, `NotReady`. + type: string + required: + - component + - reason + - status + type: object + type: array + lastUpdated: + description: LastUpdated time of the deployment status + format: date-time + type: string + phase: + description: Phase indicates the state of the deployment + type: string + reason: + type: string + type: object + type: object + served: true storage: true subresources: status: {} diff --git a/deploy/kustomize/olm-catalog/kustomization.yaml b/deploy/kustomize/olm-catalog/kustomization.yaml index 543c52fe5e..9bff5576eb 100644 --- a/deploy/kustomize/olm-catalog/kustomization.yaml +++ b/deploy/kustomize/olm-catalog/kustomization.yaml @@ -3,6 +3,7 @@ bases: resources: - ../../common/pmem-csi.intel.com_v1alpha1_deployment_cr.yaml +- ../../common/pmem-csi.intel.com_v1beta1_deployment_cr.yaml - ../../crd/pmem-csi.intel.com_deployments.yaml images: diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 4b0933341b..ae067fc550 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -287,6 +287,12 @@ Resource requirements depend on the workload. To generate some load, run make test_e2e TEST_E2E_FOCUS=lvm-production.*late.binding.*stress.test ``` +Alternatively, could run [`hack/stress-driver.sh`](hack/stress-driver.sh) +helper script to generate the load on the driver +```console +ROUNDS=500 VOL_COUNT=5 ./hack/stress-driver.sh +``` + Now resource recommendations can be retrieved with: ```console @@ -295,6 +301,9 @@ kubectl describe vpa kubectl get vpa pmem-csi-node -o jsonpath='{range .status.recommendation.containerRecommendations[*]}{.containerName}{":\n\tRequests: "}{.lowerBound}{"\n\tLimits: "}{.upperBound}{"\n"}{end}' ``` +The default resource requirements used for the driver deployments by the operator +are chosen from the VPA recommendations described in this section. + ## Switching device mode If device mode is switched between LVM and direct(aka ndctl), please keep diff --git a/docs/design.md b/docs/design.md index ed6830c5f3..362ccbda6b 100644 --- a/docs/design.md +++ b/docs/design.md @@ -421,7 +421,7 @@ tools and APIs. The driver deployment is controlled by a cluster-scoped [custom resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) named [`Deployment`](./install.md#pmem-csi-deployment-crd) in the -`pmem-csi.intel.com/v1alpha1` API group. The operator runs inside the cluster +`pmem-csi.intel.com/v1beta1` API group. The operator runs inside the cluster and listens for deployment changes. It makes sure that the required Kubernetes objects are created for a driver deployment. Refer to [Deployment CRD](./install.md#deployment) for details. diff --git a/docs/install.md b/docs/install.md index 185ca15eeb..8e42b88917 100644 --- a/docs/install.md +++ b/docs/install.md @@ -275,7 +275,7 @@ Here is a minimal example driver deployment created with a custom resource: ``` ShellSession $ kubectl create -f - < Annotations: -API Version: pmem-csi.intel.com/v1alpha1 +API Version: pmem-csi.intel.com/v1beta1 Kind: Deployment Metadata: Creation Timestamp: 2020-10-07T07:31:58Z Generation: 1 Managed Fields: - API Version: pmem-csi.intel.com/v1alpha1 + API Version: pmem-csi.intel.com/v1beta1 Fields Type: FieldsV1 fieldsV1: f:spec: @@ -320,7 +320,7 @@ Metadata: Manager: kubectl-create Operation: Update Time: 2020-10-07T07:31:58Z - API Version: pmem-csi.intel.com/v1alpha1 + API Version: pmem-csi.intel.com/v1beta1 Fields Type: FieldsV1 fieldsV1: f:status: @@ -333,7 +333,7 @@ Metadata: Operation: Update Time: 2020-10-07T07:32:22Z Resource Version: 1235740 - Self Link: /apis/pmem-csi.intel.com/v1alpha1/deployments/pmem-csi.intel.com + Self Link: /apis/pmem-csi.intel.com/v1beta1/deployments/pmem-csi.intel.com UID: d8635490-53fa-4eec-970d-cd4c76f53b23 Spec: Device Mode: lvm @@ -1200,13 +1200,15 @@ The current API for PMEM-CSI `Deployment` resources is: |Field | Type | Description | |---|---|---| -| apiVersion | string | `pmem-csi.intel.com/v1alpha1`| +| apiVersion | string | `pmem-csi.intel.com/v1beta1` or `pmem-csi.intel.com/v1alpha1` | | kind | string | `Deployment`| | metadata | [ObjectMeta](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata) | Object metadata, name used for CSI driver and as prefix for sub-objects | | spec | [DeploymentSpec](#deployment-spec) | Specification of the desired behavior of the deployment | #### DeploymentSpec +Below specification fields are valid in all API revisions unless noted otherwise in the description. + |Field | Type | Description | Default Value | |---|---|---|---| | image | string | PMEM-CSI docker image name used for the deployment | the same image as the operator1 | @@ -1215,8 +1217,12 @@ The current API for PMEM-CSI `Deployment` resources is: | pullPolicy | string | Docker image pull policy. either one of `Always`, `Never`, `IfNotPresent` | `IfNotPresent` | | logLevel | integer | PMEM-CSI driver logging level | 3 | | deviceMode | string | Device management mode to use. Supports one of `lvm` or `direct` | `lvm` -| controllerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller pod | -| nodeResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the pods running on node(s) | +| controllerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller pod.
3_Deprecated and only available in `v1alpha1` API revision._ | +| nodeResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the pods running on node(s).
_3Deprecated and only available in `v1alpha1` API revision._ | +| controllerDriverResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller driver container running on master node. Available in `v1beta1` onwords. | +| nodeDriverResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the driver container running on worker node(s).
_Available in `v1beta1` onwards._ | +| provisionerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the [external provisioner](https://kubernetes-csi.github.io/docs/external-provisioner.html) sidecar container.
_Available in `v1beta1` onwards._ | +| nodeRegistrarResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the [driver registrar](https://kubernetes-csi.github.io/docs/node-driver-registrar.html) sidecar container running on worker node(s).
_Available in `v1beta1` onwards._ | | registryCert | string | Encoded tls certificate signed by a certificate authority used for driver's controller registry server | generated by operator self-signed CA | | nodeControllerCert | string | Encoded tls certificate signed by a certificate authority used for driver's node controllers | generated by operator self-signed CA | | registryKey | string | Encoded RSA private key used for signing by `registryCert` | generated by the operator | @@ -1237,6 +1243,10 @@ appropriate values: operator dynamically chooses suitable image versions. Users have to take care of that themselves when overriding the values. +3 Pod level resource requests(`nodeResources` and `controllerResources`) +are deprecated infavor of per-container(`nodeDriverResources`, `nodeRegistrarResources`, +`controllerDriverResources` and `provisionerResources`) resource requests. + **WARNING**: although all fields can be modified and changes will be propagated to the deployed driver, not all changes are safe. In particular, changing the `deviceMode` will not work when there are diff --git a/hack/stress-driver.sh b/hack/stress-driver.sh new file mode 100755 index 0000000000..6d6ff97d9a --- /dev/null +++ b/hack/stress-driver.sh @@ -0,0 +1,27 @@ +# !/bin/bash + +# Utility script to stress the driver by creating and deleting the volumes. It +# runs by default 100 runs, each run it creates and deletes 10(5 xfs and 5 ext) +# volumes of each 4GB size. Hence, it requires approximately 40 GB of PMEM memory +# +# The main intention of this script is to determine the default resource requirements +# to be set for the driver deployed via the operator. +# This is expected to run by setting up the metrics-server and VirtualPodAutoscaler +# describe in the 'Performance and resource measurements' section in the +# developer documentation(DEPLOYMENT.md). + +ROUNDS=${ROUNDS:-100} +VOL_COUNT=${VOL_COUNT:-5} # 5 ext4 + 5 xfs = 10 * 4Gi ~= 40Gi + +for i in $(seq 1 1 $ROUNDS) ; do + echo "Round #$i:" + echo "Creating volumes..." + for j in $(seq 1 1 $VOL_COUNT) ; do + sed -e "s;\(.*name:\)\(.*\);\1\2-$j;g" < deploy/common/pmem-pvc.yaml | kubectl create -f - + done + echo "Deleting all pvc..." + while [ "$(kubectl get pvc --no-headers | wc -l)" -ne "0" ]; do + kubectl delete pvc --all + kubectl delete pv --all + done +done diff --git a/pkg/apis/addtoscheme_pmemcsi_v1beta1.go b/pkg/apis/addtoscheme_pmemcsi_v1beta1.go new file mode 100644 index 0000000000..52404e0425 --- /dev/null +++ b/pkg/apis/addtoscheme_pmemcsi_v1beta1.go @@ -0,0 +1,16 @@ +/* +Copyright 2020 The Kubernetes Authors. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package apis + +import ( + "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, v1beta1.SchemeBuilder.AddToScheme) +} diff --git a/pkg/apis/pmemcsi/base/defaults.go b/pkg/apis/pmemcsi/base/defaults.go index a0a6d04b9a..092bb7a7ed 100644 --- a/pkg/apis/pmemcsi/base/defaults.go +++ b/pkg/apis/pmemcsi/base/defaults.go @@ -34,6 +34,26 @@ const ( // DefaultRegistrarImage default node driver registrar image to use DefaultRegistrarImage = defaultRegistrarImageName + ":" + defaultRegistrarImageTag + // Below resource requests and limits are derived(with minor adjustments) from + // recommendations reported by VirtualPodAutoscaler(LowerBound -> Requests and UpperBound -> Limits) + + // DefaultControllerResourceRequestCPU default CPU resource request used for controller driver container + DefaultControllerResourceRequestCPU = "12m" // MilliSeconds + // DefaultControllerResourceRequestMemory default memory resource request used for controller driver container + DefaultControllerResourceRequestMemory = "128Mi" // MB + // DefaultNodeResourceRequestCPU default CPU resource request used for node driver container + DefaultNodeResourceRequestCPU = "100m" // MilliSeconds + // DefaultNodeResourceRequestMemory default memory resource request used for node driver container + DefaultNodeResourceRequestMemory = "250Mi" // MB + // DefaultNodeRegistrarRequestCPU default CPU resource request used for node registrar container + DefaultNodeRegistrarRequestCPU = "12m" // MilliSeconds + // DefaultNodeRegistrarRequestMemory default memory resource request used for node registrar container + DefaultNodeRegistrarRequestMemory = "128Mi" // MB + // DefaultProvisionerRequestCPU default CPU resource request used for provisioner container + DefaultProvisionerRequestCPU = "12m" // MilliSeconds + // DefaultProvisionerRequestMemory default memory resource request used for node registrar container + DefaultProvisionerRequestMemory = "128Mi" // MB + // DefaultControllerResourceLimitCPU default CPU resource limit used for controller driver container DefaultControllerResourceLimitCPU = "500m" // MilliSeconds // DefaultControllerResourceLimitMemory default memory resource limit used for controller driver container @@ -42,6 +62,14 @@ const ( DefaultNodeResourceLimitCPU = "600m" // MilliSeconds // DefaultNodeResourceLimitMemory default memory resource limit used for node driver container DefaultNodeResourceLimitMemory = "500Mi" // MB + // DefaultNodeRegistrarLimitCPU default CPU resource limit used for node registrar container + DefaultNodeRegistrarLimitCPU = "100m" // MilliSeconds + // DefaultNodeRegistrarLimitMemory default memory resource limit used for node registrar container + DefaultNodeRegistrarLimitMemory = "128Mi" // MB + // DefaultProvisionerLimitCPU default CPU resource limit used for provisioner container + DefaultProvisionerLimitCPU = "250m" // MilliSeconds + // DefaultProvisionerLimitMemory default memory resource limit used for node registrar container + DefaultProvisionerLimitMemory = "250Mi" // MB // DefaultDeviceMode default device manger used for deployment DefaultDeviceMode = DeviceModeLVM diff --git a/pkg/apis/pmemcsi/v1alpha1/deployment_types.go b/pkg/apis/pmemcsi/v1alpha1/deployment_types.go index 38481ec720..52d0e1402f 100644 --- a/pkg/apis/pmemcsi/v1alpha1/deployment_types.go +++ b/pkg/apis/pmemcsi/v1alpha1/deployment_types.go @@ -73,6 +73,10 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error { if d.Spec.ControllerResources == nil { d.Spec.ControllerResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultControllerResourceRequestCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultControllerResourceRequestMemory), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(base.DefaultControllerResourceLimitCPU), corev1.ResourceMemory: resource.MustParse(base.DefaultControllerResourceLimitMemory), @@ -82,6 +86,10 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error { if d.Spec.NodeResources == nil { d.Spec.NodeResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultNodeResourceRequestCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultNodeResourceRequestMemory), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(base.DefaultNodeResourceLimitCPU), corev1.ResourceMemory: resource.MustParse(base.DefaultNodeResourceLimitMemory), diff --git a/pkg/apis/pmemcsi/v1beta1/deployment_types.go b/pkg/apis/pmemcsi/v1beta1/deployment_types.go new file mode 100644 index 0000000000..860a88808e --- /dev/null +++ b/pkg/apis/pmemcsi/v1beta1/deployment_types.go @@ -0,0 +1,152 @@ +/* +Copyright 2020 The Kubernetes Authors. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package v1beta1 + +import ( + "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DeploymentSpec defines the desired state of Deployment +// +k8s:deepcopy-gen=true +type DeploymentSpec struct { + // Important: Run "make operator-generate-k8s" to regenerate code after modifying this file + + base.DeploymentSpec `json:",inline"` + + // ProvisionerResources Compute resources required by provisioner sidecar container + ProvisionerResources *corev1.ResourceRequirements `json:"provisionerResources,omitempty"` + // NodeRegistrarResources Compute resources required by node registrar sidecar container + NodeRegistrarResources *corev1.ResourceRequirements `json:"nodeRegistrarResources,omitempty"` + // NodeDriverResources Compute resources required by driver container running on worker nodes + NodeDriverResources *corev1.ResourceRequirements `json:"nodeDriverResources,omitempty"` + // ControllerDriverResources Compute resources required by driver container running on master node + ControllerDriverResources *corev1.ResourceRequirements `json:"controllerDriverResources,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Deployment is the Schema for the deployments API +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=deployments,scope=Cluster +// +kubebuilder:printcolumn:name="DeviceMode",type=string,JSONPath=`.spec.deviceMode` +// +kubebuilder:printcolumn:name="NodeSelector",type=string,JSONPath=`.spec.nodeSelector` +// +kubebuilder:printcolumn:name="Image",type=string,JSONPath=`.spec.image` +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:storageversion +type Deployment struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DeploymentSpec `json:"spec,omitempty"` + Status base.DeploymentStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// DeploymentList contains a list of Deployment +type DeploymentList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Deployment `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Deployment{}, &DeploymentList{}) +} + +func (d *Deployment) SetCondition(t base.DeploymentConditionType, state corev1.ConditionStatus, reason string) { + d.Status.SetCondition(t, state, reason) +} + +func (d *Deployment) SetDriverStatus(t base.DriverType, status, reason string) { + d.Status.SetDriverStatus(t, status, reason) +} + +// EnsureDefaults make sure that the deployment object has all defaults set properly +func (d *Deployment) EnsureDefaults(operatorImage string) error { + if err := d.Spec.EnsureDefaults(operatorImage); err != nil { + return err + } + + if d.Spec.ControllerDriverResources == nil { + d.Spec.ControllerDriverResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultControllerResourceRequestCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultControllerResourceRequestMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultControllerResourceLimitCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultControllerResourceLimitMemory), + }, + } + } + + if d.Spec.ProvisionerResources == nil { + d.Spec.ProvisionerResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultProvisionerRequestCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultProvisionerRequestMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultProvisionerLimitCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultProvisionerLimitMemory), + }, + } + } + + if d.Spec.NodeDriverResources == nil { + d.Spec.NodeDriverResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultNodeResourceRequestCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultNodeResourceRequestMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultNodeResourceLimitCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultNodeResourceLimitMemory), + }, + } + } + + if d.Spec.NodeRegistrarResources == nil { + d.Spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultNodeRegistrarRequestCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultNodeRegistrarRequestMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(base.DefaultNodeRegistrarLimitCPU), + corev1.ResourceMemory: resource.MustParse(base.DefaultNodeRegistrarLimitMemory), + }, + } + } + + return nil +} + +// GetOwnerReference returns self owner reference could be used by other object +// to add this deployment to it's owner reference list. +func (d *Deployment) GetOwnerReference() metav1.OwnerReference { + blockOwnerDeletion := true + isController := true + return metav1.OwnerReference{ + APIVersion: d.APIVersion, + Kind: d.Kind, + Name: d.GetName(), + UID: d.GetUID(), + BlockOwnerDeletion: &blockOwnerDeletion, + Controller: &isController, + } +} + +// HaveCertificatesConfigured checks if the configured deployment +func (d *Deployment) HaveCertificatesConfigured() (bool, error) { + return d.Spec.HaveCertificatesConfigured() +} diff --git a/pkg/apis/pmemcsi/v1beta1/deployment_types_test.go b/pkg/apis/pmemcsi/v1beta1/deployment_types_test.go new file mode 100644 index 0000000000..202bbd1e28 --- /dev/null +++ b/pkg/apis/pmemcsi/v1beta1/deployment_types_test.go @@ -0,0 +1,210 @@ +/* +Copyright 2019 Intel Corporation. + +SPDX-License-Identifier: Apache-2.0 +*/ +package v1beta1_test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/intel/pmem-csi/pkg/apis" + "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes/scheme" +) + +func TestDeploymentType(t *testing.T) { + RegisterFailHandler(Fail) + + err := apis.AddToScheme(scheme.Scheme) + Expect(err).Should(BeNil(), "Failed to add api schema") + + RunSpecs(t, "PMEM Operator API test suite") +} + +var _ = Describe("Operator", func() { + + BeforeEach(func() { + }) + + Context("API", func() { + // + // TODO: Add input-validation tests for the API spec fields + // + It("shall set defaults for empty deployment", func() { + d := api.Deployment{} + err := d.EnsureDefaults("") + Expect(err).ShouldNot(HaveOccurred(), "ensure defaults") + + Expect(d.Spec.LogLevel).Should(BeEquivalentTo(base.DefaultLogLevel), "default logging level mismatch") + Expect(d.Spec.DeviceMode).Should(BeEquivalentTo(base.DefaultDeviceMode), "default driver mode mismatch") + Expect(d.Spec.Image).Should(BeEquivalentTo(base.DefaultDriverImage), "default driver image mismatch") + Expect(d.Spec.PullPolicy).Should(BeEquivalentTo(base.DefaultImagePullPolicy), "default image pull policy mismatch") + Expect(d.Spec.ProvisionerImage).Should(BeEquivalentTo(base.DefaultProvisionerImage), "default provisioner image mismatch") + Expect(d.Spec.NodeRegistrarImage).Should(BeEquivalentTo(base.DefaultRegistrarImage), "default node driver registrar image mismatch") + + Expect(d.Spec.ControllerDriverResources).ShouldNot(BeNil(), "default controller driver resources not set") + rs := d.Spec.ControllerDriverResources.Requests + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultControllerResourceRequestCPU), "controller driver 'cpu' resource request mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultControllerResourceRequestMemory), "controller driver 'memory' resource request mismatch") + rs = d.Spec.ControllerDriverResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultControllerResourceLimitCPU), "controller driver 'cpu' resource limit mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultControllerResourceLimitMemory), "controller driver 'memory' resource limit mismatch") + + Expect(d.Spec.NodeDriverResources).ShouldNot(BeNil(), "default node driver resources not set") + rs = d.Spec.NodeDriverResources.Requests + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultNodeResourceRequestCPU), "node driver 'cpu' resource request mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultNodeResourceRequestMemory), "node driver 'cpu' resource request mismatch") + rs = d.Spec.NodeDriverResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultNodeResourceLimitCPU), "node driver 'cpu' resource limit mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultNodeResourceLimitMemory), "node driver 'cpu' resource limit mismatch") + + Expect(d.Spec.NodeRegistrarResources).ShouldNot(BeNil(), "default node registrar resources not set") + rs = d.Spec.NodeRegistrarResources.Requests + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultNodeRegistrarRequestCPU), "node registrar 'cpu' resource request mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultNodeRegistrarRequestMemory), "node registrar 'cpu' resource request mismatch") + rs = d.Spec.NodeRegistrarResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultNodeRegistrarLimitCPU), "node registrar 'cpu' resource limit mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultNodeRegistrarLimitMemory), "node registrar 'cpu' resource limit mismatch") + + Expect(d.Spec.ProvisionerResources).ShouldNot(BeNil(), "default provisioner resources not set") + rs = d.Spec.ProvisionerResources.Requests + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultProvisionerRequestCPU), "provisioner 'cpu' resource request mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultProvisionerRequestMemory), "provisioner 'cpu' resource request mismatch") + rs = d.Spec.ProvisionerResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(base.DefaultProvisionerLimitCPU), "provisioner 'cpu' resource limit mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(base.DefaultProvisionerLimitMemory), "provisioner 'cpu' resource limit mismatch") + }) + + It("shall be able to set values", func() { + yaml := `kind: Deployment +apiVersion: pmem-csi.intel.com/v1beta1 +metadata: + name: test-deployment +spec: + logLevel: 10 + deviceMode: direct + image: test-driver:v0.0.0 + imagePullPolicy: Never + provisionerImage: test-provisioner:v0.0.0 + nodeRegistrarImage: test-driver-registrar:v0.0.0 + controllerDriverResources: + requests: + cpu: 1000m + memory: 10Mi + nodeDriverResources: + requests: + cpu: 2000m + memory: 100Mi + nodeRegistrarResources: + requests: + cpu: 100m + memory: 250Mi + provisionerResources: + requests: + cpu: 50m + memory: 150Mi +` + decode := scheme.Codecs.UniversalDeserializer().Decode + + obj, _, err := decode([]byte(yaml), nil, nil) + Expect(err).Should(BeNil(), "Failed to parse deployment") + Expect(obj).ShouldNot(BeNil(), "Nil deployment object") + + d := obj.(*api.Deployment) + err = d.EnsureDefaults("") + Expect(err).ShouldNot(HaveOccurred(), "ensure defaults") + + Expect(d.Spec.LogLevel).Should(BeEquivalentTo(10), "logging level mismatch") + Expect(d.Spec.DeviceMode).Should(BeEquivalentTo("direct"), "driver mode mismatch") + Expect(d.Spec.Image).Should(BeEquivalentTo("test-driver:v0.0.0"), "driver image mismatch") + Expect(d.Spec.PullPolicy).Should(BeEquivalentTo("Never"), "image pull policy mismatch") + Expect(d.Spec.ProvisionerImage).Should(BeEquivalentTo("test-provisioner:v0.0.0"), "provisioner image mismatch") + Expect(d.Spec.NodeRegistrarImage).Should(BeEquivalentTo("test-driver-registrar:v0.0.0"), "node driver registrar image mismatch") + + Expect(d.Spec.ControllerDriverResources).ShouldNot(BeNil(), "controller driver resources not set") + rs := d.Spec.ControllerDriverResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("1000m"))).Should(BeZero(), "controller driver 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("10Mi"))).Should(BeZero(), "controller driver 'memory' resource requests mismatch") + + Expect(d.Spec.NodeDriverResources).ShouldNot(BeNil(), "node driver resources not set") + rs = d.Spec.NodeDriverResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("2000m"))).Should(BeZero(), "node driver 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("100Mi"))).Should(BeZero(), "node driver 'memory' resource requests mismatch") + + Expect(d.Spec.NodeRegistrarResources).ShouldNot(BeNil(), "node registrar resources not set") + rs = d.Spec.NodeRegistrarResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("100m"))).Should(BeZero(), "node registrar 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("250Mi"))).Should(BeZero(), "node registrar 'memory' resource request mismatch") + + Expect(d.Spec.ProvisionerResources).ShouldNot(BeNil(), "provisioner resources not set") + rs = d.Spec.ProvisionerResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("50m"))).Should(BeZero(), "provisioner 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("150Mi"))).Should(BeZero(), "provisioner 'memory' resource requests mismatch") + }) + + It("should have valid json schema", func() { + + crdFile := os.Getenv("REPO_ROOT") + "/deploy/crd/pmem-csi.intel.com_deployments.yaml" + data, err := ioutil.ReadFile(crdFile) + Expect(err).ShouldNot(HaveOccurred(), "load crd data") + crd := &apiextensions.CustomResourceDefinition{} + + deserializer := scheme.Codecs.UniversalDeserializer() + _, _, err = deserializer.Decode(data, nil, crd) + Expect(err).ShouldNot(HaveOccurred(), "decode crd file") + + var crdProp *apiextensions.JSONSchemaProps + for _, v := range crd.Spec.Versions { + if v.Name == api.SchemeBuilder.GroupVersion.Version { + crdProp = v.Schema.OpenAPIV3Schema + } + } + Expect(crdProp).ShouldNot(BeNil(), "Nil CRD schmea") + Expect(crdProp.Type).Should(BeEquivalentTo("object"), "Deployment JSON schema type mismatch") + spec, ok := crdProp.Properties["spec"] + Expect(ok).Should(BeTrue(), "Deployment JSON schema does not have 'spec'") + status, ok := crdProp.Properties["status"] + Expect(ok).Should(BeTrue(), "Deployment JSON schema does not have 'status'") + + specProperties := map[string]string{ + "logLevel": "integer", + "image": "string", + "imagePullPolicy": "string", + "provisionerImage": "string", + "nodeRegistrarImage": "string", + "controllerDriverResources": "object", + "nodeDriverResources": "object", + "provisionerResources": "object", + "nodeRegistrarResources": "object", + "kubeletDir": "string", + } + + for key := range spec.Properties { + By(key) + } + + for prop, tipe := range specProperties { + jsonProp, ok := spec.Properties[prop] + Expect(ok).Should(BeTrue(), "Missing %q property in deployment spec", prop) + Expect(jsonProp.Type).Should(BeEquivalentTo(tipe), "%q property type mismatch", prop) + } + + statusProperties := map[string]string{ + "phase": "string", + } + for prop, tipe := range statusProperties { + jsonProp, ok := status.Properties[prop] + Expect(ok).Should(BeTrue(), "Missing %q property in deployment status", prop) + Expect(jsonProp.Type).Should(BeEquivalentTo(tipe), "%q property type mismatch", prop) + } + }) + }) +}) diff --git a/pkg/apis/pmemcsi/v1beta1/doc.go b/pkg/apis/pmemcsi/v1beta1/doc.go new file mode 100644 index 0000000000..2d96850b43 --- /dev/null +++ b/pkg/apis/pmemcsi/v1beta1/doc.go @@ -0,0 +1,10 @@ +/* +Copyright 2020 The Kubernetes Authors. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package v1beta1 contains API Schema definitions for the pmem-csi v1beta1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=pmem-csi.intel.com +package v1beta1 diff --git a/pkg/apis/pmemcsi/v1beta1/register.go b/pkg/apis/pmemcsi/v1beta1/register.go new file mode 100644 index 0000000000..725abfb28d --- /dev/null +++ b/pkg/apis/pmemcsi/v1beta1/register.go @@ -0,0 +1,25 @@ +/* +Copyright 2020 The Kubernetes Authors. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// NOTE: Boilerplate only. Ignore this file. + +// Package v1beta1 contains API Schema definitions for the pmem-csi v1beta1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=pmem-csi.intel.com +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "pmem-csi.intel.com", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} +) diff --git a/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go b/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..41ba78eaa1 --- /dev/null +++ b/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,105 @@ +// +build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Deployment) DeepCopyInto(out *Deployment) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Deployment. +func (in *Deployment) DeepCopy() *Deployment { + if in == nil { + return nil + } + out := new(Deployment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Deployment) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentList) DeepCopyInto(out *DeploymentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Deployment, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentList. +func (in *DeploymentList) DeepCopy() *DeploymentList { + if in == nil { + return nil + } + out := new(DeploymentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DeploymentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { + *out = *in + in.DeploymentSpec.DeepCopyInto(&out.DeploymentSpec) + if in.ProvisionerResources != nil { + in, out := &in.ProvisionerResources, &out.ProvisionerResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.NodeRegistrarResources != nil { + in, out := &in.NodeRegistrarResources, &out.NodeRegistrarResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.NodeDriverResources != nil { + in, out := &in.NodeDriverResources, &out.NodeDriverResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.ControllerDriverResources != nil { + in, out := &in.ControllerDriverResources, &out.ControllerDriverResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec. +func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { + if in == nil { + return nil + } + out := new(DeploymentSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/deployments/load.go b/pkg/deployments/load.go index 786154ce01..3b320a1f8e 100644 --- a/pkg/deployments/load.go +++ b/pkg/deployments/load.go @@ -15,7 +15,7 @@ import ( "github.com/intel/pmem-csi/deploy" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" "github.com/intel/pmem-csi/pkg/version" corev1 "k8s.io/api/core/v1" @@ -86,12 +86,20 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode base.DeviceM case "CSIDriver": obj.SetName(deployment.GetName()) case "StatefulSet": - if err := patchPodTemplate(obj, deployment, deployment.Spec.ControllerResources); err != nil { + resources := map[string]*corev1.ResourceRequirements{ + "pmem-driver": deployment.Spec.ControllerDriverResources, + "external-provisioner": deployment.Spec.ProvisionerResources, + } + if err := patchPodTemplate(obj, deployment, resources); err != nil { // TODO: avoid panic panic(fmt.Errorf("set controller resources: %v", err)) } case "DaemonSet": - if err := patchPodTemplate(obj, deployment, deployment.Spec.NodeResources); err != nil { + resources := map[string]*corev1.ResourceRequirements{ + "pmem-driver": deployment.Spec.NodeDriverResources, + "driver-registrar": deployment.Spec.NodeRegistrarResources, + } + if err := patchPodTemplate(obj, deployment, resources); err != nil { // TODO: avoid panic panic(fmt.Errorf("set node resources: %v", err)) } @@ -111,7 +119,7 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode base.DeviceM return loadObjects(kubernetes, deviceMode, patchYAML, patchUnstructured) } -func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, resources *corev1.ResourceRequirements) error { +func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, resources map[string]*corev1.ResourceRequirements) error { if resources == nil { return nil } @@ -136,22 +144,27 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, } // Convert through JSON. - resourcesObj := map[string]interface{}{} - data, err := json.Marshal(resources) - if err != nil { - return err - } - if err := json.Unmarshal(data, &resourcesObj); err != nil { - return err + resourceObj := func(r *corev1.ResourceRequirements) (map[string]interface{}, error) { + obj := map[string]interface{}{} + data, err := json.Marshal(r) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &obj); err != nil { + return nil, err + } + return obj, nil } containers := spec["containers"].([]interface{}) for _, container := range containers { - // Mimick the current operator behavior - // (https://github.com/intel/pmem-csi/issues/616) and apply - // the resource requirements to all containers. container := container.(map[string]interface{}) - container["resources"] = resourcesObj + containerName := container["name"].(string) + obj, err := resourceObj(resources[containerName]) + if err != nil { + return err + } + container["resources"] = obj // Override driver name in env var. env := container["env"] @@ -167,7 +180,7 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, } var image string - switch container["name"].(string) { + switch containerName { case "external-provisioner": image = deployment.Spec.ProvisionerImage case "driver-registrar": diff --git a/pkg/deployments/load_test.go b/pkg/deployments/load_test.go index a6419cbbd9..5593e2c354 100644 --- a/pkg/deployments/load_test.go +++ b/pkg/deployments/load_test.go @@ -15,7 +15,7 @@ import ( "github.com/intel/pmem-csi/deploy" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" "github.com/intel/pmem-csi/pkg/deployments" "github.com/intel/pmem-csi/pkg/version" ) diff --git a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go index 2d74d3d98f..346a5e3dc6 100644 --- a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go +++ b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go @@ -17,7 +17,7 @@ import ( "runtime" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" grpcserver "github.com/intel/pmem-csi/pkg/grpc-server" pmemtls "github.com/intel/pmem-csi/pkg/pmem-csi-operator/pmem-tls" pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc" @@ -1313,7 +1313,7 @@ func (d *PmemCSIDriver) getControllerContainer() corev1.Container { }, }, Ports: d.getMetricsPorts(controllerMetricsPort), - Resources: *d.Spec.ControllerResources, + Resources: *d.Spec.ControllerDriverResources, TerminationMessagePath: "/tmp/termination-log", SecurityContext: &corev1.SecurityContext{ ReadOnlyRootFilesystem: &true, @@ -1396,7 +1396,7 @@ func (d *PmemCSIDriver) getNodeDriverContainer() corev1.Container { }, }, Ports: d.getMetricsPorts(nodeMetricsPort), - Resources: *d.Spec.NodeResources, + Resources: *d.Spec.NodeDriverResources, SecurityContext: &corev1.SecurityContext{ Privileged: &true, // Node driver must run as root user @@ -1430,7 +1430,7 @@ func (d *PmemCSIDriver) getProvisionerContainer() corev1.Container { }, }, Ports: d.getMetricsPorts(provisionerMetricsPort), - Resources: *d.Spec.ControllerResources, + Resources: *d.Spec.ProvisionerResources, SecurityContext: &corev1.SecurityContext{ ReadOnlyRootFilesystem: &true, }, @@ -1467,7 +1467,7 @@ func (d *PmemCSIDriver) getNodeRegistrarContainer() corev1.Container { Value: d.GetName(), }, }, - Resources: *d.Spec.NodeResources, + Resources: *d.Spec.NodeRegistrarResources, } } diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go index a38212ebd8..445ea57beb 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller.go @@ -15,7 +15,7 @@ import ( "time" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - pmemcsiv1alpha1 "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + pmemcsiv1beta1 "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" "github.com/intel/pmem-csi/pkg/k8sutil" pmemcontroller "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller" "github.com/intel/pmem-csi/pkg/version" @@ -110,7 +110,7 @@ func add(mgr manager.Manager, r *ReconcileDeployment) error { } // Watch for changes to primary resource Deployment - if err := c.Watch(&source.Kind{Type: &pmemcsiv1alpha1.Deployment{}}, &handler.EnqueueRequestForObject{}, p); err != nil { + if err := c.Watch(&source.Kind{Type: &pmemcsiv1beta1.Deployment{}}, &handler.EnqueueRequestForObject{}, p); err != nil { klog.Errorf("Deployment.Add: watch error: %v", err) return err } @@ -167,7 +167,7 @@ func add(mgr manager.Manager, r *ReconcileDeployment) error { for _, resource := range currentObjects { if err := c.Watch(&source.Kind{Type: resource}, &handler.EnqueueRequestForOwner{ IsController: true, - OwnerType: &pmemcsiv1alpha1.Deployment{}, + OwnerType: &pmemcsiv1beta1.Deployment{}, }, sop); err != nil { klog.Errorf("Deployment.Add: watch error: %v", err) return err @@ -181,7 +181,7 @@ func add(mgr manager.Manager, r *ReconcileDeployment) error { var _ reconcile.Reconciler = &ReconcileDeployment{} // ReconcileHook function to be invoked on reconciling a deployment. -type ReconcileHook *func(d *pmemcsiv1alpha1.Deployment) +type ReconcileHook *func(d *pmemcsiv1beta1.Deployment) // ReconcileDeployment reconciles a Deployment object type ReconcileDeployment struct { @@ -193,7 +193,7 @@ type ReconcileDeployment struct { // container image used for deploying the operator containerImage string // known deployments - deployments map[string]*pmemcsiv1alpha1.Deployment + deployments map[string]*pmemcsiv1beta1.Deployment // deploymentsMutex protects concurrent access to deployments deploymentsMutex sync.Mutex // reconcileMutex synchronizes concurrent reconcile calls @@ -237,7 +237,7 @@ func NewReconcileDeployment(client client.Client, opts pmemcontroller.Controller k8sVersion: opts.K8sVersion, namespace: opts.Namespace, containerImage: opts.DriverImage, - deployments: map[string]*pmemcsiv1alpha1.Deployment{}, + deployments: map[string]*pmemcsiv1beta1.Deployment{}, reconcileHooks: map[ReconcileHook]struct{}{}, }, nil } @@ -257,7 +257,7 @@ func (r *ReconcileDeployment) Reconcile(request reconcile.Request) (reconcile.Re requeueDelayOnError := 2 * time.Minute // Fetch the Deployment instance - deployment := &pmemcsiv1alpha1.Deployment{} + deployment := &pmemcsiv1beta1.Deployment{} err = r.client.Get(context.TODO(), request.NamespacedName, deployment) if err != nil { klog.V(3).Infof("Failed to retrieve object '%s' to reconcile", request.Name) @@ -364,7 +364,7 @@ func (r *ReconcileDeployment) Delete(obj runtime.Object) error { } // PatchDeploymentStatus patches the give given deployment CR status -func (r *ReconcileDeployment) PatchDeploymentStatus(dep *pmemcsiv1alpha1.Deployment, patch client.Patch) error { +func (r *ReconcileDeployment) PatchDeploymentStatus(dep *pmemcsiv1beta1.Deployment, patch client.Patch) error { dep.Status.LastUpdated = metav1.Now() // Passing a copy of CR to patch as the fake client used in tests // will write back the changes to both status and spec. @@ -376,13 +376,13 @@ func (r *ReconcileDeployment) PatchDeploymentStatus(dep *pmemcsiv1alpha1.Deploym return nil } -func (r *ReconcileDeployment) saveDeployment(d *pmemcsiv1alpha1.Deployment) { +func (r *ReconcileDeployment) saveDeployment(d *pmemcsiv1beta1.Deployment) { r.deploymentsMutex.Lock() defer r.deploymentsMutex.Unlock() r.deployments[d.Name] = d } -func (r *ReconcileDeployment) getDeployment(name string) *pmemcsiv1alpha1.Deployment { +func (r *ReconcileDeployment) getDeployment(name string) *pmemcsiv1beta1.Deployment { r.deploymentsMutex.Lock() defer r.deploymentsMutex.Unlock() return r.deployments[name] diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go index b42241f576..e3d21064b9 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go @@ -17,7 +17,7 @@ import ( "github.com/intel/pmem-csi/deploy" "github.com/intel/pmem-csi/pkg/apis" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" pmemcontroller "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller" "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller/deployment" "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller/deployment/testcases" @@ -49,6 +49,8 @@ type pmemDeployment struct { image, pullPolicy, provisionerImage, registrarImage string controllerCPU, controllerMemory string nodeCPU, nodeMemory string + provisionerCPU, provisionerMemory string + nodeRegistarCPU, nodeRegistrarMemory string caCert, regCert, regKey, ncCert, ncKey []byte kubeletDir string } @@ -57,7 +59,7 @@ func getDeployment(d *pmemDeployment) *api.Deployment { dep := &api.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", - APIVersion: "pmem-csi.intel.com/v1alpha1", + APIVersion: "pmem-csi.intel.com/v1beta1", }, ObjectMeta: metav1.ObjectMeta{ Name: d.name, @@ -77,7 +79,7 @@ func getDeployment(d *pmemDeployment) *api.Deployment { spec.ProvisionerImage = d.provisionerImage spec.NodeRegistrarImage = d.registrarImage if d.controllerCPU != "" || d.controllerMemory != "" { - spec.ControllerResources = &corev1.ResourceRequirements{ + spec.ControllerDriverResources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(d.controllerCPU), corev1.ResourceMemory: resource.MustParse(d.controllerMemory), @@ -85,13 +87,29 @@ func getDeployment(d *pmemDeployment) *api.Deployment { } } if d.nodeCPU != "" || d.nodeMemory != "" { - spec.NodeResources = &corev1.ResourceRequirements{ + spec.NodeDriverResources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(d.nodeCPU), corev1.ResourceMemory: resource.MustParse(d.nodeMemory), }, } } + if d.provisionerCPU != "" || d.provisionerMemory != "" { + spec.ProvisionerResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(d.provisionerCPU), + corev1.ResourceMemory: resource.MustParse(d.provisionerMemory), + }, + } + } + if d.nodeRegistarCPU != "" || d.nodeRegistrarMemory != "" { + spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(d.nodeRegistarCPU), + corev1.ResourceMemory: resource.MustParse(d.nodeRegistrarMemory), + }, + } + } spec.CACert = d.caCert spec.RegistryCert = d.regCert spec.RegistryPrivateKey = d.regKey diff --git a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go index 9ab08cd5ee..d7f626edfb 100644 --- a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go +++ b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go @@ -12,7 +12,7 @@ import ( "fmt" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" pmemtls "github.com/intel/pmem-csi/pkg/pmem-csi-operator/pmem-tls" corev1 "k8s.io/api/core/v1" @@ -42,22 +42,38 @@ func UpdateTests() []UpdateTest { "nodeRegistrarImage": func(d *api.Deployment) { d.Spec.NodeRegistrarImage = "still-no-such-registrar-image" }, - "controllerResources": func(d *api.Deployment) { - d.Spec.ControllerResources = &corev1.ResourceRequirements{ + "controllerDriverResources": func(d *api.Deployment) { + d.Spec.ControllerDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("201m"), corev1.ResourceMemory: resource.MustParse("101Mi"), }, } }, - "nodeResources": func(d *api.Deployment) { - d.Spec.NodeResources = &corev1.ResourceRequirements{ + "nodeDriverResources": func(d *api.Deployment) { + d.Spec.NodeDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("501m"), corev1.ResourceMemory: resource.MustParse("501Mi"), }, } }, + "provisionerResources": func(d *api.Deployment) { + d.Spec.ProvisionerResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("101m"), + corev1.ResourceMemory: resource.MustParse("101Mi"), + }, + } + }, + "nodeRegistrarResources": func(d *api.Deployment) { + d.Spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("301m"), + corev1.ResourceMemory: resource.MustParse("301Mi"), + }, + } + }, "TLS": func(d *api.Deployment) { SetTLSOrDie(&d.Spec) }, @@ -103,18 +119,46 @@ func UpdateTests() []UpdateTest { "a": "b", }, }, - ControllerResources: &corev1.ResourceRequirements{ + ControllerDriverResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("200m"), corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - NodeResources: &corev1.ResourceRequirements{ + NodeDriverResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("500Mi"), }, }, + ProvisionerResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + }, + NodeRegistrarResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("30m"), + corev1.ResourceMemory: resource.MustParse("30Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + }, }, } SetTLSOrDie(&full.Spec) diff --git a/pkg/pmem-csi-operator/main.go b/pkg/pmem-csi-operator/main.go index 038c5ca91d..8a6aa0735f 100644 --- a/pkg/pmem-csi-operator/main.go +++ b/pkg/pmem-csi-operator/main.go @@ -16,7 +16,7 @@ import ( "k8s.io/klog/v2" "github.com/intel/pmem-csi/pkg/apis" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" "github.com/intel/pmem-csi/pkg/k8sutil" "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller" diff --git a/test/e2e/deploy/deploy.go b/test/e2e/deploy/deploy.go index 976e917fc5..4c73e2e1a7 100644 --- a/test/e2e/deploy/deploy.go +++ b/test/e2e/deploy/deploy.go @@ -31,7 +31,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework/skipper" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" pmemexec "github.com/intel/pmem-csi/pkg/exec" "github.com/onsi/ginkgo" diff --git a/test/e2e/deploy/operator.go b/test/e2e/deploy/operator.go index 1c5bb081a5..c37878e00b 100644 --- a/test/e2e/deploy/operator.go +++ b/test/e2e/deploy/operator.go @@ -17,7 +17,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" "github.com/intel/pmem-csi/pkg/apis" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" "github.com/onsi/gomega" ) diff --git a/test/e2e/operator/deployment_api.go b/test/e2e/operator/deployment_api.go index 4c8dac408d..4e817f8f6b 100644 --- a/test/e2e/operator/deployment_api.go +++ b/test/e2e/operator/deployment_api.go @@ -14,7 +14,7 @@ import ( "time" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" "github.com/intel/pmem-csi/pkg/exec" "github.com/intel/pmem-csi/pkg/k8sutil" "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller/deployment/testcases" @@ -153,18 +153,30 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { PullPolicy: corev1.PullNever, Image: dummyImage, }, - ControllerResources: &corev1.ResourceRequirements{ + ControllerDriverResources: &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("200m"), corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - NodeResources: &corev1.ResourceRequirements{ + NodeDriverResources: &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("500Mi"), }, }, + ProvisionerResources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("210m"), + corev1.ResourceMemory: resource.MustParse("110Mi"), + }, + }, + NodeRegistrarResources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, }, }, } @@ -242,18 +254,30 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { spec.Image = "test-driver-image" spec.PullPolicy = corev1.PullNever spec.ProvisionerImage = "test-provisioner" - spec.ControllerResources = &corev1.ResourceRequirements{ + spec.ControllerDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("200m"), corev1.ResourceMemory: resource.MustParse("100Mi"), }, } - spec.NodeResources = &corev1.ResourceRequirements{ + spec.NodeDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("500Mi"), }, } + spec.ProvisionerResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + } testcases.SetTLSOrDie(spec) deployment = deploy.UpdateDeploymentCR(f, deployment) diff --git a/test/e2e/operator/validate/validate.go b/test/e2e/operator/validate/validate.go index 09b21a52f0..65c2e698bc 100644 --- a/test/e2e/operator/validate/validate.go +++ b/test/e2e/operator/validate/validate.go @@ -22,7 +22,7 @@ import ( "time" "github.com/intel/pmem-csi/pkg/apis/pmemcsi/base" - api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1alpha1" + api "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" "github.com/intel/pmem-csi/pkg/deployments" operatordeployment "github.com/intel/pmem-csi/pkg/pmem-csi-operator/controller/deployment" "github.com/intel/pmem-csi/pkg/version"