feat: stateful set persistent volume claim retention policy (#5946)

* feat: stateful set persistent volume claim retention policy

* refactor: naming convention and test case update

* feat: adding statefulSetPersistentVolumeClaimRetentionPolicy crd yaml

k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml

@@ -32463,6 +32463,25 @@ spec:
               serverConfig:
                 description: Server definition
                 type: string
+              statefulSetPersistentVolumeClaimRetentionPolicy:
+                description: StatefulSetPersistentVolumeClaimRetentionPolicy policy
+                  for stateful set pvc
+                properties:
+                  whenDeleted:
+                    description: WhenDeleted specifies what happens to PVCs created
+                      from StatefulSet VolumeClaimTemplates when the StatefulSet is
+                      deleted. The default policy of `Retain` causes PVCs to not be
+                      affected by StatefulSet deletion. The `Delete` policy causes
+                      those PVCs to be deleted.
+                    type: string
+                  whenScaled:
+                    description: WhenScaled specifies what happens to PVCs created
+                      from StatefulSet VolumeClaimTemplates when the StatefulSet is
+                      scaled down. The default policy of `Retain` causes PVCs to not
+                      be affected by a scaledown. The `Delete` policy causes the associated
+                      PVCs for any excess pods above the replica count to be deleted.
+                    type: string
+                type: object
             required:
             - serverConfig
             type: object

k8s/yaml/crds.yaml

@@ -32471,6 +32471,25 @@ spec:
               serverConfig:
                 description: Server definition
                 type: string
+              statefulSetPersistentVolumeClaimRetentionPolicy:
+                description: StatefulSetPersistentVolumeClaimRetentionPolicy policy
+                  for stateful set pvc
+                properties:
+                  whenDeleted:
+                    description: WhenDeleted specifies what happens to PVCs created
+                      from StatefulSet VolumeClaimTemplates when the StatefulSet is
+                      deleted. The default policy of `Retain` causes PVCs to not be
+                      affected by StatefulSet deletion. The `Delete` policy causes
+                      those PVCs to be deleted.
+                    type: string
+                  whenScaled:
+                    description: WhenScaled specifies what happens to PVCs created
+                      from StatefulSet VolumeClaimTemplates when the StatefulSet is
+                      scaled down. The default policy of `Retain` causes PVCs to not
+                      be affected by a scaledown. The `Delete` policy causes the associated
+                      PVCs for any excess pods above the replica count to be deleted.
+                    type: string
+                type: object
             required:
             - serverConfig
             type: object

operator/apis/mlops/v1alpha1/server_types.go

@@ -10,6 +10,7 @@ the Change License after the Change Date as each is defined in accordance with t
 package v1alpha1
 
 import (
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"knative.dev/pkg/apis"
@@ -37,6 +38,8 @@ type ServerSpec struct {
 	// PodSpec overrides
 	// Slices such as containers would be appended not overridden
 	PodSpec *PodSpec `json:"podSpec,omitempty"`
+	// StatefulSetPersistentVolumeClaimRetentionPolicy policy for stateful set pvc
+	StatefulSetPersistentVolumeClaimRetentionPolicy *appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy `json:"statefulSetPersistentVolumeClaimRetentionPolicy,omitempty"`
 	// Scaling spec
 	ScalingSpec `json:",inline"`
 	// +Optional

operator/config/crd/bases/mlops.seldon.io_servers.yaml

@@ -8850,6 +8850,25 @@ spec:
               serverConfig:
                 description: Server definition
                 type: string
+              statefulSetPersistentVolumeClaimRetentionPolicy:
+                description: StatefulSetPersistentVolumeClaimRetentionPolicy policy
+                  for stateful set pvc
+                properties:
+                  whenDeleted:
+                    description: WhenDeleted specifies what happens to PVCs created
+                      from StatefulSet VolumeClaimTemplates when the StatefulSet is
+                      deleted. The default policy of `Retain` causes PVCs to not be
+                      affected by StatefulSet deletion. The `Delete` policy causes
+                      those PVCs to be deleted.
+                    type: string
+                  whenScaled:
+                    description: WhenScaled specifies what happens to PVCs created
+                      from StatefulSet VolumeClaimTemplates when the StatefulSet is
+                      scaled down. The default policy of `Retain` causes PVCs to not
+                      be affected by a scaledown. The `Delete` policy causes the associated
+                      PVCs for any excess pods above the replica count to be deleted.
+                    type: string
+                type: object
             required:
             - serverConfig
             type: object

operator/controllers/reconcilers/server/server_reconciler.go

@@ -137,6 +137,7 @@ func (s *ServerReconciler) createStatefulSetReconciler(server *mlopsv1alpha1.Ser
 		podSpec,
 		serverConfig.Spec.VolumeClaimTemplates,
 		&server.Spec.ScalingSpec,
+		server.Spec.StatefulSetPersistentVolumeClaimRetentionPolicy,
 		serverConfig.ObjectMeta,
 		annotator)
 	return statefulSetReconciler, nil

operator/controllers/reconcilers/server/server_reconciler_test.go

@@ -72,6 +72,40 @@ func TestServerReconcile(t *testing.T) {
 			expectedSvcNames:        []string{"myserver-0"},
 			expectedStatefulSetName: "myserver",
 		},
+		{
+			name: "Test StatefulSetPersistentVolumeClaimRetentionPolicy",
+			serverConfig: &mlopsv1alpha1.ServerConfig{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      mlserverConfigName,
+					Namespace: constants.SeldonNamespace,
+				},
+				Spec: mlopsv1alpha1.ServerConfigSpec{
+					PodSpec: v1.PodSpec{
+						Containers: []v1.Container{
+							{
+								Name:  "mlserver",
+								Image: "seldonio/mlserver:0.5",
+							},
+						},
+					},
+				},
+			},
+			server: &mlopsv1alpha1.Server{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "myserver",
+					Namespace: constants.SeldonNamespace,
+				},
+				Spec: mlopsv1alpha1.ServerSpec{
+					ServerConfig: mlserverConfigName,
+					StatefulSetPersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
+						WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
+						WhenScaled:  appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
+					},
+				},
+			},
+			expectedSvcNames:        []string{"myserver-0"},
+			expectedStatefulSetName: "myserver",
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

operator/controllers/reconcilers/server/statefulset_reconciler.go

@@ -41,14 +41,15 @@ func NewServerStatefulSetReconciler(
 	podSpec *v1.PodSpec,
 	volumeClaimTemplates []mlopsv1alpha1.PersistentVolumeClaim,
 	scaling *mlopsv1alpha1.ScalingSpec,
+	volumeClaimRetentionPolicy *appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy,
 	serverConfigMeta metav1.ObjectMeta,
 	annotator *patch.Annotator,
 ) *ServerStatefulSetReconciler {
 	labels := utils.MergeMaps(meta.Labels, serverConfigMeta.Labels)
 	annotations := utils.MergeMaps(meta.Annotations, serverConfigMeta.Annotations)
 	return &ServerStatefulSetReconciler{
 		ReconcilerConfig: common,
-		StatefulSet:      toStatefulSet(meta, podSpec, volumeClaimTemplates, scaling, labels, annotations),
+		StatefulSet:      toStatefulSet(meta, podSpec, volumeClaimTemplates, scaling, volumeClaimRetentionPolicy, labels, annotations),
 		Annotator:        annotator,
 	}
 }
@@ -61,6 +62,7 @@ func toStatefulSet(meta metav1.ObjectMeta,
 	podSpec *v1.PodSpec,
 	volumeClaimTemplates []mlopsv1alpha1.PersistentVolumeClaim,
 	scaling *mlopsv1alpha1.ScalingSpec,
+	volumeClaimRetentionPolicy *appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy,
 	labels map[string]string,
 	annotations map[string]string) *appsv1.StatefulSet {
 	labels[constants.KubernetesNameLabelKey] = constants.ServerLabelValue
@@ -88,7 +90,8 @@ func toStatefulSet(meta metav1.ObjectMeta,
 				},
 				Spec: *podSpec,
 			},
-			PodManagementPolicy: appsv1.ParallelPodManagement,
+			PodManagementPolicy:                  appsv1.ParallelPodManagement,
+			PersistentVolumeClaimRetentionPolicy: volumeClaimRetentionPolicy,
 		},
 	}
 

operator/controllers/reconcilers/server/statefulset_reconciler_test.go

@@ -41,14 +41,15 @@ func TestStatefulSetReconcile(t *testing.T) {
 	g.Expect(err).To(BeNil())
 
 	type test struct {
-		name                 string
-		metaServer           metav1.ObjectMeta
-		metaServerConfig     metav1.ObjectMeta
-		podSpec              *v1.PodSpec
-		volumeClaimTemplates []mlopsv1alpha1.PersistentVolumeClaim
-		scaling              *mlopsv1alpha1.ScalingSpec
-		existing             *appsv1.StatefulSet
-		expectedReconcileOp  constants.ReconcileOperation
+		name                                            string
+		metaServer                                      metav1.ObjectMeta
+		metaServerConfig                                metav1.ObjectMeta
+		podSpec                                         *v1.PodSpec
+		volumeClaimTemplates                            []mlopsv1alpha1.PersistentVolumeClaim
+		scaling                                         *mlopsv1alpha1.ScalingSpec
+		statefulSetPersistentVolumeClaimRetentionPolicy *appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy
+		existing                                        *appsv1.StatefulSet
+		expectedReconcileOp                             constants.ReconcileOperation
 	}
 
 	getIntPtr := func(i int32) *int32 {
@@ -334,6 +335,7 @@ func TestStatefulSetReconcile(t *testing.T) {
 				test.podSpec,
 				test.volumeClaimTemplates,
 				test.scaling,
+				test.statefulSetPersistentVolumeClaimRetentionPolicy,
 				test.metaServerConfig,
 				annotator)
 			rop, err := r.getReconcileOperation()
@@ -355,14 +357,15 @@ func TestToStatefulSet(t *testing.T) {
 	g := NewGomegaWithT(t)
 
 	type test struct {
-		name                 string
-		meta                 metav1.ObjectMeta
-		podSpec              *v1.PodSpec
-		labels               map[string]string
-		annotations          map[string]string
-		volumeClaimTemplates []mlopsv1alpha1.PersistentVolumeClaim
-		scaling              *mlopsv1alpha1.ScalingSpec
-		statefulSet          *appsv1.StatefulSet
+		name                                            string
+		meta                                            metav1.ObjectMeta
+		podSpec                                         *v1.PodSpec
+		labels                                          map[string]string
+		annotations                                     map[string]string
+		volumeClaimTemplates                            []mlopsv1alpha1.PersistentVolumeClaim
+		statefulSetPersistentVolumeClaimRetentionPolicy *appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy
+		scaling                                         *mlopsv1alpha1.ScalingSpec
+		statefulSet                                     *appsv1.StatefulSet
 	}
 
 	getIntPtr := func(i int32) *int32 {
@@ -459,11 +462,113 @@ func TestToStatefulSet(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "Basic",
+			meta: metav1.ObjectMeta{
+				Name:      "foo",
+				Namespace: "default",
+			},
+			podSpec: &v1.PodSpec{
+				Containers: []v1.Container{
+					{
+						Name:    "c1",
+						Image:   "myimagec1:1",
+						Command: []string{"cmd"},
+					},
+				},
+				NodeName: "node",
+			},
+			labels:      map[string]string{"l1": "l1val"},
+			annotations: map[string]string{"a1": "a1val"},
+			volumeClaimTemplates: []mlopsv1alpha1.PersistentVolumeClaim{
+				{
+					Name: "model-repository",
+					Spec: v1.PersistentVolumeClaimSpec{
+						AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+						Resources: v1.VolumeResourceRequirements{
+							Requests: v1.ResourceList{
+								v1.ResourceStorage: oneG,
+							},
+						},
+					},
+				},
+			},
+			scaling: &mlopsv1alpha1.ScalingSpec{
+				Replicas: getIntPtr(2),
+			},
+			statefulSetPersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
+				WhenDeleted: "Delete",
+				WhenScaled:  "Delete",
+			},
+			statefulSet: &appsv1.StatefulSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "foo",
+					Namespace: "default",
+					Labels: map[string]string{
+						constants.KubernetesNameLabelKey: constants.ServerLabelValue,
+						"l1":                             "l1val"},
+					Annotations: map[string]string{"a1": "a1val"},
+				},
+				Spec: appsv1.StatefulSetSpec{
+					ServiceName: "foo",
+					Replicas:    getIntPtr(2),
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{constants.ServerLabelNameKey: "foo"},
+					},
+					Template: v1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels: map[string]string{constants.ServerLabelNameKey: "foo",
+								constants.KubernetesNameLabelKey: constants.ServerLabelValue,
+								"l1":                             "l1val"},
+							Annotations: map[string]string{"a1": "a1val"},
+							Name:        "foo",
+							Namespace:   "default",
+						},
+						Spec: v1.PodSpec{
+							Containers: []v1.Container{
+								{
+									Name:    "c1",
+									Image:   "myimagec1:1",
+									Command: []string{"cmd"},
+								},
+							},
+							NodeName: "node",
+						},
+					},
+					PodManagementPolicy: appsv1.ParallelPodManagement,
+					VolumeClaimTemplates: []v1.PersistentVolumeClaim{
+						{
+							ObjectMeta: metav1.ObjectMeta{
+								Name: "model-repository",
+							},
+							Spec: v1.PersistentVolumeClaimSpec{
+								AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+								Resources: v1.VolumeResourceRequirements{
+									Requests: v1.ResourceList{
+										v1.ResourceStorage: oneG,
+									},
+								},
+							},
+						},
+					},
+					PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
+						WhenDeleted: "Delete",
+						WhenScaled:  "Delete",
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			statefulSet := toStatefulSet(test.meta, test.podSpec, test.volumeClaimTemplates, test.scaling, test.labels, test.annotations)
+			statefulSet := toStatefulSet(test.meta,
+				test.podSpec,
+				test.volumeClaimTemplates,
+				test.scaling,
+				test.statefulSetPersistentVolumeClaimRetentionPolicy,
+				test.labels,
+				test.annotations)
 			g.Expect(equality.Semantic.DeepEqual(statefulSet, test.statefulSet)).To(BeTrue())
 		})
 	}
@@ -523,6 +628,7 @@ func TestLabelsAnnotations(t *testing.T) {
 				&v1.PodSpec{},
 				[]mlopsv1alpha1.PersistentVolumeClaim{},
 				&mlopsv1alpha1.ScalingSpec{},
+				&appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{},
 				test.metaServerConfig,
 				annotator)
 			for k, v := range test.expectedLabels {