diff --git a/api/v1alpha1/const.go b/api/v1alpha1/const.go index 6f6e9f6a..a3e13e6f 100644 --- a/api/v1alpha1/const.go +++ b/api/v1alpha1/const.go @@ -61,7 +61,8 @@ const ( AnnotationNodeHost = "ydb.tech/node-host" AnnotationNodeDomain = "ydb.tech/node-domain" - AnnotationValueTrue = "true" + RemoteFinalizerKey = "ydb.tech/remote-finalizer" + StorageFinalizerKey = "ydb.tech/storage-finalizer" legacyTenantNameFormat = "/%s/%s" ) diff --git a/internal/annotations/annotations.go b/internal/annotations/annotations.go index 5fa06070..9789663a 100644 --- a/internal/annotations/annotations.go +++ b/internal/annotations/annotations.go @@ -1,15 +1,86 @@ package annotations +import ( + corev1 "k8s.io/api/core/v1" + + "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" +) + const ( - PrimaryResourceStorageAnnotation = "ydb.tech/primary-resource-storage" - PrimaryResourceDatabaseAnnotation = "ydb.tech/primary-resource-database" - RemoteResourceVersionAnnotation = "ydb.tech/remote-resource-version" - ConfigurationChecksum = "ydb.tech/configuration-checksum" - StorageFinalizerKey = "ydb.tech/storage-finalizer" - RemoteFinalizerKey = "ydb.tech/remote-finalizer" - LastAppliedAnnotation = "ydb.tech/last-applied" + PrimaryResourceStorage = "ydb.tech/primary-resource-storage" + PrimaryResourceDatabase = "ydb.tech/primary-resource-database" + RemoteResourceVersion = "ydb.tech/remote-resource-version" + ConfigurationChecksum = "ydb.tech/configuration-checksum" + LastApplied = "ydb.tech/last-applied" +) + +var ( + AcceptedDNSPolicy = []string{ + string(corev1.DNSClusterFirstWithHostNet), + string(corev1.DNSClusterFirst), + string(corev1.DNSDefault), + string(corev1.DNSNone), + } + SupportedAnnotations = map[string]struct{}{ + v1alpha1.AnnotationSkipInitialization: {}, + v1alpha1.AnnotationUpdateStrategyOnDelete: {}, + v1alpha1.AnnotationUpdateDNSPolicy: {}, + v1alpha1.AnnotationDisableLivenessProbe: {}, + v1alpha1.AnnotationDataCenter: {}, + v1alpha1.AnnotationGRPCPublicHost: {}, + v1alpha1.AnnotationNodeHost: {}, + v1alpha1.AnnotationNodeDomain: {}, + } ) +type Annotations map[string]string + +func Common(objAnnotations Annotations) Annotations { + annotations := Annotations{} + + annotations.Merge(getSupportedAnnotations(objAnnotations)) + + return annotations +} + +func (an Annotations) Merge(other map[string]string) map[string]string { + if other == nil { + return an + } + + for k, v := range other { + an[k] = v + } + + return an +} + +func (an Annotations) AsMap() map[string]string { + return an +} + +func (an Annotations) Copy() Annotations { + res := Annotations{} + + for k, v := range an { + res[k] = v + } + + return res +} + +func getSupportedAnnotations(annotations map[string]string) map[string]string { + common := make(map[string]string) + + for key, value := range annotations { + if _, exists := SupportedAnnotations[key]; exists { + common[key] = value + } + } + + return common +} + func CompareLastAppliedAnnotation(map1, map2 map[string]string) bool { value1 := getLastAppliedAnnotation(map1) value2 := getLastAppliedAnnotation(map2) @@ -18,7 +89,7 @@ func CompareLastAppliedAnnotation(map1, map2 map[string]string) bool { func getLastAppliedAnnotation(annotations map[string]string) string { for key, value := range annotations { - if key == LastAppliedAnnotation { + if key == LastApplied { return value } } diff --git a/internal/annotations/annotations_test.go b/internal/annotations/annotations_test.go new file mode 100644 index 00000000..07e47107 --- /dev/null +++ b/internal/annotations/annotations_test.go @@ -0,0 +1,47 @@ +package annotations_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" +) + +func TestLabels(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Label suite") +} + +var _ = Describe("Testing annotations", func() { + It("merges two sets of annotations", func() { + fstLabels := ydbannotations.Annotations{ + "a": "a", + "b": "b", + } + + sndLabels := ydbannotations.Annotations{ + "c": "c", + "d": "d", + } + + Expect(fstLabels.Merge(sndLabels)).To(BeEquivalentTo(map[string]string{ + "a": "a", + "b": "b", + "c": "c", + "d": "d", + })) + }) + + It("sets correct defaults", func() { + Expect(ydbannotations.Common(map[string]string{ + "ydb.tech/skip-initialization": "true", + "ydb.tech/node-host": "ydb-testing.k8s-c.yandex.net", + "ydb.tech/last-applied": "some-body", + "sample-annotation": "test", + })).To(BeEquivalentTo(map[string]string{ + "ydb.tech/skip-initialization": "true", + "ydb.tech/node-host": "ydb-testing.k8s-c.yandex.net", + })) + }) +}) diff --git a/internal/controllers/database/init.go b/internal/controllers/database/init.go index 910f6d6c..d791df2f 100644 --- a/internal/controllers/database/init.go +++ b/internal/controllers/database/init.go @@ -3,6 +3,7 @@ package database import ( "context" "fmt" + "strconv" ydb "github.com/ydb-platform/ydb-go-sdk/v3" corev1 "k8s.io/api/core/v1" @@ -34,15 +35,17 @@ func (r *Reconciler) setInitPipelineStatus( } // This block is special internal logic that skips all Database initialization. - if value, ok := database.Annotations[v1alpha1.AnnotationSkipInitialization]; ok && value == v1alpha1.AnnotationValueTrue { - r.Log.Info("Database initialization disabled (with annotation), proceed with caution") - r.Recorder.Event( - database, - corev1.EventTypeWarning, - "SkippingInit", - "Skipping initialization due to skip annotation present, be careful!", - ) - return r.setInitDatabaseCompleted(ctx, database, "Database initialization not performed because initialization is skipped") + if value, ok := database.Annotations[v1alpha1.AnnotationSkipInitialization]; ok { + if isTrue, _ := strconv.ParseBool(value); isTrue { + r.Log.Info("Database initialization disabled (with annotation), proceed with caution") + r.Recorder.Event( + database, + corev1.EventTypeWarning, + "SkippingInit", + "Skipping initialization due to skip annotation present, be careful!", + ) + return r.setInitDatabaseCompleted(ctx, database, "Database initialization not performed because initialization is skipped") + } } if meta.IsStatusConditionTrue(database.Status.Conditions, OldDatabaseInitializedCondition) { diff --git a/internal/controllers/remotedatabasenodeset/controller.go b/internal/controllers/remotedatabasenodeset/controller.go index ae0d662c..dca15d5a 100644 --- a/internal/controllers/remotedatabasenodeset/controller.go +++ b/internal/controllers/remotedatabasenodeset/controller.go @@ -6,7 +6,7 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" + apilabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" @@ -25,7 +25,7 @@ import ( "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck - ydblabels "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" + "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) @@ -72,15 +72,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // to registering our finalizer. - if !controllerutil.ContainsFinalizer(remoteDatabaseNodeSet, ydbannotations.RemoteFinalizerKey) { - controllerutil.AddFinalizer(remoteDatabaseNodeSet, ydbannotations.RemoteFinalizerKey) + if !controllerutil.ContainsFinalizer(remoteDatabaseNodeSet, v1alpha1.RemoteFinalizerKey) { + controllerutil.AddFinalizer(remoteDatabaseNodeSet, v1alpha1.RemoteFinalizerKey) if err := r.RemoteClient.Update(ctx, remoteDatabaseNodeSet); err != nil { return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } } } else { // The object is being deleted - if controllerutil.ContainsFinalizer(remoteDatabaseNodeSet, ydbannotations.RemoteFinalizerKey) { + if controllerutil.ContainsFinalizer(remoteDatabaseNodeSet, v1alpha1.RemoteFinalizerKey) { // our finalizer is present, so lets handle any external dependency if err := r.deleteExternalResources(ctx, remoteDatabaseNodeSet); err != nil { // if fail to delete the external dependency here, return with error @@ -89,7 +89,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } // remove our finalizer from the list and update it. - controllerutil.RemoveFinalizer(remoteDatabaseNodeSet, ydbannotations.RemoteFinalizerKey) + controllerutil.RemoveFinalizer(remoteDatabaseNodeSet, v1alpha1.RemoteFinalizerKey) if err := r.RemoteClient.Update(ctx, remoteDatabaseNodeSet); err != nil { return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } @@ -119,7 +119,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.C requests := make([]reconcile.Request, 0) annotations := mapObj.GetAnnotations() - primaryResourceName, exist := annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] + primaryResourceName, exist := annotations[ydbannotations.PrimaryResourceDatabase] if exist { databaseNodeSets := &v1alpha1.DatabaseNodeSetList{} if err := r.Client.List( @@ -192,10 +192,10 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.C Complete(r) } -func buildLocalSelector() (labels.Selector, error) { - labelRequirements := []labels.Requirement{} - localClusterRequirement, err := labels.NewRequirement( - ydblabels.RemoteClusterKey, +func buildLocalSelector() (apilabels.Selector, error) { + labelRequirements := []apilabels.Requirement{} + localClusterRequirement, err := apilabels.NewRequirement( + labels.RemoteClusterKey, selection.Exists, []string{}, ) @@ -203,13 +203,13 @@ func buildLocalSelector() (labels.Selector, error) { return nil, err } labelRequirements = append(labelRequirements, *localClusterRequirement) - return labels.NewSelector().Add(labelRequirements...), nil + return apilabels.NewSelector().Add(labelRequirements...), nil } -func BuildRemoteSelector(remoteCluster string) (labels.Selector, error) { - labelRequirements := []labels.Requirement{} - remoteClusterRequirement, err := labels.NewRequirement( - ydblabels.RemoteClusterKey, +func BuildRemoteSelector(remoteCluster string) (apilabels.Selector, error) { + labelRequirements := []apilabels.Requirement{} + remoteClusterRequirement, err := apilabels.NewRequirement( + labels.RemoteClusterKey, selection.Equals, []string{remoteCluster}, ) @@ -217,7 +217,7 @@ func BuildRemoteSelector(remoteCluster string) (labels.Selector, error) { return nil, err } labelRequirements = append(labelRequirements, *remoteClusterRequirement) - return labels.NewSelector().Add(labelRequirements...), nil + return apilabels.NewSelector().Add(labelRequirements...), nil } func (r *Reconciler) deleteExternalResources( diff --git a/internal/controllers/remotedatabasenodeset/controller_test.go b/internal/controllers/remotedatabasenodeset/controller_test.go index 25252795..fb96cd0c 100644 --- a/internal/controllers/remotedatabasenodeset/controller_test.go +++ b/internal/controllers/remotedatabasenodeset/controller_test.go @@ -529,25 +529,25 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { return err } - primaryResourceStorage, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceStorageAnnotation] + primaryResourceStorage, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceStorage] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceStorageAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceStorage, remoteSecret.Name) } if primaryResourceStorage != foundRemoteStorageNodeSet.Spec.StorageRef.Name { return fmt.Errorf("primaryResourceName %s does not equal storageRef name %s", primaryResourceStorage, foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name) } - primaryResourceDatabase, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] + primaryResourceDatabase, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceDatabase] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceDatabaseAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceDatabase, remoteSecret.Name) } if primaryResourceDatabase != foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name { return fmt.Errorf("primaryResourceName %s does not equal databaseRef name %s", primaryResourceDatabase, foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name) } - remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersionAnnotation] + remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersion] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersionAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersion, remoteSecret.Name) } if localSecret.GetResourceVersion() != remoteRV { return fmt.Errorf("localRV %s does not equal remoteRV %s", localSecret.GetResourceVersion(), remoteRV) @@ -620,22 +620,22 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { return err } - _, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceStorageAnnotation] + _, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceStorage] if exist { - return fmt.Errorf("annotation %s still exist on remoteSecret %s", ydbannotations.PrimaryResourceStorageAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s still exist on remoteSecret %s", ydbannotations.PrimaryResourceStorage, remoteSecret.Name) } - primaryResourceDatabase, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] + primaryResourceDatabase, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceDatabase] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceDatabaseAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceDatabase, remoteSecret.Name) } if primaryResourceDatabase != foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name { return fmt.Errorf("primaryResourceName %s does not equal databaseRef name %s", primaryResourceDatabase, foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name) } - remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersionAnnotation] + remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersion] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersionAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersion, remoteSecret.Name) } if localSecret.GetResourceVersion() != remoteRV { return fmt.Errorf("localRV %s does not equal remoteRV %s", localSecret.GetResourceVersion(), remoteRV) @@ -686,17 +686,17 @@ var _ = Describe("RemoteDatabaseNodeSet controller tests", func() { return err } - primaryResourceDatabase, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] + primaryResourceDatabase, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceDatabase] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceDatabaseAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceDatabase, remoteSecret.Name) } if primaryResourceDatabase != foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name { return fmt.Errorf("primaryResourceName %s does not equal databaseRef name %s", primaryResourceDatabase, foundRemoteDatabaseNodeSet.Spec.DatabaseRef.Name) } - remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersionAnnotation] + remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersion] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersionAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersion, remoteSecret.Name) } if localSecret.GetResourceVersion() != remoteRV { return fmt.Errorf("localRV %s does not equal remoteRV %s", localSecret.GetResourceVersion(), remoteRV) diff --git a/internal/controllers/remotedatabasenodeset/remote_objects.go b/internal/controllers/remotedatabasenodeset/remote_objects.go index d600e93d..2ae3c8d8 100644 --- a/internal/controllers/remotedatabasenodeset/remote_objects.go +++ b/internal/controllers/remotedatabasenodeset/remote_objects.go @@ -278,7 +278,7 @@ func (r *Reconciler) removeUnusedRemoteObjects( // Remove annotation if no one another DatabaseNodeSet if !existInDatabase { // Try to update existing object in local cluster by rawPatch - patch := []byte(fmt.Sprintf(`{"metadata": {"annotations": {"%s": null}}}`, ydbannotations.PrimaryResourceStorageAnnotation)) + patch := []byte(fmt.Sprintf(`{"metadata": {"annotations": {"%s": null}}}`, ydbannotations.PrimaryResourceStorage)) updateErr := r.Client.Patch(ctx, localObj, client.RawPatch(types.StrategicMergePatchType, patch)) if updateErr != nil { r.Recorder.Event( @@ -298,7 +298,7 @@ func (r *Reconciler) removeUnusedRemoteObjects( } // Delete resource if annotation `ydb.tech/primary-resource-storage` does not exist - _, existInStorage := localObj.GetAnnotations()[ydbannotations.PrimaryResourceStorageAnnotation] + _, existInStorage := localObj.GetAnnotations()[ydbannotations.PrimaryResourceStorage] if !existInStorage { // Try to delete unused resource from local cluster deleteErr := r.Client.Delete(ctx, localObj) diff --git a/internal/controllers/remotestoragenodeset/controller.go b/internal/controllers/remotestoragenodeset/controller.go index 91ce95ea..3bafd749 100644 --- a/internal/controllers/remotestoragenodeset/controller.go +++ b/internal/controllers/remotestoragenodeset/controller.go @@ -72,15 +72,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // to registering our finalizer. - if !controllerutil.ContainsFinalizer(remoteStorageNodeSet, ydbannotations.RemoteFinalizerKey) { - controllerutil.AddFinalizer(remoteStorageNodeSet, ydbannotations.RemoteFinalizerKey) + if !controllerutil.ContainsFinalizer(remoteStorageNodeSet, v1alpha1.RemoteFinalizerKey) { + controllerutil.AddFinalizer(remoteStorageNodeSet, v1alpha1.RemoteFinalizerKey) if err := r.RemoteClient.Update(ctx, remoteStorageNodeSet); err != nil { return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } } } else { // The object is being deleted - if controllerutil.ContainsFinalizer(remoteStorageNodeSet, ydbannotations.RemoteFinalizerKey) { + if controllerutil.ContainsFinalizer(remoteStorageNodeSet, v1alpha1.RemoteFinalizerKey) { // our finalizer is present, so lets handle any external dependency if err := r.deleteExternalResources(ctx, remoteStorageNodeSet); err != nil { // if fail to delete the external dependency here, return with error @@ -89,7 +89,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } // remove our finalizer from the list and update it. - controllerutil.RemoveFinalizer(remoteStorageNodeSet, ydbannotations.RemoteFinalizerKey) + controllerutil.RemoveFinalizer(remoteStorageNodeSet, v1alpha1.RemoteFinalizerKey) if err := r.RemoteClient.Update(ctx, remoteStorageNodeSet); err != nil { return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } @@ -119,7 +119,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, remoteCluster *cluster.C requests := make([]reconcile.Request, 0) annotations := mapObj.GetAnnotations() - primaryResourceName, exist := annotations[ydbannotations.PrimaryResourceStorageAnnotation] + primaryResourceName, exist := annotations[ydbannotations.PrimaryResourceStorage] if exist { storageNodeSets := &v1alpha1.StorageNodeSetList{} if err := r.Client.List( diff --git a/internal/controllers/remotestoragenodeset/controller_test.go b/internal/controllers/remotestoragenodeset/controller_test.go index 0c5666c6..c801943e 100644 --- a/internal/controllers/remotestoragenodeset/controller_test.go +++ b/internal/controllers/remotestoragenodeset/controller_test.go @@ -25,7 +25,7 @@ import ( "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" testobjects "github.com/ydb-platform/ydb-kubernetes-operator/e2e/tests/test-objects" - ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + annotations ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/remotestoragenodeset" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage" @@ -400,17 +400,17 @@ var _ = Describe("RemoteStorageNodeSet controller tests", func() { return err } - primaryResourceName, exist := remoteSecret.Annotations[ydbannotations.PrimaryResourceStorageAnnotation] + primaryResourceName, exist := remoteSecret.Annotations[annotations.PrimaryResourceStorage] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.PrimaryResourceStorageAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", annotations.PrimaryResourceStorage, remoteSecret.Name) } if primaryResourceName != foundRemoteStorageNodeSet.Spec.StorageRef.Name { return fmt.Errorf("primaryResourceName %s does not equal storageRef name %s", primaryResourceName, foundRemoteStorageNodeSet.Spec.StorageRef.Name) } - remoteRV, exist := remoteSecret.Annotations[ydbannotations.RemoteResourceVersionAnnotation] + remoteRV, exist := remoteSecret.Annotations[annotations.RemoteResourceVersion] if !exist { - return fmt.Errorf("annotation %s does not exist on remoteSecret %s", ydbannotations.RemoteResourceVersionAnnotation, remoteSecret.Name) + return fmt.Errorf("annotation %s does not exist on remoteSecret %s", annotations.RemoteResourceVersion, remoteSecret.Name) } if localSecret.GetResourceVersion() != remoteRV { return fmt.Errorf("localRV %s does not equal remoteRV %s", localSecret.GetResourceVersion(), remoteRV) diff --git a/internal/controllers/remotestoragenodeset/remote_objects.go b/internal/controllers/remotestoragenodeset/remote_objects.go index a54f6a39..eec14fe4 100644 --- a/internal/controllers/remotestoragenodeset/remote_objects.go +++ b/internal/controllers/remotestoragenodeset/remote_objects.go @@ -277,7 +277,7 @@ func (r *Reconciler) removeUnusedRemoteObjects( // Remove annotation if no one another StorageNodeSet if !existInStorage { - patch := []byte(fmt.Sprintf(`{"metadata": {"annotations": {"%s": null}}}`, ydbannotations.PrimaryResourceStorageAnnotation)) + patch := []byte(fmt.Sprintf(`{"metadata": {"annotations": {"%s": null}}}`, ydbannotations.PrimaryResourceStorage)) updateErr := r.Client.Patch(ctx, localObj, client.RawPatch(types.StrategicMergePatchType, patch)) if updateErr != nil { r.Recorder.Event( @@ -297,7 +297,7 @@ func (r *Reconciler) removeUnusedRemoteObjects( } // Delete resource if annotation `ydb.tech/primary-resource-database` does not exist - _, existInDatabase := localObj.GetAnnotations()[ydbannotations.PrimaryResourceDatabaseAnnotation] + _, existInDatabase := localObj.GetAnnotations()[ydbannotations.PrimaryResourceDatabase] if !existInDatabase { // Try to delete unused resource from local cluster deleteErr := r.Client.Delete(ctx, localObj) diff --git a/internal/controllers/storage/controller.go b/internal/controllers/storage/controller.go index 34239316..98a2c883 100644 --- a/internal/controllers/storage/controller.go +++ b/internal/controllers/storage/controller.go @@ -26,7 +26,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" . "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" ) @@ -88,15 +87,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // to registering our finalizer. - if !controllerutil.ContainsFinalizer(resource, ydbannotations.StorageFinalizerKey) { - controllerutil.AddFinalizer(resource, ydbannotations.StorageFinalizerKey) + if !controllerutil.ContainsFinalizer(resource, v1alpha1.StorageFinalizerKey) { + controllerutil.AddFinalizer(resource, v1alpha1.StorageFinalizerKey) if err := r.Client.Update(ctx, resource); err != nil { return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } } } else { // The object is being deleted - if controllerutil.ContainsFinalizer(resource, ydbannotations.StorageFinalizerKey) { + if controllerutil.ContainsFinalizer(resource, v1alpha1.StorageFinalizerKey) { // our finalizer is present, so lets handle any external dependency if err := r.checkExistingDatabases(ctx, resource); err != nil { // if fail to check dependency existence, return with error @@ -105,7 +104,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } // remove our finalizer from the list and update it. - controllerutil.RemoveFinalizer(resource, ydbannotations.StorageFinalizerKey) + controllerutil.RemoveFinalizer(resource, v1alpha1.StorageFinalizerKey) if err := r.Client.Update(ctx, resource); err != nil { return ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err } diff --git a/internal/controllers/storage/controller_test.go b/internal/controllers/storage/controller_test.go index ded230bb..53dd3b57 100644 --- a/internal/controllers/storage/controller_test.go +++ b/internal/controllers/storage/controller_test.go @@ -18,7 +18,7 @@ import ( "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" testobjects "github.com/ydb-platform/ydb-kubernetes-operator/e2e/tests/test-objects" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/resources" @@ -123,7 +123,7 @@ var _ = Describe("Storage controller medium tests", func() { }, &foundStorage)).Should(Succeed()) foundConfigurationChecksumAnnotation := false - if podAnnotations[annotations.ConfigurationChecksum] == resources.SHAChecksum(foundStorage.Spec.Configuration) { + if podAnnotations[ydbannotations.ConfigurationChecksum] == resources.GetSHA256Checksum(foundStorage.Spec.Configuration) { foundConfigurationChecksumAnnotation = true } Expect(foundConfigurationChecksumAnnotation).To(BeTrue()) diff --git a/internal/controllers/storage/init.go b/internal/controllers/storage/init.go index a9dc1b17..f08011d1 100644 --- a/internal/controllers/storage/init.go +++ b/internal/controllers/storage/init.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "strconv" "strings" "time" @@ -44,15 +45,17 @@ func (r *Reconciler) setInitPipelineStatus( } // This block is special internal logic that skips all Storage initialization. - if value, ok := storage.Annotations[v1alpha1.AnnotationSkipInitialization]; ok && value == v1alpha1.AnnotationValueTrue { - r.Log.Info("Storage initialization disabled (with annotation), proceed with caution") - r.Recorder.Event( - storage, - corev1.EventTypeWarning, - "SkippingInit", - "Skipping initialization due to skip annotation present, be careful!", - ) - return r.setInitStorageCompleted(ctx, storage, "Storage initialization not performed because initialization is skipped") + if value, ok := storage.Annotations[v1alpha1.AnnotationSkipInitialization]; ok { + if isTrue, _ := strconv.ParseBool(value); isTrue { + r.Log.Info("Storage initialization disabled (with annotation), proceed with caution") + r.Recorder.Event( + storage, + corev1.EventTypeWarning, + "SkippingInit", + "Skipping initialization due to skip annotation present, be careful!", + ) + return r.setInitStorageCompleted(ctx, storage, "Storage initialization not performed because initialization is skipped") + } } if meta.IsStatusConditionTrue(storage.Status.Conditions, OldStorageInitializedCondition) { @@ -303,7 +306,7 @@ func (r *Reconciler) createInitBlobstorageJob( ctx context.Context, storage *resources.StorageClusterBuilder, ) error { - builder := resources.GetInitJobBuilder(storage.DeepCopy()) + builder := storage.GetInitJobBuilder() newResource := builder.Placeholder(storage) _, err := resources.CreateOrUpdateOrMaybeIgnore(ctx, r.Client, newResource, func() error { var err error diff --git a/internal/labels/label.go b/internal/labels/label.go index 19ff5d94..5a0659d7 100644 --- a/internal/labels/label.go +++ b/internal/labels/label.go @@ -1,9 +1,5 @@ package labels -import ( - "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" -) - // https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ const ( // NameKey The name of a higher level application this one is part of @@ -31,8 +27,9 @@ const ( StorageGeneration = "ydb.tech/storage-generation" DatabaseGeneration = "ydb.tech/database-generation" - StorageComponent = "storage-node" - DynamicComponent = "dynamic-node" + StorageComponent = "storage-node" + DynamicComponent = "dynamic-node" + BlobstorageInitComponent = "blobstorage-init" GRPCComponent = "grpc" InterconnectComponent = "interconnect" @@ -50,28 +47,6 @@ func Common(name string, defaultLabels Labels) Labels { return l } -func StorageLabels(cluster *v1alpha1.Storage) Labels { - l := Common(cluster.Name, cluster.Labels) - - l.Merge(cluster.Spec.AdditionalLabels) - l.Merge(map[string]string{ - ComponentKey: StorageComponent, - }) - - return l -} - -func DatabaseLabels(database *v1alpha1.Database) Labels { - l := Common(database.Name, database.Labels) - - l.Merge(database.Spec.AdditionalLabels) - l.Merge(map[string]string{ - ComponentKey: DynamicComponent, - }) - - return l -} - func (l Labels) AsMap() map[string]string { return l } @@ -98,16 +73,6 @@ func (l Labels) Merge(other map[string]string) map[string]string { return l } -func (l Labels) MergeInPlace(other map[string]string) map[string]string { - result := l.Copy() - - for k, v := range other { - result[k] = v - } - - return result -} - func makeCommonLabels(other map[string]string, instance string) map[string]string { common := make(map[string]string) diff --git a/internal/resources/database.go b/internal/resources/database.go index 2328207c..672be046 100644 --- a/internal/resources/database.go +++ b/internal/resources/database.go @@ -7,7 +7,7 @@ import ( "k8s.io/client-go/rest" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/configuration/schema" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/metrics" @@ -28,6 +28,19 @@ func NewDatabase(ydbCr *api.Database) DatabaseBuilder { return DatabaseBuilder{Database: cr, Storage: nil} } +func (b *DatabaseBuilder) NewLabels() labels.Labels { + l := labels.Common(b.Name, b.Labels) + l.Merge(map[string]string{labels.ComponentKey: labels.DynamicComponent}) + + return l +} + +func (b *DatabaseBuilder) NewAnnotations() ydbannotations.Annotations { + annotations := ydbannotations.Common(b.Annotations) + + return annotations +} + func (b *DatabaseBuilder) Unwrap() *api.Database { return b.DeepCopy() } @@ -37,13 +50,20 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc return []ResourceBuilder{} } - databaseLabels := labels.DatabaseLabels(b.Unwrap()) + databaseLabels := b.NewLabels() + databaseAnnotations := b.NewAnnotations() statefulSetLabels := databaseLabels.Copy() + statefulSetLabels.Merge(b.Spec.AdditionalLabels) statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name}) - statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) - statefulSetAnnotations[annotations.ConfigurationChecksum] = SHAChecksum(b.Spec.Configuration) + statefulSetAnnotations := databaseAnnotations.Copy() + statefulSetAnnotations.Merge(b.Spec.AdditionalAnnotations) + if b.Spec.Configuration != "" { + statefulSetAnnotations.Merge(map[string]string{ydbannotations.ConfigurationChecksum: GetSHA256Checksum(b.Spec.Configuration)}) + } else { + statefulSetAnnotations.Merge(map[string]string{ydbannotations.ConfigurationChecksum: GetSHA256Checksum(b.Storage.Spec.Configuration)}) + } grpcServiceLabels := databaseLabels.Copy() grpcServiceLabels.Merge(b.Spec.Service.GRPC.AdditionalLabels) @@ -123,7 +143,7 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc api.DatabaseEncryptionKeySecretDir, api.DatabaseEncryptionKeySecretFile, ), - ID: SHAChecksum(b.Spec.StorageClusterRef.Name), + ID: GetSHA256Checksum(b.Spec.StorageClusterRef.Name), Pin: b.Spec.Encryption.Pin, Version: 1, }, @@ -219,16 +239,23 @@ func (b *DatabaseBuilder) GetResourceBuilders(restConfig *rest.Config) []Resourc }, ) } else { - optionalBuilders = append(optionalBuilders, b.getNodeSetBuilders(databaseLabels)...) + optionalBuilders = append( + optionalBuilders, + b.getNodeSetBuilders(databaseLabels, databaseAnnotations)...) } return optionalBuilders } -func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []ResourceBuilder { +func (b *DatabaseBuilder) getNodeSetBuilders( + databaseLabels labels.Labels, + databaseAnnotations ydbannotations.Annotations, +) []ResourceBuilder { var nodeSetBuilders []ResourceBuilder for _, nodeSetSpecInline := range b.Spec.NodeSets { + nodeSetName := fmt.Sprintf("%s-%s", b.Name, nodeSetSpecInline.Name) + nodeSetLabels := databaseLabels.Copy() nodeSetLabels.Merge(nodeSetSpecInline.Labels) nodeSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: nodeSetSpecInline.Name}) @@ -236,11 +263,12 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res nodeSetLabels.Merge(map[string]string{labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster}) } - nodeSetAnnotations := CopyDict(b.Annotations) - if nodeSetSpecInline.Annotations != nil { - for k, v := range nodeSetSpecInline.Annotations { - nodeSetAnnotations[k] = v - } + nodeSetAnnotations := databaseAnnotations.Copy() + nodeSetAnnotations.Merge(nodeSetSpecInline.Annotations) + if b.Spec.Configuration != "" { + nodeSetAnnotations.Merge(map[string]string{ydbannotations.ConfigurationChecksum: GetSHA256Checksum(b.Spec.Configuration)}) + } else { + nodeSetAnnotations.Merge(map[string]string{ydbannotations.ConfigurationChecksum: GetSHA256Checksum(b.Storage.Spec.Configuration)}) } databaseNodeSetSpec := b.recastDatabaseNodeSetSpecInline(nodeSetSpecInline.DeepCopy()) @@ -250,7 +278,7 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res &RemoteDatabaseNodeSetBuilder{ Object: b, - Name: b.Name + "-" + nodeSetSpecInline.Name, + Name: nodeSetName, Labels: nodeSetLabels, Annotations: nodeSetAnnotations, @@ -263,7 +291,7 @@ func (b *DatabaseBuilder) getNodeSetBuilders(databaseLabels labels.Labels) []Res &DatabaseNodeSetBuilder{ Object: b, - Name: b.Name + "-" + nodeSetSpecInline.Name, + Name: nodeSetName, Labels: nodeSetLabels, Annotations: nodeSetAnnotations, diff --git a/internal/resources/database_statefulset.go b/internal/resources/database_statefulset.go index 20964efa..1925c8eb 100644 --- a/internal/resources/database_statefulset.go +++ b/internal/resources/database_statefulset.go @@ -7,6 +7,7 @@ import ( "log" "regexp" "strconv" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -16,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/ptr" ) @@ -63,9 +65,11 @@ func (b *DatabaseStatefulSetBuilder) Build(obj client.Object) error { Template: b.buildPodTemplateSpec(), } - if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateStrategyOnDelete]; ok && value == api.AnnotationValueTrue { - sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ - Type: "OnDelete", + if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateStrategyOnDelete]; ok { + if isTrue, _ := strconv.ParseBool(value); isTrue { + sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ + Type: "OnDelete", + } } } @@ -144,12 +148,10 @@ func (b *DatabaseStatefulSetBuilder) buildPodTemplateSpec() corev1.PodTemplateSp } if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateDNSPolicy]; ok { - switch value { - case string(corev1.DNSClusterFirstWithHostNet), string(corev1.DNSClusterFirst), string(corev1.DNSDefault), string(corev1.DNSNone): - podTemplate.Spec.DNSPolicy = corev1.DNSPolicy(value) - case "": - podTemplate.Spec.DNSPolicy = corev1.DNSClusterFirst - default: + for _, acceptedPolicy := range ydbannotations.AcceptedDNSPolicy { + if value == acceptedPolicy { + podTemplate.Spec.DNSPolicy = corev1.DNSPolicy(value) + } } } @@ -422,13 +424,17 @@ func (b *DatabaseStatefulSetBuilder) buildContainer() corev1.Container { }, } - if value, ok := b.ObjectMeta.Annotations[api.AnnotationDisableLivenessProbe]; !ok || value != api.AnnotationValueTrue { - container.LivenessProbe = &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(api.GRPCPort), - }, + container.LivenessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(api.GRPCPort), }, + }, + } + + if value, ok := b.ObjectMeta.Annotations[api.AnnotationDisableLivenessProbe]; ok { + if isTrue, _ := strconv.ParseBool(value); isTrue { + container.LivenessProbe = nil } } @@ -696,6 +702,9 @@ func (b *DatabaseStatefulSetBuilder) buildContainerArgs() ([]string, []string) { } if value, ok := b.ObjectMeta.Annotations[api.AnnotationNodeHost]; ok { + if !strings.HasPrefix(value, "$(NODE_NAME).") { + value = fmt.Sprintf("%s.%s", "$(NODE_NAME)", value) + } args = append(args, "--node-host", value, diff --git a/internal/resources/databasenodeset.go b/internal/resources/databasenodeset.go index b3e1d041..26413a30 100644 --- a/internal/resources/databasenodeset.go +++ b/internal/resources/databasenodeset.go @@ -8,7 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" ) @@ -22,10 +22,6 @@ type DatabaseNodeSetBuilder struct { DatabaseNodeSetSpec api.DatabaseNodeSetSpec } -type DatabaseNodeSetResource struct { - *api.DatabaseNodeSet -} - func (b *DatabaseNodeSetBuilder) Build(obj client.Object) error { dns, ok := obj.(*api.DatabaseNodeSet) if !ok { @@ -54,30 +50,48 @@ func (b *DatabaseNodeSetBuilder) Placeholder(cr client.Object) client.Object { } } -func (b *DatabaseNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { - ydbCr := api.RecastDatabaseNodeSet(b.Unwrap()) - databaseLabels := labels.DatabaseLabels(ydbCr) +type DatabaseNodeSetResource struct { + *api.DatabaseNodeSet +} - statefulSetName := b.Name - statefulSetLabels := databaseLabels.Copy() - statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: statefulSetName}) +func (b *DatabaseNodeSetResource) NewLabels() labels.Labels { + l := labels.Common(b.Name, b.Labels) + l.Merge(map[string]string{labels.ComponentKey: labels.DynamicComponent}) databaseNodeSetName := b.Labels[labels.DatabaseNodeSetComponent] - statefulSetLabels.Merge(map[string]string{labels.DatabaseNodeSetComponent: databaseNodeSetName}) + l.Merge(map[string]string{labels.DatabaseNodeSetComponent: databaseNodeSetName}) + if remoteCluster, exist := b.Labels[labels.RemoteClusterKey]; exist { - statefulSetLabels.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster}) + l.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster}) } - statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) - statefulSetAnnotations[annotations.ConfigurationChecksum] = SHAChecksum(b.Spec.Configuration) + return l +} + +func (b *DatabaseNodeSetResource) NewAnnotations() ydbannotations.Annotations { + annotations := ydbannotations.Common(b.Annotations) + + return annotations +} + +func (b *DatabaseNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { + nodeSetLabels := b.NewLabels() + nodeSetAnnotations := b.NewAnnotations() + + statefulSetLabels := nodeSetLabels.Copy() + statefulSetLabels.Merge(b.Spec.AdditionalLabels) + statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name}) + + statefulSetAnnotations := nodeSetAnnotations.Copy() + statefulSetAnnotations.Merge(b.Spec.AdditionalAnnotations) var resourceBuilders []ResourceBuilder resourceBuilders = append(resourceBuilders, &DatabaseStatefulSetBuilder{ - Database: ydbCr, + Database: api.RecastDatabaseNodeSet(b.Unwrap()), RestConfig: restConfig, - Name: statefulSetName, + Name: b.Name, Labels: statefulSetLabels, Annotations: statefulSetAnnotations, }, diff --git a/internal/resources/predicate.go b/internal/resources/predicate.go index 65e1a241..f9e8ca06 100644 --- a/internal/resources/predicate.go +++ b/internal/resources/predicate.go @@ -7,13 +7,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" ) func LastAppliedAnnotationPredicate() predicate.Predicate { return predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - return !annotations.CompareLastAppliedAnnotation( + return !ydbannotations.CompareLastAppliedAnnotation( e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations(), ) diff --git a/internal/resources/remotedatabasenodeset.go b/internal/resources/remotedatabasenodeset.go index 93fcf5c1..8e1f41a3 100644 --- a/internal/resources/remotedatabasenodeset.go +++ b/internal/resources/remotedatabasenodeset.go @@ -61,16 +61,13 @@ func (b *RemoteDatabaseNodeSetBuilder) Placeholder(cr client.Object) client.Obje func (b *RemoteDatabaseNodeSetResource) GetResourceBuilders() []ResourceBuilder { var resourceBuilders []ResourceBuilder - nodeSetAnnotations := CopyDict(b.Annotations) - delete(nodeSetAnnotations, ydbannotations.LastAppliedAnnotation) - resourceBuilders = append(resourceBuilders, &DatabaseNodeSetBuilder{ Object: b, Name: b.Name, Labels: b.Labels, - Annotations: nodeSetAnnotations, + Annotations: b.Annotations, DatabaseNodeSetSpec: b.Spec, }, @@ -169,8 +166,8 @@ func (b *RemoteDatabaseNodeSetResource) SetPrimaryResourceAnnotations(obj client annotations[key] = value } - if _, exist := annotations[ydbannotations.PrimaryResourceDatabaseAnnotation]; !exist { - annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] = b.Spec.DatabaseRef.Name + if _, exist := annotations[ydbannotations.PrimaryResourceDatabase]; !exist { + annotations[ydbannotations.PrimaryResourceDatabase] = b.Spec.DatabaseRef.Name } obj.SetAnnotations(annotations) @@ -179,7 +176,7 @@ func (b *RemoteDatabaseNodeSetResource) SetPrimaryResourceAnnotations(obj client func (b *RemoteDatabaseNodeSetResource) UnsetPrimaryResourceAnnotations(obj client.Object) { annotations := make(map[string]string) for key, value := range obj.GetAnnotations() { - if key != annotations[ydbannotations.PrimaryResourceDatabaseAnnotation] { + if key != annotations[ydbannotations.PrimaryResourceDatabase] { annotations[key] = value } } diff --git a/internal/resources/remotestoragenodeset.go b/internal/resources/remotestoragenodeset.go index c8dbf8d9..a627025f 100644 --- a/internal/resources/remotestoragenodeset.go +++ b/internal/resources/remotestoragenodeset.go @@ -62,7 +62,7 @@ func (b *RemoteStorageNodeSetResource) GetResourceBuilders() []ResourceBuilder { var resourceBuilders []ResourceBuilder nodeSetAnnotations := CopyDict(b.Annotations) - delete(nodeSetAnnotations, ydbannotations.LastAppliedAnnotation) + delete(nodeSetAnnotations, ydbannotations.LastApplied) resourceBuilders = append(resourceBuilders, &StorageNodeSetBuilder{ @@ -150,8 +150,8 @@ func (b *RemoteStorageNodeSetResource) SetPrimaryResourceAnnotations(obj client. annotations[key] = value } - if _, exist := annotations[ydbannotations.PrimaryResourceStorageAnnotation]; !exist { - annotations[ydbannotations.PrimaryResourceStorageAnnotation] = b.Spec.StorageRef.Name + if _, exist := annotations[ydbannotations.PrimaryResourceStorage]; !exist { + annotations[ydbannotations.PrimaryResourceStorage] = b.Spec.StorageRef.Name } obj.SetAnnotations(annotations) @@ -160,7 +160,7 @@ func (b *RemoteStorageNodeSetResource) SetPrimaryResourceAnnotations(obj client. func (b *RemoteStorageNodeSetResource) UnsetPrimaryResourceAnnotations(obj client.Object) { annotations := make(map[string]string) for key, value := range obj.GetAnnotations() { - if key != annotations[ydbannotations.PrimaryResourceStorageAnnotation] { + if key != annotations[ydbannotations.PrimaryResourceStorage] { annotations[key] = value } } diff --git a/internal/resources/resource.go b/internal/resources/resource.go index c48965cf..c2ce8482 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -82,7 +82,7 @@ type ResourceBuilder interface { } var ( - annotator = patch.NewAnnotator(ydbannotations.LastAppliedAnnotation) + annotator = patch.NewAnnotator(ydbannotations.LastApplied) patchMaker = patch.NewPatchMaker(annotator) ) @@ -265,8 +265,8 @@ func UpdateResource(oldObj, newObj client.Object) client.Object { func CopyPrimaryResourceObjectAnnotation(obj client.Object, oldAnnotations map[string]string) { annotations := CopyDict(obj.GetAnnotations()) for key, value := range oldAnnotations { - if key == ydbannotations.PrimaryResourceDatabaseAnnotation || - key == ydbannotations.PrimaryResourceStorageAnnotation { + if key == ydbannotations.PrimaryResourceDatabase || + key == ydbannotations.PrimaryResourceStorage { annotations[key] = value } } @@ -278,7 +278,7 @@ func SetRemoteResourceVersionAnnotation(obj client.Object, resourceVersion strin for key, value := range obj.GetAnnotations() { annotations[key] = value } - annotations[ydbannotations.RemoteResourceVersionAnnotation] = resourceVersion + annotations[ydbannotations.RemoteResourceVersion] = resourceVersion obj.SetAnnotations(annotations) } @@ -564,7 +564,7 @@ func buildCAStorePatchingCommandArgs( return command, args } -func SHAChecksum(text string) string { +func GetSHA256Checksum(text string) string { hasher := sha256.New() hasher.Write([]byte(text)) return hex.EncodeToString(hasher.Sum(nil)) @@ -581,3 +581,14 @@ func CompareMaps(map1, map2 map[string]string) bool { } return true } + +func isSignAlgorithmSupported(alg string) bool { + supportedAlgs := jwt.GetAlgorithms() + + for _, supportedAlg := range supportedAlgs { + if alg == supportedAlg { + return true + } + } + return false +} diff --git a/internal/resources/storage.go b/internal/resources/storage.go index 9eaef577..6366d681 100644 --- a/internal/resources/storage.go +++ b/internal/resources/storage.go @@ -1,12 +1,14 @@ package resources import ( + "fmt" + "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/rest" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/metrics" ) @@ -29,14 +31,66 @@ func (b *StorageClusterBuilder) Unwrap() *api.Storage { return b.DeepCopy() } +func (b *StorageClusterBuilder) NewLabels() labels.Labels { + l := labels.Common(b.Name, b.Labels) + l.Merge(map[string]string{labels.ComponentKey: labels.StorageComponent}) + + return l +} + +func (b *StorageClusterBuilder) NewAnnotations() ydbannotations.Annotations { + annotations := ydbannotations.Common(b.Annotations) + annotations.Merge(map[string]string{ydbannotations.ConfigurationChecksum: GetSHA256Checksum(b.Spec.Configuration)}) + + return annotations +} + +func (b *StorageClusterBuilder) NewInitJobLabels() labels.Labels { + l := labels.Common(b.Name, b.Labels) + + if b.Spec.InitJob != nil { + l.Merge(b.Spec.InitJob.AdditionalLabels) + } + l.Merge(map[string]string{labels.ComponentKey: labels.BlobstorageInitComponent}) + + return l +} + +func (b *StorageClusterBuilder) NewInitJobAnnotations() ydbannotations.Annotations { + annotations := ydbannotations.Common(b.Annotations) + + if b.Spec.InitJob != nil { + annotations.Merge(b.Spec.InitJob.AdditionalLabels) + } + annotations.Merge(map[string]string{ydbannotations.ConfigurationChecksum: GetSHA256Checksum(b.Spec.Configuration)}) + + return annotations +} + +func (b *StorageClusterBuilder) GetInitJobBuilder() ResourceBuilder { + jobName := fmt.Sprintf(InitJobNameFormat, b.Name) + jobLabels := b.NewInitJobLabels() + jobAnnotations := b.NewInitJobAnnotations() + + return &StorageInitJobBuilder{ + Storage: b.Unwrap(), + + Name: jobName, + Labels: jobLabels, + Annotations: jobAnnotations, + } +} + func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { - storageLabels := labels.StorageLabels(b.Unwrap()) + storageLabels := b.NewLabels() + storageAnnotations := b.NewAnnotations() statefulSetLabels := storageLabels.Copy() + statefulSetLabels.Merge(b.Spec.AdditionalLabels) statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name}) - statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) - statefulSetAnnotations[annotations.ConfigurationChecksum] = SHAChecksum(b.Spec.Configuration) + statefulSetAnnotations := storageAnnotations.Copy() + statefulSetAnnotations.Merge(b.Spec.AdditionalAnnotations) grpcServiceLabels := storageLabels.Copy() grpcServiceLabels.Merge(b.Spec.Service.GRPC.AdditionalLabels) @@ -110,7 +164,10 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R }, ) } else { - optionalBuilders = append(optionalBuilders, b.getNodeSetBuilders(storageLabels)...) + optionalBuilders = append( + optionalBuilders, + b.getNodeSetBuilders(storageLabels, storageAnnotations)..., + ) } return append( @@ -158,10 +215,15 @@ func (b *StorageClusterBuilder) GetResourceBuilders(restConfig *rest.Config) []R ) } -func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels) []ResourceBuilder { +func (b *StorageClusterBuilder) getNodeSetBuilders( + storageLabels labels.Labels, + storageAnnotations ydbannotations.Annotations, +) []ResourceBuilder { var nodeSetBuilders []ResourceBuilder for _, nodeSetSpecInline := range b.Spec.NodeSets { + nodeSetName := fmt.Sprintf("%s-%s", b.Name, nodeSetSpecInline.Name) + nodeSetLabels := storageLabels.Copy() nodeSetLabels.Merge(nodeSetSpecInline.Labels) nodeSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: nodeSetSpecInline.Name}) @@ -169,12 +231,8 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels) nodeSetLabels.Merge(map[string]string{labels.RemoteClusterKey: nodeSetSpecInline.Remote.Cluster}) } - nodeSetAnnotations := CopyDict(b.Annotations) - if nodeSetSpecInline.Annotations != nil { - for k, v := range nodeSetSpecInline.Annotations { - nodeSetAnnotations[k] = v - } - } + nodeSetAnnotations := storageAnnotations.Copy() + nodeSetAnnotations.Merge(nodeSetSpecInline.Annotations) storageNodeSetSpec := b.recastStorageNodeSetSpecInline(nodeSetSpecInline.DeepCopy()) if nodeSetSpecInline.Remote != nil { @@ -183,7 +241,7 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels) &RemoteStorageNodeSetBuilder{ Object: b, - Name: b.Name + "-" + nodeSetSpecInline.Name, + Name: nodeSetName, Labels: nodeSetLabels, Annotations: nodeSetAnnotations, @@ -196,7 +254,7 @@ func (b *StorageClusterBuilder) getNodeSetBuilders(storageLabels labels.Labels) &StorageNodeSetBuilder{ Object: b, - Name: b.Name + "-" + nodeSetSpecInline.Name, + Name: nodeSetName, Labels: nodeSetLabels, Annotations: nodeSetAnnotations, diff --git a/internal/resources/storage_init_job.go b/internal/resources/storage_init_job.go index a18cbd7c..46db784f 100644 --- a/internal/resources/storage_init_job.go +++ b/internal/resources/storage_init_job.go @@ -11,7 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" "github.com/ydb-platform/ydb-kubernetes-operator/internal/ptr" ) @@ -25,6 +25,16 @@ type StorageInitJobBuilder struct { Annotations map[string]string } +func NewInitJob(ydbCr *api.Storage) StorageInitJobBuilder { + cr := ydbCr.DeepCopy() + + return StorageInitJobBuilder{Storage: cr} +} + +func (b *StorageInitJobBuilder) Unwrap() *api.Storage { + return b.DeepCopy() +} + func (b *StorageInitJobBuilder) Build(obj client.Object) error { job, ok := obj.(*batchv1.Job) if !ok { @@ -70,7 +80,7 @@ func GetInitJobBuilder(storage *api.Storage) ResourceBuilder { } if storage.Spec.InitJob.AdditionalAnnotations != nil { jobAnnotations = CopyDict(storage.Spec.InitJob.AdditionalAnnotations) - jobAnnotations[annotations.ConfigurationChecksum] = SHAChecksum(storage.Spec.Configuration) + jobAnnotations[ydbannotations.ConfigurationChecksum] = GetSHA256Checksum(storage.Spec.Configuration) } } @@ -93,8 +103,8 @@ func (b *StorageInitJobBuilder) buildInitJobPodTemplateSpec() corev1.PodTemplate } podTemplate := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: b.Labels, - Annotations: b.Annotations, + Labels: CopyDict(b.Labels), + Annotations: CopyDict(b.Annotations), }, Spec: corev1.PodSpec{ Containers: []corev1.Container{b.buildInitJobContainer()}, @@ -138,12 +148,10 @@ func (b *StorageInitJobBuilder) buildInitJobPodTemplateSpec() corev1.PodTemplate } if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateDNSPolicy]; ok { - switch value { - case string(corev1.DNSClusterFirstWithHostNet), string(corev1.DNSClusterFirst), string(corev1.DNSDefault), string(corev1.DNSNone): - podTemplate.Spec.DNSPolicy = corev1.DNSPolicy(value) - case "": - podTemplate.Spec.DNSPolicy = corev1.DNSClusterFirst - default: + for _, acceptedPolicy := range ydbannotations.AcceptedDNSPolicy { + if value == acceptedPolicy { + podTemplate.Spec.DNSPolicy = corev1.DNSPolicy(value) + } } } diff --git a/internal/resources/storage_statefulset.go b/internal/resources/storage_statefulset.go index f4ec2d3a..a9044f36 100644 --- a/internal/resources/storage_statefulset.go +++ b/internal/resources/storage_statefulset.go @@ -80,9 +80,11 @@ func (b *StorageStatefulSetBuilder) Build(obj client.Object) error { Template: b.buildPodTemplateSpec(), } - if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateStrategyOnDelete]; ok && value == api.AnnotationValueTrue { - sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ - Type: "OnDelete", + if value, ok := b.ObjectMeta.Annotations[api.AnnotationUpdateStrategyOnDelete]; ok { + if isTrue, _ := strconv.ParseBool(value); isTrue { + sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ + Type: "OnDelete", + } } } @@ -157,9 +159,8 @@ func (b *StorageStatefulSetBuilder) buildPodTemplateSpec() corev1.PodTemplateSpe switch value { case string(corev1.DNSClusterFirstWithHostNet), string(corev1.DNSClusterFirst), string(corev1.DNSDefault), string(corev1.DNSNone): podTemplate.Spec.DNSPolicy = corev1.DNSPolicy(value) - case "": - podTemplate.Spec.DNSPolicy = corev1.DNSClusterFirst default: + podTemplate.Spec.DNSPolicy = corev1.DNSClusterFirst } } @@ -383,13 +384,17 @@ func (b *StorageStatefulSetBuilder) buildContainer() corev1.Container { // todo Resources: containerResources, } - if value, ok := b.ObjectMeta.Annotations[api.AnnotationDisableLivenessProbe]; !ok || value != api.AnnotationValueTrue { - container.LivenessProbe = &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(api.GRPCPort), - }, + container.LivenessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(api.GRPCPort), }, + }, + } + + if value, ok := b.ObjectMeta.Annotations[api.AnnotationDisableLivenessProbe]; ok { + if isTrue, _ := strconv.ParseBool(value); isTrue { + container.LivenessProbe = nil } } diff --git a/internal/resources/storagenodeset.go b/internal/resources/storagenodeset.go index 7f69425f..b87aaff8 100644 --- a/internal/resources/storagenodeset.go +++ b/internal/resources/storagenodeset.go @@ -8,7 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1" - "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" + ydbannotations "github.com/ydb-platform/ydb-kubernetes-operator/internal/annotations" "github.com/ydb-platform/ydb-kubernetes-operator/internal/labels" ) @@ -22,10 +22,6 @@ type StorageNodeSetBuilder struct { StorageNodeSetSpec api.StorageNodeSetSpec } -type StorageNodeSetResource struct { - *api.StorageNodeSet -} - func (b *StorageNodeSetBuilder) Build(obj client.Object) error { sns, ok := obj.(*api.StorageNodeSet) if !ok { @@ -54,31 +50,49 @@ func (b *StorageNodeSetBuilder) Placeholder(cr client.Object) client.Object { } } -func (b *StorageNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { - ydbCr := api.RecastStorageNodeSet(b.Unwrap()) - storageLabels := labels.StorageLabels(ydbCr) +type StorageNodeSetResource struct { + *api.StorageNodeSet +} - statefulSetName := b.Name - statefulSetLabels := storageLabels.Copy() - statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: statefulSetName}) +func (b *StorageNodeSetResource) NewLabels() labels.Labels { + l := labels.Common(b.Name, b.Labels) + l.Merge(map[string]string{labels.ComponentKey: labels.StorageComponent}) storageNodeSetName := b.Labels[labels.StorageNodeSetComponent] - statefulSetLabels.Merge(map[string]string{labels.StorageNodeSetComponent: storageNodeSetName}) + l.Merge(map[string]string{labels.StorageNodeSetComponent: storageNodeSetName}) + if remoteCluster, exist := b.Labels[labels.RemoteClusterKey]; exist { - statefulSetLabels.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster}) + l.Merge(map[string]string{labels.RemoteClusterKey: remoteCluster}) } - statefulSetAnnotations := CopyDict(b.Spec.AdditionalAnnotations) - statefulSetAnnotations[annotations.ConfigurationChecksum] = SHAChecksum(b.Spec.Configuration) + return l +} + +func (b *StorageNodeSetResource) NewAnnotations() ydbannotations.Annotations { + annotations := ydbannotations.Common(b.Annotations) + + return annotations +} + +func (b *StorageNodeSetResource) GetResourceBuilders(restConfig *rest.Config) []ResourceBuilder { + nodeSetLabels := b.NewLabels() + nodeSetAnnotations := b.NewAnnotations() + + statefulSetLabels := nodeSetLabels.Copy() + statefulSetLabels.Merge(b.Spec.AdditionalLabels) + statefulSetLabels.Merge(map[string]string{labels.StatefulsetComponent: b.Name}) + + statefulSetAnnotations := nodeSetAnnotations.Copy() + statefulSetAnnotations.Merge(b.Spec.AdditionalAnnotations) var resourceBuilders []ResourceBuilder resourceBuilders = append( resourceBuilders, &StorageStatefulSetBuilder{ - Storage: ydbCr, + Storage: api.RecastStorageNodeSet(b.StorageNodeSet), RestConfig: restConfig, - Name: statefulSetName, + Name: b.Name, Labels: statefulSetLabels, Annotations: statefulSetAnnotations, },