feat: add submariner plugin config

Signed-off-by: Tom <[email protected]>

pkg/apis/fleet/v1alpha1/types.go

@@ -87,6 +87,8 @@ type PluginConfig struct {
 	DistributedStorage *DistributedStorageConfig `json:"distributedStorage,omitempty"`
 	// Flagger defines the configuretion for the kurator rollout engine.
 	Flagger *FlaggerConfig `json:"flagger,omitempty"`
+	// SubMariner defines the configuration for the kurator network management.
+	SubMariner *SubMarinerConfig `json:"submariner,omitempty"`
 }
 
 type MetricConfig struct {
@@ -569,6 +571,34 @@ type FlaggerConfig struct {
 	PublicTestloader bool `json:"publicTestloader,omitempty"`
 }
 
+type SubMarinerConfig struct {
+	// Chart defines the helm chart config of the submariner.
+	// default value is
+	//
+	// ```yaml
+	// chart:
+	//   repository: https://submariner-io.github.io/submariner-charts/charts
+	//   name: submariner
+	//   version: 0.14.9
+	//   targetNamespace: submariner
+	// ```
+	//
+
+	// ```yaml
+	// extraArgs:
+	//   operator:
+	//			image:
+	//     		pullPolicy: "IfNotPresent"
+	// ```
+
+	// +optional
+	Chart *ChartConfig `json:"chart,omitempty"`
+	// ExtraArgs is the set of extra arguments for submariner, and example will be provided in the future.
+	//
+	// +optional
+	ExtraArgs apiextensionsv1.JSON `json:"extraArgs,omitempty"`
+}
+
 // Provider only can be istio now.
 // TODO: add Linkerd, APP Mesh, NGINX, Kuma, Gateway, Gloo
 type Provider string

pkg/fleet-manager/fleet_plugin.go

@@ -58,6 +58,7 @@ func (f *FleetManager) reconcilePlugins(ctx context.Context, fleet *fleetapi.Fle
 			f.reconcileBackupPlugin,
 			f.reconcileDistributedStoragePlugin,
 			f.reconcileFlaggerPlugin,
+			f.reconcileSubmarinerPlugin,
 		}
 
 		resultsChannel := make(chan reconcileResult, len(funcs))

pkg/fleet-manager/fleet_plugin_submariner.go

@@ -0,0 +1,77 @@
+/*
+Copyright Kurator Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fleet
+
+import (
+	"context"
+	"time"
+
+	"helm.sh/helm/v3/pkg/kube"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1"
+	"kurator.dev/kurator/pkg/fleet-manager/plugin"
+	"kurator.dev/kurator/pkg/infra/util"
+)
+
+// reconcileSubmarinerPlugin reconciles the Submariner plugin.
+// The fleetClusters parameter is currently unused, but is included to match the function signature of other functions in reconcilePlugins.
+func (f *FleetManager) reconcileSubmarinerPlugin(ctx context.Context, fleet *fleetapi.Fleet, fleetClusters map[ClusterKey]*FleetCluster) (kube.ResourceList, ctrl.Result, error) {
+	log := ctrl.LoggerFrom(ctx)
+
+	submarinerCfg := fleet.Spec.Plugin.SubMariner
+
+	if submarinerCfg == nil {
+		// reconcilePluginResources will delete all resources if plugin is nil
+		return nil, ctrl.Result{}, nil
+	}
+
+	fleetNN := types.NamespacedName{
+		Namespace: fleet.Namespace,
+		Name:      fleet.Name,
+	}
+
+	fleetOwnerRef := ownerReference(fleet)
+	var resources kube.ResourceList
+
+	for key, cluster := range fleetClusters {
+		b, err := plugin.RenderSubmarinerOperator(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{
+			Name:       key.Name,
+			SecretName: cluster.Secret,
+			SecretKey:  cluster.SecretKey,
+		}, submarinerCfg)
+		if err != nil {
+			return nil, ctrl.Result{}, err
+		}
+
+		// apply submariner helm resources
+		submarinerResources, err := util.PatchResources(b)
+		if err != nil {
+			return nil, ctrl.Result{}, err
+		}
+		resources = append(resources, submarinerResources...)
+	}
+
+	log.V(4).Info("wait for submariner helm release to be reconciled")
+	if !f.helmReleaseReady(ctx, fleet, resources) {
+		// wait for HelmRelease to be ready
+		return nil, ctrl.Result{
+			// HelmRelease check interval is 1m, so we set 30s here
+			RequeueAfter: 30 * time.Second,
+		}, nil
+	}
+
+	return resources, ctrl.Result{}, nil
+}

pkg/fleet-manager/manifests/plugins/submariner-k8s-broker.yaml

@@ -0,0 +1,5 @@
+type: default
+repo: https://submariner-io.github.io/submariner-charts/charts
+name: submariner-k8s-broker
+version: 0.18.0
+targetNamespace: submariner

pkg/fleet-manager/manifests/plugins/submariner-operator.yaml

@@ -0,0 +1,5 @@
+type: default
+repo: https://submariner-io.github.io/submariner-charts/charts
+name: submariner-operator
+version: 0.18.0
+targetNamespace: submariner

pkg/fleet-manager/plugin/plugin.go

@@ -32,25 +32,29 @@ import (
 )
 
 const (
-	MetricPluginName          = "metric"
-	GrafanaPluginName         = "grafana"
-	KyvernoPluginName         = "kyverno"
-	BackupPluginName          = "backup"
-	StorageOperatorPluginName = "storage-operator"
-	ClusterStoragePluginName  = "cluster-storage"
-	FlaggerPluginName         = "flagger"
-	PublicTestloaderName      = "testloader"
-
-	ThanosComponentName        = "thanos"
-	PrometheusComponentName    = "prometheus"
-	GrafanaComponentName       = "grafana"
-	KyvernoComponentName       = "kyverno"
-	KyvernoPolicyComponentName = "kyverno-policies"
-	VeleroComponentName        = "velero"
-	RookOperatorComponentName  = "rook"
-	RookClusterComponentName   = "rook-ceph"
-	FlaggerComponentName       = "flagger"
-	TestloaderComponentName    = "testloader"
+	MetricPluginName             = "metric"
+	GrafanaPluginName            = "grafana"
+	KyvernoPluginName            = "kyverno"
+	BackupPluginName             = "backup"
+	StorageOperatorPluginName    = "storage-operator"
+	ClusterStoragePluginName     = "cluster-storage"
+	FlaggerPluginName            = "flagger"
+	PublicTestloaderName         = "testloader"
+	SubMarinerBrokerPluginName   = "submariner-k8s-broker"
+	SubMarinerOperatorPluginName = "submariner-operator"
+
+	ThanosComponentName             = "thanos"
+	PrometheusComponentName         = "prometheus"
+	GrafanaComponentName            = "grafana"
+	KyvernoComponentName            = "kyverno"
+	KyvernoPolicyComponentName      = "kyverno-policies"
+	VeleroComponentName             = "velero"
+	RookOperatorComponentName       = "rook"
+	RookClusterComponentName        = "rook-ceph"
+	FlaggerComponentName            = "flagger"
+	TestloaderComponentName         = "testloader"
+	SubMarinerBrokerComponentName   = "submariner-k8s-broker"
+	SubMarinerOperatorComponentName = "submariner-operator"
 
 	OCIReposiotryPrefix = "oci://"
 )
@@ -444,6 +448,69 @@ func RenderRolloutTestloader(
 	})
 }
 
+func RenderSubmarinerOperator(
+	fsys fs.FS,
+	fleetNN types.NamespacedName,
+	fleetRef *metav1.OwnerReference,
+	cluster KubeConfigSecretRef,
+	subMarinerConfig *fleetv1a1.SubMarinerConfig,
+) ([]byte, error) {
+	// get and merge the chart config
+	c, err := getFleetPluginChart(fsys, SubMarinerOperatorComponentName)
+	if err != nil {
+		return nil, err
+	}
+	mergeChartConfig(c, subMarinerConfig.Chart)
+	c.TargetNamespace = fleetNN.Namespace // submariner chart is fleet scoped
+
+	values, err := toMap(subMarinerConfig.ExtraArgs)
+	if err != nil {
+		return nil, err
+	}
+
+	return renderFleetPlugin(fsys, FleetPluginConfig{
+		Name:           SubMarinerOperatorPluginName,
+		Component:      SubMarinerOperatorComponentName,
+		Fleet:          fleetNN,
+		Cluster:        &cluster,
+		OwnerReference: fleetRef,
+		Chart:          *c,
+		Values:         values,
+	})
+}
+
+
+func RenderSubmarinerBroker(
+	fsys fs.FS,
+	fleetNN types.NamespacedName,
+	fleetRef *metav1.OwnerReference,
+	cluster KubeConfigSecretRef,
+	subMarinerConfig *fleetv1a1.SubMarinerConfig,
+) ([]byte, error) {
+	// get and merge the chart config
+	c, err := getFleetPluginChart(fsys, SubMarinerBrokerComponentName)
+	if err != nil {
+		return nil, err
+	}
+	mergeChartConfig(c, subMarinerConfig.Chart)
+	c.TargetNamespace = fleetNN.Namespace // submariner chart is fleet scoped
+
+	values, err := toMap(subMarinerConfig.ExtraArgs)
+	if err != nil {
+		return nil, err
+	}
+
+	return renderFleetPlugin(fsys, FleetPluginConfig{
+		Name:           SubMarinerBrokerPluginName,
+		Component:      SubMarinerBrokerComponentName,
+		Fleet:          fleetNN,
+		Cluster:        &cluster,
+		OwnerReference: fleetRef,
+		Chart:          *c,
+		Values:         values,
+	})
+}
+
 // According to distributedStorageCfg, generate the configuration for rook-ceph
 func buildStorageClusterValue(distributedStorageCfg fleetv1a1.DistributedStorageConfig) map[string]interface{} {
 	customValues := make(map[string]interface{})

pkg/fleet-manager/plugin/plugin_test.go

@@ -644,3 +644,81 @@ func TestRendeRolloutTestloader(t *testing.T) {
 		})
 	}
 }
+
+func TestRenderSubmarinerBroker(t *testing.T) {
+	cases := []struct {
+		name   string
+		fleet  types.NamespacedName
+		ref    *metav1.OwnerReference
+		config *v1alpha1.SubMarinerConfig
+	}{
+		{
+			name: "default",
+			fleet: types.NamespacedName{
+				Name:      "fleet-1",
+				Namespace: "default",
+			},
+			ref: &metav1.OwnerReference{
+				APIVersion: v1alpha1.GroupVersion.String(),
+				Kind:       "Fleet",
+				Name:       "fleet-1",
+				UID:        "xxxxxx",
+			},
+			config: &v1alpha1.SubMarinerConfig{},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := RenderSubmarinerBroker(manifestFS, tc.fleet, tc.ref, KubeConfigSecretRef{
+				Name:       "cluster1",
+				SecretName: "cluster1",
+				SecretKey:  "kubeconfig.yaml",
+			}, tc.config)
+			assert.NoError(t, err)
+
+			getExpected, err := getExpected("submariner-broker", tc.name)
+			assert.NoError(t, err)
+			assert.Equal(t, string(getExpected), string(got))
+		})
+	}
+}
+
+func TestRenderSubmarinerOperator(t *testing.T) {
+	cases := []struct {
+		name   string
+		fleet  types.NamespacedName
+		ref    *metav1.OwnerReference
+		config *v1alpha1.SubMarinerConfig
+	}{
+		{
+			name: "default",
+			fleet: types.NamespacedName{
+				Name:      "fleet-1",
+				Namespace: "default",
+			},
+			ref: &metav1.OwnerReference{
+				APIVersion: v1alpha1.GroupVersion.String(),
+				Kind:       "Fleet",
+				Name:       "fleet-1",
+				UID:        "xxxxxx",
+			},
+			config: &v1alpha1.SubMarinerConfig{},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := RenderSubmarinerOperator(manifestFS, tc.fleet, tc.ref, KubeConfigSecretRef{
+				Name:       "cluster1",
+				SecretName: "cluster1",
+				SecretKey:  "kubeconfig.yaml",
+			}, tc.config)
+			assert.NoError(t, err)
+
+			getExpected, err := getExpected("submariner-operator", tc.name)
+			assert.NoError(t, err)
+			assert.Equal(t, string(getExpected), string(got))
+		})
+	}
+}

pkg/fleet-manager/plugin/testdata/submariner-broker/default.yaml

@@ -0,0 +1,53 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: "submariner-k8s-broker-cluster1"
+  namespace: "default"
+  labels:
+    app.kubernetes.io/managed-by: fleet-manager
+    fleet.kurator.dev/name: "fleet-1"
+    fleet.kurator.dev/plugin: "submariner-k8s-broker"
+    fleet.kurator.dev/component: "submariner-k8s-broker"
+  ownerReferences:
+  - apiVersion: "fleet.kurator.dev/v1alpha1"
+    kind: "Fleet"
+    name: "fleet-1"
+    uid: "xxxxxx"
+spec:
+  type: "default"
+  interval: 5m0s
+  url: "https://submariner-io.github.io/submariner-charts/charts"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: "submariner-k8s-broker-cluster1"
+  namespace: "default"
+  labels:
+    app.kubernetes.io/managed-by: fleet-manager
+    fleet.kurator.dev/name: "fleet-1"
+    fleet.kurator.dev/plugin: "submariner-k8s-broker"
+    fleet.kurator.dev/component: "submariner-k8s-broker"
+  ownerReferences:
+  - apiVersion: "fleet.kurator.dev/v1alpha1"
+    kind: "Fleet"
+    name: "fleet-1"
+    uid: "xxxxxx"
+spec:
+  chart:
+    spec:
+      chart: "submariner-k8s-broker"
+      version: "0.18.0"
+      sourceRef:
+        kind: HelmRepository
+        name: "submariner-k8s-broker-cluster1"
+  interval: 1m0s
+  install:
+    createNamespace: true
+  targetNamespace: "default"
+  storageNamespace: "default"
+  timeout: 15m0s
+  kubeConfig:
+    secretRef:
+      name: cluster1
+      key: kubeconfig.yaml