diff --git a/docs/content/en/references/apps_v1alpha1_types.html b/docs/content/en/references/apps_v1alpha1_types.html index f8596a8c3..ebefed348 100644 --- a/docs/content/en/references/apps_v1alpha1_types.html +++ b/docs/content/en/references/apps_v1alpha1_types.html @@ -1707,13 +1707,12 @@

RolloutConfig trafficRoutingProvider
-string +Kurator fleet/v1alpha1.Provider -(Optional)

TrafficRoutingProvider defines traffic routing provider. -Kurator only supports istio for now. +Kurator supports istio,kuma,nginx for now. Other provider will be added later.

@@ -2213,6 +2212,57 @@

TrafficRoutingConfig +host
+ +string + + + +(Optional) +

Host defines the domain name you specify. +Only for NGINX +Fill in the host field with your own domain as the host of the Ingress. +The actual created Ingress is as follows:

+
apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+name: nginx
+namespace: application.syncPolicies.rollout.workload.namespace
+labels:
+app: application.syncPolicies.rollout.ServiceName
+annotations:
+kubernetes.io/ingress.class: "nginx"
+spec:
+rules:
+- host: ""
+http:
+paths:
+- pathType: Prefix
+path: "/"
+backend:
+service:
+name: application.syncPolicies.rollout.ServiceName
+port:
+number: application.syncPolicies.rollout.port
+
+ + + + +protocol
+ +string + + + +(Optional) +

Protocol defines the protocol used by Kuma +Only for kuma +Defaults to http

+ + + + canaryStrategy
diff --git a/docs/content/en/references/fleet_v1alpha1_types.html b/docs/content/en/references/fleet_v1alpha1_types.html index 83843a4b5..5694cf081 100644 --- a/docs/content/en/references/fleet_v1alpha1_types.html +++ b/docs/content/en/references/fleet_v1alpha1_types.html @@ -338,6 +338,7 @@

ChartConfig

(Appears on: BackupConfig, +Config, DistributedStorageConfig, FlaggerConfig, GrafanaConfig, @@ -399,6 +400,65 @@

ChartConfig +

Config +

+

+(Appears on: +FlaggerConfig) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+chart
+ + +ChartConfig + + +		 +(Optional) +

Chart defines the helm chart config of the TrafficRoutingProvider. +For Example, using the following configuration to change the version of nginx installed.

+
repo: https://kubernetes.github.io/ingress-nginx
+version: 4.10.0
+
+
+extraArgs
+ + +Kubernetes /apiextensions/v1.JSON + + +		 +(Optional) +

ExtraArgs is the set of extra arguments for TrafficRoutingProvider’s chart. +You can pass in values according to your needs. +For Example, using the following configuration to change the port that Prometheus uses to scrape metrics.

+
values:
+controller:
+podAnnotations:
+prometheus.io/port: 10378
+
+
+
+

Device

@@ -665,10 +725,11 @@

FlaggerConfig -(Optional)

TrafficRoutingProvider defines traffic routing provider. And Kurator will install flagger in trafficRoutingProvider’s namespace For example, If you use istio as a provider, flager will be installed in istio’s namespace istio-system. +And if you use istio as a provider, you need to install it manually. +Otherwise, you can configure it in ProviderConfig (or use the default configuration) and Kurator will automatically deploy it. Other provider will be added later.

@@ -685,6 +746,23 @@

FlaggerConfig you can also specify a private testloader in the Application.Spec.SyncPolicies.Rollout.TestLoader

+ + +config
+ + +Config + + + + +(Optional) +

ProviderConfig defines the helm configuration for the TrafficRoutingProvider. +You can pass in a custom helm configuration to install the TrafficRoutingProvider +And default value is in ./pkg/fleet-manager/manifests/plugins/ +Currently only used for Kuma and Nginx

+ + diff --git a/docs/proposals/rollout/Add_Gateway_Plugin_Support.md b/docs/proposals/rollout/Add_Gateway_Plugin_Support.md index b6d7a34fb..5fcb0002c 100644 --- a/docs/proposals/rollout/Add_Gateway_Plugin_Support.md +++ b/docs/proposals/rollout/Add_Gateway_Plugin_Support.md @@ -42,31 +42,34 @@ We will delve into the API design required to support these configurations. The type TrafficRoutingConfig struct { ... // for NGINX - // The default created ingress is as follows, (replace app.example.com with your own domain, and change the path matching rules as needed) + // The default created ingress is as follows, (Fill in `host` with your own domain) + // ```yaml // apiVersion: networking.k8s.io/v1 // kind: Ingress // metadata: - // name: application.syncPolicies.rollout.name - // namespace: application.syncPolicies.rollout.namespace + // name: nginx + // namespace: application.syncPolicies.rollout.workload.namespace // labels: - // app: application.syncPolicies.rollout.name + // app: application.syncPolicies.rollout.ServiceName // annotations: // kubernetes.io/ingress.class: "nginx" // spec: // rules: - // - host: "app.example.com" + // - host: "" // http: // paths: // - pathType: Prefix // path: "/" // backend: // service: - // name: application.syncPolicies.rollout.name + // name: application.syncPolicies.rollout.ServiceName // port: - // number: 80 + // number: application.syncPolicies.rollout.port + // ``` // +optional - Ingress *ingressv1.IngressRule `json:"ingress,omitempty"` - // for Kuma + Host string `json:"host,omitempty"` + + // for kuma // Defaults to http // +optional Protocol string `json:"protocol,omitempty"` diff --git a/examples/rollout/ab-testingNginx.yaml b/examples/rollout/ab-testingNginx.yaml new file mode 100644 index 000000000..f9d29e65c --- /dev/null +++ b/examples/rollout/ab-testingNginx.yaml @@ -0,0 +1,82 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: abtesting-nginx-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: nginx + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + analysisTimes: 3 + timeoutSeconds: 60 + host: "app.example.com" + match: + - headers: + x-canary: + exact: "insider" + - headers: + cookie: + exact: "canary" + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: nginx-request-success-rate + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://ingress-nginx-flagger-kurator-member-prometheus.ingress-nginx:9090 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://app.example.com/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s diff --git a/examples/rollout/blue_greenNginx.yaml b/examples/rollout/blue_greenNginx.yaml new file mode 100644 index 000000000..05f236426 --- /dev/null +++ b/examples/rollout/blue_greenNginx.yaml @@ -0,0 +1,75 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: blue-green-nginx-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: nginx + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + analysisTimes: 3 + timeoutSeconds: 60 + host: "app.example.com" + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: nginx-request-success-rate + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://ingress-nginx-flagger-kurator-member-prometheus.ingress-nginx:9090 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://app.example.com/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s diff --git a/examples/rollout/canaryKuma.yaml b/examples/rollout/canaryKuma.yaml new file mode 100644 index 000000000..30c3a0c6a --- /dev/null +++ b/examples/rollout/canaryKuma.yaml @@ -0,0 +1,76 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: rollout-kuma-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: kuma + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + timeoutSeconds: 60 + canaryStrategy: + maxWeight: 50 + stepWeight: 10 + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: kuma-request-success-rate + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://prometheus-server.mesh-observability:80 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s \ No newline at end of file diff --git a/examples/rollout/canaryNginx.yaml b/examples/rollout/canaryNginx.yaml new file mode 100644 index 000000000..e1f39e493 --- /dev/null +++ b/examples/rollout/canaryNginx.yaml @@ -0,0 +1,77 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: rollout-nginx-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: nginx + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + timeoutSeconds: 60 + canaryStrategy: + maxWeight: 50 + stepWeight: 10 + host: "app.example.com" + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: nginx-request-success-rate + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://ingress-nginx-flagger-kurator-member-prometheus.ingress-nginx:9090 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://app.example.com/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s \ No newline at end of file diff --git a/examples/rollout/canaryWithCustomMetric.yaml b/examples/rollout/canaryWithCustomMetric.yaml index e30ba67d7..f7b51fe02 100644 --- a/examples/rollout/canaryWithCustomMetric.yaml +++ b/examples/rollout/canaryWithCustomMetric.yaml @@ -47,14 +47,14 @@ spec: intervalSeconds: 90 thresholdRange: min: 99 - - name: my-metric + - name: istio-request-success-rate intervalSeconds: 90 thresholdRange: max: 99 customMetric: provider: type: prometheus - address: http://flagger-prometheus.ingress-nginx:9090 + address: http://prometheus.istio-system:9090 query: | sum( rate( diff --git a/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml b/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml index ee74280a4..ced86e244 100644 --- a/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml +++ b/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml @@ -1618,6 +1618,36 @@ spec: type: object type: object type: object + host: + description: |- + Host defines the domain name you specify. + Only for NGINX + Fill in the host field with your own domain as the host of the Ingress. + The actual created Ingress is as follows: + ```yaml + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: nginx + namespace: application.syncPolicies.rollout.workload.namespace + labels: + app: application.syncPolicies.rollout.ServiceName + annotations: + kubernetes.io/ingress.class: "nginx" + spec: + rules: + - host: "" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: application.syncPolicies.rollout.ServiceName + port: + number: application.syncPolicies.rollout.port + ``` + type: string hosts: description: Defaults to the RolloutConfig.ServiceName items: @@ -1923,6 +1953,12 @@ spec: type: object type: object type: array + protocol: + description: |- + Protocol defines the protocol used by Kuma + Only for kuma + Defaults to http + type: string retries: description: |- Retries describes the retry policy to use when a HTTP request fails. @@ -1978,7 +2014,7 @@ spec: trafficRoutingProvider: description: |- TrafficRoutingProvider defines traffic routing provider. - Kurator only supports istio for now. + Kurator supports istio,kuma,nginx for now. Other provider will be added later. type: string workload: @@ -2005,6 +2041,7 @@ spec: - port - rolloutPolicy - serviceName + - trafficRoutingProvider - workload type: object type: object diff --git a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml index a9d39296e..26ca859df 100644 --- a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml +++ b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml @@ -2494,6 +2494,47 @@ spec: Default value depends on the kind of the component. type: string type: object + config: + description: |- + ProviderConfig defines the helm configuration for the TrafficRoutingProvider. + You can pass in a custom helm configuration to install the TrafficRoutingProvider + And default value is in `./pkg/fleet-manager/manifests/plugins/` + Currently only used for Kuma and Nginx + properties: + chart: + description: |- + Chart defines the helm chart config of the TrafficRoutingProvider. + For Example, using the following configuration to change the version of nginx installed. + ```yaml + repo: https://kubernetes.github.io/ingress-nginx + version: 4.10.0 + ``` + properties: + name: + description: |- + Name defines the name of the chart. + Default value depends on the kind of the component. + type: string + repository: + description: |- + Repository defines the repository of chart. + Default value depends on the kind of the component. + type: string + version: + description: |- + Version defines the version of the chart. + Default value depends on the kind of the component. + type: string + type: object + extraArgs: + description: "ExtraArgs is the set of extra arguments + for TrafficRoutingProvider's chart.\nYou can pass in + values according to your needs.\nFor Example, using + the following configuration to change the port that + Prometheus uses to scrape metrics.\n```yaml\nvalues:\n\tcontroller:\n\t + \ podAnnotations:\n\t\tprometheus.io/port: 10378\n```" + x-kubernetes-preserve-unknown-fields: true + type: object extraArgs: description: |- ExtraArgs is the set of extra arguments for flagger chart. @@ -2517,8 +2558,12 @@ spec: TrafficRoutingProvider defines traffic routing provider. And Kurator will install flagger in trafficRoutingProvider's namespace For example, If you use `istio` as a provider, flager will be installed in istio's namespace `istio-system`. + And if you use `istio` as a provider, you need to install it manually. + Otherwise, you can configure it in ProviderConfig (or use the default configuration) and Kurator will automatically deploy it. Other provider will be added later. type: string + required: + - trafficRoutingProvider type: object grafana: description: Grafana defines the configuration for the grafana diff --git a/manifests/charts/fleet-manager/templates/webhooks.yaml b/manifests/charts/fleet-manager/templates/webhooks.yaml index b2c4fc76d..1fc15926b 100644 --- a/manifests/charts/fleet-manager/templates/webhooks.yaml +++ b/manifests/charts/fleet-manager/templates/webhooks.yaml @@ -28,3 +28,34 @@ webhooks: resources: - applications sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/kurator-serving-cert + creationTimestamp: null + name: fleet-manager-mutating-webhook-configuration +webhooks: + - admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: kurator-webhook-service-fleet + namespace: {{ .Release.Namespace }} + path: /mutate-apps-kurator-dev-v1alpha1-application # do not change this + failurePolicy: Fail + matchPolicy: Equivalent + name: mutation.application.apps.kurator.dev + rules: + - apiGroups: + - apps.kurator.dev + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - applications + sideEffects: None diff --git a/pkg/apis/apps/v1alpha1/types.go b/pkg/apis/apps/v1alpha1/types.go index 43bbbaf9b..a65ccf48e 100644 --- a/pkg/apis/apps/v1alpha1/types.go +++ b/pkg/apis/apps/v1alpha1/types.go @@ -23,6 +23,7 @@ import ( kustomizev1beta2 "github.com/fluxcd/kustomize-controller/api/v1beta2" sourcev1beta2 "github.com/fluxcd/source-controller/api/v1beta2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" ) // +genclient @@ -117,10 +118,9 @@ type RolloutConfig struct { TestLoader *bool `json:"testLoader,omitempty"` // TrafficRoutingProvider defines traffic routing provider. - // Kurator only supports istio for now. + // Kurator supports istio,kuma,nginx for now. // Other provider will be added later. - // +optional - TrafficRoutingProvider string `json:"trafficRoutingProvider,omitempty"` + TrafficRoutingProvider fleetapi.Provider `json:"trafficRoutingProvider"` // Workload specifies what workload to deploy the test to. // Workload of type Deployment or DaemonSet. @@ -239,6 +239,42 @@ type TrafficRoutingConfig struct { // +optional CorsPolicy *istiov1alpha3.CorsPolicy `json:"corsPolicy,omitempty"` + // Host defines the domain name you specify. + // Only for NGINX + // Fill in the host field with your own domain as the host of the Ingress. + // The actual created Ingress is as follows: + // ```yaml + // apiVersion: networking.k8s.io/v1 + // kind: Ingress + // metadata: + // name: nginx + // namespace: application.syncPolicies.rollout.workload.namespace + // labels: + // app: application.syncPolicies.rollout.ServiceName + // annotations: + // kubernetes.io/ingress.class: "nginx" + // spec: + // rules: + // - host: "" + // http: + // paths: + // - pathType: Prefix + // path: "/" + // backend: + // service: + // name: application.syncPolicies.rollout.ServiceName + // port: + // number: application.syncPolicies.rollout.port + // ``` + // +optional + Host string `json:"host,omitempty"` + + // Protocol defines the protocol used by Kuma + // Only for kuma + // Defaults to http + // +optional + Protocol string `json:"protocol,omitempty"` + // CanaryStrategy defines parameters for Canary Deployment. // Note: Kurator determines A/B Testing, Blue/Green Deployment, or Canary Deployment // based on the presence of content in the canaryStrategy field. diff --git a/pkg/apis/fleet/v1alpha1/types.go b/pkg/apis/fleet/v1alpha1/types.go index 80ae44a2b..3cb9f0229 100644 --- a/pkg/apis/fleet/v1alpha1/types.go +++ b/pkg/apis/fleet/v1alpha1/types.go @@ -562,13 +562,42 @@ type FlaggerConfig struct { // TrafficRoutingProvider defines traffic routing provider. // And Kurator will install flagger in trafficRoutingProvider's namespace // For example, If you use `istio` as a provider, flager will be installed in istio's namespace `istio-system`. + // And if you use `istio` as a provider, you need to install it manually. + // Otherwise, you can configure it in ProviderConfig (or use the default configuration) and Kurator will automatically deploy it. // Other provider will be added later. - // +optional - TrafficRoutingProvider Provider `json:"trafficRoutingProvider,omitempty"` + TrafficRoutingProvider Provider `json:"trafficRoutingProvider"` // PublicTestloader defines whether to install the publictestloader or not. // In addition to the public testloader you can configure here, // you can also specify a private testloader in the Application.Spec.SyncPolicies.Rollout.TestLoader PublicTestloader bool `json:"publicTestloader,omitempty"` + // ProviderConfig defines the helm configuration for the TrafficRoutingProvider. + // You can pass in a custom helm configuration to install the TrafficRoutingProvider + // And default value is in `./pkg/fleet-manager/manifests/plugins/` + // Currently only used for Kuma and Nginx + // +optional + ProviderConfig *Config `json:"config,omitempty"` +} + +type Config struct { + // Chart defines the helm chart config of the TrafficRoutingProvider. + // For Example, using the following configuration to change the version of nginx installed. + // ```yaml + // repo: https://kubernetes.github.io/ingress-nginx + // version: 4.10.0 + // ``` + // +optional + Chart *ChartConfig `json:"chart,omitempty"` + // ExtraArgs is the set of extra arguments for TrafficRoutingProvider's chart. + // You can pass in values according to your needs. + // For Example, using the following configuration to change the port that Prometheus uses to scrape metrics. + // ```yaml + // values: + // controller: + // podAnnotations: + // prometheus.io/port: 10378 + // ``` + // +optional + ExtraArgs apiextensionsv1.JSON `json:"extraArgs,omitempty"` } type SubMarinerOperatorConfig struct { @@ -624,6 +653,8 @@ type Provider string const ( Istio Provider = "istio" + Kuma Provider = "kuma" + Nginx Provider = "nginx" ) // FleetStatus defines the observed state of the fleet diff --git a/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go index e3c148740..a25d40220 100644 --- a/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go @@ -105,6 +105,28 @@ func (in *ChartConfig) DeepCopy() *ChartConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Config) DeepCopyInto(out *Config) { + *out = *in + if in.Chart != nil { + in, out := &in.Chart, &out.Chart + *out = new(ChartConfig) + **out = **in + } + in.ExtraArgs.DeepCopyInto(&out.ExtraArgs) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. +func (in *Config) DeepCopy() *Config { + if in == nil { + return nil + } + out := new(Config) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Device) DeepCopyInto(out *Device) { *out = *in @@ -220,6 +242,11 @@ func (in *FlaggerConfig) DeepCopyInto(out *FlaggerConfig) { **out = **in } in.ExtraArgs.DeepCopyInto(&out.ExtraArgs) + if in.ProviderConfig != nil { + in, out := &in.ProviderConfig, &out.ProviderConfig + *out = new(Config) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/client/client.go b/pkg/client/client.go index 2b445831c..f68f41fb9 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -33,6 +33,7 @@ import ( helmclient "helm.sh/helm/v3/pkg/kube" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + ingressv1 "k8s.io/api/networking/v1" crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -87,6 +88,9 @@ func NewClient(rest genericclioptions.RESTClientGetter) (*Client, error) { if err := appsv1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("failed to add appv1 api to scheme: %v", err) } + if err := ingressv1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to add ingress api to scheme: %v", err) + } // create controller-runtime client with scheme ctrlRuntimeClient, err := client.New(c, client.Options{Scheme: scheme}) if err != nil { diff --git a/pkg/fleet-manager/application/rollout_helper.go b/pkg/fleet-manager/application/rollout_helper.go index cbdff8fe2..ec197dfbb 100644 --- a/pkg/fleet-manager/application/rollout_helper.go +++ b/pkg/fleet-manager/application/rollout_helper.go @@ -19,12 +19,15 @@ package application import ( "context" "fmt" + "strings" "time" flaggerv1b1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" "github.com/pkg/errors" + "istio.io/istio/pkg/util/sets" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + ingressv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -43,13 +46,22 @@ import ( const ( // kurator rollout labels RolloutIdentifier = "kurator.dev/rollout" - sidecarInject = "istio-injection" - + istioInject = "istio-injection" + kumaInject = "kuma.io/sidecar-injection" // StatusSyncInterval specifies the interval for requeueing when synchronizing status. It determines how frequently the status should be checked and updated. StatusSyncInterval = 30 * time.Second currentClusterKind = "currentCluster" currentClusterName = "host" + // resources config + ingressAPIVersion = "networking.k8s.io/v1" + ingressKind = "Ingress" + ingressName = "nginx" + ingressLabelKey = "app" + ingressAnnotationKey = "kubernetes.io/ingress.class" + ingressAnnotationValue = "nginx" + + kumaAnnotation = "9898.service.kuma.io/protocol" ) func (a *ApplicationManager) fetchRolloutClusters(ctx context.Context, @@ -115,18 +127,37 @@ func (a *ApplicationManager) syncRolloutPolicyForCluster(ctx context.Context, annotation := map[string]string{ RolloutIdentifier: policyName, } + provider := rolloutPolicy.TrafficRoutingProvider for clusterKey, fleetCluster := range destinationClusters { fleetClusterClient := fleetCluster.Client.CtrlRuntimeClient() - - // If the trafficRoutingProvider is Istio, add the sidecar injection label to the workload's namespace. - if rolloutPolicy.TrafficRoutingProvider == "istio" { - err := enableIstioSidecarInjection(ctx, fleetClusterClient, rolloutPolicy.Workload.Namespace) + switch provider { + // If the trafficRoutingProvider is Istio or Kuma, add the sidecar injection label/Annotations to the workload's namespace. + case fleetapi.Istio, fleetapi.Kuma: + err := enableSidecarInjection(ctx, fleetClusterClient, rolloutPolicy.Workload.Namespace, provider, annotation) if err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to set namespace %s istio-injection enable", rolloutPolicy.Workload.Namespace) + return ctrl.Result{}, errors.Wrapf(err, "failed to set namespace %s %s's sidecar inject enable", rolloutPolicy.Workload.Namespace, provider) } - } + case fleetapi.Nginx: + // Canaries in the same namespace reference the same ingress + ingress := &ingressv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: ingressName, + Namespace: rolloutPolicy.Workload.Namespace, + }, + } + result, err := controllerutil.CreateOrUpdate(ctx, fleetClusterClient, ingress, func() error { + ingress.SetAnnotations(annotation) + return renderNginxIngress(ingress, rolloutPolicy) + }) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to operate ingress") + } + log.Info("sync nginx", "result:", result) + default: + return ctrl.Result{}, errors.Errorf("unknown provider type %s", provider) + } // if delete private testloader when rollout polity has changed if rolloutPolicy.TestLoader == nil || !*rolloutPolicy.TestLoader { testloaderDeploy := &appsv1.Deployment{} @@ -270,6 +301,10 @@ func (a *ApplicationManager) deleteResourcesInMemberClusters(ctx context.Context } for _, cluster := range destinationClusters { newClient := cluster.Client.CtrlRuntimeClient() + + if err := deleteIngressCreatedByKurator(ctx, newClient, rolloutPolicy); err != nil { + return err + } testloaderDeploy := &appsv1.Deployment{} if err := deleteResourceCreatedByKurator(ctx, testloaderNamespaceName, newClient, testloaderDeploy); err != nil { return errors.Wrapf(err, "failed to delete testloader deployment") @@ -291,7 +326,7 @@ func (a *ApplicationManager) deleteResourcesInMemberClusters(ctx context.Context return nil } -func enableIstioSidecarInjection(ctx context.Context, kubeClient client.Client, namespace string) error { +func enableSidecarInjection(ctx context.Context, kubeClient client.Client, namespace string, provider fleetapi.Provider, annotation map[string]string) error { log := ctrl.LoggerFrom(ctx) ns := &corev1.Namespace{} @@ -303,9 +338,13 @@ func enableIstioSidecarInjection(ctx context.Context, kubeClient client.Client, // if no found, create a namespace if apierrors.IsNotFound(err) { ns.SetName(namespace) - ns.SetLabels(map[string]string{ - sidecarInject: "enabled", - }) + switch provider { + case fleetapi.Kuma: + annotation[kumaInject] = "enabled" + case fleetapi.Istio: + ns.SetLabels(map[string]string{istioInject: "enabled"}) + } + ns.SetAnnotations(annotation) if createErr := kubeClient.Create(ctx, ns); createErr != nil { return errors.Wrapf(createErr, "failed to create namespace %s", namespacedName.Namespace) } @@ -314,8 +353,21 @@ func enableIstioSidecarInjection(ctx context.Context, kubeClient client.Client, return err } } else { - ns := addLabels(ns, sidecarInject, "enabled") - if updateErr := kubeClient.Update(ctx, ns); updateErr != nil { + var newNs client.Object + switch provider { + case fleetapi.Kuma: + newNs, err = addAnnotations(ns, map[string]string{ + kumaInject: "enabled", + }) + if err != nil { + return err + } + case fleetapi.Istio: + if newNs, err = addLabels(ns, map[string]string{istioInject: "enabled"}); err != nil { + return err + } + } + if updateErr := kubeClient.Update(ctx, newNs); updateErr != nil { return errors.Wrapf(updateErr, "failed to update namespace %s", namespacedName.Namespace) } } @@ -397,6 +449,94 @@ func deleteResourceCreatedByKurator(ctx context.Context, namespaceName types.Nam return nil } +func deleteIngressCreatedByKurator(ctx context.Context, kubeClient client.Client, rollout *applicationapi.RolloutConfig) error { + ingress := &ingressv1.Ingress{} + namespaceName := types.NamespacedName{ + Namespace: rollout.Workload.Namespace, + Name: ingressName, + } + if err := kubeClient.Get(ctx, namespaceName, ingress); err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to get ingress %s in %s", namespaceName.Name, namespaceName.Namespace) + } + } else { + // verify if the ingress were created by kurator + annotations := ingress.GetAnnotations() + if _, exist := annotations[RolloutIdentifier]; exist { + labels := ingress.GetLabels() + if set := sets.New(strings.Split(labels[ingressLabelKey], ",")...); set.Contains(rollout.ServiceName) { + if set.Len() == 1 { + if deleteErr := kubeClient.Delete(ctx, ingress); deleteErr != nil { + return errors.Wrapf(deleteErr, "failed to Delete ingress %s in %s", namespaceName.Name, namespaceName.Namespace) + } + } else { + // There are still other canaries using this ingress + if err := updateIngressRulesAndLabels(ctx, kubeClient, ingress, rollout, namespaceName, set); err != nil { + return err + } + } + } + } + } + return nil +} + +func updateIngressRulesAndLabels(ctx context.Context, kubeClient client.Client, ingress *ingressv1.Ingress, rollout *applicationapi.RolloutConfig, namespaceName types.NamespacedName, set sets.Set[string]) error { + newRules := make([]ingressv1.IngressRule, 0) + for _, rule := range ingress.Spec.Rules { + if rule.Host != rollout.RolloutPolicy.TrafficRouting.Host { + newRules = append(newRules, rule) + } + } + ingress.Spec.Rules = newRules + + labels := ingress.GetLabels() + labels[ingressLabelKey] = strings.Join(set.Delete(rollout.ServiceName).UnsortedList(), ",") + ingress.SetLabels(labels) + + if err := kubeClient.Update(ctx, ingress); err != nil { + return errors.Wrapf(err, "failed to Update ingress %s in %s", namespaceName.Name, namespaceName.Namespace) + } + return nil +} + +// create/update ingress configuration +func renderNginxIngress(ingress *ingressv1.Ingress, rollout *applicationapi.RolloutConfig) error { + if labels := ingress.GetLabels(); labels == nil || labels[ingressLabelKey] == "" { + ingress.SetLabels(map[string]string{ingressLabelKey: rollout.ServiceName}) + } else { + labels[ingressLabelKey] = strings.Join(sets.New(strings.Split(labels[ingressLabelKey], ",")...).Insert(rollout.ServiceName).UnsortedList(), ",") + ingress.SetLabels(labels) + } + if _, err := addAnnotations(ingress, map[string]string{ + ingressAnnotationKey: ingressAnnotationValue, + }); err != nil { + return err + } + Prefix := ingressv1.PathTypePrefix + rule := ingressv1.IngressRule{ + Host: rollout.RolloutPolicy.TrafficRouting.Host, + IngressRuleValue: ingressv1.IngressRuleValue{ + HTTP: &ingressv1.HTTPIngressRuleValue{ + Paths: []ingressv1.HTTPIngressPath{{ + PathType: &Prefix, + Path: "/", + Backend: ingressv1.IngressBackend{ + Service: &ingressv1.IngressServiceBackend{ + Name: rollout.ServiceName, + Port: ingressv1.ServiceBackendPort{ + Number: rollout.Port, + }, + }, + }, + }}, + }, + }, + } + ingress.Spec.Rules = append(ingress.Spec.Rules, rule) + return nil +} + // create/update canary configuration func renderCanary(rolloutPolicy applicationapi.RolloutConfig, canaryInCluster *flaggerv1b1.Canary) *flaggerv1b1.Canary { canaryInCluster.ObjectMeta.Namespace = rolloutPolicy.Workload.Namespace @@ -404,7 +544,7 @@ func renderCanary(rolloutPolicy applicationapi.RolloutConfig, canaryInCluster *f canaryInCluster.TypeMeta.Kind = "Canary" canaryInCluster.TypeMeta.APIVersion = "flagger.app/v1beta1" canaryInCluster.Spec = flaggerv1b1.CanarySpec{ - Provider: rolloutPolicy.TrafficRoutingProvider, + Provider: string(rolloutPolicy.TrafficRoutingProvider), TargetRef: flaggerv1b1.LocalObjectReference{ APIVersion: rolloutPolicy.Workload.APIVersion, Kind: rolloutPolicy.Workload.Kind, @@ -415,7 +555,16 @@ func renderCanary(rolloutPolicy applicationapi.RolloutConfig, canaryInCluster *f RevertOnDeletion: rolloutPolicy.RolloutPolicy.RevertOnDeletion, Suspend: rolloutPolicy.RolloutPolicy.Suspend, } - + switch rolloutPolicy.TrafficRoutingProvider { + case fleetapi.Nginx: + canaryInCluster.Spec.IngressRef = &flaggerv1b1.LocalObjectReference{ + APIVersion: ingressAPIVersion, + Kind: ingressKind, + Name: ingressName, + } + case fleetapi.Kuma: + canaryInCluster.SetAnnotations(map[string]string{"kuma.io/mesh": "default"}) + } return canaryInCluster } @@ -425,17 +574,22 @@ func renderCanaryService(rolloutPolicy applicationapi.RolloutConfig, service *co } ports := service.Spec.Ports canaryService := &flaggerv1b1.CanaryService{ - Name: rolloutPolicy.ServiceName, - Port: rolloutPolicy.Port, - Gateways: rolloutPolicy.RolloutPolicy.TrafficRouting.Gateways, - Hosts: rolloutPolicy.RolloutPolicy.TrafficRouting.Hosts, - Retries: rolloutPolicy.RolloutPolicy.TrafficRouting.Retries, - Headers: rolloutPolicy.RolloutPolicy.TrafficRouting.Headers, - CorsPolicy: rolloutPolicy.RolloutPolicy.TrafficRouting.CorsPolicy, - Primary: (*flaggerv1b1.CustomMetadata)(rolloutPolicy.Primary), - Canary: (*flaggerv1b1.CustomMetadata)(rolloutPolicy.Preview), + Name: rolloutPolicy.ServiceName, + Port: rolloutPolicy.Port, + } + switch rolloutPolicy.TrafficRoutingProvider { + case fleetapi.Istio: + canaryService.Gateways = rolloutPolicy.RolloutPolicy.TrafficRouting.Gateways + canaryService.Hosts = rolloutPolicy.RolloutPolicy.TrafficRouting.Hosts + canaryService.Retries = rolloutPolicy.RolloutPolicy.TrafficRouting.Retries + canaryService.Headers = rolloutPolicy.RolloutPolicy.TrafficRouting.Headers + canaryService.CorsPolicy = rolloutPolicy.RolloutPolicy.TrafficRouting.CorsPolicy + case fleetapi.Kuma: + annotations := &flaggerv1b1.CustomMetadata{Annotations: map[string]string{kumaAnnotation: rolloutPolicy.RolloutPolicy.TrafficRouting.Protocol}} + canaryService.Apex = annotations + canaryService.Canary = annotations + canaryService.Primary = annotations } - Timeout := fmt.Sprintf("%d", rolloutPolicy.RolloutPolicy.TrafficRouting.TimeoutSeconds) + "s" canaryService.Timeout = Timeout @@ -555,18 +709,41 @@ func generateWebhookUrl(name, namespace string) string { return url } -func addLabels(obj client.Object, key, value string) client.Object { - labels := obj.GetLabels() - // prevent nil pointer panic - if labels == nil { - obj.SetLabels(map[string]string{ - key: value, - }) - return obj - } - labels[key] = value - obj.SetLabels(labels) - return obj +func addLabels(obj client.Object, labels map[string]string) (client.Object, error) { + existingLabels := obj.GetLabels() + if existingLabels == nil { + obj.SetLabels(labels) + } else { + for k, v := range labels { + if value, exists := existingLabels[k]; exists { + if value != v { + return nil, errors.New("label key conflict") + } + } else { + existingLabels[k] = v + } + } + obj.SetLabels(existingLabels) + } + return obj, nil +} +func addAnnotations(obj client.Object, ann map[string]string) (client.Object, error) { + annotations := obj.GetAnnotations() + if annotations == nil { + obj.SetAnnotations(ann) + } else { + for k, v := range ann { + if value, exists := annotations[k]; exists { + if value != v { + return nil, errors.New("annotation key conflict") + } + } else { + annotations[k] = v + } + } + obj.SetAnnotations(annotations) + } + return obj, nil } func mergeMap(map1, map2 map[string]*applicationapi.RolloutStatus) map[string]*applicationapi.RolloutStatus { diff --git a/pkg/fleet-manager/application/rollout_helper_test.go b/pkg/fleet-manager/application/rollout_helper_test.go index 43bd83e55..640a78a11 100644 --- a/pkg/fleet-manager/application/rollout_helper_test.go +++ b/pkg/fleet-manager/application/rollout_helper_test.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" applicationapi "kurator.dev/kurator/pkg/apis/apps/v1alpha1" + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" ) func generateRolloutPolicy(installPrivateTestloader *bool) applicationapi.RolloutConfig { @@ -39,7 +40,7 @@ func generateRolloutPolicy(installPrivateTestloader *bool) applicationapi.Rollou rolloutPolicy := applicationapi.RolloutConfig{ TestLoader: installPrivateTestloader, - TrafficRoutingProvider: "istio", + TrafficRoutingProvider: fleetapi.Istio, Workload: &applicationapi.CrossNamespaceObjectReference{ APIVersion: "appv1/deployment", Kind: "Deployment", @@ -594,8 +595,7 @@ func Test_renderCanaryAnalysis(t *testing.T) { func Test_addLables(t *testing.T) { type args struct { obj client.Object - key string - value string + label map[string]string } tests := []struct { name string @@ -617,8 +617,9 @@ func Test_addLables(t *testing.T) { }, }, }, - key: "istio-injection", - value: "ebabled", + label: map[string]string{ + "istio-injection": "enabled", + }, }, want: &corev1.Namespace{ TypeMeta: metav1.TypeMeta{ @@ -629,7 +630,7 @@ func Test_addLables(t *testing.T) { Name: "webapp", Labels: map[string]string{ "xxx": "abc", - "istio-injection": "ebabled", + "istio-injection": "enabled", }, }, }, @@ -646,8 +647,7 @@ func Test_addLables(t *testing.T) { Name: "webapp", }, }, - key: "XXX", - value: "abc", + label: map[string]string{"XXX": "abc"}, }, want: &corev1.Namespace{ TypeMeta: metav1.TypeMeta{ @@ -665,7 +665,7 @@ func Test_addLables(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := addLabels(tt.args.obj, tt.args.key, tt.args.value); !reflect.DeepEqual(got, tt.want) { + if got, _ := addLabels(tt.args.obj, tt.args.label); !reflect.DeepEqual(got, tt.want) { t.Errorf("addLablesOrAnnotaions() = %v, want %v", got, tt.want) } }) diff --git a/pkg/fleet-manager/fleet_plugin.go b/pkg/fleet-manager/fleet_plugin.go index f1833d525..f8ca1395b 100644 --- a/pkg/fleet-manager/fleet_plugin.go +++ b/pkg/fleet-manager/fleet_plugin.go @@ -40,6 +40,8 @@ const ( ) func (f *FleetManager) reconcilePlugins(ctx context.Context, fleet *fleetapi.Fleet, fleetClusters map[ClusterKey]*FleetCluster) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + log = log.WithValues("fleet", types.NamespacedName{Name: fleet.Name, Namespace: fleet.Namespace}) var resources kube.ResourceList if fleet.Spec.Plugin != nil { @@ -59,6 +61,7 @@ func (f *FleetManager) reconcilePlugins(ctx context.Context, fleet *fleetapi.Fle f.reconcileDistributedStoragePlugin, f.reconcileFlaggerPlugin, f.reconcileSubmarinerPlugin, + f.reconcileProviderPlugin, } resultsChannel := make(chan reconcileResult, len(funcs)) @@ -102,6 +105,7 @@ func (f *FleetManager) reconcilePlugins(ctx context.Context, fleet *fleetapi.Fle } } + log.Info("All plugin Resources succeed") return f.reconcilePluginResources(ctx, fleet, resources) } diff --git a/pkg/fleet-manager/fleet_plugin_provider.go b/pkg/fleet-manager/fleet_plugin_provider.go new file mode 100644 index 000000000..1b944a73e --- /dev/null +++ b/pkg/fleet-manager/fleet_plugin_provider.go @@ -0,0 +1,75 @@ +/* +Copyright Kurator Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "time" + + "helm.sh/helm/v3/pkg/kube" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" + "kurator.dev/kurator/pkg/fleet-manager/plugin" + "kurator.dev/kurator/pkg/infra/util" +) + +// reconcileProviderPlugin reconciles the Provider plugin. +func (f *FleetManager) reconcileProviderPlugin(ctx context.Context, fleet *fleetapi.Fleet, fleetClusters map[ClusterKey]*FleetCluster) (kube.ResourceList, ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + flaggerCfg := fleet.Spec.Plugin.Flagger + if flaggerCfg == nil || flaggerCfg.TrafficRoutingProvider == fleetapi.Istio { + // reconcilePluginResources will delete all resources if plugin is nil + return nil, ctrl.Result{}, nil + } + + fleetNN := types.NamespacedName{ + Namespace: fleet.Namespace, + Name: fleet.Name, + } + + fleetOwnerRef := ownerReference(fleet) + var resources kube.ResourceList + + for key, cluster := range fleetClusters { + b, err := plugin.RenderProvider(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{ + Name: key.Name, + SecretName: cluster.Secret, + SecretKey: cluster.SecretKey, + }, flaggerCfg) + if err != nil { + return nil, ctrl.Result{}, err + } + + // apply provider helm resources + providerResources, err := util.PatchResources(b) + if err != nil { + return nil, ctrl.Result{}, err + } + resources = append(resources, providerResources...) + } + + log.V(4).Info("wait for provider helm release to be reconciled") + if !f.helmReleaseReady(ctx, fleet, resources) { + // wait for HelmRelease to be ready + return nil, ctrl.Result{ + // HelmRelease check interval is 1m, so we set 30s here + RequeueAfter: 30 * time.Second, + }, nil + } + + return resources, ctrl.Result{}, nil +} diff --git a/pkg/fleet-manager/manifests/plugins/kuma.yaml b/pkg/fleet-manager/manifests/plugins/kuma.yaml new file mode 100644 index 000000000..75c9d95c1 --- /dev/null +++ b/pkg/fleet-manager/manifests/plugins/kuma.yaml @@ -0,0 +1,5 @@ +type: default +repo: https://kumahq.github.io/charts +name: kuma +version: 2.7.3 +targetNamespace: kuma-system diff --git a/pkg/fleet-manager/manifests/plugins/nginx.yaml b/pkg/fleet-manager/manifests/plugins/nginx.yaml new file mode 100644 index 000000000..b50aac0e1 --- /dev/null +++ b/pkg/fleet-manager/manifests/plugins/nginx.yaml @@ -0,0 +1,12 @@ +type: default +repo: https://kubernetes.github.io/ingress-nginx +name: ingress-nginx +version: 4.x +targetNamespace: ingress-nginx +values: + controller: + metrics: + enabled: true + podAnnotations: + prometheus.io/scrape: true + prometheus.io/port: 10254 \ No newline at end of file diff --git a/pkg/fleet-manager/plugin/plugin.go b/pkg/fleet-manager/plugin/plugin.go index 8b8627c17..86d96ff05 100644 --- a/pkg/fleet-manager/plugin/plugin.go +++ b/pkg/fleet-manager/plugin/plugin.go @@ -61,6 +61,8 @@ const ( var ProviderNamespace = map[fleetv1a1.Provider]string{ "istio": "istio-system", + "kuma": "kuma-system", + "nginx": "ingress-nginx", } type GrafanaDataSource struct { @@ -401,6 +403,14 @@ func RenderFlagger( c.TargetNamespace = ProviderNamespace[flaggerConfig.TrafficRoutingProvider] values, err := toMap(flaggerConfig.ExtraArgs) + if flaggerConfig.TrafficRoutingProvider == fleetv1a1.Nginx { + values = transform.MergeMaps(values, map[string]interface{}{ + "prometheus": map[string]interface{}{ + "install": true, + }, + "meshProvider": "nginx", + }) + } if err != nil { return nil, err } @@ -416,6 +426,41 @@ func RenderFlagger( }) } +func RenderProvider( + fsys fs.FS, + fleetNN types.NamespacedName, + fleetRef *metav1.OwnerReference, + cluster KubeConfigSecretRef, + flaggerConfig *fleetv1a1.FlaggerConfig, +) ([]byte, error) { + name := string(flaggerConfig.TrafficRoutingProvider) + // get and merge the chart config + c, err := getFleetPluginChart(fsys, name) + if err != nil { + return nil, err + } + + values := map[string]interface{}{} + if providerConfig := flaggerConfig.ProviderConfig; providerConfig != nil { + mergeChartConfig(c, providerConfig.Chart) + values, err = toMap(providerConfig.ExtraArgs) + if err != nil { + return nil, err + } + } + c.TargetNamespace = ProviderNamespace[flaggerConfig.TrafficRoutingProvider] + + return renderFleetPlugin(fsys, FleetPluginConfig{ + Name: name, + Component: name, + Fleet: fleetNN, + Cluster: &cluster, + OwnerReference: fleetRef, + Chart: *c, + Values: values, + }) +} + func RenderRolloutTestloader( fsys fs.FS, fleetNN types.NamespacedName, diff --git a/pkg/webhooks/application_webhook.go b/pkg/webhooks/application_webhook.go index af01abb3c..c4e000de6 100644 --- a/pkg/webhooks/application_webhook.go +++ b/pkg/webhooks/application_webhook.go @@ -22,6 +22,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,9 +30,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "kurator.dev/kurator/pkg/apis/apps/v1alpha1" + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" ) var _ webhook.CustomValidator = &ApplicationWebhook{} +var _ webhook.CustomDefaulter = &ApplicationWebhook{} type ApplicationWebhook struct { Client client.Reader @@ -41,11 +44,14 @@ func (wh *ApplicationWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&v1alpha1.Application{}). WithValidator(wh). + WithDefaulter(wh). Complete() } -func (wh *ApplicationWebhook) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { +func (wh *ApplicationWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { in, ok := obj.(*v1alpha1.Application) + log := ctrl.LoggerFrom(ctx) + log.Info("All field Validate succeed") if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Application but got a %T", obj)) } @@ -131,3 +137,48 @@ func (wh *ApplicationWebhook) ValidateUpdate(_ context.Context, oldObj, newObj r func (wh *ApplicationWebhook) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { return nil, nil } + +func (wh *ApplicationWebhook) Default(ctx context.Context, obj runtime.Object) error { + log := ctrl.LoggerFrom(ctx) + log.Info("setting default") + app, ok := obj.(*v1alpha1.Application) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Application but got a %T", obj)) + } + if err := defaultApp(ctx, app); err != nil { + return err + } + return nil +} + +func defaultApp(ctx context.Context, app *v1alpha1.Application) error { + log := ctrl.LoggerFrom(ctx) + log = log.WithValues("application", types.NamespacedName{Name: app.Name, Namespace: app.Namespace}) + if err := defaultSyncPolicies(ctx, app.Spec.SyncPolicies); err != nil { + return err + } + log.Info("All field set default") + return nil +} + +func defaultSyncPolicies(ctx context.Context, SyncPolicies []*v1alpha1.ApplicationSyncPolicy) error { + log := ctrl.LoggerFrom(ctx) + log = log.WithValues("application") + for i, syncPoliciy := range SyncPolicies { + if syncPoliciy.Rollout == nil { + continue + } + switch syncPoliciy.Rollout.TrafficRoutingProvider { + case fleetapi.Nginx: + if syncPoliciy.Rollout.RolloutPolicy.TrafficRouting.Host == "" { + return apierrors.NewBadRequest(fmt.Sprintf("expected application.syncPolicies.Rollout.RolloutPolicy.TrafficRouting.Host in index %d", i)) + } + case fleetapi.Kuma: + if syncPoliciy.Rollout.RolloutPolicy.TrafficRouting.Protocol == "" { + syncPoliciy.Rollout.RolloutPolicy.TrafficRouting.Protocol = "http" + } + } + } + log.Info("set SyncPolicies default success") + return nil +}