From 9cc74876a123ae33d0da10d2a490c2dd430eca4b Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Thu, 30 Jan 2025 14:42:54 -0500 Subject: [PATCH 1/5] feat(meshfaultinjection): support GRPC protocol Signed-off-by: Lukasz Dziedziak --- .../policies/meshfaultinjection/plugin/v1alpha1/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugins/policies/meshfaultinjection/plugin/v1alpha1/plugin.go b/pkg/plugins/policies/meshfaultinjection/plugin/v1alpha1/plugin.go index 7365266e29b4..7ee81adbae01 100644 --- a/pkg/plugins/policies/meshfaultinjection/plugin/v1alpha1/plugin.go +++ b/pkg/plugins/policies/meshfaultinjection/plugin/v1alpha1/plugin.go @@ -181,7 +181,7 @@ func configure( protocol core_mesh.Protocol, ) error { switch protocol { - case core_mesh.ProtocolHTTP, core_mesh.ProtocolHTTP2: + case core_mesh.ProtocolHTTP, core_mesh.ProtocolHTTP2, core_mesh.ProtocolGRPC: for _, rule := range fromRules { conf := rule.Conf.(api.Conf) from := rule.Subset From 72c0c56e2dcd45fa19a9b3e1bd9a072408323c17 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Thu, 30 Jan 2025 15:14:43 -0500 Subject: [PATCH 2/5] test(e2e): speedup universal tests Signed-off-by: Lukasz Dziedziak --- test/e2e_env/universal/meshretry/grpc.go | 36 ++++++++------ test/e2e_env/universal/retry/retry.go | 63 ++++++++++-------------- 2 files changed, 46 insertions(+), 53 deletions(-) diff --git a/test/e2e_env/universal/meshretry/grpc.go b/test/e2e_env/universal/meshretry/grpc.go index 1565553383f4..80045f99c7f9 100644 --- a/test/e2e_env/universal/meshretry/grpc.go +++ b/test/e2e_env/universal/meshretry/grpc.go @@ -45,23 +45,27 @@ func GrpcRetry() { E2EAfterAll(func() { Expect(universal.Cluster.DeleteMeshApps(meshName)).To(Succeed()) - Expect(universal.Cluster.GetKumactlOptions().RunKumactl("delete", "dataplane", "fake-echo-server", "-m", meshName)).To(Succeed()) Expect(universal.Cluster.DeleteMesh(meshName)).To(Succeed()) }) It("should retry on GRPC connection failure", func() { - echoServerDataplane := fmt.Sprintf(` -type: Dataplane + faultInjection := fmt.Sprintf(` +type: MeshFaultInjection mesh: "%s" -name: fake-echo-server -networking: - address: 241.0.0.1 - inbound: - - port: 7777 - servicePort: 7777 - tags: - kuma.io/service: test-server - kuma.io/protocol: grpc +name: mesh-fault-injecton-500-grpc +spec: + targetRef: + kind: MeshService + name: test-server + from: + - targetRef: + kind: MeshService + name: test-client + default: + http: + - abort: + httpStatus: 500 + percentage: "50.0" `, meshName) meshRetryPolicy := fmt.Sprintf(` type: MeshRetry @@ -100,8 +104,8 @@ spec: g.Expect(grpcSuccessStats(g)).To(stats.BeGreaterThanZero()) }, "30s", "1s").Should(Succeed()) - By("Adding a faulty dataplane") - Expect(universal.Cluster.Install(YamlUniversal(echoServerDataplane))).To(Succeed()) + By("Adding a fault injection") + Expect(universal.Cluster.Install(YamlUniversal(faultInjection))).To(Succeed()) By("Clean counters") Expect(admin.ResetCounters()).To(Succeed()) @@ -112,7 +116,7 @@ spec: defer func() { lastFailureStats = failureStats.Stats[0] }() g.Expect(failureStats).To(stats.BeGreaterThanZero()) g.Expect(failureStats).To(stats.BeGreaterThan(lastFailureStats)) - }, "60s", "5s").MustPassRepeatedly(3).Should(Succeed()) + }, "30s", "1s").MustPassRepeatedly(3).Should(Succeed()) By("Apply a MeshRetry policy") Expect(universal.Cluster.Install(YamlUniversal(meshRetryPolicy))).To(Succeed()) @@ -127,6 +131,6 @@ spec: defer func() { lastFailureStats = failureStats.Stats[0] }() g.Expect(failureStats).To(Not(stats.BeGreaterThan(lastFailureStats))) g.Expect(grpcSuccessStats(g)).To(stats.BeGreaterThanZero()) - }, "30s", "5s").MustPassRepeatedly(3).Should(Succeed()) + }, "30s", "1s").MustPassRepeatedly(3).Should(Succeed()) }) } diff --git a/test/e2e_env/universal/retry/retry.go b/test/e2e_env/universal/retry/retry.go index 32f84b53db37..ddc04d275a09 100644 --- a/test/e2e_env/universal/retry/retry.go +++ b/test/e2e_env/universal/retry/retry.go @@ -2,7 +2,6 @@ package retry import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,24 +35,25 @@ func Policy() { E2EAfterAll(func() { Expect(universal.Cluster.DeleteMeshApps(meshName)).To(Succeed()) - Expect(universal.Cluster.GetKumactlOptions().RunKumactl("delete", "dataplane", "fake-echo-server", "-m", meshName)).To(Succeed()) Expect(universal.Cluster.DeleteMesh(meshName)).To(Succeed()) }) It("should retry on HTTP connection failure", func() { - echoServerDataplane := fmt.Sprintf(` -type: Dataplane + fiPolicy := fmt.Sprintf(` +type: FaultInjection mesh: "%s" -name: fake-echo-server -networking: - address: 241.0.0.1 - inbound: - - port: 7777 - servicePort: 7777 - tags: - kuma.io/service: test-server - kuma.io/protocol: http -`, meshName) +name: fi-retry +sources: + - match: + kuma.io/service: demo-client +destinations: + - match: + kuma.io/service: test-server + kuma.io/protocol: http +conf: + abort: + httpStatus: 500 + percentage: 50`, meshName) retryPolicy := fmt.Sprintf(` type: Retry mesh: "%s" @@ -75,32 +75,22 @@ conf: universal.Cluster, "demo-client", "test-server.mesh", ) g.Expect(err).ToNot(HaveOccurred()) - }).Should(Succeed()) - Consistently(func(g Gomega) { - // -m 8 to wait for 8 seconds to beat the default 5s connect timeout - _, err := client.CollectEchoResponse( - universal.Cluster, "demo-client", "test-server.mesh", - client.WithMaxTime(8), - ) - g.Expect(err).ToNot(HaveOccurred()) - }).Should(Succeed()) + }, "30s", "1s").MustPassRepeatedly(3).Should(Succeed()) - By("Adding a faulty dataplane") - Expect(universal.Cluster.Install(YamlUniversal(echoServerDataplane))).To(Succeed()) + By("Adding a fault injection") + Expect(universal.Cluster.Install(YamlUniversal(fiPolicy))).To(Succeed()) By("Check some errors happen") - var errs []error - for i := 0; i < 50; i++ { - time.Sleep(time.Millisecond * 100) - _, err := client.CollectEchoResponse( + Eventually(func(g Gomega) { + response, err := client.CollectFailure( universal.Cluster, "demo-client", "test-server.mesh", - client.WithMaxTime(8), + client.NoFail(), + client.OutputFormat(`{ "received": { "status": %{response_code} } }`), ) - if err != nil { - errs = append(errs, err) - } - } - Expect(errs).ToNot(BeEmpty()) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(response.ResponseCode).To(Equal(500)) + }, "30s", "100ms").Should(Succeed()) By("Apply a retry policy") Expect(universal.Cluster.Install(YamlUniversal(retryPolicy))).To(Succeed()) @@ -109,9 +99,8 @@ conf: Eventually(func(g Gomega) { _, err := client.CollectEchoResponse( universal.Cluster, "demo-client", "test-server.mesh", - client.WithMaxTime(8), ) g.Expect(err).ToNot(HaveOccurred()) - }, "1m", "1s", MustPassRepeatedly(3)).Should(Succeed()) + }, "30s", "1s").MustPassRepeatedly(3).Should(Succeed()) }) } From fa698bd32bff9b46abfc23b9985a1995d3b2f676 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Thu, 30 Jan 2025 17:25:16 -0500 Subject: [PATCH 3/5] fix Signed-off-by: Lukasz Dziedziak --- test/e2e_env/universal/meshretry/grpc.go | 4 ++-- test/e2e_env/universal/retry/retry.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e_env/universal/meshretry/grpc.go b/test/e2e_env/universal/meshretry/grpc.go index 80045f99c7f9..32ab54e5eb7b 100644 --- a/test/e2e_env/universal/meshretry/grpc.go +++ b/test/e2e_env/universal/meshretry/grpc.go @@ -116,7 +116,7 @@ spec: defer func() { lastFailureStats = failureStats.Stats[0] }() g.Expect(failureStats).To(stats.BeGreaterThanZero()) g.Expect(failureStats).To(stats.BeGreaterThan(lastFailureStats)) - }, "30s", "1s").MustPassRepeatedly(3).Should(Succeed()) + }, "30s", "5s").Should(Succeed()) By("Apply a MeshRetry policy") Expect(universal.Cluster.Install(YamlUniversal(meshRetryPolicy))).To(Succeed()) @@ -131,6 +131,6 @@ spec: defer func() { lastFailureStats = failureStats.Stats[0] }() g.Expect(failureStats).To(Not(stats.BeGreaterThan(lastFailureStats))) g.Expect(grpcSuccessStats(g)).To(stats.BeGreaterThanZero()) - }, "30s", "1s").MustPassRepeatedly(3).Should(Succeed()) + }, "30s", "5s").Should(Succeed()) }) } diff --git a/test/e2e_env/universal/retry/retry.go b/test/e2e_env/universal/retry/retry.go index ddc04d275a09..b191be104b09 100644 --- a/test/e2e_env/universal/retry/retry.go +++ b/test/e2e_env/universal/retry/retry.go @@ -101,6 +101,6 @@ conf: universal.Cluster, "demo-client", "test-server.mesh", ) g.Expect(err).ToNot(HaveOccurred()) - }, "30s", "1s").MustPassRepeatedly(3).Should(Succeed()) + }, "60s", "1s").MustPassRepeatedly(3).Should(Succeed()) }) } From 7786cb180ad6b27bc7cbfaccbaf36020b5c2f543 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Fri, 31 Jan 2025 05:46:16 -0500 Subject: [PATCH 4/5] fix Signed-off-by: Lukasz Dziedziak --- test/e2e_env/universal/meshretry/grpc.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/e2e_env/universal/meshretry/grpc.go b/test/e2e_env/universal/meshretry/grpc.go index 32ab54e5eb7b..53f548f73e88 100644 --- a/test/e2e_env/universal/meshretry/grpc.go +++ b/test/e2e_env/universal/meshretry/grpc.go @@ -2,6 +2,7 @@ package meshretry import ( "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -28,8 +29,6 @@ func GrpcRetry() { WithProtocol("grpc"), WithTransparentProxy(true), )). - // remove default policies after https://github.com/kumahq/kuma/issues/3325 - Install(TrafficRouteUniversal(meshName)). Setup(universal.Cluster) Expect(err).ToNot(HaveOccurred()) @@ -64,7 +63,7 @@ spec: default: http: - abort: - httpStatus: 500 + httpStatus: 503 percentage: "50.0" `, meshName) meshRetryPolicy := fmt.Sprintf(` @@ -124,7 +123,7 @@ spec: By("Clean counters") Expect(admin.ResetCounters()).To(Succeed()) lastFailureStats = stats.StatItem{Name: "", Value: float64(0)} - + time.Sleep(1*time.Hour) By("Eventually all requests succeed consistently") Eventually(func(g Gomega) { failureStats := grpcFailureStats(g) From 0b5daea7f72371aa604f6f0afe01f60286c5ae5d Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Fri, 31 Jan 2025 06:07:54 -0500 Subject: [PATCH 5/5] remove wait Signed-off-by: Lukasz Dziedziak --- test/e2e_env/universal/meshretry/grpc.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/e2e_env/universal/meshretry/grpc.go b/test/e2e_env/universal/meshretry/grpc.go index 53f548f73e88..fb501f0a74f2 100644 --- a/test/e2e_env/universal/meshretry/grpc.go +++ b/test/e2e_env/universal/meshretry/grpc.go @@ -2,7 +2,6 @@ package meshretry import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -123,7 +122,7 @@ spec: By("Clean counters") Expect(admin.ResetCounters()).To(Succeed()) lastFailureStats = stats.StatItem{Name: "", Value: float64(0)} - time.Sleep(1*time.Hour) + By("Eventually all requests succeed consistently") Eventually(func(g Gomega) { failureStats := grpcFailureStats(g)