feat: make router-tests less flaky

.github/workflows/router-ci.yaml

@@ -156,15 +156,19 @@ jobs:
         with:
           cache-dependency-path: |
             router-tests/go.sum
-      - uses: nick-fields/retry@v3
+      - name: Run Integration tests
+        working-directory: ./router-tests
+        run: make test test_params="-run '^Test[^(Flaky)]' --timeout=5m --parallel 10"
+      - name: Run Flaky Integration tests
+        uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 5
           retry_wait_seconds: 5
           retry_on: error
           command: |
             cd router-tests
-            make test test_params="--timeout=5m"
+            make test test_params="-run '^TestFlaky' --timeout=5m -p 1 --parallel 1"
 
   image_scan:
     if: github.event.pull_request.head.repo.full_name == github.repository

demo/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/wundergraph/cosmo/composition-go v0.0.0-20240124120900-5effe48a4a1d
 	github.com/wundergraph/cosmo/router v0.0.0-20250119174948-4b991294658e
 	github.com/wundergraph/cosmo/router-tests v0.0.0-20241213115435-a249dba8c52a
-	github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.145
+	github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.146
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
 	go.opentelemetry.io/otel v1.28.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.23.1

demo/go.sum

@@ -312,8 +312,8 @@ github.com/wundergraph/cosmo/router v0.0.0-20250119174948-4b991294658e h1:ee4fu7
 github.com/wundergraph/cosmo/router v0.0.0-20250119174948-4b991294658e/go.mod h1:ImqCvxvvNOy1UxbuTnFtin/CDBFHoFqrZly3rC2z+e0=
 github.com/wundergraph/cosmo/router-tests v0.0.0-20241213115435-a249dba8c52a h1:GVLe85f5g+G0IOorDBBNTfm5Ua9DO0vuVY7ReSTOEbQ=
 github.com/wundergraph/cosmo/router-tests v0.0.0-20241213115435-a249dba8c52a/go.mod h1:I+SFviFnd3BHlPmYn+ckmzQyDB9+/c8RZJo4t6VQAds=
-github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.145 h1:3JuBmRux6YB/UZgh6COvgLXzQhMIsdHV7A02NsYdAVE=
-github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.145/go.mod h1:B7eV0Qh8Lop9QzIOQcsvKp3S0ejfC6mgyWoJnI917yQ=
+github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.146 h1:C9+jjMgbU/RJTiFGC0HNHan4LxrY7fIhmbZRoqZryLk=
+github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.146/go.mod h1:B7eV0Qh8Lop9QzIOQcsvKp3S0ejfC6mgyWoJnI917yQ=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

router-tests/cache_warmup_test.go

@@ -570,7 +570,8 @@ func TestCacheWarmup(t *testing.T) {
 	})
 }
 
-func TestCacheWarmupMetrics(t *testing.T) {
+// Is set as Flaky so that when running the tests it will be run separately and retried if it fails
+func TestFlakyCacheWarmupMetrics(t *testing.T) {
 	t.Run("should emit planning times metrics during warmup", func(t *testing.T) {
 		t.Parallel()
 

router-tests/complexity_limits_test.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"github.com/wundergraph/cosmo/router-tests/testenv"
@@ -152,6 +153,8 @@ func TestComplexityLimits(t *testing.T) {
 				require.Contains(t, testSpan.Attributes(), otel.WgQueryDepth.Int(3))
 				require.Contains(t, testSpan.Attributes(), otel.WgQueryDepthCacheHit.Bool(false))
 				exporter.Reset()
+				// wait to let cache get consistent
+				time.Sleep(100 * time.Millisecond)
 
 				failedRes2, _ := xEnv.MakeGraphQLRequest(testenv.GraphQLRequest{
 					Query: `{ employee(id:1) { id details { forename surname } } }`,
@@ -163,6 +166,8 @@ func TestComplexityLimits(t *testing.T) {
 				require.Contains(t, testSpan2.Attributes(), otel.WgQueryDepth.Int(3))
 				require.Contains(t, testSpan2.Attributes(), otel.WgQueryDepthCacheHit.Bool(true))
 				exporter.Reset()
+				// wait to let cache get consistent
+				time.Sleep(100 * time.Millisecond)
 
 				successRes := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{
 					Query: `query { employees { id } }`,
@@ -172,6 +177,8 @@ func TestComplexityLimits(t *testing.T) {
 				require.Contains(t, testSpan3.Attributes(), otel.WgQueryDepth.Int(2))
 				require.Contains(t, testSpan3.Attributes(), otel.WgQueryDepthCacheHit.Bool(false))
 				exporter.Reset()
+				// wait to let cache get consistent
+				time.Sleep(100 * time.Millisecond)
 
 				successRes2 := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{
 					Query: `query { employees { id } }`,

router-tests/config_hot_reload_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/wundergraph/cosmo/router/pkg/routerconfig"
 
 	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/wundergraph/cosmo/router-tests/testenv"
 	"github.com/wundergraph/cosmo/router/core"
@@ -238,28 +239,35 @@ func TestConfigHotReload(t *testing.T) {
 			},
 		}, func(t *testing.T, xEnv *testenv.Environment) {
 
-			var done atomic.Bool
-
+			var startedReq atomic.Bool
 			go func() {
-				defer done.Store(true)
-
+				startedReq.Store(true)
 				res, err := xEnv.MakeGraphQLRequestWithContext(context.Background(), testenv.GraphQLRequest{
 					Query: `{ employees { id } }`,
 				})
 				require.NoError(t, err)
-				require.Equal(t, res.Response.StatusCode, 200)
-				require.Equal(t, `{"errors":[{"message":"Failed to fetch from Subgraph 'employees'."}],"data":{"employees":null}}`, res.Body)
+				assert.Equal(t, res.Response.StatusCode, 200)
+				assert.Equal(t, `{"errors":[{"message":"Failed to fetch from Subgraph 'employees'."}],"data":{"employees":null}}`, res.Body)
 			}()
 
 			// Let's wait a bit to make sure all requests are in flight
 			// otherwise the shutdown will be too fast and the wait-group will not be done fully
+			require.Eventually(t, func() bool {
+				return startedReq.Load()
+			}, time.Second*10, time.Millisecond*100)
 			time.Sleep(time.Millisecond * 100)
 
-			xEnv.Shutdown()
+			var done atomic.Bool
+			go func() {
+				defer done.Store(true)
+
+				err := xEnv.Router.Shutdown(context.Background())
+				assert.ErrorContains(t, err, context.DeadlineExceeded.Error())
+			}()
 
 			require.Eventually(t, func() bool {
 				return done.Load()
-			}, time.Second*5, time.Millisecond*100)
+			}, time.Second*20, time.Millisecond*100)
 		})
 	})
 
@@ -314,13 +322,15 @@ func TestConfigHotReload(t *testing.T) {
 
 			// Swap config
 			require.NoError(t, pm.updateConfig(pm.initConfig, "old-1"))
-
 			err = conn.ReadJSON(&msg)
 
-			// Ensure that the connection is closed. In the future, we might want to send a complete message to the client
+			// If the operation happen fast enough, ensure that the connection is closed.
+			// In the future, we might want to send a complete message to the client
 			// and wait until in-flight messages are delivered before closing the connection
-			var wsErr *websocket.CloseError
-			require.ErrorAs(t, err, &wsErr)
+			if err != nil {
+				var wsErr *websocket.CloseError
+				require.ErrorAs(t, err, &wsErr)
+			}
 
 			require.NoError(t, conn.Close())
 

router-tests/events_config_test.go → router-tests/events/events_config_test.go

@@ -1,4 +1,4 @@
-package integration_test
+package events_test
 
 import (
 	"github.com/stretchr/testify/assert"