[wip] debugging flakes

test/e2e-tests.sh

@@ -16,6 +16,8 @@
 
 source $(dirname $0)/e2e-common.sh
 
+export ENABLE_GKE_TELEMETRY=true
+
 # Script entry point.
 initialize $@  --skip-istio-addon
 

third_party/istio-latest/istio-ci-mesh.yaml

@@ -18,6 +18,7 @@ spec:
   values:
     global:
       proxy:
+        logLevel: debug
         autoInject: enabled
     pilot:
       autoscaleMin: 3
@@ -29,6 +30,8 @@ spec:
         autoscaleEnabled: false
 
   meshConfig:
+    accessLogFile: /dev/stdout
+    accessLogEncoding: JSON
     defaultConfig:
       terminationDrainDuration: "20s"
 

third_party/istio-latest/istio-ci-mesh/istio.yaml

@@ -7691,6 +7691,8 @@ metadata:
 data:
   meshNetworks: 'networks: {}'
   mesh: |-
+    accessLogEncoding: JSON
+    accessLogFile: /dev/stdout
     defaultConfig:
       discoveryAddress: istiod.istio-system.svc:15012
       proxyMetadata: {}
@@ -7772,7 +7774,7 @@ data:
           "includeIPRanges": "*",
           "includeInboundPorts": "*",
           "includeOutboundPorts": "",
-          "logLevel": "warning",
+          "logLevel": "debug",
           "privileged": false,
           "readinessFailureThreshold": 30,
           "readinessInitialDelaySeconds": 1,
@@ -9047,7 +9049,7 @@ spec:
             - router
             - --domain
             - $(POD_NAMESPACE).svc.cluster.local
-            - --proxyLogLevel=warning
+            - --proxyLogLevel=debug
             - --proxyComponentLogLevel=misc:error
             - --log_output_level=default:info
           env:

third_party/istio-latest/istio-ci-no-mesh.yaml

@@ -16,6 +16,9 @@ apiVersion: install.istio.io/v1alpha1
 kind: IstioOperator
 spec:
   values:
+    global:
+      proxy:
+        logLevel: debug
     pilot:
       autoscaleMin: 3
       autoscaleMax: 10
@@ -26,6 +29,8 @@ spec:
         autoscaleEnabled: false
 
   meshConfig:
+    accessLogFile: /dev/stdout
+    accessLogEncoding: JSON
     defaultConfig:
       terminationDrainDuration: "20s"
 

third_party/istio-latest/istio-ci-no-mesh/istio.yaml

@@ -7691,6 +7691,8 @@ metadata:
 data:
   meshNetworks: 'networks: {}'
   mesh: |-
+    accessLogEncoding: JSON
+    accessLogFile: /dev/stdout
     defaultConfig:
       discoveryAddress: istiod.istio-system.svc:15012
       proxyMetadata: {}
@@ -7772,7 +7774,7 @@ data:
           "includeIPRanges": "*",
           "includeInboundPorts": "*",
           "includeOutboundPorts": "",
-          "logLevel": "warning",
+          "logLevel": "debug",
           "privileged": false,
           "readinessFailureThreshold": 30,
           "readinessInitialDelaySeconds": 1,
@@ -9047,7 +9049,7 @@ spec:
             - router
             - --domain
             - $(POD_NAMESPACE).svc.cluster.local
-            - --proxyLogLevel=warning
+            - --proxyLogLevel=debug
             - --proxyComponentLogLevel=misc:error
             - --log_output_level=default:info
           env:

third_party/istio-latest/istio-kind-no-mesh.yaml

@@ -18,6 +18,7 @@ spec:
   values:
     global:
       proxy:
+        logLevel: debug
         clusterDomain: cluster.local
     pilot:
       autoscaleMin: 1
@@ -30,6 +31,8 @@ spec:
         type: NodePort
 
   meshConfig:
+    accessLogFile: /dev/stdout
+    accessLogEncoding: JSON
     defaultConfig:
       terminationDrainDuration: "20s"
 

third_party/istio-latest/istio-kind-no-mesh/istio.yaml

@@ -7691,6 +7691,8 @@ metadata:
 data:
   meshNetworks: 'networks: {}'
   mesh: |-
+    accessLogEncoding: JSON
+    accessLogFile: /dev/stdout
     defaultConfig:
       discoveryAddress: istiod.istio-system.svc:15012
       proxyMetadata: {}
@@ -7772,7 +7774,7 @@ data:
           "includeIPRanges": "*",
           "includeInboundPorts": "*",
           "includeOutboundPorts": "",
-          "logLevel": "warning",
+          "logLevel": "debug",
           "privileged": false,
           "readinessFailureThreshold": 30,
           "readinessInitialDelaySeconds": 1,
@@ -9047,7 +9049,7 @@ spec:
             - router
             - --domain
             - $(POD_NAMESPACE).svc.cluster.local
-            - --proxyLogLevel=warning
+            - --proxyLogLevel=debug
             - --proxyComponentLogLevel=misc:error
             - --log_output_level=default:info
           env:

vendor/knative.dev/networking/test/conformance/ingress/run.go

@@ -50,13 +50,13 @@ var stableTests = map[string]func(t *testing.T){
 
 var betaTests = map[string]func(t *testing.T){
 	// Add your conformance test for beta features
-	"host-rewrite": TestRewriteHost,
-	"headers/tags": TestTagHeaders,
+	// "host-rewrite": TestRewriteHost,
+	// "headers/tags": TestTagHeaders,
 }
 
 var alphaTests = map[string]func(t *testing.T){
 	// Add your conformance test for alpha features
-	"httpoption": TestHTTPOption,
+	// "httpoption": TestHTTPOption,
 }
 
 // RunConformance will run ingress conformance tests

vendor/knative.dev/networking/test/conformance/ingress/update.go

@@ -127,6 +127,7 @@ func TestUpdate(t *testing.T) {
 
 		t.Logf("Rolling out %q w/ %q", nextName, sentinel)
 
+		time.Sleep(10 * time.Second)
 		// Update the Ingress, and wait for it to report ready.
 		UpdateIngressReady(ctx, t, clients, ing.Name, v1alpha1.IngressSpec{
 			Rules: []v1alpha1.IngressRule{{

vendor/knative.dev/networking/test/conformance/ingress/util.go

@@ -1128,6 +1128,8 @@ func RuntimeRequestWithExpectations(ctx context.Context, t *testing.T, client *h
 		opt(req)
 	}
 
+	id := rand.Int()
+	req.Header.Set("id", strconv.Itoa(id))
 	resp, err := client.Do(req)
 
 	if err != nil {
@@ -1141,7 +1143,7 @@ func RuntimeRequestWithExpectations(ctx context.Context, t *testing.T, client *h
 
 	for _, e := range responseExpectations {
 		if err := e(resp); err != nil {
-			t.Error("Error meeting response expectations:", err)
+			t.Errorf("[%d] Error meeting response expectations: %v", id, err)
 			DumpResponse(ctx, t, resp)
 			return nil
 		}
@@ -1150,7 +1152,7 @@ func RuntimeRequestWithExpectations(ctx context.Context, t *testing.T, client *h
 	if resp.StatusCode == http.StatusOK {
 		b, err := io.ReadAll(resp.Body)
 		if err != nil {
-			t.Error("Unable to read response body:", err)
+			t.Errorf("[%d] Unable to read response body: %v", id, err)
 			DumpResponse(ctx, t, resp)
 			return nil
 		}

vendor/knative.dev/networking/test/test_images/runtime/handlers/handler.go

@@ -30,14 +30,14 @@ import (
 
 // InitHandlers initializes all handlers.
 func InitHandlers(mux *http.ServeMux) {
-	mux.HandleFunc("/", withHeaders(withRequestLog(runtimeHandler)))
+	mux.HandleFunc("/", withHeaders(runtimeHandler))
 
-	h := probe.NewHandler(withRequestLog(withKubeletProbeHeaderCheck))
+	h := probe.NewHandler(http.HandlerFunc(withKubeletProbeHeaderCheck))
 	mux.HandleFunc(nethttp.HealthCheckPath, h.ServeHTTP)
 }
 
 // withRequestLog logs each request before handling it.
-func withRequestLog(next http.HandlerFunc) http.HandlerFunc {
+func WithRequestLog(next http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		reqDump, err := httputil.DumpRequest(r, true)
 		if err != nil {
@@ -73,6 +73,6 @@ func withHeaders(next http.HandlerFunc) http.HandlerFunc {
 func writeJSON(w http.ResponseWriter, o interface{}) {
 	w.WriteHeader(http.StatusOK)
 	e := json.NewEncoder(w)
-	e.SetIndent("", "\t")
+	// e.SetIndent("", "\t")
 	e.Encode(o)
 }

vendor/knative.dev/networking/test/test_images/runtime/main.go

@@ -48,11 +48,13 @@ func main() {
 	mux := http.NewServeMux()
 	handlers.InitHandlers(mux)
 
+	handler := handlers.WithRequestLog(mux.ServeHTTP)
+
 	if cert, key := os.Getenv("CERT"), os.Getenv("KEY"); cert != "" && key != "" {
 		log.Print("Server starting on port with TLS ", port)
-		test.ListenAndServeTLSGracefullyWithHandler(cert, key, ":"+port, mux)
+		test.ListenAndServeTLSGracefullyWithHandler(cert, key, ":"+port, handler)
 	} else {
 		log.Print("Server starting on port ", port)
-		test.ListenAndServeGracefullyWithHandler(":"+port, mux)
+		test.ListenAndServeGracefullyWithHandler(":"+port, handler)
 	}
 }