Ci e2e service.0

ci/jenkins/ssh-config

@@ -1,8 +1,8 @@
-Host CONTROLPLANENODE
+Host SSHCONFIGNODENAME
     SendEnv LANG LC_*
     HashKnownHosts yes
     GSSAPIAuthentication yes
     GSSAPIDelegateCredentials no
-    HostName CONTROLPLANENODEIP
+    HostName SSHCONFIGNODEIP
     Port 22
     User capv

ci/jenkins/test-vmc.sh

@@ -375,10 +375,8 @@ function deliver_antrea {
         docker save -o flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator:${DOCKER_IMG_VERSION}
     fi
 
-
-    kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}' | while read control_plane_ip; do
-        ${SCP_WITH_ANTREA_CI_KEY} $GIT_CHECKOUT_DIR/build/yamls/*.yml capv@${control_plane_ip}:~
-    done
+    control_plane_ip="$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')"
+    ${SCP_WITH_ANTREA_CI_KEY} $GIT_CHECKOUT_DIR/build/yamls/*.yml capv@${control_plane_ip}:~
 
     IPs=($(kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | xargs))
     for i in "${!IPs[@]}"
@@ -448,28 +446,33 @@ function run_e2e {
     export KUBECONFIG=$GIT_CHECKOUT_DIR/jenkins/out/kubeconfig
 
     mkdir -p $GIT_CHECKOUT_DIR/test/e2e/infra/vagrant/playbook/kube
-    cp -f $GIT_CHECKOUT_DIR/jenkins/out/kubeconfig $GIT_CHECKOUT_DIR/test/e2e/infra/vagrant/playbook/kube/config
+    CLUSTER_KUBECONFIG="${GIT_CHECKOUT_DIR}/jenkins/out/kubeconfig"
+    CLUSTER_SSHCONFIG="${GIT_CHECKOUT_DIR}/jenkins/out/ssh-config"
 
     echo "=== Generate ssh-config ==="
-    cp -f $GIT_CHECKOUT_DIR/ci/jenkins/ssh-config $GIT_CHECKOUT_DIR/test/e2e/infra/vagrant/ssh-config
-    control_plane_name="$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $1}')"
-    control_plane_ip="$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')"
-    echo "=== Control-plane Node ip: ${control_plane_ip} ==="
-    sed -i "s/CONTROLPLANENODEIP/${control_plane_ip}/g" $GIT_CHECKOUT_DIR/test/e2e/infra/vagrant/ssh-config
+    kubectl get nodes -o wide --no-headers=true | awk '{print $1}' | while read sshconfig_nodename; do
+        echo "Generating ssh-config for Node ${sshconfig_nodename}"
+        sshconfig_nodeip="$(kubectl get node "${sshconfig_nodename}" -o jsonpath='{.status.addresses[0].address}')"
+        cp "${GIT_CHECKOUT_DIR}/ci/jenkins/ssh-config" "${CLUSTER_SSHCONFIG}.new"
+        sed -i "s/SSHCONFIGNODEIP/${sshconfig_nodeip}/g" "${CLUSTER_SSHCONFIG}.new"
+        sed -i "s/SSHCONFIGNODENAME/${sshconfig_nodename}/g" "${CLUSTER_SSHCONFIG}.new"
+        echo "    IdentityFile ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key" >> "${CLUSTER_SSHCONFIG}.new"
+        cat "${CLUSTER_SSHCONFIG}.new" >> "${CLUSTER_SSHCONFIG}"
+    done
+
     echo "=== Move kubeconfig to control-plane Node ==="
+    control_plane_ip="$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}')"
     ${SSH_WITH_ANTREA_CI_KEY} -n capv@${control_plane_ip} "if [ ! -d ".kube" ]; then mkdir .kube; fi"
     ${SCP_WITH_ANTREA_CI_KEY} $GIT_CHECKOUT_DIR/jenkins/out/kubeconfig capv@${control_plane_ip}:~/.kube/config
-    sed -i "s/CONTROLPLANENODE/${control_plane_name}/g" $GIT_CHECKOUT_DIR/test/e2e/infra/vagrant/ssh-config
-    echo "    IdentityFile ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key" >> $GIT_CHECKOUT_DIR/test/e2e/infra/vagrant/ssh-config
 
     set +e
     mkdir -p ${GIT_CHECKOUT_DIR}/antrea-test-logs
     if [[ "$COVERAGE" == true ]]; then
         rm -rf ${GIT_CHECKOUT_DIR}/e2e-coverage
         mkdir -p ${GIT_CHECKOUT_DIR}/e2e-coverage
-        go test -v -timeout=100m antrea.io/antrea/test/e2e --logs-export-dir ${GIT_CHECKOUT_DIR}/antrea-test-logs --prometheus --coverage --coverage-dir ${GIT_CHECKOUT_DIR}/e2e-coverage
+        go test -v -timeout=100m antrea.io/antrea/test/e2e --logs-export-dir ${GIT_CHECKOUT_DIR}/antrea-test-logs --prometheus --coverage --coverage-dir ${GIT_CHECKOUT_DIR}/e2e-coverage --provider remote --remote.sshconfig "${CLUSTER_SSHCONFIG}" --remote.kubeconfig "${CLUSTER_KUBECONFIG}"
     else
-        go test -v -timeout=100m antrea.io/antrea/test/e2e --logs-export-dir ${GIT_CHECKOUT_DIR}/antrea-test-logs --prometheus
+        go test -v -timeout=100m antrea.io/antrea/test/e2e --logs-export-dir ${GIT_CHECKOUT_DIR}/antrea-test-logs --prometheus --provider remote --remote.sshconfig "${CLUSTER_SSHCONFIG}" --remote.kubeconfig "${CLUSTER_KUBECONFIG}"
     fi
 
     test_rc=$?

ci/jenkins/test.sh

@@ -428,7 +428,22 @@ function run_e2e_windows {
     mkdir -p "${WORKDIR}/.kube"
     mkdir -p "${WORKDIR}/.ssh"
     cp -f "${WORKDIR}/kube.conf" "${WORKDIR}/.kube/config"
-    cp -f "${WORKDIR}/ssh-config" "${WORKDIR}/.ssh/config"
+    echo "=== Generate ssh-config ==="
+    SSH_CONFIG_DST="${WORKDIR}/.ssh/config"
+    kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 != role {print $1}' | while read sshconfig_nodename; do
+        echo "Generating ssh-config for Node ${sshconfig_nodename}"
+        sshconfig_nodeip="$(kubectl get node "${sshconfig_nodename}" -o jsonpath='{.status.addresses[0].address}')"
+        cp ci/jenkins/ssh-config "${SSH_CONFIG_DST}.new"
+        sed -i "s/SSHCONFIGNODEIP/${sshconfig_nodeip}/g" "${SSH_CONFIG_DST}.new"
+        sed -i "s/SSHCONFIGNODENAME/${sshconfig_nodename}/g" "${SSH_CONFIG_DST}.new"
+        if [[ "${sshconfig_nodename}" =~ "win" ]]; then
+            sed -i "s/capv/administrator/g" "${SSH_CONFIG_DST}.new"
+        else
+            sed -i "s/capv/jenkins/g" "${SSH_CONFIG_DST}.new"
+        fi
+        echo "    IdentityFile ${WORKDIR}/.ssh/id_rsa" >> "${SSH_CONFIG_DST}.new"
+        cat "${SSH_CONFIG_DST}.new" >> "${SSH_CONFIG_DST}"
+    done
 
     set +e
     mkdir -p `pwd`/antrea-test-logs

ci/kind/test-e2e-kind.sh

@@ -137,9 +137,9 @@ function run_test {
   fi
   sleep 1
   if $coverage; then
-      go test -v -timeout=70m antrea.io/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --coverage --coverage-dir $ANTREA_COV_DIR
+      go test -v -timeout=70m antrea.io/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --coverage --coverage-dir $ANTREA_COV_DIR -run TestClusterIPHostAccess
   else
-      go test -v -timeout=65m antrea.io/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR
+      go test -v -timeout=65m antrea.io/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR -run TestClusterIPHostAccess
   fi
   $TESTBED_CMD destroy kind
 }

test/e2e/fixtures.go

@@ -38,6 +38,12 @@ func skipIfProviderIs(tb testing.TB, name string, reason string) {
 	}
 }
 
+func skipIfProviderIsNot(tb testing.TB, name string, reason string) {
+	if testOptions.providerName != name {
+		tb.Skipf("Skipping test if provider is not '%s': %s", name, reason)
+	}
+}
+
 func skipIfNumNodesLessThan(tb testing.TB, required int) {
 	if clusterInfo.numNodes < required {
 		tb.Skipf("Skipping test as it requires %d different Nodes but cluster only has %d", required, clusterInfo.numNodes)

test/e2e/framework.go

@@ -399,7 +399,7 @@ func collectClusterInfo() error {
 
 	retrieveCIDRs := func(cmd string, reg string) ([]string, error) {
 		res := make([]string, 2)
-		rc, stdout, _, err := RunCommandOnNode(clusterInfo.controlPlaneNodeName, cmd)
+		rc, stdout, _, err := RunCommandOnNode(controlPlaneNodeName(), cmd)
 		if err != nil || rc != 0 {
 			return res, fmt.Errorf("error when running the following command `%s` on control-plane Node: %v, %s", cmd, err, stdout)
 		}
@@ -948,9 +948,13 @@ func (data *TestData) createBusyboxPodOnNode(name string, nodeName string) error
 // The Pod will be scheduled on the specified Node (if nodeName is not empty).
 func (data *TestData) createHostNetworkBusyboxPodOnNode(name string, nodeName string) error {
 	sleepDuration := 3600 // seconds
-	return data.createPodOnNode(name, nodeName, busyboxImage, []string{"sleep", strconv.Itoa(sleepDuration)}, nil, nil, nil, false, func(pod *corev1.Pod) {
-		pod.Spec.HostNetwork = true
-	})
+	return data.createPodOnNode(name, nodeName, busyboxImage, []string{"sleep", strconv.Itoa(sleepDuration)}, nil, nil, nil, true, nil)
+}
+
+// createHostNetworkAgnhostPodOnNode creates a host network Pod in the test namespace with a single agnhost container.
+func (data *TestData) createHostNetworkAgnhostPodOnNode(name string, nodeName string) error {
+	sleepDuration := 3600 // seconds
+	return data.createPodOnNode(name, nodeName, agnhostImage, []string{"sleep", strconv.Itoa(sleepDuration)}, nil, nil, nil, true, nil)
 }
 
 // createBusyboxPod creates a Pod in the test namespace with a single busybox container.

test/e2e/service_test.go

@@ -0,0 +1,114 @@
+// Copyright 2021 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// TestClusterIPHostAccess tests traffic from host to Cluster IP Service.
+func TestClusterIPHostAccess(t *testing.T) {
+	// TODO: Support for dual-stack and IPv6-only clusters
+	skipIfIPv6Cluster(t)
+
+	data, err := setupTest(t)
+	if err != nil {
+		t.Fatalf("Error when setting up test: %v", err)
+	}
+	defer teardownTest(t, data)
+
+	svcName := "nginx"
+	svc, cleanup := data.createClusterIPServiceAllInOne(t, svcName, controlPlaneNodeName())
+	defer cleanup()
+	t.Logf("%s Service is ready", svcName)
+
+	var winNode string
+	if len(clusterInfo.windowsNodes) != 0 {
+		idx := clusterInfo.windowsNodes[0]
+		winNode = clusterInfo.nodes[idx].name
+	}
+
+	curlSvc := func(node string) {
+		rc, stdout, stderr, err := RunCommandOnNode(node, fmt.Sprintf("curl %s:80", svc.Spec.ClusterIP))
+		if rc != 0 || err != nil {
+			t.Errorf("Error when running command on Node '%s', rc: %d, stdout: %s, stderr: %s, error: %v",
+				node, rc, stdout, stderr, err)
+		} else {
+			t.Logf("curl from Node '%s' succeeded\n", node)
+		}
+	}
+	t.Logf("Try to curl Cluster IP Service from a Linux host")
+	curlSvc(controlPlaneNodeName())
+	if winNode != "" {
+		t.Logf("Try to curl Cluster IP Service from a Windows host")
+		curlSvc(winNode)
+	}
+}
+
+// TestClusterIPHostAccessOnKind tests traffic from a HostNetwork Pod to Cluster IP Service when provider is Kind.
+func TestClusterIPHostAccessOnKind(t *testing.T) {
+	skipIfProviderIsNot(t, "kind", "TestClusterIPHostAccess tests on providers other than Kind.")
+	// TODO: Support for dual-stack and IPv6-only clusters
+	skipIfIPv6Cluster(t)
+
+	data, err := setupTest(t)
+	if err != nil {
+		t.Fatalf("Error when setting up test: %v", err)
+	}
+	defer teardownTest(t, data)
+
+	svcName := "nginx"
+	node := controlPlaneNodeName()
+	svc, cleanup := data.createClusterIPServiceAllInOne(t, svcName, node)
+	defer cleanup()
+	t.Logf("%s Service is ready", svcName)
+
+	client := "client"
+	require.NoError(t, data.createHostNetworkAgnhostPodOnNode(client, node))
+	defer data.deletePod(testNamespace, client)
+	_, err = data.podWaitForIPs(defaultTimeout, client, testNamespace)
+	require.NoError(t, err)
+	require.NoError(t, data.podWaitForRunning(defaultTimeout, client, testNamespace))
+
+	cmd := []string{"curl", fmt.Sprintf("%s:80", svc.Spec.ClusterIP)}
+	stdout, stderr, err := data.runCommandFromPod(testNamespace, client, agnhostContainerName, cmd)
+	if err != nil {
+		t.Errorf("Error when curling from a HostNetwork Pod to Cluster IP Service, stdout: %s, stderr: %s: %v",
+			stdout, stderr, err)
+	} else {
+		t.Logf("curl from a HostNetwork Pod on Node '%s' succeeded\n", node)
+	}
+}
+
+func (data *TestData) createClusterIPServiceAllInOne(t *testing.T, name string, node string) (*corev1.Service, func()) {
+	ipv4Protocol := corev1.IPv4Protocol
+	require.NoError(t, data.createNginxPod(name, node))
+	_, err := data.podWaitForIPs(defaultTimeout, name, testNamespace)
+	require.NoError(t, err)
+	require.NoError(t, data.podWaitForRunning(defaultTimeout, name, testNamespace))
+	svc, err := data.createNginxClusterIPService(name, false, &ipv4Protocol)
+	require.NoError(t, err)
+
+	cleanup := func() {
+		data.deletePodAndWait(defaultTimeout, name)
+		data.deleteServiceAndWait(defaultTimeout, name)
+	}
+
+	return svc, cleanup
+}