Refactor connectivity tests

Makefile

@@ -104,7 +104,7 @@ external_network:
 			--opt "com.docker.network.driver.mtu=9000" \
 			--opt "com.docker.network.bridge.name=mini_lab_ext" \
 			--opt "com.docker.network.bridge.enable_ip_masquerade=true" && \
-		sudo ip route add 203.0.113.128/25 via 203.0.113.2 dev mini_lab_ext; fi
+		sudo ip route add 203.0.113.128/25 via 203.0.113.128 dev mini_lab_ext; fi
 
 .PHONY: env
 env:
@@ -133,15 +133,15 @@ _privatenet: env
 
 .PHONY: machine
 machine: _privatenet
-	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks $(shell docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: firewall
 firewall: _privatenet
 	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: public-ip
 public-ip:
-	@docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network ip list --name test --network internet-mini-lab -o template --template "{{ .ipaddress }}"
+	@docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network ip create --name test --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 -o template --template "{{ .ipaddress }}"
 
 .PHONY: ls
 ls: env
@@ -247,11 +247,10 @@ ssh-machine:
 	))
 	ssh -F files/ssh/config $(machine) $(COMMAND)
 
-.PHONY: connect-to-www
-connect-to-www:
-	@echo "Attempting to connect to container www..."
+.PHONY: test-connectivity-to-external-service
+test-connectivity-to-external-service:
 	@for i in $$(seq 1 $(MAX_RETRIES)); do \
-		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent http://203.0.113.3" > /dev/null 2>&1; then \
+		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent http://203.0.113.10" > /dev/null 2>&1; then \
 			echo "Connected successfully"; \
 			exit 0; \
 		else \

README.md

@@ -14,6 +14,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
     - [Reinstall machine](#reinstall-machine)
     - [Free machine](#free-machine)
 - [Flavors](#flavors)
+- [Network Topology](#network-topology)
 
 <!-- /TOC -->
 
@@ -204,3 +205,9 @@ In order to start specific flavor, you can define the flavor as follows:
 export MINI_LAB_FLAVOR=sonic
 make
 ```
+
+## Network topology
+
+An Nginx is running inside of the www container to allow automatic testing of outgoing connections.
+
+![Network topology](docs/network.svg)

files/inet/frr.conf → files/exit/frr.conf

@@ -1,5 +1,5 @@
 frr defaults datacenter
-hostname inet
+hostname exit
 !
 log syslog informational
 !
@@ -16,8 +16,8 @@ interface eth2
  ipv6 nd ra-interval 6
  no ipv6 nd suppress-ra
 !
-interface ext
- ip address 203.0.113.2/24
+interface mini_lab_ext
+ ip address 203.0.113.128/24
 !
 interface lo
  ip address 10.0.0.21/32

files/inet/network.sh → files/exit/network.sh

@@ -3,7 +3,7 @@ set -o errexit -o xtrace
 
 ip link add vrfInternet type vrf table 1000
 ip link set dev vrfInternet up
-ip link set dev ext master vrfInternet
+ip link set dev mini_lab_ext master vrfInternet
 
 ip link add name bridge type bridge stp_state 0
 ip link set dev bridge type bridge vlan_filtering 1

files/external_service/network.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -o errexit -o xtrace
+
+ip addr add 203.0.113.10/24 dev mini_lab_ext
+ip route add 203.0.113.128/25 via 203.0.113.128 dev mini_lab_ext

files/rules.yaml

@@ -31,6 +31,6 @@ ingress:
       - 22
     protocol: TCP
     from:
-      - 203.0.113.1/25
+      - 203.0.113.1/32
     to:
       - 203.0.113.128/25

mini-lab.capms.yaml

@@ -7,6 +7,17 @@ mgmt:
 
 topology:
   nodes:
+    exit:
+      kind: linux
+      image: quay.io/frrouting/frr:10.0.1
+      network-mode: none
+      binds:
+        - files/exit/daemons:/etc/frr/daemons
+        - files/exit/frr.conf:/etc/frr/frr.conf
+        - files/exit/vtysh.conf:/etc/frr/vtysh.conf
+        - files/exit/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
     mini_lab_ext:
       kind: bridge
     leaf01:
@@ -25,16 +36,6 @@ topology:
       binds:
         - /dev:/dev
         - files/ssh/id_rsa.pub:/authorized_keys
-    inet:
-      kind: linux
-      image: quay.io/frrouting/frr:10.0.1
-      binds:
-        - files/inet/daemons:/etc/frr/daemons
-        - files/inet/frr.conf:/etc/frr/frr.conf
-        - files/inet/vtysh.conf:/etc/frr/vtysh.conf
-        - files/inet/network.sh:/root/network.sh
-      exec:
-        - sh /root/network.sh
     vms:
       kind: linux
       image: ${MINI_LAB_VM_IMAGE}
@@ -45,13 +46,13 @@ topology:
         QEMU_MACHINE_CPU_CORES: 2
         QEMU_MACHINE_DISK_SIZE: 20G
   links:
-    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+    - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"]
       mtu: 9000
     - endpoints: ["leaf01:Ethernet0", "vms:lan0"]
     - endpoints: ["leaf02:Ethernet0", "vms:lan1"]
     - endpoints: ["leaf01:Ethernet1", "vms:lan2"]
     - endpoints: ["leaf02:Ethernet1", "vms:lan3"]
     - endpoints: ["leaf01:Ethernet2", "vms:lan4"]
     - endpoints: ["leaf02:Ethernet2", "vms:lan5"]
-    - endpoints: ["leaf01:Ethernet120", "inet:eth1"]
-    - endpoints: ["leaf02:Ethernet120", "inet:eth2"]
+    - endpoints: ["leaf01:Ethernet120", "exit:eth1"]
+    - endpoints: ["leaf02:Ethernet120", "exit:eth2"]

mini-lab.cumulus.yaml

@@ -16,42 +16,45 @@ topology:
         - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz
         - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys
   nodes:
+    exit:
+      kind: linux
+      image: quay.io/frrouting/frr:10.0.1
+      network-mode: none
+      binds:
+        - files/exit/daemons:/etc/frr/daemons
+        - files/exit/frr.conf:/etc/frr/frr.conf
+        - files/exit/vtysh.conf:/etc/frr/vtysh.conf
+        - files/exit/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
+    external_service:
+      kind: linux
+      image: docker.io/library/nginx:alpine-slim
+      network-mode: none
+      binds:
+        - files/external_service/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
     mini_lab_ext:
       kind: bridge
     leaf01:
       kind: cvx
     leaf02:
       kind: cvx
-    inet:
-      kind: linux
-      image: quay.io/frrouting/frr:10.0.1
-      binds:
-        - files/inet/daemons:/etc/frr/daemons
-        - files/inet/frr.conf:/etc/frr/frr.conf
-        - files/inet/vtysh.conf:/etc/frr/vtysh.conf
-        - files/inet/network.sh:/root/network.sh
-      exec:
-        - sh /root/network.sh
     vms:
       kind: linux
       image: ${MINI_LAB_VM_IMAGE}
       binds:
         - /dev:/dev
         - scripts:/mini-lab
-    www:
-      kind: linux
-      image: docker.io/library/nginx:alpine-slim
-      network-mode: none
-      exec:
-        - ip addr add 203.0.113.3/24 dev ext
-        - ip route add 203.0.113.128/25 via 203.0.113.2 dev ext
   links:
-    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+    - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"]
+      mtu: 9000
+    - endpoints: ["external_service:mini_lab_ext", "mini_lab_ext:external_service"]
       mtu: 9000
-    - endpoints: ["www:ext", "mini_lab_ext:www"]
     - endpoints: ["leaf01:swp1", "vms:lan0"]
     - endpoints: ["leaf02:swp1", "vms:lan1"]
     - endpoints: ["leaf01:swp2", "vms:lan2"]
     - endpoints: ["leaf02:swp2", "vms:lan3"]
-    - endpoints: ["leaf01:swp31", "inet:eth1"]
-    - endpoints: ["leaf02:swp31", "inet:eth2"]
+    - endpoints: ["leaf01:swp31", "exit:eth1"]
+    - endpoints: ["leaf02:swp31", "exit:eth2"]

mini-lab.sonic.yaml

@@ -6,6 +6,25 @@ mgmt:
 
 topology:
   nodes:
+    exit:
+      kind: linux
+      image: quay.io/frrouting/frr:10.0.1
+      network-mode: none
+      binds:
+        - files/exit/daemons:/etc/frr/daemons
+        - files/exit/frr.conf:/etc/frr/frr.conf
+        - files/exit/vtysh.conf:/etc/frr/vtysh.conf
+        - files/exit/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
+    external_service:
+      kind: linux
+      image: docker.io/library/nginx:alpine-slim
+      network-mode: none
+      binds:
+        - files/external_service/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
     mini_lab_ext:
       kind: bridge
     leaf01:
@@ -24,36 +43,20 @@ topology:
       binds:
         - /dev:/dev
         - files/ssh/id_rsa.pub:/authorized_keys
-    inet:
-      kind: linux
-      image: quay.io/frrouting/frr:10.0.1
-      binds:
-        - files/inet/daemons:/etc/frr/daemons
-        - files/inet/frr.conf:/etc/frr/frr.conf
-        - files/inet/vtysh.conf:/etc/frr/vtysh.conf
-        - files/inet/network.sh:/root/network.sh
-      exec:
-        - sh /root/network.sh
     vms:
       kind: linux
       image: ${MINI_LAB_VM_IMAGE}
       binds:
         - /dev:/dev
         - scripts:/mini-lab
-    www:
-      kind: linux
-      image: docker.io/library/nginx:alpine-slim
-      network-mode: none
-      exec:
-        - ip addr add 203.0.113.3/24 dev ext
-        - ip route add 203.0.113.128/25 via 203.0.113.2 dev ext
   links:
-    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+    - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"]
+      mtu: 9000
+    - endpoints: ["external_service:mini_lab_ext", "mini_lab_ext:external_service"]
       mtu: 9000
-    - endpoints: ["www:ext", "mini_lab_ext:www"]
     - endpoints: ["leaf01:Ethernet0", "vms:lan0"]
     - endpoints: ["leaf02:Ethernet0", "vms:lan1"]
     - endpoints: ["leaf01:Ethernet1", "vms:lan2"]
     - endpoints: ["leaf02:Ethernet1", "vms:lan3"]
-    - endpoints: ["leaf01:Ethernet120", "inet:eth1"]
-    - endpoints: ["leaf02:Ethernet120", "inet:eth2"]
+    - endpoints: ["leaf01:Ethernet120", "exit:eth1"]
+    - endpoints: ["leaf02:Ethernet120", "exit:eth2"]

test/integration.sh

@@ -43,9 +43,26 @@ done
 echo "$phoned/$minPhoned machines have phoned home"
 
 echo "Test connectivity to outside"
-make connect-to-www
+make test-connectivity-to-external-service
 
 echo "Test connectivity from outside"
-ssh -F files/ssh/config metal@$(make public-ip) -C exit
+public_ip=$(make public-ip)
+make ssh-machine COMMAND="sudo ip addr add ${public_ip}/32 dev lo"
+
+for i in $(seq 1 10); do
+  if ssh -F files/ssh/config metal@"${public_ip}" -C exit > /dev/null 2>&1; then
+    echo "Connected successfully"
+    break
+  else
+    echo "Connection failed"
+    if [ $i -lt 10 ]; then
+      echo "Retrying in 1 second..."
+      sleep 1
+    else
+      echo "Max retries reached"
+      exit 1
+    fi
+  fi
+done
 
 echo "Successfully started mini-lab"