FM-363: Block sync test (#459)

* FM-363: Combine stop and rm into destroy

* FM-363: Try to connect a full node to the existing one

* FM-363: Test that the full node is syncing

* FM-363: Fix init script

.github/actions/install-tools/action.yaml

@@ -29,8 +29,11 @@ runs:
         repo-token: ${{ inputs.repo-token }}
 
     # For compiling Solidity contracts
-    - name: Install Foundry
-      uses: foundry-rs/foundry-toolchain@v1
+    # - name: Install Foundry
+    #   uses: foundry-rs/foundry-toolchain@v1
+
+    - name: 'Install jq'
+      uses: dcarbone/[email protected]
 
     # See https://github.com/docker/setup-buildx-action
     - name: Set up Docker Buildx

fendermint/testing/Makefile/common.toml

@@ -40,6 +40,7 @@ run_task = { name = [
 dependencies = [
   "test-data-dir",
   "test-data-env",
+  "test-node-dir",
   "docker-network-create",
   "cometbft-init",
   "fendermint-init",
@@ -59,12 +60,9 @@ dependencies = []
 [tasks.teardown]
 # `dependencies` doesn't seem to work with `cleanup_task`.
 run_task = { name = [
-  "cometbft-stop",
-  "cometbft-rm",
-  "fendermint-stop",
-  "fendermint-rm",
-  "ethapi-stop",
-  "ethapi-rm",
+  "cometbft-destroy",
+  "fendermint-destroy",
+  "ethapi-destroy",
   "docker-network-rm",
   "test-data-dir-rm",
 ] }
@@ -73,8 +71,6 @@ run_task = { name = [
 [tasks.test-data-dir]
 script = """
 mkdir -p ${TEST_DATA_DIR}/keys;
-mkdir -p ${TEST_DATA_DIR}/${NODE_NAME}/fendermint/data/logs;
-mkdir -p ${TEST_DATA_DIR}/${NODE_NAME}/cometbft;
 cp -r ${TEST_SCRIPTS_DIR} ${TEST_DATA_DIR}/scripts
 """
 
@@ -87,3 +83,14 @@ rm -rf ${TEST_DATA_DIR}
 script = """
 touch ${TEST_DATA_DIR}/.env
 """
+
+[tasks.test-node-dir]
+script = """
+mkdir -p ${TEST_DATA_DIR}/${NODE_NAME}/fendermint/data/logs;
+mkdir -p ${TEST_DATA_DIR}/${NODE_NAME}/cometbft;
+"""
+
+[tasks.test-node-dir-rm]
+script = """
+rm -rf ${TEST_DATA_DIR}/${NODE_NAME}
+"""

fendermint/testing/smoke-test/Makefile.toml

@@ -15,7 +15,7 @@ extend = [
 ]
 
 env_files = [
-  { path = "./smoke.env" },
+  { path = "./scripts/smoke.env" },
   { path = "../Makefile/common.env" },
   { path = "../Makefile/ci.env", profile = "ci" },
 ]

fendermint/testing/snapshot-test/Makefile.toml

@@ -12,12 +12,12 @@ extend = [
 ]
 
 env_files = [
-  { path = "./snapshot.env" },
+  { path = "./scripts/snapshot.env" },
   { path = "../Makefile/common.env" },
   { path = "../Makefile/ci.env", profile = "ci" },
 ]
 
-# Overriding th env file to enable snapshotting.
+# Overriding the env file to enable snapshotting.
 [tasks.test-data-env]
 script = """
 cat << EOF > ${TEST_DATA_DIR}/.env
@@ -32,9 +32,9 @@ EOF
 
 [tasks.test]
 clear = true
-dependencies = ["snapshot-wait", "snapshot-created"]
+dependencies = ["snapshot-wait", "snapshot-created", "fullnode-sync"]
 
-# Wait enought time that some snapshots should be exported.
+# Wait enough time that some snapshots should be exported.
 [tasks.snapshot-wait]
 extend = "wait"
 env = { "CARGO_MAKE_WAIT_MILLISECONDS" = "15000" }
@@ -48,3 +48,68 @@ if [ -z "$(ls -A $FM_SNAPSOTS_DIR)" ]; then
    exit 1
 fi
 """
+
+
+# Set up a full node that syncs with the default one, then stop it.
+[tasks.fullnode-sync]
+run_task = { name = [
+  "fullnode-setup",
+  "fullnode-test",
+], fork = true, cleanup_task = "fullnode-teardown" }
+
+
+[tasks.fullnode-setup]
+dependencies = ["cometbft-export-node-id"]
+env_files = [{ path = "./scripts/fullnode.env" }]
+run_task = { name = [
+  "test-node-dir",
+  "cometbft-init",
+  "fullnode-set-seed",
+  "fullnode-copy-genesis",
+  "fendermint-start",
+  "cometbft-start",
+  "cometbft-wait",
+  "fendermint-logs",
+  "cometbft-logs",
+] }
+
+
+# Set the persistent peer address to that of the default node-0.
+[tasks.fullnode-set-seed]
+env_files = [{ path = "./scripts/fullnode.env" }]
+script = """
+CMT_SEED_ID=$(cat $BASE_DIR/$SEED_NODE_NAME/node-id)
+CMT_PERSISTENT_PEERS=$CMT_SEED_ID@$SEED_CMT_CONTAINER_NAME:26656
+sed -i'' -e "s|persistent_peers = \\"\\"|persistent_peers = \\"$CMT_PERSISTENT_PEERS\\"|" $BASE_DIR/${NODE_NAME}/cometbft/config/config.toml
+"""
+
+
+# Get the genesis from node-0
+[tasks.fullnode-copy-genesis]
+env_files = [{ path = "./scripts/fullnode.env" }]
+script = """
+cp $BASE_DIR/${SEED_NODE_NAME}/cometbft/config/genesis.json \
+   $BASE_DIR/${NODE_NAME}/cometbft/config/genesis.json
+"""
+
+# See if it managed to sync.
+[tasks.fullnode-test]
+env_files = [{ path = "./scripts/fullnode.env" }]
+script = """
+EARLIEST=$(curl -s localhost:${CMT_RPC_HOST_PORT}/status | jq -r ".result.sync_info.earliest_block_height")
+LATEST=$(curl -s localhost:${CMT_RPC_HOST_PORT}/status | jq -r ".result.sync_info.latest_block_height")
+
+if [ "$EARLIEST" = "$LATEST" ]; then
+  echo "ERROR: The chain is not syncing!"
+  exit 1
+fi
+"""
+
+
+[tasks.fullnode-teardown]
+env_files = [{ path = "./scripts/fullnode.env" }]
+run_task = { name = [
+  "cometbft-destroy",
+  "fendermint-destroy",
+  "test-node-dir-rm",
+] }

fendermint/testing/snapshot-test/scripts/fullnode.env

@@ -0,0 +1,10 @@
+SEED_NODE_NAME=node-0
+SEED_CMT_CONTAINER_NAME=snapshot-cometbft
+NODE_NAME=node-1
+FM_CONTAINER_NAME=snapshot-fendermint-1
+CMT_CONTAINER_NAME=snapshot-cometbft-1
+CMT_DIR=${TEST_DATA_DIR}/${NODE_NAME}/cometbft
+CMT_P2P_HOST_PORT=26666
+CMT_RPC_HOST_PORT=26667
+CMT_MAX_NUM_OUTBOUND_PEERS=1
+CMT_WAIT_MILLIS=20000

fendermint/testing/snapshot-test/scripts/init.sh

@@ -28,6 +28,11 @@ for NAME in victoria veronica vivienne; do
     add-account --public-key $KEYS_DIR/$NAME.pk \
                 --balance 1000 \
                 --kind ethereum
+
+  # Convert FM validator key to CMT
+  fendermint \
+    key into-tendermint --secret-key $KEYS_DIR/$NAME.sk \
+      --out $KEYS_DIR/$NAME.priv_validator_key.json
 done
 
 # Add a validator
@@ -42,7 +47,6 @@ fendermint \
   genesis --genesis-file $GENESIS_FILE \
   into-tendermint --out $CMT_DIR/config/genesis.json
 
-# Convert FM validator key to CMT
-fendermint \
-  key into-tendermint --secret-key $KEYS_DIR/$VALIDATOR_NAME.sk \
-    --out $CMT_DIR/config/priv_validator_key.json
+# Copy the default validator key
+cp $KEYS_DIR/$VALIDATOR_NAME.priv_validator_key.json \
+   $CMT_DIR/config/priv_validator_key.json

infra/scripts/cometbft.toml

@@ -24,6 +24,7 @@ docker run \
   --publish ${CMT_RPC_HOST_PORT}:26657 \
   --publish ${CMT_P2P_HOST_PORT}:26656 \
   --volume ${CMT_DIR}:/cometbft \
+  --env-file ${ENV_FILE} \
   --env CMT_PROXY_APP=tcp://${FM_CONTAINER_NAME}:26658 \
   --env CMT_PEX=false \
   --env CMT_MAX_SUBSCRIPTION_CLIENTS=10 \
@@ -156,20 +157,22 @@ id=`docker exec ${CMT_CONTAINER_NAME} cometbft show-node-id`
 echo $id
 """
 
+[tasks.cometbft-export-node-id]
+script = """
+id=`docker exec ${CMT_CONTAINER_NAME} cometbft show-node-id`
+echo $id > ${TEST_DATA_DIR}/${NODE_NAME}/node-id
+"""
+
 [tasks.cometbft-seed]
 dependencies = ["get-external-ip"]
 script = """
 id=`docker exec ${CMT_CONTAINER_NAME} cometbft show-node-id`
 echo $id@${NODE_EXTERNAL_IP}:${CMT_P2P_HOST_PORT}
 """
 
-[tasks.cometbft-rm]
-extend = "docker-rm"
-env = { "CONTAINER_NAME" = "${CMT_CONTAINER_NAME}" }
-
-[tasks.cometbft-stop]
-extend = "docker-stop"
+[tasks.cometbft-destroy]
 env = { "CONTAINER_NAME" = "${CMT_CONTAINER_NAME}" }
+run_task = "docker-destroy"
 
 [tasks.cometbft-logs]
 extend = "docker-logs"
@@ -205,16 +208,10 @@ sed -i'' -e "s|allow_duplicate_ip = false|allow_duplicate_ip = true|" ${CMT_DIR}
 """
 
 [tasks.cometbft-config]
-dependencies = [
+run_task = { name = [
     "cometbft-init",
     "set-seeds",
     "addr-book-enable",
     "set-external-addr",
     "duplicate-ip-enable",
-]
-
-
-[tasks.node-id]
-command = "docker"
-args = ["exec", "${CONTAINER_NAME}", "cometbft show-node-id"]
-ignore_errors = true
+] }

infra/scripts/docker.toml

@@ -18,6 +18,12 @@ command = "docker"
 args = ["rm", "--force", "${CONTAINER_NAME}"]
 ignore_errors = true
 
+[tasks.docker-destroy]
+run_task = { name = [
+  "docker-stop",
+  "docker-rm",
+] }
+
 [tasks.docker-logs]
 command = "docker"
 args = ["logs", "${CONTAINER_NAME}"]

infra/scripts/ethapi.toml

@@ -19,13 +19,9 @@ dependencies = ["docker-network-create"]
 extend = "ethapi-run"
 env = { "CMD" = "eth run", "FLAGS" = "-d" }
 
-[tasks.ethapi-rm]
-extend = "docker-rm"
-env = { "CONTAINER_NAME" = "${ETHAPI_CONTAINER_NAME}" }
-
-[tasks.ethapi-stop]
-extend = "docker-stop"
+[tasks.ethapi-destroy]
 env = { "CONTAINER_NAME" = "${ETHAPI_CONTAINER_NAME}" }
+run_task = "docker-destroy"
 
 [tasks.ethapi-logs]
 extend = "docker-logs"

infra/scripts/fendermint.toml

@@ -149,13 +149,9 @@ else
 fi
 """
 
-[tasks.fendermint-rm]
-extend = "docker-rm"
-env = { "CONTAINER_NAME" = "${FM_CONTAINER_NAME}" }
-
-[tasks.fendermint-stop]
-extend = "docker-stop"
+[tasks.fendermint-destroy]
 env = { "CONTAINER_NAME" = "${FM_CONTAINER_NAME}" }
+run_task = "docker-destroy"
 
 [tasks.fendermint-logs]
 extend = "docker-logs"

infra/scripts/subnet.toml

@@ -3,7 +3,7 @@
 ########################################################################################################################
 
 [tasks.bootstrap]
-env = { "NETWORK_NAME"="${NETWORK_NAME}", "CMT_DIR" = "${BASE_DIR}/bootstrap/cometbft" }
+env = { "NETWORK_NAME" = "${NETWORK_NAME}", "CMT_DIR" = "${BASE_DIR}/bootstrap/cometbft" }
 run_task = "new-bootstrap"
 
 [tasks.new-bootstrap]
@@ -28,10 +28,8 @@ run_task = "cometbft-node-id"
 
 [tasks.bootstrap-down]
 dependencies = [
-    "cometbft-stop",
-    "cometbft-rm",
-    "fendermint-stop",
-    "fendermint-rm",
+    "cometbft-destroy",
+    "fendermint-destroy",
 ]
 
 [tasks.bootstrap-restart]
@@ -72,7 +70,7 @@ run_task = { name = [
     "cometbft-start",
     "cometbft-wait",
     "ethapi-start",
-]}
+] }
 
 [tasks.child-fullnode-down]
 run_task = "testnode-down"
@@ -148,4 +146,4 @@ CometBFT node ID:
 CometBFT P2P:
 \thttp://0.0.0.0:${CMT_P2P_HOST_PORT}
 EOF
-"""
+"""

infra/scripts/testnode.toml

@@ -25,13 +25,13 @@ dependencies = [
 ]
 
 [tasks.testnode-clear]
-script="""
+script = """
 echo clearing all IPC data
 rm -rf ${BASE_DIR}
 """
 
 [tasks.testnode-mkdir]
-script="""
+script = """
 echo creating directories: $BASE_DIR $FM_DIR $CMT_DIR
 mkdir -p $BASE_DIR
 mkdir -p $FM_DIR
@@ -48,19 +48,16 @@ run_task = { name = [
     "cometbft-start",
     "cometbft-wait",
     "ethapi-start",
-]}
+] }
 
 [tasks.testnode-down]
 # `dependencies` doesn't seem to work with `cleanup_task`.
 run_task = { name = [
-    "cometbft-stop",
-    "cometbft-rm",
-    "fendermint-stop",
-    "fendermint-rm",
-    "ethapi-stop",
-    "ethapi-rm",
-    "docker-network-rm"
-]}
+    "cometbft-destroy",
+    "fendermint-destroy",
+    "ethapi-destroy",
+    "docker-network-rm",
+] }
 
 # This task create all necessary data structures to run Fendermint:
 # the genesis file with necessary entities and cryptographic keys.
@@ -72,7 +69,7 @@ dependencies = [
     "genesis-add-validator",
     "genesis-new-gateway",
     "genesis-write",
-    "testnode-export-keys"
+    "testnode-export-keys",
 ]
 
 [tasks.testnode-export-keys]
@@ -108,4 +105,4 @@ Fendermint API:
 CometBFT API:
 \thttp://0.0.0.0:26657
 EOF
-"""
+"""