IPC-333: child validator and bootstrap deployment scripts (#320)

* fm-333: child validator and bootstrap deployment scripts

* ipc-333: child subnet validator and bootstrap deployment scripts

* ipc-333: expose CMT/ETHAPI ports and fix docker deployment

* ipc-333: add docs

* fm-333: address review

* ipc-333: add top-down configuration to fendermint start

* ipc-333: address review

* ipc-333: second round of reviews and fixes

* ipc-333: update bootstrap ip

---------

Signed-off-by: Alfonso de la Rocha <[email protected]>

Makefile

@@ -13,7 +13,6 @@ IPC_ACTORS_ABI        := .make/.ipc-actors-abi
 # Note that without `:=`, just `=`, it should evaluate it every time it appears in a target.
 IPC_ACTORS_DIR         = $(shell $(IPC_ACTORS_FIND))
 IPC_ACTORS_OUT         = $(IPC_ACTORS_DIR)/out
-
 FENDERMINT_CODE       := $(shell find . -type f \( -name "*.rs" -o -name "Cargo.toml" \) | grep -v target)
 
 # Override PROFILE env var to choose between `local | ci`

README.md

@@ -9,6 +9,8 @@ Fendermint is an effort to implement [IPC with Tendermint Core](https://docs.goo
 
 Please have a look in the [docs](./docs/README.md) to see an overview of the project, how to run the components, and previous demos.
 
+## IPC
+Fendermint is built with support for [IPC](https://github.com/consensus-shipyard/ipc) by design. If you are looking to deploy the infrastructure Fendermint-based IPC subnet, refer to the [IPC main repo](https://github.com/consensus-shipyard/ipc), or have a look at the [IPC infrastructure docs](./docs/ipc.md).
 
 ## Testing
 

docs/README.md

@@ -6,6 +6,7 @@ The following documentation should help getting oriented:
 * [Getting started with Tendermint](./tendermint.md)
 * [Running Fendermint](./running.md)
 * [Checkpointing](./checkpointing.md)
+* [Running IPC infrastructure](./ipc.md)
 
 You can also check out the previous demos:
 * [Milestone-1 Demo](./demos/milestone-1/README.md)

docs/ipc.md

@@ -0,0 +1,79 @@
+# IPC
+
+This documentation will guide you through the different utils provided in Fendermint for the deployment of Fendermint-based IPC subnets. All node processes are run inside Docker containers in your local environment.
+
+This docs are only focused on the infrastructure deployment, for an end-to-end walk through of spawning IPC subnets refer to the [IPC quickstart](https://github.com/consensus-shipyard/ipc/blob/main/docs/quickstart-calibration.md).
+
+## Prerequisites
+
+On Linux (links and instructions for Ubuntu):
+
+- Install system packages: `sudo apt install build-essential clang cmake pkg-config libssl-dev protobuf-compiler`.
+- Install Rust. See [instructions](https://www.rust-lang.org/tools/install).
+- Install cargo-make: `cargo install --force cargo-make`.
+- Install Docker. See [instructions](https://docs.docker.com/engine/install/ubuntu/).
+- Install Foundry. See [instructions](https://book.getfoundry.sh/getting-started/installation).
+
+## Deploy subnet bootstrap
+In order not to expose directly the network address information from validators, subnets leverage the use of bootstrap nodes (or `seeds` in CometBFT parlance), for new nodes to discover peers in the network and connect to the subnet's validators. To run a bootstrap node you can run the following command from the root of the repo:
+```bash
+cargo make --makefile infra/Makefile.toml bootstrap
+```
+You'll see that by the end of the output, this command should output the network address of your bootstrap. You can use this endpoint to include this bootstrap node as a seed in the `seeds` configuration of CometBFT.
+```console
+[cargo-make] INFO - Running Task: cometbft-wait
+[cargo-make] INFO - Running Task: cometbft-node-id
+[email protected]:26656
+[cargo-make] INFO - Build Done in 13.38 seconds.
+```
+
+If at any time you need to query the endpoint of your bootstrap, you can run:
+```bash
+cargo make --makefile infra/Makefile.toml bootstrap-id
+```
+
+`cargo-make bootstrap` supports the following environment variables to customize the deployment:
+- `CMT_P2P_HOST_PORT` (optional): Specifies the listening port for the bootstraps P2p interface in the localhost for CometBFT. This is the address that needs to be shared with other peers if they want to use the bootstrap as a `seed` to discover connections.
+- `CMT_RPC_HOST_PORT` (optional): Specifies the listening port in the localhost for CometBFT's RPC.
+- `NODE_NAME` (optional): Node name information to attach to the containers of the deployment. This will be needed to deploy more than one bootstrap in the same local environment.
+
+Finally, to remove the bootstrap you can run:
+```
+cargo make --makefile infra/Makefile.toml bootstrap-down
+```
+
+
+## Deploy child subnet validator
+Once a child subnet has been bootstrapped in its parent, its subnet actor has been deployed, and has fulfilled its minimum requirements in terms of validators and minimum collateral, validators in the subnet can deploy their infrastructure to spawn the child subnet.
+
+In order to spawn a validator node in a child subnet, you need to run:
+```bash
+cargo make --makefile infra/Makefile.toml \
+    -e VALIDATOR_PRIV_KEY=<VALIDATOR_PRIV_KEY> \
+    -e SUBNET_ID=<SUBNET_ID> \
+    -e CMT_P2P_HOST_PORT=<COMETBFT_P2P_PORT> \
+    -e CMT_RPC_HOST_PORT=<COMETBFT_RPC_PORT> \
+    -e ETHAPI_HOST_PORT=<ETH_RPC_PORT> \
+    -e BOOTSTRAPS=<BOOTSTRAP_ENDPOINT>
+    -e PARENT_REGISTRY=<PARENT_REGISTRY_CONTRACT_ADDR> \
+    -e PARENT_GATEWAY=<GATEWAY_REGISTRY_CONTRACT_ADDR> \
+    child-validator
+```
+This command will run the infrastructure for a Fendermint validator in the child subnet. It will generate the genesis of the subnet from the information in its parent, and will run the validator's infrastructure with the specific configuration passed in the command.
+
+`cargo-make child-validator` supports the following environment variables to customize the deployment:
+- `CMT_P2P_HOST_PORT` (optional): Specifies the listening port in the localhost for the P2P interface of the CometBFT node.
+- `CMT_RPC_HOST_PORT` (optional): Specifies the listening port in the localhost for CometBFT's RPC.
+- `ETHAPI_HOST_PORT` (optional): Specifies the listening port in the localhost for the ETH RPC of the node.
+- `NODE_NAME` (optional): Name for the node deployment. Along with `CMT_P2P_HOST_PORT`, `CMT_RPC_HOST_PORT` and `ETHAPI_HOST_PORT`, these variables come really handy for the deployment of several validator nodes over the same system.
+- `VALIDATOR_PRIV_KEY`: Path of the private key for your validator (it should be the corresponding one used to join the subnet in the parent).
+- `SUBNET_ID`: SubnetID for the child subnet.
+- `BOOTSTRAPS`: Comma separated list of bootstraps (or seeds in CometBFT parlance).
+- `PARENT_ENDPOINT`: Public endpoint that the validator should use to connect to the parent.
+- `PARENT_REGISTRY`: Ethereum address of the IPC registry contract in the parent
+- `PARENT_GATEWAY`: Ethereum address of the IPC gateway contract in the parent.
+
+Finally, to remove the bootstrap you can run:
+```
+cargo make --makefile infra/Makefile.toml child-validator-down
+```

fendermint/app/src/cmd/genesis.rs

@@ -10,7 +10,7 @@ use ipc_provider::IpcProvider;
 use std::path::PathBuf;
 
 use fendermint_vm_actor_interface::eam::EthAddress;
-use fendermint_vm_core::Timestamp;
+use fendermint_vm_core::{chainid, Timestamp};
 use fendermint_vm_genesis::{
     ipc, Account, Actor, ActorMeta, Collateral, Genesis, Multisig, SignerAddr, Validator,
     ValidatorKey,
@@ -189,9 +189,13 @@ where
 fn into_tendermint(genesis_file: &PathBuf, args: &GenesisIntoTendermintArgs) -> anyhow::Result<()> {
     let genesis = read_genesis(genesis_file)?;
     let genesis_json = serde_json::to_value(&genesis)?;
+
+    let chain_id: u64 = chainid::from_str_hashed(&genesis.chain_name)?.into();
+    let chain_id = chain_id.to_string();
+
     let tmg = tendermint::Genesis {
         genesis_time: tendermint::time::Time::from_unix_timestamp(genesis.timestamp.as_secs(), 0)?,
-        chain_id: tendermint::chain::Id::try_from(genesis.chain_name)?,
+        chain_id: tendermint::chain::Id::try_from(chain_id)?,
         initial_height: 0,
         // Values are based on the default produced by `tendermint init`
         consensus_params: tendermint::consensus::Params {

fendermint/testing/smoke-test/Makefile.toml

@@ -129,7 +129,7 @@ docker run \
   --name ${CMT_CONTAINER_NAME} \
   --user $(id -u) \
   --network ${NETWORK_NAME} \
-  --publish 26657:${CMT_HOST_PORT} \
+  --publish ${CMT_HOST_PORT}:26657 \
   --volume ${TEST_DATA_DIR}/cometbft:/cometbft \
   --env CMT_PROXY_APP=tcp://${FM_CONTAINER_NAME}:26658 \
   --env CMT_PEX=false \
@@ -179,7 +179,7 @@ docker run \
   --init \
   --user $(id -u) \
   --network ${NETWORK_NAME} \
-  --publish 8545:${ETHAPI_HOST_PORT} \
+  --publish ${ETHAPI_HOST_PORT}:8545 \
   --env TENDERMINT_WS_URL=ws://${CMT_CONTAINER_NAME}:26657/websocket \
   --env LOG_LEVEL=debug \
   --env RUST_BACKTRACE=1 \

infra/Makefile.toml

@@ -5,18 +5,38 @@ extend = [
   { path = "scripts/ethapi.toml" },
   { path = "scripts/node.toml" },
   { path = "scripts/testnet.toml" },
-  { path = "scripts/testnode.toml" }
+  { path = "scripts/testnode.toml" },
+  { path = "scripts/subnet.toml" }
 ]
 
 [config]
 default_to_workspace = false
 
 [env]
-CHAIN_NAME = { value = "root", condition = { env_not_set = ["CHAIN_NAME"] }}
+# General network-specific parameters
+SUBNET_ID = { value = "/r0", condition = { env_not_set = ["SUBNET_ID"] }}
 BALANCE = { value = "1000", condition = { env_not_set = ["BALANCE"] }}
 BASE_FEE = { value = "1000", condition = { env_not_set = ["BASE_FEE"] }}
 TIMESTAMP = { value = "1680101412", condition = { env_not_set = ["TIMESTAMP"] }}
-BASE_DIR="${HOME}/.ipc/${CHAIN_NAME}"
+POWER_SCALE = { value = "3", condition = { env_not_set = ["POWER_SCALE"] }}
+CMT_P2P_HOST_PORT = { value = "26656", condition = { env_not_set = ["CMT_P2P_HOST_PORT"] }}
+CMT_RPC_HOST_PORT = { value = "26657", condition = { env_not_set = ["CMT_RPC_HOST_PORT"] }}
+ETHAPI_HOST_PORT = { value = "8545", condition = { env_not_set = ["ETHAPI_HOST_PORT"] }}
+
+# IPC subnet related parameters
+# Use calibration as default value
+NODE_NAME = { value = "ipc-node", condition = { env_not_set = ["NODE_NAME"] } }
+PARENT_ENDPOINT = { value = "https://api.calibration.node.glif.io/rpc/v1", condition = { env_not_set = ["PARENT_ENDPOINT"] } }
+PARENT_GATEWAY = { value = "0x56948d2CFaa2EF355B8C08Ac925202db212146D1", condition = { env_not_set = ["PARENT_GATEWAY"] } }
+PARENT_REGISTRY = { value = "0x6A4884D2B6A597792dC68014D4B7C117cca5668e", condition = { env_not_set = ["PARENT_REGISTRY"] } }
+NETWORK_TYPE = { value = "test", condition = { env_not_set = ["NETWORK_TYPE"] } }
+VALIDATOR_PRIV_KEY = { value = "~/.ipc/priv.key", condition = { env_not_set = ["VALIDATOR_PRIV_KEY"] } }
+TOPDOWN_CHAIN_HEAD_DELAY = { value = "10", condition = { env_not_set = ["TOPDOWN_CHAIN_HEAD_DELAY"] } }
+# Comma-separated list of bootstrap nodes to be used by the CometBFT node.
+BOOTSTRAPS = { value = "", condition = { env_not_set = ["BOOTSTRAPS"] } }
+
+# Deployment-related
+BASE_DIR="${HOME}/.ipc/${SUBNET_ID}"
 FM_DIR="${BASE_DIR}/fendermint"
 CMT_DIR="${BASE_DIR}/cometbft"
 
@@ -26,7 +46,7 @@ KEY_NAME="validator_key"
 PUB_KEY_PATH="${KEYS_DIR}/${KEY_NAME}.pk"
 PRIV_KEY_PATH="${KEYS_DIR}/${KEY_NAME}.sk"
 
-NETWORK_NAME = "${CHAIN_NAME}"
+NETWORK_NAME = "${SUBNET_ID}"
 
 CMT_CONTAINER_NAME = "${NETWORK_NAME}-cometbft"
 FM_CONTAINER_NAME = "${NETWORK_NAME}-fendermint"
@@ -37,8 +57,6 @@ FM_DOCKER_IMAGE = "fendermint:latest"
 TEST_DATA_DIR = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/fendermint/testing/smoke-test/test-data"
 TEST_SCRIPTS_DIR = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/fendermint/testing/smoke-test/scripts"
 ACTORS_BUNDLE = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/../builtin-actors/output/bundle.car"
-CMT_HOST_PORT = 26657
-ETHAPI_HOST_PORT = 8545
 # If this wasn't present, any wait task is skipped.
 CARGO_MAKE_WAIT_MILLISECONDS = 5000
 # This wait time seems to work locally.
@@ -50,7 +68,7 @@ VERBOSITY = ""
 script="""
 echo
 echo Chain info:
-echo - Chain: ${CHAIN_NAME}
+echo - Chain: ${SUBNET_ID}
 echo - Balance: ${BALANCE}
 echo - Base Fee: ${BASE_FEE}
 echo - Timestamp: ${TIMESTAMP}
@@ -87,9 +105,9 @@ script = ['''
     echo - info: Print the setup information
     echo
     echo Most tasks use these environment variables:
-    echo - CHAIN_NAME (default '${CHAIN_NAME}'): the target IPC subnet
+    echo - SUBNET_ID (default '${SUBNET_ID}'): the target IPC subnet
     echo
-    echo Run 'cargo make -e CHAIN_NAME=chain -e BALANCE=100 -e BASE_FEE=200 ... COMMAND' to populate the variables from CLI or
+    echo Run 'cargo make -e SUBNET_ID=chain -e BALANCE=100 -e BASE_FEE=200 ... COMMAND' to populate the variables from CLI or
     echo Run 'cargo make --env-file=/PATH/.env COMMAND' to populate the variables from the file before running the command.
     echo
     echo Run 'cargo make --list-all-steps' for a complete list of available tasks.

infra/docker-compose.yml

@@ -28,7 +28,7 @@ services:
     image: "fendermint:latest"
     environment:
       - FM_DATA_DIR=/data/fendermint/data
-      - FM_CHAIN_NAME=$CHAIN_NAME
+      - FM_CHAIN_NAME=$SUBNET_ID
       - LOG_LEVEL=info
     volumes:
       - $BASE_DIR/node${NODE_ID}:/data
@@ -59,4 +59,4 @@ services:
 networks:
   testnet:
     name: ${NETWORK_NAME}
-    external: true
+    external: true

infra/scripts/cometbft.toml

@@ -21,7 +21,8 @@ docker run \
   --name ${CMT_CONTAINER_NAME} \
   --user $(id -u) \
   --network ${NETWORK_NAME} \
-  --publish 26657:${CMT_HOST_PORT} \
+  --publish ${CMT_RPC_HOST_PORT}:26657 \
+  --publish ${CMT_P2P_HOST_PORT}:26656 \
   --volume ${CMT_DIR}:/cometbft \
   --env CMT_PROXY_APP=tcp://${FM_CONTAINER_NAME}:26658 \
   --env CMT_PEX=false \
@@ -33,8 +34,8 @@ dependencies = ["cometbft-pull", "docker-network-create"]
 [tasks.cometbft-node-id]
 script = """
 id=`docker exec ${CMT_CONTAINER_NAME} cometbft show-node-id`
-ip=`docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${CMT_CONTAINER_NAME}`
-echo $id@$ip:26656
+ip=`curl ifconfig.me`
+echo $id@$ip:${CMT_P2P_HOST_PORT}
 """
 
 [tasks.cometbft-rm]

infra/scripts/ethapi.toml

@@ -6,7 +6,7 @@ docker run \
   --init \
   --user $(id -u) \
   --network ${NETWORK_NAME} \
-  --publish 8545:${ETHAPI_HOST_PORT} \
+  --publish ${ETHAPI_HOST_PORT}:8545 \
   --env TENDERMINT_WS_URL=ws://${CMT_CONTAINER_NAME}:26657/websocket \
   --env LOG_LEVEL=debug \
   --env RUST_BACKTRACE=1 \

infra/scripts/fendermint.toml

@@ -16,6 +16,33 @@ docker run \
   --env LOG_LEVEL=info \
   --entrypoint ${ENTRY} \
   ${FM_DOCKER_IMAGE} \
+  --network=${NETWORK_TYPE} ${CMD}
+"""
+dependencies = ["docker-network-create", "fendermint-deps"]
+
+[tasks.fendermint-start-ipc]
+extend = "fendermint-run"
+env = { "ENTRY" = "fendermint", "CMD" = "run", "FLAGS" = "-d" }
+
+[tasks.fendermint-run-ipc]
+script = """
+docker run \
+  ${FLAGS} \
+  --name ${FM_CONTAINER_NAME} \
+  --init \
+  --user $(id -u) \
+  --network ${NETWORK_NAME} \
+  --volume ${BASE_DIR}:/data \
+  --env FM_DATA_DIR=/data/fendermint/data \
+  --env FM_CHAIN_NAME=${NETWORK_NAME} \
+  --env FM_IPC__SUBNET_ID=${NETWORK_NAME} \
+  --env FM_IPC__TOPDOWN__CHAIN_HEAD_DELAY=${TOPDOWN_CHAIN_HEAD_DELAY} \
+  --env FM_IPC__TOPDOWN__PARENT_HTTP_ENDPOINT=${PARENT_ENDPOINT} \
+  --env FM_IPC__TOPDOWN__PARENT_REGISTRY=${PARENT_REGISTRY} \
+  --env FM_IPC__TOPDOWN__PARENT_GATEWAY=${PARENT_GATEWAY} \
+  --env LOG_LEVEL=info \
+  --entrypoint ${ENTRY} \
+  ${FM_DOCKER_IMAGE} \
   ${CMD}
 """
 dependencies = ["docker-network-create", "fendermint-deps"]

infra/scripts/node.toml

@@ -1,19 +1,36 @@
 ########################################################################################################################
 # Node helpers
 ########################################################################################################################
+
+[tasks.bootstrap]
+env = { "NETWORK_NAME"="${NETWORK_NAME}", "CMT_DIR" = "${BASE_DIR}/bootstrap/cometbft", "CMT_CONTAINER_NAME" = "cometbft-${NODE_NAME}-bootstrap" }
+run_task = "new-node"
+
+[tasks.bootstrap-id]
+env = { "CMT_CONTAINER_NAME" = "cometbft-${NODE_NAME}-bootstrap" }
+run_task = "cometbft-node-id"
+
+[tasks.bootstrap-down]
+env = { "CMT_CONTAINER_NAME" = "cometbft-${NODE_NAME}-bootstrap" }
+run_task = "node-down"
+
 [tasks.node-down]
 dependencies = [
     "cometbft-stop",
     "cometbft-rm"
 ]
 
+
 [tasks.new-node]
 dependencies = [
     "node-down",
-    "node-init",
+    "bootstrap-mkdir",
     "cometbft-init",
     "seed-enable",
-    "cometbft-start"
+    "set-seeds",
+    "cometbft-start",
+    "cometbft-wait",
+    "cometbft-node-id",
 ]
 
 [tasks.node-init]
@@ -28,16 +45,28 @@ echo clearing all IPC data
 rm -rf ${BASE_DIR}
 """
 
+[tasks.bootstrap-mkdir]
+script="""
+echo creating directories: $BASE_DIR $CMT_DIR
+mkdir -p $BASE_DIR
+mkdir -p $CMT_DIR
+"""
+
 [tasks.node-mkdir]
 script="""
 echo creating directories: $BASE_DIR $FM_DIR $CMT_DIR
 mkdir -p $BASE_DIR
-mkdir -p $FM_DIR
 
+mkdir -p $FM_DIR
 mkdir -p $CMT_DIR
 """
 
 [tasks.seed-enable]
 script="""
-sed -i'bak' "s/seed_mode = false/seed_mode = true/" ${BASE_DIR}/cometbft/config/config.toml
+sed -i'bak' "s/seed_mode = false/seed_mode = true/" ${CMT_DIR}/config/config.toml
+"""
+
+[tasks.set-seeds]
+script="""
+sed -i 's/seeds = ""/seeds = "${BOOTSTRAPS}"/g' ${CMT_DIR}/config/config.toml
 """