Merge pull request #25 from ephemery-testnet/genesis-generator

Change genesis generation to use `ethereum-genesis-generator`

.github/workflows/build-testnet-config.yml

@@ -14,21 +14,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 18
 
     - name: Build testnet configs
       run: bash ./scripts/build-genesis.sh
 
+    - name: Build validator names
+      run: bash ./scripts/build-validator-names.sh
+
     - name: Build bootnode enr
       run: bash ./scripts/build-boot-enr.sh
 

.github/workflows/scheduled-release.yml

@@ -38,21 +38,12 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Set up Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 18
-
     - name: Build testnet configs
       run: bash ./scripts/build-genesis.sh
 
+    - name: Build validator names
+      run: bash ./scripts/build-validator-names.sh
+
     - name: Build bootnode enr
       run: bash ./scripts/build-boot-enr.sh
 

.gitignore

@@ -1,2 +1,3 @@
 apps/eth2-testnet-genesis
-dist
+dist
+temp

bootnode-keys/bootnode_eph1.pub

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4SY0jmB1xS3HCZpewkPG
+t3mt4AITFd2DRIeUqDXo41z2ZTk9VSdPU8rkBxjK9isjXINTCuB0fi+zeN+Aq4aF
+KcP9PijPkiH42VczFPuyPqTXlUym9ANC+AYfX275TEHTLZOn0CgwZmgzt3bREfaC
+vAeLcSJCQ64HhfzqhpN+tsA6gl5OB4TGYT7mnSyGDdTf7AHc2AQlADm/mYD9YEeI
+wkQbzgGx3f8lQ4OlH1bRRNS5gqtAbwpfwxjh84KF7KGNTF6HSmL10DD5xjBzcKD1
+4KwllVEdcO1PmYft+z9Idyr/H/krAGBx3UtZUczuVGvwCzsfmDIywA4pGUYU7U8e
+gwIDAQAB
+-----END PUBLIC KEY-----

cl-bootnodes.txt

@@ -1,3 +1,4 @@
+teku:bootnode_eph1:137.74.203.240:9100
 enr:-Iq4QNMYHuJGbnXyBj6FPS2UkOQ-hnxT-mIdNMMr7evR9UYtLemaluorL6J10RoUG1V4iTPTEbl3huijSNs5_ssBWFiGAYhBNHOzgmlkgnY0gmlwhIlKy_CJc2VjcDI1NmsxoQNULnJBzD8Sakd9EufSXhM4rQTIkhKBBTmWVJUtLCp8KoN1ZHCCIyk
 enr:-Iq4QIc297-de1P6hznMX2cIdVsQkve9BD9NUsJ7vVQa7eh5UpekA9rLid5A-yLiS3gZwOGugYZPi58x76zNs2cEQFCGAYhBJlTYgmlkgnY0gmlwhEFtmi6Jc2VjcDI1NmsxoQJDyix-IHa_mVwLBEN9NeG8I-RUjNQK_MGxk9OqRQUAtIN1ZHCCIyg
 lighthouse:bootnode_pk2:135.181.140.168:9040

cl-config.yaml

@@ -8,7 +8,7 @@ CONFIG_NAME: testnet # needs to exist because of Prysm. Otherwise it conflicts w
 MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: $MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
 # Mar-01-2021 08:53:32 AM +UTC
 # This is an invalid valid and should be updated when you create the genesis
-MIN_GENESIS_TIME: $CL_TIMESTAMP
+MIN_GENESIS_TIME: $GENESIS_TIMESTAMP
 GENESIS_FORK_VERSION: $GENESIS_FORK_VERSION
 GENESIS_DELAY: 300
 
@@ -33,20 +33,20 @@ CAPELLA_FORK_VERSION: $CAPELLA_FORK_VERSION
 CAPELLA_FORK_EPOCH: 0
 # Deneb
 DENEB_FORK_VERSION: $DENEB_FORK_VERSION
-DENEB_FORK_EPOCH: 5
+DENEB_FORK_EPOCH: $DENEB_FORK_EPOCH
 
 # Time parameters
 # ---------------------------------------------------------------
 # 12 seconds
-SECONDS_PER_SLOT: 12
+SECONDS_PER_SLOT: $SLOT_DURATION_IN_SECONDS
 # 14 (estimate from Eth1 mainnet)
-SECONDS_PER_ETH1_BLOCK: 14
+SECONDS_PER_ETH1_BLOCK: $SLOT_DURATION_IN_SECONDS
 # 2**8 (= 256) epochs ~27 hours
 MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
 # 2**8 (= 256) epochs ~27 hours
 SHARD_COMMITTEE_PERIOD: 256
 # 2**11 (= 2,048) Eth1 blocks ~8 hours
-ETH1_FOLLOW_DISTANCE: 12
+ETH1_FOLLOW_DISTANCE: $ETH1_FOLLOW_DISTANCE
 
 
 # Validator cycle
@@ -56,13 +56,13 @@ INACTIVITY_SCORE_BIAS: 4
 # 2**4 (= 16)
 INACTIVITY_SCORE_RECOVERY_RATE: 16
 # 2**4 * 10**9 (= 16,000,000,000) Gwei
-EJECTION_BALANCE: 16000000000
+EJECTION_BALANCE: $EJECTION_BALANCE
 # 2**2 (= 4)
 MIN_PER_EPOCH_CHURN_LIMIT: 4
 # 2**16 (= 65,536)
 CHURN_LIMIT_QUOTIENT: 65536
 # [New in Deneb:EIP7514] 2**3 (= 8)
-MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8
+MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: $MAX_CHURN
 
 # Fork choice
 # ---------------------------------------------------------------

el-config.yaml

@@ -1,5 +1,6 @@
 chain_id: ${CHAIN_ID}
 deposit_contract_address: "${DEPOSIT_CONTRACT_ADDRESS}"
+el_premine: {}
 el_premine_addrs: {
   # Sepolia genesis (core devs)
   "0xa2A6d93439144FFE4D27c9E088dCD8b783946263": "1000000ETH",
@@ -33,3 +34,7 @@ el_premine_addrs: {
   "0x992775d32fd0ec76b95C5E76CeEA92ED5a4bE1F9": 10000000ETH, # https://faucet.bordel.wtf/
 }
 genesis_timestamp: ${GENESIS_TIMESTAMP}
+genesis_delay: ${GENESIS_DELAY}
+genesis_gaslimit: ${GENESIS_GASLIMIT}
+slot_duration_in_seconds: ${SLOT_DURATION_IN_SECONDS}
+deneb_fork_epoch: ${DENEB_FORK_EPOCH}

scripts/build-boot-enr.sh

@@ -7,9 +7,9 @@ get_github_release() {
     sed -E 's/.*"([^"]+)".*/\1/'                                    # Pluck JSON value
 }
 
-if ! [ -d ./apps/lighthouse ]; then
-  mkdir ./apps/lighthouse
-  cd ./apps/lighthouse
+if ! [ -d ./temp/lighthouse ]; then
+  mkdir -p ./temp/lighthouse
+  cd ./temp/lighthouse
 
   lighthouse_release=$(get_github_release sigp/lighthouse)
   wget "https://github.com/sigp/lighthouse/releases/download/$lighthouse_release/lighthouse-${lighthouse_release}-x86_64-unknown-linux-gnu-portable.tar.gz"
@@ -23,7 +23,15 @@ tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
 mkdir -p $tmp_dir
 mkdir -p ./dist/bootnode-keys
 
+if [ -f ./dist/boot_enr.txt ]; then
+  rm ./dist/boot_enr.txt
+fi
+if [ -f ./dist/bootstrap_nodes.txt ]; then
+  rm ./dist/bootstrap_nodes.txt
+fi
+
 add_bootnode_enr() {
+  echo "add enr: $1"
   echo "$1" >> ./dist/bootstrap_nodes.txt
   echo "- $1" >> ./dist/boot_enr.txt
 }
@@ -44,24 +52,46 @@ cat ./cl-bootnodes.txt | while read line ; do
       add_bootnode_enr $line
     elif [ ${bootnode_data[0]} = "lh_bootnode" ]; then
       rm -rf $tmp_dir/*
-      ./apps/lighthouse/lighthouse boot_node --testnet-dir ./dist --datadir $tmp_dir --port ${bootnode_data[3]} --enr-address ${bootnode_data[2]} &
+      ./temp/lighthouse/lighthouse boot_node --testnet-dir ./dist --datadir $tmp_dir --port ${bootnode_data[3]} --enr-address ${bootnode_data[2]} &
       sleep 2
       killall lighthouse
       sleep 2
       bootnode_enr=$(cat $tmp_dir/beacon/network/enr.dat)
-      echo "$bootnode_enr" >> ./dist/bootnode-keys/${bootnode_data[1]}.enr
-      add_bootnode_key ${bootnode_data[1]} $tmp_dir/beacon/network/key
-      add_bootnode_enr $bootnode_enr
+      if [ ! -z "$bootnode_enr" ]; then
+        echo "$bootnode_enr" >> ./dist/bootnode-keys/${bootnode_data[1]}.enr
+        add_bootnode_key ${bootnode_data[1]} $tmp_dir/beacon/network/key
+        add_bootnode_enr $bootnode_enr
+      fi
     elif [ ${bootnode_data[0]} = "lighthouse" ]; then
       rm -rf $tmp_dir/*
-      ./apps/lighthouse/lighthouse bn --testnet-dir ./dist --datadir $tmp_dir --enr-address ${bootnode_data[2]} --enr-udp-port ${bootnode_data[3]} --port ${bootnode_data[3]} &
+      ./temp/lighthouse/lighthouse bn --testnet-dir ./dist --datadir $tmp_dir --enr-address ${bootnode_data[2]} --enr-udp-port ${bootnode_data[3]} --port ${bootnode_data[3]} &
       sleep 10
       killall lighthouse
       sleep 2
       bootnode_enr=$(cat $tmp_dir/beacon/network/enr.dat)
-      echo "$bootnode_enr" >> ./dist/bootnode-keys/${bootnode_data[1]}.enr
-      add_bootnode_key ${bootnode_data[1]} $tmp_dir/beacon/network/key
-      add_bootnode_enr $bootnode_enr
+      if [ ! -z "$bootnode_enr" ]; then
+        echo "$bootnode_enr" >> ./dist/bootnode-keys/${bootnode_data[1]}.enr
+        add_bootnode_key ${bootnode_data[1]} $tmp_dir/beacon/network/key
+        add_bootnode_enr $bootnode_enr
+      fi
+    elif [ ${bootnode_data[0]} = "teku" ]; then
+      rm -rf $tmp_dir/*
+      echo -n 0x$(openssl rand -hex 32 | tr -d "\n") > $tmp_dir/jwtsecret
+      docker run -d --restart unless-stopped --name teku-node -u $UID -v ./dist:/testnet:ro -p 5052:5052 -v $tmp_dir:/data consensys/teku:latest \
+        --network=/testnet/config.yaml --initial-state=/testnet/genesis.ssz \
+        --ee-endpoint=http://172.17.0.1:8651 --ee-jwt-secret-file=/data/jwtsecret \
+        --data-path=/data --p2p-enabled=true --p2p-interface=0.0.0.0 --p2p-advertised-ip=${bootnode_data[2]} --p2p-port=${bootnode_data[3]} --p2p-advertised-port=${bootnode_data[3]} \
+        --rest-api-enabled --rest-api-interface=0.0.0.0 --rest-api-host-allowlist=* --rest-api-port=5052 \
+        --Xpeer-rate-limit=100000 --Xpeer-request-limit=1000 --ignore-weak-subjectivity-period-enabled --data-storage-non-canonical-blocks-enabled=true
+      sleep 10
+      bootnode_enr=$(curl -s http://127.0.0.1:5052/eth/v1/node/identity | jq -r .data.enr)
+      docker rm -f teku-node
+      sleep 2
+      if [ ! -z "$bootnode_enr" ]; then
+        echo "$bootnode_enr" >> ./dist/bootnode-keys/${bootnode_data[1]}.enr
+        add_bootnode_key ${bootnode_data[1]} $tmp_dir/beacon/kvstore/generated-node-key.dat
+        add_bootnode_enr $bootnode_enr
+      fi
     fi
 done
 

scripts/build-genesis.sh

@@ -1,94 +1,52 @@
 #!/bin/bash -e
 source ./values.env
-mkdir -p ./dist
 
-setup_apps(){
-    set -x
-    cd ./apps
-
-    if ! [ -d "./eth2-testnet-genesis" ]; then
-        git clone https://github.com/protolambda/eth2-testnet-genesis.git
-        cd eth2-testnet-genesis
-        go install .
-        cd ..
-    fi
-
-    if [ -d "./el-gen" ]; then
-        cd el-gen
-        pip3 install -r requirements.txt
-        cd ..
-    fi
-
-    go install github.com/protolambda/eth2-val-tools@latest
-    go install github.com/protolambda/zcli@latest
-
-    cd ..
-}
-
-gen_el_config(){
-    set -x
-    if ! [ -f "./dist/genesis.json" ]; then
-        tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
-        mkdir -p ./dist
-        envsubst < ./el-config.yaml > $tmp_dir/genesis-config.yaml
-        python3 ./apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml      > ./dist/genesis.json
-        python3 ./apps/el-gen/genesis_chainspec.py $tmp_dir/genesis-config.yaml > ./dist/chainspec.json
-        python3 ./apps/el-gen/genesis_besu.py $tmp_dir/genesis-config.yaml > ./dist/besu.json
-        cp ./el-bootnodes.txt ./dist/boot_enode.txt
-        cp ./el-bootnodes.txt ./dist/bootnode.txt
-    else
-        echo "el genesis already exists. skipping generation..."
+gen_all_config(){
+    if [ -d ./dist ]; then
+        rm -rf ./dist
     fi
-}
-
-gen_cl_config(){
-    set -x
-    # Consensus layer: Check if genesis already exists
-    if ! [ -f "./dist/genesis.ssz" ]; then
-        tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
-        mkdir -p ./dist
-        # Replace environment vars in files
-        envsubst < ./cl-config.yaml > ./dist/config.yaml
-
-        # Replace MIN_GENESIS_TIME on config
-        sed "s/^MIN_GENESIS_TIME:.*/MIN_GENESIS_TIME: ${GENESIS_TIMESTAMP}/" ./dist/config.yaml > $tmp_dir/config.yaml
-        mv $tmp_dir/config.yaml ./dist/config.yaml
-
-        # Create deposit_contract.txt and deploy_block.txt
-        grep DEPOSIT_CONTRACT_ADDRESS ./dist/config.yaml | cut -d " " -f2 > ./dist/deposit_contract.txt
-        echo $DEPOSIT_CONTRACT_BLOCK > ./dist/deploy_block.txt
-        echo $DEPOSIT_CONTRACT_BLOCK > ./dist/deposit_contract_block.txt
-
-        # Create a dummy validator with iteration number in pubkey (required to get a unique forkdigest for each genesis iteration)
-        dummyaddr=$(echo $ITERATION_NUMBER | awk '{printf("%040x\n", $1)}')
-        echo "0xb54b2811832ff970d1b3e048271e4fc9c0f4dcccac17683724f972203a6130d8ee7c26ec9bde0183fcede171deaddc4b:0x010000000000000000000000$dummyaddr:16000000000" > $tmp_dir/validators.txt
-
-        # collect validators
-        cat ./validators/*.txt | sed 's/#.*$//g' >> $tmp_dir/validators.txt
 
-        # Generate genesis
-        eth2-testnet-genesis capella \
-        --config ./dist/config.yaml \
-        --additional-validators $tmp_dir/validators.txt \
-        --eth1-config ./dist/genesis.json \
-        --tranches-dir ./dist/tranches \
-        --state-output ./dist/genesis.ssz
-
-    else
-        echo "cl genesis already exists. skipping generation..."
+    # prepare input configs for ethereum-genesis-generator
+    if [ -d ./temp ]; then
+        rm -rf ./temp
     fi
-}
-
-gen_all_config(){
-    setup_apps
+    mkdir -p ./temp/output
+    mkdir -p ./temp/input/cl
+    cp ./cl-config.yaml ./temp/input/cl/config.yaml
+    echo "" > ./temp/input/cl/mnemonics.yaml
+    mkdir -p ./temp/input/el
+    cp ./el-config.yaml ./temp/input/el/genesis-config.yaml
+    cp ./values.env ./temp/input/values.env
+
+    # Create a dummy validator with iteration number in pubkey (required to get a unique forkdigest for each genesis iteration)
+    dummyaddr=$(echo $ITERATION_NUMBER | awk '{printf("%040x\n", $1)}')
+    echo "0xb54b2811832ff970d1b3e048271e4fc9c0f4dcccac17683724f972203a6130d8ee7c26ec9bde0183fcede171deaddc4b:0x010000000000000000000000$dummyaddr:16000000000" > ./temp/input/validators.txt
+
+    # collect validators
+    cat ./validators/*.txt | sed 's/#.*$//g' >> ./temp/input/validators.txt
+
+    # run ethereum-genesis-generator
+    docker run --rm -u $UID -v $PWD/temp/output:/data \
+        -v $PWD/temp/input:/config \
+        ethpandaops/ethereum-genesis-generator:latest \
+        all
+
+    # copy config folder structure
+    cp ./temp/output/custom_config_data -r ./dist
+    rm ./dist/mnemonics.yaml
+    rm -rf ./temp
+
+    # build retention vars
     touch ./dist/retention.vars
     echo 'export ITERATION_NUMBER="'"${ITERATION_NUMBER}"'"' >> ./dist/retention.vars
     echo 'export ITERATION_RELEASE="ephemery-'"${ITERATION_NUMBER}"'"' >> ./dist/retention.vars
     echo 'export GENESIS_TIMESTAMP="'"${GENESIS_TIMESTAMP}"'"' >> ./dist/retention.vars
     echo 'export GENESIS_RESET_INTERVAL="'"${GENESIS_INTERVAL}"'"' >> ./dist/retention.vars
     echo 'export CHAIN_ID="'"${CHAIN_ID}"'"' >> ./dist/retention.vars
-    gen_el_config
-    gen_cl_config
+
+    # copy el bootnodes
+    cp ./el-bootnodes.txt ./dist/boot_enode.txt
+    cp ./el-bootnodes.txt ./dist/bootnode.txt
 
     ls -lah ./dist
 }

scripts/build-nodevars.sh

@@ -3,9 +3,6 @@ set -x
 
 source ./values.env
 
-# get genesis details
-zcli pretty capella BeaconState ./dist/genesis.ssz > ./dist/parsedBeaconState.json
-
 echo 'ITERATION_NUMBER="'"$ITERATION_NUMBER"'"' >> ./dist/nodevars_env.txt
 echo 'CHAIN_ID="'"$CHAIN_ID"'"' >> ./dist/nodevars_env.txt
 echo 'BOOTNODE_ENR="'"$(cat ./dist/bootstrap_nodes.txt | sed -E '/^$/d' | head -n1)"'"' >> ./dist/nodevars_env.txt