From 8683bbeefb688364f5f9e445dffb42fc266dd8f0 Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Wed, 13 Dec 2023 04:18:33 -0500 Subject: [PATCH] Rename `ExportGenesisStateCommand` to `ExportGenesisHeadCommand` and make it respect custom genesis block builders (#2331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2326. This PR both fixes a logic bug and replaces an incorrect name. ## Bug Fix: Respecting custom genesis builder Prior to this PR the standard logic for creating a genesis block was repeated inside of cumulus. This PR removes that duplicated logic, and calls into the proper `BuildGenesisBlock` implementation. One consequence is that if the genesis block has already been initialized, it will not be re-created, but rather read from the database like it is for other node invocations. So you need to watch out for old unpurged data during the development process. Offchain tools may need to be updated accordingly. I've already filed https://github.com/paritytech/zombienet/issues/1519 ## Rename: It doesn't export state. It exports head data. The name export-genesis-state was always wrong, nad it's never too late to right a wrong. I've changed the name of the struct to `ExportGenesisHeadCommand`. There is still the question of what to do with individual nodes' public CLIs. I have updated the parachain template to a reasonable default that preserves compatibility with tools that will expect `export-genesis-state` to still work. And I've chosen not to modify the public CLIs of any other nodes in the repo. I'll leave it up to their individual owners/maintains to decide whether that is appropriate. --------- Co-authored-by: Joshy Orndorff Co-authored-by: Bastian Köcher Co-authored-by: Bastian Köcher --- Cargo.lock | 1 + cumulus/client/cli/Cargo.toml | 1 + cumulus/client/cli/src/lib.rs | 85 ++++++------------- cumulus/parachain-template/node/src/cli.rs | 7 +- .../parachain-template/node/src/command.rs | 4 +- cumulus/polkadot-parachain/src/cli.rs | 3 +- cumulus/polkadot-parachain/src/command.rs | 4 +- cumulus/test/service/src/cli.rs | 37 +------- cumulus/test/service/src/genesis.rs | 43 +++++++++- cumulus/test/service/src/lib.rs | 4 +- cumulus/test/service/src/main.rs | 44 +++------- .../test-parachains/adder/collator/src/cli.rs | 6 +- .../undying/collator/src/cli.rs | 6 +- prdoc/pr_2331.prdoc | 17 ++++ 14 files changed, 119 insertions(+), 143 deletions(-) create mode 100644 prdoc/pr_2331.prdoc diff --git a/Cargo.lock b/Cargo.lock index ba8cccca1edc..0e2194829c22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3324,6 +3324,7 @@ dependencies = [ "sc-cli", "sc-client-api", "sc-service", + "sp-blockchain", "sp-core", "sp-runtime", "url", diff --git a/cumulus/client/cli/Cargo.toml b/cumulus/client/cli/Cargo.toml index c1b27673c406..af4912e2b2d4 100644 --- a/cumulus/client/cli/Cargo.toml +++ b/cumulus/client/cli/Cargo.toml @@ -18,3 +18,4 @@ sc-chain-spec = { path = "../../../substrate/client/chain-spec" } sc-service = { path = "../../../substrate/client/service" } sp-core = { path = "../../../substrate/primitives/core" } sp-runtime = { path = "../../../substrate/primitives/runtime" } +sp-blockchain = { path = "../../../substrate/primitives/blockchain" } diff --git a/cumulus/client/cli/src/lib.rs b/cumulus/client/cli/src/lib.rs index 7e78afe6efb4..1cebecb00431 100644 --- a/cumulus/client/cli/src/lib.rs +++ b/cumulus/client/cli/src/lib.rs @@ -23,20 +23,18 @@ use std::{ io::{self, Write}, net::SocketAddr, path::PathBuf, + sync::Arc, }; use codec::Encode; use sc_chain_spec::ChainSpec; -use sc_client_api::ExecutorProvider; +use sc_client_api::HeaderBackend; use sc_service::{ config::{PrometheusConfig, TelemetryEndpoints}, BasePath, TransactionPoolOptions, }; use sp_core::hexdisplay::HexDisplay; -use sp_runtime::{ - traits::{Block as BlockT, Hash as HashT, Header as HeaderT, Zero}, - StateVersion, -}; +use sp_runtime::traits::{Block as BlockT, Zero}; use url::Url; /// The `purge-chain` command used to remove the whole chain: the parachain and the relay chain. @@ -129,9 +127,9 @@ impl sc_cli::CliConfiguration for PurgeChainCmd { } } -/// Command for exporting the genesis state of the parachain +/// Command for exporting the genesis head data of the parachain #[derive(Debug, clap::Parser)] -pub struct ExportGenesisStateCommand { +pub struct ExportGenesisHeadCommand { /// Output file name or stdout if unspecified. #[arg()] pub output: Option, @@ -145,24 +143,29 @@ pub struct ExportGenesisStateCommand { pub shared_params: sc_cli::SharedParams, } -impl ExportGenesisStateCommand { - /// Run the export-genesis-state command - pub fn run( - &self, - chain_spec: &dyn ChainSpec, - client: &impl ExecutorProvider, - ) -> sc_cli::Result<()> { - let state_version = sc_chain_spec::resolve_state_version_from_wasm( - &chain_spec.build_storage()?, - client.executor(), - )?; - - let block: Block = generate_genesis_block(chain_spec, state_version)?; - let raw_header = block.header().encode(); +impl ExportGenesisHeadCommand { + /// Run the export-genesis-head command + pub fn run(&self, client: Arc) -> sc_cli::Result<()> + where + B: BlockT, + C: HeaderBackend + 'static, + { + let genesis_hash = client.hash(Zero::zero())?.ok_or(sc_cli::Error::Client( + sp_blockchain::Error::Backend( + "Failed to lookup genesis block hash when exporting genesis head data.".into(), + ), + ))?; + let genesis_header = client.header(genesis_hash)?.ok_or(sc_cli::Error::Client( + sp_blockchain::Error::Backend( + "Failed to lookup genesis header by hash when exporting genesis head data.".into(), + ), + ))?; + + let raw_header = genesis_header.encode(); let output_buf = if self.raw { raw_header } else { - format!("0x{:?}", HexDisplay::from(&block.header().encode())).into_bytes() + format!("0x{:?}", HexDisplay::from(&genesis_header.encode())).into_bytes() }; if let Some(output) = &self.output { @@ -175,43 +178,7 @@ impl ExportGenesisStateCommand { } } -/// Generate the genesis block from a given ChainSpec. -pub fn generate_genesis_block( - chain_spec: &dyn ChainSpec, - genesis_state_version: StateVersion, -) -> Result { - let storage = chain_spec.build_storage()?; - - let child_roots = storage.children_default.iter().map(|(sk, child_content)| { - let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - child_content.data.clone().into_iter().collect(), - genesis_state_version, - ); - (sk.clone(), state_root.encode()) - }); - let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - storage.top.clone().into_iter().chain(child_roots).collect(), - genesis_state_version, - ); - - let extrinsics_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( - Vec::new(), - genesis_state_version, - ); - - Ok(Block::new( - <::Header as HeaderT>::new( - Zero::zero(), - extrinsics_root, - state_root, - Default::default(), - Default::default(), - ), - Default::default(), - )) -} - -impl sc_cli::CliConfiguration for ExportGenesisStateCommand { +impl sc_cli::CliConfiguration for ExportGenesisHeadCommand { fn shared_params(&self) -> &sc_cli::SharedParams { &self.shared_params } diff --git a/cumulus/parachain-template/node/src/cli.rs b/cumulus/parachain-template/node/src/cli.rs index 098f59b0f373..73ef996b7504 100644 --- a/cumulus/parachain-template/node/src/cli.rs +++ b/cumulus/parachain-template/node/src/cli.rs @@ -24,8 +24,11 @@ pub enum Subcommand { /// Remove the whole chain. PurgeChain(cumulus_client_cli::PurgeChainCmd), - /// Export the genesis state of the parachain. - ExportGenesisState(cumulus_client_cli::ExportGenesisStateCommand), + /// Export the genesis head data of the parachain. + /// + /// Head data is the encoded block header. + #[command(alias = "export-genesis-state")] + ExportGenesisHead(cumulus_client_cli::ExportGenesisHeadCommand), /// Export the genesis wasm of the parachain. ExportGenesisWasm(cumulus_client_cli::ExportGenesisWasmCommand), diff --git a/cumulus/parachain-template/node/src/command.rs b/cumulus/parachain-template/node/src/command.rs index 4dd8463f6be6..6ddb68a359a7 100644 --- a/cumulus/parachain-template/node/src/command.rs +++ b/cumulus/parachain-template/node/src/command.rs @@ -162,12 +162,12 @@ pub fn run() -> Result<()> { cmd.run(config, polkadot_config) }) }, - Some(Subcommand::ExportGenesisState(cmd)) => { + Some(Subcommand::ExportGenesisHead(cmd)) => { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| { let partials = new_partial(&config)?; - cmd.run(&*config.chain_spec, &*partials.client) + cmd.run(partials.client) }) }, Some(Subcommand::ExportGenesisWasm(cmd)) => { diff --git a/cumulus/polkadot-parachain/src/cli.rs b/cumulus/polkadot-parachain/src/cli.rs index 63e4baf27aeb..fec6e144e40f 100644 --- a/cumulus/polkadot-parachain/src/cli.rs +++ b/cumulus/polkadot-parachain/src/cli.rs @@ -45,7 +45,8 @@ pub enum Subcommand { PurgeChain(cumulus_client_cli::PurgeChainCmd), /// Export the genesis state of the parachain. - ExportGenesisState(cumulus_client_cli::ExportGenesisStateCommand), + #[command(alias = "export-genesis-state")] + ExportGenesisHead(cumulus_client_cli::ExportGenesisHeadCommand), /// Export the genesis wasm of the parachain. ExportGenesisWasm(cumulus_client_cli::ExportGenesisWasmCommand), diff --git a/cumulus/polkadot-parachain/src/command.rs b/cumulus/polkadot-parachain/src/command.rs index 98dcc2fea4a6..d1b020cc7c90 100644 --- a/cumulus/polkadot-parachain/src/command.rs +++ b/cumulus/polkadot-parachain/src/command.rs @@ -530,10 +530,10 @@ pub fn run() -> Result<()> { cmd.run(config, polkadot_config) }) }, - Some(Subcommand::ExportGenesisState(cmd)) => { + Some(Subcommand::ExportGenesisHead(cmd)) => { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| { - construct_partials!(config, |partials| cmd.run(&*config.chain_spec, &*partials.client)) + construct_partials!(config, |partials| cmd.run(partials.client)) }) }, Some(Subcommand::ExportGenesisWasm(cmd)) => { diff --git a/cumulus/test/service/src/cli.rs b/cumulus/test/service/src/cli.rs index ef1159a3c1f8..3dc5b8e31016 100644 --- a/cumulus/test/service/src/cli.rs +++ b/cumulus/test/service/src/cli.rs @@ -16,6 +16,7 @@ use std::{net::SocketAddr, path::PathBuf}; +use cumulus_client_cli::{ExportGenesisHeadCommand, ExportGenesisWasmCommand}; use polkadot_service::{ChainSpec, ParaId, PrometheusConfig}; use sc_cli::{ CliConfiguration, DefaultConfigurationValues, ImportParams, KeystoreParams, NetworkParams, @@ -60,45 +61,13 @@ pub enum Subcommand { BuildSpec(sc_cli::BuildSpecCmd), /// Export the genesis state of the parachain. - ExportGenesisState(ExportGenesisStateCommand), + #[command(alias = "export-genesis-state")] + ExportGenesisHead(ExportGenesisHeadCommand), /// Export the genesis wasm of the parachain. ExportGenesisWasm(ExportGenesisWasmCommand), } -#[derive(Debug, clap::Parser)] -#[group(skip)] -pub struct ExportGenesisStateCommand { - #[arg(default_value_t = 2000u32)] - pub parachain_id: u32, - - #[command(flatten)] - pub base: cumulus_client_cli::ExportGenesisStateCommand, -} - -impl CliConfiguration for ExportGenesisStateCommand { - fn shared_params(&self) -> &SharedParams { - &self.base.shared_params - } -} - -/// Command for exporting the genesis wasm file. -#[derive(Debug, clap::Parser)] -#[group(skip)] -pub struct ExportGenesisWasmCommand { - #[arg(default_value_t = 2000u32)] - pub parachain_id: u32, - - #[command(flatten)] - pub base: cumulus_client_cli::ExportGenesisWasmCommand, -} - -impl CliConfiguration for ExportGenesisWasmCommand { - fn shared_params(&self) -> &SharedParams { - &self.base.shared_params - } -} - #[derive(Debug)] pub struct RelayChainCli { /// The actual relay chain cli object. diff --git a/cumulus/test/service/src/genesis.rs b/cumulus/test/service/src/genesis.rs index d4a9a2256264..be4b0427b2ee 100644 --- a/cumulus/test/service/src/genesis.rs +++ b/cumulus/test/service/src/genesis.rs @@ -15,11 +15,50 @@ // along with Cumulus. If not, see . use codec::Encode; -use cumulus_client_cli::generate_genesis_block; use cumulus_primitives_core::ParaId; use cumulus_test_runtime::Block; use polkadot_primitives::HeadData; -use sp_runtime::traits::Block as BlockT; +use sc_chain_spec::ChainSpec; +use sp_runtime::{ + traits::{Block as BlockT, Hash as HashT, Header as HeaderT, Zero}, + StateVersion, +}; + +/// Generate a simple test genesis block from a given ChainSpec. +pub fn generate_genesis_block( + chain_spec: &dyn ChainSpec, + genesis_state_version: StateVersion, +) -> Result { + let storage = chain_spec.build_storage()?; + + let child_roots = storage.children_default.iter().map(|(sk, child_content)| { + let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( + child_content.data.clone().into_iter().collect(), + genesis_state_version, + ); + (sk.clone(), state_root.encode()) + }); + let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( + storage.top.clone().into_iter().chain(child_roots).collect(), + genesis_state_version, + ); + + let extrinsics_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( + Vec::new(), + genesis_state_version, + ); + + Ok(Block::new( + <::Header as HeaderT>::new( + Zero::zero(), + extrinsics_root, + state_root, + Default::default(), + Default::default(), + ), + Default::default(), + )) +} /// Returns the initial head data for a parachain ID. pub fn initial_head_data(para_id: ParaId) -> HeadData { diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 627d060d8a0c..586c4603c76a 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -22,7 +22,9 @@ pub mod bench_utils; pub mod chain_spec; -mod genesis; + +/// Utilities for creating test genesis block and head data +pub mod genesis; use runtime::AccountId; use sc_executor::{HeapAllocStrategy, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY}; diff --git a/cumulus/test/service/src/main.rs b/cumulus/test/service/src/main.rs index 55a0f12d671a..aace92ca965d 100644 --- a/cumulus/test/service/src/main.rs +++ b/cumulus/test/service/src/main.rs @@ -16,16 +16,14 @@ mod cli; -use std::{io::Write, sync::Arc}; +use std::sync::Arc; use cli::{RelayChainCli, Subcommand, TestCollatorCli}; -use cumulus_client_cli::generate_genesis_block; use cumulus_primitives_core::{relay_chain::CollatorPair, ParaId}; -use cumulus_test_service::AnnounceBlockFn; +use cumulus_test_service::{new_partial, AnnounceBlockFn}; use polkadot_service::runtime_traits::AccountIdConversion; use sc_cli::{CliConfiguration, SubstrateCli}; -use sp_core::{hexdisplay::HexDisplay, Encode, Pair}; -use sp_runtime::traits::Block; +use sp_core::Pair; pub fn wrap_announce_block() -> Box AnnounceBlockFn> { tracing::info!("Block announcements disabled."); @@ -44,38 +42,16 @@ fn main() -> Result<(), sc_cli::Error> { runner.sync_run(|config| cmd.run(config.chain_spec, config.network)) }, - Some(Subcommand::ExportGenesisState(params)) => { - let mut builder = sc_cli::LoggerBuilder::new(""); - builder.with_profiling(sc_tracing::TracingReceiver::Log, ""); - let _ = builder.init(); - - let spec = - cli.load_spec(¶ms.base.shared_params.chain.clone().unwrap_or_default())?; - let state_version = cumulus_test_service::runtime::VERSION.state_version(); - - let block: parachains_common::Block = generate_genesis_block(&*spec, state_version)?; - let raw_header = block.header().encode(); - let output_buf = if params.base.raw { - raw_header - } else { - format!("0x{:?}", HexDisplay::from(&block.header().encode())).into_bytes() - }; - - if let Some(output) = ¶ms.base.output { - std::fs::write(output, output_buf)?; - } else { - std::io::stdout().write_all(&output_buf)?; - } - - Ok(()) + Some(Subcommand::ExportGenesisHead(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.sync_run(|mut config| { + let partial = new_partial(&mut config, false)?; + cmd.run(partial.client) + }) }, Some(Subcommand::ExportGenesisWasm(cmd)) => { let runner = cli.create_runner(cmd)?; - runner.sync_run(|_config| { - let parachain_id = ParaId::from(cmd.parachain_id); - let spec = cumulus_test_service::get_chain_spec(Some(parachain_id)); - cmd.base.run(&spec) - }) + runner.sync_run(|config| cmd.run(&*config.chain_spec)) }, None => { let log_filters = cli.run.normalize().log_filters(); diff --git a/polkadot/parachain/test-parachains/adder/collator/src/cli.rs b/polkadot/parachain/test-parachains/adder/collator/src/cli.rs index 14b259706835..f81e4cc0fff6 100644 --- a/polkadot/parachain/test-parachains/adder/collator/src/cli.rs +++ b/polkadot/parachain/test-parachains/adder/collator/src/cli.rs @@ -24,16 +24,16 @@ use sc_cli::SubstrateCli; pub enum Subcommand { /// Export the genesis state of the parachain. #[command(name = "export-genesis-state")] - ExportGenesisState(ExportGenesisStateCommand), + ExportGenesisState(ExportGenesisHeadCommand), /// Export the genesis wasm of the parachain. #[command(name = "export-genesis-wasm")] ExportGenesisWasm(ExportGenesisWasmCommand), } -/// Command for exporting the genesis state of the parachain +/// Command for exporting the genesis head data of the parachain #[derive(Debug, Parser)] -pub struct ExportGenesisStateCommand {} +pub struct ExportGenesisHeadCommand {} /// Command for exporting the genesis wasm file. #[derive(Debug, Parser)] diff --git a/polkadot/parachain/test-parachains/undying/collator/src/cli.rs b/polkadot/parachain/test-parachains/undying/collator/src/cli.rs index d04122f2f689..9572887a51a2 100644 --- a/polkadot/parachain/test-parachains/undying/collator/src/cli.rs +++ b/polkadot/parachain/test-parachains/undying/collator/src/cli.rs @@ -25,16 +25,16 @@ use std::path::PathBuf; pub enum Subcommand { /// Export the genesis state of the parachain. #[command(name = "export-genesis-state")] - ExportGenesisState(ExportGenesisStateCommand), + ExportGenesisState(ExportGenesisHeadCommand), /// Export the genesis wasm of the parachain. #[command(name = "export-genesis-wasm")] ExportGenesisWasm(ExportGenesisWasmCommand), } -/// Command for exporting the genesis state of the parachain +/// Command for exporting the genesis head data of the parachain #[derive(Debug, Parser)] -pub struct ExportGenesisStateCommand { +pub struct ExportGenesisHeadCommand { /// Output file name or stdout if unspecified. #[arg()] pub output: Option, diff --git a/prdoc/pr_2331.prdoc b/prdoc/pr_2331.prdoc new file mode 100644 index 000000000000..e3daf4c45bd4 --- /dev/null +++ b/prdoc/pr_2331.prdoc @@ -0,0 +1,17 @@ +title: Rename `ExportGenesisStateCommand` to `ExportGenesisHeadCommand` + +doc: + - audience: Node Operator + description: | + The `export-genesis-state` subcommand is now called `export-gensis-head`, but + `export-genesis-state` stays as an alias to not break any scripts. + + - audience: Node Dev + description: | + The struct `ExportGenesisStateCommand` is now called `ExportGenesisHeadCommand`. + So, you only need to rename the import and usage. The `run` function is now + taking only a `client` as argument to fetch the genesis header. This way + the exported genesis head is respecting custom genesis block builders. + +crates: + - name: "cumulus-client-cli"