diff --git a/Cargo.lock b/Cargo.lock index 1bcbf12..c61b460 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3942,9 +3942,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.4" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -3954,9 +3954,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -3965,9 +3965,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" @@ -4287,9 +4287,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.197" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -4315,9 +4315,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -7155,6 +7155,8 @@ dependencies = [ "reqwest", "rustc_version", "rustls", + "serde", + "serde_yaml", "solana-accounts-db", "solana-clap-v3-utils", "solana-core", diff --git a/Cargo.toml b/Cargo.toml index 9843f97..a36e78d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ openssl = "0.10.66" rand = "0.8.5" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.11", default-features = false, features = ["quic"] } +serde = "1.0.208" +serde_yaml = "0.9.34" solana-accounts-db = "1.18.20" solana-clap-v3-utils = "1.18.20" solana-core = "1.18.20" diff --git a/PROGRESS.md b/PROGRESS.md index e3414bf..5383274 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -79,11 +79,11 @@ By here: - We can deploy bootstrap, N validators, M RPC nodes, and C clients with various command line configurations - We can control the how and where we deploy kubernetes pods -- [ ] Other Features +- [x] Other Features - [x] Heterogeneous Clusters (i.e. multiple validator versions) - [x] Deploy specific commit - [x] Generic Clients - - [ ] Deploy with user-defined stake distribution + - [x] Deploy with user-defined stake distribution By here: - We can deploy bootstrap, N validators, M RPC nodes, and C clients with various command line configurations @@ -91,4 +91,11 @@ By here: - We can deploy multiple cluster versions and have them interact with each other - We can define a stake distribution for our cluster +Features TODO +- [ ] Heterogenous Agave/Firedancer clusters +- [ ] Latency and packet drop simulation +- [ ] Feature gate activation +- [ ] High Level: Usage scheduling + - based on a user's deployment scale, need time-based user multiplexing of infrastructure + DONE diff --git a/README.md b/README.md index f3fef73..36bd961 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,17 @@ In Validator Lab we can deploy and test new validator features quickly and easil ## How to run +### Requirements +1) Docker. Create `docker` group add user to `docker` group +``` +sudo usermod -aG docker $USER +newgrp docker +``` +2) jq +``` +sudo apt install jq +``` + ### Setup Ensure you have the proper permissions to connect to the Monogon Kubernetes endpoint. Reach out to Leo on slack if you need the key (you do if you haven't asked him in the past). @@ -79,13 +90,15 @@ cargo run --bin cluster -- --base-image # e.g. ubuntu:20.04 --image-name # e.g. cluster-image # validator config + --skip-primordial-accounts --full-rpc - --internal-node-sol - --internal-node-stake-sol + --internal-node-sol + --internal-node-stake-sol # kubernetes config --cpu-requests --memory-requests # deploy with clients + bench-tps -c --client-type --client-to-run @@ -107,6 +120,41 @@ For client Version >= 2.0.0 --bench-tps-args 'tx-count=5000 keypair-multiplier=4 threads=16 num-lamports-per-account=200000000 sustained tpu-connection-pool-size=8 thread-batch-sleep-ms=0 commitment-config=processed' ``` +## Baking Validator Stakes into Genesis +- You can bake validator accounts and delegated stakes into genesis creation by passing in `--validator-balances-file `. This way when the cluster boots up, all validators will consistently be in the leader schedule and no need to wait for stake to warm up. In the validator balances file, you can set specific validator balances and stake amounts. +The validator balances file has the following yaml format: +``` +--- +v0: + balances_lamports: + stake_lamports: +v1: + balances_lamports: + stake_lamports: +... +vN: + balances_lamports: + stake_lamports: +``` +^ Note, the file must have the `v0`, `v1`, ..., `vN` format. The number of validators in this file must match `--num-validators ` + +For example, we could create: `validator-balances.yml` and have it look like: +``` +--- +v0: + balance_lamports: 400000000000 + stake_lamports: 40000000000 +v1: + balance_lamports: 200000000000 + stake_lamports: 20000000000 +v2: + balance_lamports: 300000000000 + stake_lamports: 30000000000 +``` + +- If you do not want to bake stakes into genesis and instead want the stake to warm up after deplyoyment, pass in the flag `--skip-primordial-stakes` and leave out `--validator-balances` +- `--internal-node-sol`, `--internal-node-stake-sol`, are `--comission` are only valid with `--skip-primordial-stakes` + ## Metrics 1) Setup metrics database: ``` @@ -128,7 +176,7 @@ You can add in RPC nodes. These sit behind a load balancer. Load balancer distri --num-rpc-nodes ``` -## Heterogeneous Clusters +## Heterogeneous Agave Clusters You can deploy a cluster with heterogeneous validator versions For example, say you want to deploy a cluster with the following nodes: * 1 bootstrap, 3 validators, 1 rpc-node, and 1 client running some agave-repo local commit @@ -155,7 +203,9 @@ cargo run --bin cluster -- -n --registry --release-channe For steps (2) and (3), when using `--no-bootstrap`, we assume that the directory at `--cluster-data-path ` has the correct genesis, bootstrap identity, and faucet account stored. These are all created in step (1). -Note: We can't deploy heterogeneous clusters across v1.17 and v1.18 due to feature differences. Hope to fix this in the future. Have something where we can specifically define which features to enable. +Notes: +1) We can't deploy heterogeneous clusters across v1.17 and v1.18 due to feature differences. Hope to fix this in the future. Have something where we can specifically define which features to enable. +2) Heterogenous clusters with primordial stakes baked into genesis is not supported yet ## Querying the RPC from outside the cluster The cluster now has an external IP/port that can be queried to reach the cluster RPC. The external RPC port will be logged during cluster boot, e.g.: diff --git a/src/genesis.rs b/src/genesis.rs index a5fd9a9..85c8490 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -2,12 +2,14 @@ use { crate::{fetch_spl, new_spinner_progress_bar, NodeType, SOLANA_RELEASE, SUN, WRITING}, log::*, rand::Rng, + serde::{Deserialize, Serialize}, solana_core::gen_keys::GenKeys, solana_sdk::{ native_token::sol_to_lamports, - signature::{write_keypair_file, Keypair}, + signature::{write_keypair_file, Keypair, Signer}, }, std::{ + collections::HashMap, error::Error, fs::{File, OpenOptions}, io::{self, BufRead, BufWriter, Read, Write}, @@ -24,6 +26,40 @@ pub const DEFAULT_INTERNAL_NODE_SOL: f64 = 100.0; pub const DEFAULT_BOOTSTRAP_NODE_STAKE_SOL: f64 = 10.0; pub const DEFAULT_BOOTSTRAP_NODE_SOL: f64 = 100.0; pub const DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE: u64 = 42; +const VALIDATOR_ACCOUNTS_KEYPAIR_COUNT: usize = 3; +const RPC_ACCOUNTS_KEYPAIR_COUNT: usize = 1; + +#[derive(Debug, Deserialize)] +struct ValidatorStakes { + balance_lamports: u64, + stake_lamports: u64, +} + +fn generate_filename(node_type: &NodeType, account_type: &str, index: usize) -> String { + match node_type { + NodeType::Bootstrap => format!("{node_type}/{account_type}.json"), + NodeType::Standard | NodeType::RPC => { + format!("{node_type}-{account_type}-{index}.json") + } + NodeType::Client(_, _) => panic!("Client type not supported"), + } +} + +#[derive(Serialize, Deserialize)] +struct ValidatorAccountsFile { + validator_accounts: Vec, +} + +/// Info needed to create a staked validator account, +/// including relevant balances and vote- and stake-account addresses +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct StakedValidatorAccountInfo { + pub balance_lamports: u64, + pub stake_lamports: u64, + pub identity_account: String, + pub vote_account: String, + pub stake_account: String, +} fn parse_spl_genesis_file( spl_file: &PathBuf, @@ -53,6 +89,7 @@ fn parse_spl_genesis_file( Ok(args) } +#[derive(Debug)] pub struct GenesisFlags { pub hashes_per_tick: String, pub slots_per_epoch: Option, @@ -61,37 +98,13 @@ pub struct GenesisFlags { pub enable_warmup_epochs: bool, pub max_genesis_archive_unpacked_size: Option, pub cluster_type: String, - pub bootstrap_validator_sol: Option, - pub bootstrap_validator_stake_sol: Option, + pub bootstrap_validator_sol: f64, + pub bootstrap_validator_stake_sol: f64, pub commission: u8, -} - -impl std::fmt::Display for GenesisFlags { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "GenesisFlags {{\n\ - hashes_per_tick: {:?},\n\ - slots_per_epoch: {:?},\n\ - target_lamports_per_signature: {:?},\n\ - faucet_lamports: {:?},\n\ - enable_warmup_epochs: {},\n\ - max_genesis_archive_unpacked_size: {:?},\n\ - cluster_type: {}\n\ - bootstrap_validator_sol: {:?},\n\ - bootstrap_validator_stake_sol: {:?},\n\ - }}", - self.hashes_per_tick, - self.slots_per_epoch, - self.target_lamports_per_signature, - self.faucet_lamports, - self.enable_warmup_epochs, - self.max_genesis_archive_unpacked_size, - self.cluster_type, - self.bootstrap_validator_sol, - self.bootstrap_validator_stake_sol, - ) - } + pub internal_node_sol: f64, + pub internal_node_stake_sol: f64, + pub skip_primordial_stakes: bool, + pub validator_accounts_file: Option, } fn append_client_accounts_to_file( @@ -121,11 +134,18 @@ fn append_client_accounts_to_file( pub struct Genesis { config_dir: PathBuf, key_generator: GenKeys, + pub validator_stakes_file: Option, + validator_accounts: HashMap, pub flags: GenesisFlags, } impl Genesis { - pub fn new(config_dir: PathBuf, flags: GenesisFlags, retain_previous_genesis: bool) -> Self { + pub fn new( + config_dir: PathBuf, + validator_stakes_file: Option, + flags: GenesisFlags, + retain_previous_genesis: bool, + ) -> Self { // if we are deploying a heterogeneous cluster // all deployments after the first must retain the original genesis directory if !retain_previous_genesis { @@ -140,6 +160,8 @@ impl Genesis { Self { config_dir, key_generator: GenKeys::new(seed), + validator_stakes_file, + validator_accounts: HashMap::default(), flags, } } @@ -173,16 +195,12 @@ impl Genesis { } }; - let account_types: Vec = if let Some(tag) = deployment_tag { - account_types - .into_iter() - .map(|acct| format!("{}-{}", acct, tag)) - .collect() - } else { - account_types + let account_types: Vec = match deployment_tag { + Some(tag) => account_types .into_iter() - .map(|acct| acct.to_string()) - .collect() + .map(|acct| format!("{acct}-{tag}")) + .collect(), + None => account_types.into_iter().map(String::from).collect(), }; let total_accounts_to_generate = number_of_accounts * account_types.len(); @@ -190,34 +208,86 @@ impl Genesis { .key_generator .gen_n_keypairs(total_accounts_to_generate as u64); + if node_type == NodeType::Standard { + self.initialize_validator_accounts(&node_type, &keypairs); + } + self.write_accounts_to_file(&node_type, &account_types, &keypairs)?; Ok(()) } fn write_accounts_to_file( - &self, + &mut self, node_type: &NodeType, account_types: &[String], keypairs: &[Keypair], ) -> Result<(), Box> { - for (i, keypair) in keypairs.iter().enumerate() { - let account_index = i / account_types.len(); - let account = &account_types[i % account_types.len()]; - let filename = match node_type { - NodeType::Bootstrap => { - format!("{node_type}/{account}.json") + let chunk_size = match node_type { + NodeType::Bootstrap | NodeType::Standard => VALIDATOR_ACCOUNTS_KEYPAIR_COUNT, + NodeType::RPC => RPC_ACCOUNTS_KEYPAIR_COUNT, + NodeType::Client(_, _) => return Err("Client type not supported".into()), + }; + for (i, account_type_keypair) in keypairs.chunks_exact(chunk_size).enumerate() { + match node_type { + NodeType::Bootstrap | NodeType::Standard => { + // Create a filename for each type of account based on node type and index + let identity_filename = + generate_filename(node_type, account_types[0].as_str(), i); + let stake_filename = generate_filename(node_type, account_types[1].as_str(), i); + let vote_filename = generate_filename(node_type, account_types[2].as_str(), i); + + write_keypair_file( + &account_type_keypair[0], + self.config_dir.join(identity_filename), + )?; + write_keypair_file( + &account_type_keypair[1], + self.config_dir.join(vote_filename), + )?; + write_keypair_file( + &account_type_keypair[2], + self.config_dir.join(stake_filename), + )?; } - NodeType::Standard | NodeType::RPC => { - format!("{node_type}-{account}-{account_index}.json") + NodeType::RPC => { + let identity_filename = + generate_filename(node_type, account_types[0].as_str(), i); + write_keypair_file( + &account_type_keypair[0], + self.config_dir.join(identity_filename), + )?; } - NodeType::Client(_, _) => panic!("Client type not supported"), + NodeType::Client(_, _) => return Err("Client type not supported".into()), + } + } + + Ok(()) + } + + fn initialize_validator_accounts(&mut self, node_type: &NodeType, keypairs: &[Keypair]) { + if node_type != &NodeType::Standard { + return; + } + for (i, account_type_keypair) in keypairs + .chunks_exact(VALIDATOR_ACCOUNTS_KEYPAIR_COUNT) + .enumerate() + { + let identity_account = account_type_keypair[0].pubkey().to_string(); + let vote_account = account_type_keypair[1].pubkey().to_string(); + let stake_account = account_type_keypair[2].pubkey().to_string(); + + let validator_account = StakedValidatorAccountInfo { + balance_lamports: 0, + stake_lamports: 0, + identity_account, + vote_account, + stake_account, }; - let outfile = self.config_dir.join(&filename); - write_keypair_file(keypair, outfile)?; + let key = format!("v{i}"); + self.validator_accounts.insert(key, validator_account); } - Ok(()) } pub fn create_client_accounts( @@ -310,19 +380,9 @@ impl Genesis { fn setup_genesis_flags(&self) -> Result, Box> { let mut args = vec![ "--bootstrap-validator-lamports".to_string(), - sol_to_lamports( - self.flags - .bootstrap_validator_sol - .unwrap_or(DEFAULT_BOOTSTRAP_NODE_SOL), - ) - .to_string(), + sol_to_lamports(self.flags.bootstrap_validator_sol).to_string(), "--bootstrap-validator-stake-lamports".to_string(), - sol_to_lamports( - self.flags - .bootstrap_validator_stake_sol - .unwrap_or(DEFAULT_BOOTSTRAP_NODE_STAKE_SOL), - ) - .to_string(), + sol_to_lamports(self.flags.bootstrap_validator_stake_sol).to_string(), "--hashes-per-tick".to_string(), self.flags.hashes_per_tick.clone(), "--max-genesis-archive-unpacked-size".to_string(), @@ -378,6 +438,22 @@ impl Genesis { args.push(path); } + if let Some(validator_accounts_file) = &self.flags.validator_accounts_file { + args.push("--validator-accounts-file".to_string()); + args.push( + validator_accounts_file + .clone() + .into_os_string() + .into_string() + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?, + ); + } + if let Some(slots_per_epoch) = self.flags.slots_per_epoch { args.push("--slots-per-epoch".to_string()); args.push(slots_per_epoch.to_string()); @@ -436,4 +512,136 @@ impl Genesis { Ok(()) } + + pub fn create_snapshot(&self, exec_path: &Path) -> Result<(), Box> { + let warp_slot = 1; + let executable_path: PathBuf = exec_path.join("agave-ledger-tool"); + let args = vec![ + "-l".to_string(), + self.config_dir + .join("bootstrap-validator") + .into_os_string() + .into_string() + .unwrap(), + "create-snapshot".to_string(), + "0".to_string(), + self.config_dir + .join("bootstrap-validator") + .into_os_string() + .into_string() + .unwrap(), + "--warp-slot".to_string(), + warp_slot.to_string(), + ]; + let output = Command::new(executable_path) + .args(&args) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .map_err(Box::new)?; + + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).into()); + } + info!("Snapshot creation complete"); + Ok(()) + } + + pub fn get_bank_hash(&self, exec_path: &Path) -> Result> { + let executable_path: PathBuf = exec_path.join("agave-ledger-tool"); + let agave_output = Command::new(executable_path) + .args([ + "-l", + self.config_dir + .join("bootstrap-validator") + .into_os_string() + .into_string() + .unwrap() + .as_str(), + "verify", + "--halt-at-slot", + "0", + "--print-bank-hash", + "--output", + "json", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + .stdout + .expect("Failed to capture agave-ledger-tool output"); + + // get bank hash + let jq_output = Command::new("jq") + .arg("-r") + .arg(".hash") + .stdin(agave_output) + .output()?; + + let bank_hash = String::from_utf8_lossy(&jq_output.stdout) + .trim() + .to_string(); + + info!("bankHash: {bank_hash}"); + + Ok(bank_hash) + } + + pub fn load_validator_genesis_stakes_from_file(&mut self) -> io::Result<()> { + let validator_stakes_file = match &self.validator_stakes_file { + Some(file) => file, + None => { + warn!("validator_stakes_file is None"); + return Ok(()); + } + }; + let file = File::open(validator_stakes_file)?; + let validator_stakes: HashMap = serde_yaml::from_reader(file) + .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{err:?}")))?; + + if validator_stakes.len() != self.validator_accounts.len() { + return Err(io::Error::new( + io::ErrorKind::Other, + format!( + "Number of validator stakes ({}) does not match number of validator accounts ({})", + validator_stakes.len(), self.validator_accounts.len() + ), + )); + } + + // match `validator_stakes` with corresponding `StakedValidatorAccountInfo` and update balance and stake + for (key, stake) in validator_stakes { + if let Some(validator_account) = self.validator_accounts.get_mut(&key) { + validator_account.balance_lamports = stake.balance_lamports; + validator_account.stake_lamports = stake.stake_lamports; + } else { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Validator account for key '{key}' not found"), + )); + } + } + + self.write_validator_genesis_accouts_to_file()?; + Ok(()) + } + + // Creates yaml file solana-genesis can read in for `--validator-stakes-file ` + // Yaml file created with the following format dictated in agave/genesis/README.md + // See: https://github.com/anza-xyz/agave/blob/master/genesis/README.md#3-through-the-validator-accounts-file-flag + fn write_validator_genesis_accouts_to_file(&mut self) -> std::io::Result<()> { + let accounts_file = ValidatorAccountsFile { + validator_accounts: self.validator_accounts.values().cloned().collect(), + }; + + let output_file = self.config_dir.join("validator-genesis-accounts.yml"); + self.flags.validator_accounts_file = Some(output_file.clone()); + + let file = File::create(&output_file)?; + serde_yaml::to_writer(file, &accounts_file) + .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{err:?}")))?; + + info!("Validator genesis accounts successfully written to {output_file:?}"); + Ok(()) + } } diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 70ca068..32b4c80 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -89,6 +89,10 @@ impl<'a> Kubernetes<'a> { self.validator_config.shred_version = Some(shred_version); } + pub fn set_bank_hash(&mut self, bank_hash: String) { + self.validator_config.bank_hash = Some(bank_hash); + } + async fn get_namespaces(&self) -> Result, kube::Error> { let namespaces: Api = Api::all(self.k8s_client.clone()); let namespace_list = namespaces.list(&ListParams::default()).await?; @@ -346,6 +350,21 @@ impl<'a> Kubernetes<'a> { Self::generate_full_rpc_flags(&mut flags); } + if let Some(shred_version) = self.validator_config.shred_version { + flags.push("--expected-shred-version".to_string()); + flags.push(shred_version.to_string()); + } + + if let Some(bank_hash) = &self.validator_config.bank_hash { + flags.push("--expected-bank-hash".to_string()); + flags.push(bank_hash.to_string()); + } + + if !self.validator_config.skip_primordial_stakes { + flags.push("--wait-for-supermajority".to_string()); + flags.push("1".to_string()); + } + flags } @@ -550,20 +569,32 @@ impl<'a> Kubernetes<'a> { Self::generate_full_rpc_flags(&mut flags); } - flags.push("--internal-node-stake-sol".to_string()); - flags.push(self.validator_config.internal_node_stake_sol.to_string()); + if self.validator_config.skip_primordial_stakes { + flags.push("--internal-node-stake-sol".to_string()); + flags.push(self.validator_config.internal_node_stake_sol.to_string()); - flags.push("--commission".to_string()); - flags.push(self.validator_config.commission.to_string()); + flags.push("--internal-node-sol".to_string()); + flags.push(self.validator_config.internal_node_sol.to_string()); - flags.push("--internal-node-sol".to_string()); - flags.push(self.validator_config.internal_node_sol.to_string()); + flags.push("--commission".to_string()); + flags.push(self.validator_config.commission.to_string()); + } if let Some(shred_version) = self.validator_config.shred_version { flags.push("--expected-shred-version".to_string()); flags.push(shred_version.to_string()); } + if let Some(bank_hash) = &self.validator_config.bank_hash { + flags.push("--expected-bank-hash".to_string()); + flags.push(bank_hash.to_string()); + } + + if !self.validator_config.skip_primordial_stakes { + flags.push("--wait-for-supermajority".to_string()); + flags.push("1".to_string()); + } + self.add_known_validators_if_exists(&mut flags); flags diff --git a/src/main.rs b/src/main.rs index 63416ed..d569466 100644 --- a/src/main.rs +++ b/src/main.rs @@ -238,14 +238,16 @@ fn parse_matches() -> clap::ArgMatches { .long("internal-node-sol") .takes_value(true) .default_value(&DEFAULT_INTERNAL_NODE_SOL.to_string()) - .help("Amount to fund internal nodes in genesis config."), + .conflicts_with("validator_balances_file") + .help("Amount to fund internal nodes in genesis"), ) .arg( Arg::with_name("internal_node_stake_sol") .long("internal-node-stake-sol") .takes_value(true) .default_value(&DEFAULT_INTERNAL_NODE_STAKE_SOL.to_string()) - .help("Amount to stake internal nodes (Sol)."), + .conflicts_with("validator_balances_file") + .help("Amount to stake internal nodes (Sol) in genesis"), ) .arg( Arg::with_name("commission") @@ -253,12 +255,33 @@ fn parse_matches() -> clap::ArgMatches { .value_name("PERCENTAGE") .takes_value(true) .default_value("100") + .conflicts_with("validator_balances_file") .help("The commission taken by nodes on staking rewards (0-100)") ) + .arg( + Arg::with_name("skip_primordial_stakes") + .long("skip-primordial-stakes") + .help("Do not bake validator stake accounts into genesis. + Validators will be funded and staked after the cluster boots. + This will result in several epochs for all of the stake to warm up"), + ) + .arg( + Arg::with_name("validator_balances_file") + .long("validator-balances-file") + .value_name("FILENAME") + .takes_value(true) + .help("The location of validator balances and stake balances for validator accounts"), + ) + .group( + ArgGroup::with_name("validation_stake_config") + .args(&["skip_primordial_stakes", "validator_balances_file"]) + .required(true) // Only one of these args must be present + .multiple(false), // Passing both in will result in an error + ) .arg( Arg::with_name("no_restart") .long("no-restart") - .help("Validator config. If set, validators will not restart after \ + .help("Validator config. If set, validators will not restart after exiting for any reason."), ) //RPC config @@ -547,6 +570,12 @@ async fn main() -> Result<(), Box> { let commission = value_t_or_exit!(matches, "commission", u8); + let internal_node_stake_sol = value_t_or_exit!(matches, "internal_node_stake_sol", f64); + let internal_node_sol = + value_t_or_exit!(matches, "internal_node_sol", f64) + internal_node_stake_sol; + + let skip_primordial_stakes = matches.is_present("skip_primordial_stakes"); + let genesis_flags = GenesisFlags { hashes_per_tick: matches .value_of("hashes_per_tick") @@ -581,33 +610,26 @@ async fn main() -> Result<(), Box> { .value_of("cluster_type") .unwrap_or_default() .to_string(), - bootstrap_validator_sol: matches - .value_of("bootstrap_validator_sol") - .map(|value_str| { - value_str - .parse() - .expect("Invalid value for bootstrap_validator_sol") - }), - bootstrap_validator_stake_sol: matches.value_of("bootstrap_validator_stake_sol").map( - |value_str| { - value_str - .parse() - .expect("Invalid value for bootstrap_validator_stake_sol") - }, + bootstrap_validator_sol: value_t_or_exit!(matches, "bootstrap_validator_sol", f64), + bootstrap_validator_stake_sol: value_t_or_exit!( + matches, + "bootstrap_validator_stake_sol", + f64 ), commission, + internal_node_sol, + internal_node_stake_sol, + skip_primordial_stakes, + validator_accounts_file: None, }; - let internal_node_stake_sol = value_t_or_exit!(matches, "internal_node_stake_sol", f64); - let internal_node_sol = - value_t_or_exit!(matches, "internal_node_sol", f64) + internal_node_stake_sol; - let limit_ledger_size = value_t_or_exit!(matches, "limit_ledger_size", u64); let mut validator_config = ValidatorConfig { internal_node_sol, internal_node_stake_sol, commission, shred_version: None, // set after genesis created + bank_hash: None, //set after snapshot created max_ledger_size: if limit_ledger_size < DEFAULT_MIN_MAX_LEDGER_SHREDS { clap::Error::with_description( format!("The provided --limit-ledger-size value was too small, the minimum value is {DEFAULT_MIN_MAX_LEDGER_SHREDS}"), @@ -623,6 +645,7 @@ async fn main() -> Result<(), Box> { enable_full_rpc: matches.is_present("enable_full_rpc"), known_validators: vec![], restart: !matches.is_present("no_restart"), + skip_primordial_stakes, }; if num_rpc_nodes == 0 && !validator_config.enable_full_rpc { @@ -671,10 +694,20 @@ async fn main() -> Result<(), Box> { let retain_previous_genesis = !deploy_bootstrap_validator; let mut genesis = Genesis::new( config_directory.clone(), + matches + .value_of("validator_balances_file") + .map(PathBuf::from), genesis_flags, retain_previous_genesis, ); + // generate standard validator accounts + genesis.generate_accounts(NodeType::Standard, num_validators, Some(&image_tag))?; + info!("Generated {num_validators} validator account(s)"); + + genesis.generate_accounts(NodeType::RPC, num_rpc_nodes, Some(&image_tag))?; + info!("Generated {num_rpc_nodes} rpc account(s)"); + if deploy_bootstrap_validator { genesis.generate_faucet()?; info!("Generated faucet account"); @@ -682,19 +715,23 @@ async fn main() -> Result<(), Box> { genesis.generate_accounts(NodeType::Bootstrap, 1, None)?; info!("Generated bootstrap account"); + if genesis.validator_stakes_file.is_some() { + genesis.load_validator_genesis_stakes_from_file()?; + } + // creates genesis and writes to binary file genesis .generate(cluster_data_root.get_root_path(), &exec_path) .await?; info!("Genesis created"); - } - // generate standard validator accounts - genesis.generate_accounts(NodeType::Standard, num_validators, Some(&image_tag))?; - info!("Generated {num_validators} validator account(s)"); + if !skip_primordial_stakes { + genesis.create_snapshot(&exec_path)?; - genesis.generate_accounts(NodeType::RPC, num_rpc_nodes, Some(&image_tag))?; - info!("Generated {num_rpc_nodes} rpc account(s)"); + let bank_hash = genesis.get_bank_hash(&exec_path)?; + kub_controller.set_bank_hash(bank_hash); + } + } let ledger_dir = config_directory.join("bootstrap-validator"); let shred_version = LedgerHelper::get_shred_version(&ledger_dir)?; diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index 236c150..4e9f7ef 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -65,6 +65,9 @@ while [[ -n $1 ]]; do elif [[ $1 = --dev-halt-at-slot ]]; then # not enabled in net.sh args+=("$1" "$2") shift 2 + elif [[ $1 = --expected-shred-version ]]; then + args+=("$1" "$2") + shift 2 elif [[ $1 = --dynamic-port-range ]]; then # not enabled in net.sh args+=("$1" "$2") shift 2 @@ -280,6 +283,7 @@ EOF exit 1 } +run_validator_stake_setup=true positional_args=() while [[ -n $1 ]]; do if [[ ${1:0:1} = - ]]; then @@ -409,6 +413,7 @@ while [[ -n $1 ]]; do shift 2 elif [[ $1 == --wait-for-supermajority ]]; then args+=("$1" "$2") + run_validator_stake_setup=false shift 2 elif [[ $1 == --expected-bank-hash ]]; then args+=("$1" "$2") @@ -605,10 +610,12 @@ run_delegate_stake() { solana --keypair $IDENTITY_FILE stake-account validator-accounts/stake.json } -echo "get airdrop and create vote account" -setup_validator -echo "create stake account and delegate stake" -run_delegate_stake +if $run_validator_stake_setup; then + echo "get airdrop and create vote account" + setup_validator + echo "create stake account and delegate stake" + run_delegate_stake +fi echo running validator: diff --git a/src/validator_config.rs b/src/validator_config.rs index 93fe515..c5bb946 100644 --- a/src/validator_config.rs +++ b/src/validator_config.rs @@ -13,4 +13,6 @@ pub struct ValidatorConfig { pub enable_full_rpc: bool, pub known_validators: Vec, pub restart: bool, + pub bank_hash: Option, + pub skip_primordial_stakes: bool, }