diff --git a/Cargo.lock b/Cargo.lock index 9e73d20..6d15f2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4737,6 +4737,24 @@ dependencies = [ "url 2.5.0", ] +[[package]] +name = "solana-clap-v3-utils" +version = "1.18.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab05348a84bbf3e81a70117d24ca46ce62343530ddfe9e8d84fb95a8061d59b" +dependencies = [ + "chrono", + "clap 3.2.25", + "rpassword", + "solana-remote-wallet", + "solana-sdk", + "solana-zk-token-sdk", + "thiserror", + "tiny-bip39", + "uriparse", + "url 2.5.0", +] + [[package]] name = "solana-cli-config" version = "1.18.9" @@ -7136,6 +7154,7 @@ dependencies = [ "rustc_version", "rustls", "solana-accounts-db", + "solana-clap-v3-utils", "solana-core", "solana-ledger", "solana-logger", diff --git a/Cargo.toml b/Cargo.toml index 3b630c4..8202d2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" [dependencies] bzip2 = "0.4.4" -clap = { version = "3.2.22", features = ["cargo"] } +clap = { version = "3.2.22", features = ["cargo"] } console = "0.15.8" git2 = "0.18.3" indicatif = "0.17.8" @@ -20,6 +20,7 @@ rand = "0.8.5" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.11", default-features = false, features = ["quic"] } solana-accounts-db = "1.18.8" +solana-clap-v3-utils = "1.18.8" solana-core = "1.18.8" solana-ledger = "1.18.8" solana-logger = "1.18.8" diff --git a/PROGRESS.md b/PROGRESS.md index e0b7f63..7298a03 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -19,43 +19,43 @@ - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client -- [ ] Create & Deploy Secrets + - [x] Client +- [x] Create & Deploy Secrets - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client -- [ ] Create & Deploy Selector + - [x] Client +- [x] Create & Deploy Selector - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client -- [ ] Create & Deploy Replica Set + - [x] Client +- [x] Create & Deploy Replica Set - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client -- [ ] Create & Deploy Services + - [x] Client +- [x] Create & Deploy Services - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client - [x] Check Bootstrap is deployed and running - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) -- [ ] Add metrics +- [x] Add metrics - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client -- [ ] Create accounts + - [x] Client +- [x] Create accounts - [x] Validator (regular) - [x] RPC - - [ ] Client -- [ ] Add feature flags to configure: + - [x] Client +- [x] Add feature flags to configure: - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client Above, we start with bootstrap, and then we do validators (regular), and then we do RPCs, then Clients - By the end of the Bootstrap set of PRs, we can diff --git a/README.md b/README.md index 4e539ac..86cdfc0 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,12 @@ cargo run --bin cluster -- # kubernetes config --cpu-requests --memory-requests + # deploy with clients + -c + --client-type + --client-to-run + --client-wait-for-n-nodes + --bench-tps-args ``` ## Metrics @@ -81,7 +87,7 @@ cargo run --bin cluster -- ``` #### RPC Nodes -You can add in RPC nodes. These sit behind a load balancer. Load balancer distributed loads across all RPC nodes that the bootstrap. Set the number of RPC nodes with: +You can add in RPC nodes. These sit behind a load balancer. Load balancer distributed loads across all RPC nodes and the bootstrap. Set the number of RPC nodes with: ``` --num-rpc-nodes ``` diff --git a/src/client_config.rs b/src/client_config.rs new file mode 100644 index 0000000..e38ca98 --- /dev/null +++ b/src/client_config.rs @@ -0,0 +1,12 @@ +use solana_sdk::pubkey::Pubkey; + +#[derive(Clone, Debug)] +pub struct ClientConfig { + pub num_clients: usize, + pub client_type: String, + pub client_to_run: String, + pub bench_tps_args: Vec, + pub client_target_node: Option, + pub client_duration_seconds: u64, + pub client_wait_for_n_nodes: Option, +} diff --git a/src/cluster_images.rs b/src/cluster_images.rs index 9e5fa6c..1e7b100 100644 --- a/src/cluster_images.rs +++ b/src/cluster_images.rs @@ -14,7 +14,7 @@ pub struct ClusterImages { bootstrap: Option, validator: Option, rpc: Option, - _clients: Vec, + clients: Vec, } impl ClusterImages { @@ -23,7 +23,7 @@ impl ClusterImages { ValidatorType::Bootstrap => self.bootstrap = Some(item), ValidatorType::Standard => self.validator = Some(item), ValidatorType::RPC => self.rpc = Some(item), - _ => panic!("{validator_type} not implemented yet!"), + ValidatorType::Client(_) => self.clients.push(item), } } @@ -45,6 +45,15 @@ impl ClusterImages { .ok_or_else(|| "Validator is not available".into()) } + pub fn client(&mut self, client_index: usize) -> Result<&mut Validator, Box> { + if self.clients.is_empty() { + return Err("No Clients available".to_string().into()); + } + self.clients + .get_mut(client_index) + .ok_or_else(|| "Client index out of bounds".to_string().into()) + } + pub fn get_validators(&self) -> impl Iterator { self.bootstrap .iter() @@ -52,4 +61,16 @@ impl ClusterImages { .chain(self.rpc.iter()) .filter_map(Some) } + + pub fn get_clients(&self) -> impl Iterator { + self.clients.iter() + } + + pub fn get_clients_mut(&mut self) -> impl Iterator { + self.clients.iter_mut() + } + + pub fn get_all(&self) -> impl Iterator { + self.get_validators().chain(self.get_clients()) + } } diff --git a/src/docker.rs b/src/docker.rs index fb5e240..0b0df72 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -45,11 +45,18 @@ impl DockerImage { // Put DockerImage in format for building, pushing, and pulling impl Display for DockerImage { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "{}/{}-{}:{}", - self.registry, self.validator_type, self.image_name, self.tag - ) + match self.validator_type { + ValidatorType::Client(index) => write!( + f, + "{}/{}-{}-{}:{}", + self.registry, self.validator_type, index, self.image_name, self.tag + ), + ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => write!( + f, + "{}/{}-{}:{}", + self.registry, self.validator_type, self.image_name, self.tag + ), + } } } @@ -72,17 +79,15 @@ impl DockerConfig { docker_image: &DockerImage, ) -> Result<(), Box> { let validator_type = docker_image.validator_type(); - match validator_type { - ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => (), - ValidatorType::Client => { - return Err(format!( - "Build docker image for validator type: {validator_type} not supported yet" - ) - .into()); + let docker_path = match validator_type { + ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => { + solana_root_path.join(format!("docker-build/{validator_type}")) } - } + ValidatorType::Client(index) => { + solana_root_path.join(format!("docker-build/{validator_type}-{index}")) + } + }; - let docker_path = solana_root_path.join(format!("docker-build/{validator_type}")); self.create_base_image( solana_root_path, docker_image, @@ -100,7 +105,7 @@ impl DockerConfig { docker_path: &PathBuf, validator_type: &ValidatorType, ) -> Result<(), Box> { - self.create_dockerfile(validator_type, docker_path, None)?; + self.create_dockerfile(validator_type, docker_path, solana_root_path, None)?; // We use std::process::Command here because Docker-rs is very slow building dockerfiles // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. @@ -130,7 +135,6 @@ impl DockerConfig { return Err(output.status.to_string().into()); } progress_bar.finish_and_clear(); - info!("{validator_type} image build complete"); Ok(()) } @@ -141,7 +145,7 @@ impl DockerConfig { validator_type: &ValidatorType, ) -> Result<(), Box> { let script_path = docker_dir.join(file_name); - let script_content = validator_type.script()?; + let script_content = validator_type.script(); StartupScripts::write_script_to_file(script_content, &script_path).map_err(|e| e.into()) } @@ -149,6 +153,7 @@ impl DockerConfig { &self, validator_type: &ValidatorType, docker_path: &PathBuf, + solana_root_path: &Path, content: Option<&str>, ) -> Result<(), Box> { if docker_path.exists() { @@ -156,23 +161,19 @@ impl DockerConfig { } fs::create_dir_all(docker_path)?; - match validator_type { + let file_name = format!("{validator_type}-startup-script.sh"); + Self::write_startup_script_to_docker_directory(&file_name, docker_path, validator_type)?; + StartupScripts::write_script_to_file( + StartupScripts::common(), + &docker_path.join("common.sh"), + )?; + + let startup_script_directory = match validator_type { ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => { - let file_name = format!("{validator_type}-startup-script.sh"); - Self::write_startup_script_to_docker_directory( - &file_name, - docker_path, - validator_type, - )?; - StartupScripts::write_script_to_file( - StartupScripts::common(), - &docker_path.join("common.sh"), - )?; + format!("./docker-build/{validator_type}") } - ValidatorType::Client => todo!(), - } - - let startup_script_directory = format!("./docker-build/{validator_type}"); + ValidatorType::Client(index) => format!("./docker-build/{validator_type}-{index}"), + }; let solana_build_directory = if let DeployMethod::ReleaseChannel(_) = self.deploy_method { "solana-release" } else { @@ -191,13 +192,15 @@ USER solana COPY --chown=solana:solana {startup_script_directory} /home/solana/k8s-cluster-scripts RUN chmod +x /home/solana/k8s-cluster-scripts/* COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger -COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/ +COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/bin/ COPY --chown=solana:solana ./{solana_build_directory}/version.yml /home/solana/ -ENV PATH="/home/solana/.cargo/bin:${{PATH}}" +ENV PATH="/home/solana/bin:${{PATH}}" WORKDIR /home/solana +{} "#, - self.base_image + self.base_image, + self.insert_client_accounts_if_present(solana_root_path, validator_type)? ); debug!("dockerfile: {dockerfile:?}"); @@ -208,6 +211,40 @@ WORKDIR /home/solana Ok(()) } + fn insert_client_accounts_if_present( + &self, + solana_root_path: &Path, + validator_type: &ValidatorType, + ) -> Result> { + match validator_type { + ValidatorType::Client(index) => { + let bench_tps_path = + solana_root_path.join(format!("config-k8s/bench-tps-{index}.yml")); + if bench_tps_path.exists() { + Ok(format!( + r#" +COPY --chown=solana:solana ./config-k8s/bench-tps-{index}.yml /home/solana/client-accounts.yml + "# + )) + } else { + Err(format!("{bench_tps_path:?} does not exist!").into()) + } + } + ValidatorType::Bootstrap => { + let client_accounts_path = solana_root_path.join("config-k8s/client-accounts.yml"); + if client_accounts_path.exists() { + Ok(r#" +COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana + "# + .to_string()) + } else { + Ok("".to_string()) + } + } + ValidatorType::Standard | ValidatorType::RPC => Ok("".to_string()), + } + } + pub fn push_image(docker_image: &DockerImage) -> Result> { let command = format!("docker push '{docker_image}'"); let child = Command::new("sh") diff --git a/src/genesis.rs b/src/genesis.rs index 1de9d20..18ba974 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -1,5 +1,7 @@ use { - crate::{fetch_spl, new_spinner_progress_bar, ValidatorType, SUN}, + crate::{ + fetch_spl, new_spinner_progress_bar, release::DeployMethod, ValidatorType, SUN, WRITING, + }, log::*, rand::Rng, solana_core::gen_keys::GenKeys, @@ -9,10 +11,10 @@ use { }, std::{ error::Error, - fs::File, - io::Read, + fs::{File, OpenOptions}, + io::{self, BufRead, BufWriter, Read, Write}, path::{Path, PathBuf}, - process::Command, + process::{Child, Command, Stdio}, result::Result, }, }; @@ -23,6 +25,7 @@ pub const DEFAULT_INTERNAL_NODE_STAKE_SOL: f64 = 10.0; pub const DEFAULT_INTERNAL_NODE_SOL: f64 = 100.0; pub const DEFAULT_BOOTSTRAP_NODE_STAKE_SOL: f64 = 10.0; pub const DEFAULT_BOOTSTRAP_NODE_SOL: f64 = 100.0; +pub const DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE: u64 = 42; fn parse_spl_genesis_file( spl_file: &PathBuf, @@ -92,6 +95,30 @@ impl std::fmt::Display for GenesisFlags { } } +fn append_client_accounts_to_file( + bench_tps_account_path: &PathBuf, //bench-tps-i.yml + client_accounts_path: &PathBuf, //client-accounts.yml +) -> io::Result<()> { + // Open the bench-tps-i.yml file for reading. + let input = File::open(bench_tps_account_path)?; + let reader = io::BufReader::new(input); + + // Open (or create) client-accounts.yml + let output = OpenOptions::new() + .create(true) + .append(true) + .open(client_accounts_path)?; + let mut writer = BufWriter::new(output); + + // Skip first line since it is a header aka "---" in a yaml + for line in reader.lines().skip(1) { + let line = line?; + writeln!(writer, "{line}")?; + } + + Ok(()) +} + pub struct Genesis { config_dir: PathBuf, key_generator: GenKeys, @@ -128,10 +155,6 @@ impl Genesis { validator_type: ValidatorType, number_of_accounts: usize, ) -> Result<(), Box> { - if validator_type == ValidatorType::Client { - return Err("Client valdiator_type in generate_accounts not allowed".into()); - } - info!("generating {number_of_accounts} {validator_type} accounts..."); let account_types = match validator_type { @@ -141,7 +164,9 @@ impl Genesis { ValidatorType::RPC => { vec!["identity"] // no vote or stake account for RPC } - ValidatorType::Client => panic!("Client type not supported"), + ValidatorType::Client(_) => { + return Err("Client valdiator_type in generate_accounts not allowed".into()) + } }; let total_accounts_to_generate = number_of_accounts * account_types.len(); @@ -170,7 +195,7 @@ impl Genesis { ValidatorType::Standard | ValidatorType::RPC => { format!("{validator_type}-{account}-{account_index}.json") } - ValidatorType::Client => panic!("Client type not supported"), + ValidatorType::Client(_) => panic!("Client type not supported"), }; let outfile = self.config_dir.join(&filename); @@ -179,6 +204,92 @@ impl Genesis { Ok(()) } + pub fn create_client_accounts( + &mut self, + number_of_clients: usize, + bench_tps_args: &[String], + target_lamports_per_signature: u64, + config_dir: &Path, + deploy_method: &DeployMethod, + solana_root_path: &Path, + ) -> Result<(), Box> { + if number_of_clients == 0 { + return Ok(()); + } + + let client_accounts_file = config_dir.join("client-accounts.yml"); + + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{WRITING}Creating and writing client accounts...")); + + info!("generating {number_of_clients} client account(s)..."); + let children: Result, _> = (0..number_of_clients) + .map(|i| { + Self::create_client_account( + i, + config_dir, + target_lamports_per_signature, + bench_tps_args, + deploy_method, + solana_root_path, + ) + }) + .collect(); + + for child in children? { + let output = child.wait_with_output()?; + if !output.status.success() { + return Err(output.status.to_string().into()); + } + } + + for i in 0..number_of_clients { + let account_path = config_dir.join(format!("bench-tps-{i}.yml")); + append_client_accounts_to_file(&account_path, &client_accounts_file)?; + } + progress_bar.finish_and_clear(); + info!("client-accounts.yml creation for genesis complete"); + + Ok(()) + } + + fn create_client_account( + client_index: usize, + config_dir: &Path, + target_lamports_per_signature: u64, + bench_tps_args: &[String], + deploy_method: &DeployMethod, + solana_root_path: &Path, + ) -> Result> { + info!("client account: {client_index}"); + let mut args = Vec::new(); + let account_path = config_dir.join(format!("bench-tps-{client_index}.yml")); + args.push("--write-client-keys".to_string()); + args.push(account_path.into_os_string().into_string().map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?); + args.push("--target-lamports-per-signature".to_string()); + args.push(target_lamports_per_signature.to_string()); + + args.extend_from_slice(bench_tps_args); + + let executable_path = if let DeployMethod::ReleaseChannel(_) = deploy_method { + solana_root_path.join("solana-release/bin/solana-bench-tps") + } else { + solana_root_path.join("farf/bin/solana-bench-tps") + }; + let child = Command::new(executable_path) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + + Ok(child) + } + fn setup_genesis_flags(&self) -> Result, Box> { let mut args = vec![ "--bootstrap-validator-lamports".to_string(), @@ -258,6 +369,22 @@ impl Genesis { args.push(lamports_per_signature.to_string()); } + if self.config_dir.join("client-accounts.yml").exists() { + args.push("--primordial-accounts-file".to_string()); + args.push( + self.config_dir + .join("client-accounts.yml") + .into_os_string() + .into_string() + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?, + ); + } + Ok(args) } diff --git a/src/kubernetes.rs b/src/kubernetes.rs index f35c6d6..560bedb 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,5 +1,6 @@ use { crate::{ + client_config::ClientConfig, docker::DockerImage, k8s_helpers::{self, SecretType}, validator_config::ValidatorConfig, @@ -44,6 +45,7 @@ pub struct Kubernetes<'a> { k8s_client: Client, namespace: String, validator_config: &'a mut ValidatorConfig, + client_config: ClientConfig, pod_requests: PodRequests, pub metrics: Option, } @@ -52,6 +54,7 @@ impl<'a> Kubernetes<'a> { pub async fn new( namespace: &str, validator_config: &'a mut ValidatorConfig, + client_config: ClientConfig, pod_requests: PodRequests, metrics: Option, ) -> Kubernetes<'a> { @@ -59,6 +62,7 @@ impl<'a> Kubernetes<'a> { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), validator_config, + client_config, pod_requests, metrics, } @@ -173,6 +177,32 @@ impl<'a> Kubernetes<'a> { k8s_helpers::create_secret(secret_name, secrets) } + pub fn create_client_secret( + &self, + client_index: usize, + config_dir: &Path, + ) -> Result> { + let secret_name = format!("client-accounts-secret-{client_index}"); + let faucet_key_path = config_dir.join("faucet.json"); + let identity_key_path = config_dir.join(format!("validator-identity-{}.json", 0)); + + let mut secrets = BTreeMap::new(); + secrets.insert( + "faucet".to_string(), + SecretType::File { + path: faucet_key_path, + }, + ); + secrets.insert( + "identity".to_string(), + SecretType::File { + path: identity_key_path, + }, + ); + + k8s_helpers::create_secret(secret_name, secrets) + } + pub fn add_known_validator(&mut self, pubkey: Pubkey) { self.validator_config.known_validators.push(pubkey); info!("pubkey added to known validators: {:?}", pubkey); @@ -270,6 +300,32 @@ impl<'a> Kubernetes<'a> { flags } + fn generate_client_command_flags(&self) -> Vec { + let mut flags = vec![]; + + flags.push(self.client_config.client_to_run.clone()); //client to run + if !self.client_config.bench_tps_args.is_empty() { + flags.push(self.client_config.bench_tps_args.join(" ")); + } + + flags.push(self.client_config.client_type.clone()); + + if let Some(target_node) = self.client_config.client_target_node { + flags.push("--target-node".to_string()); + flags.push(target_node.to_string()); + } + + flags.push("--duration".to_string()); + flags.push(self.client_config.client_duration_seconds.to_string()); + + if let Some(num_nodes) = self.client_config.client_wait_for_n_nodes { + flags.push("--num-nodes".to_string()); + flags.push(num_nodes.to_string()); + } + + flags + } + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } @@ -586,4 +642,50 @@ impl<'a> Kubernetes<'a> { Some(readiness_probe), ) } + + pub fn create_client_replica_set( + &mut self, + image: &DockerImage, + secret_name: Option, + label_selector: &BTreeMap, + client_index: usize, + ) -> Result> { + let mut env_vars = self.set_non_bootstrap_environment_variables(); + if self.metrics.is_some() { + env_vars.push(self.get_metrics_env_var_secret()) + } + env_vars.append(&mut self.set_load_balancer_environment_variables()); + + let accounts_volume = Some(vec![Volume { + name: format!("client-accounts-volume-{}", client_index), + secret: Some(SecretVolumeSource { + secret_name, + ..Default::default() + }), + ..Default::default() + }]); + + let accounts_volume_mount = Some(vec![VolumeMount { + name: format!("client-accounts-volume-{}", client_index), + mount_path: "/home/solana/client-accounts".to_string(), + ..Default::default() + }]); + + let mut command = + vec!["/home/solana/k8s-cluster-scripts/client-startup-script.sh".to_string()]; + command.extend(self.generate_client_command_flags()); + + k8s_helpers::create_replica_set( + format!("client-{client_index}"), + self.namespace.clone(), + label_selector.clone(), + image.clone(), + env_vars, + command.clone(), + accounts_volume, + accounts_volume_mount, + self.pod_requests.requests.clone(), + None, + ) + } } diff --git a/src/ledger_helper.rs b/src/ledger_helper.rs index 9f369b3..e484738 100644 --- a/src/ledger_helper.rs +++ b/src/ledger_helper.rs @@ -1,6 +1,5 @@ use { crate::genesis::DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, - log::*, solana_accounts_db::hardened_unpack::open_genesis_config, solana_sdk::shred_version::compute_shred_version, std::{error::Error, path::Path}, @@ -25,7 +24,6 @@ impl LedgerHelper { let genesis_config = open_genesis_config(ledger_dir, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE); let shred_version = compute_shred_version(&genesis_config?.hash(), None); - info!("Shred Version: {}", shred_version); Ok(shred_version) } } diff --git a/src/lib.rs b/src/lib.rs index ed47de0..f0a695d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -53,16 +53,16 @@ pub enum ValidatorType { #[strum(serialize = "rpc-node")] RPC, #[strum(serialize = "client")] - Client, + Client(/* client index */ usize), } impl ValidatorType { - fn script(&self) -> Result<&'static str, Box> { + fn script(&self) -> &'static str { match self { - ValidatorType::Bootstrap => Ok(startup_scripts::StartupScripts::bootstrap()), - ValidatorType::Standard => Ok(startup_scripts::StartupScripts::validator()), - ValidatorType::RPC => Ok(startup_scripts::StartupScripts::rpc()), - _ => Err(format!("ValidatorType {:?} not supported!", self).into()), + ValidatorType::Bootstrap => startup_scripts::StartupScripts::bootstrap(), + ValidatorType::Standard => startup_scripts::StartupScripts::validator(), + ValidatorType::RPC => startup_scripts::StartupScripts::rpc(), + ValidatorType::Client(_) => startup_scripts::StartupScripts::client(), } } } @@ -100,6 +100,7 @@ impl Metrics { } } +pub mod client_config; pub mod cluster_images; pub mod docker; pub mod genesis; @@ -116,6 +117,7 @@ static PACKAGE: Emoji = Emoji("📦 ", ""); static ROCKET: Emoji = Emoji("🚀 ", ""); static SUN: Emoji = Emoji("🌞 ", ""); static TRUCK: Emoji = Emoji("🚚 ", ""); +static WRITING: Emoji = Emoji("🖊️ ", ""); /// Creates a new process bar for processing that will take an unknown amount of time pub fn new_spinner_progress_bar() -> ProgressBar { @@ -279,3 +281,23 @@ pub async fn fetch_spl(solana_root_path: &Path) -> Result<(), Box) -> Vec { + if let Some(args) = bench_tps_args { + let mut val_args: Vec<_> = args + .split_whitespace() + .filter_map(|arg| arg.split_once('=')) + .flat_map(|(key, value)| vec![format!("--{key}"), value.to_string()]) + .collect(); + + let flag_args_iter = args + .split_whitespace() + .filter(|arg| arg.split_once('=').is_none()) + .map(|flag| format!("--{flag}")); + + val_args.extend(flag_args_iter); + val_args + } else { + Vec::new() // Return empty vec if no args provided + } +} diff --git a/src/main.rs b/src/main.rs index a9c60a8..2c57946 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use { clap::{command, value_t_or_exit, Arg, ArgGroup}, log::*, + solana_clap_v3_utils::input_parsers::pubkey_of, solana_ledger::blockstore_cleanup_service::{ DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS, }, @@ -8,15 +9,18 @@ use { std::{fs, path::PathBuf, result::Result}, strum::VariantNames, validator_lab::{ + client_config::ClientConfig, cluster_images::ClusterImages, docker::{DockerConfig, DockerImage}, genesis::{ Genesis, GenesisFlags, DEFAULT_BOOTSTRAP_NODE_SOL, DEFAULT_BOOTSTRAP_NODE_STAKE_SOL, - DEFAULT_FAUCET_LAMPORTS, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL, + DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE, DEFAULT_FAUCET_LAMPORTS, + DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, }, kubernetes::{Kubernetes, PodRequests}, ledger_helper::LedgerHelper, + parse_and_format_bench_tps_args, release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, validator_config::ValidatorConfig, @@ -75,10 +79,6 @@ fn parse_matches() -> clap::ArgMatches { .takes_value(true) .default_value("0") .help("Number of validators to deploy") - .validator(|s| match s.parse::() { - Ok(n) if n >= 0 => Ok(()), - _ => Err(String::from("number_of_validators should be >= 0")), - }), ) // Genesis Config .arg( @@ -237,10 +237,70 @@ fn parse_matches() -> clap::ArgMatches { .takes_value(true) .default_value("0") .help("Number of rpc nodes") - .validator(|s| match s.parse::() { - Ok(n) if n >= 0 => Ok(()), - _ => Err(String::from("number_of_rpc_nodes should be >= 0")), - }), + ) + // Client Config + .arg( + Arg::with_name("number_of_clients") + .long("num-clients") + .short('c') + .takes_value(true) + .default_value("0") + .help("Number of clients") + ) + .arg( + Arg::with_name("client_type") + .long("client-type") + .takes_value(true) + .default_value("tpu-client") + .possible_values(["tpu-client", "rpc-client"]) + .help("Client Config. Set Client Type"), + ) + .arg( + Arg::with_name("client_to_run") + .long("client-to-run") + .takes_value(true) + .default_value("bench-tps") + .possible_values(["bench-tps", "idle"]) + .help("Client Config. Set Client to run"), + ) + .arg( + Arg::with_name("bench_tps_args") + .long("bench-tps-args") + .value_name("KEY VALUE") + .takes_value(true) + .multiple(true) + .number_of_values(1) + .help("Client Config. + User can optionally provide extraArgs that are transparently + supplied to the client program as command line parameters. + For example, + --bench-tps-args 'tx-count=5000 thread-batch-sleep-ms=250' + This will start bench-tps clients, and supply '--tx-count 5000 --thread-batch-sleep-ms 250' + to the bench-tps client."), + ) + .arg( + Arg::with_name("client_target_node") + .long("client-target-node") + .takes_value(true) + .value_name("PUBKEY") + .help("Client Config. Optional: Specify an exact node to send transactions to + Not supported yet. TODO..."), + ) + .arg( + Arg::with_name("client_duration_seconds") + .long("client-duration-seconds") + .takes_value(true) + .default_value("7500") + .value_name("SECS") + .help("Client Config. Seconds to run benchmark, then exit"), + ) + .arg( + Arg::with_name("client_wait_for_n_nodes") + .long("client-wait-for-n-nodes") + .short('N') + .takes_value(true) + .value_name("NUM") + .help("Client Config. Optional: Wait for NUM nodes to converge"), ) // kubernetes config .arg( @@ -310,6 +370,21 @@ async fn main() -> Result<(), Box> { let num_validators = value_t_or_exit!(matches, "number_of_validators", usize); let num_rpc_nodes = value_t_or_exit!(matches, "number_of_rpc_nodes", usize); + let client_config = ClientConfig { + num_clients: value_t_or_exit!(matches, "number_of_clients", usize), + client_type: matches.value_of("client_type").unwrap().to_string(), + client_to_run: matches.value_of("client_to_run").unwrap().to_string(), + bench_tps_args: parse_and_format_bench_tps_args(matches.value_of("bench_tps_args")), + client_target_node: pubkey_of(&matches, "client_target_node"), + client_duration_seconds: value_t_or_exit!(matches, "client_duration_seconds", u64), + client_wait_for_n_nodes: matches + .value_of("client_wait_for_n_nodes") + .map(|value_str| { + value_str + .parse() + .expect("Invalid value for client_wait_for_n_nodes") + }), + }; let deploy_method = if let Some(local_path) = matches.value_of("local_path") { DeployMethod::Local(local_path.to_owned()) @@ -448,6 +523,7 @@ async fn main() -> Result<(), Box> { let mut kub_controller = Kubernetes::new( environment_config.namespace, &mut validator_config, + client_config.clone(), pod_requests, metrics, ) @@ -463,7 +539,7 @@ async fn main() -> Result<(), Box> { } build_config.prepare().await?; - info!("Validator setup prepared successfully"); + info!("Setup Validator Environment"); let config_directory = solana_root.get_root_path().join("config-k8s"); let mut genesis = Genesis::new(config_directory.clone(), genesis_flags); @@ -474,11 +550,21 @@ async fn main() -> Result<(), Box> { genesis.generate_accounts(ValidatorType::Bootstrap, 1)?; info!("Generated bootstrap account"); + genesis.create_client_accounts( + client_config.num_clients, + &client_config.bench_tps_args, + DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE, + &config_directory, + &deploy_method, + solana_root.get_root_path(), + )?; + info!("Client accounts created"); + // creates genesis and writes to binary file genesis .generate(solana_root.get_root_path(), &build_path) .await?; - info!("Created genesis successfully"); + info!("Genesis created"); // generate standard validator accounts genesis.generate_accounts(ValidatorType::Standard, num_validators)?; @@ -490,6 +576,7 @@ async fn main() -> Result<(), Box> { let ledger_dir = config_directory.join("bootstrap-validator"); let shred_version = LedgerHelper::get_shred_version(&ledger_dir)?; kub_controller.set_shred_version(shred_version); + info!("Shred Version: {shred_version}"); //unwraps are safe here. since their requirement is enforced by argmatches let docker = DockerConfig::new( @@ -534,14 +621,24 @@ async fn main() -> Result<(), Box> { cluster_images.set_item(rpc_node, ValidatorType::RPC); } + for client_index in 0..client_config.num_clients { + let client = Validator::new(DockerImage::new( + registry_name.clone(), + ValidatorType::Client(client_index), + image_name.clone(), + image_tag.clone(), + )); + cluster_images.set_item(client, ValidatorType::Client(client_index)); + } + if build_config.docker_build() { - for v in cluster_images.get_validators() { + for v in cluster_images.get_all() { docker.build_image(solana_root.get_root_path(), v.image())?; - info!("{} image built successfully", v.validator_type()); + info!("Built {} image", v.validator_type()); } - docker.push_images(cluster_images.get_validators())?; - info!("Validator images pushed successfully"); + docker.push_images(cluster_images.get_all())?; + info!("Pushed {} docker images", cluster_images.get_all().count()); } // metrics secret create once and use by all pods @@ -601,10 +698,7 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(bootstrap_validator.replica_set()) .await?; - info!( - "{} deployed successfully", - bootstrap_validator.replica_set_name() - ); + info!("Deployed {}", bootstrap_validator.replica_set_name()); // create and deploy bootstrap-service let bootstrap_service = kub_controller.create_service( @@ -612,7 +706,7 @@ async fn main() -> Result<(), Box> { bootstrap_validator.service_labels(), ); kub_controller.deploy_service(&bootstrap_service).await?; - info!("bootstrap validator service deployed successfully"); + info!("Deployed Bootstrap Balidator Service"); // load balancer service. only create one and use for all bootstrap/rpc nodes // service selector matches bootstrap selector @@ -624,7 +718,7 @@ async fn main() -> Result<(), Box> { //deploy load balancer kub_controller.deploy_service(&load_balancer).await?; - info!("load balancer service deployed successfully"); + info!("Deployed Load Balancer Service"); // wait for bootstrap replicaset to deploy while !kub_controller @@ -646,7 +740,7 @@ async fn main() -> Result<(), Box> { let rpc_secret = kub_controller.create_rpc_secret(rpc_index, &config_directory)?; rpc_node.set_secret(rpc_secret); kub_controller.deploy_secret(rpc_node.secret()).await?; - info!("Deployed RPC node {rpc_index} Secret"); + info!("Deployed RPC Node {rpc_index} Secret"); let identity_path = config_directory.join(format!("rpc-node-identity-{rpc_index}.json")); @@ -688,90 +782,122 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(rpc_node.replica_set()) .await?; - info!("rpc node replica set ({rpc_index}) deployed successfully"); + info!("Deployed RPC Node Replica Set ({rpc_index})"); let rpc_service = kub_controller.create_service( - &format!("rpc-node-selector-{rpc_index}"), + &format!("rpc-node-service-{rpc_index}"), rpc_node.service_labels(), ); kub_controller.deploy_service(&rpc_service).await?; - info!("rpc node service ({rpc_index}) deployed successfully"); + info!("Deployed RPC Node Service ({rpc_index})"); rpc_nodes.push(rpc_node.replica_set_name().clone()); } // wait for at least one rpc node to deploy - loop { - let mut ready = false; + 'outer: loop { for rpc_name in &rpc_nodes { if kub_controller.is_replica_set_ready(rpc_name).await? { - ready = true; - break; + break 'outer; } } - if ready { - break; - } - info!("no rpc nodes ready yet"); + info!("RPC Nodes not ready yet"); tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - if num_validators == 0 { - info!("No validators to deploy. Returning"); - return Ok(()); + if num_validators > 0 { + let validator = cluster_images.validator()?; + for validator_index in 0..num_validators { + // Create and deploy validators secrets + let validator_secret = + kub_controller.create_validator_secret(validator_index, &config_directory)?; + validator.set_secret(validator_secret); + kub_controller.deploy_secret(validator.secret()).await?; + info!("Deployed Validator {validator_index} Secret"); + + let identity_path = + config_directory.join(format!("validator-identity-{validator_index}.json")); + let validator_keypair = + read_keypair_file(identity_path).expect("Failed to read validator keypair file"); + + validator.add_label( + "validator/name", + &format!("validator-{validator_index}"), + LabelType::Service, + ); + validator.add_label( + "validator/type", + validator.validator_type().to_string(), + LabelType::Info, + ); + validator.add_label( + "validator/identity", + validator_keypair.pubkey().to_string(), + LabelType::Info, + ); + + let replica_set = kub_controller.create_validator_replica_set( + validator.image(), + validator.secret().metadata.name.clone(), + &validator.all_labels(), + validator_index, + )?; + validator.set_replica_set(replica_set); + + kub_controller + .deploy_replicas_set(validator.replica_set()) + .await?; + info!("Deployed Validator Replica Set ({validator_index})"); + + let validator_service = kub_controller.create_service( + &format!("validator-service-{validator_index}"), + validator.service_labels(), + ); + kub_controller.deploy_service(&validator_service).await?; + info!("Deployed Validator Service ({validator_index})"); + } } - let validator = cluster_images.validator()?; - - for validator_index in 0..num_validators { - // Create and deploy validators secrets - let validator_secret = - kub_controller.create_validator_secret(validator_index, &config_directory)?; - validator.set_secret(validator_secret); - kub_controller.deploy_secret(validator.secret()).await?; - info!("Deployed Validator {validator_index} secret"); - - let identity_path = - config_directory.join(format!("validator-identity-{validator_index}.json")); - let validator_keypair = - read_keypair_file(identity_path).expect("Failed to read validator keypair file"); - - validator.add_label( - "validator/name", - &format!("validator-{validator_index}"), + + for client in cluster_images.get_clients_mut() { + let client_index = if let ValidatorType::Client(index) = client.validator_type() { + *index + } else { + return Err("Invalid Validator Type in Client".into()); + }; + + let client_secret = kub_controller.create_client_secret(client_index, &config_directory)?; + client.set_secret(client_secret); + + kub_controller.deploy_secret(client.secret()).await?; + info!("Deployed Client {client_index} Secret"); + + client.add_label( + "client/name", + format!("client-{client_index}"), LabelType::Service, ); - validator.add_label( - "validator/type", - validator.validator_type().to_string(), - LabelType::Info, - ); - validator.add_label( - "validator/identity", - validator_keypair.pubkey().to_string(), - LabelType::Info, - ); - let replica_set = kub_controller.create_validator_replica_set( - validator.image(), - validator.secret().metadata.name.clone(), - &validator.all_labels(), - validator_index, + let client_replica_set = kub_controller.create_client_replica_set( + client.image(), + client.secret().metadata.name.clone(), + &client.all_labels(), + client_index, )?; - validator.set_replica_set(replica_set); + client.set_replica_set(client_replica_set); kub_controller - .deploy_replicas_set(validator.replica_set()) + .deploy_replicas_set(client.replica_set()) .await?; - info!("validator replica set ({validator_index}) deployed successfully"); + info!("Deployed Client Replica Set ({client_index})"); - let validator_service = kub_controller.create_service( - &format!("validator-service-{validator_index}"), - validator.service_labels(), + let client_service = kub_controller.create_service( + &format!("client-service-{client_index}"), + client.service_labels(), ); - kub_controller.deploy_service(&validator_service).await?; - info!("validator service ({validator_index}) deployed successfully"); + kub_controller.deploy_service(&client_service).await?; + info!("Deployed Client Service ({client_index})"); } Ok(()) diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index 945a637..781f9a5 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -32,8 +32,8 @@ source /home/solana/k8s-cluster-scripts/common.sh no_restart=0 # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" @@ -238,8 +238,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" @@ -658,8 +658,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" @@ -936,6 +936,123 @@ done "# } + pub fn client() -> &'static str { + r#"#!/bin/bash + +clientToRun="$1" +benchTpsExtraArgs="$2" +clientType= + +# check if benchTpsExtraArgs is set. if not then it will get set to client-type. Which then needs to get handled appropriately +if [[ "$benchTpsExtraArgs" == "tpu-client" || "$benchTpsExtraArgs" == "rpc-client" ]]; then + clientType=$benchTpsExtraArgs + benchTpsExtraArgs= + shift 2 +else + clientType="${3:-tpu-client}" + shift 3 + # Convert string to array + IFS=' ' read -r -a argsArray <<< "$benchTpsExtraArgs" + + # Loop through the array and check for the specific flag + for arg in "${argsArray[@]}"; do + if [ "$arg" == "--use-rpc-client" ]; then + clientType="rpc-client" + break + elif [ "$arg" == "--use-tpu-client" ]; then + clientType="tpu-client" + break + fi + done +fi + +runtime_args=() +while [[ -n $1 ]]; do + if [[ ${1:0:1} = - ]]; then + if [[ $1 = --target-node ]]; then + echo "WARNING: --target-node not supported yet...not included" + shift 2 + elif [[ $1 = --duration ]]; then + runtime_args+=("$1" "$2") + shift 2 + elif [[ $1 = --num-nodes ]]; then + runtime_args+=("$1" "$2") + shift 2 + else + echo "Unknown argument: $1" + solana-bench-tps --help + exit 1 + fi + else + echo "Unknown argument: $1" + solana-bench-tps --help + exit 1 + fi +done + +echo "get airdrop for client" +solana airdrop 5000000 -k ./client-accounts/identity.json -u "http://$LOAD_BALANCER_RPC_ADDRESS" + +missing() { + echo "Error: $1 not specified" + exit 1 +} + +RPC_CLIENT=false +case "$clientType" in + tpu-client) + RPC_CLIENT=false + ;; + rpc-client) + RPC_CLIENT=true + ;; + *) + echo "Unexpected clientType: \"$clientType\"" + exit 1 + ;; +esac +case $clientToRun in +bench-tps) + args=() + + if ${RPC_CLIENT}; then + args+=(--use-rpc-client) + else + args+=(--use-tpu-client) + fi + + entrypointIp="${BOOTSTRAP_GOSSIP_ADDRESS:0:-5}" + url="$entrypointIp:8899" + + args+=(--bind-address "$entrypointIp") + args+=(--client-node-id ./client-accounts/identity.json) + + clientCommand="\ + solana-bench-tps \ + --sustained \ + $benchTpsExtraArgs \ + --read-client-keys ./client-accounts.yml \ + --url "http://$url" + ${args[*]} \ + ${runtime_args[*]} \ + " + ;; +idle) + # In net/remote/remote-client.sh, we add faucet keypair here + # but in this case we already do that in the docker container + # by default + while true; do sleep 3600; done + ;; +*) + echo "Unknown client name: $clientToRun" + exit 1 +esac + +echo "client command to run: $clientCommand" +$clientCommand +"# + } + pub fn common() -> &'static str { r#" # |source| this file