From e8a74a9057b17177f48e121293fb47af236193f8 Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 10 May 2024 16:59:50 +0000 Subject: [PATCH 1/9] create client accounts for n clients --- PROGRESS.md | 2 +- src/client_config.rs | 4 ++ src/genesis.rs | 111 +++++++++++++++++++++++++++++++++++++++++-- src/lib.rs | 2 + src/main.rs | 29 ++++++++++- 5 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 src/client_config.rs diff --git a/PROGRESS.md b/PROGRESS.md index e0b7f63..4c436fa 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -50,7 +50,7 @@ - [ ] Create accounts - [x] Validator (regular) - [x] RPC - - [ ] Client + - [x] Client - [ ] Add feature flags to configure: - [x] Bootstrap - [x] Validator (regular) diff --git a/src/client_config.rs b/src/client_config.rs new file mode 100644 index 0000000..a4d206e --- /dev/null +++ b/src/client_config.rs @@ -0,0 +1,4 @@ +#[derive(Clone, Debug)] +pub struct ClientConfig { + pub num_clients: usize, +} diff --git a/src/genesis.rs b/src/genesis.rs index 1de9d20..def8361 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -1,5 +1,7 @@ use { - crate::{fetch_spl, new_spinner_progress_bar, ValidatorType, SUN}, + crate::{ + fetch_spl, new_spinner_progress_bar, release::DeployMethod, ValidatorType, SUN, WRITING, + }, log::*, rand::Rng, solana_core::gen_keys::GenKeys, @@ -9,10 +11,10 @@ use { }, std::{ error::Error, - fs::File, - io::Read, + fs::{File, OpenOptions}, + io::{self, BufRead, BufWriter, Read, Write}, path::{Path, PathBuf}, - process::Command, + process::{Child, Command, Stdio}, result::Result, }, }; @@ -23,6 +25,7 @@ pub const DEFAULT_INTERNAL_NODE_STAKE_SOL: f64 = 10.0; pub const DEFAULT_INTERNAL_NODE_SOL: f64 = 100.0; pub const DEFAULT_BOOTSTRAP_NODE_STAKE_SOL: f64 = 10.0; pub const DEFAULT_BOOTSTRAP_NODE_SOL: f64 = 100.0; +pub const DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE: u64 = 42; fn parse_spl_genesis_file( spl_file: &PathBuf, @@ -92,6 +95,34 @@ impl std::fmt::Display for GenesisFlags { } } +fn append_client_accounts_to_file( + in_file: &PathBuf, //bench-tps-i.yml + out_file: &PathBuf, //client-accounts.yml +) -> io::Result<()> { + // Open the bench-tps-i.yml file for reading. + let input = File::open(in_file)?; + let reader = io::BufReader::new(input); + + // Open (or create) client-accounts.yml + let output = OpenOptions::new() + .create(true) + .append(true) + .open(out_file)?; + let mut writer = BufWriter::new(output); + + // Enumerate the lines of the input file, starting from 1. + for (index, line) in reader.lines().enumerate().map(|(i, l)| (i + 1, l)) { + let line = line?; + + // Skip first line since it is a header aka "---" in a yaml + if (index as u64) > 1 { + writeln!(writer, "{line}")?; + } + } + + Ok(()) +} + pub struct Genesis { config_dir: PathBuf, key_generator: GenKeys, @@ -179,6 +210,78 @@ impl Genesis { Ok(()) } + pub fn create_client_accounts( + &mut self, + number_of_clients: usize, + target_lamports_per_signature: u64, + config_dir: &Path, + deploy_method: &DeployMethod, + solana_root_path: &Path, + ) -> Result<(), Box> { + if number_of_clients == 0 { + return Ok(()); + } + let client_accounts_file = config_dir.join("client-accounts.yml"); + + info!("generating {number_of_clients} client accounts..."); + + let children: Result, _> = (0..number_of_clients) + .map(|i| { + info!("client account: {i}"); + let mut args = Vec::new(); + let account_path = config_dir.join(format!("bench-tps-{i}.yml")); + args.push("--write-client-keys".to_string()); + args.push(account_path.into_os_string().into_string().map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?); + args.push("--target-lamports-per-signature".to_string()); + args.push(target_lamports_per_signature.to_string()); + + let executable_path = if let DeployMethod::ReleaseChannel(_) = deploy_method { + solana_root_path.join("solana-release/bin/solana-bench-tps") + } else { + solana_root_path.join("farf/bin/solana-bench-tps") + }; + + Self::create_client_account(&args, &executable_path) + }) + .collect(); + + for child in children? { + let output = child.wait_with_output()?; + if !output.status.success() { + return Err(output.status.to_string().into()); + } + } + + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{WRITING}Writing client accounts...")); + for i in 0..number_of_clients { + let account_path = config_dir.join(format!("bench-tps-{i}.yml")); + append_client_accounts_to_file(&account_path, &client_accounts_file)?; + } + progress_bar.finish_and_clear(); + info!("client-accounts.yml creation for genesis complete"); + + Ok(()) + } + + fn create_client_account( + args: &Vec, + executable_path: &PathBuf, + ) -> Result> { + let child = Command::new(executable_path) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + + Ok(child) + } + fn setup_genesis_flags(&self) -> Result, Box> { let mut args = vec![ "--bootstrap-validator-lamports".to_string(), diff --git a/src/lib.rs b/src/lib.rs index ed47de0..26537c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -100,6 +100,7 @@ impl Metrics { } } +pub mod client_config; pub mod cluster_images; pub mod docker; pub mod genesis; @@ -116,6 +117,7 @@ static PACKAGE: Emoji = Emoji("📦 ", ""); static ROCKET: Emoji = Emoji("🚀 ", ""); static SUN: Emoji = Emoji("🌞 ", ""); static TRUCK: Emoji = Emoji("🚚 ", ""); +static WRITING: Emoji = Emoji("🖊️ ", ""); /// Creates a new process bar for processing that will take an unknown amount of time pub fn new_spinner_progress_bar() -> ProgressBar { diff --git a/src/main.rs b/src/main.rs index a9c60a8..9491b3f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,11 +8,13 @@ use { std::{fs, path::PathBuf, result::Result}, strum::VariantNames, validator_lab::{ + client_config::ClientConfig, cluster_images::ClusterImages, docker::{DockerConfig, DockerImage}, genesis::{ Genesis, GenesisFlags, DEFAULT_BOOTSTRAP_NODE_SOL, DEFAULT_BOOTSTRAP_NODE_STAKE_SOL, - DEFAULT_FAUCET_LAMPORTS, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL, + DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE, DEFAULT_FAUCET_LAMPORTS, + DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, }, kubernetes::{Kubernetes, PodRequests}, @@ -242,6 +244,19 @@ fn parse_matches() -> clap::ArgMatches { _ => Err(String::from("number_of_rpc_nodes should be >= 0")), }), ) + // Client Configurations + .arg( + Arg::with_name("number_of_clients") + .long("num-clients") + .short('c') + .takes_value(true) + .default_value("0") + .help("Number of clients ") + .validator(|s| match s.parse::() { + Ok(n) if n >= 0 => Ok(()), + _ => Err(String::from("number_of_clients should be >= 0")), + }), + ) // kubernetes config .arg( Arg::with_name("cpu_requests") @@ -310,6 +325,9 @@ async fn main() -> Result<(), Box> { let num_validators = value_t_or_exit!(matches, "number_of_validators", usize); let num_rpc_nodes = value_t_or_exit!(matches, "number_of_rpc_nodes", usize); + let client_config = ClientConfig { + num_clients: value_t_or_exit!(matches, "number_of_clients", usize), + }; let deploy_method = if let Some(local_path) = matches.value_of("local_path") { DeployMethod::Local(local_path.to_owned()) @@ -474,6 +492,15 @@ async fn main() -> Result<(), Box> { genesis.generate_accounts(ValidatorType::Bootstrap, 1)?; info!("Generated bootstrap account"); + genesis.create_client_accounts( + client_config.num_clients, + DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE, + &config_directory, + &deploy_method, + solana_root.get_root_path(), + )?; + info!("Client accounts created successfully"); + // creates genesis and writes to binary file genesis .generate(solana_root.get_root_path(), &build_path) From 942237f8edb7db1fb284822669fafecd709745bf Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 10 May 2024 19:25:13 +0000 Subject: [PATCH 2/9] build and push client images --- PROGRESS.md | 2 +- src/cluster_images.rs | 21 ++++++- src/docker.rs | 107 ++++++++++++++++++++++++----------- src/genesis.rs | 24 ++++++-- src/lib.rs | 12 ++-- src/main.rs | 19 ++++++- src/startup_scripts.rs | 123 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 261 insertions(+), 47 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 4c436fa..96c7f4d 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -19,7 +19,7 @@ - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client - [ ] Create & Deploy Secrets - [x] Bootstrap - [x] Validator (regular) diff --git a/src/cluster_images.rs b/src/cluster_images.rs index 9e5fa6c..460121d 100644 --- a/src/cluster_images.rs +++ b/src/cluster_images.rs @@ -14,7 +14,7 @@ pub struct ClusterImages { bootstrap: Option, validator: Option, rpc: Option, - _clients: Vec, + clients: Vec, } impl ClusterImages { @@ -23,7 +23,7 @@ impl ClusterImages { ValidatorType::Bootstrap => self.bootstrap = Some(item), ValidatorType::Standard => self.validator = Some(item), ValidatorType::RPC => self.rpc = Some(item), - _ => panic!("{validator_type} not implemented yet!"), + ValidatorType::Client(_) => self.clients.push(item), } } @@ -45,6 +45,15 @@ impl ClusterImages { .ok_or_else(|| "Validator is not available".into()) } + pub fn client(&mut self, client_index: usize) -> Result<&mut Validator, Box> { + if self.clients.is_empty() { + return Err("No Clients available".to_string().into()); + } + self.clients + .get_mut(client_index) + .ok_or_else(|| "Client index out of bounds".to_string().into()) + } + pub fn get_validators(&self) -> impl Iterator { self.bootstrap .iter() @@ -52,4 +61,12 @@ impl ClusterImages { .chain(self.rpc.iter()) .filter_map(Some) } + + pub fn get_clients(&self) -> impl Iterator { + self.clients.iter() + } + + pub fn get_all(&self) -> impl Iterator { + self.get_validators().chain(self.get_clients()) + } } diff --git a/src/docker.rs b/src/docker.rs index fb5e240..f652f2c 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -45,11 +45,18 @@ impl DockerImage { // Put DockerImage in format for building, pushing, and pulling impl Display for DockerImage { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "{}/{}-{}:{}", - self.registry, self.validator_type, self.image_name, self.tag - ) + match self.validator_type { + ValidatorType::Client(index) => write!( + f, + "{}/{}-{}-{}:{}", + self.registry, self.validator_type, index, self.image_name, self.tag + ), + ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => write!( + f, + "{}/{}-{}:{}", + self.registry, self.validator_type, self.image_name, self.tag + ), + } } } @@ -72,17 +79,17 @@ impl DockerConfig { docker_image: &DockerImage, ) -> Result<(), Box> { let validator_type = docker_image.validator_type(); - match validator_type { - ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => (), - ValidatorType::Client => { - return Err(format!( - "Build docker image for validator type: {validator_type} not supported yet" - ) - .into()); + let docker_path = match validator_type { + ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => { + solana_root_path.join(format!("docker-build/{validator_type}")) } - } + ValidatorType::Client(index) => { + solana_root_path.join(format!("docker-build/{validator_type}-{index}")) + } + }; + + info!("docker path: {:?}", docker_path); - let docker_path = solana_root_path.join(format!("docker-build/{validator_type}")); self.create_base_image( solana_root_path, docker_image, @@ -100,7 +107,7 @@ impl DockerConfig { docker_path: &PathBuf, validator_type: &ValidatorType, ) -> Result<(), Box> { - self.create_dockerfile(validator_type, docker_path, None)?; + self.create_dockerfile(validator_type, docker_path, solana_root_path, None)?; // We use std::process::Command here because Docker-rs is very slow building dockerfiles // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. @@ -116,6 +123,8 @@ impl DockerConfig { dockerfile.display() ); + info!("command: {command}"); + let output = Command::new("sh") .arg("-c") .arg(&command) @@ -141,7 +150,7 @@ impl DockerConfig { validator_type: &ValidatorType, ) -> Result<(), Box> { let script_path = docker_dir.join(file_name); - let script_content = validator_type.script()?; + let script_content = validator_type.script(); StartupScripts::write_script_to_file(script_content, &script_path).map_err(|e| e.into()) } @@ -149,6 +158,7 @@ impl DockerConfig { &self, validator_type: &ValidatorType, docker_path: &PathBuf, + solana_root_path: &Path, content: Option<&str>, ) -> Result<(), Box> { if docker_path.exists() { @@ -156,23 +166,19 @@ impl DockerConfig { } fs::create_dir_all(docker_path)?; - match validator_type { + let file_name = format!("{validator_type}-startup-script.sh"); + Self::write_startup_script_to_docker_directory(&file_name, docker_path, validator_type)?; + StartupScripts::write_script_to_file( + StartupScripts::common(), + &docker_path.join("common.sh"), + )?; + + let startup_script_directory = match validator_type { ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => { - let file_name = format!("{validator_type}-startup-script.sh"); - Self::write_startup_script_to_docker_directory( - &file_name, - docker_path, - validator_type, - )?; - StartupScripts::write_script_to_file( - StartupScripts::common(), - &docker_path.join("common.sh"), - )?; + format!("./docker-build/{validator_type}") } - ValidatorType::Client => todo!(), - } - - let startup_script_directory = format!("./docker-build/{validator_type}"); + ValidatorType::Client(index) => format!("./docker-build/{validator_type}-{index}"), + }; let solana_build_directory = if let DeployMethod::ReleaseChannel(_) = self.deploy_method { "solana-release" } else { @@ -200,6 +206,11 @@ WORKDIR /home/solana self.base_image ); + let dockerfile = format!( + "{dockerfile}\n{}", + self.insert_client_accounts_if_present(solana_root_path, validator_type)?, + ); + debug!("dockerfile: {dockerfile:?}"); std::fs::write( docker_path.join("Dockerfile"), @@ -208,6 +219,40 @@ WORKDIR /home/solana Ok(()) } + fn insert_client_accounts_if_present( + &self, + solana_root_path: &Path, + validator_type: &ValidatorType, + ) -> Result> { + match validator_type { + ValidatorType::Client(index) => { + let bench_tps_path = + solana_root_path.join(format!("config-k8s/bench-tps-{index}.yml")); + if bench_tps_path.exists() { + Ok(format!( + r#" +COPY --chown=solana:solana ./config-k8s/bench-tps-{index}.yml /home/solana/client-accounts.yml + "#, + )) + } else { + Err(format!("{:?} does not exist!", bench_tps_path).into()) + } + } + ValidatorType::Bootstrap => { + let client_accounts_path = solana_root_path.join("config-k8s/client-accounts.yml"); + if client_accounts_path.exists() { + Ok(r#" +COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana + "# + .to_string()) + } else { + Err(format!("{:?} does not exist!", client_accounts_path).into()) + } + } + ValidatorType::Standard | ValidatorType::RPC => Ok("".to_string()), + } + } + pub fn push_image(docker_image: &DockerImage) -> Result> { let command = format!("docker push '{docker_image}'"); let child = Command::new("sh") diff --git a/src/genesis.rs b/src/genesis.rs index def8361..7c0c2cf 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -159,7 +159,7 @@ impl Genesis { validator_type: ValidatorType, number_of_accounts: usize, ) -> Result<(), Box> { - if validator_type == ValidatorType::Client { + if let ValidatorType::Client(_) = validator_type { return Err("Client valdiator_type in generate_accounts not allowed".into()); } @@ -172,7 +172,7 @@ impl Genesis { ValidatorType::RPC => { vec!["identity"] // no vote or stake account for RPC } - ValidatorType::Client => panic!("Client type not supported"), + ValidatorType::Client(_) => panic!("Client type not supported"), }; let total_accounts_to_generate = number_of_accounts * account_types.len(); @@ -201,7 +201,7 @@ impl Genesis { ValidatorType::Standard | ValidatorType::RPC => { format!("{validator_type}-{account}-{account_index}.json") } - ValidatorType::Client => panic!("Client type not supported"), + ValidatorType::Client(_) => panic!("Client type not supported"), }; let outfile = self.config_dir.join(&filename); @@ -223,7 +223,7 @@ impl Genesis { } let client_accounts_file = config_dir.join("client-accounts.yml"); - info!("generating {number_of_clients} client accounts..."); + info!("generating {number_of_clients} client account(s)..."); let children: Result, _> = (0..number_of_clients) .map(|i| { @@ -361,6 +361,22 @@ impl Genesis { args.push(lamports_per_signature.to_string()); } + if self.config_dir.join("client-accounts.yml").exists() { + args.push("--primordial-accounts-file".to_string()); + args.push( + self.config_dir + .join("client-accounts.yml") + .into_os_string() + .into_string() + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?, + ); + } + Ok(args) } diff --git a/src/lib.rs b/src/lib.rs index 26537c9..262acac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -53,16 +53,16 @@ pub enum ValidatorType { #[strum(serialize = "rpc-node")] RPC, #[strum(serialize = "client")] - Client, + Client(/* client index */ usize), } impl ValidatorType { - fn script(&self) -> Result<&'static str, Box> { + fn script(&self) -> &'static str { match self { - ValidatorType::Bootstrap => Ok(startup_scripts::StartupScripts::bootstrap()), - ValidatorType::Standard => Ok(startup_scripts::StartupScripts::validator()), - ValidatorType::RPC => Ok(startup_scripts::StartupScripts::rpc()), - _ => Err(format!("ValidatorType {:?} not supported!", self).into()), + ValidatorType::Bootstrap => startup_scripts::StartupScripts::bootstrap(), + ValidatorType::Standard => startup_scripts::StartupScripts::validator(), + ValidatorType::RPC => startup_scripts::StartupScripts::rpc(), + ValidatorType::Client(_) => startup_scripts::StartupScripts::client(), } } } diff --git a/src/main.rs b/src/main.rs index 9491b3f..58a2dce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -561,14 +561,27 @@ async fn main() -> Result<(), Box> { cluster_images.set_item(rpc_node, ValidatorType::RPC); } + for client_index in 0..client_config.num_clients { + let client = Validator::new(DockerImage::new( + registry_name.clone(), + ValidatorType::Client(client_index), + image_name.clone(), + image_tag.clone(), + )); + cluster_images.set_item(client, ValidatorType::Client(client_index)); + } + if build_config.docker_build() { - for v in cluster_images.get_validators() { + for v in cluster_images.get_all() { docker.build_image(solana_root.get_root_path(), v.image())?; info!("{} image built successfully", v.validator_type()); } - docker.push_images(cluster_images.get_validators())?; - info!("Validator images pushed successfully"); + docker.push_images(cluster_images.get_all())?; + info!( + "{} docker images pushed successfully", + cluster_images.get_all().count() + ); } // metrics secret create once and use by all pods diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index 945a637..11db0e2 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -936,6 +936,129 @@ done "# } + pub fn client() -> &'static str { + r#"#!/bin/bash + +clientToRun="$1" +benchTpsExtraArgs="$2" +clientType= + +# check if benchTpsExtraArgs is set. if not then it will get set to client-type. Which then needs to get handled appropriately +if [[ "$benchTpsExtraArgs" == "tpu-client" || "$benchTpsExtraArgs" == "rpc-client" ]]; then + clientType=$benchTpsExtraArgs + benchTpsExtraArgs= + shift 2 +else + clientType="${3:-tpu-client}" + shift 3 + # Convert string to array + IFS=' ' read -r -a argsArray <<< "$benchTpsExtraArgs" + + # Loop through the array and check for the specific flag + for arg in "${argsArray[@]}"; do + if [ "$arg" == "--use-rpc-client" ]; then + clientType="rpc-client" + break + elif [ "$arg" == "--use-tpu-client" ]; then + clientType="tpu-client" + break + fi + done +fi + +runtime_args=() +while [[ -n $1 ]]; do + if [[ ${1:0:1} = - ]]; then + if [[ $1 = --target-node ]]; then + echo "WARNING: --target-node not supported yet...not included" + shift 2 + elif [[ $1 = --duration ]]; then + runtime_args+=("$1" "$2") + shift 2 + elif [[ $1 = --num-nodes ]]; then + runtime_args+=("$1" "$2") + shift 2 + else + echo "Unknown argument: $1" + solana-bench-tps --help + exit 1 + fi + else + echo "Unknown argument: $1" + solana-bench-tps --help + exit 1 + fi +done + +echo "get airdrop for client" +solana airdrop 5000000 -k ./client-accounts/identity.json -u "http://$LOAD_BALANCER_RPC_ADDRESS" + +missing() { + echo "Error: $1 not specified" + exit 1 +} + +threadCount=$(nproc) +if [[ $threadCount -gt 4 ]]; then + threadCount=4 +fi + +echo "threadCount: $threadCount" + +RPC_CLIENT=false +case "$clientType" in + tpu-client) + RPC_CLIENT=false + ;; + rpc-client) + RPC_CLIENT=true + ;; + *) + echo "Unexpected clientType: \"$clientType\"" + exit 1 + ;; +esac +case $clientToRun in +bench-tps) + args=() + + if ${RPC_CLIENT}; then + args+=(--use-rpc-client) + fi + + entrypointIp="${BOOTSTRAP_GOSSIP_ADDRESS:0:-5}" + url="$entrypointIp:8899" + + args+=(--bind-address "$entrypointIp") + args+=(--client-node-id ./client-accounts/identity.json) + + clientCommand="\ + solana-bench-tps \ + --sustained \ + --threads $threadCount \ + $benchTpsExtraArgs \ + --read-client-keys ./client-accounts.yml \ + --url "http://$url" + ${args[*]} \ + ${runtime_args[*]} \ + " + ;; +idle) + # In net/remote/remote-client.sh, we add faucet keypair here + # but in this case we already do that in the docker container + # by default + while true; do sleep 3600; done + ;; +*) + echo "Unknown client name: $clientToRun" + exit 1 +esac + +echo "client command to run: $clientCommand" +$clientCommand +"# + } + pub fn common() -> &'static str { r#" # |source| this file From 51bfc4ad60cdbc2fc47aba1c3e0d14886f6124ed Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 13 May 2024 18:32:09 +0000 Subject: [PATCH 3/9] create and deploy client secrets --- PROGRESS.md | 2 +- src/cluster_images.rs | 4 ++ src/kubernetes.rs | 26 ++++++++++ src/main.rs | 115 +++++++++++++++++++++++------------------- 4 files changed, 94 insertions(+), 53 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 96c7f4d..81fe9d4 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -24,7 +24,7 @@ - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client - [ ] Create & Deploy Selector - [x] Bootstrap - [x] Validator (regular) diff --git a/src/cluster_images.rs b/src/cluster_images.rs index 460121d..1e7b100 100644 --- a/src/cluster_images.rs +++ b/src/cluster_images.rs @@ -66,6 +66,10 @@ impl ClusterImages { self.clients.iter() } + pub fn get_clients_mut(&mut self) -> impl Iterator { + self.clients.iter_mut() + } + pub fn get_all(&self) -> impl Iterator { self.get_validators().chain(self.get_clients()) } diff --git a/src/kubernetes.rs b/src/kubernetes.rs index f35c6d6..fcb8938 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -173,6 +173,32 @@ impl<'a> Kubernetes<'a> { k8s_helpers::create_secret(secret_name, secrets) } + pub fn create_client_secret( + &self, + client_index: usize, + config_dir: &Path, + ) -> Result> { + let secret_name = format!("client-accounts-secret-{client_index}"); + let faucet_key_path = config_dir.join("faucet.json"); + let identity_key_path = config_dir.join(format!("validator-identity-{}.json", 0)); + + let mut secrets = BTreeMap::new(); + secrets.insert( + "faucet".to_string(), + SecretType::File { + path: faucet_key_path, + }, + ); + secrets.insert( + "identity".to_string(), + SecretType::File { + path: identity_key_path, + }, + ); + + k8s_helpers::create_secret(secret_name, secrets) + } + pub fn add_known_validator(&mut self, pubkey: Pubkey) { self.validator_config.known_validators.push(pubkey); info!("pubkey added to known validators: {:?}", pubkey); diff --git a/src/main.rs b/src/main.rs index 58a2dce..5899114 100644 --- a/src/main.rs +++ b/src/main.rs @@ -758,60 +758,71 @@ async fn main() -> Result<(), Box> { } } - if num_validators == 0 { - info!("No validators to deploy. Returning"); - return Ok(()); + if num_validators > 0 { + let validator = cluster_images.validator()?; + for validator_index in 0..num_validators { + // Create and deploy validators secrets + let validator_secret = + kub_controller.create_validator_secret(validator_index, &config_directory)?; + validator.set_secret(validator_secret); + kub_controller.deploy_secret(validator.secret()).await?; + info!("Deployed Validator {validator_index} secret"); + + let identity_path = + config_directory.join(format!("validator-identity-{validator_index}.json")); + let validator_keypair = + read_keypair_file(identity_path).expect("Failed to read validator keypair file"); + + validator.add_label( + "validator/name", + &format!("validator-{validator_index}"), + LabelType::Service, + ); + validator.add_label( + "validator/type", + validator.validator_type().to_string(), + LabelType::Info, + ); + validator.add_label( + "validator/identity", + validator_keypair.pubkey().to_string(), + LabelType::Info, + ); + + let replica_set = kub_controller.create_validator_replica_set( + validator.image(), + validator.secret().metadata.name.clone(), + &validator.all_labels(), + validator_index, + )?; + validator.set_replica_set(replica_set); + + kub_controller + .deploy_replicas_set(validator.replica_set()) + .await?; + info!("validator replica set ({validator_index}) deployed successfully"); + + let validator_service = kub_controller.create_service( + &format!("validator-service-{validator_index}"), + validator.service_labels(), + ); + kub_controller.deploy_service(&validator_service).await?; + info!("validator service ({validator_index}) deployed successfully"); + } } - let validator = cluster_images.validator()?; - - for validator_index in 0..num_validators { - // Create and deploy validators secrets - let validator_secret = - kub_controller.create_validator_secret(validator_index, &config_directory)?; - validator.set_secret(validator_secret); - kub_controller.deploy_secret(validator.secret()).await?; - info!("Deployed Validator {validator_index} secret"); - - let identity_path = - config_directory.join(format!("validator-identity-{validator_index}.json")); - let validator_keypair = - read_keypair_file(identity_path).expect("Failed to read validator keypair file"); - - validator.add_label( - "validator/name", - &format!("validator-{validator_index}"), - LabelType::Service, - ); - validator.add_label( - "validator/type", - validator.validator_type().to_string(), - LabelType::Info, - ); - validator.add_label( - "validator/identity", - validator_keypair.pubkey().to_string(), - LabelType::Info, - ); - let replica_set = kub_controller.create_validator_replica_set( - validator.image(), - validator.secret().metadata.name.clone(), - &validator.all_labels(), - validator_index, - )?; - validator.set_replica_set(replica_set); - - kub_controller - .deploy_replicas_set(validator.replica_set()) - .await?; - info!("validator replica set ({validator_index}) deployed successfully"); - - let validator_service = kub_controller.create_service( - &format!("validator-service-{validator_index}"), - validator.service_labels(), - ); - kub_controller.deploy_service(&validator_service).await?; - info!("validator service ({validator_index}) deployed successfully"); + for client in cluster_images.get_clients_mut() { + let client_index = if let ValidatorType::Client(index) = client.validator_type() { + *index + } else { + return Err("Invalid Validator Type in Client".into()); + }; + + let client_secret = kub_controller.create_client_secret(client_index, &config_directory)?; + client.set_secret(client_secret); + + kub_controller.deploy_secret(client.secret()).await?; + info!("Deployed Client {client_index} Secret"); } Ok(()) From 3532ae15bbb3e355874f0f0b2172067123ea413e Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 13 May 2024 20:43:50 +0000 Subject: [PATCH 4/9] add client args and deploy replica set --- PROGRESS.md | 4 +- README.md | 6 +++ src/client_config.rs | 8 ++++ src/kubernetes.rs | 76 ++++++++++++++++++++++++++++++++++ src/lib.rs | 16 ++++++++ src/main.rs | 97 +++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 203 insertions(+), 4 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 81fe9d4..00abb18 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -29,12 +29,12 @@ - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client - [ ] Create & Deploy Replica Set - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client - [ ] Create & Deploy Services - [x] Bootstrap - [x] Validator (regular) diff --git a/README.md b/README.md index 4e539ac..a5612b8 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,12 @@ cargo run --bin cluster -- # kubernetes config --cpu-requests --memory-requests + # deploy with clients + -c + --client-type + --client-to-run + --num-nodes + --bench-tps-args ``` ## Metrics diff --git a/src/client_config.rs b/src/client_config.rs index a4d206e..15fe2af 100644 --- a/src/client_config.rs +++ b/src/client_config.rs @@ -1,4 +1,12 @@ +use solana_sdk::pubkey::Pubkey; + #[derive(Clone, Debug)] pub struct ClientConfig { pub num_clients: usize, + pub client_type: String, + pub client_to_run: String, + pub bench_tps_args: Option>, + pub target_node: Option, + pub duration: u64, + pub num_nodes: Option, } diff --git a/src/kubernetes.rs b/src/kubernetes.rs index fcb8938..44a28c9 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,5 +1,6 @@ use { crate::{ + client_config::ClientConfig, docker::DockerImage, k8s_helpers::{self, SecretType}, validator_config::ValidatorConfig, @@ -44,6 +45,7 @@ pub struct Kubernetes<'a> { k8s_client: Client, namespace: String, validator_config: &'a mut ValidatorConfig, + client_config: ClientConfig, pod_requests: PodRequests, pub metrics: Option, } @@ -52,6 +54,7 @@ impl<'a> Kubernetes<'a> { pub async fn new( namespace: &str, validator_config: &'a mut ValidatorConfig, + client_config: ClientConfig, pod_requests: PodRequests, metrics: Option, ) -> Kubernetes<'a> { @@ -59,6 +62,7 @@ impl<'a> Kubernetes<'a> { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), validator_config, + client_config, pod_requests, metrics, } @@ -296,6 +300,32 @@ impl<'a> Kubernetes<'a> { flags } + fn generate_client_command_flags(&self) -> Vec { + let mut flags = vec![]; + + flags.push(self.client_config.client_to_run.clone()); //client to run + if let Some(bench_tps_args) = &self.client_config.bench_tps_args { + flags.push(bench_tps_args.join(" ")); + } + + flags.push(self.client_config.client_type.clone()); + + if let Some(target_node) = self.client_config.target_node { + flags.push("--target-node".to_string()); + flags.push(target_node.to_string()); + } + + flags.push("--duration".to_string()); + flags.push(self.client_config.duration.to_string()); + + if let Some(num_nodes) = self.client_config.num_nodes { + flags.push("--num-nodes".to_string()); + flags.push(num_nodes.to_string()); + } + + flags + } + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } @@ -612,4 +642,50 @@ impl<'a> Kubernetes<'a> { Some(readiness_probe), ) } + + pub fn create_client_replica_set( + &mut self, + image: &DockerImage, + secret_name: Option, + label_selector: &BTreeMap, + client_index: usize, + ) -> Result> { + let mut env_vars = self.set_non_bootstrap_environment_variables(); + if self.metrics.is_some() { + env_vars.push(self.get_metrics_env_var_secret()) + } + env_vars.append(&mut self.set_load_balancer_environment_variables()); + + let accounts_volume = Some(vec![Volume { + name: format!("client-accounts-volume-{}", client_index), + secret: Some(SecretVolumeSource { + secret_name, + ..Default::default() + }), + ..Default::default() + }]); + + let accounts_volume_mount = Some(vec![VolumeMount { + name: format!("client-accounts-volume-{}", client_index), + mount_path: "/home/solana/client-accounts".to_string(), + ..Default::default() + }]); + + let mut command = + vec!["/home/solana/k8s-cluster-scripts/client-startup-script.sh".to_string()]; + command.extend(self.generate_client_command_flags()); + + k8s_helpers::create_replica_set( + format!("client-{client_index}"), + self.namespace.clone(), + label_selector.clone(), + image.clone(), + env_vars, + command.clone(), + accounts_volume, + accounts_volume_mount, + self.pod_requests.requests.clone(), + None, + ) + } } diff --git a/src/lib.rs b/src/lib.rs index 262acac..6c11f60 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -281,3 +281,19 @@ pub async fn fetch_spl(solana_root_path: &Path) -> Result<(), Box) -> Option> { + bench_tps_args.map(|args| { + let mut val_args: Vec<_> = args + .split_whitespace() + .filter_map(|arg| arg.split_once('=')) + .flat_map(|(key, value)| vec![format!("--{}", key), value.to_string()]) + .collect(); + let flag_args_iter = args + .split_whitespace() + .filter(|arg| arg.split_once('=').is_none()) + .map(|flag| format!("--{}", flag)); + val_args.extend(flag_args_iter); + val_args + }) +} diff --git a/src/main.rs b/src/main.rs index 5899114..6bf9225 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ use { solana_ledger::blockstore_cleanup_service::{ DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS, }, - solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, + solana_sdk::{pubkey::Pubkey, signature::keypair::read_keypair_file, signer::Signer}, std::{fs, path::PathBuf, result::Result}, strum::VariantNames, validator_lab::{ @@ -19,6 +19,7 @@ use { }, kubernetes::{Kubernetes, PodRequests}, ledger_helper::LedgerHelper, + parse_and_format_bench_tps_args, release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, validator_config::ValidatorConfig, @@ -244,7 +245,7 @@ fn parse_matches() -> clap::ArgMatches { _ => Err(String::from("number_of_rpc_nodes should be >= 0")), }), ) - // Client Configurations + // Client Config .arg( Arg::with_name("number_of_clients") .long("num-clients") @@ -257,6 +258,58 @@ fn parse_matches() -> clap::ArgMatches { _ => Err(String::from("number_of_clients should be >= 0")), }), ) + .arg( + Arg::with_name("client_type") + .long("client-type") + .takes_value(true) + .default_value("tpu-client") + .possible_values(["tpu-client", "rpc-client"]) + .help("Client Config. Set Client Type"), + ) + .arg( + Arg::with_name("client_to_run") + .long("client-to-run") + .takes_value(true) + .default_value("bench-tps") + .possible_values(["bench-tps", "idle"]) + .help("Client Config. Set Client to run"), + ) + .arg( + Arg::with_name("bench_tps_args") + .long("bench-tps-args") + .value_name("KEY VALUE") + .takes_value(true) + .multiple(true) + .number_of_values(1) + .help("Client Config. + User can optionally provide extraArgs that are transparently + supplied to the client program as command line parameters. + For example, + --bench-tps-args 'tx-count=5000 thread-batch-sleep-ms=250' + This will start bench-tps clients, and supply '--tx-count 5000 --thread-batch-sleep-ms 250' + to the bench-tps client."), + ) + .arg( + Arg::with_name("target_node") + .long("target-node") + .takes_value(true) + .help("Client Config. Optional: Specify an exact node to send transactions to. use: --target-node . + Not supported yet. TODO..."), + ) + .arg( + Arg::with_name("duration") + .long("duration") + .takes_value(true) + .default_value("7500") + .help("Client Config. Seconds to run benchmark, then exit; default is forever use: --duration "), + ) + .arg( + Arg::with_name("num_nodes") + .long("num-nodes") + .short('N') + .takes_value(true) + .help("Client Config. Optional: Wait for NUM nodes to converge: --num-nodes "), + ) // kubernetes config .arg( Arg::with_name("cpu_requests") @@ -327,6 +380,26 @@ async fn main() -> Result<(), Box> { let num_rpc_nodes = value_t_or_exit!(matches, "number_of_rpc_nodes", usize); let client_config = ClientConfig { num_clients: value_t_or_exit!(matches, "number_of_clients", usize), + client_type: matches + .value_of("client_type") + .unwrap_or_default() + .to_string(), + client_to_run: matches + .value_of("client_to_run") + .unwrap_or_default() + .to_string(), + bench_tps_args: parse_and_format_bench_tps_args(matches.value_of("bench_tps_args")), + target_node: match matches.value_of("target_node") { + Some(s) => match s.parse::() { + Ok(pubkey) => Some(pubkey), + Err(e) => return Err(format!("failed to parse pubkey in target_node: {e}").into()), + }, + None => None, + }, + duration: value_t_or_exit!(matches, "duration", u64), + num_nodes: matches + .value_of("num_nodes") + .map(|value_str| value_str.parse().expect("Invalid value for num_nodes")), }; let deploy_method = if let Some(local_path) = matches.value_of("local_path") { @@ -466,6 +539,7 @@ async fn main() -> Result<(), Box> { let mut kub_controller = Kubernetes::new( environment_config.namespace, &mut validator_config, + client_config.clone(), pod_requests, metrics, ) @@ -823,6 +897,25 @@ async fn main() -> Result<(), Box> { kub_controller.deploy_secret(client.secret()).await?; info!("Deployed Client {client_index} Secret"); + + client.add_label( + "client/name", + format!("client-{}", client_index), + LabelType::Service, + ); + + let client_replica_set = kub_controller.create_client_replica_set( + client.image(), + client.secret().metadata.name.clone(), + &client.all_labels(), + client_index, + )?; + client.set_replica_set(client_replica_set); + + kub_controller + .deploy_replicas_set(client.replica_set()) + .await?; + info!("client replica set ({client_index}) deployed successfully"); } Ok(()) From f430fa2d95fbdee591650af2de679343a705ffa1 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 13 May 2024 21:21:54 +0000 Subject: [PATCH 5/9] add client service and add client args to write-client-keys --- PROGRESS.md | 6 +++--- src/client_config.rs | 2 +- src/docker.rs | 4 ---- src/genesis.rs | 5 +++++ src/kubernetes.rs | 4 ++-- src/lib.rs | 14 +++++++++----- src/main.rs | 10 +++++++++- 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 00abb18..a6a1737 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -39,14 +39,14 @@ - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client - [x] Check Bootstrap is deployed and running - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) - [ ] Add metrics - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client - [ ] Create accounts - [x] Validator (regular) - [x] RPC @@ -55,7 +55,7 @@ - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - - [ ] Client + - [x] Client Above, we start with bootstrap, and then we do validators (regular), and then we do RPCs, then Clients - By the end of the Bootstrap set of PRs, we can diff --git a/src/client_config.rs b/src/client_config.rs index 15fe2af..7323389 100644 --- a/src/client_config.rs +++ b/src/client_config.rs @@ -5,7 +5,7 @@ pub struct ClientConfig { pub num_clients: usize, pub client_type: String, pub client_to_run: String, - pub bench_tps_args: Option>, + pub bench_tps_args: Vec, pub target_node: Option, pub duration: u64, pub num_nodes: Option, diff --git a/src/docker.rs b/src/docker.rs index f652f2c..ad26824 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -88,8 +88,6 @@ impl DockerConfig { } }; - info!("docker path: {:?}", docker_path); - self.create_base_image( solana_root_path, docker_image, @@ -123,8 +121,6 @@ impl DockerConfig { dockerfile.display() ); - info!("command: {command}"); - let output = Command::new("sh") .arg("-c") .arg(&command) diff --git a/src/genesis.rs b/src/genesis.rs index 7c0c2cf..640bec0 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -213,6 +213,7 @@ impl Genesis { pub fn create_client_accounts( &mut self, number_of_clients: usize, + bench_tps_args: &[String], target_lamports_per_signature: u64, config_dir: &Path, deploy_method: &DeployMethod, @@ -240,6 +241,10 @@ impl Genesis { args.push("--target-lamports-per-signature".to_string()); args.push(target_lamports_per_signature.to_string()); + if !bench_tps_args.is_empty() { + args.extend(bench_tps_args.to_owned()); + } + let executable_path = if let DeployMethod::ReleaseChannel(_) = deploy_method { solana_root_path.join("solana-release/bin/solana-bench-tps") } else { diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 44a28c9..0c62d58 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -304,8 +304,8 @@ impl<'a> Kubernetes<'a> { let mut flags = vec![]; flags.push(self.client_config.client_to_run.clone()); //client to run - if let Some(bench_tps_args) = &self.client_config.bench_tps_args { - flags.push(bench_tps_args.join(" ")); + if !self.client_config.bench_tps_args.is_empty() { + flags.push(self.client_config.bench_tps_args.join(" ")); } flags.push(self.client_config.client_type.clone()); diff --git a/src/lib.rs b/src/lib.rs index 6c11f60..f0a695d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -282,18 +282,22 @@ pub async fn fetch_spl(solana_root_path: &Path) -> Result<(), Box) -> Option> { - bench_tps_args.map(|args| { +pub fn parse_and_format_bench_tps_args(bench_tps_args: Option<&str>) -> Vec { + if let Some(args) = bench_tps_args { let mut val_args: Vec<_> = args .split_whitespace() .filter_map(|arg| arg.split_once('=')) - .flat_map(|(key, value)| vec![format!("--{}", key), value.to_string()]) + .flat_map(|(key, value)| vec![format!("--{key}"), value.to_string()]) .collect(); + let flag_args_iter = args .split_whitespace() .filter(|arg| arg.split_once('=').is_none()) - .map(|flag| format!("--{}", flag)); + .map(|flag| format!("--{flag}")); + val_args.extend(flag_args_iter); val_args - }) + } else { + Vec::new() // Return empty vec if no args provided + } } diff --git a/src/main.rs b/src/main.rs index 6bf9225..761a33a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -568,6 +568,7 @@ async fn main() -> Result<(), Box> { genesis.create_client_accounts( client_config.num_clients, + &client_config.bench_tps_args, DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE, &config_directory, &deploy_method, @@ -805,7 +806,7 @@ async fn main() -> Result<(), Box> { info!("rpc node replica set ({rpc_index}) deployed successfully"); let rpc_service = kub_controller.create_service( - &format!("rpc-node-selector-{rpc_index}"), + &format!("rpc-node-service-{rpc_index}"), rpc_node.service_labels(), ); kub_controller.deploy_service(&rpc_service).await?; @@ -916,6 +917,13 @@ async fn main() -> Result<(), Box> { .deploy_replicas_set(client.replica_set()) .await?; info!("client replica set ({client_index}) deployed successfully"); + + let client_service = kub_controller.create_service( + &format!("client-service-{client_index}"), + client.service_labels(), + ); + kub_controller.deploy_service(&client_service).await?; + info!("client service ({client_index}) deployed successfully"); } Ok(()) From 065dd0c1d145e0b206e29b82d5482f6cf1d0a785 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 13 May 2024 23:15:46 +0000 Subject: [PATCH 6/9] address jon nits from https://github.com/anza-xyz/validator-lab/pull/40 --- README.md | 2 +- src/docker.rs | 4 +-- src/main.rs | 65 ++++++++++++++---------------------------- src/startup_scripts.rs | 12 ++++---- 4 files changed, 30 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index a5612b8..4082f20 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ cargo run --bin cluster -- ``` #### RPC Nodes -You can add in RPC nodes. These sit behind a load balancer. Load balancer distributed loads across all RPC nodes that the bootstrap. Set the number of RPC nodes with: +You can add in RPC nodes. These sit behind a load balancer. Load balancer distributed loads across all RPC nodes and the bootstrap. Set the number of RPC nodes with: ``` --num-rpc-nodes ``` diff --git a/src/docker.rs b/src/docker.rs index ad26824..447247f 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -193,9 +193,9 @@ USER solana COPY --chown=solana:solana {startup_script_directory} /home/solana/k8s-cluster-scripts RUN chmod +x /home/solana/k8s-cluster-scripts/* COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger -COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/ +COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/bin/ COPY --chown=solana:solana ./{solana_build_directory}/version.yml /home/solana/ -ENV PATH="/home/solana/.cargo/bin:${{PATH}}" +ENV PATH="/home/solana/bin:${{PATH}}" WORKDIR /home/solana "#, diff --git a/src/main.rs b/src/main.rs index 761a33a..966d66a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -78,10 +78,6 @@ fn parse_matches() -> clap::ArgMatches { .takes_value(true) .default_value("0") .help("Number of validators to deploy") - .validator(|s| match s.parse::() { - Ok(n) if n >= 0 => Ok(()), - _ => Err(String::from("number_of_validators should be >= 0")), - }), ) // Genesis Config .arg( @@ -240,10 +236,6 @@ fn parse_matches() -> clap::ArgMatches { .takes_value(true) .default_value("0") .help("Number of rpc nodes") - .validator(|s| match s.parse::() { - Ok(n) if n >= 0 => Ok(()), - _ => Err(String::from("number_of_rpc_nodes should be >= 0")), - }), ) // Client Config .arg( @@ -253,10 +245,6 @@ fn parse_matches() -> clap::ArgMatches { .takes_value(true) .default_value("0") .help("Number of clients ") - .validator(|s| match s.parse::() { - Ok(n) if n >= 0 => Ok(()), - _ => Err(String::from("number_of_clients should be >= 0")), - }), ) .arg( Arg::with_name("client_type") @@ -574,13 +562,13 @@ async fn main() -> Result<(), Box> { &deploy_method, solana_root.get_root_path(), )?; - info!("Client accounts created successfully"); + info!("Client accounts created"); // creates genesis and writes to binary file genesis .generate(solana_root.get_root_path(), &build_path) .await?; - info!("Created genesis successfully"); + info!("Genesis created"); // generate standard validator accounts genesis.generate_accounts(ValidatorType::Standard, num_validators)?; @@ -649,14 +637,11 @@ async fn main() -> Result<(), Box> { if build_config.docker_build() { for v in cluster_images.get_all() { docker.build_image(solana_root.get_root_path(), v.image())?; - info!("{} image built successfully", v.validator_type()); + info!("Built {} image", v.validator_type()); } docker.push_images(cluster_images.get_all())?; - info!( - "{} docker images pushed successfully", - cluster_images.get_all().count() - ); + info!("Pushed {} docker images", cluster_images.get_all().count()); } // metrics secret create once and use by all pods @@ -673,7 +658,7 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_secret(bootstrap_validator.secret()) .await?; - info!("Deployed Bootstrap Secret"); + info!("Bootstrap Secret deployed"); // Create Bootstrap labels // Bootstrap needs two labels, one for each service. @@ -716,10 +701,7 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(bootstrap_validator.replica_set()) .await?; - info!( - "{} deployed successfully", - bootstrap_validator.replica_set_name() - ); + info!("{} deployed", bootstrap_validator.replica_set_name()); // create and deploy bootstrap-service let bootstrap_service = kub_controller.create_service( @@ -727,7 +709,7 @@ async fn main() -> Result<(), Box> { bootstrap_validator.service_labels(), ); kub_controller.deploy_service(&bootstrap_service).await?; - info!("bootstrap validator service deployed successfully"); + info!("Bootstrap Balidator Service deployed"); // load balancer service. only create one and use for all bootstrap/rpc nodes // service selector matches bootstrap selector @@ -739,7 +721,7 @@ async fn main() -> Result<(), Box> { //deploy load balancer kub_controller.deploy_service(&load_balancer).await?; - info!("load balancer service deployed successfully"); + info!("Load Balancer Service deployed"); // wait for bootstrap replicaset to deploy while !kub_controller @@ -761,7 +743,7 @@ async fn main() -> Result<(), Box> { let rpc_secret = kub_controller.create_rpc_secret(rpc_index, &config_directory)?; rpc_node.set_secret(rpc_secret); kub_controller.deploy_secret(rpc_node.secret()).await?; - info!("Deployed RPC node {rpc_index} Secret"); + info!("RPC Node {rpc_index} Secret deployed"); let identity_path = config_directory.join(format!("rpc-node-identity-{rpc_index}.json")); @@ -803,32 +785,27 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(rpc_node.replica_set()) .await?; - info!("rpc node replica set ({rpc_index}) deployed successfully"); + info!("RPC Node Replica Set ({rpc_index}) deployed"); let rpc_service = kub_controller.create_service( &format!("rpc-node-service-{rpc_index}"), rpc_node.service_labels(), ); kub_controller.deploy_service(&rpc_service).await?; - info!("rpc node service ({rpc_index}) deployed successfully"); + info!("RPC Node Service ({rpc_index}) deployed"); rpc_nodes.push(rpc_node.replica_set_name().clone()); } // wait for at least one rpc node to deploy - loop { - let mut ready = false; + 'outer: loop { for rpc_name in &rpc_nodes { if kub_controller.is_replica_set_ready(rpc_name).await? { - ready = true; - break; + break 'outer; } } - if ready { - break; - } - info!("no rpc nodes ready yet"); + info!("RPC Nodes not ready yet"); tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } @@ -841,7 +818,7 @@ async fn main() -> Result<(), Box> { kub_controller.create_validator_secret(validator_index, &config_directory)?; validator.set_secret(validator_secret); kub_controller.deploy_secret(validator.secret()).await?; - info!("Deployed Validator {validator_index} secret"); + info!("Validator {validator_index} Secret deployed"); let identity_path = config_directory.join(format!("validator-identity-{validator_index}.json")); @@ -875,14 +852,14 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(validator.replica_set()) .await?; - info!("validator replica set ({validator_index}) deployed successfully"); + info!("Validator Replica Set ({validator_index}) deployed"); let validator_service = kub_controller.create_service( &format!("validator-service-{validator_index}"), validator.service_labels(), ); kub_controller.deploy_service(&validator_service).await?; - info!("validator service ({validator_index}) deployed successfully"); + info!("Validator Service ({validator_index}) deployed"); } } @@ -897,11 +874,11 @@ async fn main() -> Result<(), Box> { client.set_secret(client_secret); kub_controller.deploy_secret(client.secret()).await?; - info!("Deployed Client {client_index} Secret"); + info!("Client {client_index} Secret deployed"); client.add_label( "client/name", - format!("client-{}", client_index), + format!("client-{client_index}"), LabelType::Service, ); @@ -916,14 +893,14 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(client.replica_set()) .await?; - info!("client replica set ({client_index}) deployed successfully"); + info!("Client Replica Set ({client_index}) deployed"); let client_service = kub_controller.create_service( &format!("client-service-{client_index}"), client.service_labels(), ); kub_controller.deploy_service(&client_service).await?; - info!("client service ({client_index}) deployed successfully"); + info!("Client Service ({client_index}) deployed"); } Ok(()) diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index 11db0e2..dc17a11 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -32,8 +32,8 @@ source /home/solana/k8s-cluster-scripts/common.sh no_restart=0 # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" @@ -238,8 +238,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" @@ -658,8 +658,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" From 004e6159b09abafcce3246b3e6bd23beb540a1ff Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 14 May 2024 15:30:38 +0000 Subject: [PATCH 7/9] fix bug. need to explicitly call --use-tpu-client in releases pre v1.19 --- src/startup_scripts.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index dc17a11..7412057 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -1024,6 +1024,8 @@ bench-tps) if ${RPC_CLIENT}; then args+=(--use-rpc-client) + else + args+=(--use-tpu-client) fi entrypointIp="${BOOTSTRAP_GOSSIP_ADDRESS:0:-5}" From 03f818747f627dfe91f9247c183a9c159ee2bd95 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 20 May 2024 21:39:48 +0000 Subject: [PATCH 8/9] clean up. fix a couple bugs. remove explicit thread setting in client --- Cargo.lock | 19 +++++++++ Cargo.toml | 3 +- src/client_config.rs | 6 +-- src/docker.rs | 20 ++++----- src/genesis.rs | 93 ++++++++++++++++++++++-------------------- src/kubernetes.rs | 6 +-- src/ledger_helper.rs | 2 - src/main.rs | 85 +++++++++++++++++++------------------- src/startup_scripts.rs | 20 +++------ 9 files changed, 130 insertions(+), 124 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e73d20..6d15f2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4737,6 +4737,24 @@ dependencies = [ "url 2.5.0", ] +[[package]] +name = "solana-clap-v3-utils" +version = "1.18.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab05348a84bbf3e81a70117d24ca46ce62343530ddfe9e8d84fb95a8061d59b" +dependencies = [ + "chrono", + "clap 3.2.25", + "rpassword", + "solana-remote-wallet", + "solana-sdk", + "solana-zk-token-sdk", + "thiserror", + "tiny-bip39", + "uriparse", + "url 2.5.0", +] + [[package]] name = "solana-cli-config" version = "1.18.9" @@ -7136,6 +7154,7 @@ dependencies = [ "rustc_version", "rustls", "solana-accounts-db", + "solana-clap-v3-utils", "solana-core", "solana-ledger", "solana-logger", diff --git a/Cargo.toml b/Cargo.toml index 3b630c4..8202d2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" [dependencies] bzip2 = "0.4.4" -clap = { version = "3.2.22", features = ["cargo"] } +clap = { version = "3.2.22", features = ["cargo"] } console = "0.15.8" git2 = "0.18.3" indicatif = "0.17.8" @@ -20,6 +20,7 @@ rand = "0.8.5" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.11", default-features = false, features = ["quic"] } solana-accounts-db = "1.18.8" +solana-clap-v3-utils = "1.18.8" solana-core = "1.18.8" solana-ledger = "1.18.8" solana-logger = "1.18.8" diff --git a/src/client_config.rs b/src/client_config.rs index 7323389..e38ca98 100644 --- a/src/client_config.rs +++ b/src/client_config.rs @@ -6,7 +6,7 @@ pub struct ClientConfig { pub client_type: String, pub client_to_run: String, pub bench_tps_args: Vec, - pub target_node: Option, - pub duration: u64, - pub num_nodes: Option, + pub client_target_node: Option, + pub client_duration_seconds: u64, + pub client_wait_for_n_nodes: Option, } diff --git a/src/docker.rs b/src/docker.rs index 447247f..aa94893 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -135,7 +135,6 @@ impl DockerConfig { return Err(output.status.to_string().into()); } progress_bar.finish_and_clear(); - info!("{validator_type} image build complete"); Ok(()) } @@ -193,18 +192,15 @@ USER solana COPY --chown=solana:solana {startup_script_directory} /home/solana/k8s-cluster-scripts RUN chmod +x /home/solana/k8s-cluster-scripts/* COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger -COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/bin/ +COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/ COPY --chown=solana:solana ./{solana_build_directory}/version.yml /home/solana/ -ENV PATH="/home/solana/bin:${{PATH}}" +ENV PATH="/home/solana/.cargo/bin:${{PATH}}" WORKDIR /home/solana +{} "#, - self.base_image - ); - - let dockerfile = format!( - "{dockerfile}\n{}", - self.insert_client_accounts_if_present(solana_root_path, validator_type)?, + self.base_image, + self.insert_client_accounts_if_present(solana_root_path, validator_type)? ); debug!("dockerfile: {dockerfile:?}"); @@ -228,10 +224,10 @@ WORKDIR /home/solana Ok(format!( r#" COPY --chown=solana:solana ./config-k8s/bench-tps-{index}.yml /home/solana/client-accounts.yml - "#, + "# )) } else { - Err(format!("{:?} does not exist!", bench_tps_path).into()) + Err(format!("{bench_tps_path:?} does not exist!").into()) } } ValidatorType::Bootstrap => { @@ -242,7 +238,7 @@ COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana "# .to_string()) } else { - Err(format!("{:?} does not exist!", client_accounts_path).into()) + Ok("".to_string()) } } ValidatorType::Standard | ValidatorType::RPC => Ok("".to_string()), diff --git a/src/genesis.rs b/src/genesis.rs index 640bec0..18ba974 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -96,28 +96,24 @@ impl std::fmt::Display for GenesisFlags { } fn append_client_accounts_to_file( - in_file: &PathBuf, //bench-tps-i.yml - out_file: &PathBuf, //client-accounts.yml + bench_tps_account_path: &PathBuf, //bench-tps-i.yml + client_accounts_path: &PathBuf, //client-accounts.yml ) -> io::Result<()> { // Open the bench-tps-i.yml file for reading. - let input = File::open(in_file)?; + let input = File::open(bench_tps_account_path)?; let reader = io::BufReader::new(input); // Open (or create) client-accounts.yml let output = OpenOptions::new() .create(true) .append(true) - .open(out_file)?; + .open(client_accounts_path)?; let mut writer = BufWriter::new(output); - // Enumerate the lines of the input file, starting from 1. - for (index, line) in reader.lines().enumerate().map(|(i, l)| (i + 1, l)) { + // Skip first line since it is a header aka "---" in a yaml + for line in reader.lines().skip(1) { let line = line?; - - // Skip first line since it is a header aka "---" in a yaml - if (index as u64) > 1 { - writeln!(writer, "{line}")?; - } + writeln!(writer, "{line}")?; } Ok(()) @@ -159,10 +155,6 @@ impl Genesis { validator_type: ValidatorType, number_of_accounts: usize, ) -> Result<(), Box> { - if let ValidatorType::Client(_) = validator_type { - return Err("Client valdiator_type in generate_accounts not allowed".into()); - } - info!("generating {number_of_accounts} {validator_type} accounts..."); let account_types = match validator_type { @@ -172,7 +164,9 @@ impl Genesis { ValidatorType::RPC => { vec!["identity"] // no vote or stake account for RPC } - ValidatorType::Client(_) => panic!("Client type not supported"), + ValidatorType::Client(_) => { + return Err("Client valdiator_type in generate_accounts not allowed".into()) + } }; let total_accounts_to_generate = number_of_accounts * account_types.len(); @@ -222,36 +216,23 @@ impl Genesis { if number_of_clients == 0 { return Ok(()); } + let client_accounts_file = config_dir.join("client-accounts.yml"); - info!("generating {number_of_clients} client account(s)..."); + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!("{WRITING}Creating and writing client accounts...")); + info!("generating {number_of_clients} client account(s)..."); let children: Result, _> = (0..number_of_clients) .map(|i| { - info!("client account: {i}"); - let mut args = Vec::new(); - let account_path = config_dir.join(format!("bench-tps-{i}.yml")); - args.push("--write-client-keys".to_string()); - args.push(account_path.into_os_string().into_string().map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid Unicode data in path: {:?}", err), - ) - })?); - args.push("--target-lamports-per-signature".to_string()); - args.push(target_lamports_per_signature.to_string()); - - if !bench_tps_args.is_empty() { - args.extend(bench_tps_args.to_owned()); - } - - let executable_path = if let DeployMethod::ReleaseChannel(_) = deploy_method { - solana_root_path.join("solana-release/bin/solana-bench-tps") - } else { - solana_root_path.join("farf/bin/solana-bench-tps") - }; - - Self::create_client_account(&args, &executable_path) + Self::create_client_account( + i, + config_dir, + target_lamports_per_signature, + bench_tps_args, + deploy_method, + solana_root_path, + ) }) .collect(); @@ -262,8 +243,6 @@ impl Genesis { } } - let progress_bar = new_spinner_progress_bar(); - progress_bar.set_message(format!("{WRITING}Writing client accounts...")); for i in 0..number_of_clients { let account_path = config_dir.join(format!("bench-tps-{i}.yml")); append_client_accounts_to_file(&account_path, &client_accounts_file)?; @@ -275,9 +254,33 @@ impl Genesis { } fn create_client_account( - args: &Vec, - executable_path: &PathBuf, + client_index: usize, + config_dir: &Path, + target_lamports_per_signature: u64, + bench_tps_args: &[String], + deploy_method: &DeployMethod, + solana_root_path: &Path, ) -> Result> { + info!("client account: {client_index}"); + let mut args = Vec::new(); + let account_path = config_dir.join(format!("bench-tps-{client_index}.yml")); + args.push("--write-client-keys".to_string()); + args.push(account_path.into_os_string().into_string().map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?); + args.push("--target-lamports-per-signature".to_string()); + args.push(target_lamports_per_signature.to_string()); + + args.extend_from_slice(bench_tps_args); + + let executable_path = if let DeployMethod::ReleaseChannel(_) = deploy_method { + solana_root_path.join("solana-release/bin/solana-bench-tps") + } else { + solana_root_path.join("farf/bin/solana-bench-tps") + }; let child = Command::new(executable_path) .args(args) .stdout(Stdio::null()) diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 0c62d58..560bedb 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -310,15 +310,15 @@ impl<'a> Kubernetes<'a> { flags.push(self.client_config.client_type.clone()); - if let Some(target_node) = self.client_config.target_node { + if let Some(target_node) = self.client_config.client_target_node { flags.push("--target-node".to_string()); flags.push(target_node.to_string()); } flags.push("--duration".to_string()); - flags.push(self.client_config.duration.to_string()); + flags.push(self.client_config.client_duration_seconds.to_string()); - if let Some(num_nodes) = self.client_config.num_nodes { + if let Some(num_nodes) = self.client_config.client_wait_for_n_nodes { flags.push("--num-nodes".to_string()); flags.push(num_nodes.to_string()); } diff --git a/src/ledger_helper.rs b/src/ledger_helper.rs index 9f369b3..e484738 100644 --- a/src/ledger_helper.rs +++ b/src/ledger_helper.rs @@ -1,6 +1,5 @@ use { crate::genesis::DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, - log::*, solana_accounts_db::hardened_unpack::open_genesis_config, solana_sdk::shred_version::compute_shred_version, std::{error::Error, path::Path}, @@ -25,7 +24,6 @@ impl LedgerHelper { let genesis_config = open_genesis_config(ledger_dir, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE); let shred_version = compute_shred_version(&genesis_config?.hash(), None); - info!("Shred Version: {}", shred_version); Ok(shred_version) } } diff --git a/src/main.rs b/src/main.rs index 966d66a..2c57946 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,11 @@ use { clap::{command, value_t_or_exit, Arg, ArgGroup}, log::*, + solana_clap_v3_utils::input_parsers::pubkey_of, solana_ledger::blockstore_cleanup_service::{ DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS, }, - solana_sdk::{pubkey::Pubkey, signature::keypair::read_keypair_file, signer::Signer}, + solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, std::{fs, path::PathBuf, result::Result}, strum::VariantNames, validator_lab::{ @@ -244,7 +245,7 @@ fn parse_matches() -> clap::ArgMatches { .short('c') .takes_value(true) .default_value("0") - .help("Number of clients ") + .help("Number of clients") ) .arg( Arg::with_name("client_type") @@ -278,25 +279,28 @@ fn parse_matches() -> clap::ArgMatches { to the bench-tps client."), ) .arg( - Arg::with_name("target_node") - .long("target-node") + Arg::with_name("client_target_node") + .long("client-target-node") .takes_value(true) - .help("Client Config. Optional: Specify an exact node to send transactions to. use: --target-node . + .value_name("PUBKEY") + .help("Client Config. Optional: Specify an exact node to send transactions to Not supported yet. TODO..."), ) .arg( - Arg::with_name("duration") - .long("duration") + Arg::with_name("client_duration_seconds") + .long("client-duration-seconds") .takes_value(true) .default_value("7500") - .help("Client Config. Seconds to run benchmark, then exit; default is forever use: --duration "), + .value_name("SECS") + .help("Client Config. Seconds to run benchmark, then exit"), ) .arg( - Arg::with_name("num_nodes") - .long("num-nodes") + Arg::with_name("client_wait_for_n_nodes") + .long("client-wait-for-n-nodes") .short('N') .takes_value(true) - .help("Client Config. Optional: Wait for NUM nodes to converge: --num-nodes "), + .value_name("NUM") + .help("Client Config. Optional: Wait for NUM nodes to converge"), ) // kubernetes config .arg( @@ -368,26 +372,18 @@ async fn main() -> Result<(), Box> { let num_rpc_nodes = value_t_or_exit!(matches, "number_of_rpc_nodes", usize); let client_config = ClientConfig { num_clients: value_t_or_exit!(matches, "number_of_clients", usize), - client_type: matches - .value_of("client_type") - .unwrap_or_default() - .to_string(), - client_to_run: matches - .value_of("client_to_run") - .unwrap_or_default() - .to_string(), + client_type: matches.value_of("client_type").unwrap().to_string(), + client_to_run: matches.value_of("client_to_run").unwrap().to_string(), bench_tps_args: parse_and_format_bench_tps_args(matches.value_of("bench_tps_args")), - target_node: match matches.value_of("target_node") { - Some(s) => match s.parse::() { - Ok(pubkey) => Some(pubkey), - Err(e) => return Err(format!("failed to parse pubkey in target_node: {e}").into()), - }, - None => None, - }, - duration: value_t_or_exit!(matches, "duration", u64), - num_nodes: matches - .value_of("num_nodes") - .map(|value_str| value_str.parse().expect("Invalid value for num_nodes")), + client_target_node: pubkey_of(&matches, "client_target_node"), + client_duration_seconds: value_t_or_exit!(matches, "client_duration_seconds", u64), + client_wait_for_n_nodes: matches + .value_of("client_wait_for_n_nodes") + .map(|value_str| { + value_str + .parse() + .expect("Invalid value for client_wait_for_n_nodes") + }), }; let deploy_method = if let Some(local_path) = matches.value_of("local_path") { @@ -543,7 +539,7 @@ async fn main() -> Result<(), Box> { } build_config.prepare().await?; - info!("Validator setup prepared successfully"); + info!("Setup Validator Environment"); let config_directory = solana_root.get_root_path().join("config-k8s"); let mut genesis = Genesis::new(config_directory.clone(), genesis_flags); @@ -580,6 +576,7 @@ async fn main() -> Result<(), Box> { let ledger_dir = config_directory.join("bootstrap-validator"); let shred_version = LedgerHelper::get_shred_version(&ledger_dir)?; kub_controller.set_shred_version(shred_version); + info!("Shred Version: {shred_version}"); //unwraps are safe here. since their requirement is enforced by argmatches let docker = DockerConfig::new( @@ -658,7 +655,7 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_secret(bootstrap_validator.secret()) .await?; - info!("Bootstrap Secret deployed"); + info!("Deployed Bootstrap Secret"); // Create Bootstrap labels // Bootstrap needs two labels, one for each service. @@ -701,7 +698,7 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(bootstrap_validator.replica_set()) .await?; - info!("{} deployed", bootstrap_validator.replica_set_name()); + info!("Deployed {}", bootstrap_validator.replica_set_name()); // create and deploy bootstrap-service let bootstrap_service = kub_controller.create_service( @@ -709,7 +706,7 @@ async fn main() -> Result<(), Box> { bootstrap_validator.service_labels(), ); kub_controller.deploy_service(&bootstrap_service).await?; - info!("Bootstrap Balidator Service deployed"); + info!("Deployed Bootstrap Balidator Service"); // load balancer service. only create one and use for all bootstrap/rpc nodes // service selector matches bootstrap selector @@ -721,7 +718,7 @@ async fn main() -> Result<(), Box> { //deploy load balancer kub_controller.deploy_service(&load_balancer).await?; - info!("Load Balancer Service deployed"); + info!("Deployed Load Balancer Service"); // wait for bootstrap replicaset to deploy while !kub_controller @@ -743,7 +740,7 @@ async fn main() -> Result<(), Box> { let rpc_secret = kub_controller.create_rpc_secret(rpc_index, &config_directory)?; rpc_node.set_secret(rpc_secret); kub_controller.deploy_secret(rpc_node.secret()).await?; - info!("RPC Node {rpc_index} Secret deployed"); + info!("Deployed RPC Node {rpc_index} Secret"); let identity_path = config_directory.join(format!("rpc-node-identity-{rpc_index}.json")); @@ -785,14 +782,14 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(rpc_node.replica_set()) .await?; - info!("RPC Node Replica Set ({rpc_index}) deployed"); + info!("Deployed RPC Node Replica Set ({rpc_index})"); let rpc_service = kub_controller.create_service( &format!("rpc-node-service-{rpc_index}"), rpc_node.service_labels(), ); kub_controller.deploy_service(&rpc_service).await?; - info!("RPC Node Service ({rpc_index}) deployed"); + info!("Deployed RPC Node Service ({rpc_index})"); rpc_nodes.push(rpc_node.replica_set_name().clone()); } @@ -818,7 +815,7 @@ async fn main() -> Result<(), Box> { kub_controller.create_validator_secret(validator_index, &config_directory)?; validator.set_secret(validator_secret); kub_controller.deploy_secret(validator.secret()).await?; - info!("Validator {validator_index} Secret deployed"); + info!("Deployed Validator {validator_index} Secret"); let identity_path = config_directory.join(format!("validator-identity-{validator_index}.json")); @@ -852,14 +849,14 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(validator.replica_set()) .await?; - info!("Validator Replica Set ({validator_index}) deployed"); + info!("Deployed Validator Replica Set ({validator_index})"); let validator_service = kub_controller.create_service( &format!("validator-service-{validator_index}"), validator.service_labels(), ); kub_controller.deploy_service(&validator_service).await?; - info!("Validator Service ({validator_index}) deployed"); + info!("Deployed Validator Service ({validator_index})"); } } @@ -874,7 +871,7 @@ async fn main() -> Result<(), Box> { client.set_secret(client_secret); kub_controller.deploy_secret(client.secret()).await?; - info!("Client {client_index} Secret deployed"); + info!("Deployed Client {client_index} Secret"); client.add_label( "client/name", @@ -893,14 +890,14 @@ async fn main() -> Result<(), Box> { kub_controller .deploy_replicas_set(client.replica_set()) .await?; - info!("Client Replica Set ({client_index}) deployed"); + info!("Deployed Client Replica Set ({client_index})"); let client_service = kub_controller.create_service( &format!("client-service-{client_index}"), client.service_labels(), ); kub_controller.deploy_service(&client_service).await?; - info!("Client Service ({client_index}) deployed"); + info!("Deployed Client Service ({client_index})"); } Ok(()) diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index 7412057..99f0c4d 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -32,8 +32,8 @@ source /home/solana/k8s-cluster-scripts/common.sh no_restart=0 # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/bin/agave-validator" -solana_validator="/home/solana/bin/solana-validator" +agave_validator="/home/solana/.cargo/bin/agave-validator" +solana_validator="/home/solana/.cargo/bin/solana-validator" # Initialize program variable program="" @@ -238,8 +238,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/bin/agave-validator" -solana_validator="/home/solana/bin/solana-validator" +agave_validator="/home/solana/.cargo/bin/agave-validator" +solana_validator="/home/solana/.cargo/bin/solana-validator" # Initialize program variable program="" @@ -658,8 +658,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/bin/agave-validator" -solana_validator="/home/solana/bin/solana-validator" +agave_validator="/home/solana/.cargo/bin/agave-validator" +solana_validator="/home/solana/.cargo/bin/solana-validator" # Initialize program variable program="" @@ -998,13 +998,6 @@ missing() { exit 1 } -threadCount=$(nproc) -if [[ $threadCount -gt 4 ]]; then - threadCount=4 -fi - -echo "threadCount: $threadCount" - RPC_CLIENT=false case "$clientType" in tpu-client) @@ -1037,7 +1030,6 @@ bench-tps) clientCommand="\ solana-bench-tps \ --sustained \ - --threads $threadCount \ $benchTpsExtraArgs \ --read-client-keys ./client-accounts.yml \ --url "http://$url" From 211a23f571fa8bbc7362e994bb72ca1295dcd6f5 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 22 May 2024 15:46:49 +0000 Subject: [PATCH 9/9] address comments --- PROGRESS.md | 14 +++++++------- README.md | 2 +- src/docker.rs | 4 ++-- src/startup_scripts.rs | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index a6a1737..7298a03 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -20,38 +20,38 @@ - [x] Validator (regular) - [x] RPC nodes - [x] Client -- [ ] Create & Deploy Secrets +- [x] Create & Deploy Secrets - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - [x] Client -- [ ] Create & Deploy Selector +- [x] Create & Deploy Selector - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - [x] Client -- [ ] Create & Deploy Replica Set +- [x] Create & Deploy Replica Set - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - [x] Client -- [ ] Create & Deploy Services +- [x] Create & Deploy Services - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - [x] Client - [x] Check Bootstrap is deployed and running - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) -- [ ] Add metrics +- [x] Add metrics - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes - [x] Client -- [ ] Create accounts +- [x] Create accounts - [x] Validator (regular) - [x] RPC - [x] Client -- [ ] Add feature flags to configure: +- [x] Add feature flags to configure: - [x] Bootstrap - [x] Validator (regular) - [x] RPC nodes diff --git a/README.md b/README.md index 4082f20..86cdfc0 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ cargo run --bin cluster -- -c --client-type --client-to-run - --num-nodes + --client-wait-for-n-nodes --bench-tps-args ``` diff --git a/src/docker.rs b/src/docker.rs index aa94893..0b0df72 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -192,9 +192,9 @@ USER solana COPY --chown=solana:solana {startup_script_directory} /home/solana/k8s-cluster-scripts RUN chmod +x /home/solana/k8s-cluster-scripts/* COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger -COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/ +COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/bin/ COPY --chown=solana:solana ./{solana_build_directory}/version.yml /home/solana/ -ENV PATH="/home/solana/.cargo/bin:${{PATH}}" +ENV PATH="/home/solana/bin:${{PATH}}" WORKDIR /home/solana {} diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index 99f0c4d..781f9a5 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -32,8 +32,8 @@ source /home/solana/k8s-cluster-scripts/common.sh no_restart=0 # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" @@ -238,8 +238,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program="" @@ -658,8 +658,8 @@ ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS # Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` -agave_validator="/home/solana/.cargo/bin/agave-validator" -solana_validator="/home/solana/.cargo/bin/solana-validator" +agave_validator="/home/solana/bin/agave-validator" +solana_validator="/home/solana/bin/solana-validator" # Initialize program variable program=""