Add clients

Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 
 [dependencies]
 bzip2 = "0.4.4"
-clap = { version = "3.2.22", features = ["cargo"] } 
+clap = { version = "3.2.22", features = ["cargo"] }
 console = "0.15.8"
 git2 = "0.18.3"
 indicatif = "0.17.8"
@@ -20,6 +20,7 @@ rand = "0.8.5"
 reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] }
 rustls = { version = "0.21.11", default-features = false, features = ["quic"] }
 solana-accounts-db = "1.18.8"
+solana-clap-v3-utils = "1.18.8"
 solana-core = "1.18.8"
 solana-ledger = "1.18.8"
 solana-logger = "1.18.8"

PROGRESS.md

@@ -19,43 +19,43 @@
     - [x] Bootstrap
     - [x] Validator (regular)
     - [x] RPC nodes
-    - [ ] Client
-- [ ] Create & Deploy Secrets
+    - [x] Client
+- [x] Create & Deploy Secrets
     - [x] Bootstrap
     - [x] Validator (regular)
     - [x] RPC nodes
-    - [ ] Client
-- [ ] Create & Deploy Selector
+    - [x] Client
+- [x] Create & Deploy Selector
     - [x] Bootstrap
     - [x] Validator (regular)
     - [x] RPC nodes
-    - [ ] Client
-- [ ] Create & Deploy Replica Set
+    - [x] Client
+- [x] Create & Deploy Replica Set
     - [x] Bootstrap
     - [x] Validator (regular)
     - [x] RPC nodes
-    - [ ] Client
-- [ ] Create & Deploy Services
+    - [x] Client
+- [x] Create & Deploy Services
     - [x] Bootstrap
     - [x] Validator (regular)
     - [x] RPC nodes
-    - [ ] Client
+    - [x] Client
 - [x] Check Bootstrap is deployed and running
 - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes)
-- [ ] Add metrics
+- [x] Add metrics
     - [x] Bootstrap
     - [x] Validator (regular)
     - [x] RPC nodes
-    - [ ] Client
-- [ ] Create accounts
+    - [x] Client
+- [x] Create accounts
     - [x] Validator (regular)
     - [x] RPC
-    - [ ] Client
-- [ ] Add feature flags to configure:
+    - [x] Client
+- [x] Add feature flags to configure:
     - [x] Bootstrap
     - [x] Validator (regular)
     - [x] RPC nodes
-    - [ ] Client
+    - [x] Client
 
 Above, we start with bootstrap, and then we do validators (regular), and then we do RPCs, then Clients
 - By the end of the Bootstrap set of PRs, we can

README.md

@@ -63,6 +63,12 @@ cargo run --bin cluster --
     # kubernetes config
     --cpu-requests <cores>
     --memory-requests <memory>
+    # deploy with clients
+    -c <num-clients>
+    --client-type <client-type e.g. tpu-client>
+    --client-to-run <type-of-client e.g. bench-tps>
+    --client-wait-for-n-nodes <wait-for-N-nodes-to-converge-before-starting-client>
+    --bench-tps-args <bench-tps-args e.g. tx-count=25000>
 ```
 
 ## Metrics
@@ -81,7 +87,7 @@ cargo run --bin cluster --
 ```
 
 #### RPC Nodes
-You can add in RPC nodes. These sit behind a load balancer. Load balancer distributed loads across all RPC nodes that the bootstrap. Set the number of RPC nodes with:
+You can add in RPC nodes. These sit behind a load balancer. Load balancer distributed loads across all RPC nodes and the bootstrap. Set the number of RPC nodes with:
 ```
 --num-rpc-nodes <num-nodes>
 ```

src/client_config.rs

@@ -0,0 +1,12 @@
+use solana_sdk::pubkey::Pubkey;
+
+#[derive(Clone, Debug)]
+pub struct ClientConfig {
+    pub num_clients: usize,
+    pub client_type: String,
+    pub client_to_run: String,
+    pub bench_tps_args: Vec<String>,
+    pub client_target_node: Option<Pubkey>,
+    pub client_duration_seconds: u64,
+    pub client_wait_for_n_nodes: Option<u64>,
+}

src/cluster_images.rs

@@ -14,7 +14,7 @@ pub struct ClusterImages {
     bootstrap: Option<Validator>,
     validator: Option<Validator>,
     rpc: Option<Validator>,
-    _clients: Vec<Validator>,
+    clients: Vec<Validator>,
 }
 
 impl ClusterImages {
@@ -23,7 +23,7 @@ impl ClusterImages {
             ValidatorType::Bootstrap => self.bootstrap = Some(item),
             ValidatorType::Standard => self.validator = Some(item),
             ValidatorType::RPC => self.rpc = Some(item),
-            _ => panic!("{validator_type} not implemented yet!"),
+            ValidatorType::Client(_) => self.clients.push(item),
         }
     }
 
@@ -45,11 +45,32 @@ impl ClusterImages {
             .ok_or_else(|| "Validator is not available".into())
     }
 
+    pub fn client(&mut self, client_index: usize) -> Result<&mut Validator, Box<dyn Error>> {
+        if self.clients.is_empty() {
+            return Err("No Clients available".to_string().into());
+        }
+        self.clients
+            .get_mut(client_index)
+            .ok_or_else(|| "Client index out of bounds".to_string().into())
+    }
+
     pub fn get_validators(&self) -> impl Iterator<Item = &Validator> {
         self.bootstrap
             .iter()
             .chain(self.validator.iter())
             .chain(self.rpc.iter())
             .filter_map(Some)
     }
+
+    pub fn get_clients(&self) -> impl Iterator<Item = &Validator> {
+        self.clients.iter()
+    }
+
+    pub fn get_clients_mut(&mut self) -> impl Iterator<Item = &mut Validator> {
+        self.clients.iter_mut()
+    }
+
+    pub fn get_all(&self) -> impl Iterator<Item = &Validator> {
+        self.get_validators().chain(self.get_clients())
+    }
 }

src/docker.rs

@@ -45,11 +45,18 @@ impl DockerImage {
 // Put DockerImage in format for building, pushing, and pulling
 impl Display for DockerImage {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(
-            f,
-            "{}/{}-{}:{}",
-            self.registry, self.validator_type, self.image_name, self.tag
-        )
+        match self.validator_type {
+            ValidatorType::Client(index) => write!(
+                f,
+                "{}/{}-{}-{}:{}",
+                self.registry, self.validator_type, index, self.image_name, self.tag
+            ),
+            ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => write!(
+                f,
+                "{}/{}-{}:{}",
+                self.registry, self.validator_type, self.image_name, self.tag
+            ),
+        }
     }
 }
 
@@ -72,17 +79,15 @@ impl DockerConfig {
         docker_image: &DockerImage,
     ) -> Result<(), Box<dyn Error>> {
         let validator_type = docker_image.validator_type();
-        match validator_type {
-            ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => (),
-            ValidatorType::Client => {
-                return Err(format!(
-                    "Build docker image for validator type: {validator_type} not supported yet"
-                )
-                .into());
+        let docker_path = match validator_type {
+            ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => {
+                solana_root_path.join(format!("docker-build/{validator_type}"))
             }
-        }
+            ValidatorType::Client(index) => {
+                solana_root_path.join(format!("docker-build/{validator_type}-{index}"))
+            }
+        };
 
-        let docker_path = solana_root_path.join(format!("docker-build/{validator_type}"));
         self.create_base_image(
             solana_root_path,
             docker_image,
@@ -100,7 +105,7 @@ impl DockerConfig {
         docker_path: &PathBuf,
         validator_type: &ValidatorType,
     ) -> Result<(), Box<dyn Error>> {
-        self.create_dockerfile(validator_type, docker_path, None)?;
+        self.create_dockerfile(validator_type, docker_path, solana_root_path, None)?;
 
         // We use std::process::Command here because Docker-rs is very slow building dockerfiles
         // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively.
@@ -130,7 +135,6 @@ impl DockerConfig {
             return Err(output.status.to_string().into());
         }
         progress_bar.finish_and_clear();
-        info!("{validator_type} image build complete");
 
         Ok(())
     }
@@ -141,38 +145,35 @@ impl DockerConfig {
         validator_type: &ValidatorType,
     ) -> Result<(), Box<dyn Error>> {
         let script_path = docker_dir.join(file_name);
-        let script_content = validator_type.script()?;
+        let script_content = validator_type.script();
         StartupScripts::write_script_to_file(script_content, &script_path).map_err(|e| e.into())
     }
 
     fn create_dockerfile(
         &self,
         validator_type: &ValidatorType,
         docker_path: &PathBuf,
+        solana_root_path: &Path,
         content: Option<&str>,
     ) -> Result<(), Box<dyn Error>> {
         if docker_path.exists() {
             fs::remove_dir_all(docker_path)?;
         }
         fs::create_dir_all(docker_path)?;
 
-        match validator_type {
+        let file_name = format!("{validator_type}-startup-script.sh");
+        Self::write_startup_script_to_docker_directory(&file_name, docker_path, validator_type)?;
+        StartupScripts::write_script_to_file(
+            StartupScripts::common(),
+            &docker_path.join("common.sh"),
+        )?;
+
+        let startup_script_directory = match validator_type {
             ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => {
-                let file_name = format!("{validator_type}-startup-script.sh");
-                Self::write_startup_script_to_docker_directory(
-                    &file_name,
-                    docker_path,
-                    validator_type,
-                )?;
-                StartupScripts::write_script_to_file(
-                    StartupScripts::common(),
-                    &docker_path.join("common.sh"),
-                )?;
+                format!("./docker-build/{validator_type}")
             }
-            ValidatorType::Client => todo!(),
-        }
-
-        let startup_script_directory = format!("./docker-build/{validator_type}");
+            ValidatorType::Client(index) => format!("./docker-build/{validator_type}-{index}"),
+        };
         let solana_build_directory = if let DeployMethod::ReleaseChannel(_) = self.deploy_method {
             "solana-release"
         } else {
@@ -191,13 +192,15 @@ USER solana
 COPY --chown=solana:solana  {startup_script_directory} /home/solana/k8s-cluster-scripts
 RUN chmod +x /home/solana/k8s-cluster-scripts/*
 COPY --chown=solana:solana ./config-k8s/bootstrap-validator  /home/solana/ledger
-COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/
+COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/bin/
 COPY --chown=solana:solana ./{solana_build_directory}/version.yml /home/solana/
-ENV PATH="/home/solana/.cargo/bin:${{PATH}}"
+ENV PATH="/home/solana/bin:${{PATH}}"
 
 WORKDIR /home/solana
+{}
 "#,
-            self.base_image
+            self.base_image,
+            self.insert_client_accounts_if_present(solana_root_path, validator_type)?
         );
 
         debug!("dockerfile: {dockerfile:?}");
@@ -208,6 +211,40 @@ WORKDIR /home/solana
         Ok(())
     }
 
+    fn insert_client_accounts_if_present(
+        &self,
+        solana_root_path: &Path,
+        validator_type: &ValidatorType,
+    ) -> Result<String, Box<dyn Error>> {
+        match validator_type {
+            ValidatorType::Client(index) => {
+                let bench_tps_path =
+                    solana_root_path.join(format!("config-k8s/bench-tps-{index}.yml"));
+                if bench_tps_path.exists() {
+                    Ok(format!(
+                        r#"
+COPY --chown=solana:solana ./config-k8s/bench-tps-{index}.yml /home/solana/client-accounts.yml
+                    "#
+                    ))
+                } else {
+                    Err(format!("{bench_tps_path:?} does not exist!").into())
+                }
+            }
+            ValidatorType::Bootstrap => {
+                let client_accounts_path = solana_root_path.join("config-k8s/client-accounts.yml");
+                if client_accounts_path.exists() {
+                    Ok(r#"
+COPY --chown=solana:solana ./config-k8s/client-accounts.yml /home/solana
+                    "#
+                    .to_string())
+                } else {
+                    Ok("".to_string())
+                }
+            }
+            ValidatorType::Standard | ValidatorType::RPC => Ok("".to_string()),
+        }
+    }
+
     pub fn push_image(docker_image: &DockerImage) -> Result<Child, Box<dyn Error>> {
         let command = format!("docker push '{docker_image}'");
         let child = Command::new("sh")