fix bug, add rpc nodes (#40)

* wip

* fix script format bug and common.sh copy

* validators connecting through gossip but now showing up on solana-ul validators. show they are caught up though with catchup command

* default to --enable-warmup-epochs to true

* create rpc accounts

* build/push rpc-nodes, add rpc-node startup script, deploy rpc node secrets

* deploy rpc replica sets. but will not work until service added

* add rpc services

* wait for >= 1 rpc node before deploying validators

PROGRESS.md

@@ -18,43 +18,43 @@
 - [x] Docker Build & Push
     - [x] Bootstrap
     - [x] Validator (regular)
-    - [ ] RPC nodes
+    - [x] RPC nodes
     - [ ] Client
 - [ ] Create & Deploy Secrets
     - [x] Bootstrap
     - [x] Validator (regular)
-    - [ ] RPC nodes
+    - [x] RPC nodes
     - [ ] Client
 - [ ] Create & Deploy Selector
     - [x] Bootstrap
     - [x] Validator (regular)
-    - [ ] RPC nodes
+    - [x] RPC nodes
     - [ ] Client
 - [ ] Create & Deploy Replica Set
     - [x] Bootstrap
     - [x] Validator (regular)
-    - [ ] RPC nodes
+    - [x] RPC nodes
     - [ ] Client
 - [ ] Create & Deploy Services
     - [x] Bootstrap
-    - [ ] Validator (regular)
-    - [ ] RPC nodes
+    - [x] Validator (regular)
+    - [x] RPC nodes
     - [ ] Client
 - [x] Check Bootstrap is deployed and running
 - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes)
 - [ ] Add metrics
     - [x] Bootstrap
     - [x] Validator (regular)
-    - [ ] RPC nodes
+    - [x] RPC nodes
     - [ ] Client
 - [ ] Create accounts
     - [x] Validator (regular)
-    - [ ] RPC
+    - [x] RPC
     - [ ] Client
 - [ ] Add feature flags to configure:
-    - [ ] Bootstrap
-    - [ ] Validator (regular)
-    - [ ] RPC nodes
+    - [x] Bootstrap
+    - [x] Validator (regular)
+    - [x] RPC nodes
     - [ ] Client
 
 Above, we start with bootstrap, and then we do validators (regular), and then we do RPCs, then Clients

README.md

@@ -45,7 +45,6 @@ cargo run --bin cluster --
     --num_validators <number-of-non-bootstrap-voting-validators>
     # genesis config. Optional: Many of these have defaults
     --hashes-per-tick <hashes-per-tick>
-    --enable-warmup-epochs <true|false>
     --faucet-lamports <faucet-lamports>
     --bootstrap-validator-sol <validator-sol>
     --bootstrap-validator-stake-sol <validator-stake>
@@ -81,6 +80,12 @@ cargo run --bin cluster --
 --metrics-password <metrics-password>   # from (1)
 ```
 
+#### RPC Nodes
+You can add in RPC nodes. These sit behind a load balancer. Load balancer distributed loads across all RPC nodes that the bootstrap. Set the number of RPC nodes with:
+```
+--num-rpc-nodes <num-nodes>
+```
+
 
 ## Kubernetes Cheatsheet
 Create namespace:

src/cluster_images.rs

@@ -13,7 +13,7 @@ use {
 pub struct ClusterImages {
     bootstrap: Option<Validator>,
     validator: Option<Validator>,
-    _rpc: Option<Validator>,
+    rpc: Option<Validator>,
     _clients: Vec<Validator>,
 }
 
@@ -22,6 +22,7 @@ impl ClusterImages {
         match validator_type {
             ValidatorType::Bootstrap => self.bootstrap = Some(item),
             ValidatorType::Standard => self.validator = Some(item),
+            ValidatorType::RPC => self.rpc = Some(item),
             _ => panic!("{validator_type} not implemented yet!"),
         }
     }
@@ -38,11 +39,17 @@ impl ClusterImages {
             .ok_or_else(|| "Validator is not available".into())
     }
 
+    pub fn rpc(&mut self) -> Result<&mut Validator, Box<dyn Error>> {
+        self.rpc
+            .as_mut()
+            .ok_or_else(|| "Validator is not available".into())
+    }
+
     pub fn get_validators(&self) -> impl Iterator<Item = &Validator> {
         self.bootstrap
             .iter()
             .chain(self.validator.iter())
-            .chain(self._rpc.iter())
+            .chain(self.rpc.iter())
             .filter_map(Some)
     }
 }

src/docker.rs

@@ -73,8 +73,8 @@ impl DockerConfig {
     ) -> Result<(), Box<dyn Error>> {
         let validator_type = docker_image.validator_type();
         match validator_type {
-            ValidatorType::Bootstrap | ValidatorType::Standard => (),
-            ValidatorType::RPC | ValidatorType::Client => {
+            ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => (),
+            ValidatorType::Client => {
                 return Err(format!(
                     "Build docker image for validator type: {validator_type} not supported yet"
                 )
@@ -157,20 +157,19 @@ impl DockerConfig {
         fs::create_dir_all(docker_path)?;
 
         match validator_type {
-            ValidatorType::Bootstrap | ValidatorType::Standard => {
-                let files_to_copy = [
-                    format!("{validator_type}-startup-script.sh"),
-                    "common.sh".to_string(),
-                ];
-                for file_name in files_to_copy.iter() {
-                    Self::write_startup_script_to_docker_directory(
-                        file_name,
-                        docker_path,
-                        validator_type,
-                    )?;
-                }
+            ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => {
+                let file_name = format!("{validator_type}-startup-script.sh");
+                Self::write_startup_script_to_docker_directory(
+                    &file_name,
+                    docker_path,
+                    validator_type,
+                )?;
+                StartupScripts::write_script_to_file(
+                    StartupScripts::common(),
+                    &docker_path.join("common.sh"),
+                )?;
             }
-            ValidatorType::RPC | ValidatorType::Client => todo!(),
+            ValidatorType::Client => todo!(),
         }
 
         let startup_script_directory = format!("./docker-build/{validator_type}");

src/k8s_helpers.rs

@@ -17,8 +17,8 @@ use {
 };
 
 pub enum SecretType {
-    Value { v: String },
-    File { path: PathBuf },
+    Value { v: String },    // will be read by pod via ENV variable
+    File { path: PathBuf }, // will be read by pod as .json file
 }
 
 fn build_secret(name: String, data: BTreeMap<String, ByteString>) -> Secret {
@@ -43,7 +43,7 @@ pub fn create_secret(
             SecretType::File { path } => {
                 let content = std::fs::read(&path)
                     .map_err(|err| format!("Failed to read file '{:?}': {}", path, err))?;
-                Ok((label, ByteString(content)))
+                Ok((format!("{label}.json"), ByteString(content)))
             }
         })
         .collect::<Result<BTreeMap<String, ByteString>, Box<dyn Error>>>()?;

src/kubernetes.rs

@@ -9,8 +9,8 @@ use {
         api::{
             apps::v1::ReplicaSet,
             core::v1::{
-                EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretKeySelector,
-                SecretVolumeSource, Service, Volume, VolumeMount,
+                EnvVar, EnvVarSource, ExecAction, Namespace, ObjectFieldSelector, Probe, Secret,
+                SecretKeySelector, SecretVolumeSource, Service, Volume, VolumeMount,
             },
         },
         apimachinery::pkg::api::resource::Quantity,
@@ -149,6 +149,30 @@ impl<'a> Kubernetes<'a> {
         k8s_helpers::create_secret(secret_name.to_string(), secrets)
     }
 
+    pub fn create_rpc_secret(
+        &self,
+        rpc_index: usize,
+        config_dir: &Path,
+    ) -> Result<Secret, Box<dyn Error>> {
+        let secret_name = format!("rpc-node-account-secret-{rpc_index}");
+
+        let mut secrets = BTreeMap::new();
+        secrets.insert(
+            "identity".to_string(),
+            SecretType::File {
+                path: config_dir.join(format!("rpc-node-identity-{rpc_index}.json")),
+            },
+        );
+        secrets.insert(
+            "faucet".to_string(),
+            SecretType::File {
+                path: config_dir.join("faucet.json"),
+            },
+        );
+
+        k8s_helpers::create_secret(secret_name, secrets)
+    }
+
     pub fn add_known_validator(&mut self, pubkey: Pubkey) {
         self.validator_config.known_validators.push(pubkey);
         info!("pubkey added to known validators: {:?}", pubkey);
@@ -197,8 +221,11 @@ impl<'a> Kubernetes<'a> {
             ..Default::default()
         }]);
 
-        let mut command =
-            vec!["/home/solana/k8s-cluster-scripts/bootstrap-startup-script.sh".to_string()];
+        let command_path = format!(
+            "/home/solana/k8s-cluster-scripts/{}-startup-script.sh",
+            ValidatorType::Bootstrap
+        );
+        let mut command = vec![command_path];
         command.extend(self.generate_bootstrap_command_flags());
 
         k8s_helpers::create_replica_set(
@@ -375,23 +402,23 @@ impl<'a> Kubernetes<'a> {
             k8s_helpers::create_environment_variable(
                 "LOAD_BALANCER_RPC_ADDRESS".to_string(),
                 Some(
-                    "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:8899"
+                    "bootstrap-and-rpc-node-lb-service.$(NAMESPACE).svc.cluster.local:8899"
                         .to_string(),
                 ),
                 None,
             ),
             k8s_helpers::create_environment_variable(
                 "LOAD_BALANCER_GOSSIP_ADDRESS".to_string(),
                 Some(
-                    "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:8001"
+                    "bootstrap-and-rpc-node-lb-service.$(NAMESPACE).svc.cluster.local:8001"
                         .to_string(),
                 ),
                 None,
             ),
             k8s_helpers::create_environment_variable(
                 "LOAD_BALANCER_FAUCET_ADDRESS".to_string(),
                 Some(
-                    "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:9900"
+                    "bootstrap-and-rpc-node-lb-service.$(NAMESPACE).svc.cluster.local:9900"
                         .to_string(),
                 ),
                 None,
@@ -471,4 +498,92 @@ impl<'a> Kubernetes<'a> {
             None,
         )
     }
+
+    fn generate_rpc_command_flags(&self) -> Vec<String> {
+        let mut flags: Vec<String> = Vec::new();
+        self.generate_command_flags(&mut flags);
+        if let Some(shred_version) = self.validator_config.shred_version {
+            flags.push("--expected-shred-version".to_string());
+            flags.push(shred_version.to_string());
+        }
+
+        self.add_known_validators_if_exists(&mut flags);
+
+        flags
+    }
+
+    pub fn create_rpc_replica_set(
+        &mut self,
+        image: &DockerImage,
+        secret_name: Option<String>,
+        label_selector: &BTreeMap<String, String>,
+        rpc_index: usize,
+    ) -> Result<ReplicaSet, Box<dyn Error>> {
+        let mut env_vars = vec![EnvVar {
+            name: "MY_POD_IP".to_string(),
+            value_from: Some(EnvVarSource {
+                field_ref: Some(ObjectFieldSelector {
+                    field_path: "status.podIP".to_string(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }),
+            ..Default::default()
+        }];
+        env_vars.append(&mut self.set_non_bootstrap_environment_variables());
+        env_vars.append(&mut self.set_load_balancer_environment_variables());
+
+        if self.metrics.is_some() {
+            env_vars.push(self.get_metrics_env_var_secret())
+        }
+
+        let accounts_volume = Some(vec![Volume {
+            name: format!("rpc-node-accounts-volume-{}", rpc_index),
+            secret: Some(SecretVolumeSource {
+                secret_name,
+                ..Default::default()
+            }),
+            ..Default::default()
+        }]);
+
+        let accounts_volume_mount = Some(vec![VolumeMount {
+            name: format!("rpc-node-accounts-volume-{}", rpc_index),
+            mount_path: "/home/solana/rpc-node-accounts".to_string(),
+            ..Default::default()
+        }]);
+
+        let mut command =
+            vec!["/home/solana/k8s-cluster-scripts/rpc-node-startup-script.sh".to_string()];
+        command.extend(self.generate_rpc_command_flags());
+
+        let exec_action = ExecAction {
+            command: Some(vec![
+                String::from("/bin/bash"),
+                String::from("-c"),
+                String::from(
+                    "solana -u http://$MY_POD_IP:8899 balance -k rpc-node-accounts/identity.json",
+                ),
+            ]),
+        };
+
+        let readiness_probe = Probe {
+            exec: Some(exec_action),
+            initial_delay_seconds: Some(20),
+            period_seconds: Some(20),
+            ..Default::default()
+        };
+
+        k8s_helpers::create_replica_set(
+            format!("{}-{rpc_index}", ValidatorType::RPC),
+            self.namespace.clone(),
+            label_selector.clone(),
+            image.clone(),
+            env_vars,
+            command.clone(),
+            accounts_volume,
+            accounts_volume_mount,
+            self.pod_requests.requests.clone(),
+            Some(readiness_probe),
+        )
+    }
 }

src/lib.rs

@@ -61,6 +61,7 @@ impl ValidatorType {
         match self {
             ValidatorType::Bootstrap => Ok(startup_scripts::StartupScripts::bootstrap()),
             ValidatorType::Standard => Ok(startup_scripts::StartupScripts::validator()),
+            ValidatorType::RPC => Ok(startup_scripts::StartupScripts::rpc()),
             _ => Err(format!("ValidatorType {:?} not supported!", self).into()),
         }
     }