diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index d45196e..b35298f 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -1,12 +1,11 @@ use { - crate::ValidatorType, + crate::{docker::DockerImage, ValidatorType}, k8s_openapi::{ api::{ apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ - Container, EnvVar, PodSecurityContext, - PodSpec, PodTemplateSpec, Probe, ResourceRequirements, Secret, - Volume, VolumeMount, + Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, + ResourceRequirements, Secret, Volume, VolumeMount, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -52,8 +51,7 @@ pub fn create_replica_set( name: &ValidatorType, namespace: &str, label_selector: &BTreeMap, - container_name: &str, - image_name: &str, + image_name: &DockerImage, environment_variables: Vec, command: &[String], volumes: Option>, @@ -68,7 +66,7 @@ pub fn create_replica_set( }), spec: Some(PodSpec { containers: vec![Container { - name: container_name.to_string(), + name: format!("{}-{}", image_name.validator_type(), "container"), image: Some(image_name.to_string()), image_pull_policy: Some("Always".to_string()), env: Some(environment_variables), diff --git a/src/kubernetes.rs b/src/kubernetes.rs index cb597a5..538584f 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,11 +1,11 @@ use { - crate::{k8s_helpers, ValidatorType, validator_config::ValidatorConfig}, + crate::{docker::DockerImage, k8s_helpers, validator_config::ValidatorConfig, ValidatorType}, k8s_openapi::{ api::{ apps::v1::ReplicaSet, core::v1::{ - EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, - Secret, SecretVolumeSource, Volume, VolumeMount, + EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, + Volume, VolumeMount, }, }, apimachinery::pkg::api::resource::Quantity, @@ -15,6 +15,7 @@ use { Client, }, log::*, + solana_sdk::{pubkey::Pubkey, signature::keypair::read_keypair_file, signer::Signer}, std::{collections::BTreeMap, error::Error, path::PathBuf}, }; @@ -44,8 +45,11 @@ pub struct Kubernetes<'a> { } impl<'a> Kubernetes<'a> { - pub async fn new(namespace: &str, validator_config: &'a mut ValidatorConfig, pod_requests: PodRequests -) -> Kubernetes<'a> { + pub async fn new( + namespace: &str, + validator_config: &'a mut ValidatorConfig, + pod_requests: PodRequests, + ) -> Kubernetes<'a> { Self { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), @@ -67,7 +71,7 @@ impl<'a> Kubernetes<'a> { } pub fn create_bootstrap_secret( - &self, + &mut self, secret_name: &str, config_dir: &PathBuf, ) -> Result> { @@ -76,6 +80,12 @@ impl<'a> Kubernetes<'a> { let vote_key_path = config_dir.join("bootstrap-validator/vote-account.json"); let stake_key_path = config_dir.join("bootstrap-validator/stake-account.json"); + let bootstrap_keypair = read_keypair_file(identity_key_path.clone()) + .expect("Failed to read bootstrap validator keypair file"); + + //TODO: need to fix and not read the json path twice + self.add_known_validator(bootstrap_keypair.pubkey()); + let key_files = vec![ (faucet_key_path, "faucet"), (identity_key_path, "identity"), @@ -86,6 +96,17 @@ impl<'a> Kubernetes<'a> { k8s_helpers::create_secret_from_files(secret_name, &key_files) } + fn add_known_validator(&mut self, pubkey: Pubkey) { + if let Some(ref mut known_validators) = self.validator_config.known_validators { + known_validators.push(pubkey); + } else { + let new_known_validators = vec![pubkey]; + self.validator_config.known_validators = Some(new_known_validators); + } + + info!("pubkey added to known validators: {:?}", pubkey); + } + pub async fn deploy_secret(&self, secret: &Secret) -> Result { let secrets_api: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); @@ -94,12 +115,11 @@ impl<'a> Kubernetes<'a> { pub fn create_bootstrap_validator_replica_set( &mut self, - container_name: &str, - image_name: &str, + image_name: &DockerImage, secret_name: Option, label_selector: &BTreeMap, ) -> Result> { - let mut env_vars = vec![EnvVar { + let env_vars = vec![EnvVar { name: "MY_POD_IP".to_string(), value_from: Some(EnvVarSource { field_ref: Some(ObjectFieldSelector { @@ -111,7 +131,6 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]; - let accounts_volume = Some(vec![Volume { name: "bootstrap-accounts-volume".into(), secret: Some(SecretVolumeSource { @@ -131,15 +150,10 @@ impl<'a> Kubernetes<'a> { vec!["/home/solana/k8s-cluster-scripts/bootstrap-startup-script.sh".to_string()]; command.extend(self.generate_bootstrap_command_flags()); - for c in command.iter() { - debug!("bootstrap command: {}", c); - } - k8s_helpers::create_replica_set( &ValidatorType::Bootstrap, self.namespace.as_str(), label_selector, - container_name, image_name, env_vars, &command, @@ -150,6 +164,40 @@ impl<'a> Kubernetes<'a> { ) } + fn generate_command_flags(&self, flags: &mut Vec) { + if self.validator_config.tpu_enable_udp { + flags.push("--tpu-enable-udp".to_string()); + } + if self.validator_config.tpu_disable_quic { + flags.push("--tpu-disable-quic".to_string()); + } + if self.validator_config.skip_poh_verify { + flags.push("--skip-poh-verify".to_string()); + } + if self.validator_config.no_snapshot_fetch { + flags.push("--no-snapshot-fetch".to_string()); + } + if self.validator_config.require_tower { + flags.push("--require-tower".to_string()); + } + if self.validator_config.enable_full_rpc { + flags.push("--enable-rpc-transaction-history".to_string()); + flags.push("--enable-extended-tx-metadata-storage".to_string()); + } + + if let Some(limit_ledger_size) = self.validator_config.max_ledger_size { + flags.push("--limit-ledger-size".to_string()); + flags.push(limit_ledger_size.to_string()); + } + } + + fn generate_bootstrap_command_flags(&self) -> Vec { + let mut flags: Vec = Vec::new(); + self.generate_command_flags(&mut flags); + + flags + } + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } diff --git a/src/main.rs b/src/main.rs index 18ebb39..efd24b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,19 @@ use { - clap::{command, Arg, ArgGroup}, + clap::{command, value_t_or_exit, Arg, ArgGroup}, log::*, - solana_ledger::blockstore_cleanup_service::{DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS}, + solana_ledger::blockstore_cleanup_service::{ + DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS, + }, solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, std::fs, strum::VariantNames, validator_lab::{ docker::{DockerConfig, DockerImage}, - genesis::{Genesis, GenesisFlags, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL}, + genesis::{Genesis, GenesisFlags}, kubernetes::{Kubernetes, PodRequests}, release::{BuildConfig, BuildType, DeployMethod}, - SolanaRoot, ValidatorType, validator_config::ValidatorConfig, + SolanaRoot, ValidatorType, }, }; @@ -198,7 +200,7 @@ fn parse_matches() -> clap::ArgMatches { Off by default since validator won't restart if the pod restarts"), ) .arg( - Arg::with_name("full_rpc") + Arg::with_name("enable_full_rpc") .long("full-rpc") .help("Validator config. Support full RPC services on all nodes"), ) @@ -334,8 +336,6 @@ async fn main() { let mut validator_config = ValidatorConfig { tpu_enable_udp: matches.is_present("tpu_enable_udp"), tpu_disable_quic: matches.is_present("tpu_disable_quic"), - shred_version: None, // set after genesis created - bank_hash: None, // set after genesis created max_ledger_size: if matches.is_present("limit_ledger_size") { let limit_ledger_size = match matches.value_of("limit_ledger_size") { Some(_) => value_t_or_exit!(matches, "limit_ledger_size", u64), @@ -355,7 +355,6 @@ async fn main() { no_snapshot_fetch: matches.is_present("no_snapshot_fetch"), require_tower: matches.is_present("require_tower"), enable_full_rpc: matches.is_present("enable_full_rpc"), - entrypoints: Vec::new(), known_validators: None, }; @@ -364,7 +363,12 @@ async fn main() { matches.value_of("memory_requests").unwrap().to_string(), ); - let kub_controller = Kubernetes::new(environment_config.namespace, &mut validator_config, pod_requests).await; + let mut kub_controller = Kubernetes::new( + environment_config.namespace, + &mut validator_config, + pod_requests, + ) + .await; match kub_controller.namespace_exists().await { Ok(true) => (), Ok(false) => { @@ -426,7 +430,7 @@ async fn main() { ); let validator_type = ValidatorType::Bootstrap; - let docker_image = DockerImage::new( + let bootstrap_docker_image = DockerImage::new( matches.value_of("registry_name").unwrap().to_string(), validator_type, matches.value_of("image_name").unwrap().to_string(), @@ -437,18 +441,21 @@ async fn main() { ); if build_config.docker_build() { - match docker.build_image(solana_root.get_root_path(), &docker_image) { - Ok(_) => info!("{} image built successfully", docker_image.validator_type()), + match docker.build_image(solana_root.get_root_path(), &bootstrap_docker_image) { + Ok(_) => info!( + "{} image built successfully", + bootstrap_docker_image.validator_type() + ), Err(err) => { error!("Exiting........ {}", err); return; } } - match DockerConfig::push_image(&docker_image) { + match DockerConfig::push_image(&bootstrap_docker_image) { Ok(_) => info!( "{} image pushed successfully", - docker_image.validator_type() + bootstrap_docker_image.validator_type() ), Err(err) => { error!("Exiting........ {}", err); @@ -493,9 +500,8 @@ async fn main() { bootstrap_keypair.pubkey().to_string(), ); - let bootstrap_replica_set = match kub_controller.create_bootstrap_validator_replica_set( - bootstrap_container_name, - bootstrap_image_name, + let _bootstrap_replica_set = match kub_controller.create_bootstrap_validator_replica_set( + &bootstrap_docker_image, bootstrap_secret.metadata.name.clone(), &bootstrap_rs_labels, ) { diff --git a/src/validator_config.rs b/src/validator_config.rs index b60776b..35f4a4b 100644 --- a/src/validator_config.rs +++ b/src/validator_config.rs @@ -1,18 +1,13 @@ -use solana_sdk::{ - hash::Hash, pubkey::Pubkey, -}; +use solana_sdk::pubkey::Pubkey; pub struct ValidatorConfig { pub tpu_enable_udp: bool, pub tpu_disable_quic: bool, - pub shred_version: Option, - pub bank_hash: Option, pub max_ledger_size: Option, pub skip_poh_verify: bool, pub no_snapshot_fetch: bool, pub require_tower: bool, pub enable_full_rpc: bool, - pub entrypoints: Vec, pub known_validators: Option>, } @@ -31,26 +26,20 @@ impl std::fmt::Display for ValidatorConfig { "Runtime Config\n\ tpu_enable_udp: {}\n\ tpu_disable_quic: {}\n\ - shred_version: {:?}\n\ - bank_hash: {:?}\n\ max_ledger_size: {:?}\n\ skip_poh_verify: {}\n\ no_snapshot_fetch: {}\n\ require_tower: {}\n\ enable_full_rpc: {}\n\ - entrypoints: {:?}\n\ known_validators: {:?}", self.tpu_enable_udp, self.tpu_disable_quic, - self.shred_version, - self.bank_hash, self.max_ledger_size, self.skip_poh_verify, self.no_snapshot_fetch, self.require_tower, self.enable_full_rpc, - self.entrypoints.join(", "), known_validators, ) } -} \ No newline at end of file +}