From f025ae7096e37363ab80121117ac499b3d3797cb Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 1 Apr 2024 20:52:23 +0000 Subject: [PATCH 1/7] add bootstrap validator selector --- PROGRESS.md | 2 +- src/k8s_helpers.rs | 6 ++++++ src/kubernetes.rs | 6 +++++- src/main.rs | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 5b6a368..8db6e91 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -24,7 +24,7 @@ - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Selector - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 52b64a1..b6d80d4 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -28,3 +28,9 @@ pub fn create_secret_from_files( Ok(create_secret(secret_name, data)) } + +pub fn create_selector(key: &str, value: &str) -> BTreeMap { + let mut btree = BTreeMap::new(); + btree.insert(key.to_string(), value.to_string()); + btree +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 71d92f0..27aecff 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -5,7 +5,7 @@ use { api::{Api, ListParams, PostParams}, Client, }, - std::{error::Error, path::Path}, + std::{collections::BTreeMap, error::Error, path::Path}, }; pub struct Kubernetes { @@ -58,4 +58,8 @@ impl Kubernetes { Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); secrets_api.create(&PostParams::default(), secret).await } + + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { + k8s_helpers::create_selector(key, value) + } } diff --git a/src/main.rs b/src/main.rs index 4614426..5f6694c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use { clap::{command, Arg, ArgGroup}, log::*, + solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, std::fs, strum::VariantNames, validator_lab::{ @@ -376,4 +377,22 @@ async fn main() { return; } } + + // Bootstrap needs two labels. Because it is going to have two services. + // One via Load Balancer, one direct + let mut bootstrap_rs_labels = + kub_controller.create_selector("validator/lb", "load-balancer-selector"); + bootstrap_rs_labels.insert( + "validator/name".to_string(), + "bootstrap-validator-selector".to_string(), + ); + bootstrap_rs_labels.insert("validator/type".to_string(), "bootstrap".to_string()); + + let identity_path = config_directory.join("bootstrap-validator/identity.json"); + let bootstrap_keypair = + read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); + bootstrap_rs_labels.insert( + "validator/identity".to_string(), + bootstrap_keypair.pubkey().to_string(), + ); } From 9596709051701957dd40a693dda3a322cdaf4746 Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 18 Apr 2024 22:40:49 +0000 Subject: [PATCH 2/7] address chido: hard code startup scripts --- src/docker.rs | 45 +++-- src/k8s_helpers.rs | 2 +- src/lib.rs | 3 +- src/main.rs | 29 ++-- src/scripts/common.sh | 123 -------------- ...p-startup-script.sh => startup_scripts.rs} | 155 +++++++++++++++++- 6 files changed, 191 insertions(+), 166 deletions(-) delete mode 100644 src/scripts/common.sh rename src/{scripts/bootstrap-startup-script.sh => startup_scripts.rs} (56%) diff --git a/src/docker.rs b/src/docker.rs index 3ab5e1a..59a9285 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -1,5 +1,8 @@ use { - crate::{new_spinner_progress_bar, release::DeployMethod, ValidatorType, BUILD, ROCKET}, + crate::{ + new_spinner_progress_bar, release::DeployMethod, startup_scripts::StartupScripts, + ValidatorType, BUILD, ROCKET, + }, log::*, std::{ error::Error, @@ -65,7 +68,6 @@ impl DockerConfig { pub fn build_image( &self, solana_root_path: &Path, - lab_path: &Path, docker_image: &DockerImage, ) -> Result<(), Box> { let validator_type = docker_image.validator_type(); @@ -82,7 +84,6 @@ impl DockerConfig { let docker_path = solana_root_path.join(format!("docker-build/{validator_type}")); self.create_base_image( solana_root_path, - lab_path, docker_image, &docker_path, &validator_type, @@ -94,12 +95,11 @@ impl DockerConfig { fn create_base_image( &self, solana_root_path: &Path, - lab_path: &Path, docker_image: &DockerImage, docker_path: &PathBuf, validator_type: &ValidatorType, ) -> Result<(), Box> { - self.create_dockerfile(validator_type, docker_path, lab_path, None)?; + self.create_dockerfile(validator_type, docker_path, None)?; // We use std::process::Command here because Docker-rs is very slow building dockerfiles // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. @@ -137,22 +137,18 @@ impl DockerConfig { Ok(()) } - fn copy_file_to_docker( - source_dir: &Path, - docker_dir: &Path, + fn write_startup_script_to_docker_directory( file_name: &str, + docker_dir: &Path, ) -> std::io::Result<()> { - let source_path = source_dir.join("src/scripts").join(file_name); - let destination_path = docker_dir.join(file_name); - fs::copy(source_path, destination_path)?; - Ok(()) + let script_path = docker_dir.join(file_name); + StartupScripts::write_script_to_file(StartupScripts::bootstrap(), &script_path) } fn create_dockerfile( &self, validator_type: &ValidatorType, docker_path: &PathBuf, - lab_path: &Path, content: Option<&str>, ) -> Result<(), Box> { if docker_path.exists() { @@ -160,21 +156,19 @@ impl DockerConfig { } fs::create_dir_all(docker_path)?; - if let DeployMethod::Local(_) = self.deploy_method { - if validator_type == &ValidatorType::Bootstrap { - let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"]; - for file_name in files_to_copy.iter() { - Self::copy_file_to_docker(lab_path, docker_path, file_name)?; - } + if validator_type == &ValidatorType::Bootstrap { + let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"]; + for file_name in files_to_copy.iter() { + Self::write_startup_script_to_docker_directory(file_name, docker_path)?; } } - let (solana_build_directory, startup_script_directory) = - if let DeployMethod::ReleaseChannel(_) = self.deploy_method { - ("solana-release", "./src/scripts".to_string()) - } else { - ("farf", format!("./docker-build/{validator_type}")) - }; + let startup_script_directory = format!("./docker-build/{validator_type}"); + let solana_build_directory = if let DeployMethod::ReleaseChannel(_) = self.deploy_method { + "solana-release" + } else { + "farf" + }; let dockerfile = format!( r#" @@ -186,6 +180,7 @@ RUN apt-get update && apt-get install -y iputils-ping curl vim && \ USER solana COPY --chown=solana:solana {startup_script_directory} /home/solana/k8s-cluster-scripts +RUN chmod +x /home/solana/k8s-cluster-scripts/* COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger COPY --chown=solana:solana ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/ COPY --chown=solana:solana ./{solana_build_directory}/version.yml /home/solana/ diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index b6d80d4..9a0d8d3 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -23,7 +23,7 @@ pub fn create_secret_from_files( for (file_path, key_type) in key_files { let file_content = std::fs::read(file_path) .map_err(|err| format!("Failed to read file '{:?}': {}", file_path, err))?; - data.insert(format!("{}.json", key_type), ByteString(file_content)); + data.insert(format!("{key_type}.json"), ByteString(file_content)); } Ok(create_secret(secret_name, data)) diff --git a/src/lib.rs b/src/lib.rs index 8a94ad7..7da8288 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,7 @@ const UPGRADEABLE_LOADER: &str = "BPFLoaderUpgradeab1e11111111111111111111111"; #[derive(Clone, Debug)] pub struct EnvironmentConfig<'a> { pub namespace: &'a str, - pub lab_path: PathBuf, // path to the validator-lab directory + pub build_directory: Option, // path to the validator-lab directory } pub struct SolanaRoot { @@ -54,6 +54,7 @@ pub mod genesis; pub mod k8s_helpers; pub mod kubernetes; pub mod release; +pub mod startup_scripts; static BUILD: Emoji = Emoji("👷 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); diff --git a/src/main.rs b/src/main.rs index 5f6694c..cb59a11 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,7 @@ use { clap::{command, Arg, ArgGroup}, log::*, solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, - std::fs, + std::{fs, path::PathBuf}, strum::VariantNames, validator_lab::{ docker::{DockerConfig, DockerImage}, @@ -44,6 +44,7 @@ fn parse_matches() -> clap::ArgMatches { Arg::with_name("release_channel") .long("release-channel") .takes_value(true) + .requires("build_directory") .help("Pulls specific release version. e.g. v1.17.2"), ) .group( @@ -52,12 +53,12 @@ fn parse_matches() -> clap::ArgMatches { .required(true), ) .arg( - Arg::with_name("validator_lab_directory") - .long("validator-lab-dir") + Arg::with_name("build_directory") + .long("build-dir") .takes_value(true) - .required(true) - .help("Absolute path to validator lab directory. - e.g. /home/sol/validator-lab"), + .conflicts_with("local_path") + .help("Absolute path to build directory for release-channel + e.g. /home/sol/validator-lab-build"), ) // Genesis Config .arg( @@ -169,7 +170,7 @@ async fn main() { let matches = parse_matches(); let environment_config = EnvironmentConfig { namespace: matches.value_of("cluster_namespace").unwrap_or_default(), - lab_path: matches.value_of("validator_lab_directory").unwrap().into(), + build_directory: matches.value_of("build_directory").map(PathBuf::from), }; let deploy_method = if let Some(local_path) = matches.value_of("local_path") { @@ -187,7 +188,9 @@ async fn main() { (root, path) } DeployMethod::ReleaseChannel(_) => { - let root = SolanaRoot::new_from_path(environment_config.lab_path.clone()); + // unwrap safe since required if release-channel used + let root = + SolanaRoot::new_from_path(environment_config.build_directory.unwrap().clone()); let path = root.get_root_path().join("solana-release/bin"); (root, path) } @@ -336,11 +339,7 @@ async fn main() { ); if build_config.docker_build() { - match docker.build_image( - solana_root.get_root_path(), - &environment_config.lab_path, - &docker_image, - ) { + match docker.build_image(solana_root.get_root_path(), &docker_image) { Ok(_) => info!("{} image built successfully", docker_image.validator_type()), Err(err) => { error!("Exiting........ {err}"); @@ -365,7 +364,7 @@ async fn main() { { Ok(secret) => secret, Err(err) => { - error!("Failed to create bootstrap secret! {}", err); + error!("Failed to create bootstrap secret! {err}"); return; } }; @@ -373,7 +372,7 @@ async fn main() { match kub_controller.deploy_secret(&bootstrap_secret).await { Ok(_) => info!("Deployed Bootstrap Secret"), Err(err) => { - error!("{}", err); + error!("{err}"); return; } } diff --git a/src/scripts/common.sh b/src/scripts/common.sh deleted file mode 100644 index dd7a32d..0000000 --- a/src/scripts/common.sh +++ /dev/null @@ -1,123 +0,0 @@ -# |source| this file -# -# Common utilities shared by other scripts in this directory -# -# The following directive disable complaints about unused variables in this -# file: -# shellcheck disable=2034 - -prebuild= -if [[ $1 = "--prebuild" ]]; then - prebuild=true -fi - -if [[ $(uname) != Linux ]]; then - # Protect against unsupported configurations to prevent non-obvious errors - # later. Arguably these should be fatal errors but for now prefer tolerance. - if [[ -n $SOLANA_CUDA ]]; then - echo "Warning: CUDA is not supported on $(uname)" - SOLANA_CUDA= - fi -fi - -if [[ -n $USE_INSTALL || ! -f "$SOLANA_ROOT"/Cargo.toml ]]; then - # echo "define if solana program" - solana_program() { - # echo "call if solana program" - declare program="$1" - if [[ -z $program ]]; then - printf "solana" - else - printf "solana-%s" "$program" - fi - } -else - echo "define else solana program" - solana_program() { - echo "call if solana program" - declare program="$1" - declare crate="$program" - if [[ -z $program ]]; then - crate="cli" - program="solana" - else - program="solana-$program" - fi - - if [[ -n $NDEBUG ]]; then - maybe_release=--release - fi - - # Prebuild binaries so that CI sanity check timeout doesn't include build time - if [[ $prebuild ]]; then - ( - set -x - # shellcheck disable=SC2086 # Don't want to double quote - cargo $CARGO_TOOLCHAIN build $maybe_release --bin $program - ) - fi - - printf "cargo $CARGO_TOOLCHAIN run $maybe_release --bin %s %s -- " "$program" - } -fi - -solana_bench_tps=$(solana_program bench-tps) -solana_faucet=$(solana_program faucet) -solana_validator=$(solana_program validator) -solana_validator_cuda="$solana_validator --cuda" -solana_genesis=$(solana_program genesis) -solana_gossip=$(solana_program gossip) -solana_keygen=$(solana_program keygen) -solana_ledger_tool=$(solana_program ledger-tool) -solana_cli=$(solana_program) - -export RUST_BACKTRACE=1 - -# https://gist.github.com/cdown/1163649 -urlencode() { - declare s="$1" - declare l=$((${#s} - 1)) - for i in $(seq 0 $l); do - declare c="${s:$i:1}" - case $c in - [a-zA-Z0-9.~_-]) - echo -n "$c" - ;; - *) - printf '%%%02X' "'$c" - ;; - esac - done -} - -default_arg() { - declare name=$1 - declare value=$2 - - for arg in "${args[@]}"; do - if [[ $arg = "$name" ]]; then - return - fi - done - - if [[ -n $value ]]; then - args+=("$name" "$value") - else - args+=("$name") - fi -} - -replace_arg() { - declare name=$1 - declare value=$2 - - default_arg "$name" "$value" - - declare index=0 - for arg in "${args[@]}"; do - index=$((index + 1)) - if [[ $arg = "$name" ]]; then - args[$index]="$value" - fi - done -} diff --git a/src/scripts/bootstrap-startup-script.sh b/src/startup_scripts.rs similarity index 56% rename from src/scripts/bootstrap-startup-script.sh rename to src/startup_scripts.rs index 56e0a7e..b00b1bb 100644 --- a/src/scripts/bootstrap-startup-script.sh +++ b/src/startup_scripts.rs @@ -1,3 +1,25 @@ +use std::{ + fs::{self, File}, + io::{Result, Write}, + path::Path, +}; + +pub struct StartupScripts; + +impl StartupScripts { + pub fn write_script_to_file(script: &str, script_path: &Path) -> Result<()> { + if script_path.exists() { + // If script already exists, delete it. + // prevents versioning issues. + fs::remove_file(script_path)?; + } + let mut file = File::create(script_path)?; + file.write_all(script.as_bytes())?; + Ok(()) + } + + pub fn bootstrap() -> &'static str { + r#" #!/bin/bash set -e @@ -174,7 +196,7 @@ while true; do while true; do if [[ -z $pid ]] || ! kill -0 "$pid"; then - echo "############## validator exited, restarting ##############" + echo "\############## validator exited, restarting ##############" break fi sleep 1 @@ -182,3 +204,134 @@ while true; do kill_node done + "# + } + + pub fn common() -> &'static str { + r#" +# |source| this file +# +# Common utilities shared by other scripts in this directory +# +# The following directive disable complaints about unused variables in this +# file: +# shellcheck disable=2034 + +prebuild= +if [[ $1 = "--prebuild" ]]; then + prebuild=true +fi + +if [[ $(uname) != Linux ]]; then + # Protect against unsupported configurations to prevent non-obvious errors + # later. Arguably these should be fatal errors but for now prefer tolerance. + if [[ -n $SOLANA_CUDA ]]; then + echo "Warning: CUDA is not supported on $(uname)" + SOLANA_CUDA= + fi +fi + +if [[ -n $USE_INSTALL || ! -f "$SOLANA_ROOT"/Cargo.toml ]]; then + # echo "define if solana program" + solana_program() { + # echo "call if solana program" + declare program="$1" + if [[ -z $program ]]; then + printf "solana" + else + printf "solana-%s" "$program" + fi + } +else + echo "define else solana program" + solana_program() { + echo "call if solana program" + declare program="$1" + declare crate="$program" + if [[ -z $program ]]; then + crate="cli" + program="solana" + else + program="solana-$program" + fi + + if [[ -n $NDEBUG ]]; then + maybe_release=--release + fi + + # Prebuild binaries so that CI sanity check timeout doesn't include build time + if [[ $prebuild ]]; then + ( + set -x + # shellcheck disable=SC2086 # Don't want to double quote + cargo $CARGO_TOOLCHAIN build $maybe_release --bin $program + ) + fi + + printf "cargo $CARGO_TOOLCHAIN run $maybe_release --bin %s %s -- " "$program" + } +fi + +solana_bench_tps=$(solana_program bench-tps) +solana_faucet=$(solana_program faucet) +solana_validator=$(solana_program validator) +solana_validator_cuda="$solana_validator --cuda" +solana_genesis=$(solana_program genesis) +solana_gossip=$(solana_program gossip) +solana_keygen=$(solana_program keygen) +solana_ledger_tool=$(solana_program ledger-tool) +solana_cli=$(solana_program) + +export RUST_BACKTRACE=1 + +# https://gist.github.com/cdown/1163649 +urlencode() { + declare s="$1" + declare l=$((${#s} - 1)) + for i in $(seq 0 $l); do + declare c="${s:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) + echo -n "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} + +default_arg() { + declare name=$1 + declare value=$2 + + for arg in "${args[@]}"; do + if [[ $arg = "$name" ]]; then + return + fi + done + + if [[ -n $value ]]; then + args+=("$name" "$value") + else + args+=("$name") + fi +} + +replace_arg() { + declare name=$1 + declare value=$2 + + default_arg "$name" "$value" + + declare index=0 + for arg in "${args[@]}"; do + index=$((index + 1)) + if [[ $arg = "$name" ]]; then + args[$index]="$value" + fi + done +} + "# + } +} From 8d5f9a30c419aac56254f03052dd033d4c576455 Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 19 Apr 2024 17:51:17 +0000 Subject: [PATCH 3/7] add library and validator structs --- Cargo.lock | 1 + Cargo.toml | 1 + src/docker.rs | 34 +++++++++++++------ src/lib.rs | 31 +++++++++++++++++ src/library.rs | 44 ++++++++++++++++++++++++ src/main.rs | 87 +++++++++++++++++++++++++++++------------------- src/validator.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 236 insertions(+), 46 deletions(-) create mode 100644 src/library.rs create mode 100644 src/validator.rs diff --git a/Cargo.lock b/Cargo.lock index ce16dc9..fd13d4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7132,6 +7132,7 @@ dependencies = [ "kube", "log", "rand 0.8.5", + "rayon", "reqwest", "rustc_version", "rustls", diff --git a/Cargo.toml b/Cargo.toml index c98d591..44c552a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ k8s-openapi ={ version = "0.20.0", features = ["v1_28"] } kube = "0.87.2" log = "0.4.21" rand = "0.8.5" +rayon = "1.9.0" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.10", default-features = false, features = ["quic"] } solana-core = "1.18.8" diff --git a/src/docker.rs b/src/docker.rs index 59a9285..d429568 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -1,9 +1,10 @@ use { crate::{ new_spinner_progress_bar, release::DeployMethod, startup_scripts::StartupScripts, - ValidatorType, BUILD, ROCKET, + validator::Validator, DockerPushThreadError, ValidatorType, BUILD, ROCKET, }, log::*, + rayon::prelude::*, std::{ error::Error, fmt::{self, Display, Formatter}, @@ -13,6 +14,7 @@ use { }, }; +#[derive(Clone)] pub struct DockerImage { registry: String, validator_type: ValidatorType, @@ -199,14 +201,13 @@ WORKDIR /home/solana Ok(()) } - pub fn push_image(docker_image: &DockerImage) -> Result<(), Box> { + pub fn push_image(docker_image: &DockerImage) -> Result<(), Box> { let progress_bar = new_spinner_progress_bar(); progress_bar.set_message(format!( - "{ROCKET}Pushing {} image to registry...", - docker_image.validator_type() + "{ROCKET}Pushing {docker_image} image to registry...", )); - let command = format!("docker push '{}'", docker_image); - let output = match Command::new("sh") + let command = format!("docker push '{docker_image}'"); + let output = Command::new("sh") .arg("-c") .arg(&command) .stdout(Stdio::null()) @@ -214,15 +215,26 @@ WORKDIR /home/solana .spawn() .expect("Failed to execute command") .wait_with_output() - { - Ok(res) => Ok(res), - Err(err) => Err(Box::new(err) as Box), - }?; + .expect("Failed to push image"); if !output.status.success() { - return Err(output.status.to_string().into()); + return Err(Box::new(DockerPushThreadError::from( + output.status.to_string(), + ))); } progress_bar.finish_and_clear(); Ok(()) } + + pub fn push_images<'a, I>(&self, validators: I) -> Result<(), Box> + where + I: IntoIterator, + { + info!("Pushing images..."); + validators + .into_iter() + .collect::>() // Collect into Vec and thread push + .par_iter() + .try_for_each(|validator| Self::push_image(validator.image())) + } } diff --git a/src/lib.rs b/src/lib.rs index 7da8288..1c9293d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,8 @@ use { log::*, reqwest::Client, std::{ + error::Error, + fmt::{self, Display, Formatter}, fs::File, io::{BufReader, Cursor, Read, Write}, path::{Path, PathBuf}, @@ -49,12 +51,41 @@ pub enum ValidatorType { Client, } +#[derive(Debug)] +struct DockerPushThreadError { + message: String, +} + +impl Display for DockerPushThreadError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message) + } +} + +impl Error for DockerPushThreadError {} + +impl From for DockerPushThreadError { + fn from(message: String) -> Self { + DockerPushThreadError { message } + } +} + +impl From<&str> for DockerPushThreadError { + fn from(message: &str) -> Self { + DockerPushThreadError { + message: message.to_string(), + } + } +} + pub mod docker; pub mod genesis; pub mod k8s_helpers; pub mod kubernetes; +pub mod library; pub mod release; pub mod startup_scripts; +pub mod validator; static BUILD: Emoji = Emoji("👷 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); diff --git a/src/library.rs b/src/library.rs new file mode 100644 index 0000000..3b66931 --- /dev/null +++ b/src/library.rs @@ -0,0 +1,44 @@ +use { + crate::{validator::Validator, ValidatorType}, + std::{error::Error, result::Result}, +}; + +// Holds all validators. +// 1) Bootstrap +// 2) Standard Validator -> One image for each validator (not implemented yet) +// 3) RPC Node -> One image for each RPC node (not implemented yet) +// 4) Clients -> Each client has its own image (not implemented yet) + +pub struct Library { + validators: Vec>, + _clients: Option>, +} + +impl Default for Library { + fn default() -> Self { + Self { + validators: vec![None; 1], + _clients: None, + } + } +} + +impl Library { + pub fn set_item(&mut self, item: Validator, validator_type: ValidatorType) { + match validator_type { + ValidatorType::Bootstrap => self.validators[0] = Some(item), + _ => panic!("{validator_type} not implemented yet!"), + } + } + + pub fn bootstrap(&mut self) -> Result<&mut Validator, Box> { + self.validators + .get_mut(0) + .and_then(Option::as_mut) + .ok_or_else(|| "No Bootstrap validator found.".to_string().into()) + } + + pub fn get_validators(&self) -> impl Iterator { + self.validators.iter().filter_map(Option::as_ref) + } +} diff --git a/src/main.rs b/src/main.rs index cb59a11..35a2dc3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,9 @@ use { DEFAULT_FAUCET_LAMPORTS, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, }, kubernetes::Kubernetes, + library::Library, release::{BuildConfig, BuildType, DeployMethod}, + validator::{LabelType, Validator}, EnvironmentConfig, SolanaRoot, ValidatorType, }, }; @@ -330,46 +332,56 @@ async fn main() { deploy_method, ); - let validator_type = ValidatorType::Bootstrap; - let docker_image = DockerImage::new( - matches.value_of("registry_name").unwrap().to_string(), - validator_type, - matches.value_of("image_name").unwrap().to_string(), - matches.value_of("image_tag").unwrap().to_string(), - ); + let registry_name = matches.value_of("registry_name").unwrap().to_string(); + let image_name = matches.value_of("image_name").unwrap().to_string(); + let image_tag = matches + .value_of("image_tag") + .unwrap_or_default() + .to_string(); + + let mut validator_library = Library::default(); + + let bootstrap_validator = Validator::new(DockerImage::new( + registry_name.clone(), + ValidatorType::Bootstrap, + image_name.clone(), + image_tag.clone(), + )); + validator_library.set_item(bootstrap_validator, ValidatorType::Bootstrap); if build_config.docker_build() { - match docker.build_image(solana_root.get_root_path(), &docker_image) { - Ok(_) => info!("{} image built successfully", docker_image.validator_type()), - Err(err) => { - error!("Exiting........ {err}"); - return; + for v in validator_library.get_validators() { + match docker.build_image(solana_root.get_root_path(), v.image()) { + Ok(_) => info!("{} image built successfully", v.validator_type()), + Err(err) => { + error!("Failed to build docker image {err}"); + return; + } } } - match DockerConfig::push_image(&docker_image) { - Ok(_) => info!( - "{} image pushed successfully", - docker_image.validator_type() - ), + match docker.push_images(validator_library.get_validators()) { + Ok(_) => info!("Validator images pushed successfully"), Err(err) => { - error!("Error. Failed to build imge: {err}"); + error!("Failed to push Validator docker image {err}"); return; } } } - let bootstrap_secret = match kub_controller - .create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) - { - Ok(secret) => secret, + let bootstrap_validator = validator_library.bootstrap().expect("should be bootstrap"); + match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) { + Ok(secret) => bootstrap_validator.set_secret(secret), Err(err) => { error!("Failed to create bootstrap secret! {err}"); return; } }; - match kub_controller.deploy_secret(&bootstrap_secret).await { + match kub_controller + .deploy_secret(bootstrap_validator.secret()) + .await + { Ok(_) => info!("Deployed Bootstrap Secret"), Err(err) => { error!("{err}"); @@ -377,21 +389,26 @@ async fn main() { } } - // Bootstrap needs two labels. Because it is going to have two services. - // One via Load Balancer, one direct - let mut bootstrap_rs_labels = - kub_controller.create_selector("validator/lb", "load-balancer-selector"); - bootstrap_rs_labels.insert( - "validator/name".to_string(), - "bootstrap-validator-selector".to_string(), - ); - bootstrap_rs_labels.insert("validator/type".to_string(), "bootstrap".to_string()); - + // Create bootstrap labels let identity_path = config_directory.join("bootstrap-validator/identity.json"); let bootstrap_keypair = read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); - bootstrap_rs_labels.insert( - "validator/identity".to_string(), + // Bootstrap needs two labels. It will have two services. + // One for Load Balancer, one direct + bootstrap_validator.add_label( + "load-balancer/name", + "load-balancer-selector", + LabelType::Service, + ); + bootstrap_validator.add_label( + "service/name", + "bootstrap-validator-selector", + LabelType::Service, + ); + bootstrap_validator.add_label("validator/type", "bootstrap", LabelType::Info); + bootstrap_validator.add_label( + "validator/identity", bootstrap_keypair.pubkey().to_string(), + LabelType::Info, ); } diff --git a/src/validator.rs b/src/validator.rs new file mode 100644 index 0000000..82f5640 --- /dev/null +++ b/src/validator.rs @@ -0,0 +1,84 @@ +use { + crate::{docker::DockerImage, ValidatorType}, + k8s_openapi::api::{apps::v1::ReplicaSet, core::v1::Secret}, + std::{collections::BTreeMap, string::String}, +}; + +pub enum LabelType { + Info, + Service, +} + +#[derive(Clone)] +pub struct Validator { + validator_type: ValidatorType, + image: DockerImage, + secret: Secret, + info_labels: BTreeMap, + replica_set: ReplicaSet, + service_labels: BTreeMap, +} + +impl Validator { + pub fn new(image: DockerImage) -> Self { + Self { + validator_type: image.validator_type(), + image, + secret: Secret::default(), + info_labels: BTreeMap::new(), + replica_set: ReplicaSet::default(), + service_labels: BTreeMap::new(), + } + } + + pub fn image(&self) -> &DockerImage { + &self.image + } + + pub fn secret(&self) -> &Secret { + &self.secret + } + + pub fn validator_type(&self) -> &ValidatorType { + &self.validator_type + } + + pub fn add_label(&mut self, key: K, value: V, label_type: LabelType) + where + K: Into, + V: Into, + { + match label_type { + LabelType::Info => { + self.info_labels.insert(key.into(), value.into()); + } + LabelType::Service => { + self.service_labels.insert(key.into(), value.into()); + } + } + } + + pub fn replica_set_labels(&self) -> &BTreeMap { + &self.info_labels + } + + pub fn service_labels(&self) -> &BTreeMap { + &self.service_labels + } + + pub fn set_secret(&mut self, secret: Secret) { + self.secret = secret; + } + + pub fn set_replica_set(&mut self, replica_set: ReplicaSet) { + self.replica_set = replica_set; + } + + pub fn replica_set(&self) -> &ReplicaSet { + &self.replica_set + } + + pub fn replica_set_name(&self) -> &String { + self.replica_set.metadata.name.as_ref().unwrap() + } +} From d5a071abb1f51d795cb0b4a0cc9e102b3578394f Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 19 Apr 2024 19:05:55 +0000 Subject: [PATCH 4/7] upgrade rustls --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd13d4b..42088f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4130,9 +4130,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" dependencies = [ "log", "ring 0.17.8", diff --git a/Cargo.toml b/Cargo.toml index 44c552a..5484393 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ log = "0.4.21" rand = "0.8.5" rayon = "1.9.0" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } -rustls = { version = "0.21.10", default-features = false, features = ["quic"] } +rustls = { version = "0.21.11", default-features = false, features = ["quic"] } solana-core = "1.18.8" solana-logger = "1.18.8" solana-sdk = "1.18.8" From 68ac58f724ba3ba6d105268895e8051ea010548f Mon Sep 17 00:00:00 2001 From: greg Date: Sun, 21 Apr 2024 21:56:17 +0000 Subject: [PATCH 5/7] address Jon comments from PR #7 --- src/docker.rs | 7 ++--- src/lib.rs | 80 ++++++++++++++++++++++++++++++++------------------- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/src/docker.rs b/src/docker.rs index d429568..8469032 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -117,7 +117,7 @@ impl DockerConfig { dockerfile.display() ); - let output = match Command::new("sh") + let output = Command::new("sh") .arg("-c") .arg(&command) .stdout(Stdio::null()) @@ -125,10 +125,7 @@ impl DockerConfig { .spawn() .expect("Failed to execute command") .wait_with_output() - { - Ok(res) => Ok(res), - Err(err) => Err(Box::new(err) as Box), - }?; + .map_err(Box::new)?; if !output.status.success() { return Err(output.status.to_string().into()); diff --git a/src/lib.rs b/src/lib.rs index 1c9293d..f1f1059 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,13 @@ impl SolanaRoot { } } +struct GenesisProgram<'a> { + name: &'a str, + version: &'a str, + address: &'a str, + loader: &'a str, +} + #[derive(Debug, Clone, Copy, PartialEq, Display)] pub enum ValidatorType { #[strum(serialize = "bootstrap-validator")] @@ -172,12 +179,13 @@ async fn fetch_program( ) -> Result<(), Box> { let so_filename = format!("spl_{}-{}.so", name.replace('-', "_"), version); let so_path = solana_root_path.join(&so_filename); + let so_name = format!("spl_{}.so", name.replace('-', "_")); if !so_path.exists() { info!("Downloading {} {}", name, version); let url = format!( "https://github.com/solana-labs/solana-program-library/releases/download/{}-v{}/{}", - name, version, so_filename + name, version, so_name ); download_to_temp(&url, &so_path) @@ -192,44 +200,56 @@ pub async fn fetch_spl(solana_root_path: &Path) -> Result<(), Box Date: Sun, 21 Apr 2024 21:59:48 +0000 Subject: [PATCH 6/7] Jon nit from PR #8. rm build_path --- src/release.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/release.rs b/src/release.rs index a30650e..76ab83a 100644 --- a/src/release.rs +++ b/src/release.rs @@ -30,7 +30,6 @@ pub struct BuildConfig { build_type: BuildType, solana_root_path: PathBuf, docker_build: bool, - _build_path: PathBuf, } impl BuildConfig { @@ -40,17 +39,11 @@ impl BuildConfig { solana_root_path: &Path, docker_build: bool, ) -> Self { - let build_path = match deploy_method { - DeployMethod::Local(_) => solana_root_path.join("farf/bin"), - DeployMethod::ReleaseChannel(_) => solana_root_path.join("solana-release/bin"), - }; - BuildConfig { deploy_method, build_type, solana_root_path: solana_root_path.to_path_buf(), docker_build, - _build_path: build_path, } } From 146a54689d68e5b05e8f394f25f33d365196b779 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 22 Apr 2024 05:20:40 +0000 Subject: [PATCH 7/7] chido. change up secret handling to be more flexible --- src/k8s_helpers.rs | 37 ++++++++++++++++++++++++++----------- src/kubernetes.rs | 41 ++++++++++++++++++++++++++++++++--------- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 9a0d8d3..9041678 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -1,10 +1,19 @@ use { k8s_openapi::{api::core::v1::Secret, ByteString}, kube::api::ObjectMeta, - std::{collections::BTreeMap, error::Error, path::PathBuf}, + std::{ + collections::{BTreeMap, HashMap}, + error::Error, + path::PathBuf, + }, }; -fn create_secret(name: &str, data: BTreeMap) -> Secret { +pub enum SecretType { + Value { v: String }, + File { path: PathBuf }, +} + +fn build_secret(name: &str, data: BTreeMap) -> Secret { Secret { metadata: ObjectMeta { name: Some(name.to_string()), @@ -15,18 +24,24 @@ fn create_secret(name: &str, data: BTreeMap) -> Secret { } } -pub fn create_secret_from_files( +pub fn create_secret( secret_name: &str, - key_files: &[(PathBuf, &str)], //[pathbuf, key type] + secrets: HashMap, ) -> Result> { - let mut data = BTreeMap::new(); - for (file_path, key_type) in key_files { - let file_content = std::fs::read(file_path) - .map_err(|err| format!("Failed to read file '{:?}': {}", file_path, err))?; - data.insert(format!("{key_type}.json"), ByteString(file_content)); + let mut data: BTreeMap = BTreeMap::new(); + for (label, value) in secrets { + match value { + SecretType::Value { v } => { + data.insert(label, ByteString(v.into_bytes())); + } + SecretType::File { path } => { + let file_content = std::fs::read(&path) + .map_err(|err| format!("Failed to read file '{:?}': {}", path, err))?; + data.insert(label, ByteString(file_content)); + } + } } - - Ok(create_secret(secret_name, data)) + Ok(build_secret(secret_name, data)) } pub fn create_selector(key: &str, value: &str) -> BTreeMap { diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 27aecff..f15fcb5 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,11 +1,15 @@ use { - crate::k8s_helpers, + crate::k8s_helpers::{self, SecretType}, k8s_openapi::api::core::v1::{Namespace, Secret}, kube::{ api::{Api, ListParams, PostParams}, Client, }, - std::{collections::BTreeMap, error::Error, path::Path}, + std::{ + collections::{BTreeMap, HashMap}, + error::Error, + path::Path, + }, }; pub struct Kubernetes { @@ -43,14 +47,33 @@ impl Kubernetes { let vote_key_path = config_dir.join("bootstrap-validator/vote-account.json"); let stake_key_path = config_dir.join("bootstrap-validator/stake-account.json"); - let key_files = vec![ - (faucet_key_path, "faucet"), - (identity_key_path, "identity"), - (vote_key_path, "vote"), - (stake_key_path, "stake"), - ]; + let mut secrets = HashMap::new(); + secrets.insert( + "faucet".to_string(), + SecretType::File { + path: faucet_key_path, + }, + ); + secrets.insert( + "identity".to_string(), + SecretType::File { + path: identity_key_path, + }, + ); + secrets.insert( + "vote".to_string(), + SecretType::File { + path: vote_key_path, + }, + ); + secrets.insert( + "stake".to_string(), + SecretType::File { + path: stake_key_path, + }, + ); - k8s_helpers::create_secret_from_files(secret_name, &key_files) + k8s_helpers::create_secret(secret_name, secrets) } pub async fn deploy_secret(&self, secret: &Secret) -> Result {