From b96aa7a2ca9c3d808150c347e92b112bb177fbbc Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 29 Mar 2024 16:10:08 +0000 Subject: [PATCH 01/26] wip. bug in docker build --- PROGRESS.md | 2 +- src/docker.rs | 169 +++++++++++++++++++++ src/lib.rs | 3 +- src/main.rs | 68 +++++++++ src/release.rs | 27 +++- src/scripts/bootstrap-startup-scripts.sh | 184 +++++++++++++++++++++++ src/scripts/common.sh | 123 +++++++++++++++ 7 files changed, 569 insertions(+), 7 deletions(-) create mode 100644 src/docker.rs create mode 100644 src/scripts/bootstrap-startup-scripts.sh create mode 100644 src/scripts/common.sh diff --git a/PROGRESS.md b/PROGRESS.md index 45820d4..5c5701f 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -16,7 +16,7 @@ - [x] Generate faucet and bootstrap accounts - [x] Build genesis - [ ] Docker Build - - [ ] Build Bootstrap Image + - [x] Build Bootstrap Image - [ ] Push Image to registry - [ ] Create & Deploy Secrets - [ ] Bootstrap diff --git a/src/docker.rs b/src/docker.rs new file mode 100644 index 0000000..d132cc5 --- /dev/null +++ b/src/docker.rs @@ -0,0 +1,169 @@ +use { + crate::{ + boxed_error, new_spinner_progress_bar, ValidatorType, BUILD, + }, + log::*, + std::{ + error::Error, + fs, + path::PathBuf, + process::{Command, Output, Stdio}, + }, +}; + +pub struct DockerConfig { + pub base_image: String, + pub image_name: String, + pub tag: String, + pub registry: String, + deploy_method: String, +} + +impl DockerConfig { + pub fn new( + base_image: String, + image_name: String, + tag: String, + registry: String, + deploy_method: String, + ) -> Self { + DockerConfig { + base_image, + image_name, + tag, + registry, + deploy_method, + } + } + + pub fn build_image(&self, solana_root_path: PathBuf, validator_type: &ValidatorType) -> Result<(), Box> { + let image_name = format!("{}-{}", validator_type, self.image_name); + let docker_path = solana_root_path.join(format!("{}/{}", "docker-build", validator_type)); + match self.create_base_image(solana_root_path, image_name, docker_path, validator_type) { + Ok(res) => { + if res.status.success() { + info!("Successfully created base Image"); + Ok(()) + } else { + error!("Failed to build base image"); + Err(boxed_error!(String::from_utf8_lossy(&res.stderr))) + } + } + Err(err) => Err(err), + } + } + + pub fn create_base_image( + &self, + solana_root_path: PathBuf, + image_name: String, + docker_path: PathBuf, + validator_type: &ValidatorType, + ) -> Result> { + let dockerfile_path = self.create_dockerfile(validator_type, docker_path, None)?; + + trace!("Tmp: {}", dockerfile_path.as_path().display()); + trace!("Exists: {}", dockerfile_path.as_path().exists()); + + // We use std::process::Command here because Docker-rs is very slow building dockerfiles + // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. + // so we result to using std::process::Command + let dockerfile = dockerfile_path.join("Dockerfile"); + let context_path = solana_root_path.display().to_string(); + + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!( + "{BUILD}Building {} docker image...", + validator_type + )); + + let command = format!( + "docker build -t {}/{}:{} -f {:?} {}", + self.registry, image_name, self.tag, dockerfile, context_path + ); + info!("command: {}", command); + let output = match Command::new("sh") + .arg("-c") + .arg(&command) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Failed to execute command") + .wait_with_output() + { + Ok(res) => Ok(res), + Err(err) => Err(Box::new(err) as Box), + }; + progress_bar.finish_and_clear(); + info!("{} image build complete", validator_type); + + output + } + + pub fn create_dockerfile( + &self, + validator_type: &ValidatorType, + docker_path: PathBuf, + content: Option<&str>, + ) -> Result> { + match validator_type { + ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => (), + _ => { + return Err(boxed_error!(format!( + "Invalid validator type: {}. Exiting...", + validator_type + ))); + } + } + + if docker_path.exists() { + fs::remove_dir_all(&docker_path)?; + } + fs::create_dir_all(&docker_path)?; + + let solana_build_directory = if self.deploy_method == "tar" { + "solana-release" + } else { + "farf" + }; + + //TODO: I Removed some stuff from this dockerfile. may need to add some stuff back in + let dockerfile = format!( + r#" +FROM {} +RUN apt-get update +RUN apt-get install -y iputils-ping curl vim bzip2 + +RUN useradd -ms /bin/bash solana +RUN adduser solana sudo +USER solana + +RUN mkdir -p /home/solana/k8s-cluster-scripts +COPY ./src/scripts /home/solana/k8s-cluster-scripts + +RUN mkdir -p /home/solana/ledger +COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger + +RUN mkdir -p /home/solana/.cargo/bin + +COPY ./{solana_build_directory}/bin/ /home/solana/.cargo/bin/ +COPY ./{solana_build_directory}/version.yml /home/solana/ + +RUN mkdir -p /home/solana/config +ENV PATH="/home/solana/.cargo/bin:${{PATH}}" + +WORKDIR /home/solana + +"#, + self.base_image + ); + + debug!("dockerfile: {}", dockerfile); + std::fs::write( + docker_path.as_path().join("Dockerfile"), + content.unwrap_or(dockerfile.as_str()), + ) + .expect("saved Dockerfile"); + Ok(docker_path) + } +} diff --git a/src/lib.rs b/src/lib.rs index 67cfe23..6297fa0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,8 +58,9 @@ pub mod genesis; pub mod kubernetes; pub mod release; -static SUN: Emoji = Emoji("🌞 ", ""); +static BUILD: Emoji = Emoji("👷 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); +static SUN: Emoji = Emoji("🌞 ", ""); static TRUCK: Emoji = Emoji("🚚 ", ""); /// Creates a new process bar for processing that will take an unknown amount of time diff --git a/src/main.rs b/src/main.rs index 3319913..11e265e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ use { std::fs, strum::VariantNames, validator_lab::{ + docker::DockerConfig, genesis::{Genesis, GenesisFlags}, kubernetes::Kubernetes, release::{BuildConfig, BuildType, DeployMethod}, @@ -108,6 +109,47 @@ fn parse_matches() -> clap::ArgMatches { .takes_value(true) .help("Genesis config. bootstrap validator stake sol"), ) + //Docker config + .arg( + Arg::with_name("docker_build") + .long("docker-build") + .requires("registry_name") + .requires("image_name") + .requires("base_image") + .requires("image_tag") + .help("Build Docker images. Build new docker images"), + ) + .arg( + Arg::with_name("registry_name") + .long("registry") + .takes_value(true) + .required(true) + .help("Registry to push docker image to"), + ) + .arg( + Arg::with_name("image_name") + .long("image-name") + .takes_value(true) + .default_value("k8s-cluster-image") + .required(true) + .help("Docker image name. Will be prepended with validator_type (bootstrap or validator)"), + ) + .arg( + Arg::with_name("base_image") + .long("base-image") + .takes_value(true) + .default_value("ubuntu:20.04") + .required(true) + .help("Docker base image"), + ) + .arg( + Arg::with_name("image_tag") + .long("tag") + .takes_value(true) + .required(true) + .default_value("latest") + .help("Docker image tag."), + ) .get_matches() } @@ -265,4 +307,30 @@ async fn main() { return; } } + + //unwraps are safe here. since their requirement is enforced by argmatches + let docker = DockerConfig::new( + matches + .value_of("base_image") + .unwrap_or_default() + .to_string(), + matches.value_of("image_name").unwrap().to_string(), + matches + .value_of("image_tag") + .unwrap_or_default() + .to_string(), + matches.value_of("registry_name").unwrap().to_string(), + deploy_method.to_string(), + ); + + if build_config.docker_build() { + let image_type = ValidatorType::Bootstrap; + match docker.build_image(solana_root.get_root_path(), &image_type) { + Ok(_) => info!("{} image built successfully", image_type), + Err(err) => { + error!("Exiting........ {}", err); + return; + } + } + } } diff --git a/src/release.rs b/src/release.rs index d498bff..5316c6b 100644 --- a/src/release.rs +++ b/src/release.rs @@ -29,19 +29,36 @@ pub struct BuildConfig { deploy_method: DeployMethod, build_type: BuildType, solana_root_path: PathBuf, + docker_build: bool, } impl BuildConfig { pub fn new( deploy_method: DeployMethod, build_type: BuildType, - solana_root_path: &Path, - ) -> Self { - Self { + solana_root_path: &PathBuf, + docker_build: bool, + ) -> Result> { + let build_path = match deploy_method { + DeployMethod::Local(_) => solana_root_path.join("farf/bin"), + DeployMethod::ReleaseChannel(_) => solana_root_path.join("solana-release/bin"), + }; + + Ok(BuildConfig { deploy_method, build_type, - solana_root_path: solana_root_path.to_path_buf(), - } + build_path, + solana_root_path: solana_root_path.clone(), + docker_build, + }) + } + + pub fn build_path(&self) -> PathBuf { + self.build_path.clone() + } + + pub fn docker_build(&self) -> bool { + self.docker_build } pub async fn prepare(&self) -> Result<(), Box> { diff --git a/src/scripts/bootstrap-startup-scripts.sh b/src/scripts/bootstrap-startup-scripts.sh new file mode 100644 index 0000000..56e0a7e --- /dev/null +++ b/src/scripts/bootstrap-startup-scripts.sh @@ -0,0 +1,184 @@ +#!/bin/bash +set -e + +# start faucet +nohup solana-faucet --keypair bootstrap-accounts/faucet.json & + +# Start the bootstrap validator node +# shellcheck disable=SC1091 +source /home/solana/k8s-cluster-scripts/common.sh + +program="agave-validator" + +no_restart=0 + +echo "PROGRAM: $program" + +args=() +while [[ -n $1 ]]; do + if [[ ${1:0:1} = - ]]; then + if [[ $1 = --init-complete-file ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 = --gossip-host ]]; then # set with env variables + args+=("$1" "$2") + shift 2 + elif [[ $1 = --gossip-port ]]; then # set with env variables + args+=("$1" "$2") + shift 2 + elif [[ $1 = --dev-halt-at-slot ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --dynamic-port-range ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --limit-ledger-size ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 = --no-rocksdb-compaction ]]; then # not enabled in net.sh + args+=("$1") + shift + elif [[ $1 = --enable-rpc-transaction-history ]]; then # enabled through full-rpc + args+=("$1") + shift + elif [[ $1 = --rpc-pubsub-enable-block-subscription ]]; then # not enabled in net.sh + args+=("$1") + shift + elif [[ $1 = --enable-cpi-and-log-storage ]]; then # not enabled in net.sh + args+=("$1") + shift + elif [[ $1 = --enable-extended-tx-metadata-storage ]]; then # enabled through full-rpc + args+=("$1") + shift + elif [[ $1 = --enable-rpc-bigtable-ledger-storage ]]; then + args+=("$1") + shift + elif [[ $1 = --tpu-disable-quic ]]; then + args+=("$1") + shift + elif [[ $1 = --tpu-enable-udp ]]; then + args+=("$1") + shift + elif [[ $1 = --rpc-send-batch-ms ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --rpc-send-batch-size ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 = --skip-poh-verify ]]; then + args+=("$1") + shift + elif [[ $1 = --no-restart ]]; then # not enabled in net.sh + no_restart=1 + shift + elif [[ $1 == --wait-for-supermajority ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --expected-bank-hash ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --accounts ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --maximum-snapshots-to-retain ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + elif [[ $1 == --no-snapshot-fetch ]]; then + args+=("$1") + shift + elif [[ $1 == --accounts-db-skip-shrink ]]; then + args+=("$1") + shift + elif [[ $1 == --require-tower ]]; then + args+=("$1") + shift + elif [[ $1 = --log-messages-bytes-limit ]]; then # not enabled in net.sh + args+=("$1" "$2") + shift 2 + else + echo "Unknown argument: $1" + $program --help + exit 1 + fi + else + echo "Unknown argument: $1" + $program --help + exit 1 + fi +done + +# These keypairs are created by ./setup.sh and included in the genesis config +identity=bootstrap-accounts/identity.json +vote_account=bootstrap-accounts/vote.json + +ledger_dir=/home/solana/ledger +[[ -d "$ledger_dir" ]] || { + echo "$ledger_dir does not exist" + exit 1 +} + +args+=( + --no-os-network-limits-test \ + --no-wait-for-vote-to-start-leader \ + --snapshot-interval-slots 200 \ + --identity "$identity" \ + --vote-account "$vote_account" \ + --ledger ledger \ + --log - \ + --gossip-host "$MY_POD_IP" \ + --gossip-port 8001 \ + --rpc-port 8899 \ + --rpc-faucet-address "$MY_POD_IP":9900 \ + --no-poh-speed-test \ + --no-incremental-snapshots \ + --full-rpc-api \ + --allow-private-addr \ + --enable-rpc-transaction-history +) + +echo "Bootstrap Args" +for arg in "${args[@]}"; do + echo "$arg" +done + +pid= +kill_node() { + # Note: do not echo anything from this function to ensure $pid is actually + # killed when stdout/stderr are redirected + set +ex + if [[ -n $pid ]]; then + declare _pid=$pid + pid= + kill "$_pid" || true + wait "$_pid" || true + fi +} + +kill_node_and_exit() { + kill_node + exit +} + +trap 'kill_node_and_exit' INT TERM ERR + +while true; do + echo "$program ${args[*]}" + $program "${args[@]}" & + pid=$! + echo "pid: $pid" + + if ((no_restart)); then + wait "$pid" + exit $? + fi + + while true; do + if [[ -z $pid ]] || ! kill -0 "$pid"; then + echo "############## validator exited, restarting ##############" + break + fi + sleep 1 + done + + kill_node +done diff --git a/src/scripts/common.sh b/src/scripts/common.sh new file mode 100644 index 0000000..dd7a32d --- /dev/null +++ b/src/scripts/common.sh @@ -0,0 +1,123 @@ +# |source| this file +# +# Common utilities shared by other scripts in this directory +# +# The following directive disable complaints about unused variables in this +# file: +# shellcheck disable=2034 + +prebuild= +if [[ $1 = "--prebuild" ]]; then + prebuild=true +fi + +if [[ $(uname) != Linux ]]; then + # Protect against unsupported configurations to prevent non-obvious errors + # later. Arguably these should be fatal errors but for now prefer tolerance. + if [[ -n $SOLANA_CUDA ]]; then + echo "Warning: CUDA is not supported on $(uname)" + SOLANA_CUDA= + fi +fi + +if [[ -n $USE_INSTALL || ! -f "$SOLANA_ROOT"/Cargo.toml ]]; then + # echo "define if solana program" + solana_program() { + # echo "call if solana program" + declare program="$1" + if [[ -z $program ]]; then + printf "solana" + else + printf "solana-%s" "$program" + fi + } +else + echo "define else solana program" + solana_program() { + echo "call if solana program" + declare program="$1" + declare crate="$program" + if [[ -z $program ]]; then + crate="cli" + program="solana" + else + program="solana-$program" + fi + + if [[ -n $NDEBUG ]]; then + maybe_release=--release + fi + + # Prebuild binaries so that CI sanity check timeout doesn't include build time + if [[ $prebuild ]]; then + ( + set -x + # shellcheck disable=SC2086 # Don't want to double quote + cargo $CARGO_TOOLCHAIN build $maybe_release --bin $program + ) + fi + + printf "cargo $CARGO_TOOLCHAIN run $maybe_release --bin %s %s -- " "$program" + } +fi + +solana_bench_tps=$(solana_program bench-tps) +solana_faucet=$(solana_program faucet) +solana_validator=$(solana_program validator) +solana_validator_cuda="$solana_validator --cuda" +solana_genesis=$(solana_program genesis) +solana_gossip=$(solana_program gossip) +solana_keygen=$(solana_program keygen) +solana_ledger_tool=$(solana_program ledger-tool) +solana_cli=$(solana_program) + +export RUST_BACKTRACE=1 + +# https://gist.github.com/cdown/1163649 +urlencode() { + declare s="$1" + declare l=$((${#s} - 1)) + for i in $(seq 0 $l); do + declare c="${s:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) + echo -n "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} + +default_arg() { + declare name=$1 + declare value=$2 + + for arg in "${args[@]}"; do + if [[ $arg = "$name" ]]; then + return + fi + done + + if [[ -n $value ]]; then + args+=("$name" "$value") + else + args+=("$name") + fi +} + +replace_arg() { + declare name=$1 + declare value=$2 + + default_arg "$name" "$value" + + declare index=0 + for arg in "${args[@]}"; do + index=$((index + 1)) + if [[ $arg = "$name" ]]; then + args[$index]="$value" + fi + done +} From 08f084ac2af412bdbc377d88aee5673f3703436c Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 29 Mar 2024 20:34:11 +0000 Subject: [PATCH 02/26] build docker bootstrap image --- README.md | 8 +- src/docker.rs | 91 ++++++++++++------- src/genesis.rs | 2 +- src/main.rs | 12 ++- src/release.rs | 2 +- ...scripts.sh => bootstrap-startup-script.sh} | 0 6 files changed, 77 insertions(+), 38 deletions(-) rename src/scripts/{bootstrap-startup-scripts.sh => bootstrap-startup-script.sh} (100%) diff --git a/README.md b/README.md index 2c0d47d..2440a41 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ cargo run --bin cluster -- --release-channel # note: MUST include the "v" ``` -#### Build from Local Repo and Configure Genesis +#### Build from Local Repo and Configure Genesis and Bootstrap Validator Image Example: ``` cargo run --bin cluster -- @@ -51,4 +51,10 @@ cargo run --bin cluster -- --max-genesis-archive-unpacked-size --target-lamports-per-signature --slots-per-epoch + # docker config + --registry # e.g. gregcusack + --docker-build + --tag # e.g. v1 + --base-image # e.g. ubuntu:20.04 + --image-name # e.g. cluster-image ``` \ No newline at end of file diff --git a/src/docker.rs b/src/docker.rs index d132cc5..a1431f8 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -1,12 +1,11 @@ use { - crate::{ - boxed_error, new_spinner_progress_bar, ValidatorType, BUILD, - }, + crate::{new_spinner_progress_bar, release::DeployMethod, ValidatorType, BUILD}, log::*, std::{ + env, error::Error, fs, - path::PathBuf, + path::{Path, PathBuf}, process::{Command, Output, Stdio}, }, }; @@ -16,7 +15,7 @@ pub struct DockerConfig { pub image_name: String, pub tag: String, pub registry: String, - deploy_method: String, + deploy_method: DeployMethod, } impl DockerConfig { @@ -25,7 +24,7 @@ impl DockerConfig { image_name: String, tag: String, registry: String, - deploy_method: String, + deploy_method: DeployMethod, ) -> Self { DockerConfig { base_image, @@ -36,7 +35,21 @@ impl DockerConfig { } } - pub fn build_image(&self, solana_root_path: PathBuf, validator_type: &ValidatorType) -> Result<(), Box> { + pub fn build_image( + &self, + solana_root_path: PathBuf, + validator_type: &ValidatorType, + ) -> Result<(), Box> { + match validator_type { + ValidatorType::Bootstrap => (), + ValidatorType::Standard | ValidatorType::RPC | ValidatorType::Client => { + return Err(format!( + "Build docker image for validator type: {} not supported yet", + validator_type + ) + .into()); + } + } let image_name = format!("{}-{}", validator_type, self.image_name); let docker_path = solana_root_path.join(format!("{}/{}", "docker-build", validator_type)); match self.create_base_image(solana_root_path, image_name, docker_path, validator_type) { @@ -46,16 +59,16 @@ impl DockerConfig { Ok(()) } else { error!("Failed to build base image"); - Err(boxed_error!(String::from_utf8_lossy(&res.stderr))) + Err(String::from_utf8_lossy(&res.stderr).into()) } } Err(err) => Err(err), } } - pub fn create_base_image( + fn create_base_image( &self, - solana_root_path: PathBuf, + solana_root_path: PathBuf, image_name: String, docker_path: PathBuf, validator_type: &ValidatorType, @@ -100,34 +113,46 @@ impl DockerConfig { output } - pub fn create_dockerfile( + fn copy_file_to_docker( + source_dir: &Path, + docker_dir: &Path, + file_name: &str, + ) -> std::io::Result<()> { + let source_path = source_dir.join("src/scripts").join(file_name); + let destination_path = docker_dir.join(file_name); + fs::copy(&source_path, &destination_path)?; + Ok(()) + } + + fn create_dockerfile( &self, validator_type: &ValidatorType, docker_path: PathBuf, content: Option<&str>, - ) -> Result> { - match validator_type { - ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => (), - _ => { - return Err(boxed_error!(format!( - "Invalid validator type: {}. Exiting...", - validator_type - ))); - } - } - + ) -> Result> { if docker_path.exists() { fs::remove_dir_all(&docker_path)?; } fs::create_dir_all(&docker_path)?; - let solana_build_directory = if self.deploy_method == "tar" { - "solana-release" - } else { - "farf" - }; + if let DeployMethod::Local(_) = self.deploy_method { + if validator_type == &ValidatorType::Bootstrap { + let manifest_path = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR")); + let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"]; + for file_name in files_to_copy.iter() { + Self::copy_file_to_docker(&manifest_path, &docker_path, file_name)?; + } + } + } + + let (solana_build_directory, startup_script_directory) = + if let DeployMethod::ReleaseChannel(_) = self.deploy_method { + ("solana-release", "./src/scripts".to_string()) + } else { + ("farf", format!("./docker-build/{}", validator_type)) + }; - //TODO: I Removed some stuff from this dockerfile. may need to add some stuff back in let dockerfile = format!( r#" FROM {} @@ -139,8 +164,9 @@ RUN adduser solana sudo USER solana RUN mkdir -p /home/solana/k8s-cluster-scripts -COPY ./src/scripts /home/solana/k8s-cluster-scripts - +# TODO: this needs to be changed for non bootstrap, this should be ./src/scripts/-startup-scripts.sh +COPY {startup_script_directory}/bootstrap-startup-script.sh /home/solana/k8s-cluster-scripts + RUN mkdir -p /home/solana/ledger COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger @@ -160,10 +186,9 @@ WORKDIR /home/solana debug!("dockerfile: {}", dockerfile); std::fs::write( - docker_path.as_path().join("Dockerfile"), + docker_path.join("Dockerfile"), content.unwrap_or(dockerfile.as_str()), - ) - .expect("saved Dockerfile"); + )?; Ok(docker_path) } } diff --git a/src/genesis.rs b/src/genesis.rs index 129db00..be3325b 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -276,7 +276,7 @@ impl Genesis { let fetch_spl_file = solana_root_path.join("fetch-spl.sh"); fetch_spl(&fetch_spl_file)?; - // add in spl stuff + // add in spl let spl_file = solana_root_path.join("spl-genesis-args.sh"); parse_spl_genesis_file(&spl_file) } diff --git a/src/main.rs b/src/main.rs index 11e265e..e7a9408 100644 --- a/src/main.rs +++ b/src/main.rs @@ -222,7 +222,15 @@ async fn main() { } } - let build_config = BuildConfig::new(deploy_method, build_type, solana_root.get_root_path()); + let build_config = BuildConfig::new( + deploy_method.clone(), + build_type, + &solana_root.get_root_path(), + matches.is_present("docker_build"), + ) + .unwrap_or_else(|err| { + panic!("Error creating BuildConfig: {}", err); + }); let genesis_flags = GenesisFlags { hashes_per_tick: matches @@ -320,7 +328,7 @@ async fn main() { .unwrap_or_default() .to_string(), matches.value_of("registry_name").unwrap().to_string(), - deploy_method.to_string(), + deploy_method, ); if build_config.docker_build() { diff --git a/src/release.rs b/src/release.rs index 5316c6b..00c84e3 100644 --- a/src/release.rs +++ b/src/release.rs @@ -11,7 +11,7 @@ use { strum_macros::{EnumString, IntoStaticStr, VariantNames}, }; -#[derive(Debug, Clone)] +#[derive(Debug, PartialEq, Clone)] pub enum DeployMethod { Local(String), ReleaseChannel(String), diff --git a/src/scripts/bootstrap-startup-scripts.sh b/src/scripts/bootstrap-startup-script.sh similarity index 100% rename from src/scripts/bootstrap-startup-scripts.sh rename to src/scripts/bootstrap-startup-script.sh From 836d00617e0f817acf10bc80b52acfd91b80f04a Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 8 Apr 2024 17:13:18 +0000 Subject: [PATCH 03/26] cargo-fmt/clippy. change to docker-build default --- README.md | 1 - src/docker.rs | 54 +++++++++++++++++++++++--------------------------- src/genesis.rs | 11 ++++------ src/lib.rs | 3 ++- src/main.rs | 40 ++++++++++++++----------------------- src/release.rs | 30 +++++++++++++--------------- 6 files changed, 60 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 2440a41..b469538 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,6 @@ cargo run --bin cluster -- --slots-per-epoch # docker config --registry # e.g. gregcusack - --docker-build --tag # e.g. v1 --base-image # e.g. ubuntu:20.04 --image-name # e.g. cluster-image diff --git a/src/docker.rs b/src/docker.rs index a1431f8..1abccde 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -37,22 +37,21 @@ impl DockerConfig { pub fn build_image( &self, - solana_root_path: PathBuf, + solana_root_path: &Path, validator_type: &ValidatorType, ) -> Result<(), Box> { match validator_type { ValidatorType::Bootstrap => (), ValidatorType::Standard | ValidatorType::RPC | ValidatorType::Client => { return Err(format!( - "Build docker image for validator type: {} not supported yet", - validator_type + "Build docker image for validator type: {validator_type} not supported yet" ) .into()); } } - let image_name = format!("{}-{}", validator_type, self.image_name); - let docker_path = solana_root_path.join(format!("{}/{}", "docker-build", validator_type)); - match self.create_base_image(solana_root_path, image_name, docker_path, validator_type) { + let image_name = format!("{validator_type}-{}", self.image_name); + let docker_path = solana_root_path.join(format!("docker-build/{validator_type}")); + match self.create_base_image(solana_root_path, image_name, &docker_path, validator_type) { Ok(res) => { if res.status.success() { info!("Successfully created base Image"); @@ -68,33 +67,30 @@ impl DockerConfig { fn create_base_image( &self, - solana_root_path: PathBuf, + solana_root_path: &Path, image_name: String, - docker_path: PathBuf, + docker_path: &PathBuf, validator_type: &ValidatorType, ) -> Result> { - let dockerfile_path = self.create_dockerfile(validator_type, docker_path, None)?; + self.create_dockerfile(validator_type, docker_path, None)?; - trace!("Tmp: {}", dockerfile_path.as_path().display()); - trace!("Exists: {}", dockerfile_path.as_path().exists()); + trace!("Tmp: {}", docker_path.as_path().display()); + trace!("Exists: {}", docker_path.as_path().exists()); // We use std::process::Command here because Docker-rs is very slow building dockerfiles // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. // so we result to using std::process::Command - let dockerfile = dockerfile_path.join("Dockerfile"); + let dockerfile = docker_path.join("Dockerfile"); let context_path = solana_root_path.display().to_string(); let progress_bar = new_spinner_progress_bar(); - progress_bar.set_message(format!( - "{BUILD}Building {} docker image...", - validator_type - )); + progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...",)); let command = format!( - "docker build -t {}/{}:{} -f {:?} {}", - self.registry, image_name, self.tag, dockerfile, context_path + "docker build -t {}/{image_name}:{} -f {dockerfile:?} {context_path}", + self.registry, self.tag, ); - info!("command: {}", command); + let output = match Command::new("sh") .arg("-c") .arg(&command) @@ -108,7 +104,7 @@ impl DockerConfig { Err(err) => Err(Box::new(err) as Box), }; progress_bar.finish_and_clear(); - info!("{} image build complete", validator_type); + info!("{validator_type} image build complete"); output } @@ -120,20 +116,20 @@ impl DockerConfig { ) -> std::io::Result<()> { let source_path = source_dir.join("src/scripts").join(file_name); let destination_path = docker_dir.join(file_name); - fs::copy(&source_path, &destination_path)?; + fs::copy(source_path, destination_path)?; Ok(()) } fn create_dockerfile( &self, validator_type: &ValidatorType, - docker_path: PathBuf, + docker_path: &PathBuf, content: Option<&str>, - ) -> Result> { + ) -> Result<(), Box> { if docker_path.exists() { - fs::remove_dir_all(&docker_path)?; + fs::remove_dir_all(docker_path)?; } - fs::create_dir_all(&docker_path)?; + fs::create_dir_all(docker_path)?; if let DeployMethod::Local(_) = self.deploy_method { if validator_type == &ValidatorType::Bootstrap { @@ -141,7 +137,7 @@ impl DockerConfig { PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR")); let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"]; for file_name in files_to_copy.iter() { - Self::copy_file_to_docker(&manifest_path, &docker_path, file_name)?; + Self::copy_file_to_docker(&manifest_path, docker_path, file_name)?; } } } @@ -150,7 +146,7 @@ impl DockerConfig { if let DeployMethod::ReleaseChannel(_) = self.deploy_method { ("solana-release", "./src/scripts".to_string()) } else { - ("farf", format!("./docker-build/{}", validator_type)) + ("farf", format!("./docker-build/{validator_type}")) }; let dockerfile = format!( @@ -184,11 +180,11 @@ WORKDIR /home/solana self.base_image ); - debug!("dockerfile: {}", dockerfile); + debug!("dockerfile: {dockerfile:?}"); std::fs::write( docker_path.join("Dockerfile"), content.unwrap_or(dockerfile.as_str()), )?; - Ok(docker_path) + Ok(()) } } diff --git a/src/genesis.rs b/src/genesis.rs index be3325b..eaa902c 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -250,7 +250,7 @@ impl Genesis { .for_each(|account_type| { args.push( self.config_dir - .join(format!("bootstrap-validator/{}.json", account_type)) + .join(format!("bootstrap-validator/{account_type}.json")) .to_string_lossy() .to_string(), ); @@ -269,10 +269,7 @@ impl Genesis { args } - pub fn setup_spl_args( - &self, - solana_root_path: &PathBuf, - ) -> Result, Box> { + pub fn setup_spl_args(&self, solana_root_path: &Path) -> Result, Box> { let fetch_spl_file = solana_root_path.join("fetch-spl.sh"); fetch_spl(&fetch_spl_file)?; @@ -283,8 +280,8 @@ impl Genesis { pub fn generate( &mut self, - solana_root_path: &PathBuf, - build_path: &PathBuf, + solana_root_path: &Path, + build_path: &Path, ) -> Result<(), Box> { let mut args = self.setup_genesis_flags(); let mut spl_args = self.setup_spl_args(solana_root_path)?; diff --git a/src/lib.rs b/src/lib.rs index 6297fa0..c1ad9cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,6 +54,7 @@ pub enum ValidatorType { Client, } +pub mod docker; pub mod genesis; pub mod kubernetes; pub mod release; @@ -79,7 +80,7 @@ pub fn cat_file(path: &PathBuf) -> std::io::Result<()> { let mut file = File::open(path)?; let mut contents = String::new(); file.read_to_string(&mut contents)?; - info!("{:?}:\n{}", path.file_name(), contents); + info!("{:?}:\n{contents}", path.file_name()); Ok(()) } diff --git a/src/main.rs b/src/main.rs index e7a9408..1f9bcbe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,7 +77,7 @@ fn parse_matches() -> clap::ArgMatches { Arg::with_name("enable_warmup_epochs") .long("enable-warmup-epochs") .takes_value(true) - .possible_values(&["true", "false"]) + .possible_values(["true", "false"]) .default_value("true") .help("Genesis config. enable warmup epoch. defaults to true"), ) @@ -90,7 +90,7 @@ fn parse_matches() -> clap::ArgMatches { .arg( Arg::with_name("cluster_type") .long("cluster-type") - .possible_values(&["development", "devnet", "testnet", "mainnet-beta"]) + .possible_values(["development", "devnet", "testnet", "mainnet-beta"]) .takes_value(true) .default_value("development") .help( @@ -111,13 +111,9 @@ fn parse_matches() -> clap::ArgMatches { ) //Docker config .arg( - Arg::with_name("docker_build") - .long("docker-build") - .requires("registry_name") - .requires("image_name") - .requires("base_image") - .requires("image_tag") - .help("Build Docker images. Build new docker images"), + Arg::with_name("skip_docker_build") + .long("skip-docker-build") + .help("Skips build Docker images"), ) .arg( Arg::with_name("registry_name") @@ -131,7 +127,6 @@ fn parse_matches() -> clap::ArgMatches { .long("image-name") .takes_value(true) .default_value("k8s-cluster-image") - .required(true) .help("Docker image name. Will be prepended with validator_type (bootstrap or validator)"), ) .arg( @@ -139,14 +134,12 @@ fn parse_matches() -> clap::ArgMatches { .long("base-image") .takes_value(true) .default_value("ubuntu:20.04") - .required(true) .help("Docker base image"), ) .arg( Arg::with_name("image_tag") .long("tag") .takes_value(true) - .required(true) .default_value("latest") .help("Docker image tag."), ) @@ -225,12 +218,9 @@ async fn main() { let build_config = BuildConfig::new( deploy_method.clone(), build_type, - &solana_root.get_root_path(), - matches.is_present("docker_build"), - ) - .unwrap_or_else(|err| { - panic!("Error creating BuildConfig: {}", err); - }); + solana_root.get_root_path(), + !matches.is_present("skip_docker_build"), + ); let genesis_flags = GenesisFlags { hashes_per_tick: matches @@ -285,14 +275,14 @@ async fn main() { match build_config.prepare().await { Ok(_) => info!("Validator setup prepared successfully"), Err(err) => { - error!("Error: {}", err); + error!("Error: {err}"); return; } } let mut genesis = Genesis::new(solana_root.get_root_path(), genesis_flags); match genesis.generate_faucet() { - Ok(_) => (), + Ok(_) => info!("Generated faucet account"), Err(err) => { error!("generate faucet error! {err}"); return; @@ -300,7 +290,7 @@ async fn main() { } match genesis.generate_accounts(ValidatorType::Bootstrap, 1) { - Ok(_) => (), + Ok(_) => info!("Generated bootstrap account"), Err(err) => { error!("generate accounts error! {err}"); return; @@ -309,9 +299,9 @@ async fn main() { // creates genesis and writes to binary file match genesis.generate(solana_root.get_root_path(), &build_path) { - Ok(_) => (), + Ok(_) => info!("Created genesis successfully"), Err(err) => { - error!("generate genesis error! {}", err); + error!("generate genesis error! {err}"); return; } } @@ -334,9 +324,9 @@ async fn main() { if build_config.docker_build() { let image_type = ValidatorType::Bootstrap; match docker.build_image(solana_root.get_root_path(), &image_type) { - Ok(_) => info!("{} image built successfully", image_type), + Ok(_) => info!("{image_type} image built successfully"), Err(err) => { - error!("Exiting........ {}", err); + error!("Error. Failed to build imge: {err}"); return; } } diff --git a/src/release.rs b/src/release.rs index 00c84e3..49fb08f 100644 --- a/src/release.rs +++ b/src/release.rs @@ -30,27 +30,28 @@ pub struct BuildConfig { build_type: BuildType, solana_root_path: PathBuf, docker_build: bool, + build_path: PathBuf, } impl BuildConfig { pub fn new( deploy_method: DeployMethod, build_type: BuildType, - solana_root_path: &PathBuf, + solana_root_path: &Path, docker_build: bool, - ) -> Result> { + ) -> Self { let build_path = match deploy_method { DeployMethod::Local(_) => solana_root_path.join("farf/bin"), DeployMethod::ReleaseChannel(_) => solana_root_path.join("solana-release/bin"), }; - Ok(BuildConfig { + BuildConfig { deploy_method, build_type, - build_path, - solana_root_path: solana_root_path.clone(), + solana_root_path: solana_root_path.to_path_buf(), docker_build, - }) + build_path, + } } pub fn build_path(&self) -> PathBuf { @@ -81,7 +82,7 @@ impl BuildConfig { async fn setup_tar_deploy(&self, release_channel: &String) -> Result> { let file_name = "solana-release"; let tar_filename = format!("{file_name}.tar.bz2"); - info!("tar file: {}", tar_filename); + info!("tar file: {tar_filename}"); self.download_release_from_channel(&tar_filename, release_channel) .await?; @@ -145,14 +146,14 @@ impl BuildConfig { let tag_object = solana_repo.revparse_single(tag)?.id(); // Check if the commit associated with the tag is the same as the current commit if tag_object == commit { - info!("The current commit is associated with tag: {}", tag); + info!("The current commit is associated with tag: {tag}"); note = tag_object.to_string(); break; } } // Write to branch/tag and commit to version.yml - let content = format!("channel: devbuild {}\ncommit: {}", note, commit); + let content = format!("channel: devbuild {note}\ncommit: {commit}"); std::fs::write(self.solana_root_path.join("farf/version.yml"), content) .expect("Failed to write version.yml"); @@ -165,22 +166,19 @@ impl BuildConfig { tar_filename: &str, release_channel: &String, ) -> Result<(), Box> { - info!("Downloading release from channel: {}", release_channel); + info!("Downloading release from channel: {release_channel}"); let file_path = self.solana_root_path.join(tar_filename); // Remove file if let Err(err) = fs::remove_file(&file_path) { if err.kind() != std::io::ErrorKind::NotFound { - return Err(format!("{}: {:?}", "Error while removing file:", err).into()); + return Err(format!("{err}: {:?}", "Error while removing file:").into()); } } let download_url = format!( - "{}{}{}", - "https://release.solana.com/", - release_channel, - "/solana-release-x86_64-unknown-linux-gnu.tar.bz2" + "https://release.solana.com/{release_channel}/solana-release-x86_64-unknown-linux-gnu.tar.bz2" ); - info!("download_url: {}", download_url); + info!("download_url: {download_url}"); download_to_temp( download_url.as_str(), From e5097ad63f7a5d3b4e0f14b322555ad551604a73 Mon Sep 17 00:00:00 2001 From: greg Date: Fri, 29 Mar 2024 23:02:02 +0000 Subject: [PATCH 04/26] push bootstrap to dockerhub registry. modify dockerconfig. create DockerImage struct --- PROGRESS.md | 4 +- src/docker.rs | 130 ++++++++++++++++++++++++++++++++++---------------- src/lib.rs | 1 + src/main.rs | 27 ++++++++--- 4 files changed, 114 insertions(+), 48 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 5c5701f..84521ce 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -15,9 +15,9 @@ - [x] Create Genesis - [x] Generate faucet and bootstrap accounts - [x] Build genesis -- [ ] Docker Build +- [x] Docker Build - [x] Build Bootstrap Image - - [ ] Push Image to registry + - [x] Push Image to registry - [ ] Create & Deploy Secrets - [ ] Bootstrap - [ ] Validator (regular) diff --git a/src/docker.rs b/src/docker.rs index 1abccde..22bb244 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -1,36 +1,64 @@ use { - crate::{new_spinner_progress_bar, release::DeployMethod, ValidatorType, BUILD}, + crate::{new_spinner_progress_bar, release::DeployMethod, ValidatorType, BUILD, ROCKET}, log::*, std::{ env, error::Error, + fmt::{self, Display, Formatter}, fs, path::{Path, PathBuf}, - process::{Command, Output, Stdio}, + process::{Command, Stdio}, }, }; -pub struct DockerConfig { - pub base_image: String, - pub image_name: String, - pub tag: String, - pub registry: String, - deploy_method: DeployMethod, +pub struct DockerImage { + registry: String, + validator_type: ValidatorType, + image_name: String, + tag: String, } -impl DockerConfig { +impl DockerImage { + // Constructor to create a new instance of DockerImage pub fn new( - base_image: String, + registry: String, + validator_type: ValidatorType, image_name: String, tag: String, - registry: String, - deploy_method: DeployMethod, ) -> Self { - DockerConfig { - base_image, + DockerImage { + registry, + validator_type, image_name, tag, - registry, + } + } + + pub fn validator_type(&self) -> ValidatorType { + self.validator_type + } +} + +// Put DockerImage in format for building, pushing, and pulling +impl Display for DockerImage { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "{}/{}-{}:{}", + self.registry, self.validator_type, self.image_name, self.tag + ) + } +} + +pub struct DockerConfig { + pub base_image: String, + deploy_method: DeployMethod, +} + +impl DockerConfig { + pub fn new(base_image: String, deploy_method: DeployMethod) -> Self { + DockerConfig { + base_image, deploy_method, } } @@ -38,8 +66,9 @@ impl DockerConfig { pub fn build_image( &self, solana_root_path: &Path, - validator_type: &ValidatorType, + docker_image: &DockerImage, ) -> Result<(), Box> { + let validator_type = docker_image.validator_type(); match validator_type { ValidatorType::Bootstrap => (), ValidatorType::Standard | ValidatorType::RPC | ValidatorType::Client => { @@ -49,34 +78,27 @@ impl DockerConfig { .into()); } } - let image_name = format!("{validator_type}-{}", self.image_name); + let docker_path = solana_root_path.join(format!("docker-build/{validator_type}")); - match self.create_base_image(solana_root_path, image_name, &docker_path, validator_type) { - Ok(res) => { - if res.status.success() { - info!("Successfully created base Image"); - Ok(()) - } else { - error!("Failed to build base image"); - Err(String::from_utf8_lossy(&res.stderr).into()) - } - } - Err(err) => Err(err), - } + self.create_base_image( + solana_root_path, + docker_image, + &docker_path, + &validator_type, + )?; + + Ok(()) } fn create_base_image( &self, solana_root_path: &Path, - image_name: String, + docker_image: &DockerImage, docker_path: &PathBuf, validator_type: &ValidatorType, - ) -> Result> { + ) -> Result<(), Box> { self.create_dockerfile(validator_type, docker_path, None)?; - trace!("Tmp: {}", docker_path.as_path().display()); - trace!("Exists: {}", docker_path.as_path().exists()); - // We use std::process::Command here because Docker-rs is very slow building dockerfiles // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. // so we result to using std::process::Command @@ -86,10 +108,7 @@ impl DockerConfig { let progress_bar = new_spinner_progress_bar(); progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...",)); - let command = format!( - "docker build -t {}/{image_name}:{} -f {dockerfile:?} {context_path}", - self.registry, self.tag, - ); + let command = format!("docker build -t {docker_image} -f {dockerfile:?} {context_path}"); let output = match Command::new("sh") .arg("-c") @@ -102,11 +121,15 @@ impl DockerConfig { { Ok(res) => Ok(res), Err(err) => Err(Box::new(err) as Box), - }; + }?; + + if !output.status.success() { + return Err(output.status.to_string().into()); + } progress_bar.finish_and_clear(); info!("{validator_type} image build complete"); - output + Ok(()) } fn copy_file_to_docker( @@ -187,4 +210,31 @@ WORKDIR /home/solana )?; Ok(()) } + + pub fn push_image(docker_image: &DockerImage) -> Result<(), Box> { + let progress_bar = new_spinner_progress_bar(); + progress_bar.set_message(format!( + "{ROCKET}Pushing {} image to registry...", + docker_image.validator_type() + )); + let command = format!("docker push '{}'", docker_image); + let output = match Command::new("sh") + .arg("-c") + .arg(&command) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Failed to execute command") + .wait_with_output() + { + Ok(res) => Ok(res), + Err(err) => Err(Box::new(err) as Box), + }?; + + if !output.status.success() { + return Err(output.status.to_string().into()); + } + progress_bar.finish_and_clear(); + Ok(()) + } } diff --git a/src/lib.rs b/src/lib.rs index c1ad9cb..98f8e21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,6 +61,7 @@ pub mod release; static BUILD: Emoji = Emoji("👷 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); +static ROCKET: Emoji = Emoji("🚀 ", ""); static SUN: Emoji = Emoji("🌞 ", ""); static TRUCK: Emoji = Emoji("🚚 ", ""); diff --git a/src/main.rs b/src/main.rs index 1f9bcbe..ac021c8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ use { std::fs, strum::VariantNames, validator_lab::{ - docker::DockerConfig, + docker::{DockerConfig, DockerImage}, genesis::{Genesis, GenesisFlags}, kubernetes::Kubernetes, release::{BuildConfig, BuildType, DeployMethod}, @@ -312,19 +312,34 @@ async fn main() { .value_of("base_image") .unwrap_or_default() .to_string(), + deploy_method, + ); + + let validator_type = ValidatorType::Bootstrap; + let docker_image = DockerImage::new( + matches.value_of("registry_name").unwrap().to_string(), + validator_type, matches.value_of("image_name").unwrap().to_string(), matches .value_of("image_tag") .unwrap_or_default() .to_string(), - matches.value_of("registry_name").unwrap().to_string(), - deploy_method, ); if build_config.docker_build() { - let image_type = ValidatorType::Bootstrap; - match docker.build_image(solana_root.get_root_path(), &image_type) { - Ok(_) => info!("{image_type} image built successfully"), + match docker.build_image(solana_root.get_root_path(), &docker_image) { + Ok(_) => info!("{} image built successfully", docker_image.validator_type()), + Err(err) => { + error!("Exiting........ {err}"); + return; + } + } + + match DockerConfig::push_image(&docker_image) { + Ok(_) => info!( + "{} image pushed successfully", + docker_image.validator_type() + ), Err(err) => { error!("Error. Failed to build imge: {err}"); return; From 115b5f5b92d1196a839fb9011a94b7f4969d05fb Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 1 Apr 2024 20:16:58 +0000 Subject: [PATCH 05/26] add create and deploy bootstrap secrets --- PROGRESS.md | 2 +- src/genesis.rs | 3 +-- src/k8s_helpers.rs | 30 ++++++++++++++++++++++++++++++ src/kubernetes.rs | 32 ++++++++++++++++++++++++++++++-- src/lib.rs | 1 + src/main.rs | 22 +++++++++++++++++++++- 6 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 src/k8s_helpers.rs diff --git a/PROGRESS.md b/PROGRESS.md index 84521ce..5b6a368 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -19,7 +19,7 @@ - [x] Build Bootstrap Image - [x] Push Image to registry - [ ] Create & Deploy Secrets - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client diff --git a/src/genesis.rs b/src/genesis.rs index eaa902c..d6bb7a7 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -117,8 +117,7 @@ pub struct Genesis { } impl Genesis { - pub fn new(solana_root: &Path, flags: GenesisFlags) -> Self { - let config_dir = solana_root.join("config-k8s"); + pub fn new(config_dir: PathBuf, flags: GenesisFlags) -> Self { if config_dir.exists() { std::fs::remove_dir_all(&config_dir).unwrap(); } diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs new file mode 100644 index 0000000..52b64a1 --- /dev/null +++ b/src/k8s_helpers.rs @@ -0,0 +1,30 @@ +use { + k8s_openapi::{api::core::v1::Secret, ByteString}, + kube::api::ObjectMeta, + std::{collections::BTreeMap, error::Error, path::PathBuf}, +}; + +fn create_secret(name: &str, data: BTreeMap) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(name.to_string()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + } +} + +pub fn create_secret_from_files( + secret_name: &str, + key_files: &[(PathBuf, &str)], //[pathbuf, key type] +) -> Result> { + let mut data = BTreeMap::new(); + for (file_path, key_type) in key_files { + let file_content = std::fs::read(file_path) + .map_err(|err| format!("Failed to read file '{:?}': {}", file_path, err))?; + data.insert(format!("{}.json", key_type), ByteString(file_content)); + } + + Ok(create_secret(secret_name, data)) +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 869e901..71d92f0 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,9 +1,11 @@ use { - k8s_openapi::api::core::v1::Namespace, + crate::k8s_helpers, + k8s_openapi::api::core::v1::{Namespace, Secret}, kube::{ - api::{Api, ListParams}, + api::{Api, ListParams, PostParams}, Client, }, + std::{error::Error, path::Path}, }; pub struct Kubernetes { @@ -30,4 +32,30 @@ impl Kubernetes { Ok(exists) } + + pub fn create_bootstrap_secret( + &self, + secret_name: &str, + config_dir: &Path, + ) -> Result> { + let faucet_key_path = config_dir.join("faucet.json"); + let identity_key_path = config_dir.join("bootstrap-validator/identity.json"); + let vote_key_path = config_dir.join("bootstrap-validator/vote-account.json"); + let stake_key_path = config_dir.join("bootstrap-validator/stake-account.json"); + + let key_files = vec![ + (faucet_key_path, "faucet"), + (identity_key_path, "identity"), + (vote_key_path, "vote"), + (stake_key_path, "stake"), + ]; + + k8s_helpers::create_secret_from_files(secret_name, &key_files) + } + + pub async fn deploy_secret(&self, secret: &Secret) -> Result { + let secrets_api: Api = + Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + secrets_api.create(&PostParams::default(), secret).await + } } diff --git a/src/lib.rs b/src/lib.rs index 98f8e21..9bd325f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,7 @@ pub enum ValidatorType { pub mod docker; pub mod genesis; +pub mod k8s_helpers; pub mod kubernetes; pub mod release; diff --git a/src/main.rs b/src/main.rs index ac021c8..693dd0c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -280,7 +280,9 @@ async fn main() { } } - let mut genesis = Genesis::new(solana_root.get_root_path(), genesis_flags); + let config_directory = solana_root.get_root_path().join("config-k8s"); + let mut genesis = Genesis::new(config_directory.clone(), genesis_flags); + match genesis.generate_faucet() { Ok(_) => info!("Generated faucet account"), Err(err) => { @@ -346,4 +348,22 @@ async fn main() { } } } + + let bootstrap_secret = match kub_controller + .create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) + { + Ok(secret) => secret, + Err(err) => { + error!("Failed to create bootstrap secret! {}", err); + return; + } + }; + + match kub_controller.deploy_secret(&bootstrap_secret).await { + Ok(_) => info!("Deployed Bootstrap Secret"), + Err(err) => { + error!("{}", err); + return; + } + } } From 5402d0e2acb39a8670ed01d2804d6db37c412084 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 1 Apr 2024 20:52:23 +0000 Subject: [PATCH 06/26] add bootstrap validator selector --- PROGRESS.md | 2 +- src/k8s_helpers.rs | 6 ++++++ src/kubernetes.rs | 6 +++++- src/main.rs | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 5b6a368..8db6e91 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -24,7 +24,7 @@ - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Selector - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 52b64a1..b6d80d4 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -28,3 +28,9 @@ pub fn create_secret_from_files( Ok(create_secret(secret_name, data)) } + +pub fn create_selector(key: &str, value: &str) -> BTreeMap { + let mut btree = BTreeMap::new(); + btree.insert(key.to_string(), value.to_string()); + btree +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 71d92f0..27aecff 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -5,7 +5,7 @@ use { api::{Api, ListParams, PostParams}, Client, }, - std::{error::Error, path::Path}, + std::{collections::BTreeMap, error::Error, path::Path}, }; pub struct Kubernetes { @@ -58,4 +58,8 @@ impl Kubernetes { Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); secrets_api.create(&PostParams::default(), secret).await } + + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { + k8s_helpers::create_selector(key, value) + } } diff --git a/src/main.rs b/src/main.rs index 693dd0c..f360929 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use { clap::{command, Arg, ArgGroup}, log::*, + solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, std::fs, strum::VariantNames, validator_lab::{ @@ -366,4 +367,22 @@ async fn main() { return; } } + + // Bootstrap needs two labels. Because it is going to have two services. + // One via Load Balancer, one direct + let mut bootstrap_rs_labels = + kub_controller.create_selector("validator/lb", "load-balancer-selector"); + bootstrap_rs_labels.insert( + "validator/name".to_string(), + "bootstrap-validator-selector".to_string(), + ); + bootstrap_rs_labels.insert("validator/type".to_string(), "bootstrap".to_string()); + + let identity_path = config_directory.join("bootstrap-validator/identity.json"); + let bootstrap_keypair = + read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); + bootstrap_rs_labels.insert( + "validator/identity".to_string(), + bootstrap_keypair.pubkey().to_string(), + ); } From 2a465b8d288c1f5db9517f3de03bc6117c5e2ce8 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 1 Apr 2024 21:42:53 +0000 Subject: [PATCH 07/26] wip. adding replicaset but need validator config flags --- src/k8s_helpers.rs | 80 ++++++++++++++++++++++++++++++++++++++++++++-- src/kubernetes.rs | 69 +++++++++++++++++++++++++++++++++++++-- src/main.rs | 13 ++++++++ 3 files changed, 158 insertions(+), 4 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index b6d80d4..5297779 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -1,9 +1,20 @@ use { - k8s_openapi::{api::core::v1::Secret, ByteString}, + crate::ValidatorType, + k8s_openapi::{ + api::{ + apps::v1::{ReplicaSet, ReplicaSetSpec}, + core::v1::{ + Affinity, Container, EnvVar, PodSecurityContext, + PodSpec, PodTemplateSpec, Probe, ResourceRequirements, Secret, + Volume, VolumeMount, + }, + }, + apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, + ByteString, + }, kube::api::ObjectMeta, std::{collections::BTreeMap, error::Error, path::PathBuf}, }; - fn create_secret(name: &str, data: BTreeMap) -> Secret { Secret { metadata: ObjectMeta { @@ -34,3 +45,68 @@ pub fn create_selector(key: &str, value: &str) -> BTreeMap { btree.insert(key.to_string(), value.to_string()); btree } + +#[allow(clippy::too_many_arguments)] +pub fn create_replica_set( + name: &ValidatorType, + namespace: &str, + label_selector: &BTreeMap, + container_name: &str, + image_name: &str, + environment_variables: Vec, + command: &[String], + volumes: Option>, + volume_mounts: Option>, + readiness_probe: Option, + pod_requests: BTreeMap, +) -> Result> { + let pod_spec = PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(label_selector.clone()), + ..Default::default() + }), + spec: Some(PodSpec { + containers: vec![Container { + name: container_name.to_string(), + image: Some(image_name.to_string()), + image_pull_policy: Some("Always".to_string()), + env: Some(environment_variables), + command: Some(command.to_owned()), + volume_mounts, + readiness_probe, + resources: Some(ResourceRequirements { + requests: Some(pod_requests), + ..Default::default() + }), + ..Default::default() + }], + volumes, + security_context: Some(PodSecurityContext { + run_as_user: Some(1000), + run_as_group: Some(1000), + ..Default::default() + }), + ..Default::default() + }), + }; + + let replicas_set_spec = ReplicaSetSpec { + replicas: Some(1), + selector: LabelSelector { + match_labels: Some(label_selector.clone()), + ..Default::default() + }, + template: Some(pod_spec), + ..Default::default() + }; + + Ok(ReplicaSet { + metadata: ObjectMeta { + name: Some(format!("{}-replicaset", name)), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: Some(replicas_set_spec), + ..Default::default() + }) +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 27aecff..62473ac 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,10 +1,17 @@ use { - crate::k8s_helpers, - k8s_openapi::api::core::v1::{Namespace, Secret}, + crate::{k8s_helpers, ValidatorType}, + k8s_openapi::api::{ + apps::v1::ReplicaSet, + core::v1::{ + EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, + Secret, SecretVolumeSource, Volume, VolumeMount, + }, + }, kube::{ api::{Api, ListParams, PostParams}, Client, }, + log::*, std::{collections::BTreeMap, error::Error, path::Path}, }; @@ -59,6 +66,64 @@ impl Kubernetes { secrets_api.create(&PostParams::default(), secret).await } + pub fn create_bootstrap_validator_replica_set( + &mut self, + container_name: &str, + image_name: &str, + secret_name: Option, + label_selector: &BTreeMap, + ) -> Result> { + let mut env_vars = vec![EnvVar { + name: "MY_POD_IP".to_string(), + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: "status.podIP".to_string(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }]; + + + let accounts_volume = Some(vec![Volume { + name: "bootstrap-accounts-volume".into(), + secret: Some(SecretVolumeSource { + secret_name, + ..Default::default() + }), + ..Default::default() + }]); + + let accounts_volume_mount = Some(vec![VolumeMount { + name: "bootstrap-accounts-volume".to_string(), + mount_path: "/home/solana/bootstrap-accounts".to_string(), + ..Default::default() + }]); + + let mut command = + vec!["/home/solana/k8s-cluster-scripts/bootstrap-startup-script.sh".to_string()]; + command.extend(self.generate_bootstrap_command_flags()); + + for c in command.iter() { + debug!("bootstrap command: {}", c); + } + + k8s_helpers::create_replica_set( + &ValidatorType::Bootstrap, + self.namespace.as_str(), + label_selector, + container_name, + image_name, + env_vars, + &command, + accounts_volume, + accounts_volume_mount, + None, + self.pod_requests.requests.clone(), + ) + } + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } diff --git a/src/main.rs b/src/main.rs index f360929..8f69bbd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -385,4 +385,17 @@ async fn main() { "validator/identity".to_string(), bootstrap_keypair.pubkey().to_string(), ); + + let bootstrap_replica_set = match kub_controller.create_bootstrap_validator_replica_set( + bootstrap_container_name, + bootstrap_image_name, + bootstrap_secret.metadata.name.clone(), + &bootstrap_rs_labels, + ) { + Ok(replica_set) => replica_set, + Err(err) => { + error!("Error creating bootstrap validator replicas_set: {}", err); + return; + } + }; } From f2d65a106348b09c670b6d984f5372cf868d6983 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 1 Apr 2024 22:05:13 +0000 Subject: [PATCH 08/26] add validator config --- Cargo.lock | 1 + Cargo.toml | 1 + src/kubernetes.rs | 10 ++-- src/lib.rs | 1 + src/main.rs | 111 ++++++++++++++++++++++++++++++++++------ src/validator_config.rs | 56 ++++++++++++++++++++ 6 files changed, 159 insertions(+), 21 deletions(-) create mode 100644 src/validator_config.rs diff --git a/Cargo.lock b/Cargo.lock index ce16dc9..269a491 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7136,6 +7136,7 @@ dependencies = [ "rustc_version", "rustls", "solana-core", + "solana-ledger", "solana-logger", "solana-sdk", "strum 0.26.2", diff --git a/Cargo.toml b/Cargo.toml index c98d591..7137d4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ rand = "0.8.5" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.10", default-features = false, features = ["quic"] } solana-core = "1.18.8" +solana-ledger = "1.18.8" solana-logger = "1.18.8" solana-sdk = "1.18.8" strum = "0.26.2" diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 62473ac..208fe56 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,5 +1,5 @@ use { - crate::{k8s_helpers, ValidatorType}, + crate::{k8s_helpers, ValidatorType, validator_config::ValidatorConfig}, k8s_openapi::api::{ apps::v1::ReplicaSet, core::v1::{ @@ -15,16 +15,18 @@ use { std::{collections::BTreeMap, error::Error, path::Path}, }; -pub struct Kubernetes { +pub struct Kubernetes<'a> { k8s_client: Client, namespace: String, + validator_config: &'a mut ValidatorConfig, } -impl Kubernetes { - pub async fn new(namespace: &str) -> Kubernetes { +impl<'a> Kubernetes<'a> { + pub async fn new(namespace: &str, validator_config: &'a mut ValidatorConfig) -> Kubernetes<'a> { Self { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), + validator_config, } } diff --git a/src/lib.rs b/src/lib.rs index 9bd325f..aaa9491 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,6 +59,7 @@ pub mod genesis; pub mod k8s_helpers; pub mod kubernetes; pub mod release; +pub mod validator_config; static BUILD: Emoji = Emoji("👷 ", ""); static PACKAGE: Emoji = Emoji("📦 ", ""); diff --git a/src/main.rs b/src/main.rs index 8f69bbd..05a2736 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,17 @@ use { clap::{command, Arg, ArgGroup}, log::*, + solana_ledger::blockstore_cleanup_service::{DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS}, solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, std::fs, strum::VariantNames, validator_lab::{ docker::{DockerConfig, DockerImage}, - genesis::{Genesis, GenesisFlags}, + genesis::{Genesis, GenesisFlags, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL}, kubernetes::Kubernetes, release::{BuildConfig, BuildType, DeployMethod}, SolanaRoot, ValidatorType, + validator_config::ValidatorConfig, }, }; @@ -144,6 +146,53 @@ fn parse_matches() -> clap::ArgMatches { .default_value("latest") .help("Docker image tag."), ) + // Bootstrap/Validator Config + .arg( + Arg::with_name("tpu_enable_udp") + .long("tpu-enable-udp") + .help("Validator config. Enable UDP for tpu transactions."), + ) + .arg( + Arg::with_name("tpu_disable_quic") + .long("tpu-disable-quic") + .help("Validator config. Disable quic for tpu packet forwarding"), + ) + .arg( + Arg::with_name("limit_ledger_size") + .long("limit-ledger-size") + .takes_value(true) + .help("Validator Config. The `--limit-ledger-size` parameter allows you to specify how many ledger + shreds your node retains on disk. If you do not + include this parameter, the validator will keep the entire ledger until it runs + out of disk space. The default value attempts to keep the ledger disk usage + under 500GB. More or less disk usage may be requested by adding an argument to + `--limit-ledger-size` if desired. Check `agave-validator --help` for the + default limit value used by `--limit-ledger-size`. More information about + selecting a custom limit value is at : https://github.com/solana-labs/solana/blob/583cec922b6107e0f85c7e14cb5e642bc7dfb340/core/src/ledger_cleanup_service.rs#L15-L26"), + ) + .arg( + Arg::with_name("skip_poh_verify") + .long("skip-poh-verify") + .help("Validator config. If set, validators will skip verifying + the ledger they already have saved to disk at + boot (results in a much faster boot)"), + ) + .arg( + Arg::with_name("no_snapshot_fetch") + .long("no-snapshot-fetch") + .help("Validator config. If set, disables booting validators from a snapshot"), + ) + .arg( + Arg::with_name("require_tower") + .long("require-tower") + .help("Validator config. Refuse to start if saved tower state is not found. + Off by default since validator won't restart if the pod restarts"), + ) + .arg( + Arg::with_name("full_rpc") + .long("full-rpc") + .help("Validator config. Support full RPC services on all nodes"), + ) .get_matches() } @@ -200,22 +249,6 @@ async fn main() { ); } - let kub_controller = Kubernetes::new(environment_config.namespace).await; - match kub_controller.namespace_exists().await { - Ok(true) => (), - Ok(false) => { - error!( - "Namespace: '{}' doesn't exist. Exiting...", - environment_config.namespace - ); - return; - } - Err(err) => { - error!("Error: {err}"); - return; - } - } - let build_config = BuildConfig::new( deploy_method.clone(), build_type, @@ -273,6 +306,50 @@ async fn main() { ), }; + let mut validator_config = ValidatorConfig { + tpu_enable_udp: matches.is_present("tpu_enable_udp"), + tpu_disable_quic: matches.is_present("tpu_disable_quic"), + shred_version: None, // set after genesis created + bank_hash: None, // set after genesis created + max_ledger_size: if matches.is_present("limit_ledger_size") { + let limit_ledger_size = match matches.value_of("limit_ledger_size") { + Some(_) => value_t_or_exit!(matches, "limit_ledger_size", u64), + None => DEFAULT_MAX_LEDGER_SHREDS, + }; + if limit_ledger_size < DEFAULT_MIN_MAX_LEDGER_SHREDS { + error!( + "The provided --limit-ledger-size value was too small, the minimum value is {DEFAULT_MIN_MAX_LEDGER_SHREDS}" + ); + return; + } + Some(limit_ledger_size) + } else { + None + }, + skip_poh_verify: matches.is_present("skip_poh_verify"), + no_snapshot_fetch: matches.is_present("no_snapshot_fetch"), + require_tower: matches.is_present("require_tower"), + enable_full_rpc: matches.is_present("enable_full_rpc"), + entrypoints: Vec::new(), + known_validators: None, + }; + + let kub_controller = Kubernetes::new(environment_config.namespace, &mut validator_config).await; + match kub_controller.namespace_exists().await { + Ok(true) => (), + Ok(false) => { + error!( + "Namespace: '{}' doesn't exist. Exiting...", + environment_config.namespace + ); + return; + } + Err(err) => { + error!("Error: {}", err); + return; + } + } + match build_config.prepare().await { Ok(_) => info!("Validator setup prepared successfully"), Err(err) => { diff --git a/src/validator_config.rs b/src/validator_config.rs new file mode 100644 index 0000000..b60776b --- /dev/null +++ b/src/validator_config.rs @@ -0,0 +1,56 @@ +use solana_sdk::{ + hash::Hash, pubkey::Pubkey, +}; + +pub struct ValidatorConfig { + pub tpu_enable_udp: bool, + pub tpu_disable_quic: bool, + pub shred_version: Option, + pub bank_hash: Option, + pub max_ledger_size: Option, + pub skip_poh_verify: bool, + pub no_snapshot_fetch: bool, + pub require_tower: bool, + pub enable_full_rpc: bool, + pub entrypoints: Vec, + pub known_validators: Option>, +} + +impl std::fmt::Display for ValidatorConfig { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let known_validators = match &self.known_validators { + Some(validators) => validators + .iter() + .map(|v| v.to_string()) + .collect::>() + .join(", "), + None => "None".to_string(), + }; + write!( + f, + "Runtime Config\n\ + tpu_enable_udp: {}\n\ + tpu_disable_quic: {}\n\ + shred_version: {:?}\n\ + bank_hash: {:?}\n\ + max_ledger_size: {:?}\n\ + skip_poh_verify: {}\n\ + no_snapshot_fetch: {}\n\ + require_tower: {}\n\ + enable_full_rpc: {}\n\ + entrypoints: {:?}\n\ + known_validators: {:?}", + self.tpu_enable_udp, + self.tpu_disable_quic, + self.shred_version, + self.bank_hash, + self.max_ledger_size, + self.skip_poh_verify, + self.no_snapshot_fetch, + self.require_tower, + self.enable_full_rpc, + self.entrypoints.join(", "), + known_validators, + ) + } +} \ No newline at end of file From 15b1ae70c9435ec2834415adce134ec753a3cb0b Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 1 Apr 2024 22:13:52 +0000 Subject: [PATCH 09/26] add pod requests. need for scheduling --- src/k8s_helpers.rs | 3 ++- src/kubernetes.rs | 36 ++++++++++++++++++++++++++++++------ src/main.rs | 28 ++++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 5297779..d45196e 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -4,7 +4,7 @@ use { api::{ apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ - Affinity, Container, EnvVar, PodSecurityContext, + Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, ResourceRequirements, Secret, Volume, VolumeMount, }, @@ -15,6 +15,7 @@ use { kube::api::ObjectMeta, std::{collections::BTreeMap, error::Error, path::PathBuf}, }; + fn create_secret(name: &str, data: BTreeMap) -> Secret { Secret { metadata: ObjectMeta { diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 208fe56..7065079 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,11 +1,14 @@ use { crate::{k8s_helpers, ValidatorType, validator_config::ValidatorConfig}, - k8s_openapi::api::{ - apps::v1::ReplicaSet, - core::v1::{ - EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, - Secret, SecretVolumeSource, Volume, VolumeMount, + k8s_openapi::{ + api::{ + apps::v1::ReplicaSet, + core::v1::{ + EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, + Secret, SecretVolumeSource, Volume, VolumeMount, + }, }, + apimachinery::pkg::api::resource::Quantity, }, kube::{ api::{Api, ListParams, PostParams}, @@ -15,18 +18,39 @@ use { std::{collections::BTreeMap, error::Error, path::Path}, }; +#[derive(Debug, Clone)] +pub struct PodRequests { + requests: BTreeMap, +} + +impl PodRequests { + pub fn new(cpu_requests: String, memory_requests: String) -> PodRequests { + PodRequests { + requests: vec![ + ("cpu".to_string(), Quantity(cpu_requests)), + ("memory".to_string(), Quantity(memory_requests)), + ] + .into_iter() + .collect(), + } + } +} + pub struct Kubernetes<'a> { k8s_client: Client, namespace: String, validator_config: &'a mut ValidatorConfig, + pod_requests: PodRequests, } impl<'a> Kubernetes<'a> { - pub async fn new(namespace: &str, validator_config: &'a mut ValidatorConfig) -> Kubernetes<'a> { + pub async fn new(namespace: &str, validator_config: &'a mut ValidatorConfig, pod_requests: PodRequests +) -> Kubernetes<'a> { Self { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), validator_config, + pod_requests, } } diff --git a/src/main.rs b/src/main.rs index 05a2736..a37923b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use { validator_lab::{ docker::{DockerConfig, DockerImage}, genesis::{Genesis, GenesisFlags, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL}, - kubernetes::Kubernetes, + kubernetes::{Kubernetes, PodRequests}, release::{BuildConfig, BuildType, DeployMethod}, SolanaRoot, ValidatorType, validator_config::ValidatorConfig, @@ -193,6 +193,25 @@ fn parse_matches() -> clap::ArgMatches { .long("full-rpc") .help("Validator config. Support full RPC services on all nodes"), ) + // kubernetes config + .arg( + Arg::with_name("cpu_requests") + .long("cpu-requests") + .takes_value(true) + .default_value("20") // 20 cores + .help("Kubernetes pod config. Specify minimum CPUs required for deploying validator. + can use millicore notation as well. e.g. 500m (500 millicores) == 0.5 and is equivalent to half a core. + [default: 20]"), + ) + .arg( + Arg::with_name("memory_requests") + .long("memory-requests") + .takes_value(true) + .default_value("70Gi") // 70 Gigabytes + .help("Kubernetes pod config. Specify minimum memory required for deploying validator. + Can specify unit here (B, Ki, Mi, Gi, Ti) for bytes, kilobytes, etc (2^N notation) + e.g. 1Gi == 1024Mi == 1024Ki == 1,047,576B. [default: 70Gi]"), + ) .get_matches() } @@ -334,7 +353,12 @@ async fn main() { known_validators: None, }; - let kub_controller = Kubernetes::new(environment_config.namespace, &mut validator_config).await; + let pod_requests = PodRequests::new( + matches.value_of("cpu_requests").unwrap().to_string(), + matches.value_of("memory_requests").unwrap().to_string(), + ); + + let kub_controller = Kubernetes::new(environment_config.namespace, &mut validator_config, pod_requests).await; match kub_controller.namespace_exists().await { Ok(true) => (), Ok(false) => { From 80c74969df8b9c381ab7c1c12a113c031c8c7251 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 1 Apr 2024 23:09:52 +0000 Subject: [PATCH 10/26] create bootstrap validator replicaset --- src/k8s_helpers.rs | 12 +++---- src/kubernetes.rs | 78 +++++++++++++++++++++++++++++++++-------- src/main.rs | 40 ++++++++++++--------- src/validator_config.rs | 15 ++------ 4 files changed, 93 insertions(+), 52 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index d45196e..b35298f 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -1,12 +1,11 @@ use { - crate::ValidatorType, + crate::{docker::DockerImage, ValidatorType}, k8s_openapi::{ api::{ apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ - Container, EnvVar, PodSecurityContext, - PodSpec, PodTemplateSpec, Probe, ResourceRequirements, Secret, - Volume, VolumeMount, + Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, + ResourceRequirements, Secret, Volume, VolumeMount, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -52,8 +51,7 @@ pub fn create_replica_set( name: &ValidatorType, namespace: &str, label_selector: &BTreeMap, - container_name: &str, - image_name: &str, + image_name: &DockerImage, environment_variables: Vec, command: &[String], volumes: Option>, @@ -68,7 +66,7 @@ pub fn create_replica_set( }), spec: Some(PodSpec { containers: vec![Container { - name: container_name.to_string(), + name: format!("{}-{}", image_name.validator_type(), "container"), image: Some(image_name.to_string()), image_pull_policy: Some("Always".to_string()), env: Some(environment_variables), diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 7065079..cc94995 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,11 +1,11 @@ use { - crate::{k8s_helpers, ValidatorType, validator_config::ValidatorConfig}, + crate::{docker::DockerImage, k8s_helpers, validator_config::ValidatorConfig, ValidatorType}, k8s_openapi::{ api::{ apps::v1::ReplicaSet, core::v1::{ - EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, - Secret, SecretVolumeSource, Volume, VolumeMount, + EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, + Volume, VolumeMount, }, }, apimachinery::pkg::api::resource::Quantity, @@ -15,6 +15,7 @@ use { Client, }, log::*, + solana_sdk::{pubkey::Pubkey, signature::keypair::read_keypair_file, signer::Signer}, std::{collections::BTreeMap, error::Error, path::Path}, }; @@ -44,8 +45,11 @@ pub struct Kubernetes<'a> { } impl<'a> Kubernetes<'a> { - pub async fn new(namespace: &str, validator_config: &'a mut ValidatorConfig, pod_requests: PodRequests -) -> Kubernetes<'a> { + pub async fn new( + namespace: &str, + validator_config: &'a mut ValidatorConfig, + pod_requests: PodRequests, + ) -> Kubernetes<'a> { Self { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), @@ -67,7 +71,7 @@ impl<'a> Kubernetes<'a> { } pub fn create_bootstrap_secret( - &self, + &mut self, secret_name: &str, config_dir: &Path, ) -> Result> { @@ -76,6 +80,12 @@ impl<'a> Kubernetes<'a> { let vote_key_path = config_dir.join("bootstrap-validator/vote-account.json"); let stake_key_path = config_dir.join("bootstrap-validator/stake-account.json"); + let bootstrap_keypair = read_keypair_file(identity_key_path.clone()) + .expect("Failed to read bootstrap validator keypair file"); + + //TODO: need to fix and not read the json path twice + self.add_known_validator(bootstrap_keypair.pubkey()); + let key_files = vec![ (faucet_key_path, "faucet"), (identity_key_path, "identity"), @@ -86,6 +96,17 @@ impl<'a> Kubernetes<'a> { k8s_helpers::create_secret_from_files(secret_name, &key_files) } + fn add_known_validator(&mut self, pubkey: Pubkey) { + if let Some(ref mut known_validators) = self.validator_config.known_validators { + known_validators.push(pubkey); + } else { + let new_known_validators = vec![pubkey]; + self.validator_config.known_validators = Some(new_known_validators); + } + + info!("pubkey added to known validators: {:?}", pubkey); + } + pub async fn deploy_secret(&self, secret: &Secret) -> Result { let secrets_api: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); @@ -94,12 +115,11 @@ impl<'a> Kubernetes<'a> { pub fn create_bootstrap_validator_replica_set( &mut self, - container_name: &str, - image_name: &str, + image_name: &DockerImage, secret_name: Option, label_selector: &BTreeMap, ) -> Result> { - let mut env_vars = vec![EnvVar { + let env_vars = vec![EnvVar { name: "MY_POD_IP".to_string(), value_from: Some(EnvVarSource { field_ref: Some(ObjectFieldSelector { @@ -111,7 +131,6 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]; - let accounts_volume = Some(vec![Volume { name: "bootstrap-accounts-volume".into(), secret: Some(SecretVolumeSource { @@ -131,15 +150,10 @@ impl<'a> Kubernetes<'a> { vec!["/home/solana/k8s-cluster-scripts/bootstrap-startup-script.sh".to_string()]; command.extend(self.generate_bootstrap_command_flags()); - for c in command.iter() { - debug!("bootstrap command: {}", c); - } - k8s_helpers::create_replica_set( &ValidatorType::Bootstrap, self.namespace.as_str(), label_selector, - container_name, image_name, env_vars, &command, @@ -150,6 +164,40 @@ impl<'a> Kubernetes<'a> { ) } + fn generate_command_flags(&self, flags: &mut Vec) { + if self.validator_config.tpu_enable_udp { + flags.push("--tpu-enable-udp".to_string()); + } + if self.validator_config.tpu_disable_quic { + flags.push("--tpu-disable-quic".to_string()); + } + if self.validator_config.skip_poh_verify { + flags.push("--skip-poh-verify".to_string()); + } + if self.validator_config.no_snapshot_fetch { + flags.push("--no-snapshot-fetch".to_string()); + } + if self.validator_config.require_tower { + flags.push("--require-tower".to_string()); + } + if self.validator_config.enable_full_rpc { + flags.push("--enable-rpc-transaction-history".to_string()); + flags.push("--enable-extended-tx-metadata-storage".to_string()); + } + + if let Some(limit_ledger_size) = self.validator_config.max_ledger_size { + flags.push("--limit-ledger-size".to_string()); + flags.push(limit_ledger_size.to_string()); + } + } + + fn generate_bootstrap_command_flags(&self) -> Vec { + let mut flags: Vec = Vec::new(); + self.generate_command_flags(&mut flags); + + flags + } + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } diff --git a/src/main.rs b/src/main.rs index a37923b..d1c22e7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,19 @@ use { - clap::{command, Arg, ArgGroup}, + clap::{command, value_t_or_exit, Arg, ArgGroup}, log::*, - solana_ledger::blockstore_cleanup_service::{DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS}, + solana_ledger::blockstore_cleanup_service::{ + DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS, + }, solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, std::fs, strum::VariantNames, validator_lab::{ docker::{DockerConfig, DockerImage}, - genesis::{Genesis, GenesisFlags, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL}, + genesis::{Genesis, GenesisFlags}, kubernetes::{Kubernetes, PodRequests}, release::{BuildConfig, BuildType, DeployMethod}, - SolanaRoot, ValidatorType, validator_config::ValidatorConfig, + SolanaRoot, ValidatorType, }, }; @@ -189,7 +191,7 @@ fn parse_matches() -> clap::ArgMatches { Off by default since validator won't restart if the pod restarts"), ) .arg( - Arg::with_name("full_rpc") + Arg::with_name("enable_full_rpc") .long("full-rpc") .help("Validator config. Support full RPC services on all nodes"), ) @@ -328,8 +330,6 @@ async fn main() { let mut validator_config = ValidatorConfig { tpu_enable_udp: matches.is_present("tpu_enable_udp"), tpu_disable_quic: matches.is_present("tpu_disable_quic"), - shred_version: None, // set after genesis created - bank_hash: None, // set after genesis created max_ledger_size: if matches.is_present("limit_ledger_size") { let limit_ledger_size = match matches.value_of("limit_ledger_size") { Some(_) => value_t_or_exit!(matches, "limit_ledger_size", u64), @@ -349,7 +349,6 @@ async fn main() { no_snapshot_fetch: matches.is_present("no_snapshot_fetch"), require_tower: matches.is_present("require_tower"), enable_full_rpc: matches.is_present("enable_full_rpc"), - entrypoints: Vec::new(), known_validators: None, }; @@ -358,7 +357,12 @@ async fn main() { matches.value_of("memory_requests").unwrap().to_string(), ); - let kub_controller = Kubernetes::new(environment_config.namespace, &mut validator_config, pod_requests).await; + let mut kub_controller = Kubernetes::new( + environment_config.namespace, + &mut validator_config, + pod_requests, + ) + .await; match kub_controller.namespace_exists().await { Ok(true) => (), Ok(false) => { @@ -420,7 +424,7 @@ async fn main() { ); let validator_type = ValidatorType::Bootstrap; - let docker_image = DockerImage::new( + let bootstrap_docker_image = DockerImage::new( matches.value_of("registry_name").unwrap().to_string(), validator_type, matches.value_of("image_name").unwrap().to_string(), @@ -431,18 +435,21 @@ async fn main() { ); if build_config.docker_build() { - match docker.build_image(solana_root.get_root_path(), &docker_image) { - Ok(_) => info!("{} image built successfully", docker_image.validator_type()), + match docker.build_image(solana_root.get_root_path(), &bootstrap_docker_image) { + Ok(_) => info!( + "{} image built successfully", + bootstrap_docker_image.validator_type() + ), Err(err) => { error!("Exiting........ {err}"); return; } } - match DockerConfig::push_image(&docker_image) { + match DockerConfig::push_image(&bootstrap_docker_image) { Ok(_) => info!( "{} image pushed successfully", - docker_image.validator_type() + bootstrap_docker_image.validator_type() ), Err(err) => { error!("Error. Failed to build imge: {err}"); @@ -487,9 +494,8 @@ async fn main() { bootstrap_keypair.pubkey().to_string(), ); - let bootstrap_replica_set = match kub_controller.create_bootstrap_validator_replica_set( - bootstrap_container_name, - bootstrap_image_name, + let _bootstrap_replica_set = match kub_controller.create_bootstrap_validator_replica_set( + &bootstrap_docker_image, bootstrap_secret.metadata.name.clone(), &bootstrap_rs_labels, ) { diff --git a/src/validator_config.rs b/src/validator_config.rs index b60776b..35f4a4b 100644 --- a/src/validator_config.rs +++ b/src/validator_config.rs @@ -1,18 +1,13 @@ -use solana_sdk::{ - hash::Hash, pubkey::Pubkey, -}; +use solana_sdk::pubkey::Pubkey; pub struct ValidatorConfig { pub tpu_enable_udp: bool, pub tpu_disable_quic: bool, - pub shred_version: Option, - pub bank_hash: Option, pub max_ledger_size: Option, pub skip_poh_verify: bool, pub no_snapshot_fetch: bool, pub require_tower: bool, pub enable_full_rpc: bool, - pub entrypoints: Vec, pub known_validators: Option>, } @@ -31,26 +26,20 @@ impl std::fmt::Display for ValidatorConfig { "Runtime Config\n\ tpu_enable_udp: {}\n\ tpu_disable_quic: {}\n\ - shred_version: {:?}\n\ - bank_hash: {:?}\n\ max_ledger_size: {:?}\n\ skip_poh_verify: {}\n\ no_snapshot_fetch: {}\n\ require_tower: {}\n\ enable_full_rpc: {}\n\ - entrypoints: {:?}\n\ known_validators: {:?}", self.tpu_enable_udp, self.tpu_disable_quic, - self.shred_version, - self.bank_hash, self.max_ledger_size, self.skip_poh_verify, self.no_snapshot_fetch, self.require_tower, self.enable_full_rpc, - self.entrypoints.join(", "), known_validators, ) } -} \ No newline at end of file +} From 4fb7b4e638b6e7739fe348c6adcc24f44f88db6a Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 00:36:57 +0000 Subject: [PATCH 11/26] refactor. add validator struct to clean up main --- src/k8s_helpers.rs | 6 -- src/kubernetes.rs | 4 -- src/lib.rs | 1 + src/main.rs | 137 ++++++++++++++++++++++----------------------- src/validator.rs | 61 ++++++++++++++++++++ 5 files changed, 130 insertions(+), 79 deletions(-) create mode 100644 src/validator.rs diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index b35298f..fdec8ff 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -40,12 +40,6 @@ pub fn create_secret_from_files( Ok(create_secret(secret_name, data)) } -pub fn create_selector(key: &str, value: &str) -> BTreeMap { - let mut btree = BTreeMap::new(); - btree.insert(key.to_string(), value.to_string()); - btree -} - #[allow(clippy::too_many_arguments)] pub fn create_replica_set( name: &ValidatorType, diff --git a/src/kubernetes.rs b/src/kubernetes.rs index cc94995..999c077 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -197,8 +197,4 @@ impl<'a> Kubernetes<'a> { flags } - - pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { - k8s_helpers::create_selector(key, value) - } } diff --git a/src/lib.rs b/src/lib.rs index aaa9491..d3f6dcf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,6 +59,7 @@ pub mod genesis; pub mod k8s_helpers; pub mod kubernetes; pub mod release; +pub mod validator; pub mod validator_config; static BUILD: Emoji = Emoji("👷 ", ""); diff --git a/src/main.rs b/src/main.rs index d1c22e7..bed6958 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ use { genesis::{Genesis, GenesisFlags}, kubernetes::{Kubernetes, PodRequests}, release::{BuildConfig, BuildType, DeployMethod}, + validator::Validator, validator_config::ValidatorConfig, SolanaRoot, ValidatorType, }, @@ -423,86 +424,84 @@ async fn main() { deploy_method, ); - let validator_type = ValidatorType::Bootstrap; - let bootstrap_docker_image = DockerImage::new( - matches.value_of("registry_name").unwrap().to_string(), - validator_type, - matches.value_of("image_name").unwrap().to_string(), - matches - .value_of("image_tag") - .unwrap_or_default() - .to_string(), - ); + let registry = matches.value_of("registry_name").unwrap().to_string(); + let image_name = matches.value_of("image_name").unwrap().to_string(); + let tag = matches + .value_of("image_tag") + .unwrap_or_default() + .to_string(); + + // This will always be Some() right now. But will won't be in future when we implement + // heterogenous clusters + let bootstrap_validator = Some(Validator::new(DockerImage::new( + registry.clone(), + ValidatorType::Bootstrap, + image_name.clone(), + tag.clone(), + ))); if build_config.docker_build() { - match docker.build_image(solana_root.get_root_path(), &bootstrap_docker_image) { - Ok(_) => info!( - "{} image built successfully", - bootstrap_docker_image.validator_type() - ), + let validators = vec![&bootstrap_validator]; + for v in validators.into_iter().flatten() { + match docker.build_image(solana_root.get_root_path(), v.image()) { + Ok(_) => info!("{} image built successfully", v.validator_type()), + Err(err) => { + error!("Failed to build docker image {err}"); + return; + } + } + + match DockerConfig::push_image(v.image()) { + Ok(_) => info!("{} image pushed successfully", v.validator_type()), + Err(err) => { + error!("Failed to push docker image {err}"); + return; + } + } + } + } + + if let Some(mut bootstrap_validator) = bootstrap_validator { + match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) + { + Ok(secret) => bootstrap_validator.set_secret(secret), Err(err) => { - error!("Exiting........ {err}"); + error!("Failed to create bootstrap secret! {err}"); return; } - } + }; - match DockerConfig::push_image(&bootstrap_docker_image) { - Ok(_) => info!( - "{} image pushed successfully", - bootstrap_docker_image.validator_type() - ), + match kub_controller + .deploy_secret(bootstrap_validator.secret()) + .await + { + Ok(_) => info!("Deployed Bootstrap Secret"), Err(err) => { - error!("Error. Failed to build imge: {err}"); + error!("{err}"); return; } } - } - let bootstrap_secret = match kub_controller - .create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) - { - Ok(secret) => secret, - Err(err) => { - error!("Failed to create bootstrap secret! {}", err); - return; - } - }; + // Create bootstrap labels + let identity_path = config_directory.join("bootstrap-validator/identity.json"); + let bootstrap_keypair = + read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); + bootstrap_validator.add_label("validator/lb", "load-balancer-selector"); + bootstrap_validator.add_label("validator/name", "bootstrap-validator-selector"); + bootstrap_validator.add_label("validator/type", "bootstrap"); + bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string()); - match kub_controller.deploy_secret(&bootstrap_secret).await { - Ok(_) => info!("Deployed Bootstrap Secret"), - Err(err) => { - error!("{}", err); - return; - } + // create bootstrap replica set + match kub_controller.create_bootstrap_validator_replica_set( + bootstrap_validator.image(), + bootstrap_validator.secret().metadata.name.clone(), + bootstrap_validator.labels(), + ) { + Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), + Err(err) => { + error!("Error creating bootstrap validator replicas_set: {err}"); + return; + } + }; } - - // Bootstrap needs two labels. Because it is going to have two services. - // One via Load Balancer, one direct - let mut bootstrap_rs_labels = - kub_controller.create_selector("validator/lb", "load-balancer-selector"); - bootstrap_rs_labels.insert( - "validator/name".to_string(), - "bootstrap-validator-selector".to_string(), - ); - bootstrap_rs_labels.insert("validator/type".to_string(), "bootstrap".to_string()); - - let identity_path = config_directory.join("bootstrap-validator/identity.json"); - let bootstrap_keypair = - read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); - bootstrap_rs_labels.insert( - "validator/identity".to_string(), - bootstrap_keypair.pubkey().to_string(), - ); - - let _bootstrap_replica_set = match kub_controller.create_bootstrap_validator_replica_set( - &bootstrap_docker_image, - bootstrap_secret.metadata.name.clone(), - &bootstrap_rs_labels, - ) { - Ok(replica_set) => replica_set, - Err(err) => { - error!("Error creating bootstrap validator replicas_set: {}", err); - return; - } - }; } diff --git a/src/validator.rs b/src/validator.rs new file mode 100644 index 0000000..c643a10 --- /dev/null +++ b/src/validator.rs @@ -0,0 +1,61 @@ +use { + crate::{docker::DockerImage, ValidatorType}, + k8s_openapi::api::{apps::v1::ReplicaSet, core::v1::Secret}, + std::{collections::BTreeMap, string::String}, +}; + +pub struct Validator { + validator_type: ValidatorType, + image: DockerImage, + secret: Secret, + replica_set_labels: BTreeMap, + replica_set: ReplicaSet, +} + +impl Validator { + pub fn new(image: DockerImage) -> Self { + Self { + validator_type: image.validator_type(), + image, + secret: Secret::default(), + replica_set_labels: BTreeMap::new(), + replica_set: ReplicaSet::default(), + } + } + + pub fn image(&self) -> &DockerImage { + &self.image + } + + pub fn secret(&self) -> &Secret { + &self.secret + } + + pub fn validator_type(&self) -> &ValidatorType { + &self.validator_type + } + + pub fn add_label(&mut self, key: K, value: V) + where + K: Into, + V: Into, + { + self.replica_set_labels.insert(key.into(), value.into()); + } + + pub fn labels(&self) -> &BTreeMap { + &self.replica_set_labels + } + + pub fn set_secret(&mut self, secret: Secret) { + self.secret = secret; + } + + pub fn set_replica_set(&mut self, replica_set: ReplicaSet) { + self.replica_set = replica_set; + } + + pub fn replica_set(&self) -> &ReplicaSet { + &self.replica_set + } +} From 573248673cbd41fdd691e75304e71feada3b310e Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 00:36:57 +0000 Subject: [PATCH 12/26] refactor. add validator struct to clean up main --- src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index bed6958..8577b30 100644 --- a/src/main.rs +++ b/src/main.rs @@ -489,7 +489,6 @@ async fn main() { bootstrap_validator.add_label("validator/lb", "load-balancer-selector"); bootstrap_validator.add_label("validator/name", "bootstrap-validator-selector"); bootstrap_validator.add_label("validator/type", "bootstrap"); - bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string()); // create bootstrap replica set match kub_controller.create_bootstrap_validator_replica_set( From 1dad6c9ba8d82078cc3e7babfb9b4e588d0a4272 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 01:08:46 +0000 Subject: [PATCH 13/26] deploy bootstrap validator --- PROGRESS.md | 2 +- src/docker.rs | 2 +- src/kubernetes.rs | 11 +++++++++++ src/main.rs | 18 +++++++++++++++++- src/scripts/bootstrap-startup-script.sh | 0 src/scripts/common.sh | 0 src/validator.rs | 4 ++++ 7 files changed, 34 insertions(+), 3 deletions(-) mode change 100644 => 100755 src/scripts/bootstrap-startup-script.sh mode change 100644 => 100755 src/scripts/common.sh diff --git a/PROGRESS.md b/PROGRESS.md index 8db6e91..45e078f 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -29,7 +29,7 @@ - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Replica Set - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client diff --git a/src/docker.rs b/src/docker.rs index 22bb244..6a7253e 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -184,7 +184,7 @@ USER solana RUN mkdir -p /home/solana/k8s-cluster-scripts # TODO: this needs to be changed for non bootstrap, this should be ./src/scripts/-startup-scripts.sh -COPY {startup_script_directory}/bootstrap-startup-script.sh /home/solana/k8s-cluster-scripts +COPY {startup_script_directory} /home/solana/k8s-cluster-scripts RUN mkdir -p /home/solana/ledger COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 999c077..13a3f42 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -197,4 +197,15 @@ impl<'a> Kubernetes<'a> { flags } + + pub async fn deploy_replicas_set( + &self, + replica_set: &ReplicaSet, + ) -> Result { + let api: Api = + Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let post_params = PostParams::default(); + // Apply the ReplicaSet + api.create(&post_params, replica_set).await + } } diff --git a/src/main.rs b/src/main.rs index 8577b30..782dc1c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -374,7 +374,7 @@ async fn main() { return; } Err(err) => { - error!("Error: {}", err); + error!("Error: {err}"); return; } } @@ -502,5 +502,21 @@ async fn main() { return; } }; + + match kub_controller + .deploy_replicas_set(bootstrap_validator.replica_set()) + .await + { + Ok(_) => { + info!( + "{} deployed successfully", + bootstrap_validator.replica_set_name() + ); + } + Err(err) => { + error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); + return; + } + }; } } diff --git a/src/scripts/bootstrap-startup-script.sh b/src/scripts/bootstrap-startup-script.sh old mode 100644 new mode 100755 diff --git a/src/scripts/common.sh b/src/scripts/common.sh old mode 100644 new mode 100755 diff --git a/src/validator.rs b/src/validator.rs index c643a10..c05bb2c 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -58,4 +58,8 @@ impl Validator { pub fn replica_set(&self) -> &ReplicaSet { &self.replica_set } + + pub fn replica_set_name(&self) -> &String { + self.replica_set.metadata.name.as_ref().unwrap() + } } From f29654c443ee375e747a07bae1edb09446d6ce9f Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 8 Apr 2024 20:51:50 +0000 Subject: [PATCH 14/26] rebase for updated validator struct --- src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.rs b/src/main.rs index 782dc1c..e9dcab1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -489,6 +489,7 @@ async fn main() { bootstrap_validator.add_label("validator/lb", "load-balancer-selector"); bootstrap_validator.add_label("validator/name", "bootstrap-validator-selector"); bootstrap_validator.add_label("validator/type", "bootstrap"); + bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string()); // create bootstrap replica set match kub_controller.create_bootstrap_validator_replica_set( From 1e2315056bf06207ca469d3262cd189f3c590388 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 01:29:37 +0000 Subject: [PATCH 15/26] wip. added bootstrap service. need lb service --- src/k8s_helpers.rs | 50 +++++++++++++++++++- src/kubernetes.rs | 19 +++++++- src/main.rs | 111 +++++++++++++++++++++++++++++++-------------- src/validator.rs | 24 ++++++++-- 4 files changed, 165 insertions(+), 39 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index fdec8ff..9cd6a23 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -5,7 +5,8 @@ use { apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, Secret, Volume, VolumeMount, + ResourceRequirements, Secret, Volume, VolumeMount, Service, ServiceSpec, + ServicePort, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -103,3 +104,50 @@ pub fn create_replica_set( ..Default::default() }) } + +pub fn create_service( + service_name: &str, + namespace: &str, + label_selector: &BTreeMap, + is_load_balancer: bool, +) -> Service { + Service { + metadata: ObjectMeta { + name: Some(service_name.to_string()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: Some(ServiceSpec { + selector: Some(label_selector.clone()), + type_: if is_load_balancer { + Some("LoadBalancer".to_string()) + } else { + None + }, + cluster_ip: if is_load_balancer { + None + } else { + Some("None".to_string()) + }, + ports: Some(vec![ + ServicePort { + port: 8899, // RPC Port + name: Some("rpc-port".to_string()), + ..Default::default() + }, + ServicePort { + port: 8001, //Gossip Port + name: Some("gossip-port".to_string()), + ..Default::default() + }, + ServicePort { + port: 9900, //Faucet Port + name: Some("faucet-port".to_string()), + ..Default::default() + }, + ]), + ..Default::default() + }), + ..Default::default() + } +} \ No newline at end of file diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 13a3f42..66eadca 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -5,7 +5,7 @@ use { apps::v1::ReplicaSet, core::v1::{ EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, - Volume, VolumeMount, + Volume, VolumeMount, Service, }, }, apimachinery::pkg::api::resource::Quantity, @@ -208,4 +208,21 @@ impl<'a> Kubernetes<'a> { // Apply the ReplicaSet api.create(&post_params, replica_set).await } + + pub fn create_bootstrap_service( + &self, + service_name: &str, + label_selector: &BTreeMap, + ) -> Service { + k8s_helpers::create_service(service_name, self.namespace.as_str(), label_selector, false) + } + + pub async fn deploy_service(&self, service: &Service) -> Result { + let post_params = PostParams::default(); + // Create an API instance for Services in the specified namespace + let service_api: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + + // Create the Service object in the cluster + service_api.create(&post_params, service).await + } } diff --git a/src/main.rs b/src/main.rs index e9dcab1..52e94f2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,7 @@ use { genesis::{Genesis, GenesisFlags}, kubernetes::{Kubernetes, PodRequests}, release::{BuildConfig, BuildType, DeployMethod}, - validator::Validator, + validator::{Validator, LabelType}, validator_config::ValidatorConfig, SolanaRoot, ValidatorType, }, @@ -482,42 +482,85 @@ async fn main() { } } - // Create bootstrap labels - let identity_path = config_directory.join("bootstrap-validator/identity.json"); - let bootstrap_keypair = - read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); - bootstrap_validator.add_label("validator/lb", "load-balancer-selector"); - bootstrap_validator.add_label("validator/name", "bootstrap-validator-selector"); - bootstrap_validator.add_label("validator/type", "bootstrap"); - bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string()); - - // create bootstrap replica set - match kub_controller.create_bootstrap_validator_replica_set( - bootstrap_validator.image(), - bootstrap_validator.secret().metadata.name.clone(), - bootstrap_validator.labels(), - ) { - Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), - Err(err) => { - error!("Error creating bootstrap validator replicas_set: {err}"); - return; - } - }; + // Create bootstrap labels + let identity_path = config_directory.join("bootstrap-validator/identity.json"); + let bootstrap_keypair = + read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); + bootstrap_validator.add_label("validator/lb", "load-balancer-selector", LabelType::replica_set); + bootstrap_validator.add_label("validator/name", "bootstrap-validator-selector", LabelType::replica_set); + bootstrap_validator.add_label("validator/type", "bootstrap", LabelType::replica_set); + bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string(), LabelType::replica_set); + + // create bootstrap replica set + match kub_controller.create_bootstrap_validator_replica_set( + bootstrap_validator.image(), + bootstrap_validator.secret().metadata.name.clone(), + bootstrap_validator.replica_set_labels(), + ) { + Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), + Err(err) => { + error!("Error creating bootstrap validator replicas_set: {err}"); + return; + } + }; + + match kub_controller + .deploy_replicas_set(bootstrap_validator.replica_set()) + .await + { + Ok(_) => { + info!( + "{} deployed successfully", + bootstrap_validator.replica_set_name() + ); + } + Err(err) => { + error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); + return; + } + }; + bootstrap_validator.add_label("service/name", "bootstrap-validator-selector", LabelType::replica_set); + + let bootstrap_service = kub_controller + .create_bootstrap_service("bootstrap-validator-service", bootstrap_validator.service_labels()); + match kub_controller.deploy_service(&bootstrap_service).await { + Ok(_) => info!("bootstrap validator service deployed successfully"), + Err(err) => error!( + "Error! Failed to deploy bootstrap validator service. err: {:?}", + err + ), + } + + //load balancer service. only create one and use for all deployments + let load_balancer_label = + kub_controller.create_selector("app.kubernetes.io/lb", "load-balancer-selector"); + //create load balancer + let load_balancer = kub_controller.create_validator_load_balancer( + "bootstrap-and-non-voting-lb-service", + &load_balancer_label, + ); + + //deploy load balancer + match kub_controller.deploy_service(&load_balancer).await { + Ok(_) => info!("load balancer service deployed successfully"), + Err(err) => error!( + "Error! Failed to deploy load balancer service. err: {:?}", + err + ), + } + + // wait for bootstrap replicaset to deploy + while { match kub_controller - .deploy_replicas_set(bootstrap_validator.replica_set()) + .check_replica_set_ready(bootstrap_replica_set_name.as_str()) .await { - Ok(_) => { - info!( - "{} deployed successfully", - bootstrap_validator.replica_set_name() - ); - } - Err(err) => { - error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); - return; - } - }; + Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) + Err(_) => panic!("Error occurred while checking replica set readiness"), + } + } { + info!("replica set: {} not ready...", bootstrap_replica_set_name); + thread::sleep(Duration::from_secs(1)); } } diff --git a/src/validator.rs b/src/validator.rs index c05bb2c..56a5bda 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -4,12 +4,18 @@ use { std::{collections::BTreeMap, string::String}, }; +pub enum LabelType { + replica_set, + service, +} + pub struct Validator { validator_type: ValidatorType, image: DockerImage, secret: Secret, replica_set_labels: BTreeMap, replica_set: ReplicaSet, + service_labels: BTreeMap, } impl Validator { @@ -20,6 +26,7 @@ impl Validator { secret: Secret::default(), replica_set_labels: BTreeMap::new(), replica_set: ReplicaSet::default(), + service_labels: BTreeMap::new(), } } @@ -35,18 +42,29 @@ impl Validator { &self.validator_type } - pub fn add_label(&mut self, key: K, value: V) + pub fn add_label(&mut self, key: K, value: V, label_type: LabelType) where K: Into, V: Into, { - self.replica_set_labels.insert(key.into(), value.into()); + match label_type { + LabelType::replica_set => { + self.replica_set_labels.insert(key.into(), value.into()); + } + LabelType::service => { + self.service_labels.insert(key.into(), value.into()); + } + } } - pub fn labels(&self) -> &BTreeMap { + pub fn replica_set_labels(&self) -> &BTreeMap { &self.replica_set_labels } + pub fn service_labels(&self) -> &BTreeMap { + &self.service_labels + } + pub fn set_secret(&mut self, secret: Secret) { self.secret = secret; } From 02b28499bbdbe2b342cf3bf7305f0e8e68c20312 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 17:15:26 +0000 Subject: [PATCH 16/26] deploy and wait for validator ready. need readme update --- src/k8s_helpers.rs | 6 ++++++ src/kubernetes.rs | 30 ++++++++++++++++++++++++++++++ src/main.rs | 17 +++++++++-------- src/validator.rs | 16 ++++++++++++---- 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 9cd6a23..593aa3b 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -150,4 +150,10 @@ pub fn create_service( }), ..Default::default() } +} + +pub fn create_selector(key: &str, value: &str) -> BTreeMap { + let mut btree = BTreeMap::new(); + btree.insert(key.to_string(), value.to_string()); + btree } \ No newline at end of file diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 66eadca..c244b15 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -225,4 +225,34 @@ impl<'a> Kubernetes<'a> { // Create the Service object in the cluster service_api.create(&post_params, service).await } + + pub fn create_validator_load_balancer( + &self, + service_name: &str, + label_selector: &BTreeMap, + ) -> Service { + k8s_helpers::create_service(service_name, self.namespace.as_str(), label_selector, true) + } + + pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { + k8s_helpers::create_selector(key, value) + } + + pub async fn check_replica_set_ready( + &self, + replica_set_name: &str, + ) -> Result { + let replica_sets: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let replica_set = replica_sets.get(replica_set_name).await?; + + let desired_validators = replica_set.spec.as_ref().unwrap().replicas.unwrap_or(1); + let available_validators = replica_set + .status + .as_ref() + .unwrap() + .available_replicas + .unwrap_or(0); + + Ok(available_validators >= desired_validators) + } } diff --git a/src/main.rs b/src/main.rs index 52e94f2..25407d4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ use { validator_config::ValidatorConfig, SolanaRoot, ValidatorType, }, + std::{thread, time::Duration}, }; fn parse_matches() -> clap::ArgMatches { @@ -486,10 +487,10 @@ async fn main() { let identity_path = config_directory.join("bootstrap-validator/identity.json"); let bootstrap_keypair = read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); - bootstrap_validator.add_label("validator/lb", "load-balancer-selector", LabelType::replica_set); - bootstrap_validator.add_label("validator/name", "bootstrap-validator-selector", LabelType::replica_set); - bootstrap_validator.add_label("validator/type", "bootstrap", LabelType::replica_set); - bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string(), LabelType::replica_set); + bootstrap_validator.add_label("load-balancer/name", "load-balancer-selector", LabelType::ValidatorReplicaSet); + bootstrap_validator.add_label("service/name", "bootstrap-validator-selector", LabelType::ValidatorReplicaSet); + bootstrap_validator.add_label("validator/type", "bootstrap", LabelType::ValidatorReplicaSet); + bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string(), LabelType::ValidatorReplicaSet); // create bootstrap replica set match kub_controller.create_bootstrap_validator_replica_set( @@ -520,7 +521,7 @@ async fn main() { } }; - bootstrap_validator.add_label("service/name", "bootstrap-validator-selector", LabelType::replica_set); + bootstrap_validator.add_label("service/name", "bootstrap-validator-selector", LabelType::ValidatorReplicaSet); let bootstrap_service = kub_controller .create_bootstrap_service("bootstrap-validator-service", bootstrap_validator.service_labels()); @@ -534,7 +535,7 @@ async fn main() { //load balancer service. only create one and use for all deployments let load_balancer_label = - kub_controller.create_selector("app.kubernetes.io/lb", "load-balancer-selector"); + kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); //create load balancer let load_balancer = kub_controller.create_validator_load_balancer( "bootstrap-and-non-voting-lb-service", @@ -553,14 +554,14 @@ async fn main() { // wait for bootstrap replicaset to deploy while { match kub_controller - .check_replica_set_ready(bootstrap_replica_set_name.as_str()) + .check_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) .await { Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) Err(_) => panic!("Error occurred while checking replica set readiness"), } } { - info!("replica set: {} not ready...", bootstrap_replica_set_name); + info!("replica set: {} not ready...", bootstrap_validator.replica_set_name()); thread::sleep(Duration::from_secs(1)); } } diff --git a/src/validator.rs b/src/validator.rs index 56a5bda..4718d01 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -5,8 +5,9 @@ use { }; pub enum LabelType { - replica_set, - service, + ValidatorReplicaSet, + ValidatorService, + // ValidatorLoadBalancer, } pub struct Validator { @@ -16,10 +17,16 @@ pub struct Validator { replica_set_labels: BTreeMap, replica_set: ReplicaSet, service_labels: BTreeMap, + // load_balancer_labels: Option>, } impl Validator { pub fn new(image: DockerImage) -> Self { + // let load_balancer_labels = match image.validator_type() { + // ValidatorType::Bootstrap | ValidatorType::RPC => Some(BTreeMap::new()), + // ValidatorType::Client | ValidatorType::Standard => None, + // }; + Self { validator_type: image.validator_type(), image, @@ -27,6 +34,7 @@ impl Validator { replica_set_labels: BTreeMap::new(), replica_set: ReplicaSet::default(), service_labels: BTreeMap::new(), + // load_balancer_labels, } } @@ -48,10 +56,10 @@ impl Validator { V: Into, { match label_type { - LabelType::replica_set => { + LabelType::ValidatorReplicaSet => { self.replica_set_labels.insert(key.into(), value.into()); } - LabelType::service => { + LabelType::ValidatorService => { self.service_labels.insert(key.into(), value.into()); } } From 507888764ca8cfd3b9605075bc21b9b260a9a861 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 20:48:12 +0000 Subject: [PATCH 17/26] update readme. update progress. fix selector bug --- PROGRESS.md | 8 +- README.md | 31 ++++++++ src/k8s_helpers.rs | 6 +- src/kubernetes.rs | 8 +- src/main.rs | 177 ++++++++++++++++++++++++++------------------- src/validator.rs | 8 -- 6 files changed, 144 insertions(+), 94 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 45e078f..5503a9b 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -34,12 +34,12 @@ - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Services - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client -- [ ] Check Bootstrap is deployed and running -- [ ] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) +- [x] Check Bootstrap is deployed and running +- [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) - [ ] Add metrics - [ ] Bootstrap - [ ] Validator (regular) @@ -69,7 +69,7 @@ Above, we start with bootstrap, and then we do validators (regular), and then we - Use command line flags to set type of client, tx-count, etc - [ ] Add in kubernetes deployment flags - - [ ] CPU/Memory Requests + - [x] CPU/Memory Requests - [ ] Node Affinity -> Regions - [ ] Node Affinity -> Node Type (Equinix/Lumen) diff --git a/README.md b/README.md index b469538..c584bc8 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,35 @@ cargo run --bin cluster -- --tag # e.g. v1 --base-image # e.g. ubuntu:20.04 --image-name # e.g. cluster-image +``` + +## Kubernetes Cheatsheet +Create namespace: +``` +kubectl create ns +``` + +Delete namespace: +``` +kubectl delete ns +``` + +Get running pods: +``` +kubectl get pods -n +``` + +Get pod logs: +``` +kubectl logs -n +``` + +Exec into pod: +``` +kubectl exec -it -n -- /bin/bash +``` + +Get information about pod: +``` +kubectl describe pod -n ``` \ No newline at end of file diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 593aa3b..f508946 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -5,8 +5,8 @@ use { apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, Secret, Volume, VolumeMount, Service, ServiceSpec, - ServicePort, + ResourceRequirements, Secret, Service, ServicePort, ServiceSpec, Volume, + VolumeMount, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -156,4 +156,4 @@ pub fn create_selector(key: &str, value: &str) -> BTreeMap { let mut btree = BTreeMap::new(); btree.insert(key.to_string(), value.to_string()); btree -} \ No newline at end of file +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index c244b15..701035d 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -5,7 +5,7 @@ use { apps::v1::ReplicaSet, core::v1::{ EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, - Volume, VolumeMount, Service, + Service, Volume, VolumeMount, }, }, apimachinery::pkg::api::resource::Quantity, @@ -220,7 +220,8 @@ impl<'a> Kubernetes<'a> { pub async fn deploy_service(&self, service: &Service) -> Result { let post_params = PostParams::default(); // Create an API instance for Services in the specified namespace - let service_api: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let service_api: Api = + Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); // Create the Service object in the cluster service_api.create(&post_params, service).await @@ -242,7 +243,8 @@ impl<'a> Kubernetes<'a> { &self, replica_set_name: &str, ) -> Result { - let replica_sets: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let replica_sets: Api = + Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); let replica_set = replica_sets.get(replica_set_name).await?; let desired_validators = replica_set.spec.as_ref().unwrap().replicas.unwrap_or(1); diff --git a/src/main.rs b/src/main.rs index 25407d4..f546f3f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,18 +5,17 @@ use { DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS, }, solana_sdk::{signature::keypair::read_keypair_file, signer::Signer}, - std::fs, + std::{fs, thread, time::Duration}, strum::VariantNames, validator_lab::{ docker::{DockerConfig, DockerImage}, genesis::{Genesis, GenesisFlags}, kubernetes::{Kubernetes, PodRequests}, release::{BuildConfig, BuildType, DeployMethod}, - validator::{Validator, LabelType}, + validator::{LabelType, Validator}, validator_config::ValidatorConfig, SolanaRoot, ValidatorType, }, - std::{thread, time::Duration}, }; fn parse_matches() -> clap::ArgMatches { @@ -483,85 +482,111 @@ async fn main() { } } - // Create bootstrap labels - let identity_path = config_directory.join("bootstrap-validator/identity.json"); - let bootstrap_keypair = - read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); - bootstrap_validator.add_label("load-balancer/name", "load-balancer-selector", LabelType::ValidatorReplicaSet); - bootstrap_validator.add_label("service/name", "bootstrap-validator-selector", LabelType::ValidatorReplicaSet); - bootstrap_validator.add_label("validator/type", "bootstrap", LabelType::ValidatorReplicaSet); - bootstrap_validator.add_label("validator/identity", bootstrap_keypair.pubkey().to_string(), LabelType::ValidatorReplicaSet); - - // create bootstrap replica set - match kub_controller.create_bootstrap_validator_replica_set( - bootstrap_validator.image(), - bootstrap_validator.secret().metadata.name.clone(), - bootstrap_validator.replica_set_labels(), - ) { - Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), - Err(err) => { - error!("Error creating bootstrap validator replicas_set: {err}"); - return; - } - }; + // Create bootstrap labels + let identity_path = config_directory.join("bootstrap-validator/identity.json"); + let bootstrap_keypair = + read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); + bootstrap_validator.add_label( + "load-balancer/name", + "load-balancer-selector", + LabelType::ValidatorReplicaSet, + ); + bootstrap_validator.add_label( + "service/name", + "bootstrap-validator-selector", + LabelType::ValidatorReplicaSet, + ); + bootstrap_validator.add_label( + "validator/type", + "bootstrap", + LabelType::ValidatorReplicaSet, + ); + bootstrap_validator.add_label( + "validator/identity", + bootstrap_keypair.pubkey().to_string(), + LabelType::ValidatorReplicaSet, + ); - match kub_controller - .deploy_replicas_set(bootstrap_validator.replica_set()) - .await - { - Ok(_) => { - info!( - "{} deployed successfully", - bootstrap_validator.replica_set_name() - ); - } - Err(err) => { - error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); - return; - } - }; + // create bootstrap replica set + match kub_controller.create_bootstrap_validator_replica_set( + bootstrap_validator.image(), + bootstrap_validator.secret().metadata.name.clone(), + bootstrap_validator.replica_set_labels(), + ) { + Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), + Err(err) => { + error!("Error creating bootstrap validator replicas_set: {err}"); + return; + } + }; - bootstrap_validator.add_label("service/name", "bootstrap-validator-selector", LabelType::ValidatorReplicaSet); + match kub_controller + .deploy_replicas_set(bootstrap_validator.replica_set()) + .await + { + Ok(_) => { + info!( + "{} deployed successfully", + bootstrap_validator.replica_set_name() + ); + } + Err(err) => { + error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); + return; + } + }; - let bootstrap_service = kub_controller - .create_bootstrap_service("bootstrap-validator-service", bootstrap_validator.service_labels()); - match kub_controller.deploy_service(&bootstrap_service).await { - Ok(_) => info!("bootstrap validator service deployed successfully"), - Err(err) => error!( - "Error! Failed to deploy bootstrap validator service. err: {:?}", - err - ), - } + bootstrap_validator.add_label( + "service/name", + "bootstrap-validator-selector", + LabelType::ValidatorService, + ); - //load balancer service. only create one and use for all deployments - let load_balancer_label = - kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); - //create load balancer - let load_balancer = kub_controller.create_validator_load_balancer( - "bootstrap-and-non-voting-lb-service", - &load_balancer_label, - ); + let bootstrap_service = kub_controller.create_bootstrap_service( + "bootstrap-validator-service", + bootstrap_validator.service_labels(), + ); + match kub_controller.deploy_service(&bootstrap_service).await { + Ok(_) => info!("bootstrap validator service deployed successfully"), + Err(err) => error!( + "Error! Failed to deploy bootstrap validator service. err: {:?}", + err + ), + } - //deploy load balancer - match kub_controller.deploy_service(&load_balancer).await { - Ok(_) => info!("load balancer service deployed successfully"), - Err(err) => error!( - "Error! Failed to deploy load balancer service. err: {:?}", - err - ), - } + //load balancer service. only create one and use for all deployments + let load_balancer_label = + kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); + //create load balancer + let load_balancer = kub_controller.create_validator_load_balancer( + "bootstrap-and-non-voting-lb-service", + &load_balancer_label, + ); - // wait for bootstrap replicaset to deploy - while { - match kub_controller - .check_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) - .await - { - Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) - Err(_) => panic!("Error occurred while checking replica set readiness"), + //deploy load balancer + match kub_controller.deploy_service(&load_balancer).await { + Ok(_) => info!("load balancer service deployed successfully"), + Err(err) => error!( + "Error! Failed to deploy load balancer service. err: {:?}", + err + ), + } + + // wait for bootstrap replicaset to deploy + while { + match kub_controller + .check_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) + .await + { + Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) + Err(_) => panic!("Error occurred while checking replica set readiness"), + } + } { + info!( + "replica set: {} not ready...", + bootstrap_validator.replica_set_name() + ); + thread::sleep(Duration::from_secs(1)); } - } { - info!("replica set: {} not ready...", bootstrap_validator.replica_set_name()); - thread::sleep(Duration::from_secs(1)); } } diff --git a/src/validator.rs b/src/validator.rs index 4718d01..41d4164 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -7,7 +7,6 @@ use { pub enum LabelType { ValidatorReplicaSet, ValidatorService, - // ValidatorLoadBalancer, } pub struct Validator { @@ -17,16 +16,10 @@ pub struct Validator { replica_set_labels: BTreeMap, replica_set: ReplicaSet, service_labels: BTreeMap, - // load_balancer_labels: Option>, } impl Validator { pub fn new(image: DockerImage) -> Self { - // let load_balancer_labels = match image.validator_type() { - // ValidatorType::Bootstrap | ValidatorType::RPC => Some(BTreeMap::new()), - // ValidatorType::Client | ValidatorType::Standard => None, - // }; - Self { validator_type: image.validator_type(), image, @@ -34,7 +27,6 @@ impl Validator { replica_set_labels: BTreeMap::new(), replica_set: ReplicaSet::default(), service_labels: BTreeMap::new(), - // load_balancer_labels, } } From 1a6f5724e2a3b1d25c791c4530ed1610807aeab4 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 22:07:28 +0000 Subject: [PATCH 18/26] add metrics --- PROGRESS.md | 2 +- README.md | 17 +++ scripts/init-metrics.sh | 87 ++++++++++++ src/k8s_helpers.rs | 2 +- src/kubernetes.rs | 51 ++++++- src/lib.rs | 44 ++++++ src/main.rs | 291 ++++++++++++++++++++++++---------------- 7 files changed, 371 insertions(+), 123 deletions(-) create mode 100755 scripts/init-metrics.sh diff --git a/PROGRESS.md b/PROGRESS.md index 5503a9b..bad7044 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -41,7 +41,7 @@ - [x] Check Bootstrap is deployed and running - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) - [ ] Add metrics - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client diff --git a/README.md b/README.md index c584bc8..7b5c818 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,23 @@ cargo run --bin cluster -- --image-name # e.g. cluster-image ``` +## Metrics +1) Setup metrics database: +``` +cd scripts/ +./init-metrics -c +# enter password when promted +``` +2) add the following to your `cluster` command from above +``` +--metrics-host https://internal-metrics.solana.com # need the `https://` here +--metrics-port 8086 +--metrics-db # from (1) +--metrics-username # from (1) +--metrics-password # from (1) +``` + + ## Kubernetes Cheatsheet Create namespace: ``` diff --git a/scripts/init-metrics.sh b/scripts/init-metrics.sh new file mode 100755 index 0000000..89eae4f --- /dev/null +++ b/scripts/init-metrics.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -e + +here=$(dirname "$0") +# shellcheck source=net/common.sh +source "$here"/common.sh + +usage() { + exitcode=0 + if [[ -n "$1" ]]; then + exitcode=1 + echo "Error: $*" + fi + cat <) -> Secret { +pub fn create_secret(name: &str, data: BTreeMap) -> Secret { Secret { metadata: ObjectMeta { name: Some(name.to_string()), diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 701035d..5a0c3c2 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,14 +1,17 @@ use { - crate::{docker::DockerImage, k8s_helpers, validator_config::ValidatorConfig, ValidatorType}, + crate::{ + docker::DockerImage, k8s_helpers, validator_config::ValidatorConfig, Metrics, ValidatorType, + }, k8s_openapi::{ api::{ apps::v1::ReplicaSet, core::v1::{ - EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, - Service, Volume, VolumeMount, + EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretKeySelector, + SecretVolumeSource, Service, Volume, VolumeMount, }, }, apimachinery::pkg::api::resource::Quantity, + ByteString, }, kube::{ api::{Api, ListParams, PostParams}, @@ -42,6 +45,7 @@ pub struct Kubernetes<'a> { namespace: String, validator_config: &'a mut ValidatorConfig, pod_requests: PodRequests, + pub metrics: Option, } impl<'a> Kubernetes<'a> { @@ -49,12 +53,14 @@ impl<'a> Kubernetes<'a> { namespace: &str, validator_config: &'a mut ValidatorConfig, pod_requests: PodRequests, + metrics: Option, ) -> Kubernetes<'a> { Self { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), validator_config, pod_requests, + metrics, } } @@ -119,7 +125,7 @@ impl<'a> Kubernetes<'a> { secret_name: Option, label_selector: &BTreeMap, ) -> Result> { - let env_vars = vec![EnvVar { + let mut env_vars = vec![EnvVar { name: "MY_POD_IP".to_string(), value_from: Some(EnvVarSource { field_ref: Some(ObjectFieldSelector { @@ -131,6 +137,10 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]; + if self.metrics.is_some() { + env_vars.push(self.get_metrics_env_var_secret()) + } + let accounts_volume = Some(vec![Volume { name: "bootstrap-accounts-volume".into(), secret: Some(SecretVolumeSource { @@ -257,4 +267,37 @@ impl<'a> Kubernetes<'a> { Ok(available_validators >= desired_validators) } + + pub fn create_metrics_secret(&self) -> Result> { + let mut data = BTreeMap::new(); + if let Some(metrics) = &self.metrics { + data.insert( + "SOLANA_METRICS_CONFIG".to_string(), + ByteString(metrics.to_env_string().into_bytes()), + ); + } else { + return Err( + "Called create_metrics_secret() but metrics were not provided." + .to_string() + .into(), + ); + } + + Ok(k8s_helpers::create_secret("solana-metrics-secret", data)) + } + + pub fn get_metrics_env_var_secret(&self) -> EnvVar { + EnvVar { + name: "SOLANA_METRICS_CONFIG".to_string(), + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: Some("solana-metrics-secret".to_string()), + key: "SOLANA_METRICS_CONFIG".to_string(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } + } } diff --git a/src/lib.rs b/src/lib.rs index d3f6dcf..79e3c4e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,6 +54,50 @@ pub enum ValidatorType { Client, } +// impl std::fmt::Display for ValidatorType { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// match *self { +// ValidatorType::Bootstrap => write!(f, "bootstrap-validator"), +// ValidatorType::Standard => write!(f, "validator"), +// ValidatorType::RPC => write!(f, "rpc-node"), +// ValidatorType::Client => write!(f, "client"), +// } +// } +// } + +#[derive(Clone, Debug, Default)] +pub struct Metrics { + pub host: String, + pub port: String, + pub database: String, + pub username: String, + password: String, +} + +impl Metrics { + pub fn new( + host: String, + port: String, + database: String, + username: String, + password: String, + ) -> Self { + Metrics { + host, + port, + database, + username, + password, + } + } + pub fn to_env_string(&self) -> String { + format!( + "host={}:{},db={},u={},p={}", + self.host, self.port, self.database, self.username, self.password + ) + } +} + pub mod docker; pub mod genesis; pub mod k8s_helpers; diff --git a/src/main.rs b/src/main.rs index f546f3f..b7bfe20 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,7 +14,7 @@ use { release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, validator_config::ValidatorConfig, - SolanaRoot, ValidatorType, + Metrics, SolanaRoot, ValidatorType, }, }; @@ -215,6 +215,38 @@ fn parse_matches() -> clap::ArgMatches { Can specify unit here (B, Ki, Mi, Gi, Ti) for bytes, kilobytes, etc (2^N notation) e.g. 1Gi == 1024Mi == 1024Ki == 1,047,576B. [default: 70Gi]"), ) + //Metrics Config + .arg( + Arg::with_name("metrics_host") + .long("metrics-host") + .takes_value(true) + .requires_all(&["metrics_port", "metrics_db", "metrics_username", "metrics_password"]) + .help("Metrics Config. Optional: specify metrics host. e.g. https://internal-metrics.solana.com"), + ) + .arg( + Arg::with_name("metrics_port") + .long("metrics-port") + .takes_value(true) + .help("Metrics Config. Optional: specify metrics port. e.g. 8086"), + ) + .arg( + Arg::with_name("metrics_db") + .long("metrics-db") + .takes_value(true) + .help("Metrics Config. Optional: specify metrics database. e.g. k8s-cluster-"), + ) + .arg( + Arg::with_name("metrics_username") + .long("metrics-username") + .takes_value(true) + .help("Metrics Config. Optional: specify metrics username"), + ) + .arg( + Arg::with_name("metrics_password") + .long("metrics-password") + .takes_value(true) + .help("Metrics Config. Optional: Specify metrics password"), + ) .get_matches() } @@ -358,12 +390,24 @@ async fn main() { matches.value_of("memory_requests").unwrap().to_string(), ); + let metrics = matches.value_of("metrics_host").map(|host| { + Metrics::new( + host.to_string(), + matches.value_of("metrics_port").unwrap().to_string(), + matches.value_of("metrics_db").unwrap().to_string(), + matches.value_of("metrics_username").unwrap().to_string(), + matches.value_of("metrics_password").unwrap().to_string(), + ) + }); + let mut kub_controller = Kubernetes::new( environment_config.namespace, &mut validator_config, pod_requests, + metrics, ) .await; + match kub_controller.namespace_exists().await { Ok(true) => (), Ok(false) => { @@ -424,25 +468,23 @@ async fn main() { deploy_method, ); - let registry = matches.value_of("registry_name").unwrap().to_string(); + let registry_name = matches.value_of("registry_name").unwrap().to_string(); let image_name = matches.value_of("image_name").unwrap().to_string(); - let tag = matches + let image_tag = matches .value_of("image_tag") .unwrap_or_default() .to_string(); - // This will always be Some() right now. But will won't be in future when we implement - // heterogenous clusters - let bootstrap_validator = Some(Validator::new(DockerImage::new( - registry.clone(), + let mut bootstrap_validator = Validator::new(DockerImage::new( + registry_name.clone(), ValidatorType::Bootstrap, image_name.clone(), - tag.clone(), - ))); + image_tag.clone(), + )); if build_config.docker_build() { let validators = vec![&bootstrap_validator]; - for v in validators.into_iter().flatten() { + for v in &validators { match docker.build_image(solana_root.get_root_path(), v.image()) { Ok(_) => info!("{} image built successfully", v.validator_type()), Err(err) => { @@ -461,132 +503,147 @@ async fn main() { } } - if let Some(mut bootstrap_validator) = bootstrap_validator { - match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) - { - Ok(secret) => bootstrap_validator.set_secret(secret), + // metrics secret create once and use by all pods + if kub_controller.metrics.is_some() { + let metrics_secret = match kub_controller.create_metrics_secret() { + Ok(secret) => secret, Err(err) => { - error!("Failed to create bootstrap secret! {err}"); + error!("Failed to create metrics secret! {err}"); return; } }; - - match kub_controller - .deploy_secret(bootstrap_validator.secret()) - .await - { - Ok(_) => info!("Deployed Bootstrap Secret"), + match kub_controller.deploy_secret(&metrics_secret).await { + Ok(_) => (), Err(err) => { error!("{err}"); return; } } + }; - // Create bootstrap labels - let identity_path = config_directory.join("bootstrap-validator/identity.json"); - let bootstrap_keypair = - read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); - bootstrap_validator.add_label( - "load-balancer/name", - "load-balancer-selector", - LabelType::ValidatorReplicaSet, - ); - bootstrap_validator.add_label( - "service/name", - "bootstrap-validator-selector", - LabelType::ValidatorReplicaSet, - ); - bootstrap_validator.add_label( - "validator/type", - "bootstrap", - LabelType::ValidatorReplicaSet, - ); - bootstrap_validator.add_label( - "validator/identity", - bootstrap_keypair.pubkey().to_string(), - LabelType::ValidatorReplicaSet, - ); - - // create bootstrap replica set - match kub_controller.create_bootstrap_validator_replica_set( - bootstrap_validator.image(), - bootstrap_validator.secret().metadata.name.clone(), - bootstrap_validator.replica_set_labels(), - ) { - Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), - Err(err) => { - error!("Error creating bootstrap validator replicas_set: {err}"); - return; - } - }; - - match kub_controller - .deploy_replicas_set(bootstrap_validator.replica_set()) - .await - { - Ok(_) => { - info!( - "{} deployed successfully", - bootstrap_validator.replica_set_name() - ); - } - Err(err) => { - error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); - return; - } - }; - - bootstrap_validator.add_label( - "service/name", - "bootstrap-validator-selector", - LabelType::ValidatorService, - ); + match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) { + Ok(secret) => bootstrap_validator.set_secret(secret), + Err(err) => { + error!("Failed to create bootstrap secret! {err}"); + return; + } + }; - let bootstrap_service = kub_controller.create_bootstrap_service( - "bootstrap-validator-service", - bootstrap_validator.service_labels(), - ); - match kub_controller.deploy_service(&bootstrap_service).await { - Ok(_) => info!("bootstrap validator service deployed successfully"), - Err(err) => error!( - "Error! Failed to deploy bootstrap validator service. err: {:?}", - err - ), + match kub_controller + .deploy_secret(bootstrap_validator.secret()) + .await + { + Ok(_) => info!("Deployed Bootstrap Secret"), + Err(err) => { + error!("{err}"); + return; } + } - //load balancer service. only create one and use for all deployments - let load_balancer_label = - kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); - //create load balancer - let load_balancer = kub_controller.create_validator_load_balancer( - "bootstrap-and-non-voting-lb-service", - &load_balancer_label, - ); + // Create bootstrap labels + let identity_path = config_directory.join("bootstrap-validator/identity.json"); + let bootstrap_keypair = + read_keypair_file(identity_path).expect("Failed to read bootstrap keypair file"); + bootstrap_validator.add_label( + "load-balancer/name", + "load-balancer-selector", + LabelType::ValidatorReplicaSet, + ); + bootstrap_validator.add_label( + "service/name", + "bootstrap-validator-selector", + LabelType::ValidatorReplicaSet, + ); + bootstrap_validator.add_label( + "validator/type", + "bootstrap", + LabelType::ValidatorReplicaSet, + ); + bootstrap_validator.add_label( + "validator/identity", + bootstrap_keypair.pubkey().to_string(), + LabelType::ValidatorReplicaSet, + ); - //deploy load balancer - match kub_controller.deploy_service(&load_balancer).await { - Ok(_) => info!("load balancer service deployed successfully"), - Err(err) => error!( - "Error! Failed to deploy load balancer service. err: {:?}", - err - ), + // create bootstrap replica set + match kub_controller.create_bootstrap_validator_replica_set( + bootstrap_validator.image(), + bootstrap_validator.secret().metadata.name.clone(), + bootstrap_validator.replica_set_labels(), + ) { + Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set), + Err(err) => { + error!("Error creating bootstrap validator replicas_set: {err}"); + return; } + }; - // wait for bootstrap replicaset to deploy - while { - match kub_controller - .check_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) - .await - { - Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) - Err(_) => panic!("Error occurred while checking replica set readiness"), - } - } { + match kub_controller + .deploy_replicas_set(bootstrap_validator.replica_set()) + .await + { + Ok(_) => { info!( - "replica set: {} not ready...", + "{} deployed successfully", bootstrap_validator.replica_set_name() ); - thread::sleep(Duration::from_secs(1)); } + Err(err) => { + error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); + return; + } + }; + + bootstrap_validator.add_label( + "service/name", + "bootstrap-validator-selector", + LabelType::ValidatorService, + ); + + let bootstrap_service = kub_controller.create_bootstrap_service( + "bootstrap-validator-service", + bootstrap_validator.service_labels(), + ); + match kub_controller.deploy_service(&bootstrap_service).await { + Ok(_) => info!("bootstrap validator service deployed successfully"), + Err(err) => error!( + "Error! Failed to deploy bootstrap validator service. err: {:?}", + err + ), + } + + //load balancer service. only create one and use for all deployments + let load_balancer_label = + kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); + //create load balancer + let load_balancer = kub_controller.create_validator_load_balancer( + "bootstrap-and-non-voting-lb-service", + &load_balancer_label, + ); + + //deploy load balancer + match kub_controller.deploy_service(&load_balancer).await { + Ok(_) => info!("load balancer service deployed successfully"), + Err(err) => error!( + "Error! Failed to deploy load balancer service. err: {:?}", + err + ), + } + + // wait for bootstrap replicaset to deploy + while { + match kub_controller + .check_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) + .await + { + Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) + Err(_) => panic!("Error occurred while checking replica set readiness"), + } + } { + info!( + "replica set: {} not ready...", + bootstrap_validator.replica_set_name() + ); + thread::sleep(Duration::from_secs(1)); } } From 1b124f8befec5b09f6aa5e056bee6f057b02d1a9 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 23:33:26 +0000 Subject: [PATCH 19/26] create N validator accounts --- PROGRESS.md | 2 +- README.md | 3 +++ src/main.rs | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/PROGRESS.md b/PROGRESS.md index bad7044..3f5f63f 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -46,7 +46,7 @@ - [ ] RPC nodes - [ ] Client - [ ] Create accounts - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC - [ ] Client - [ ] Add feature flags to configure: diff --git a/README.md b/README.md index 7b5c818..aace243 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ kubectl create ns ``` cargo run --bin cluster -- -n + --num_validators --local-path ``` @@ -33,6 +34,7 @@ cargo run --bin cluster -- ``` cargo run --bin cluster -- -n + --num_validators --release-channel # note: MUST include the "v" ``` @@ -41,6 +43,7 @@ Example: ``` cargo run --bin cluster -- -n + --num_validators --local-path /home/sol/solana # genesis config. Optional: Many of these have defaults --hashes-per-tick diff --git a/src/main.rs b/src/main.rs index b7bfe20..fe1cdd0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,6 +28,17 @@ fn parse_matches() -> clap::ArgMatches { .default_value("default") .help("namespace to deploy test cluster"), ) + .arg( + Arg::with_name("number_of_validators") + .long("num-validators") + .takes_value(true) + .default_value("1") + .help("Number of validator replicas to deploy") + .validator(|s| match s.parse::() { + Ok(n) if n > 0 => Ok(()), + _ => Err(String::from("number_of_validators should be >= 0")), + }), + ) .arg( Arg::new("local_path") .long("local-path") @@ -266,6 +277,8 @@ async fn main() { namespace: matches.value_of("cluster_namespace").unwrap_or_default(), }; + let num_validators = value_t_or_exit!(matches, "number_of_validators", usize); + let deploy_method = if let Some(local_path) = matches.value_of("local_path") { DeployMethod::Local(local_path.to_owned()) } else if let Some(release_channel) = matches.value_of("release_channel") { @@ -459,6 +472,14 @@ async fn main() { } } + match genesis.generate_accounts(ValidatorType::Standard, num_validators) { + Ok(_) => (), + Err(err) => { + error!("generate accounts error! {err}"); + return; + } + } + //unwraps are safe here. since their requirement is enforced by argmatches let docker = DockerConfig::new( matches From 3d154bb704e4fc03cda12ba1cc45e6cb0d597e70 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 23:59:15 +0000 Subject: [PATCH 20/26] create and deploy validator secrets --- PROGRESS.md | 2 +- src/kubernetes.rs | 29 ++++++++++++++++++++++++++++- src/main.rs | 19 +++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 3f5f63f..9754115 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -20,7 +20,7 @@ - [x] Push Image to registry - [ ] Create & Deploy Secrets - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Selector diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 5a0c3c2..c789132 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -19,7 +19,11 @@ use { }, log::*, solana_sdk::{pubkey::Pubkey, signature::keypair::read_keypair_file, signer::Signer}, - std::{collections::BTreeMap, error::Error, path::Path}, + std::{ + collections::BTreeMap, + error::Error, + path::{Path, PathBuf}, + }, }; #[derive(Debug, Clone)] @@ -102,6 +106,29 @@ impl<'a> Kubernetes<'a> { k8s_helpers::create_secret_from_files(secret_name, &key_files) } + pub fn create_validator_secret( + &self, + validator_index: usize, + config_dir: &Path, + ) -> Result> { + let secret_name = format!("validator-accounts-secret-{}", validator_index); + + let accounts = ["identity", "vote", "stake"]; + let key_files: Vec<(PathBuf, &str)> = accounts + .iter() + .map(|&account| { + let file_name = if account == "identity" { + format!("validator-{account}-{validator_index}.json") + } else { + format!("validator-{account}-account-{validator_index}.json") + }; + (config_dir.join(file_name), account) + }) + .collect(); + + k8s_helpers::create_secret_from_files(&secret_name, &key_files) + } + fn add_known_validator(&mut self, pubkey: Pubkey) { if let Some(ref mut known_validators) = self.validator_config.known_validators { known_validators.push(pubkey); diff --git a/src/main.rs b/src/main.rs index fe1cdd0..b0e15ae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -667,4 +667,23 @@ async fn main() { ); thread::sleep(Duration::from_secs(1)); } + + // Create and deploy validators secrets + for validator_index in 0..num_validators { + let validator_secret = + match kub_controller.create_validator_secret(validator_index, &config_directory) { + Ok(secret) => secret, + Err(err) => { + error!("Failed to create validator secret! {err}"); + return; + } + }; + match kub_controller.deploy_secret(&validator_secret).await { + Ok(_) => (), + Err(err) => { + error!("{err}"); + return; + } + } + } } From d73d0113eac370dc3d4eda6d7476392cd59c02a9 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 3 Apr 2024 00:16:24 +0000 Subject: [PATCH 21/26] create validator selectors --- PROGRESS.md | 2 +- src/main.rs | 51 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 9754115..7f30bca 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -25,7 +25,7 @@ - [ ] Client - [ ] Create & Deploy Selector - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Replica Set diff --git a/src/main.rs b/src/main.rs index b0e15ae..f90e33e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -668,22 +668,53 @@ async fn main() { thread::sleep(Duration::from_secs(1)); } - // Create and deploy validators secrets + let mut validator = Validator::new(DockerImage::new( + matches.value_of("registry_name").unwrap().to_string(), + ValidatorType::Standard, + matches.value_of("image_name").unwrap().to_string(), + matches + .value_of("image_tag") + .unwrap_or_default() + .to_string(), + )); + + // Create and deploy validators secrets/selectors for validator_index in 0..num_validators { - let validator_secret = - match kub_controller.create_validator_secret(validator_index, &config_directory) { - Ok(secret) => secret, - Err(err) => { - error!("Failed to create validator secret! {err}"); - return; - } - }; - match kub_controller.deploy_secret(&validator_secret).await { + match kub_controller.create_validator_secret(validator_index, &config_directory) { + Ok(secret) => validator.set_secret(secret), + Err(err) => { + error!("Failed to create validator secret! {err}"); + return; + } + }; + + match kub_controller.deploy_secret(validator.secret()).await { Ok(_) => (), Err(err) => { error!("{err}"); return; } } + + let identity_path = + config_directory.join(format!("validator-identity-{validator_index}.json")); + let validator_keypair = + read_keypair_file(identity_path).expect("Failed to read validator keypair file"); + + validator.add_label( + "validator/name", + &format!("validator-{}", validator_index), + LabelType::ValidatorReplicaSet, + ); + validator.add_label( + "validator/type", + validator.validator_type().to_string(), + LabelType::ValidatorReplicaSet, + ); + validator.add_label( + "validator/identity", + validator_keypair.pubkey().to_string(), + LabelType::ValidatorReplicaSet, + ); } } From ca24ac1ee65e6f14b3eedd5d18b2f18052ffa986 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 3 Apr 2024 21:17:33 +0000 Subject: [PATCH 22/26] add in build and push validator docker images --- src/docker.rs | 6 +- src/main.rs | 19 +- src/scripts/validator-startup-script.sh | 398 ++++++++++++++++++++++++ 3 files changed, 409 insertions(+), 14 deletions(-) create mode 100755 src/scripts/validator-startup-script.sh diff --git a/src/docker.rs b/src/docker.rs index 6a7253e..6e60e4c 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -70,8 +70,8 @@ impl DockerConfig { ) -> Result<(), Box> { let validator_type = docker_image.validator_type(); match validator_type { - ValidatorType::Bootstrap => (), - ValidatorType::Standard | ValidatorType::RPC | ValidatorType::Client => { + ValidatorType::Bootstrap | ValidatorType::Standard => (), + ValidatorType::RPC | ValidatorType::Client => { return Err(format!( "Build docker image for validator type: {validator_type} not supported yet" ) @@ -106,7 +106,7 @@ impl DockerConfig { let context_path = solana_root_path.display().to_string(); let progress_bar = new_spinner_progress_bar(); - progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...",)); + progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...")); let command = format!("docker build -t {docker_image} -f {dockerfile:?} {context_path}"); diff --git a/src/main.rs b/src/main.rs index f90e33e..5d8e9e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -503,8 +503,15 @@ async fn main() { image_tag.clone(), )); + let mut validator = Validator::new(DockerImage::new( + registry_name.clone(), + ValidatorType::Standard, + image_name.clone(), + image_tag.clone(), + )); + if build_config.docker_build() { - let validators = vec![&bootstrap_validator]; + let validators = vec![&bootstrap_validator, &validator]; for v in &validators { match docker.build_image(solana_root.get_root_path(), v.image()) { Ok(_) => info!("{} image built successfully", v.validator_type()), @@ -668,16 +675,6 @@ async fn main() { thread::sleep(Duration::from_secs(1)); } - let mut validator = Validator::new(DockerImage::new( - matches.value_of("registry_name").unwrap().to_string(), - ValidatorType::Standard, - matches.value_of("image_name").unwrap().to_string(), - matches - .value_of("image_tag") - .unwrap_or_default() - .to_string(), - )); - // Create and deploy validators secrets/selectors for validator_index in 0..num_validators { match kub_controller.create_validator_secret(validator_index, &config_directory) { diff --git a/src/scripts/validator-startup-script.sh b/src/scripts/validator-startup-script.sh new file mode 100755 index 0000000..3e58b58 --- /dev/null +++ b/src/scripts/validator-startup-script.sh @@ -0,0 +1,398 @@ +#!/bin/bash + +# Start Validator +# shellcheck disable=SC1091 +source /home/solana/k8s-cluster-scripts/common.sh + +args=( + --no-poh-speed-test + --no-os-network-limits-test +) +airdrops_enabled=1 +# next two values will be overwritten based on command line args. default is set here: k8s-cluster/src/genesis.rs +identity=validator-accounts/identity.json +vote_account=validator-accounts/vote.json +no_restart=0 +gossip_entrypoint=$BOOTSTRAP_GOSSIP_ADDRESS +ledger_dir=/home/solana/ledger +faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS + +usage() { + if [[ -n $1 ]]; then + echo "$*" + echo + fi + cat <&1) + status=$? + + if [ $status -eq 0 ]; then + echo "Command succeeded: $description" + return 0 + else + echo "Command failed for: $description (Exit status $status)" + echo "$output" # Print the output which includes the error + + # Check for specific error message + if [[ "$output" == *"Vote account"*"already exists"* ]]; then + echo "Vote account already exists. Continuing without exiting." + vote_account_already_exists=true + return 0 + fi + if [[ "$output" == *"Stake account"*"already exists"* ]]; then + echo "Stake account already exists. Continuing without exiting." + stake_account_already_exists=true + return 0 + fi + + if [ "$retry_count" -lt $MAX_RETRIES ]; then + echo "Retrying in $RETRY_DELAY seconds..." + sleep $RETRY_DELAY + fi + fi + done + + echo "Max retry limit reached. Command still failed for: $description" + return 1 +} + +setup_validator() { + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL airdrop $node_sol $IDENTITY_FILE" "Airdrop"; then + echo "Aidrop command failed." + exit 1 + fi + + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL create-vote-account --allow-unsafe-authorized-withdrawer validator-accounts/vote.json $IDENTITY_FILE $IDENTITY_FILE -k $IDENTITY_FILE" "Create Vote Account"; then + if $vote_account_already_exists; then + echo "Vote account already exists. Skipping remaining commands." + else + echo "Create vote account failed." + exit 1 + fi + fi + + echo "created vote account" +} + +run_delegate_stake() { + echo "stake sol for account: $stake_sol" + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL create-stake-account validator-accounts/stake.json $stake_sol -k $IDENTITY_FILE" "Create Stake Account"; then + if $stake_account_already_exists; then + echo "Stake account already exists. Skipping remaining commands." + else + echo "Create stake account failed." + exit 1 + fi + fi + echo "created stake account" + + if [ "$stake_account_already_exists" != true ]; then + echo "stake account does not exist. so lets deligate" + if ! run_solana_command "solana -u $LOAD_BALANCER_RPC_URL delegate-stake validator-accounts/stake.json validator-accounts/vote.json --force -k $IDENTITY_FILE" "Delegate Stake"; then + echo "Delegate stake command failed." + exit 1 + fi + echo "delegated stake" + fi + + solana --url $LOAD_BALANCER_RPC_URL --keypair $IDENTITY_FILE stakes validator-accounts/stake.json +} + +echo "get airdrop and create vote account" +setup_validator +echo "create stake account and delegate stake" +run_delegate_stake + +echo running validator: + +echo "Validator Args" +for arg in "${args[@]}"; do + echo "$arg" +done + +while true; do + echo "$PS4$program ${args[*]}" + + $program "${args[@]}" & + pid=$! + echo "pid: $pid" + + if ((no_restart)); then + wait "$pid" + exit $? + fi + + while true; do + if [[ -z $pid ]] || ! kill -0 "$pid"; then + echo "############## validator exited, restarting ##############" + break + fi + sleep 1 + done + + kill_node +done \ No newline at end of file From 4d80096ba845406672ac72059a3ad59056d763c6 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 3 Apr 2024 00:35:27 +0000 Subject: [PATCH 23/26] wip. more work --- Cargo.lock | 1 + Cargo.toml | 1 + PROGRESS.md | 4 +- src/k8s_helpers.rs | 27 +++++++- src/kubernetes.rs | 136 ++++++++++++++++++++++++++++++++++++++++ src/ledger_helper.rs | 31 +++++++++ src/lib.rs | 1 + src/main.rs | 69 +++++++++++++++++++- src/validator_config.rs | 9 +++ 9 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 src/ledger_helper.rs diff --git a/Cargo.lock b/Cargo.lock index 269a491..d23ee06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7135,6 +7135,7 @@ dependencies = [ "reqwest", "rustc_version", "rustls", + "solana-accounts-db", "solana-core", "solana-ledger", "solana-logger", diff --git a/Cargo.toml b/Cargo.toml index 7137d4c..f649bbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ log = "0.4.21" rand = "0.8.5" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.10", default-features = false, features = ["quic"] } +solana-accounts-db = "1.18.8" solana-core = "1.18.8" solana-ledger = "1.18.8" solana-logger = "1.18.8" diff --git a/PROGRESS.md b/PROGRESS.md index 7f30bca..9bfffc3 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -30,7 +30,7 @@ - [ ] Client - [ ] Create & Deploy Replica Set - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Services @@ -42,7 +42,7 @@ - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) - [ ] Add metrics - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create accounts diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index d1289a6..aa36577 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -6,7 +6,7 @@ use { core::v1::{ Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, ResourceRequirements, Secret, Service, ServicePort, ServiceSpec, Volume, - VolumeMount, + VolumeMount, EnvVarSource, ObjectFieldSelector, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -157,3 +157,28 @@ pub fn create_selector(key: &str, value: &str) -> BTreeMap { btree.insert(key.to_string(), value.to_string()); btree } + +pub fn create_environment_variable( + name: &str, + value: Option, + field_path: Option, +) -> EnvVar { + match field_path { + Some(path) => EnvVar { + name: name.to_string(), + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: path, + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }, + None => EnvVar { + name: name.to_string(), + value, + ..Default::default() + }, + } +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index c789132..7230b3c 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,6 +1,7 @@ use { crate::{ docker::DockerImage, k8s_helpers, validator_config::ValidatorConfig, Metrics, ValidatorType, + genesis::DEFAULT_INTERNAL_NODE_SOL_TO_STAKE_SOL_RATIO, }, k8s_openapi::{ api::{ @@ -327,4 +328,139 @@ impl<'a> Kubernetes<'a> { ..Default::default() } } + + fn set_non_bootstrap_environment_variables(&self) -> Vec { + vec![ + k8s_helpers::create_environment_variable( + "NAMESPACE", + None, + Some("metadata.namespace".to_string()), + ), + k8s_helpers::create_environment_variable( + "BOOTSTRAP_RPC_ADDRESS", + Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:8899".to_string()), + None, + ), + k8s_helpers::create_environment_variable( + "BOOTSTRAP_GOSSIP_ADDRESS", + Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:8001".to_string()), + None, + ), + k8s_helpers::create_environment_variable( + "BOOTSTRAP_FAUCET_ADDRESS", + Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:9900".to_string()), + None, + ), + ] + } + + fn set_load_balancer_environment_variables(&self) -> Vec { + vec![ + k8s_helpers::create_environment_variable( + "LOAD_BALANCER_RPC_ADDRESS", + Some( + "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:8899" + .to_string(), + ), + None, + ), + k8s_helpers::create_environment_variable( + "LOAD_BALANCER_GOSSIP_ADDRESS", + Some( + "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:8001" + .to_string(), + ), + None, + ), + k8s_helpers::create_environment_variable( + "LOAD_BALANCER_FAUCET_ADDRESS", + Some( + "bootstrap-and-non-voting-lb-service.$(NAMESPACE).svc.cluster.local:9900" + .to_string(), + ), + None, + ), + ] + } + + fn generate_validator_command_flags(&self, validator_stake: &Option) -> Vec { + let mut flags: Vec = Vec::new(); + self.generate_command_flags(&mut flags); + + flags.push("--internal-node-stake-sol".to_string()); + if let Some(stake) = validator_stake { + flags.push(stake.to_string()); + } else { + flags.push(self.validator_config.internal_node_stake_sol.to_string()); + } + + flags.push("--internal-node-sol".to_string()); + if let Some(stake) = validator_stake { + flags.push((DEFAULT_INTERNAL_NODE_SOL_TO_STAKE_SOL_RATIO * stake).to_string()); + } else { + flags.push(self.validator_config.internal_node_sol.to_string()); + } + + if let Some(shred_version) = self.validator_config.shred_version { + flags.push("--expected-shred-version".to_string()); + flags.push(shred_version.to_string()); + } + + self.add_known_validators_if_exists(&mut flags); + + flags + } + + pub fn create_validator_replica_set( + &mut self, + container_name: &str, + validator_index: i32, + image_name: &str, + secret_name: Option, + label_selector: &BTreeMap, + validator_stake: &Option, + ) -> Result> { + let mut env_vars = self.set_non_bootstrap_environment_variables(); + if self.metrics.is_some() { + env_vars.push(self.get_metrics_env_var_secret()) + } + env_vars.append(&mut self.set_load_balancer_environment_variables()); + + let accounts_volume = Some(vec![Volume { + name: format!("validator-accounts-volume-{}", validator_index), + secret: Some(SecretVolumeSource { + secret_name, + ..Default::default() + }), + ..Default::default() + }]); + + let accounts_volume_mount = Some(vec![VolumeMount { + name: format!("validator-accounts-volume-{}", validator_index), + mount_path: "/home/solana/validator-accounts".to_string(), + ..Default::default() + }]); + + let mut command = + vec!["/home/solana/k8s-cluster-scripts/validator-startup-script.sh".to_string()]; + command.extend(self.generate_validator_command_flags(validator_stake)); + + for c in command.iter() { + debug!("validator command: {}", c); + } + + + k8s_helpers::create_replica_set( + &ValidatorType::Standard, + self.namespace.as_str(), + label_selector, + image_name, + env_vars, + &command, + accounts_volume, + accounts_volume_mount, + None, + self.pod_requests.requests.clone(), + ) + } } diff --git a/src/ledger_helper.rs b/src/ledger_helper.rs new file mode 100644 index 0000000..5223dde --- /dev/null +++ b/src/ledger_helper.rs @@ -0,0 +1,31 @@ +use { + crate::{genesis::DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, LEDGER_DIR}, + log::*, + solana_accounts_db::hardened_unpack::open_genesis_config, + solana_sdk::shred_version::compute_shred_version, + std::error::Error, +}; + +fn ledger_directory_exists() -> Result<(), Box> { + if !LEDGER_DIR.exists() { + return Err(format!( + "Ledger Directory does not exist, have you created genesis yet??" + ).into()); + } + Ok(()) +} + +pub struct LedgerHelper {} + +impl LedgerHelper { + pub fn get_shred_version() -> Result> { + ledger_directory_exists()?; + let genesis_config = open_genesis_config( + LEDGER_DIR.as_path(), + DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + ); + let shred_version = compute_shred_version(&genesis_config?.hash(), None); + info!("Shred Version: {}", shred_version); + Ok(shred_version) + } +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 79e3c4e..56bcec4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -102,6 +102,7 @@ pub mod docker; pub mod genesis; pub mod k8s_helpers; pub mod kubernetes; +pub mod ledger_helper; pub mod release; pub mod validator; pub mod validator_config; diff --git a/src/main.rs b/src/main.rs index 5d8e9e8..ce730f5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ use { strum::VariantNames, validator_lab::{ docker::{DockerConfig, DockerImage}, - genesis::{Genesis, GenesisFlags}, + genesis::{Genesis, GenesisFlags, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL}, kubernetes::{Kubernetes, PodRequests}, release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, @@ -207,6 +207,18 @@ fn parse_matches() -> clap::ArgMatches { .long("full-rpc") .help("Validator config. Support full RPC services on all nodes"), ) + .arg( + Arg::with_name("internal_node_sol") + .long("internal-node-sol") + .takes_value(true) + .help("Amount to fund internal nodes in genesis config."), + ) + .arg( + Arg::with_name("internal_node_stake_sol") + .long("internal-node-stake-sol") + .takes_value(true) + .help("Amount to stake internal nodes (Sol)."), + ) // kubernetes config .arg( Arg::with_name("cpu_requests") @@ -376,6 +388,18 @@ async fn main() { let mut validator_config = ValidatorConfig { tpu_enable_udp: matches.is_present("tpu_enable_udp"), tpu_disable_quic: matches.is_present("tpu_disable_quic"), + internal_node_sol: matches + .value_of("internal_node_sol") + .unwrap_or(DEFAULT_INTERNAL_NODE_SOL.to_string().as_str()) + .parse::() + .expect("Invalid value for internal_node_stake_sol") as f64, + internal_node_stake_sol: matches + .value_of("internal_node_stake_sol") + .unwrap_or(DEFAULT_INTERNAL_NODE_STAKE_SOL.to_string().as_str()) + .parse::() + .expect("Invalid value for internal_node_stake_sol") + as f64, + shred_version: None, // set after genesis created max_ledger_size: if matches.is_present("limit_ledger_size") { let limit_ledger_size = match matches.value_of("limit_ledger_size") { Some(_) => value_t_or_exit!(matches, "limit_ledger_size", u64), @@ -480,6 +504,14 @@ async fn main() { } } + match LedgerHelper::get_shred_version() { + Ok(shred_version) => kub_controller.set_shred_version(shred_version), + Err(err) => { + error!("{}", err); + return; + } + } + //unwraps are safe here. since their requirement is enforced by argmatches let docker = DockerConfig::new( matches @@ -713,5 +745,40 @@ async fn main() { validator_keypair.pubkey().to_string(), LabelType::ValidatorReplicaSet, ); + + let validator_replica_set = match kub_controller.create_validator_replica_set( + validator_container_name, + validator_index, + validator_image_name, + validator_secret.metadata.name.clone(), + &validator_labels, + &stake, + ) { + Ok(replica_set) => replica_set, + Err(err) => { + error!("Error creating validator replicas_set: {}", err); + return; + } + }; + + let _ = match kub_controller + .deploy_replicas_set(&validator_replica_set) + .await + { + Ok(rs) => { + info!( + "validator replica set ({}) deployed successfully", + validator_index + ); + rs.metadata.name.unwrap() + } + Err(err) => { + error!( + "Error! Failed to deploy validator replica set: {}. err: {:?}", + validator_index, err + ); + return; + } + }; } } diff --git a/src/validator_config.rs b/src/validator_config.rs index 35f4a4b..e2cc7f3 100644 --- a/src/validator_config.rs +++ b/src/validator_config.rs @@ -3,6 +3,9 @@ use solana_sdk::pubkey::Pubkey; pub struct ValidatorConfig { pub tpu_enable_udp: bool, pub tpu_disable_quic: bool, + pub internal_node_sol: f64, + pub internal_node_stake_sol: f64, + pub shred_version: Option, pub max_ledger_size: Option, pub skip_poh_verify: bool, pub no_snapshot_fetch: bool, @@ -26,6 +29,9 @@ impl std::fmt::Display for ValidatorConfig { "Runtime Config\n\ tpu_enable_udp: {}\n\ tpu_disable_quic: {}\n\ + internal_node_sol: {}\n\ + internal_node_stake_sol: {}\n\ + shred_version: {:?}\n\ max_ledger_size: {:?}\n\ skip_poh_verify: {}\n\ no_snapshot_fetch: {}\n\ @@ -34,6 +40,9 @@ impl std::fmt::Display for ValidatorConfig { known_validators: {:?}", self.tpu_enable_udp, self.tpu_disable_quic, + self.internal_node_sol, + self.internal_node_stake_sol, + self.shred_version, self.max_ledger_size, self.skip_poh_verify, self.no_snapshot_fetch, From 20566494cee5667f3ff86b1fa82450a543a7e442 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 3 Apr 2024 20:51:16 +0000 Subject: [PATCH 24/26] i forgot to build the validator docker images first --- PROGRESS.md | 8 +++++--- README.md | 7 +++++++ src/k8s_helpers.rs | 18 ++++++++-------- src/kubernetes.rs | 49 ++++++++++++++++++++------------------------ src/ledger_helper.rs | 22 ++++++++++---------- src/main.rs | 28 ++++++++++++------------- 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 9bfffc3..a8ff8b4 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -15,9 +15,11 @@ - [x] Create Genesis - [x] Generate faucet and bootstrap accounts - [x] Build genesis -- [x] Docker Build - - [x] Build Bootstrap Image - - [x] Push Image to registry +- [x] Docker Build and Push to registry + - [x] Bootstrap + - [ ] Validator (regular) + - [ ] RPC nodes + - [ ] Client - [ ] Create & Deploy Secrets - [x] Bootstrap - [x] Validator (regular) diff --git a/README.md b/README.md index aace243..6e25e13 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,13 @@ cargo run --bin cluster -- --tag # e.g. v1 --base-image # e.g. ubuntu:20.04 --image-name # e.g. cluster-image + # validator config + --full-rpc + --internal-node-sol + --internal-node-stake-sol + # kubernetes config + --cpu-requests + --memory-requests ``` ## Metrics diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index aa36577..9d9af58 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -1,12 +1,12 @@ use { - crate::{docker::DockerImage, ValidatorType}, + crate::docker::DockerImage, k8s_openapi::{ api::{ apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ - Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, Secret, Service, ServicePort, ServiceSpec, Volume, - VolumeMount, EnvVarSource, ObjectFieldSelector, + Container, EnvVar, EnvVarSource, ObjectFieldSelector, PodSecurityContext, PodSpec, + PodTemplateSpec, ResourceRequirements, Secret, Service, ServicePort, ServiceSpec, + Volume, VolumeMount, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -43,15 +43,14 @@ pub fn create_secret_from_files( #[allow(clippy::too_many_arguments)] pub fn create_replica_set( - name: &ValidatorType, + name: &str, namespace: &str, label_selector: &BTreeMap, - image_name: &DockerImage, + image: &DockerImage, environment_variables: Vec, command: &[String], volumes: Option>, volume_mounts: Option>, - readiness_probe: Option, pod_requests: BTreeMap, ) -> Result> { let pod_spec = PodTemplateSpec { @@ -61,13 +60,12 @@ pub fn create_replica_set( }), spec: Some(PodSpec { containers: vec![Container { - name: format!("{}-{}", image_name.validator_type(), "container"), - image: Some(image_name.to_string()), + name: format!("{}-{}", image.validator_type(), "container"), + image: Some(image.to_string()), image_pull_policy: Some("Always".to_string()), env: Some(environment_variables), command: Some(command.to_owned()), volume_mounts, - readiness_probe, resources: Some(ResourceRequirements { requests: Some(pod_requests), ..Default::default() diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 7230b3c..2a6153f 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -1,7 +1,6 @@ use { crate::{ docker::DockerImage, k8s_helpers, validator_config::ValidatorConfig, Metrics, ValidatorType, - genesis::DEFAULT_INTERNAL_NODE_SOL_TO_STAKE_SOL_RATIO, }, k8s_openapi::{ api::{ @@ -69,6 +68,10 @@ impl<'a> Kubernetes<'a> { } } + pub fn set_shred_version(&mut self, shred_version: u16) { + self.validator_config.shred_version = Some(shred_version); + } + pub async fn namespace_exists(&self) -> Result { let namespaces: Api = Api::all(self.k8s_client.clone()); let namespace_list = namespaces.list(&ListParams::default()).await?; @@ -189,7 +192,7 @@ impl<'a> Kubernetes<'a> { command.extend(self.generate_bootstrap_command_flags()); k8s_helpers::create_replica_set( - &ValidatorType::Bootstrap, + &ValidatorType::Bootstrap.to_string(), self.namespace.as_str(), label_selector, image_name, @@ -197,7 +200,6 @@ impl<'a> Kubernetes<'a> { &command, accounts_volume, accounts_volume_mount, - None, self.pod_requests.requests.clone(), ) } @@ -383,23 +385,24 @@ impl<'a> Kubernetes<'a> { ] } - fn generate_validator_command_flags(&self, validator_stake: &Option) -> Vec { + fn add_known_validators_if_exists(&self, flags: &mut Vec) { + if let Some(known_validators) = &self.validator_config.known_validators { + for key in known_validators.iter() { + flags.push("--known-validator".to_string()); + flags.push(key.to_string()); + } + } + } + + fn generate_validator_command_flags(&self) -> Vec { let mut flags: Vec = Vec::new(); self.generate_command_flags(&mut flags); flags.push("--internal-node-stake-sol".to_string()); - if let Some(stake) = validator_stake { - flags.push(stake.to_string()); - } else { - flags.push(self.validator_config.internal_node_stake_sol.to_string()); - } + flags.push(self.validator_config.internal_node_stake_sol.to_string()); flags.push("--internal-node-sol".to_string()); - if let Some(stake) = validator_stake { - flags.push((DEFAULT_INTERNAL_NODE_SOL_TO_STAKE_SOL_RATIO * stake).to_string()); - } else { - flags.push(self.validator_config.internal_node_sol.to_string()); - } + flags.push(self.validator_config.internal_node_sol.to_string()); if let Some(shred_version) = self.validator_config.shred_version { flags.push("--expected-shred-version".to_string()); @@ -413,12 +416,10 @@ impl<'a> Kubernetes<'a> { pub fn create_validator_replica_set( &mut self, - container_name: &str, - validator_index: i32, - image_name: &str, + image: &DockerImage, secret_name: Option, label_selector: &BTreeMap, - validator_stake: &Option, + validator_index: usize, ) -> Result> { let mut env_vars = self.set_non_bootstrap_environment_variables(); if self.metrics.is_some() { @@ -443,23 +444,17 @@ impl<'a> Kubernetes<'a> { let mut command = vec!["/home/solana/k8s-cluster-scripts/validator-startup-script.sh".to_string()]; - command.extend(self.generate_validator_command_flags(validator_stake)); - - for c in command.iter() { - debug!("validator command: {}", c); - } - + command.extend(self.generate_validator_command_flags()); k8s_helpers::create_replica_set( - &ValidatorType::Standard, + &format!("{}-{validator_index}", ValidatorType::Standard), self.namespace.as_str(), label_selector, - image_name, + image, env_vars, &command, accounts_volume, accounts_volume_mount, - None, self.pod_requests.requests.clone(), ) } diff --git a/src/ledger_helper.rs b/src/ledger_helper.rs index 5223dde..14ac6d2 100644 --- a/src/ledger_helper.rs +++ b/src/ledger_helper.rs @@ -1,16 +1,16 @@ use { - crate::{genesis::DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, LEDGER_DIR}, + crate::genesis::DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, log::*, solana_accounts_db::hardened_unpack::open_genesis_config, solana_sdk::shred_version::compute_shred_version, - std::error::Error, + std::{error::Error, path::PathBuf}, }; -fn ledger_directory_exists() -> Result<(), Box> { - if !LEDGER_DIR.exists() { - return Err(format!( - "Ledger Directory does not exist, have you created genesis yet??" - ).into()); +fn ledger_directory_exists(ledger_dir: &PathBuf) -> Result<(), Box> { + if !ledger_dir.exists() { + return Err( + format!("Ledger Directory does not exist, have you created genesis yet??").into(), + ); } Ok(()) } @@ -18,14 +18,14 @@ fn ledger_directory_exists() -> Result<(), Box> { pub struct LedgerHelper {} impl LedgerHelper { - pub fn get_shred_version() -> Result> { - ledger_directory_exists()?; + pub fn get_shred_version(ledger_dir: &PathBuf) -> Result> { + ledger_directory_exists(ledger_dir)?; let genesis_config = open_genesis_config( - LEDGER_DIR.as_path(), + ledger_dir.as_path(), DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, ); let shred_version = compute_shred_version(&genesis_config?.hash(), None); info!("Shred Version: {}", shred_version); Ok(shred_version) } -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index ce730f5..83d6a39 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,8 +9,11 @@ use { strum::VariantNames, validator_lab::{ docker::{DockerConfig, DockerImage}, - genesis::{Genesis, GenesisFlags, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL}, + genesis::{ + Genesis, GenesisFlags, DEFAULT_INTERNAL_NODE_SOL, DEFAULT_INTERNAL_NODE_STAKE_SOL, + }, kubernetes::{Kubernetes, PodRequests}, + ledger_helper::LedgerHelper, release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, validator_config::ValidatorConfig, @@ -504,7 +507,8 @@ async fn main() { } } - match LedgerHelper::get_shred_version() { + let ledger_dir = config_directory.join("bootstrap-validator"); + match LedgerHelper::get_shred_version(&ledger_dir) { Ok(shred_version) => kub_controller.set_shred_version(shred_version), Err(err) => { error!("{}", err); @@ -732,7 +736,7 @@ async fn main() { validator.add_label( "validator/name", - &format!("validator-{}", validator_index), + &format!("validator-{validator_index}"), LabelType::ValidatorReplicaSet, ); validator.add_label( @@ -747,16 +751,14 @@ async fn main() { ); let validator_replica_set = match kub_controller.create_validator_replica_set( - validator_container_name, + validator.image(), + validator.secret().metadata.name.clone(), + validator.replica_set_labels(), validator_index, - validator_image_name, - validator_secret.metadata.name.clone(), - &validator_labels, - &stake, ) { Ok(replica_set) => replica_set, Err(err) => { - error!("Error creating validator replicas_set: {}", err); + error!("Error creating validator replicas_set: {err}"); return; } }; @@ -766,16 +768,12 @@ async fn main() { .await { Ok(rs) => { - info!( - "validator replica set ({}) deployed successfully", - validator_index - ); + info!("validator replica set ({validator_index}) deployed successfully"); rs.metadata.name.unwrap() } Err(err) => { error!( - "Error! Failed to deploy validator replica set: {}. err: {:?}", - validator_index, err + "Error! Failed to deploy validator replica set: {validator_index}. err: {err}" ); return; } From 8c191f5a12966227624a134dfbde07d10c6173c1 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 3 Apr 2024 22:15:36 +0000 Subject: [PATCH 25/26] working. build and deploy validators --- src/docker.rs | 9 +++++++-- src/kubernetes.rs | 5 +++-- src/ledger_helper.rs | 16 ++++++++-------- ....sh => bootstrap-validator-startup-script.sh} | 0 4 files changed, 18 insertions(+), 12 deletions(-) rename src/scripts/{bootstrap-startup-script.sh => bootstrap-validator-startup-script.sh} (100%) diff --git a/src/docker.rs b/src/docker.rs index 6e60e4c..14344f0 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -155,10 +155,15 @@ impl DockerConfig { fs::create_dir_all(docker_path)?; if let DeployMethod::Local(_) = self.deploy_method { - if validator_type == &ValidatorType::Bootstrap { + if validator_type == &ValidatorType::Bootstrap + || validator_type == &ValidatorType::Standard + { let manifest_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR")); - let files_to_copy = ["bootstrap-startup-script.sh", "common.sh"]; + let files_to_copy = [ + format!("{validator_type}-startup-script.sh"), + "common.sh".to_string(), + ]; for file_name in files_to_copy.iter() { Self::copy_file_to_docker(&manifest_path, docker_path, file_name)?; } diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 2a6153f..936826f 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -187,8 +187,9 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]); - let mut command = - vec!["/home/solana/k8s-cluster-scripts/bootstrap-startup-script.sh".to_string()]; + let mut command = vec![ + "/home/solana/k8s-cluster-scripts/bootstrap-validator-startup-script.sh".to_string(), + ]; command.extend(self.generate_bootstrap_command_flags()); k8s_helpers::create_replica_set( diff --git a/src/ledger_helper.rs b/src/ledger_helper.rs index 14ac6d2..9f369b3 100644 --- a/src/ledger_helper.rs +++ b/src/ledger_helper.rs @@ -3,13 +3,15 @@ use { log::*, solana_accounts_db::hardened_unpack::open_genesis_config, solana_sdk::shred_version::compute_shred_version, - std::{error::Error, path::PathBuf}, + std::{error::Error, path::Path}, }; -fn ledger_directory_exists(ledger_dir: &PathBuf) -> Result<(), Box> { +fn ledger_directory_exists(ledger_dir: &Path) -> Result<(), Box> { if !ledger_dir.exists() { return Err( - format!("Ledger Directory does not exist, have you created genesis yet??").into(), + "Ledger Directory does not exist, have you created genesis yet??" + .to_string() + .into(), ); } Ok(()) @@ -18,12 +20,10 @@ fn ledger_directory_exists(ledger_dir: &PathBuf) -> Result<(), Box> { pub struct LedgerHelper {} impl LedgerHelper { - pub fn get_shred_version(ledger_dir: &PathBuf) -> Result> { + pub fn get_shred_version(ledger_dir: &Path) -> Result> { ledger_directory_exists(ledger_dir)?; - let genesis_config = open_genesis_config( - ledger_dir.as_path(), - DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, - ); + let genesis_config = + open_genesis_config(ledger_dir, DEFAULT_MAX_GENESIS_ARCHIVE_UNPACKED_SIZE); let shred_version = compute_shred_version(&genesis_config?.hash(), None); info!("Shred Version: {}", shred_version); Ok(shred_version) diff --git a/src/scripts/bootstrap-startup-script.sh b/src/scripts/bootstrap-validator-startup-script.sh similarity index 100% rename from src/scripts/bootstrap-startup-script.sh rename to src/scripts/bootstrap-validator-startup-script.sh From 7e53ece23687f57ea7ff82cdefaad3cbac17062a Mon Sep 17 00:00:00 2001 From: greg Date: Thu, 4 Apr 2024 04:23:54 +0000 Subject: [PATCH 26/26] update scripts to work with agave and solana validator versions --- PROGRESS.md | 4 +-- src/kubernetes.rs | 8 ++++++ src/main.rs | 12 +++++++++ src/release.rs | 10 ++++---- .../bootstrap-validator-startup-script.sh | 18 +++++++++++-- src/scripts/validator-startup-script.sh | 25 ++++++++++++++++--- 6 files changed, 64 insertions(+), 13 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index a8ff8b4..5e7c4d0 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -17,7 +17,7 @@ - [x] Build genesis - [x] Docker Build and Push to registry - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Secrets @@ -37,7 +37,7 @@ - [ ] Client - [ ] Create & Deploy Services - [x] Bootstrap - - [ ] Validator (regular) + - [x] Validator (regular) - [ ] RPC nodes - [ ] Client - [x] Check Bootstrap is deployed and running diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 936826f..15d1a0f 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -459,4 +459,12 @@ impl<'a> Kubernetes<'a> { self.pod_requests.requests.clone(), ) } + + pub fn create_validator_service( + &self, + service_name: &str, + label_selector: &BTreeMap, + ) -> Service { + k8s_helpers::create_service(service_name, self.namespace.as_str(), label_selector, false) + } } diff --git a/src/main.rs b/src/main.rs index 83d6a39..3d84f85 100644 --- a/src/main.rs +++ b/src/main.rs @@ -778,5 +778,17 @@ async fn main() { return; } }; + + // let service_name = format!("validator-service-{index}"); + let validator_service = kub_controller.create_validator_service( + &format!("validator-service-{validator_index}"), + validator.replica_set_labels(), + ); + match kub_controller.deploy_service(&validator_service).await { + Ok(_) => info!("validator service ({validator_index}) deployed successfully"), + Err(err) => { + error!("Error! Failed to deploy validator service: {validator_index}. err: {err}") + } + } } } diff --git a/src/release.rs b/src/release.rs index 49fb08f..ebda79f 100644 --- a/src/release.rs +++ b/src/release.rs @@ -63,6 +63,10 @@ impl BuildConfig { } pub async fn prepare(&self) -> Result<(), Box> { + if self.build_type == BuildType::Skip { + info!("skipping build"); + return Ok(()); + } match &self.deploy_method { DeployMethod::ReleaseChannel(channel) => match self.setup_tar_deploy(channel).await { Ok(tar_directory) => { @@ -97,11 +101,7 @@ impl BuildConfig { } fn setup_local_deploy(&self) -> Result<(), Box> { - if self.build_type != BuildType::Skip { - self.build()?; - } else { - info!("Build skipped due to --build-type skip"); - } + self.build()?; Ok(()) } diff --git a/src/scripts/bootstrap-validator-startup-script.sh b/src/scripts/bootstrap-validator-startup-script.sh index 56e0a7e..f9c72d0 100755 --- a/src/scripts/bootstrap-validator-startup-script.sh +++ b/src/scripts/bootstrap-validator-startup-script.sh @@ -7,10 +7,24 @@ nohup solana-faucet --keypair bootstrap-accounts/faucet.json & # Start the bootstrap validator node # shellcheck disable=SC1091 source /home/solana/k8s-cluster-scripts/common.sh +no_restart=0 -program="agave-validator" +# Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` +agave_validator="/home/solana/.cargo/bin/agave-validator" +solana_validator="/home/solana/.cargo/bin/solana-validator" -no_restart=0 +# Initialize program variable +program="" + +# Check if agave-validator exists and is executable +if [[ -x "$agave_validator" ]]; then + program="agave-validator" +elif [[ -x "$solana_validator" ]]; then + program="solana-validator" +else + echo "Neither agave-validator nor solana-validator could be found or is not executable." + exit 1 +fi echo "PROGRAM: $program" diff --git a/src/scripts/validator-startup-script.sh b/src/scripts/validator-startup-script.sh index 3e58b58..40295c4 100755 --- a/src/scripts/validator-startup-script.sh +++ b/src/scripts/validator-startup-script.sh @@ -9,7 +9,8 @@ args=( --no-os-network-limits-test ) airdrops_enabled=1 -# next two values will be overwritten based on command line args. default is set here: k8s-cluster/src/genesis.rs +node_sol= +stake_sol= identity=validator-accounts/identity.json vote_account=validator-accounts/vote.json no_restart=0 @@ -17,6 +18,25 @@ gossip_entrypoint=$BOOTSTRAP_GOSSIP_ADDRESS ledger_dir=/home/solana/ledger faucet_address=$LOAD_BALANCER_FAUCET_ADDRESS +# Define the paths to the validator cli. pre 1.18 is `solana-validator`. post 1.18 is `agave-validator` +agave_validator="/home/solana/.cargo/bin/agave-validator" +solana_validator="/home/solana/.cargo/bin/solana-validator" + +# Initialize program variable +program="" + +# Check if agave-validator exists and is executable +if [[ -x "$agave_validator" ]]; then + program="agave-validator" +elif [[ -x "$solana_validator" ]]; then + program="solana-validator" +else + echo "Neither agave-validator nor solana-validator could be found or is not executable." + exit 1 +fi + +echo "PROGRAM: $program" + usage() { if [[ -n $1 ]]; then echo "$*" @@ -236,9 +256,6 @@ default_arg --allow-private-addr default_arg --gossip-port 8001 default_arg --rpc-port 8899 -program="agave-validator" -echo "program: $program" - PS4="$(basename "$0"): " echo "PS4: $PS4"