Skip to content

Commit

Permalink
address jon nits from anza-xyz#38
Browse files Browse the repository at this point in the history
  • Loading branch information
gregcusack committed May 1, 2024
1 parent da24112 commit 3cd237c
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 144 deletions.
5 changes: 1 addition & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,13 @@ kubectl create ns <namespace>
cargo run --bin cluster --
-n <namespace>
--local-path <path-to-local-agave-monorepo>
--validator-lab-dir <path-to-validator-lab-directory>
```

#### Build specific Agave release
```
cargo run --bin cluster --
-n <namespace>
--release-channel <agave-version: e.g. v1.17.28> # note: MUST include the "v"
--validator-lab-dir <path-to-validator-lab-directory>
```

#### Build from Local Repo and Configure Genesis and Bootstrap Validator Image
Expand All @@ -44,7 +42,6 @@ Example:
cargo run --bin cluster --
-n <namespace>
--local-path /home/sol/solana
--validator-lab-dir /home/sol/validator-lab
# genesis config. Optional: Many of these have defaults
--hashes-per-tick <hashes-per-tick>
--enable-warmup-epochs <true|false>
Expand All @@ -66,7 +63,7 @@ cargo run --bin cluster --
```
cd scripts/
./init-metrics -c <database-name> <metrics-username>
# enter password when promted
# enter password when prompted
```
2) add the following to your `cluster` command from above
```
Expand Down
7 changes: 2 additions & 5 deletions src/kubernetes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -264,10 +264,7 @@ impl<'a> Kubernetes<'a> {
)
}

pub async fn check_replica_set_ready(
&self,
replica_set_name: &str,
) -> Result<bool, kube::Error> {
pub async fn is_replica_set_ready(&self, replica_set_name: &str) -> Result<bool, kube::Error> {
let replica_sets: Api<ReplicaSet> =
Api::namespaced(self.k8s_client.clone(), self.namespace.as_str());
let replica_set = replica_sets.get(replica_set_name).await?;
Expand All @@ -283,7 +280,7 @@ impl<'a> Kubernetes<'a> {
Ok(available_validators >= desired_validators)
}

pub fn create_metrics_secret(&self) -> Result<Secret, Box<dyn Error>> {
pub fn create_metrics_secret(&self) -> Result<Secret, Box<dyn std::error::Error>> {
let mut data = BTreeMap::new();
if let Some(metrics) = &self.metrics {
data.insert(
Expand Down
193 changes: 58 additions & 135 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use {
DEFAULT_MAX_LEDGER_SHREDS, DEFAULT_MIN_MAX_LEDGER_SHREDS,
},
solana_sdk::{signature::keypair::read_keypair_file, signer::Signer},
std::{fs, path::PathBuf},
std::{fs, path::PathBuf, result::Result},
strum::VariantNames,
validator_lab::{
cluster_images::ClusterImages,
Expand Down Expand Up @@ -257,7 +257,7 @@ fn parse_matches() -> clap::ArgMatches {
}

#[tokio::main]
async fn main() {
async fn main() -> Result<(), Box<dyn std::error::Error>> {
if std::env::var("RUST_LOG").is_err() {
std::env::set_var("RUST_LOG", "INFO");
}
Expand Down Expand Up @@ -295,16 +295,18 @@ async fn main() {

if let Ok(metadata) = fs::metadata(solana_root.get_root_path()) {
if !metadata.is_dir() {
return error!(
return Err(format!(
"Build path is not a directory: {}",
solana_root.get_root_path().display()
);
)
.into());
}
} else {
return error!(
return Err(format!(
"Build directory not found: {}",
solana_root.get_root_path().display()
);
)
.into());
}

let build_config = BuildConfig::new(
Expand Down Expand Up @@ -405,59 +407,33 @@ async fn main() {
)
.await;

match kub_controller.namespace_exists().await {
Ok(true) => (),
Ok(false) => {
error!(
"Namespace: '{}' doesn't exist. Exiting...",
environment_config.namespace
);
return;
}
Err(err) => {
error!("Error: {err}");
return;
}
let exists = kub_controller.namespace_exists().await?;
if !exists {
return Err(format!(
"Namespace: '{}' doesn't exist. Exiting...",
environment_config.namespace
)
.into());
}

match build_config.prepare().await {
Ok(_) => info!("Validator setup prepared successfully"),
Err(err) => {
error!("Error: {err}");
return;
}
}
build_config.prepare().await?;
info!("Validator setup prepared successfully");

let config_directory = solana_root.get_root_path().join("config-k8s");
let mut genesis = Genesis::new(config_directory.clone(), genesis_flags);

match genesis.generate_faucet() {
Ok(_) => info!("Generated faucet account"),
Err(err) => {
error!("generate faucet error! {err}");
return;
}
}
genesis.generate_faucet()?;
info!("Generated faucet account");

match genesis.generate_accounts(ValidatorType::Bootstrap, 1) {
Ok(_) => info!("Generated bootstrap account"),
Err(err) => {
error!("generate accounts error! {err}");
return;
}
}
genesis.generate_accounts(ValidatorType::Bootstrap, 1)?;
info!("Generated bootstrap account");

// creates genesis and writes to binary file
match genesis
genesis
.generate(solana_root.get_root_path(), &build_path)
.await
{
Ok(_) => info!("Created genesis successfully"),
Err(err) => {
error!("generate genesis error! {err}");
return;
}
}
.await?;

info!("Created genesis successfully");

//unwraps are safe here. since their requirement is enforced by argmatches
let docker = DockerConfig::new(
Expand All @@ -484,61 +460,29 @@ async fn main() {

if build_config.docker_build() {
for v in cluster_images.get_validators() {
match docker.build_image(solana_root.get_root_path(), v.image()) {
Ok(_) => info!("{} image built successfully", v.validator_type()),
Err(err) => {
error!("Failed to build docker image {err}");
return;
}
}
docker.build_image(solana_root.get_root_path(), v.image())?;
info!("{} image built successfully", v.validator_type());
}

match docker.push_images(cluster_images.get_validators()) {
Ok(_) => info!("Validator images pushed successfully"),
Err(err) => {
error!("Failed to push Validator docker image {err}");
return;
}
}
docker.push_images(cluster_images.get_validators())?;
info!("Validator images pushed successfully");
}

// metrics secret create once and use by all pods
if kub_controller.metrics.is_some() {
let metrics_secret = match kub_controller.create_metrics_secret() {
Ok(secret) => secret,
Err(err) => {
error!("Failed to create metrics secret! {err}");
return;
}
};
match kub_controller.deploy_secret(&metrics_secret).await {
Ok(_) => (),
Err(err) => {
error!("{err}");
return;
}
}
let metrics_secret = kub_controller.create_metrics_secret()?;
kub_controller.deploy_secret(&metrics_secret).await?;
};

let bootstrap_validator = cluster_images.bootstrap().expect("should be bootstrap");
match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) {
Ok(secret) => bootstrap_validator.set_secret(secret),
Err(err) => {
error!("Failed to create bootstrap secret! {err}");
return;
}
};
let bootstrap_validator = cluster_images.bootstrap()?;
let secret =
kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory)?;
bootstrap_validator.set_secret(secret);

match kub_controller
kub_controller
.deploy_secret(bootstrap_validator.secret())
.await
{
Ok(_) => info!("Deployed Bootstrap Secret"),
Err(err) => {
error!("{err}");
return;
}
}
.await?;
info!("Deployed Bootstrap Secret");

// Create Bootstrap labels
// Bootstrap needs two labels, one for each service.
Expand All @@ -564,46 +508,31 @@ async fn main() {
);

// create bootstrap replica set
match kub_controller.create_bootstrap_validator_replica_set(
let replica_set = kub_controller.create_bootstrap_validator_replica_set(
bootstrap_validator.image(),
bootstrap_validator.secret().metadata.name.clone(),
bootstrap_validator.replica_set_labels(),
) {
Ok(replica_set) => bootstrap_validator.set_replica_set(replica_set),
Err(err) => {
error!("Error creating bootstrap validator replicas_set: {err}");
return;
}
};
)?;
bootstrap_validator.set_replica_set(replica_set);

// deploy bootstrap replica set
match kub_controller
kub_controller
.deploy_replicas_set(bootstrap_validator.replica_set())
.await
{
Ok(_) => {
info!(
"{} deployed successfully",
bootstrap_validator.replica_set_name()
);
}
Err(err) => {
error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}");
return;
}
};
.await?;
info!(
"{} deployed successfully",
bootstrap_validator.replica_set_name()
);

// create and deploy bootstrap-service
let bootstrap_service = kub_controller.create_bootstrap_service(
"bootstrap-validator-service",
bootstrap_validator.service_labels(),
);
match kub_controller.deploy_service(&bootstrap_service).await {
Ok(_) => info!("bootstrap validator service deployed successfully"),
Err(err) => error!("Error! Failed to deploy bootstrap validator service. err: {err}"),
}
kub_controller.deploy_service(&bootstrap_service).await?;
info!("bootstrap validator service deployed successfully");

//load balancer service. only create one and use for all bootstrap/rpc nodes
// load balancer service. only create one and use for all bootstrap/rpc nodes
// service selector matches bootstrap selector
let load_balancer_label =
kub_controller.create_selector("load-balancer/name", "load-balancer-selector");
Expand All @@ -612,25 +541,19 @@ async fn main() {
.create_validator_load_balancer("bootstrap-and-rpc-node-lb-service", &load_balancer_label);

//deploy load balancer
match kub_controller.deploy_service(&load_balancer).await {
Ok(_) => info!("load balancer service deployed successfully"),
Err(err) => error!("Error! Failed to deploy load balancer service. err: {err}"),
}
kub_controller.deploy_service(&load_balancer).await?;
info!("load balancer service deployed successfully");

// wait for bootstrap replicaset to deploy
while {
match kub_controller
.check_replica_set_ready(bootstrap_validator.replica_set_name().as_str())
.await
{
Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false)
Err(_) => panic!("Error occurred while checking replica set readiness"),
}
} {
while !kub_controller
.is_replica_set_ready(bootstrap_validator.replica_set_name().as_str())
.await?
{
info!(
"replica set: {} not ready...",
bootstrap_validator.replica_set_name()
);
std::thread::sleep(std::time::Duration::from_secs(1));
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
Ok(())
}

0 comments on commit 3cd237c

Please sign in to comment.