From e0d11845cbee29f43137534beb014bdb96a933b3 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 01:08:46 +0000 Subject: [PATCH 1/6] deploy bootstrap validator --- PROGRESS.md | 2 +- src/kubernetes.rs | 11 +++++++++++ src/main.rs | 17 +++++++++++++++++ src/startup_scripts.rs | 0 4 files changed, 29 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/startup_scripts.rs diff --git a/PROGRESS.md b/PROGRESS.md index 8db6e91..45e078f 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -29,7 +29,7 @@ - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Replica Set - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 4c43c18..33d98f5 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -209,4 +209,15 @@ impl<'a> Kubernetes<'a> { pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } + + pub async fn deploy_replicas_set( + &self, + replica_set: &ReplicaSet, + ) -> Result { + let api: Api = + Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let post_params = PostParams::default(); + // Apply the ReplicaSet + api.create(&post_params, replica_set).await + } } diff --git a/src/main.rs b/src/main.rs index d98abee..610fca7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -513,4 +513,21 @@ async fn main() { return; } }; + + // deploy bootstrap replica set + match kub_controller + .deploy_replicas_set(bootstrap_validator.replica_set()) + .await + { + Ok(_) => { + info!( + "{} deployed successfully", + bootstrap_validator.replica_set_name() + ); + } + Err(err) => { + error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); + return; + } + }; } diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs old mode 100644 new mode 100755 From 3e8a67a92cc263080fffd6d6656777080e4ab73e Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 01:29:37 +0000 Subject: [PATCH 2/6] wip. added bootstrap service. need lb service --- src/k8s_helpers.rs | 50 +++++++++++++++++++++++++++++++++++++++++++++- src/kubernetes.rs | 19 +++++++++++++++++- src/main.rs | 42 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 950ea97..e0512c6 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -5,7 +5,8 @@ use { apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, Secret, Volume, VolumeMount, + ResourceRequirements, Secret, Volume, VolumeMount, Service, ServiceSpec, + ServicePort, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -119,3 +120,50 @@ pub fn create_replica_set( ..Default::default() }) } + +pub fn create_service( + service_name: &str, + namespace: &str, + label_selector: &BTreeMap, + is_load_balancer: bool, +) -> Service { + Service { + metadata: ObjectMeta { + name: Some(service_name.to_string()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: Some(ServiceSpec { + selector: Some(label_selector.clone()), + type_: if is_load_balancer { + Some("LoadBalancer".to_string()) + } else { + None + }, + cluster_ip: if is_load_balancer { + None + } else { + Some("None".to_string()) + }, + ports: Some(vec![ + ServicePort { + port: 8899, // RPC Port + name: Some("rpc-port".to_string()), + ..Default::default() + }, + ServicePort { + port: 8001, //Gossip Port + name: Some("gossip-port".to_string()), + ..Default::default() + }, + ServicePort { + port: 9900, //Faucet Port + name: Some("faucet-port".to_string()), + ..Default::default() + }, + ]), + ..Default::default() + }), + ..Default::default() + } +} \ No newline at end of file diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 33d98f5..ba8704f 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -10,7 +10,7 @@ use { apps::v1::ReplicaSet, core::v1::{ EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, - Volume, VolumeMount, + Volume, VolumeMount, Service, }, }, apimachinery::pkg::api::resource::Quantity, @@ -220,4 +220,21 @@ impl<'a> Kubernetes<'a> { // Apply the ReplicaSet api.create(&post_params, replica_set).await } + + pub fn create_bootstrap_service( + &self, + service_name: &str, + label_selector: &BTreeMap, + ) -> Service { + k8s_helpers::create_service(service_name, self.namespace.as_str(), label_selector, false) + } + + pub async fn deploy_service(&self, service: &Service) -> Result { + let post_params = PostParams::default(); + // Create an API instance for Services in the specified namespace + let service_api: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + + // Create the Service object in the cluster + service_api.create(&post_params, service).await + } } diff --git a/src/main.rs b/src/main.rs index 610fca7..559719b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -530,4 +530,46 @@ async fn main() { return; } }; + //////////////////////////////////////////////////////////////////////////////////////////// + /// ////////////////////////////////////////////////////////////////////////////////// + + let bootstrap_service = kub_controller + .create_bootstrap_service("bootstrap-validator-service", bootstrap_validator.service_labels()); + match kub_controller.deploy_service(&bootstrap_service).await { + Ok(_) => info!("bootstrap validator service deployed successfully"), + Err(err) => error!( + "Error! Failed to deploy bootstrap validator service. err: {:?}", + err + ), + } + + //load balancer service. only create one and use for all bootstrap/rpc nodes + // service selector matches bootstrap selector + let load_balancer_label = + kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); + //create load balancer + let load_balancer = kub_controller.create_validator_load_balancer( + "bootstrap-and-rpc-node-lb-service", + &load_balancer_label, + ); + + //deploy load balancer + match kub_controller.deploy_service(&load_balancer).await { + Ok(_) => info!("load balancer service deployed successfully"), + Err(err) => error!("Error! Failed to deploy load balancer service. err: {err}"), + } + + // wait for bootstrap replicaset to deploy + while { + match kub_controller + .check_replica_set_ready(bootstrap_validator.replica_set_name()) + .await + { + Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) + Err(_) => panic!("Error occurred while checking replica set readiness"), + } + } { + info!("replica set: {} not ready...", bootstrap_validator.replica_set_name()); + std::thread::sleep(std::time::Duration::from_secs(1)); + } } From 31d6a229f49429ac091acc6215a42956ab0fa460 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 17:15:26 +0000 Subject: [PATCH 3/6] deploy and wait for validator ready. need readme update --- src/k8s_helpers.rs | 2 +- src/kubernetes.rs | 26 ++++++++++++++++++++++++++ src/main.rs | 5 ++--- src/validator.rs | 7 +++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index e0512c6..74d7306 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -166,4 +166,4 @@ pub fn create_service( }), ..Default::default() } -} \ No newline at end of file +} diff --git a/src/kubernetes.rs b/src/kubernetes.rs index ba8704f..a7169d1 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -237,4 +237,30 @@ impl<'a> Kubernetes<'a> { // Create the Service object in the cluster service_api.create(&post_params, service).await } + + pub fn create_validator_load_balancer( + &self, + service_name: &str, + label_selector: &BTreeMap, + ) -> Service { + k8s_helpers::create_service(service_name, self.namespace.as_str(), label_selector, true) + } + + pub async fn check_replica_set_ready( + &self, + replica_set_name: &str, + ) -> Result { + let replica_sets: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let replica_set = replica_sets.get(replica_set_name).await?; + + let desired_validators = replica_set.spec.as_ref().unwrap().replicas.unwrap_or(1); + let available_validators = replica_set + .status + .as_ref() + .unwrap() + .available_replicas + .unwrap_or(0); + + Ok(available_validators >= desired_validators) + } } diff --git a/src/main.rs b/src/main.rs index 559719b..6cfe8c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,6 +20,7 @@ use { validator_config::ValidatorConfig, EnvironmentConfig, SolanaRoot, ValidatorType, }, + std::{thread, time::Duration}, }; fn parse_matches() -> clap::ArgMatches { @@ -530,8 +531,6 @@ async fn main() { return; } }; - //////////////////////////////////////////////////////////////////////////////////////////// - /// ////////////////////////////////////////////////////////////////////////////////// let bootstrap_service = kub_controller .create_bootstrap_service("bootstrap-validator-service", bootstrap_validator.service_labels()); @@ -562,7 +561,7 @@ async fn main() { // wait for bootstrap replicaset to deploy while { match kub_controller - .check_replica_set_ready(bootstrap_validator.replica_set_name()) + .check_replica_set_ready(bootstrap_validator.replica_set_name().as_str()) .await { Ok(ok) => !ok, // Continue the loop if replica set is not ready: Ok(false) diff --git a/src/validator.rs b/src/validator.rs index 82f5640..e9b2af1 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -17,10 +17,16 @@ pub struct Validator { info_labels: BTreeMap, replica_set: ReplicaSet, service_labels: BTreeMap, + // load_balancer_labels: Option>, } impl Validator { pub fn new(image: DockerImage) -> Self { + // let load_balancer_labels = match image.validator_type() { + // ValidatorType::Bootstrap | ValidatorType::RPC => Some(BTreeMap::new()), + // ValidatorType::Client | ValidatorType::Standard => None, + // }; + Self { validator_type: image.validator_type(), image, @@ -28,6 +34,7 @@ impl Validator { info_labels: BTreeMap::new(), replica_set: ReplicaSet::default(), service_labels: BTreeMap::new(), + // load_balancer_labels, } } From 05a41852216b30b42a10e4e191c1f2471bd0a437 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 20:48:12 +0000 Subject: [PATCH 4/6] update readme. update progress. fix selector bug --- PROGRESS.md | 8 ++++---- README.md | 31 +++++++++++++++++++++++++++++++ src/k8s_helpers.rs | 16 ++++++++-------- src/kubernetes.rs | 12 +++++++----- src/main.rs | 18 ++++++++++-------- src/validator.rs | 7 ------- 6 files changed, 60 insertions(+), 32 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 45e078f..5503a9b 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -34,12 +34,12 @@ - [ ] RPC nodes - [ ] Client - [ ] Create & Deploy Services - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client -- [ ] Check Bootstrap is deployed and running -- [ ] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) +- [x] Check Bootstrap is deployed and running +- [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) - [ ] Add metrics - [ ] Bootstrap - [ ] Validator (regular) @@ -69,7 +69,7 @@ Above, we start with bootstrap, and then we do validators (regular), and then we - Use command line flags to set type of client, tx-count, etc - [ ] Add in kubernetes deployment flags - - [ ] CPU/Memory Requests + - [x] CPU/Memory Requests - [ ] Node Affinity -> Regions - [ ] Node Affinity -> Node Type (Equinix/Lumen) diff --git a/README.md b/README.md index f6da3f4..53e8a7d 100644 --- a/README.md +++ b/README.md @@ -59,4 +59,35 @@ cargo run --bin cluster -- --tag # e.g. v1 --base-image # e.g. ubuntu:20.04 --image-name # e.g. cluster-image +``` + +## Kubernetes Cheatsheet +Create namespace: +``` +kubectl create ns +``` + +Delete namespace: +``` +kubectl delete ns +``` + +Get running pods: +``` +kubectl get pods -n +``` + +Get pod logs: +``` +kubectl logs -n +``` + +Exec into pod: +``` +kubectl exec -it -n -- /bin/bash +``` + +Get information about pod: +``` +kubectl describe pod -n ``` \ No newline at end of file diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 74d7306..2d92669 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -5,8 +5,8 @@ use { apps::v1::{ReplicaSet, ReplicaSetSpec}, core::v1::{ Container, EnvVar, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, Secret, Volume, VolumeMount, Service, ServiceSpec, - ServicePort, + ResourceRequirements, Secret, Service, ServicePort, ServiceSpec, Volume, + VolumeMount, }, }, apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, @@ -122,19 +122,19 @@ pub fn create_replica_set( } pub fn create_service( - service_name: &str, - namespace: &str, - label_selector: &BTreeMap, + service_name: String, + namespace: String, + label_selector: BTreeMap, is_load_balancer: bool, ) -> Service { Service { metadata: ObjectMeta { - name: Some(service_name.to_string()), - namespace: Some(namespace.to_string()), + name: Some(service_name), + namespace: Some(namespace), ..Default::default() }, spec: Some(ServiceSpec { - selector: Some(label_selector.clone()), + selector: Some(label_selector), type_: if is_load_balancer { Some("LoadBalancer".to_string()) } else { diff --git a/src/kubernetes.rs b/src/kubernetes.rs index a7169d1..95d8933 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -10,7 +10,7 @@ use { apps::v1::ReplicaSet, core::v1::{ EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, - Volume, VolumeMount, Service, + Service, Volume, VolumeMount, }, }, apimachinery::pkg::api::resource::Quantity, @@ -226,13 +226,14 @@ impl<'a> Kubernetes<'a> { service_name: &str, label_selector: &BTreeMap, ) -> Service { - k8s_helpers::create_service(service_name, self.namespace.as_str(), label_selector, false) + k8s_helpers::create_service(service_name.to_string(), self.namespace.clone(), label_selector.clone(), false) } pub async fn deploy_service(&self, service: &Service) -> Result { let post_params = PostParams::default(); // Create an API instance for Services in the specified namespace - let service_api: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let service_api: Api = + Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); // Create the Service object in the cluster service_api.create(&post_params, service).await @@ -243,14 +244,15 @@ impl<'a> Kubernetes<'a> { service_name: &str, label_selector: &BTreeMap, ) -> Service { - k8s_helpers::create_service(service_name, self.namespace.as_str(), label_selector, true) + k8s_helpers::create_service(service_name.to_string(), self.namespace.clone(), label_selector.clone(), true) } pub async fn check_replica_set_ready( &self, replica_set_name: &str, ) -> Result { - let replica_sets: Api = Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); + let replica_sets: Api = + Api::namespaced(self.k8s_client.clone(), self.namespace.as_str()); let replica_set = replica_sets.get(replica_set_name).await?; let desired_validators = replica_set.spec.as_ref().unwrap().replicas.unwrap_or(1); diff --git a/src/main.rs b/src/main.rs index 6cfe8c3..64d3d9b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,7 +20,6 @@ use { validator_config::ValidatorConfig, EnvironmentConfig, SolanaRoot, ValidatorType, }, - std::{thread, time::Duration}, }; fn parse_matches() -> clap::ArgMatches { @@ -532,14 +531,14 @@ async fn main() { } }; - let bootstrap_service = kub_controller - .create_bootstrap_service("bootstrap-validator-service", bootstrap_validator.service_labels()); + // create and deploy bootstrap-service + let bootstrap_service = kub_controller.create_bootstrap_service( + "bootstrap-validator-service", + bootstrap_validator.service_labels(), + ); match kub_controller.deploy_service(&bootstrap_service).await { Ok(_) => info!("bootstrap validator service deployed successfully"), - Err(err) => error!( - "Error! Failed to deploy bootstrap validator service. err: {:?}", - err - ), + Err(err) => error!("Error! Failed to deploy bootstrap validator service. err: {err}"), } //load balancer service. only create one and use for all bootstrap/rpc nodes @@ -568,7 +567,10 @@ async fn main() { Err(_) => panic!("Error occurred while checking replica set readiness"), } } { - info!("replica set: {} not ready...", bootstrap_validator.replica_set_name()); + info!( + "replica set: {} not ready...", + bootstrap_validator.replica_set_name() + ); std::thread::sleep(std::time::Duration::from_secs(1)); } } diff --git a/src/validator.rs b/src/validator.rs index e9b2af1..82f5640 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -17,16 +17,10 @@ pub struct Validator { info_labels: BTreeMap, replica_set: ReplicaSet, service_labels: BTreeMap, - // load_balancer_labels: Option>, } impl Validator { pub fn new(image: DockerImage) -> Self { - // let load_balancer_labels = match image.validator_type() { - // ValidatorType::Bootstrap | ValidatorType::RPC => Some(BTreeMap::new()), - // ValidatorType::Client | ValidatorType::Standard => None, - // }; - Self { validator_type: image.validator_type(), image, @@ -34,7 +28,6 @@ impl Validator { info_labels: BTreeMap::new(), replica_set: ReplicaSet::default(), service_labels: BTreeMap::new(), - // load_balancer_labels, } } From 7f5cfdba201dfcfcc1a3da17d27f47dfe7f8d704 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 2 Apr 2024 22:07:28 +0000 Subject: [PATCH 5/6] add metrics --- PROGRESS.md | 2 +- README.md | 17 ++++++++ scripts/init-metrics.sh | 87 +++++++++++++++++++++++++++++++++++++++++ src/k8s_helpers.rs | 6 +-- src/kubernetes.rs | 51 +++++++++++++++++++++--- src/lib.rs | 33 ++++++++++++++++ src/main.rs | 65 +++++++++++++++++++++++++++++- 7 files changed, 251 insertions(+), 10 deletions(-) create mode 100755 scripts/init-metrics.sh diff --git a/PROGRESS.md b/PROGRESS.md index 5503a9b..bad7044 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -41,7 +41,7 @@ - [x] Check Bootstrap is deployed and running - [x] Build and deploy Load Balancer (sits in front of bootstrap and RPC nodes) - [ ] Add metrics - - [ ] Bootstrap + - [x] Bootstrap - [ ] Validator (regular) - [ ] RPC nodes - [ ] Client diff --git a/README.md b/README.md index 53e8a7d..5c922a4 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,23 @@ cargo run --bin cluster -- --image-name # e.g. cluster-image ``` +## Metrics +1) Setup metrics database: +``` +cd scripts/ +./init-metrics -c +# enter password when promted +``` +2) add the following to your `cluster` command from above +``` +--metrics-host https://internal-metrics.solana.com # need the `https://` here +--metrics-port 8086 +--metrics-db # from (1) +--metrics-username # from (1) +--metrics-password # from (1) +``` + + ## Kubernetes Cheatsheet Create namespace: ``` diff --git a/scripts/init-metrics.sh b/scripts/init-metrics.sh new file mode 100755 index 0000000..89eae4f --- /dev/null +++ b/scripts/init-metrics.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -e + +here=$(dirname "$0") +# shellcheck source=net/common.sh +source "$here"/common.sh + +usage() { + exitcode=0 + if [[ -n "$1" ]]; then + exitcode=1 + echo "Error: $*" + fi + cat <) -> Secret { +fn build_secret(name: String, data: BTreeMap) -> Secret { Secret { metadata: ObjectMeta { - name: Some(name.to_string()), + name: Some(name), ..Default::default() }, data: Some(data), @@ -33,7 +33,7 @@ fn build_secret(name: &str, data: BTreeMap) -> Secret { } pub fn create_secret( - secret_name: &str, + secret_name: String, secrets: BTreeMap, ) -> Result> { let data = secrets diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 95d8933..51167e9 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -3,17 +3,18 @@ use { docker::DockerImage, k8s_helpers::{self, SecretType}, validator_config::ValidatorConfig, - ValidatorType, + ValidatorType, Metrics, }, k8s_openapi::{ api::{ apps::v1::ReplicaSet, core::v1::{ - EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretVolumeSource, - Service, Volume, VolumeMount, + EnvVar, EnvVarSource, Namespace, ObjectFieldSelector, Secret, SecretKeySelector, + SecretVolumeSource, Service, Volume, VolumeMount, }, }, apimachinery::pkg::api::resource::Quantity, + ByteString, }, kube::{ api::{Api, ListParams, PostParams}, @@ -45,6 +46,7 @@ pub struct Kubernetes<'a> { namespace: String, validator_config: &'a mut ValidatorConfig, pod_requests: PodRequests, + pub metrics: Option, } impl<'a> Kubernetes<'a> { @@ -52,12 +54,14 @@ impl<'a> Kubernetes<'a> { namespace: &str, validator_config: &'a mut ValidatorConfig, pod_requests: PodRequests, + metrics: Option, ) -> Kubernetes<'a> { Self { k8s_client: Client::try_default().await.unwrap(), namespace: namespace.to_owned(), validator_config, pod_requests, + metrics, } } @@ -113,7 +117,7 @@ impl<'a> Kubernetes<'a> { }, ); - k8s_helpers::create_secret(secret_name, secrets) + k8s_helpers::create_secret(secret_name.to_string(), secrets) } fn add_known_validator(&mut self, pubkey: Pubkey) { @@ -133,7 +137,7 @@ impl<'a> Kubernetes<'a> { secret_name: Option, label_selector: &BTreeMap, ) -> Result> { - let env_vars = vec![EnvVar { + let mut env_vars = vec![EnvVar { name: "MY_POD_IP".to_string(), value_from: Some(EnvVarSource { field_ref: Some(ObjectFieldSelector { @@ -145,6 +149,10 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]; + if self.metrics.is_some() { + env_vars.push(self.get_metrics_env_var_secret()) + } + let accounts_volume = Some(vec![Volume { name: "bootstrap-accounts-volume".into(), secret: Some(SecretVolumeSource { @@ -265,4 +273,37 @@ impl<'a> Kubernetes<'a> { Ok(available_validators >= desired_validators) } + + pub fn create_metrics_secret(&self) -> Result> { + let mut data = BTreeMap::new(); + if let Some(metrics) = &self.metrics { + data.insert( + "SOLANA_METRICS_CONFIG".to_string(), + SecretType::Value { v: metrics.to_env_string() }, + ); + } else { + return Err( + "Called create_metrics_secret() but metrics were not provided." + .to_string() + .into(), + ); + } + + k8s_helpers::create_secret("solana-metrics-secret".to_string(), data) + } + + pub fn get_metrics_env_var_secret(&self) -> EnvVar { + EnvVar { + name: "SOLANA_METRICS_CONFIG".to_string(), + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: Some("solana-metrics-secret".to_string()), + key: "SOLANA_METRICS_CONFIG".to_string(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } + } } diff --git a/src/lib.rs b/src/lib.rs index 101aa1f..ebe073a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,39 @@ pub enum ValidatorType { Client, } +#[derive(Clone, Debug, Default)] +pub struct Metrics { + pub host: String, + pub port: String, + pub database: String, + pub username: String, + password: String, +} + +impl Metrics { + pub fn new( + host: String, + port: String, + database: String, + username: String, + password: String, + ) -> Self { + Metrics { + host, + port, + database, + username, + password, + } + } + pub fn to_env_string(&self) -> String { + format!( + "host={}:{},db={},u={},p={}", + self.host, self.port, self.database, self.username, self.password + ) + } +} + pub mod cluster_images; pub mod docker; pub mod genesis; diff --git a/src/main.rs b/src/main.rs index 64d3d9b..762bfce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,8 @@ use { release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, validator_config::ValidatorConfig, - EnvironmentConfig, SolanaRoot, ValidatorType, + EnvironmentConfig, ValidatorType, + Metrics, SolanaRoot, }, }; @@ -221,6 +222,38 @@ fn parse_matches() -> clap::ArgMatches { Can specify unit here (B, Ki, Mi, Gi, Ti) for bytes, kilobytes, etc (2^N notation) e.g. 1Gi == 1024Mi == 1024Ki == 1,047,576B. [default: 70Gi]"), ) + //Metrics Config + .arg( + Arg::with_name("metrics_host") + .long("metrics-host") + .takes_value(true) + .requires_all(&["metrics_port", "metrics_db", "metrics_username", "metrics_password"]) + .help("Metrics Config. Optional: specify metrics host. e.g. https://internal-metrics.solana.com"), + ) + .arg( + Arg::with_name("metrics_port") + .long("metrics-port") + .takes_value(true) + .help("Metrics Config. Optional: specify metrics port. e.g. 8086"), + ) + .arg( + Arg::with_name("metrics_db") + .long("metrics-db") + .takes_value(true) + .help("Metrics Config. Optional: specify metrics database. e.g. k8s-cluster-"), + ) + .arg( + Arg::with_name("metrics_username") + .long("metrics-username") + .takes_value(true) + .help("Metrics Config. Optional: specify metrics username"), + ) + .arg( + Arg::with_name("metrics_password") + .long("metrics-password") + .takes_value(true) + .help("Metrics Config. Optional: Specify metrics password"), + ) .get_matches() } @@ -355,12 +388,24 @@ async fn main() { matches.value_of("memory_requests").unwrap().to_string(), ); + let metrics = matches.value_of("metrics_host").map(|host| { + Metrics::new( + host.to_string(), + matches.value_of("metrics_port").unwrap().to_string(), + matches.value_of("metrics_db").unwrap().to_string(), + matches.value_of("metrics_username").unwrap().to_string(), + matches.value_of("metrics_password").unwrap().to_string(), + ) + }); + let mut kub_controller = Kubernetes::new( environment_config.namespace, &mut validator_config, pod_requests, + metrics, ) .await; + match kub_controller.namespace_exists().await { Ok(true) => (), Ok(false) => { @@ -458,6 +503,24 @@ async fn main() { } } + // metrics secret create once and use by all pods + if kub_controller.metrics.is_some() { + let metrics_secret = match kub_controller.create_metrics_secret() { + Ok(secret) => secret, + Err(err) => { + error!("Failed to create metrics secret! {err}"); + return; + } + }; + match kub_controller.deploy_secret(&metrics_secret).await { + Ok(_) => (), + Err(err) => { + error!("{err}"); + return; + } + } + }; + let bootstrap_validator = cluster_images.bootstrap().expect("should be bootstrap"); match kub_controller.create_bootstrap_secret("bootstrap-accounts-secret", &config_directory) { Ok(secret) => bootstrap_validator.set_secret(secret), From f6c17c952634fedef1c266db605c8d79bc6a47eb Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 29 Apr 2024 18:52:20 +0000 Subject: [PATCH 6/6] clean up --- scripts/init-metrics.sh | 19 +++++++++++++++++-- src/kubernetes.rs | 25 ++++++++++++++++++------- src/main.rs | 37 +++++++++++++++++-------------------- src/startup_scripts.rs | 0 4 files changed, 52 insertions(+), 29 deletions(-) mode change 100755 => 100644 src/startup_scripts.rs diff --git a/scripts/init-metrics.sh b/scripts/init-metrics.sh index 89eae4f..ddd5aaa 100755 --- a/scripts/init-metrics.sh +++ b/scripts/init-metrics.sh @@ -2,8 +2,23 @@ set -e here=$(dirname "$0") -# shellcheck source=net/common.sh -source "$here"/common.sh + +# https://gist.github.com/cdown/1163649 +urlencode() { + declare s="$1" + declare l=$((${#s} - 1)) + for i in $(seq 0 $l); do + declare c="${s:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) + echo -n "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} usage() { exitcode=0 diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 51167e9..cc0f050 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -3,7 +3,7 @@ use { docker::DockerImage, k8s_helpers::{self, SecretType}, validator_config::ValidatorConfig, - ValidatorType, Metrics, + Metrics, ValidatorType, }, k8s_openapi::{ api::{ @@ -14,7 +14,6 @@ use { }, }, apimachinery::pkg::api::resource::Quantity, - ByteString, }, kube::{ api::{Api, ListParams, PostParams}, @@ -217,7 +216,7 @@ impl<'a> Kubernetes<'a> { pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } - + pub async fn deploy_replicas_set( &self, replica_set: &ReplicaSet, @@ -234,7 +233,12 @@ impl<'a> Kubernetes<'a> { service_name: &str, label_selector: &BTreeMap, ) -> Service { - k8s_helpers::create_service(service_name.to_string(), self.namespace.clone(), label_selector.clone(), false) + k8s_helpers::create_service( + service_name.to_string(), + self.namespace.clone(), + label_selector.clone(), + false, + ) } pub async fn deploy_service(&self, service: &Service) -> Result { @@ -252,9 +256,14 @@ impl<'a> Kubernetes<'a> { service_name: &str, label_selector: &BTreeMap, ) -> Service { - k8s_helpers::create_service(service_name.to_string(), self.namespace.clone(), label_selector.clone(), true) + k8s_helpers::create_service( + service_name.to_string(), + self.namespace.clone(), + label_selector.clone(), + true, + ) } - + pub async fn check_replica_set_ready( &self, replica_set_name: &str, @@ -279,7 +288,9 @@ impl<'a> Kubernetes<'a> { if let Some(metrics) = &self.metrics { data.insert( "SOLANA_METRICS_CONFIG".to_string(), - SecretType::Value { v: metrics.to_env_string() }, + SecretType::Value { + v: metrics.to_env_string(), + }, ); } else { return Err( diff --git a/src/main.rs b/src/main.rs index 762bfce..bccc954 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,8 +18,7 @@ use { release::{BuildConfig, BuildType, DeployMethod}, validator::{LabelType, Validator}, validator_config::ValidatorConfig, - EnvironmentConfig, ValidatorType, - Metrics, SolanaRoot, + EnvironmentConfig, Metrics, SolanaRoot, ValidatorType, }, }; @@ -579,20 +578,20 @@ async fn main() { // deploy bootstrap replica set match kub_controller - .deploy_replicas_set(bootstrap_validator.replica_set()) - .await - { - Ok(_) => { - info!( - "{} deployed successfully", - bootstrap_validator.replica_set_name() - ); - } - Err(err) => { - error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); - return; - } - }; + .deploy_replicas_set(bootstrap_validator.replica_set()) + .await + { + Ok(_) => { + info!( + "{} deployed successfully", + bootstrap_validator.replica_set_name() + ); + } + Err(err) => { + error!("Error! Failed to deploy bootstrap validator replicas_set. err: {err}"); + return; + } + }; // create and deploy bootstrap-service let bootstrap_service = kub_controller.create_bootstrap_service( @@ -609,10 +608,8 @@ async fn main() { let load_balancer_label = kub_controller.create_selector("load-balancer/name", "load-balancer-selector"); //create load balancer - let load_balancer = kub_controller.create_validator_load_balancer( - "bootstrap-and-rpc-node-lb-service", - &load_balancer_label, - ); + let load_balancer = kub_controller + .create_validator_load_balancer("bootstrap-and-rpc-node-lb-service", &load_balancer_label); //deploy load balancer match kub_controller.deploy_service(&load_balancer).await { diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs old mode 100755 new mode 100644