[tools] config tool fullnode-init subcommand (0LNetworkCommunity#79)

Co-authored-by: 0o-de-lally <[email protected]>

tools/cli/src/node_cli.rs

@@ -1,4 +1,4 @@
-use anyhow::anyhow;
+use anyhow::{anyhow, bail};
 use clap::Parser;
 use diem_config::config::NodeConfig;
 use libra_types::global_config_dir;
@@ -14,10 +14,16 @@ pub struct NodeCli {
 
 impl NodeCli {
     pub async fn run(&self) -> anyhow::Result<()> {
+        // validators typically aren't looking for verbose logs.
+        // but they can set it if they wish with RUST_LOG=info
+        if std::env::var("RUST_LOG").is_err() {
+            std::env::set_var("RUST_LOG", "warn")
+        }
+
         let path = self
             .config_path
             .clone()
-            .unwrap_or_else(|| global_config_dir().join("validator.yaml"));
+            .unwrap_or_else(|| find_a_config().expect("no config"));
         // A config file exists, attempt to parse the config
         let config = NodeConfig::load_from_path(path.clone()).map_err(|error| {
             anyhow!(
@@ -33,3 +39,33 @@ impl NodeCli {
         Ok(())
     }
 }
+
+/// helper to find a default config
+fn find_a_config() -> anyhow::Result<PathBuf> {
+    let d = global_config_dir();
+    let val_file = d.join("validator.yaml");
+
+    let help = "If this is not what you expected explicitly set it with --config-file <path>";
+
+    // we assume if this is set up as a validator that's the preferred profile
+    if val_file.exists() {
+        println!(
+            "\nUsing validator profile at {}.\n{}",
+            val_file.display(),
+            help
+        );
+        return Ok(val_file);
+    }
+
+    let fn_file = d.join("fullnode.yaml");
+    if fn_file.exists() {
+        println!(
+            "\nUsing fullnode profile at {}.\n{}",
+            fn_file.display(),
+            help
+        );
+        return Ok(fn_file);
+    }
+
+    bail!("ERROR: you have no node *.yaml configured in the default directory $HOME/.libra/");
+}

tools/config/Cargo.toml

@@ -23,7 +23,9 @@ libra-types = { workspace = true }
 libra-wallet = { workspace = true }
 reqwest = { workspace = true }
 serde_json = { workspace = true }
+serde_yaml = { workspace = true }
 tokio = { workspace = true }
 url = { workspace = true }
 
-
+[dev-dependencies]
+diem-temppath = { workspace = true }

tools/config/src/config_cli.rs

@@ -1,3 +1,4 @@
+use crate::fullnode_config::{download_genesis, init_fullnode_yaml};
 use crate::host::initialize_validator_configs;
 use crate::{legacy_config, make_profile};
 use anyhow::{Context, Result};
@@ -6,9 +7,9 @@ use libra_types::exports::AccountAddress;
 use libra_types::exports::AuthenticationKey;
 use libra_types::exports::Client;
 use libra_types::exports::NamedChain;
-use libra_types::global_config_dir;
 use libra_types::legacy_types::app_cfg::{self, AppCfg};
 use libra_types::type_extensions::client_ext::ClientExt;
+use libra_types::{global_config_dir, ol_progress};
 use libra_wallet::utils::read_operator_file;
 use libra_wallet::validator_files::OPERATOR_FILE;
 use std::path::PathBuf;
@@ -90,6 +91,13 @@ enum ConfigSub {
         #[clap(short, long, default_value = "false")]
         check: bool,
     },
+
+    /// Generate a fullnode dir and add fullnode.yaml from template
+    FullnodeInit {
+        /// path to libra config and data files defaults to $HOME/.libra
+        #[clap(long)]
+        home_path: Option<PathBuf>,
+    },
 }
 
 impl ConfigCli {
@@ -222,6 +230,17 @@ impl ConfigCli {
                 println!("Validators' config initialized.");
                 Ok(())
             }
+            Some(ConfigSub::FullnodeInit { home_path }) => {
+                download_genesis(home_path.to_owned()).await?;
+                println!("downloaded genesis block");
+
+                let p = init_fullnode_yaml(home_path.to_owned()).await?;
+                println!("config created at {}", p.display());
+
+                ol_progress::OLProgress::complete("fullnode configs initialized");
+
+                Ok(())
+            }
             _ => {
                 println!("Sometimes I'm right and I can be wrong. My own beliefs are in my song. The butcher, the banker, the drummer and then. Makes no difference what group I'm in.");
 

tools/config/src/fullnode_config.rs

@@ -0,0 +1,175 @@
+use diem_config::config::NodeConfig;
+use diem_types::{network_address::NetworkAddress, waypoint::Waypoint, PeerId};
+use libra_types::legacy_types::app_cfg::default_file_path;
+use std::{
+    collections::HashMap,
+    path::{Path, PathBuf},
+};
+
+/// fetch seed peers and make a yaml file from template
+pub async fn init_fullnode_yaml(home_dir: Option<PathBuf>) -> anyhow::Result<PathBuf> {
+    let waypoint = get_genesis_waypoint(home_dir.clone()).await?;
+
+    let yaml = make_fullnode_yaml(home_dir.clone(), waypoint)?;
+
+    let home = home_dir.unwrap_or_else(default_file_path);
+    let p = home.join("fullnode.yaml");
+
+    std::fs::write(&p, yaml)?;
+    let peers = fetch_seed_addresses(None).await?;
+
+    add_peers_to_yaml(&p, peers)?;
+
+    Ok(p)
+}
+
+pub fn add_peers_to_yaml(
+    path: &Path,
+    peers: HashMap<PeerId, Vec<NetworkAddress>>,
+) -> anyhow::Result<()> {
+    let string = std::fs::read_to_string(path)?;
+    let mut parsed: NodeConfig = serde_yaml::from_str(&string)?;
+
+    parsed.full_node_networks.iter_mut().for_each(move |e| {
+        if e.network_id.is_public_network() {
+            e.seed_addrs = peers.clone();
+        }
+    });
+
+    let ser = serde_yaml::to_string(&parsed)?;
+    std::fs::write(path, ser)?;
+
+    Ok(())
+}
+
+/// get seed peers from an upstream url
+pub async fn fetch_seed_addresses(
+    url: Option<&str>,
+) -> anyhow::Result<HashMap<PeerId, Vec<NetworkAddress>>> {
+    let u = url.unwrap_or(
+        "https://raw.githubusercontent.com/0LNetworkCommunity/seed-peers/main/seed_peers.yaml",
+    );
+    let res = reqwest::get(u).await?;
+    let out = res.text().await?;
+    let seeds: HashMap<PeerId, Vec<NetworkAddress>> = serde_yaml::from_str(&out)?;
+
+    Ok(seeds)
+}
+
+/// Create a fullnode yaml to bootstrap node
+pub fn make_fullnode_yaml(home_dir: Option<PathBuf>, waypoint: Waypoint) -> anyhow::Result<String> {
+    let home_dir = home_dir.unwrap_or_else(libra_types::global_config_dir);
+    let path = home_dir.display().to_string();
+
+    let template = format!(
+        "
+base:
+  role: 'full_node'
+  data_dir: '{path}/data'
+  waypoint:
+    from_config: '{waypoint}'
+
+execution:
+  genesis_file_location: '{path}/genesis.blob'
+
+state_sync:
+     state_sync_driver:
+        bootstrapping_mode: DownloadLatestStates
+        continuous_syncing_mode: ExecuteTransactionsOrApplyOutputs
+
+full_node_networks:
+- network_id: 'public'
+  listen_address: '/ip4/0.0.0.0/tcp/6182'
+
+api:
+  enabled: true
+  address: '0.0.0.0:8080'
+"
+    );
+    Ok(template)
+}
+
+/// download genesis blob
+pub async fn download_genesis(home_dir: Option<PathBuf>) -> anyhow::Result<()> {
+    let bytes = reqwest::get("https://github.com/0xzoz/blob/raw/main/genesis.blob")
+        .await?
+        .bytes()
+        .await?;
+
+    let home = home_dir.unwrap_or_else(default_file_path);
+    std::fs::create_dir_all(&home)?;
+    let p = home.join("genesis.blob");
+
+    std::fs::write(p, bytes)?;
+
+    Ok(())
+}
+
+pub async fn get_genesis_waypoint(home_dir: Option<PathBuf>) -> anyhow::Result<Waypoint> {
+    let wp_string = reqwest::get("https://raw.githubusercontent.com/0xzoz/blob/main/waypoint.txt")
+        .await?
+        .text()
+        .await?;
+
+    let home = home_dir.unwrap_or_else(default_file_path);
+    let p = home.join("waypoint.txt");
+
+    std::fs::write(p, &wp_string)?;
+
+    wp_string.parse()
+}
+
+#[tokio::test]
+async fn get_peers() {
+    let seed = fetch_seed_addresses(None).await.unwrap();
+
+    dbg!(&seed);
+}
+
+#[tokio::test]
+async fn get_yaml() {
+    use std::str::FromStr;
+    let p = diem_temppath::TempPath::new().path().to_owned();
+
+    let seeds = fetch_seed_addresses(None).await.unwrap();
+
+    let y = make_fullnode_yaml(
+        Some(p.clone()),
+        Waypoint::from_str("0:95023f4d6a7e24cac3e52cad29697184db260214210b57aef3f1031ad4d8c02c")
+            .unwrap(),
+    )
+    .unwrap();
+
+    std::fs::write(&p, y).unwrap();
+
+    // let mut parsed: NodeConfig = serde_yaml::from_str(&y).unwrap();
+    // parsed.full_node_networks.iter_mut().for_each(move |e| {
+    //   if e.network_id.is_public_network() {
+    //     e.seed_addrs = seeds.clone();
+    //   }
+    // });
+
+    add_peers_to_yaml(&p, seeds).unwrap();
+
+    let text = std::fs::read_to_string(&p).unwrap();
+
+    dbg!(&text);
+}
+
+#[tokio::test]
+async fn persist_genesis() {
+    let mut p = diem_temppath::TempPath::new();
+    p.create_as_dir().unwrap();
+    p.persist();
+
+    let path = p.path().to_owned();
+
+    download_genesis(Some(path)).await.unwrap();
+    let l = std::fs::read_dir(p.path())
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap();
+    assert!(l.file_name().to_str().unwrap().contains("genesis.blob"));
+    // dbg!(&l);
+}

tools/config/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod config_cli;
+pub mod fullnode_config;
 pub mod host;
 pub mod legacy_config;
 pub mod make_profile;