From c211e518cc2aca289537742e97539b8fb969eca1 Mon Sep 17 00:00:00 2001 From: cn-kali-team Date: Wed, 21 Feb 2024 13:20:00 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=86exp=E5=90=88=E5=B9=B6=E5=88=B0kb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helper/src/cli.rs | 12 +- helper/src/cve/mod.rs | 6 +- helper/src/exp/mod.rs | 576 ----------------- helper/src/kb/mod.rs | 610 +++++++++++++++++- helper/src/lib.rs | 11 +- nvd-cvss/src/v2/access_complexity.rs | 12 +- nvd-cvss/src/v2/authentication.rs | 16 +- nvd-cvss/src/v3/attack_complexity.rs | 16 +- nvd-cvss/src/v3/attack_vector.rs | 16 +- nvd-cvss/src/v3/impact_metrics.rs | 15 +- nvd-cvss/src/v3/mod.rs | 10 +- nvd-cvss/src/v3/user_interaction.rs | 12 +- nvd-cvss/src/v4/attack_complexity.rs | 20 +- nvd-cvss/src/v4/attack_requirements.rs | 18 +- nvd-cvss/src/v4/attack_vector.rs | 16 +- nvd-cvss/src/v4/mod.rs | 14 +- nvd-cvss/src/v4/subsequent_impact_metrics.rs | 17 +- nvd-cvss/src/v4/user_interaction.rs | 10 +- nvd-cvss/src/v4/vulnerable_impact_metrics.rs | 15 +- .../{cve_exploit => cve_knowledge_base}/db.rs | 86 +-- .../mod.rs | 17 +- nvd-model/src/exploit/db.rs | 195 ------ nvd-model/src/exploit/mod.rs | 43 -- nvd-model/src/knowledge_base/db.rs | 72 ++- nvd-model/src/knowledge_base/mod.rs | 12 +- nvd-model/src/lib.rs | 5 +- nvd-model/src/schema.rs | 35 +- nvd-server/src/api/exploit_api.rs | 19 - nvd-server/src/api/mod.rs | 7 +- nvd-yew/src/component/exp_query.rs | 126 ---- nvd-yew/src/component/exp_row.rs | 100 --- nvd-yew/src/component/exploit.rs | 106 --- nvd-yew/src/component/kb_query.rs | 9 +- nvd-yew/src/component/kb_row.rs | 39 +- nvd-yew/src/component/knowledge_base.rs | 6 +- nvd-yew/src/component/mod.rs | 12 +- nvd-yew/src/layout/nav.rs | 7 - nvd-yew/src/routes/cve.rs | 14 +- nvd-yew/src/routes/exp.rs | 188 ------ nvd-yew/src/routes/kb.rs | 9 +- nvd-yew/src/routes/mod.rs | 11 +- nvd-yew/src/services/exp.rs | 10 - nvd-yew/src/services/mod.rs | 1 - 43 files changed, 897 insertions(+), 1654 deletions(-) delete mode 100644 helper/src/exp/mod.rs rename nvd-model/src/{cve_exploit => cve_knowledge_base}/db.rs (58%) rename nvd-model/src/{cve_exploit => cve_knowledge_base}/mod.rs (52%) delete mode 100644 nvd-model/src/exploit/db.rs delete mode 100644 nvd-model/src/exploit/mod.rs delete mode 100644 nvd-server/src/api/exploit_api.rs delete mode 100644 nvd-yew/src/component/exp_query.rs delete mode 100644 nvd-yew/src/component/exp_row.rs delete mode 100644 nvd-yew/src/component/exploit.rs delete mode 100644 nvd-yew/src/routes/exp.rs delete mode 100644 nvd-yew/src/services/exp.rs diff --git a/helper/src/cli.rs b/helper/src/cli.rs index d61c690..14e6101 100644 --- a/helper/src/cli.rs +++ b/helper/src/cli.rs @@ -48,14 +48,14 @@ pub struct CPECommand { } #[derive(FromArgs, PartialEq, Debug)] -#[argh(description = "exp helper")] -#[argh(subcommand, name = "exp")] +#[argh(description = "kb helper")] +#[argh(subcommand, name = "kb")] pub struct EXPCommand { - #[argh(option, description = "import exp from files_exploits.csv")] + #[argh(option, description = "import kb from files_exploits.csv")] pub path: Option, - #[argh(option, description = "import exp from nuclei-templates path")] + #[argh(option, description = "import kb from nuclei-templates path")] pub template: Option, - #[argh(switch, description = "update exp from nuclei-templates")] + #[argh(switch, description = "update kb from nuclei-templates")] pub api: bool, } @@ -71,7 +71,7 @@ pub struct KBCommand { #[argh(description = "sync helper")] #[argh(subcommand, name = "sync")] pub struct SyncCommand { - #[argh(switch, description = "sync exp")] + #[argh(switch, description = "sync kb")] pub exp: bool, #[argh(switch, description = "sync cve")] pub cve: bool, diff --git a/helper/src/cve/mod.rs b/helper/src/cve/mod.rs index 50ee3dd..a2c06ec 100644 --- a/helper/src/cve/mod.rs +++ b/helper/src/cve/mod.rs @@ -6,18 +6,18 @@ use std::path::PathBuf; use std::str::FromStr; use diesel::mysql::MysqlConnection; +use nvd_api::ApiVersion; use nvd_api::pagination::Object; use nvd_api::v2::vulnerabilities::CveParameters; -use nvd_api::ApiVersion; use nvd_cves::v4::{CVEContainer, CVEItem}; use nvd_model::cve::{CreateCve, Cve}; use nvd_model::error::DBResult; use nvd_model::types::AnyValue; -use crate::cpe::{del_expire_product, import_vendor_product_to_db}; -use crate::exp::associate_cve_and_exploit; use crate::{create_cve_product, init_db_pool}; +use crate::cpe::{del_expire_product, import_vendor_product_to_db}; +use crate::kb::associate_cve_and_exploit; pub(crate) async fn async_cve(param: CveParameters) { let connection_pool = init_db_pool(); diff --git a/helper/src/exp/mod.rs b/helper/src/exp/mod.rs deleted file mode 100644 index 7fcbb3f..0000000 --- a/helper/src/exp/mod.rs +++ /dev/null @@ -1,576 +0,0 @@ -use std::collections::HashSet; -use std::ffi::OsStr; -use std::fmt; -use std::fs::File; -use std::marker::PhantomData; -use std::ops::DerefMut; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use chrono::{DateTime, Duration, NaiveDateTime, Utc}; -use diesel::MysqlConnection; -use octocrab::models::repos::{DiffEntryStatus, RepoCommit}; -use octocrab::{Octocrab, Page}; -use reqwest::header; -use serde::{de, Deserialize, Deserializer, Serialize}; - -use nvd_model::cve_exploit::db::CreateCveExploit; -use nvd_model::cve_exploit::CveExploit; -use nvd_model::error::DBResult; -use nvd_model::exploit::db::{CreateExploit, ExploitSource}; -use nvd_model::exploit::Exploit; -use nvd_model::types::{AnyValue, MetaData}; - -use crate::{init_db_pool, Connection}; - -mod date_format { - use chrono::{NaiveDate, NaiveDateTime, Utc}; - use serde::{self, Deserialize, Deserializer, Serializer}; - - pub(crate) const FORMAT: &str = "%Y-%m-%d"; - - pub fn serialize(date: &NaiveDateTime, serializer: S) -> Result - where - S: Serializer, - { - let s = date.to_string(); - serializer.serialize_str(&s) - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - if s.is_empty() { - return Ok(Utc::now().naive_local()); - } - match NaiveDate::parse_from_str(&s, FORMAT) { - Ok(naive_datetime) => Ok( - naive_datetime - .and_hms_opt(0, 0, 0) - .unwrap_or(Utc::now().naive_local()), - ), - Err(err) => Err(serde::de::Error::custom(err)), - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone)] -struct ExploitDB { - id: u32, - file: String, - description: String, - #[serde(with = "date_format")] - date_published: NaiveDateTime, - author: String, - r#type: String, - platform: String, - port: Option, - #[serde(with = "date_format")] - date_added: NaiveDateTime, - #[serde(with = "date_format")] - date_updated: NaiveDateTime, - verified: u8, - codes: Option, - tags: Option, - aliases: Option, - screenshot_url: Option, - application_url: Option, - source_url: Option, -} - -impl ExploitDB { - fn from_html(html: &str, item: &Item) -> Self { - let id = item.link.rsplit_once('/').unwrap_or_default().1; - let v = html.contains("") - .strip_prefix("") - .strip_prefix(", -) -> DBResult { - match Exploit::create_or_update(connection, &exploit_item) { - Ok(exp) => { - if let Some(cve_id) = cve_id { - let new_cve_exp = CreateCveExploit { - cve_id, - exploit_id: exp.id.clone(), - }; - if let Err(err) = CveExploit::create(connection, &new_cve_exp) { - println!("漏洞利用关联CVE失败：{:?}", err); - } - } - println!( - "从{}同步exploit: {}", - exploit_item.source, exploit_item.path - ); - Ok(exp) - } - Err(err) => Err(err), - } -} - -pub fn with_archive_exploit(path: PathBuf) { - let file = File::open(path).unwrap(); - let mut rdr = csv::Reader::from_reader(file); - let connection_pool = init_db_pool(); - for result in rdr.deserialize() { - // Notice that we need to provide a type hint for automatic - // deserialization. - let exploit_item: ExploitDB = result.unwrap(); - let meta = if let Some(code) = exploit_item.codes { - MetaData::from_hashset( - "tags", - code - .split(';') - .map(|s| s.to_string()) - .collect::>(), - ) - } else { - MetaData::default() - }; - let new_exp = CreateExploit { - id: uuid::Uuid::new_v4().as_bytes().to_vec(), - name: exploit_item.id.to_string(), - description: exploit_item.description, - source: ExploitSource::ExploitDb.to_string(), - path: exploit_item.file, - meta: AnyValue::new(meta.clone()), - verified: exploit_item.verified, - created_at: exploit_item.date_published, - updated_at: exploit_item.date_updated, - }; - if let Some(code_list) = meta.get_hashset("tags") { - for c in code_list { - if c.starts_with("CVE-") { - // 是CVE - if let Err(err) = create_or_update_exploit( - connection_pool.get().unwrap().deref_mut(), - new_exp.clone(), - Some(c.clone()), - ) { - println!("是CVE： import exploit err: {:?}", err); - } - } else { - // code不为空，但是不是cve - if let Err(err) = create_or_update_exploit( - connection_pool.get().unwrap().deref_mut(), - new_exp.clone(), - None, - ) { - println!("code不为空，但是不是cve： import exploit err: {:?}", err); - } - } - } - } else { - // code为空 - if let Err(err) = create_or_update_exploit( - connection_pool.get().unwrap().deref_mut(), - new_exp.clone(), - None, - ) { - println!("code为空：import exploit err: {:?}", err); - } - } - // break; - } -} - -pub async fn update_from_github() { - let connection_pool = init_db_pool(); - let commit_api = GitHubCommit::new("projectdiscovery", "nuclei-templates"); - commit_api - .update(connection_pool.get().unwrap().deref_mut()) - .await; -} - -/// 字符串转set -fn string_to_hashset<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct StringToHashSet(PhantomData>); - impl<'de> de::Visitor<'de> for StringToHashSet { - type Value = HashSet; - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("string or list of strings") - } - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - let name: Vec = value - .split(',') - .filter(|s| !s.starts_with("cve")) - .map(String::from) - .collect(); - Ok(HashSet::from_iter(name)) - } - fn visit_seq(self, visitor: S) -> Result - where - S: de::SeqAccess<'de>, - { - Deserialize::deserialize(de::value::SeqAccessDeserializer::new(visitor)) - } - } - deserializer.deserialize_any(StringToHashSet(PhantomData)) -} - -#[derive(Debug, Serialize, Deserialize, Clone, Default)] -struct Template { - id: String, - info: Info, -} - -#[derive(Debug, Serialize, Deserialize, Clone, Default)] -struct Info { - name: String, - description: String, - classification: Option, - #[serde(deserialize_with = "string_to_hashset")] - tags: HashSet, -} - -#[derive(Debug, Serialize, Deserialize, Clone, Default)] -struct Classification { - #[serde(rename = "cve-id")] - cve_id: Option, - cpe: Option, -} - -pub struct GitHubCommit { - owner: String, - repo: String, - api: Arc, -} - -fn get_yaml_file(path: PathBuf) -> Vec { - let mut yaml_file_list: Vec = Vec::new(); - if let Ok(read_dir) = Path::new(&path).read_dir() { - for element in read_dir.filter_map(|res| res.ok()) { - if element.path().is_dir() { - yaml_file_list.extend(get_yaml_file(element.path())); - } - if element.path().is_file() { - yaml_file_list.push(element.path()); - } - } - } - yaml_file_list -} - -pub fn import_from_nuclei_templates_path(path: PathBuf) { - let mut connection_pool = init_db_pool().get().unwrap(); - let conn = connection_pool.deref_mut(); - let cve_path = path.join("http").join("cves"); - let yaml_paths = get_yaml_file(cve_path); - for yaml_path in yaml_paths { - if let Ok(f) = File::open(&yaml_path) { - let template: Template = serde_yaml::from_reader(f).unwrap(); - let meta = MetaData::from_hashset("tags", template.info.tags); - let new_exp = CreateExploit { - id: uuid::Uuid::new_v4().as_bytes().to_vec(), - name: template.id.clone(), - description: template.info.description, - source: ExploitSource::NucleiTemplates.to_string(), - path: yaml_path - .strip_prefix(&path) - .unwrap() - .to_string_lossy() - .to_string(), - meta: AnyValue::new(meta), - verified: 1, - created_at: Utc::now().naive_utc(), - updated_at: Utc::now().naive_utc(), - }; - if let Err(err) = create_or_update_exploit(conn, new_exp, Some(template.id)) { - println!("import nuclei exploit err: {:?}", err); - } - } - } -} - -impl GitHubCommit { - pub fn new(owner: impl Into, repo: impl Into) -> Self { - Self { - owner: owner.into(), - repo: repo.into(), - api: octocrab::instance(), - } - } - async fn get_commit_list(&self) -> octocrab::Result> { - let now = Utc::now(); - let three_hours = now - Duration::hours(3); - println!( - "开始更新从{}到{}exp", - three_hours.to_rfc3339(), - now.to_rfc3339() - ); - self - .api - .repos(&self.owner, &self.repo) - .list_commits() - .path("http/cves") - .since(three_hours) - .send() - .await - } - async fn get_commit(&self, sha: String) -> octocrab::Result { - self.api.commits(&self.owner, &self.repo).get(sha).await - } - async fn get_template(&self, path: String) -> octocrab::Result