diff --git a/helper/Cargo.toml b/helper/Cargo.toml index 57c9d25..042ed92 100644 --- a/helper/Cargo.toml +++ b/helper/Cargo.toml @@ -29,6 +29,7 @@ similar = "2.4.0" csv = "1.3.0" octocrab = "0.34.0" openssl = { version = "0.10", features = ["vendored"] } +reqwest = { version = "0.11.11", features = ["json", "gzip", "native-tls", "socks"] } nvd-model = { path = "../nvd-model", features = ["db"] } [dev-dependencies] serde = { version = "1", features = ["derive"] } diff --git a/helper/src/import_cve.rs b/helper/src/import_cve.rs index 010c2cc..5cdb278 100644 --- a/helper/src/import_cve.rs +++ b/helper/src/import_cve.rs @@ -4,13 +4,13 @@ use diesel::mysql::MysqlConnection; use nvd_cves::v4::{CVEContainer, CVEItem}; use nvd_model::cve::{CreateCve, Cve}; use nvd_model::error::DBResult; +use nvd_model::types::AnyValue; use std::collections::HashSet; use std::fs::File; use std::io::BufReader; use std::ops::DerefMut; use std::path::PathBuf; use std::str::FromStr; -use nvd_model::types::AnyValue; pub fn import_from_archive( connection: &mut MysqlConnection, diff --git a/helper/src/import_exploit.rs b/helper/src/import_exploit.rs index 08336f4..f7b85b7 100644 --- a/helper/src/import_exploit.rs +++ b/helper/src/import_exploit.rs @@ -1,5 +1,5 @@ use crate::{init_db_pool, Connection}; -use chrono::{Duration, NaiveDateTime, Utc}; +use chrono::{DateTime, Duration, NaiveDateTime, Utc}; use diesel::MysqlConnection; use std::collections::HashSet; @@ -11,7 +11,8 @@ use nvd_model::exploit::Exploit; use nvd_model::types::{AnyValue, MetaData}; use octocrab::models::repos::{DiffEntryStatus, RepoCommit}; use octocrab::{Octocrab, Page}; -use serde::{Deserialize, Serialize}; +use reqwest::header; +use serde::{Deserialize, Deserializer, Serialize}; use std::ffi::OsStr; use std::fs::File; use std::ops::DerefMut; @@ -75,6 +76,68 @@ struct ExploitDB { source_url: Option, } +impl ExploitDB { + fn from_html(html: &str, item: &Item) -> Self { + let id = item.link.rsplit_once('/').unwrap_or_default().1; + let v = html.contains("") + .strip_prefix("") + .strip_prefix(" Err(err), @@ -349,3 +415,89 @@ impl GitHubCommit { } } } + +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +struct Rss { + channel: Channel, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +struct Channel { + item: Vec, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +struct Item { + title: String, + link: String, + description: String, + #[serde(alias = "pubDate", deserialize_with = "rfc3339_deserialize")] + published: NaiveDateTime, +} + +pub fn rfc3339_deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + if s.is_empty() { + return Ok(Utc::now().naive_local()); + } + match DateTime::parse_from_rfc2822(&s) { + Ok(naive_datetime) => Ok(naive_datetime.naive_local()), + Err(err) => Err(serde::de::Error::custom(err)), + } +} + +async fn get_info_from_exploit_url(conn: &mut Connection, item: &Item) { + let mut headers = header::HeaderMap::new(); + let ua = "Mozilla/5.0 (X11; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0"; + headers.insert(header::USER_AGENT, header::HeaderValue::from_static(ua)); + if let Ok(resp) = reqwest::ClientBuilder::new() + .default_headers(headers) + .build() + .unwrap_or_default() + .get(&item.link) + .send() + .await + { + let html = resp.text().await.unwrap_or_default(); + let exploit_item = ExploitDB::from_html(&html, item); + let meta = MetaData::default(); + let new_exp = CreateExploit { + id: uuid::Uuid::new_v4().as_bytes().to_vec(), + name: exploit_item.id.to_string(), + description: Some(exploit_item.description), + source: ExploitSource::ExploitDb.to_string(), + path: exploit_item.file, + meta: AnyValue::new(meta.clone()), + verified: exploit_item.verified, + created_at: exploit_item.date_published, + updated_at: exploit_item.date_updated, + }; + if !new_exp.path.is_empty() + && exploit_item.id != 0 + && !exploit_item.r#type.is_empty() + && !exploit_item.platform.is_empty() + { + if let Err(err) = create_or_update_exploit(conn, new_exp, exploit_item.codes) { + println!("import nuclei exploit err: {:?}", err); + } + } + } +} + +pub async fn update_from_rss() { + let connection_pool = init_db_pool(); + if let Ok(resp) = reqwest::get("https://www.exploit-db.com/rss.xml").await { + let b = resp.bytes().await.unwrap_or_default(); + let s = String::from_utf8_lossy(&b); + let rss: Rss = quick_xml::de::from_str(&s).unwrap(); + for item in rss.channel.item { + get_info_from_exploit_url(connection_pool.get().unwrap().deref_mut(), &item).await; + } + } +} diff --git a/helper/src/lib.rs b/helper/src/lib.rs index c39c59a..c2a963f 100644 --- a/helper/src/lib.rs +++ b/helper/src/lib.rs @@ -12,7 +12,7 @@ use crate::cli::{EXPCommand, SyncCommand}; use crate::import_cpe::with_archive_cpe; use crate::import_cve::with_archive_cve; use crate::import_exploit::{ - import_from_nuclei_templates_path, update_from_github, with_archive_exploit, + import_from_nuclei_templates_path, update_from_github, update_from_rss, with_archive_exploit, }; pub use cli::{CPECommand, CVECommand, NVDHelper, TopLevel}; pub use import_cpe::{create_cve_product, create_product, create_vendor}; @@ -111,6 +111,7 @@ pub async fn sync_mode(config: SyncCommand) { async_cve(param).await } if config.exp { + update_from_rss().await; update_from_github().await; } }