添加从exploit的rss订阅更新exp

helper/Cargo.toml

@@ -29,6 +29,7 @@ similar = "2.4.0"
 csv = "1.3.0"
 octocrab = "0.34.0"
 openssl = { version = "0.10", features = ["vendored"] }
+reqwest = { version = "0.11.11", features = ["json", "gzip", "native-tls", "socks"] }
 nvd-model = { path = "../nvd-model", features = ["db"] }
 [dev-dependencies]
 serde = { version = "1", features = ["derive"] }

helper/src/import_cve.rs

@@ -4,13 +4,13 @@ use diesel::mysql::MysqlConnection;
 use nvd_cves::v4::{CVEContainer, CVEItem};
 use nvd_model::cve::{CreateCve, Cve};
 use nvd_model::error::DBResult;
+use nvd_model::types::AnyValue;
 use std::collections::HashSet;
 use std::fs::File;
 use std::io::BufReader;
 use std::ops::DerefMut;
 use std::path::PathBuf;
 use std::str::FromStr;
-use nvd_model::types::AnyValue;
 
 pub fn import_from_archive(
   connection: &mut MysqlConnection,

helper/src/import_exploit.rs

@@ -1,5 +1,5 @@
 use crate::{init_db_pool, Connection};
-use chrono::{Duration, NaiveDateTime, Utc};
+use chrono::{DateTime, Duration, NaiveDateTime, Utc};
 use diesel::MysqlConnection;
 use std::collections::HashSet;
 
@@ -11,7 +11,8 @@ use nvd_model::exploit::Exploit;
 use nvd_model::types::{AnyValue, MetaData};
 use octocrab::models::repos::{DiffEntryStatus, RepoCommit};
 use octocrab::{Octocrab, Page};
-use serde::{Deserialize, Serialize};
+use reqwest::header;
+use serde::{Deserialize, Deserializer, Serialize};
 use std::ffi::OsStr;
 use std::fs::File;
 use std::ops::DerefMut;
@@ -75,6 +76,68 @@ struct ExploitDB {
   source_url: Option<String>,
 }
 
+impl ExploitDB {
+  fn from_html(html: &str, item: &Item) -> Self {
+    let id = item.link.rsplit_once('/').unwrap_or_default().1;
+    let v = html.contains("<i class=\"mdi mdi-24px mdi-check\"");
+    let mut exp = ExploitDB {
+      id: id.parse().unwrap_or_default(),
+      file: "".to_string(),
+      description: item.description.clone(),
+      date_published: item.published,
+      author: "".to_string(),
+      r#type: "".to_string(),
+      platform: "".to_string(),
+      port: None,
+      date_added: Default::default(),
+      date_updated: item.published,
+      verified: v as u8,
+      codes: None,
+      tags: None,
+      aliases: None,
+      screenshot_url: None,
+      application_url: None,
+      source_url: None,
+    };
+    for line in html.lines() {
+      if line.contains("<a href=\"https://nvd.nist.gov/vuln/detail/CVE-") {
+        if let Some(u) = line.trim().trim_end_matches('"').strip_prefix("<a href=\"") {
+          if let Some((_, id)) = u.rsplit_once('/') {
+            exp.codes = Some(id.to_string());
+          };
+        };
+      } else if line.contains("<a href=\"/?platform=") {
+        if let Some(platform) = line
+          .trim()
+          .trim_end_matches("\">")
+          .strip_prefix("<a href=\"/?platform=")
+        {
+          exp.platform = platform.to_string();
+        };
+      } else if line.contains("<a href=\"/?type=") {
+        if let Some(t) = line
+          .trim()
+          .trim_end_matches("\">")
+          .strip_prefix("<a href=\"/?type=")
+        {
+          exp.r#type = t.to_string();
+        };
+      } else if line.contains("<pre><code class=\"language-") {
+        if let Some(u) = line
+          .trim()
+          .trim_end_matches('"')
+          .strip_prefix("<pre><code class=\"language-")
+        {
+          if let Some((l, _)) = u.split_once('"') {
+            exp.file = format!("exploits/{}/{}/{}.{}", exp.platform, exp.r#type, id, l);
+          };
+        };
+      }
+    }
+    exp
+  }
+}
+
 fn create_or_update_exploit(
   connection: &mut MysqlConnection,
   exploit_item: CreateExploit,
@@ -91,7 +154,10 @@ fn create_or_update_exploit(
           println!("漏洞利用关联CVE失败：{:?}", err);
         }
       }
-      println!("从{}同步exploit{}", exploit_item.source, exploit_item.path);
+      println!(
+        "从{}同步exploit: {}",
+        exploit_item.source, exploit_item.path
+      );
       Ok(exp)
     }
     Err(err) => Err(err),
@@ -349,3 +415,89 @@ impl GitHubCommit {
     }
   }
 }
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+struct Rss {
+  channel: Channel,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+struct Channel {
+  item: Vec<Item>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+struct Item {
+  title: String,
+  link: String,
+  description: String,
+  #[serde(alias = "pubDate", deserialize_with = "rfc3339_deserialize")]
+  published: NaiveDateTime,
+}
+
+pub fn rfc3339_deserialize<'de, D>(deserializer: D) -> Result<NaiveDateTime, D::Error>
+where
+  D: Deserializer<'de>,
+{
+  let s = String::deserialize(deserializer)?;
+  if s.is_empty() {
+    return Ok(Utc::now().naive_local());
+  }
+  match DateTime::parse_from_rfc2822(&s) {
+    Ok(naive_datetime) => Ok(naive_datetime.naive_local()),
+    Err(err) => Err(serde::de::Error::custom(err)),
+  }
+}
+
+async fn get_info_from_exploit_url(conn: &mut Connection, item: &Item) {
+  let mut headers = header::HeaderMap::new();
+  let ua = "Mozilla/5.0 (X11; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0";
+  headers.insert(header::USER_AGENT, header::HeaderValue::from_static(ua));
+  if let Ok(resp) = reqwest::ClientBuilder::new()
+    .default_headers(headers)
+    .build()
+    .unwrap_or_default()
+    .get(&item.link)
+    .send()
+    .await
+  {
+    let html = resp.text().await.unwrap_or_default();
+    let exploit_item = ExploitDB::from_html(&html, item);
+    let meta = MetaData::default();
+    let new_exp = CreateExploit {
+      id: uuid::Uuid::new_v4().as_bytes().to_vec(),
+      name: exploit_item.id.to_string(),
+      description: Some(exploit_item.description),
+      source: ExploitSource::ExploitDb.to_string(),
+      path: exploit_item.file,
+      meta: AnyValue::new(meta.clone()),
+      verified: exploit_item.verified,
+      created_at: exploit_item.date_published,
+      updated_at: exploit_item.date_updated,
+    };
+    if !new_exp.path.is_empty()
+      && exploit_item.id != 0
+      && !exploit_item.r#type.is_empty()
+      && !exploit_item.platform.is_empty()
+    {
+      if let Err(err) = create_or_update_exploit(conn, new_exp, exploit_item.codes) {
+        println!("import nuclei exploit err: {:?}", err);
+      }
+    }
+  }
+}
+
+pub async fn update_from_rss() {
+  let connection_pool = init_db_pool();
+  if let Ok(resp) = reqwest::get("https://www.exploit-db.com/rss.xml").await {
+    let b = resp.bytes().await.unwrap_or_default();
+    let s = String::from_utf8_lossy(&b);
+    let rss: Rss = quick_xml::de::from_str(&s).unwrap();
+    for item in rss.channel.item {
+      get_info_from_exploit_url(connection_pool.get().unwrap().deref_mut(), &item).await;
+    }
+  }
+}

helper/src/lib.rs

@@ -12,7 +12,7 @@ use crate::cli::{EXPCommand, SyncCommand};
 use crate::import_cpe::with_archive_cpe;
 use crate::import_cve::with_archive_cve;
 use crate::import_exploit::{
-  import_from_nuclei_templates_path, update_from_github, with_archive_exploit,
+  import_from_nuclei_templates_path, update_from_github, update_from_rss, with_archive_exploit,
 };
 pub use cli::{CPECommand, CVECommand, NVDHelper, TopLevel};
 pub use import_cpe::{create_cve_product, create_product, create_vendor};
@@ -111,6 +111,7 @@ pub async fn sync_mode(config: SyncCommand) {
     async_cve(param).await
   }
   if config.exp {
+    update_from_rss().await;
     update_from_github().await;
   }
 }