Merge branch 'leaflet'

trailbaseio · Nov 5, 2024 · eedce56 · eedce56
2 parents c7a670c + d7af91c
commit eedce56
Show file tree

Hide file tree

Showing 15 changed files with 13,668 additions and 44 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/trailbase-core/bindings/LogJson.ts b/trailbase-core/bindings/LogJson.ts
@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type LogJson = { id: string, created: number, type: number, level: number, status: number, method: string, url: string, latency_ms: number, client_ip: string, referer: string, user_agent: string, data: Object | undefined, };
+export type LogJson = { id: string, created: number, type: number, level: number, status: number, method: string, url: string, latency_ms: number, client_ip: string, client_cc: string | null, referer: string, user_agent: string, data: Object | undefined, };
diff --git a/trailbase-core/bindings/Stats.ts b/trailbase-core/bindings/Stats.ts
@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type Stats = { rate: Array<[bigint, number]>, };
+export type Stats = { rate: Array<[bigint, number]>, country_codes: { [key in string]?: number } | null, };
diff --git a/trailbase-core/src/admin/list_logs.rs b/trailbase-core/src/admin/list_logs.rs
@@ -7,6 +7,7 @@ use lazy_static::lazy_static;
 use libsql::{de, params::Params, Connection};
 use log::*;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use trailbase_sqlite::query_one_row;
 use ts_rs::TS;
 use uuid::Uuid;
@@ -17,7 +18,6 @@ use crate::constants::{LOGS_RETENTION_DEFAULT, LOGS_TABLE_ID_COLUMN};
 use crate::listing::{
   build_filter_where_clause, limit_or_default, parse_query, Order, WhereClause,
 };
-use crate::logging::Log;
 use crate::table_metadata::{lookup_and_parse_table_schema, TableMetadata};
 use crate::util::id_to_b64;
 
@@ -35,15 +35,38 @@ pub struct LogJson {
 
   pub latency_ms: f64,
   pub client_ip: String,
+  pub client_cc: Option<String>,
   pub referer: String,
   pub user_agent: String,
 
   #[ts(type = "Object | undefined")]
   pub data: Option<serde_json::Value>,
 }
 
-impl From<Log> for LogJson {
-  fn from(value: Log) -> Self {
+#[derive(Debug, Clone, Deserialize)]
+struct LogQuery {
+  id: Option<[u8; 16]>,
+  created: Option<f64>,
+  r#type: i32,
+
+  level: i32,
+  status: u16,
+  method: String,
+  url: String,
+
+  // milliseconds
+  latency: f64,
+  client_ip: String,
+  client_cc: Option<String>,
+
+  referer: String,
+  user_agent: String,
+
+  data: Option<serde_json::Value>,
+}
+
+impl From<LogQuery> for LogJson {
+  fn from(value: LogQuery) -> Self {
     return LogJson {
       id: Uuid::from_bytes(value.id.unwrap()),
       created: value.created.unwrap_or(0.0),
@@ -54,6 +77,7 @@ impl From<Log> for LogJson {
       url: value.url,
       latency_ms: value.latency,
       client_ip: value.client_ip,
+      client_cc: value.client_cc,
       referer: value.referer,
       user_agent: value.user_agent,
       data: value.data,
@@ -71,7 +95,6 @@ pub struct ListLogsResponse {
   stats: Option<Stats>,
 }
 
-// FIXME: should be an admin-only api.
 pub async fn list_logs_handler(
   State(state): State<AppState>,
   RawQuery(raw_url_query): RawQuery,
@@ -166,21 +189,21 @@ async fn fetch_logs(
   cursor: Option<[u8; 16]>,
   order: Vec<(String, Order)>,
   limit: usize,
-) -> Result<Vec<Log>, Error> {
+) -> Result<Vec<LogQuery>, Error> {
   let mut params = filter_where_clause.params;
   let mut where_clause = filter_where_clause.clause;
   params.push((":limit".to_string(), libsql::Value::Integer(limit as i64)));
 
   if let Some(cursor) = cursor {
     params.push((":cursor".to_string(), libsql::Value::Blob(cursor.to_vec())));
-    where_clause = format!("{where_clause} AND _row_.id < :cursor",);
+    where_clause = format!("{where_clause} AND log.id < :cursor",);
   }
 
   let order_clause = order
     .iter()
     .map(|(col, ord)| {
       format!(
-        "_row_.{col} {}",
+        "log.{col} {}",
         match ord {
           Order::Descending => "DESC",
           Order::Ascending => "ASC",
@@ -192,9 +215,9 @@ async fn fetch_logs(
 
   let sql_query = format!(
     r#"
-      SELECT _row_.*
+      SELECT log.*, geoip_country(log.client_ip) AS client_cc
       FROM
-        (SELECT * FROM {LOGS_TABLE_NAME}) as _row_
+        (SELECT * FROM {LOGS_TABLE_NAME}) AS log
       WHERE
         {where_clause}
       ORDER BY
@@ -205,20 +228,23 @@ async fn fetch_logs(
 
   let mut rows = conn.query(&sql_query, Params::Named(params)).await?;
 
-  let mut logs: Vec<Log> = vec![];
+  let mut logs: Vec<LogQuery> = vec![];
   while let Ok(Some(row)) = rows.next().await {
     match de::from_row(&row) {
       Ok(log) => logs.push(log),
       Err(err) => warn!("failed: {err}"),
     };
   }
+
   return Ok(logs);
 }
 
 #[derive(Debug, Serialize, TS)]
 pub struct Stats {
   // List of (timestamp, value).
   rate: Vec<(i64, f64)>,
+  // Country codes.
+  country_codes: Option<HashMap<String, usize>>,
 }
 
 #[derive(Debug)]
@@ -299,7 +325,42 @@ async fn fetch_aggregate_stats(
     ));
   }
 
-  return Ok(Stats { rate });
+  if trailbase_sqlite::has_geoip_db() {
+    let cc_query = format!(
+      r#"
+    SELECT
+      country_code,
+      SUM(cnt) as count
+    FROM
+      (SELECT client_ip, COUNT(*) AS cnt, geoip_country(client_ip) as country_code FROM {LOGS_TABLE_NAME} GROUP BY client_ip)
+    GROUP BY
+      country_code
+  "#
+    );
+
+    let mut rows = conn.query(&cc_query, ()).await?;
+
+    let mut country_codes = HashMap::<String, usize>::new();
+    while let Ok(Some(row)) = rows.next().await {
+      let cc: Option<String> = row.get(0)?;
+      let count: i64 = row.get(1)?;
+
+      country_codes.insert(
+        cc.unwrap_or_else(|| "unattributed".to_string()),
+        count as usize,
+      );
+    }
+
+    return Ok(Stats {
+      rate,
+      country_codes: Some(country_codes),
+    });
+  }
+
+  return Ok(Stats {
+    rate,
+    country_codes: None,
+  });
 }
 
 #[cfg(test)]

diff --git a/trailbase-core/src/server/init.rs b/trailbase-core/src/server/init.rs
@@ -91,6 +91,12 @@ pub async fn init_app_state(
 
   let jwt = JwtHelper::init_from_path(&data_dir).await?;
 
+  // Init geoip if present.
+  let geoip_db_path = data_dir.root().join("GeoLite2-Country.mmdb");
+  if let Err(err) = trailbase_sqlite::load_geoip_db(geoip_db_path.clone()) {
+    debug!("Failed to load maxmind geoip DB '{geoip_db_path:?}': {err}");
+  }
+
   let app_state = AppState::new(
     data_dir.clone(),
     public_dir,

diff --git a/trailbase-extension/Cargo.toml b/trailbase-extension/Cargo.toml
@@ -12,6 +12,7 @@ argon2 = "0.5.3"
 base64 = "0.22.1"
 jsonschema = { version = "0.26.0", default-features = false }
 lru = { version = "0.12.3", default-features = false }
+maxminddb = "0.24.0"
 parking_lot = { version = "0.12.3", default-features = false }
 rand = "0.8.5"
 regex = "1.11.0"