diff --git a/Cargo.lock b/Cargo.lock index 11c2a27..5afeb3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,7 +138,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -249,6 +249,56 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aed08d3adb6ebe0eff737115056652670ae290f177759aac19c30456135f94c" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal-next", + "log", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.44.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709d9aa1c37abb89d40f19f5d0ad6f0d88cb1581264e571c9350fc5bb89cf1c5" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -299,6 +349,7 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", + "serde", "windows-targets 0.52.5", ] @@ -333,7 +384,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -367,6 +418,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -474,6 +535,51 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +dependencies = [ + "quote", + "syn 2.0.66", +] + +[[package]] +name = "darling" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.66", +] + +[[package]] +name = "darling_macro" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.66", +] + [[package]] name = "der" version = "0.7.9" @@ -507,6 +613,38 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "docker_credential" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31951f49556e34d90ed28342e1df7e1cb7a229c4cab0aecc627b5d91edd41d07" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -705,7 +843,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -780,6 +918,12 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -796,7 +940,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -924,6 +1068,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.26.0" @@ -934,7 +1093,9 @@ dependencies = [ "http", "hyper", "hyper-util", + "log", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -961,6 +1122,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal-next" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf569d43fa9848e510358c07b80f4adf34084ddc28c6a4a651ee8474c070dcc" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -984,6 +1160,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.5.0" @@ -1000,6 +1182,17 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -1007,7 +1200,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", + "serde", ] [[package]] @@ -1105,6 +1299,16 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.27.0" @@ -1158,7 +1362,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -1385,6 +1589,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "overload" version = "0.1.1" @@ -1429,6 +1645,31 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax 0.8.3", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax 0.8.3", + "structmeta", + "syn 2.0.66", +] + [[package]] name = "paste" version = "1.0.15" @@ -1467,7 +1708,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -1533,9 +1774,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -1637,6 +1878,17 @@ dependencies = [ "bitflags 2.5.0", ] +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.10.4" @@ -1817,6 +2069,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "2.1.2" @@ -1856,6 +2121,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -1868,6 +2142,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.23" @@ -1891,7 +2188,7 @@ checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -1905,6 +2202,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1917,13 +2225,43 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -2084,7 +2422,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap", + "indexmap 2.2.6", "log", "memchr", "once_cell", @@ -2267,6 +2605,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.66", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "subtle" version = "2.5.0" @@ -2286,9 +2647,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.61" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -2463,6 +2824,41 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "testcontainers" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025e0ac563d543e0354d984540e749859a83dbe5c0afb8d458dc48d91cef2d6a" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "dirs", + "docker_credential", + "futures", + "log", + "memchr", + "parse-display", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87dc086b8c791e5696de6de4e0d79de2cb9f8a1ac4552d46d6211a42d81d6d0f" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "1.0.60" @@ -2480,7 +2876,7 @@ checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -2566,7 +2962,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -2613,6 +3009,7 @@ dependencies = [ "bincode", "clap", "color-eyre", + "ctor", "dotenvy", "futures", "log", @@ -2625,6 +3022,8 @@ dependencies = [ "serde_yaml", "sqlx", "tantivy", + "testcontainers", + "testcontainers-modules", "tokio", "tokio-util", "uuid", @@ -2678,7 +3077,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] @@ -2792,6 +3191,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2883,7 +3283,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -2917,7 +3317,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3185,7 +3585,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.66", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cec613a..c02df8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,3 +34,6 @@ uuid = { version = "1.8.0", features = ["v7"] } [dev-dependencies] async-tempfile = "0.5.0" +ctor = "0.2.8" +testcontainers = "0.17.0" +testcontainers-modules = { version = "0.5.0", features = ["postgres"] } diff --git a/src/commands/create.rs b/src/commands/create.rs index f87b922..70c4ebd 100644 --- a/src/commands/create.rs +++ b/src/commands/create.rs @@ -6,9 +6,12 @@ use crate::{ config::{FieldType, IndexConfig}, }; -pub async fn run_create(args: CreateArgs, pool: PgPool) -> Result<()> { +pub async fn run_create(args: CreateArgs, pool: &PgPool) -> Result<()> { let config = IndexConfig::from_path(&args.config_path).await?; + run_create_from_config(&config, pool).await +} +pub async fn run_create_from_config(config: &IndexConfig, pool: &PgPool) -> Result<()> { let array_static_object_exists = config .schema .fields @@ -20,8 +23,8 @@ pub async fn run_create(args: CreateArgs, pool: PgPool) -> Result<()> { query("INSERT INTO indexes (name, config) VALUES ($1, $2)") .bind(&config.name) - .bind(&serde_json::to_value(&config)?) - .execute(&pool) + .bind(&serde_json::to_value(config)?) + .execute(pool) .await?; info!("Created index: {}", &config.name); diff --git a/src/commands/drop.rs b/src/commands/drop.rs index 7590e80..7c0b0e9 100644 --- a/src/commands/drop.rs +++ b/src/commands/drop.rs @@ -8,13 +8,13 @@ use crate::args::DropArgs; use super::get_index_path; -pub async fn run_drop(args: DropArgs, pool: PgPool) -> Result<()> { - let base_path = get_index_path(&args.name, &pool).await?; +pub async fn run_drop(args: DropArgs, pool: &PgPool) -> Result<()> { + let base_path = get_index_path(&args.name, pool).await?; let file_names: Vec<(String,)> = query_as("SELECT file_name FROM index_files WHERE index_name=$1") .bind(&args.name) - .fetch_all(&pool) + .fetch_all(pool) .await?; let file_names_len = file_names.len(); @@ -30,7 +30,7 @@ pub async fn run_drop(args: DropArgs, pool: PgPool) -> Result<()> { query("DELETE FROM indexes WHERE name=$1") .bind(&args.name) - .execute(&pool) + .execute(pool) .await?; info!( diff --git a/src/commands/index.rs b/src/commands/index.rs index f8b0bbf..4f50985 100644 --- a/src/commands/index.rs +++ b/src/commands/index.rs @@ -14,8 +14,8 @@ use crate::{args::IndexArgs, commands::field_parser::build_parsers_from_field_co use super::{dynamic_field_config, get_index_config, write_unified_index, DYNAMIC_FIELD_NAME}; -pub async fn run_index(args: IndexArgs, pool: PgPool) -> Result<()> { - let config = get_index_config(&args.name, &pool).await?; +pub async fn run_index(args: IndexArgs, pool: &PgPool) -> Result<()> { + let config = get_index_config(&args.name, pool).await?; let mut schema_builder = Schema::builder(); let dynamic_field = schema_builder.add_json_field(DYNAMIC_FIELD_NAME, dynamic_field_config()); @@ -30,8 +30,10 @@ pub async fn run_index(args: IndexArgs, pool: PgPool) -> Result<()> { index_writer.set_merge_policy(Box::new(NoMergePolicy)); let input: Box = if let Some(input) = args.input { + debug!("reading from '{}'", &input); Box::new(File::open(&input).await?) } else { + debug!("reading from stdin"); Box::new(stdin()) }; let mut reader = BufReader::new(input); @@ -78,7 +80,7 @@ pub async fn run_index(args: IndexArgs, pool: PgPool) -> Result<()> { spawn_blocking(move || index_writer.wait_merging_threads()).await??; - write_unified_index(index, &args.build_dir, &config.name, &config.path, &pool).await?; + write_unified_index(index, &args.build_dir, &config.name, &config.path, pool).await?; Ok(()) } diff --git a/src/commands/merge.rs b/src/commands/merge.rs index fcbad45..25be96a 100644 --- a/src/commands/merge.rs +++ b/src/commands/merge.rs @@ -18,10 +18,10 @@ use super::{get_index_config, open_unified_directories, write_unified_index}; const MIN_TANTIVY_MEMORY: usize = 15_000_000; -pub async fn run_merge(args: MergeArgs, pool: PgPool) -> Result<()> { - let config = get_index_config(&args.name, &pool).await?; +pub async fn run_merge(args: MergeArgs, pool: &PgPool) -> Result<()> { + let config = get_index_config(&args.name, pool).await?; - let (ids, directories): (Vec<_>, Vec<_>) = open_unified_directories(&config.path, &pool) + let (ids, directories): (Vec<_>, Vec<_>) = open_unified_directories(&config.path, pool) .await? .into_iter() .map(|(id, dir)| (id, dir.box_clone())) @@ -47,11 +47,11 @@ pub async fn run_merge(args: MergeArgs, pool: PgPool) -> Result<()> { spawn_blocking(move || index_writer.wait_merging_threads()).await??; - write_unified_index(index, &args.merge_dir, &config.name, &config.path, &pool).await?; + write_unified_index(index, &args.merge_dir, &config.name, &config.path, pool).await?; let delete_result = query("DELETE FROM index_files WHERE id = ANY($1)") .bind(&ids) - .execute(&pool) + .execute(pool) .await; for id in ids { diff --git a/src/commands/search.rs b/src/commands/search.rs index 27bc0eb..a231793 100644 --- a/src/commands/search.rs +++ b/src/commands/search.rs @@ -108,7 +108,11 @@ where rx } -pub async fn run_search(args: SearchArgs, pool: PgPool) -> Result<()> { +pub async fn run_search_with_callback( + args: SearchArgs, + pool: &PgPool, + on_doc_fn: Box, +) -> Result<()> { if args.limit == 0 { return Ok(()); } @@ -129,7 +133,7 @@ pub async fn run_search(args: SearchArgs, pool: PgPool) -> Result<()> { }) .build()?; - let config = get_index_config(&args.name, &pool).await?; + let config = get_index_config(&args.name, pool).await?; let indexed_field_names = { let mut fields = config.schema.fields.get_indexed(); @@ -141,7 +145,7 @@ pub async fn run_search(args: SearchArgs, pool: PgPool) -> Result<()> { fields }; - let directories = open_unified_directories(&config.path, &pool) + let directories = open_unified_directories(&config.path, pool) .await? .into_iter() .map(|(_, x)| x) @@ -197,7 +201,7 @@ pub async fn run_search(args: SearchArgs, pool: PgPool) -> Result<()> { let mut rx_handle = spawn(async move { let mut i = 0; while let Some(doc) = rx.recv().await { - println!("{}", doc); + on_doc_fn(doc); i += 1; if i == args.limit { break; @@ -226,3 +230,14 @@ pub async fn run_search(args: SearchArgs, pool: PgPool) -> Result<()> { Ok(()) } + +pub async fn run_search(args: SearchArgs, pool: &PgPool) -> Result<()> { + run_search_with_callback( + args, + pool, + Box::new(|doc| { + println!("{}", doc); + }), + ) + .await +} diff --git a/src/config/mod.rs b/src/config/mod.rs index e41add4..9a8d4e8 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -6,7 +6,7 @@ pub mod number; pub mod static_object; pub mod text; -use std::{ops::Deref, path::Path, vec::IntoIter}; +use std::{ops::Deref, path::Path, str::FromStr, vec::IntoIter}; use color_eyre::eyre::Result; use serde::{Deserialize, Serialize}; @@ -232,7 +232,7 @@ pub struct IndexSchema { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct IndexConfig { +pub struct IndexConfig { pub name: String, pub path: String, @@ -246,6 +246,14 @@ pub(crate) struct IndexConfig { impl IndexConfig { pub async fn from_path>(path: P) -> Result { let config_str = read_to_string(path).await?; - Ok(serde_yaml::from_str(&config_str)?) + Self::from_str(&config_str) + } +} + +impl FromStr for IndexConfig { + type Err = color_eyre::Report; + + fn from_str(s: &str) -> Result { + Ok(serde_yaml::from_str(s)?) } } diff --git a/src/main.rs b/src/main.rs index b410734..df75117 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,19 +36,19 @@ async fn async_main(args: Args) -> Result<()> { match args.subcmd { SubCommand::Create(create_args) => { - run_create(create_args, pool).await?; + run_create(create_args, &pool).await?; } SubCommand::Drop(drop_args) => { - run_drop(drop_args, pool).await?; + run_drop(drop_args, &pool).await?; } SubCommand::Index(index_args) => { - run_index(index_args, pool).await?; + run_index(index_args, &pool).await?; } SubCommand::Merge(merge_args) => { - run_merge(merge_args, pool).await?; + run_merge(merge_args, &pool).await?; } SubCommand::Search(search_args) => { - run_search(search_args, pool).await?; + run_search(search_args, &pool).await?; } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..5854797 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,55 @@ +use std::{ + fs::canonicalize, + path::{Path, PathBuf}, +}; + +use color_eyre::Result; +use sqlx::{migrate::Migrator, postgres::PgPoolOptions, PgPool}; +use testcontainers::{runners::AsyncRunner, ContainerAsync}; +use testcontainers_modules::postgres::Postgres as PostgresContainer; + +static MIGRATOR: Migrator = sqlx::migrate!(); + +const MAX_DB_CONNECTIONS: u32 = 100; + +pub struct Postgres { + /// Keep container alive (container is deleted on drop). + _container: ContainerAsync, + + /// The underlying sqlx connection to the postgres inside the container. + pub pool: PgPool, +} + +async fn open_db_pool(url: &str) -> Result { + Ok(PgPoolOptions::new() + .max_connections(MAX_DB_CONNECTIONS) + .connect(url) + .await?) +} + +pub async fn run_postgres() -> Result { + let container = PostgresContainer::default().start().await?; + let pool = open_db_pool(&format!( + "postgres://postgres:postgres@127.0.0.1:{}/postgres", + container.get_host_port_ipv4(5432).await? + )) + .await?; + + MIGRATOR.run(&pool).await?; + + Ok(Postgres { + _container: container, + pool, + }) +} + +pub fn get_test_file_path(test_file: &str) -> PathBuf { + canonicalize(&Path::new(file!())) + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .join("test_files") + .join(test_file) +} diff --git a/tests/config.rs b/tests/config.rs new file mode 100644 index 0000000..6336d51 --- /dev/null +++ b/tests/config.rs @@ -0,0 +1,71 @@ +mod common; + +use std::str::FromStr; + +use clap::Parser; +use color_eyre::Result; +use ctor::ctor; +use pretty_env_logger::formatted_timed_builder; +use tokio::sync::mpsc; +use toshokan::{ + args::{DropArgs, IndexArgs, SearchArgs}, + commands::{ + create::run_create_from_config, drop::run_drop, index::run_index, + search::run_search_with_callback, + }, + config::IndexConfig, +}; + +use crate::common::{get_test_file_path, run_postgres}; + +#[ctor] +fn init() { + color_eyre::install().unwrap(); + + let mut log_builder = formatted_timed_builder(); + log_builder.parse_filters("toshokan=trace,opendal::services=info"); + log_builder.try_init().unwrap(); +} + +#[tokio::test] +async fn test_example_config() -> Result<()> { + let postgres = run_postgres().await?; + let config = IndexConfig::from_str(include_str!("../example_config.yaml"))?; + + run_create_from_config(&config, &postgres.pool).await?; + + run_index( + IndexArgs::parse_from([ + "", + &config.name, + &get_test_file_path("hdfs-logs-multitenants-2.json").to_string_lossy(), + ]), + &postgres.pool, + ) + .await?; + + let (tx, mut rx) = mpsc::channel(1); + run_search_with_callback( + SearchArgs::parse_from([ + "", + &config.name, + "tenant_id:>50 AND severity_text:INFO", + "--limit", + "1", + ]), + &postgres.pool, + Box::new(move |doc| { + tx.try_send(doc).unwrap(); + }), + ) + .await?; + + assert_eq!( + rx.recv().await.unwrap(), + r#"{"attributes":{"class":"org.apache.hadoop.hdfs.server.datanode.DataNode"},"body":"PacketResponder: BP-108841162-10.10.34.11-1440074360971:blk_1074072698_331874, type=HAS_DOWNSTREAM_IN_PIPELINE terminating","resource":{"service":"datanode/01"},"severity_text":"INFO","tenant_id":58,"timestamp":"2016-04-13T06:46:53Z"}"# + ); + + run_drop(DropArgs::parse_from(["", &config.name]), &postgres.pool).await?; + + Ok(()) +} diff --git a/tests/test_files/hdfs-logs-multitenants-2.json b/tests/test_files/hdfs-logs-multitenants-2.json new file mode 100644 index 0000000..5b52e87 --- /dev/null +++ b/tests/test_files/hdfs-logs-multitenants-2.json @@ -0,0 +1,2 @@ +{"timestamp":1460530013,"severity_text":"INFO","body":"PacketResponder: BP-108841162-10.10.34.11-1440074360971:blk_1074072698_331874, type=HAS_DOWNSTREAM_IN_PIPELINE terminating","resource":{"service":"datanode/01"},"attributes":{"class":"org.apache.hadoop.hdfs.server.datanode.DataNode"},"tenant_id":58} +{"timestamp":1460530014,"severity_text":"INFO","body":"Receiving BP-108841162-10.10.34.11-1440074360971:blk_1074072706_331882 src: /10.10.34.33:42666 dest: /10.10.34.11:50010","resource":{"service":"datanode/01"},"attributes":{"class":"org.apache.hadoop.hdfs.server.datanode.DataNode"},"tenant_id":46}