feat(integrations): Support Qdrant 1.13 (#571)

Cargo.toml

@@ -29,7 +29,7 @@ tokio-stream = { version = "0.1" }
 tracing = { version = "0.1", features = ["log"] }
 num_cpus = { version = "1.16" }
 pin-project = { version = "1.1" }
-itertools = { version = "0.13" }
+itertools = { version = "0.14" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0" }
 strum = { version = "0.26" }
@@ -40,16 +40,16 @@ indoc = { version = "2.0" }
 regex = { version = "1.11.1" }
 uuid = { version = "1.11", features = ["v3", "v4", "serde"] }
 dyn-clone = { version = "1.0" }
-convert_case = "0.6.0"
+convert_case = "0.7.1"
 
 # Integrations
 spider = { version = "2.26" }
 async-openai = { version = "0.27.1" }
-qdrant-client = { version = "1.10", default-features = false, features = [
+qdrant-client = { version = "1.13", default-features = false, features = [
   "serde",
 ] }
 fluvio = { version = "0.24", default-features = false }
-lancedb = { version = "0.14", default-features = false }
+lancedb = { version = "0.15", default-features = false }
 arrow-array = { version = "53.3", default-features = false }
 arrow = { version = "53.3", default-features = false }
 parquet = { version = "53.3", default-features = false, features = ["async"] }
@@ -69,7 +69,7 @@ htmd = "0.1"
 ignore = "0.4"
 proc-macro2 = "1.0"
 quote = "1.0"
-redis = "0.27"
+redis = "0.28"
 reqwest = { version = "0.12.9", default-features = false }
 secrecy = "0.10.3"
 syn = "2.0"

swiftide-integrations/Cargo.toml

@@ -89,7 +89,7 @@ swiftide-test-utils = { path = "../swiftide-test-utils", features = [
   "test-utils",
 ] }
 temp-dir = { workspace = true }
-
+pretty_assertions = { workspace = true }
 arrow = { workspace = true, features = ["test_utils"] }
 
 # Used for hacking fluv to play nice

swiftide-integrations/src/qdrant/indexing_node.rs

@@ -73,6 +73,8 @@ fn try_create_vectors(
     vectors: HashMap<EmbeddedField, Embedding>,
     sparse_vectors: Option<HashMap<EmbeddedField, SparseEmbedding>>,
 ) -> Result<qdrant::Vectors> {
+    dbg!(&vector_fields);
+    dbg!(&vectors);
     if vectors.is_empty() {
         bail!("Node with empty vectors")
     } else if vectors.len() == 1 && sparse_vectors.is_none() {
@@ -113,13 +115,12 @@ fn try_create_vectors(
 mod tests {
     use std::collections::{HashMap, HashSet};
 
-    use qdrant_client::qdrant::{
-        vectors::VectorsOptions, NamedVectors, PointId, PointStruct, Value, Vector, Vectors,
-    };
+    use qdrant_client::qdrant::PointStruct;
     use swiftide_core::indexing::{EmbeddedField, Node};
     use test_case::test_case;
 
     use crate::qdrant::indexing_node::NodeWithVectors;
+    use pretty_assertions::assert_eq;
 
     static EXPECTED_UUID: &str = "d42d252d-671d-37ef-a157-8e85d0710610";
 
@@ -133,12 +134,11 @@ mod tests {
             .build().unwrap()
         ,
         HashSet::from([EmbeddedField::Combined]),
-        PointStruct { id: Some(PointId::from(EXPECTED_UUID)), payload: HashMap::from([
-            ("content".into(), Value::from("data")),
-            ("path".into(), Value::from("/path")),
-            ("m1".into(), Value::from("mv1"))]),
-            vectors: Some(Vectors { vectors_options: Some(VectorsOptions::Vector(Vector { data: vec![1.0], ..Default::default()} )) })
-        };
+        PointStruct::new(EXPECTED_UUID, vec![1.0], HashMap::from([
+            ("content", "data".into()),
+            ("path", "/path".into()),
+            ("m1", "mv1".into())])
+        );
         "Node with single vector creates struct with unnamed vector"
     )]
     #[test_case(
@@ -153,19 +153,15 @@ mod tests {
             .embed_mode(swiftide_core::indexing::EmbedMode::PerField)
             .build().unwrap(),
         HashSet::from([EmbeddedField::Chunk, EmbeddedField::Metadata("m1".into())]),
-        PointStruct { id: Some(PointId::from(EXPECTED_UUID)), payload: HashMap::from([
-            ("content".into(), Value::from("data")),
-            ("path".into(), Value::from("/path")),
-            ("m1".into(), Value::from("mv1"))]),
-            vectors: Some(Vectors { vectors_options: Some(VectorsOptions::Vectors(NamedVectors { vectors: HashMap::from([
-                ("Chunk".into(), qdrant_client::qdrant::Vector {
-                    data: vec![1.0], ..Default::default()
-                }),
-                ("Metadata: m1".into(), qdrant_client::qdrant::Vector {
-                    data: vec![2.0], ..Default::default()
-                })
-            ]) })) })
-        };
+        PointStruct::new(EXPECTED_UUID, HashMap::from([
+                ("Chunk".to_string(), vec![1.0]),
+                ("Metadata: m1".to_string(), vec![2.0])
+            ]),
+            HashMap::from([
+                ("content", "data".into()),
+                ("path", "/path".into()),
+                ("m1", "mv1".into())])
+        );
         "Node with multiple vectors creates struct with named vectors"
     )]
     #[test_case(
@@ -182,17 +178,16 @@ mod tests {
             .embed_mode(swiftide_core::indexing::EmbedMode::Both)
             .build().unwrap(),
         HashSet::from([EmbeddedField::Combined]),
-        PointStruct { id: Some(PointId::from(EXPECTED_UUID)), payload: HashMap::from([
-            ("content".into(), Value::from("data")),
-            ("path".into(), Value::from("/path")),
-            ("m1".into(), Value::from("mv1")),
-            ("m2".into(), Value::from("mv2"))]),
-            vectors: Some(Vectors { vectors_options: Some(VectorsOptions::Vectors(NamedVectors { vectors: HashMap::from([
-                ("Combined".into(), qdrant_client::qdrant::Vector {
-                    data: vec![1.0], ..Default::default()
-                })
-            ]) })) })
-        };
+        PointStruct::new(EXPECTED_UUID,
+            HashMap::from([
+                ("Combined".to_string(), vec![1.0]),
+            ]),
+            HashMap::from([
+                ("content", "data".into()),
+                ("path", "/path".into()),
+                ("m1", "mv1".into()),
+                ("m2", "mv2".into())])
+        );
         "Storing only `Combined` vector. Skipping other vectors."
     )]
     #[allow(clippy::needless_pass_by_value)]
@@ -214,6 +209,8 @@ mod tests {
             .payload
             .insert(last_updated_at_key.into(), last_updated_at.clone());
 
-        assert_eq!(point, expected_point);
+        assert_eq!(point.id, expected_point.id);
+        assert_eq!(point.payload, expected_point.payload);
+        assert_eq!(point.vectors, expected_point.vectors);
     }
 }

swiftide-integrations/src/qdrant/mod.rs

@@ -167,6 +167,11 @@ impl Qdrant {
         let size = config.vector_size.unwrap_or(self.vector_size);
         let distance = config.distance.unwrap_or(self.vector_distance);
 
+        tracing::debug!(
+            "Creating vector params: size={}, distance={:?}",
+            size,
+            distance
+        );
         qdrant::VectorParamsBuilder::new(size, distance).build()
     }
 

swiftide-integrations/src/qdrant/retrieve.rs

@@ -207,7 +207,7 @@ mod tests {
         (guard, qdrant_client)
     }
 
-    #[tokio::test]
+    #[test_log::test(tokio::test)]
     async fn test_retrieve_multiple_docs_and_filter() {
         let (_guard, qdrant_client) = setup().await;
 

swiftide-test-utils/src/test_utils.rs

@@ -35,7 +35,7 @@ pub fn openai_client(
 /// Setup Qdrant container.
 /// Returns container server and `server_url`.
 pub async fn start_qdrant() -> (ContainerAsync<GenericImage>, String) {
-    let qdrant = testcontainers::GenericImage::new("qdrant/qdrant", "v1.12.1")
+    let qdrant = testcontainers::GenericImage::new("qdrant/qdrant", "v1.13.1")
         .with_exposed_port(6334.into())
         .with_exposed_port(6333.into())
         .with_wait_for(testcontainers::core::WaitFor::http(

swiftide/tests/indexing_pipeline.rs

@@ -2,7 +2,7 @@
 //! The tests validate the functionality of the pipeline, ensuring it processes data correctly
 //! from a temporary file, simulates API responses, and stores data accurately in the Qdrant vector database.
 
-use qdrant_client::qdrant::vectors::VectorsOptions;
+use qdrant_client::qdrant::vectors_output::VectorsOptions;
 use qdrant_client::qdrant::{ScrollPointsBuilder, SearchPointsBuilder, Value};
 use swiftide::indexing::*;
 use swiftide::integrations;