Fixes to webhook and document_search

elpassion · Feb 24, 2024 · 236230f · 236230f
1 parent 6f09ce9
commit 236230f
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 7 deletions.
diff --git a/apps/api/lib/buildel/blocks/document_search.ex b/apps/api/lib/buildel/blocks/document_search.ex
@@ -124,11 +124,16 @@ defmodule Buildel.Blocks.DocumentSearch do
       |> Enum.map(fn
         %{
           "document" => document,
-          "metadata" => %{"file_name" => filename, "memory_id" => memory_id}
+          "metadata" => %{
+            "file_name" => filename,
+            "memory_id" => memory_id,
+            "chunk_id" => chunk_id
+          }
         } ->
           %{
             document_id: memory_id,
             document_name: filename,
+            chunk_id: chunk_id,
             chunk: document |> String.trim()
           }
       end)
@@ -180,11 +185,16 @@ defmodule Buildel.Blocks.DocumentSearch do
       |> Enum.map(fn
         %{
           "document" => document,
-          "metadata" => %{"file_name" => filename, "memory_id" => memory_id}
+          "metadata" => %{
+            "file_name" => filename,
+            "memory_id" => memory_id,
+            "chunk_id" => chunk_id
+          }
         } ->
           %{
             document_id: memory_id,
             document_name: filename,
+            chunk_id: chunk_id,
             chunk: document |> String.trim()
           }
       end)

diff --git a/apps/api/lib/buildel/blocks/utils/input_queue.ex b/apps/api/lib/buildel/blocks/utils/input_queue.ex
@@ -46,9 +46,9 @@ defmodule Buildel.Blocks.Utils.InputQueue do
   end
 
   def pop(%__MODULE__{queue: [_item | [item | rest]], process_item: process_item}) do
-    IO.puts("removing another item from queue")
+    IO.puts("removing another item from queue with #{rest |> Enum.count()} items left")
     process_item.(item)
 
-    %__MODULE__{queue: rest, process_item: process_item}
+    %__MODULE__{queue: [item | rest], process_item: process_item}
   end
 end
diff --git a/apps/api/lib/buildel/clients/webhook.ex b/apps/api/lib/buildel/clients/webhook.ex
@@ -11,7 +11,8 @@ defmodule Buildel.Clients.Webhook do
   def send_content(url, payload, headers \\ []) do
     headers = [{"Accept", "application/json"}, {"Content-Type", "application/json"}] ++ headers
 
-    with {:ok, response} <- HTTPoison.post(url, payload, headers) do
+    with {:ok, response} <-
+           HTTPoison.post(url, payload, headers, timeout: 60_000, recv_timeout: 60_000) do
       Logger.debug("Webhook response: #{inspect(response)}")
     else
       {:error, %HTTPoison.Error{reason: _reason} = error} ->

diff --git a/apps/api/lib/buildel/vector_db.ex b/apps/api/lib/buildel/vector_db.ex
@@ -248,7 +248,8 @@ defmodule Buildel.VectorDB.EctoAdapter do
     |> Enum.map(fn chunk ->
       %{
         "document" => chunk.document,
-        "metadata" => chunk.metadata
+        "metadata" => chunk.metadata,
+        "chunk_id" => chunk.id
       }
     end)
   end
@@ -288,7 +289,8 @@ defmodule Buildel.VectorDB.EctoAdapter do
       |> Enum.map(fn chunk ->
         %{
           "document" => chunk.document,
-          "metadata" => chunk.metadata
+          "metadata" => chunk.metadata,
+          "chunk_id" => chunk.id
         }
       end)