From 236230f4a9a43d63e7cd6cfd05ea5a83bf711892 Mon Sep 17 00:00:00 2001
From: Michal Warda <t1du53k@gmail.com>
Date: Sat, 24 Feb 2024 23:55:46 +0100
Subject: [PATCH] Fixes to webhook and document_search

---
 apps/api/lib/buildel/blocks/document_search.ex   | 14 ++++++++++++--
 apps/api/lib/buildel/blocks/utils/input_queue.ex |  4 ++--
 apps/api/lib/buildel/clients/webhook.ex          |  3 ++-
 apps/api/lib/buildel/vector_db.ex                |  6 ++++--
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/apps/api/lib/buildel/blocks/document_search.ex b/apps/api/lib/buildel/blocks/document_search.ex
index 1e3b86f9a..a31c7e8fa 100644
--- a/apps/api/lib/buildel/blocks/document_search.ex
+++ b/apps/api/lib/buildel/blocks/document_search.ex
@@ -124,11 +124,16 @@ defmodule Buildel.Blocks.DocumentSearch do
       |> Enum.map(fn
         %{
           "document" => document,
-          "metadata" => %{"file_name" => filename, "memory_id" => memory_id}
+          "metadata" => %{
+            "file_name" => filename,
+            "memory_id" => memory_id,
+            "chunk_id" => chunk_id
+          }
         } ->
           %{
             document_id: memory_id,
             document_name: filename,
+            chunk_id: chunk_id,
             chunk: document |> String.trim()
           }
       end)
@@ -180,11 +185,16 @@ defmodule Buildel.Blocks.DocumentSearch do
       |> Enum.map(fn
         %{
           "document" => document,
-          "metadata" => %{"file_name" => filename, "memory_id" => memory_id}
+          "metadata" => %{
+            "file_name" => filename,
+            "memory_id" => memory_id,
+            "chunk_id" => chunk_id
+          }
         } ->
           %{
             document_id: memory_id,
             document_name: filename,
+            chunk_id: chunk_id,
             chunk: document |> String.trim()
           }
       end)
diff --git a/apps/api/lib/buildel/blocks/utils/input_queue.ex b/apps/api/lib/buildel/blocks/utils/input_queue.ex
index 72a4529a4..4c492877e 100644
--- a/apps/api/lib/buildel/blocks/utils/input_queue.ex
+++ b/apps/api/lib/buildel/blocks/utils/input_queue.ex
@@ -46,9 +46,9 @@ defmodule Buildel.Blocks.Utils.InputQueue do
   end
 
   def pop(%__MODULE__{queue: [_item | [item | rest]], process_item: process_item}) do
-    IO.puts("removing another item from queue")
+    IO.puts("removing another item from queue with #{rest |> Enum.count()} items left")
     process_item.(item)
 
-    %__MODULE__{queue: rest, process_item: process_item}
+    %__MODULE__{queue: [item | rest], process_item: process_item}
   end
 end
diff --git a/apps/api/lib/buildel/clients/webhook.ex b/apps/api/lib/buildel/clients/webhook.ex
index 586026c84..206f2a441 100644
--- a/apps/api/lib/buildel/clients/webhook.ex
+++ b/apps/api/lib/buildel/clients/webhook.ex
@@ -11,7 +11,8 @@ defmodule Buildel.Clients.Webhook do
   def send_content(url, payload, headers \\ []) do
     headers = [{"Accept", "application/json"}, {"Content-Type", "application/json"}] ++ headers
 
-    with {:ok, response} <- HTTPoison.post(url, payload, headers) do
+    with {:ok, response} <-
+           HTTPoison.post(url, payload, headers, timeout: 60_000, recv_timeout: 60_000) do
       Logger.debug("Webhook response: #{inspect(response)}")
     else
       {:error, %HTTPoison.Error{reason: _reason} = error} ->
diff --git a/apps/api/lib/buildel/vector_db.ex b/apps/api/lib/buildel/vector_db.ex
index 22afdb3f7..ece3ce051 100644
--- a/apps/api/lib/buildel/vector_db.ex
+++ b/apps/api/lib/buildel/vector_db.ex
@@ -248,7 +248,8 @@ defmodule Buildel.VectorDB.EctoAdapter do
     |> Enum.map(fn chunk ->
       %{
         "document" => chunk.document,
-        "metadata" => chunk.metadata
+        "metadata" => chunk.metadata,
+        "chunk_id" => chunk.id
       }
     end)
   end
@@ -288,7 +289,8 @@ defmodule Buildel.VectorDB.EctoAdapter do
       |> Enum.map(fn chunk ->
         %{
           "document" => chunk.document,
-          "metadata" => chunk.metadata
+          "metadata" => chunk.metadata,
+          "chunk_id" => chunk.id
         }
       end)