02_implement_search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hybrid search for the metadata catalogue of Berlin\n",
    "\n",
    "In this notebook we prepare the data for our search application and setup the search index.\n",
    "\n",
    "- We load the data from the API.\n",
    "- We prepare the data and lemmatize the text data (for lexical search).\n",
    "- We embed via the OpenAI embedding API.\n",
    "- We test the embeddings with cosine similarity.\n",
    "- We setup the Weaviate index.\n",
    "- We create a collection with our data.\n",
    "- We test the index in regard to lexical, and vector search as well as the combination of both - hybrid search. The latter is what we use in the app.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 2 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pandarallel import pandarallel\n",
    "import numpy as np\n",
    "\n",
    "pd.options.mode.chained_assignment = None\n",
    "pd.options.display.max_rows = 500\n",
    "pd.options.display.max_seq_items = 500\n",
    "pandarallel.initialize(progress_bar=False)\n",
    "\n",
    "from time import time\n",
    "import time\n",
    "import os\n",
    "import re\n",
    "import requests\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "import pyarrow.parquet as pq\n",
    "\n",
    "import warnings\n",
    "from bs4 import MarkupResemblesLocatorWarning\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import weaviate\n",
    "from weaviate.classes.config import Property, DataType\n",
    "import weaviate.classes as wvc\n",
    "import weaviate.classes.config as wc\n",
    "\n",
    "import spacy\n",
    "from openai import OpenAI\n",
    "import json\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "warnings.filterwarnings(\"ignore\", category=MarkupResemblesLocatorWarning)\n",
    "\n",
    "load_dotenv()\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "client_openai = OpenAI(api_key=OPENAI_API_KEY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Constants**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FOLDER = \"_data/\"\n",
    "DATASETS = DATA_FOLDER + \"01_data.parq\"\n",
    "DATA_WITH_EMBEDDINGS = DATA_FOLDER + \"02_data_embedded.parq\"\n",
    "\n",
    "BASELINK_API = \"https://daten.berlin.de/datensaetze/\"\n",
    "\n",
    "# Dataset links are composed of this baselink and the identifier for each dataset.\n",
    "BASELINK_DATASHOP = (\n",
    "    \"https://datenregister.berlin.de/api/3/action/current_package_list_with_resources\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement search\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we create the actual search index for lexical and semantic search.\n",
    "\n",
    "Note that [Weaviates](https://weaviate.io/developers/weaviate) default location for the index is `~/.local/share/weaviate`. If you want to use the app on a different machine, simply copy the files from this location to the same location on the other machine. You can also set a specific path with parameter `persistence_data_path` – see [documentation here](https://weaviate.io/developers/weaviate/installation/embedded).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_parquet(DATA_WITH_EMBEDDINGS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"action\":\"startup\",\"default_vectorizer_module\":\"none\",\"level\":\"info\",\"msg\":\"the default vectorizer modules is set to \\\"none\\\", as a result all new schema classes without an explicit vectorizer setting, will use this vectorizer\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"action\":\"startup\",\"auto_schema_enabled\":true,\"level\":\"info\",\"msg\":\"auto schema enabled setting is set to \\\"true\\\"\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"No resource limits set, weaviate will use all available memory and CPU. To limit resources, set LIMIT_RESOURCES=true\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"module offload-s3 is enabled\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"warning\",\"msg\":\"Multiple vector spaces are present, GraphQL Explore and REST API list objects endpoint module include params has been disabled as a result.\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"open cluster service\",\"servers\":{\"Embedded_at_8079\":34201},\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"address\":\"192.168.178.36:34202\",\"level\":\"info\",\"msg\":\"starting cloud rpc server ...\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"starting raft sub-system ...\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"address\":\"192.168.178.36:34201\",\"level\":\"info\",\"msg\":\"tcp transport\",\"tcpMaxPool\":3,\"tcpTimeout\":10000000000,\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"loading local db\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"local DB successfully loaded\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"schema manager loaded\",\"n\":0,\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"level\":\"info\",\"metadata_only_voters\":false,\"msg\":\"construct a new raft node\",\"name\":\"Embedded_at_8079\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"action\":\"raft\",\"index\":6,\"level\":\"info\",\"msg\":\"raft initial configuration\",\"servers\":\"[[{Suffrage:Voter ID:Embedded_at_8079 Address:192.168.178.36:46519}]]\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"last_snapshot_index\":0,\"last_store_applied_index\":0,\"last_store_log_applied_index\":14,\"level\":\"info\",\"msg\":\"raft node constructed\",\"raft_applied_index\":0,\"raft_last_index\":14,\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"action\":\"raft\",\"follower\":{},\"leader-address\":\"\",\"leader-id\":\"\",\"level\":\"info\",\"msg\":\"raft entering follower state\",\"time\":\"2024-08-18T15:55:54+02:00\"}\n",
      "{\"action\":\"read_disk_use\",\"level\":\"warning\",\"msg\":\"disk usage currently at 93.21%, threshold set to 80.00%\",\"path\":\"/home/tim/.local/share/weaviate\",\"time\":\"2024-08-18T15:55:55+02:00\"}\n",
      "{\"action\":\"set_shard_read_only\",\"level\":\"warning\",\"msg\":\"Set READONLY, disk usage currently at 93.21%, threshold set to 90.00%\",\"path\":\"/home/tim/.local/share/weaviate\",\"time\":\"2024-08-18T15:55:55+02:00\"}\n",
      "{\"action\":\"bootstrap\",\"error\":\"could not join a cluster from [192.168.178.36:34201]\",\"level\":\"warning\",\"msg\":\"failed to join cluster, will notify next if voter\",\"servers\":[\"192.168.178.36:34201\"],\"time\":\"2024-08-18T15:55:55+02:00\",\"voter\":true}\n",
      "{\"action\":\"bootstrap\",\"candidates\":[{\"Suffrage\":0,\"ID\":\"Embedded_at_8079\",\"Address\":\"192.168.178.36:34201\"}],\"level\":\"info\",\"msg\":\"starting cluster bootstrapping\",\"time\":\"2024-08-18T15:55:55+02:00\"}\n",
      "{\"action\":\"bootstrap\",\"error\":\"bootstrap only works on new clusters\",\"level\":\"error\",\"msg\":\"could not bootstrapping cluster\",\"time\":\"2024-08-18T15:55:55+02:00\"}\n",
      "{\"action\":\"bootstrap\",\"level\":\"info\",\"msg\":\"notified peers this node is ready to join as voter\",\"servers\":[\"192.168.178.36:34201\"],\"time\":\"2024-08-18T15:55:55+02:00\"}\n",
      "{\"action\":\"raft\",\"last-leader-addr\":\"\",\"last-leader-id\":\"\",\"level\":\"warning\",\"msg\":\"raft heartbeat timeout reached, starting election\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"action\":\"raft\",\"level\":\"info\",\"msg\":\"raft entering candidate state\",\"node\":{},\"term\":6,\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"action\":\"raft\",\"level\":\"info\",\"msg\":\"raft election won\",\"tally\":1,\"term\":6,\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"action\":\"raft\",\"leader\":{},\"level\":\"info\",\"msg\":\"raft entering leader state\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"level\":\"info\",\"msg\":\"reload local db: update schema ...\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"index\":\"MDV\",\"level\":\"info\",\"msg\":\"reload local index\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"docker_image_tag\":\"unknown\",\"level\":\"info\",\"msg\":\"configured versions\",\"server_version\":\"1.26.1\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"action\":\"grpc_startup\",\"level\":\"info\",\"msg\":\"grpc server listening at [::]:50050\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"address\":\"192.168.178.36:34201\",\"level\":\"info\",\"msg\":\"current Leader\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"action\":\"restapi_management\",\"docker_image_tag\":\"unknown\",\"level\":\"info\",\"msg\":\"Serving weaviate at http://127.0.0.1:8079\",\"time\":\"2024-08-18T15:55:56+02:00\"}\n",
      "{\"action\":\"telemetry_push\",\"level\":\"info\",\"msg\":\"telemetry started\",\"payload\":\"\\u0026{MachineID:28b9a1b2-86c9-4c9f-b8c8-c651496e731a Type:INIT Version:1.26.1 NumObjects:0 OS:linux Arch:amd64 UsedModules:[]}\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"bootstrap\",\"level\":\"info\",\"msg\":\"node reporting ready, node has probably recovered cluster from raft config. Exiting bootstrap process\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/objects/segment-1723989004482136539\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_identifier/segment-1723988999765388109\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_identifier_searchable/segment-1723989000581954500\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_link/segment-1723989000978772120\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_link_searchable/segment-1723989001580506805\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_title/segment-1723989002240982726\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_title_searchable/segment-1723989002352460347\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_title_lemma/segment-1723989002566516944\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_title_lemma_searchable/segment-1723989002655854680\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_description/segment-1723989002718297975\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_description_searchable/segment-1723989002933860419\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_description_lemma/segment-1723989003165240999\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_description_lemma_searchable/segment-1723989003297186725\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_keyword/segment-1723989003532438022\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_keyword_searchable/segment-1723989003600202422\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_distribution/segment-1723989003745406572\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_distribution_searchable/segment-1723989003817368989\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_distribution_lemma/segment-1723989004196982855\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_distribution_lemma_searchable/segment-1723989004252962274\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property_is_study/segment-1723989004427551250\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n",
      "{\"action\":\"lsm_recover_from_active_wal\",\"class\":\"MDV\",\"index\":\"mdv\",\"level\":\"warning\",\"msg\":\"empty write-ahead-log found. Did weaviate crash prior to this or the tenant on/loaded from the cloud? Nothing to recover from this file.\",\"path\":\"/home/tim/.local/share/weaviate/mdv/ctHqTXnNa0cS/lsm/property__id/segment-1723989004441618812\",\"shard\":\"ctHqTXnNa0cS\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n"
     ]
    }
   ],
   "source": [
    "# This will either connect to an existing Weaviate instance or create a new one.\n",
    "# We only can execute this once, as is. If we want to run it again, we need to restart the kernel.\n",
    "# More info regarding the Embedded Weaviate client:\n",
    "# https://weaviate.io/developers/weaviate/installation/embedded\n",
    "# By default the index data is stored in ~/.local/share/weaviate/\n",
    "\n",
    "client = weaviate.connect_to_embedded()\n",
    "\n",
    "# # Use this line if another application on your machine is already using Weaviate on the default port.\n",
    "# # This notebook also counts as another application. If your notebook is running, you need to use a different port for the Streamlit app. Otherwise, you get an error.\n",
    "# client = weaviate.connect_to_local(port=8079, grpc_port=50050)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"action\":\"hnsw_prefill_cache_async\",\"level\":\"info\",\"msg\":\"not waiting for vector cache prefill, running in background\",\"time\":\"2024-08-18T15:55:57+02:00\",\"wait_for_cache_prefill\":false}\n",
      "{\"level\":\"info\",\"msg\":\"Completed loading shard mdv_ctHqTXnNa0cS in 68.838086ms\",\"time\":\"2024-08-18T15:55:57+02:00\"}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'hostname': 'http://127.0.0.1:8079',\n",
       " 'modules': {'generative-openai': {'documentationHref': 'https://platform.openai.com/docs/api-reference/completions',\n",
       "   'name': 'Generative Search - OpenAI'},\n",
       "  'qna-openai': {'documentationHref': 'https://platform.openai.com/docs/api-reference/completions',\n",
       "   'name': 'OpenAI Question & Answering Module'},\n",
       "  'ref2vec-centroid': {},\n",
       "  'reranker-cohere': {'documentationHref': 'https://txt.cohere.com/rerank/',\n",
       "   'name': 'Reranker - Cohere'},\n",
       "  'text2vec-cohere': {'documentationHref': 'https://docs.cohere.ai/embedding-wiki/',\n",
       "   'name': 'Cohere Module'},\n",
       "  'text2vec-huggingface': {'documentationHref': 'https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task',\n",
       "   'name': 'Hugging Face Module'},\n",
       "  'text2vec-openai': {'documentationHref': 'https://platform.openai.com/docs/guides/embeddings/what-are-embeddings',\n",
       "   'name': 'OpenAI Module'}},\n",
       " 'version': '1.26.1'}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# Get the meta endpoint description of Weaviate\n",
    "# This returns information about the Weaviate instance, including:\n",
    "# - hostname\n",
    "# - version\n",
    "# - available modules (e.g., OpenAI, Cohere, Hugging Face integrations)\n",
    "display(client.get_meta())\n",
    "\n",
    "# Check if Weaviate is live (basic health check)\n",
    "# Returns True if the instance is up and running\n",
    "print(client.is_live())\n",
    "\n",
    "# Check if Weaviate is ready to handle requests\n",
    "# Returns True if the instance is fully operational and ready to process queries\n",
    "print(client.is_ready())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"action\":\"hnsw_vector_cache_prefill\",\"count\":7000,\"index_id\":\"main\",\"level\":\"info\",\"limit\":1000000000000,\"msg\":\"prefilled vector cache\",\"time\":\"2024-08-18T15:55:57+02:00\",\"took\":323516344}\n"
     ]
    }
   ],
   "source": [
    "# Resetting or cleaning up: Delete collection.\n",
    "client.collections.delete(\"MDV\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<weaviate.collections.collection.sync.Collection at 0x759a276d2ec0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First we need to create a schema that defines how data is stored, organized and retrieved in Weaviate.\n",
    "# A schema is called a \"collection\". We can define as many collections as we want.\n",
    "# Weaviate Collections:\n",
    "# - Fundamental data organization unit, similar to database tables\n",
    "# - Group related objects with a shared schema\n",
    "# - Support vector representations for semantic search\n",
    "# - Defined by:\n",
    "#   * Properties (fields) with data types and index settings\n",
    "#   * Vector index configuration (e.g., HNSW)\n",
    "#   * Sharding and replication settings\n",
    "# - Enable both traditional filtering and vector-based operations\n",
    "# - Configurable for various indexing and search optimizations\n",
    "# Example: 'MDV' collection with properties like 'identifier', 'title', 'description'\n",
    "\n",
    "client.collections.create(\n",
    "    \"MDV\",\n",
    "    vectorizer_config=wc.Configure.Vectorizer.none(),\n",
    "    inverted_index_config=wvc.config.Configure.inverted_index(\n",
    "        bm25_b=0.75,  # default 0.75\n",
    "        bm25_k1=1.2,  # default 1.2\n",
    "        stopwords_additions=None,\n",
    "        stopwords_preset=None,\n",
    "        stopwords_removals=None,\n",
    "    ),\n",
    "    properties=[\n",
    "        Property(name=\"identifier\", data_type=DataType.TEXT),\n",
    "        Property(name=\"link\", data_type=DataType.TEXT),\n",
    "        Property(name=\"title\", data_type=DataType.TEXT),\n",
    "        Property(name=\"title_lemma\", data_type=DataType.TEXT),\n",
    "        Property(name=\"description\", data_type=DataType.TEXT),\n",
    "        Property(name=\"description_lemma\", data_type=DataType.TEXT),\n",
    "        Property(name=\"keyword\", data_type=DataType.TEXT),\n",
    "        Property(name=\"distribution\", data_type=DataType.TEXT),\n",
    "        Property(name=\"distribution_lemma\", data_type=DataType.TEXT),\n",
    "        Property(name=\"is_study\", data_type=DataType.BOOL),\n",
    "    ],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "MDV\n",
    "{\n",
    "    'MDV': *CollectionConfig(\n",
    "        name='MDV',\n",
    "        description=None,\n",
    "        generative_config=None,\n",
    "        inverted_index_config=_InvertedIndexConfig(\n",
    "            bm25=_BM25Config(b=0.75, k1=1.2),\n",
    "            cleanup_interval_seconds=60,\n",
    "            index_null_state=False,\n",
    "            index_property_length=False,\n",
    "            index_timestamps=False,\n",
    "            stopwords=_StopwordsConfig(\n",
    "                preset=<StopwordsPreset.EN: 'en'>,\n",
    "                additions=None,\n",
    "                removals=None\n",
    "            )\n",
    "        ),\n",
    "        multi_tenancy_config=_MultiTenancyConfig(\n",
    "            enabled=False,\n",
    "            auto_tenant_creation=False,\n",
    "            auto_tenant_activation=False\n",
    "        ),\n",
    "        properties=[\n",
    "            _Property(\n",
    "                name='identifier',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='link',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='title',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='title_lemma',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='description',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='description_lemma',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='keyword',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='distribution',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='distribution_lemma',\n",
    "                description=None,\n",
    "                data_type=<DataType.TEXT: 'text'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=True,\n",
    "                nested_properties=None,\n",
    "                tokenization=<Tokenization.WORD: 'word'>,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            ),\n",
    "            *Property(\n",
    "                name='is_study',\n",
    "                description=None,\n",
    "                data_type=<DataType.BOOL: 'boolean'>,\n",
    "                index_filterable=True,\n",
    "                index_range_filters=False,\n",
    "                index_searchable=False,\n",
    "                nested_properties=None,\n",
    "                tokenization=None,\n",
    "                vectorizer_config=None,\n",
    "                vectorizer='none'\n",
    "            )\n",
    "        ],\n",
    "        references=[],\n",
    "        replication_config=_ReplicationConfig(\n",
    "            factor=1,\n",
    "            async_enabled=False\n",
    "        ),\n",
    "        reranker_config=None,\n",
    "        sharding_config=_ShardingConfig(\n",
    "            virtual_per_physical=128,\n",
    "            desired_count=1,\n",
    "            actual_count=1,\n",
    "            desired_virtual_count=128,\n",
    "            actual_virtual_count=128,\n",
    "            key='_id',\n",
    "            strategy='hash',\n",
    "            function='murmur3'\n",
    "        ),\n",
    "        vector_index_config=_VectorIndexConfigHNSW(\n",
    "            quantizer=None,\n",
    "            cleanup_interval_seconds=300,\n",
    "            distance_metric=<VectorDistances.COSINE: 'cosine'>,\n",
    "            dynamic_ef_min=100,\n",
    "            dynamic_ef_max=500,\n",
    "            dynamic_ef_factor=8,\n",
    "            ef=-1,\n",
    "            ef_construction=128,\n",
    "            flat_search_cutoff=40000,\n",
    "            max_connections=32,\n",
    "            skip=False,\n",
    "            vector_cache_max_objects=1000000000000\n",
    "        ),\n",
    "        vector_index_type=<VectorIndexType.HNSW: 'hnsw'>,\n",
    "        vectorizer_config=None,\n",
    "        vectorizer=<Vectorizers.NONE: 'none'>,\n",
    "        vector_config=None\n",
    "    )\n",
    "}\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MDV\n",
      "{'MDV': _CollectionConfig(name='MDV', description=None, generative_config=None, inverted_index_config=_InvertedIndexConfig(bm25=_BM25Config(b=0.75, k1=1.2), cleanup_interval_seconds=60, index_null_state=False, index_property_length=False, index_timestamps=False, stopwords=_StopwordsConfig(preset=<StopwordsPreset.EN: 'en'>, additions=None, removals=None)), multi_tenancy_config=_MultiTenancyConfig(enabled=False, auto_tenant_creation=False, auto_tenant_activation=False), properties=[_Property(name='identifier', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='link', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='title', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='title_lemma', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='description', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='description_lemma', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='keyword', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='distribution', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='distribution_lemma', description=None, data_type=<DataType.TEXT: 'text'>, index_filterable=True, index_range_filters=False, index_searchable=True, nested_properties=None, tokenization=<Tokenization.WORD: 'word'>, vectorizer_config=None, vectorizer='none'), _Property(name='is_study', description=None, data_type=<DataType.BOOL: 'boolean'>, index_filterable=True, index_range_filters=False, index_searchable=False, nested_properties=None, tokenization=None, vectorizer_config=None, vectorizer='none')], references=[], replication_config=_ReplicationConfig(factor=1, async_enabled=False), reranker_config=None, sharding_config=_ShardingConfig(virtual_per_physical=128, desired_count=1, actual_count=1, desired_virtual_count=128, actual_virtual_count=128, key='_id', strategy='hash', function='murmur3'), vector_index_config=_VectorIndexConfigHNSW(quantizer=None, cleanup_interval_seconds=300, distance_metric=<VectorDistances.COSINE: 'cosine'>, dynamic_ef_min=100, dynamic_ef_max=500, dynamic_ef_factor=8, ef=-1, ef_construction=128, flat_search_cutoff=40000, max_connections=32, skip=False, vector_cache_max_objects=1000000000000), vector_index_type=<VectorIndexType.HNSW: 'hnsw'>, vectorizer_config=None, vectorizer=<Vectorizers.NONE: 'none'>, vector_config=None)}\n"
     ]
    }
   ],
   "source": [
    "# Collection:\n",
    "# In this case, 'MDV' is the name of a single collection. A collection in Weaviate is similar to a table in a traditional database or a collection in\n",
    "# document-oriented databases. It's a group of related objects that share the same schema.\n",
    "\n",
    "# List all collections.\n",
    "for v in client.collections.list_all().values():\n",
    "    print(v.name)\n",
    "\n",
    "# Get detailed information about all collections.\n",
    "schema = client.collections.list_all(simple=False)\n",
    "print(schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"level\":\"warning\",\"msg\":\"prop len tracker file /home/tim/.local/share/weaviate/mdv/rcw3bLauOVXL/proplengths does not exist, creating new tracker\",\"time\":\"2024-08-18T15:55:58+02:00\"}\n",
      "{\"action\":\"hnsw_prefill_cache_async\",\"level\":\"info\",\"msg\":\"not waiting for vector cache prefill, running in background\",\"time\":\"2024-08-18T15:55:58+02:00\",\"wait_for_cache_prefill\":false}\n",
      "{\"level\":\"info\",\"msg\":\"Created shard mdv_rcw3bLauOVXL in 13.907936ms\",\"time\":\"2024-08-18T15:55:58+02:00\"}\n",
      "{\"action\":\"hnsw_vector_cache_prefill\",\"count\":1000,\"index_id\":\"main\",\"level\":\"info\",\"limit\":1000000000000,\"msg\":\"prefilled vector cache\",\"time\":\"2024-08-18T15:55:58+02:00\",\"took\":268696}\n"
     ]
    }
   ],
   "source": [
    "# Now we ingest data into the collection in the form of \"items\".\n",
    "\n",
    "collection = client.collections.get(\"MDV\")\n",
    "\n",
    "with collection.batch.dynamic() as batch:\n",
    "    for data in df.to_dict(orient=\"records\"):\n",
    "        properties = {\n",
    "            \"identifier\": data[\"identifier\"],\n",
    "            \"link\": data[\"link\"],\n",
    "            \"title\": data[\"title\"],\n",
    "            \"title_lemma\": data[\"title_lemma\"],\n",
    "            \"description\": data[\"description\"],\n",
    "            \"description_lemma\": data[\"description_lemma\"],\n",
    "            \"keyword\": data[\"keyword\"],\n",
    "            \"distribution\": data[\"distribution\"],\n",
    "            \"distribution_lemma\": data[\"distribution_lemma\"],\n",
    "            \"is_study\": data[\"is_study\"],\n",
    "        }\n",
    "        batch.add_object(properties=properties, vector=data[\"embedding_openai\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Object(uuid=_WeaviateUUIDInt('0001dc52-93dc-4144-8fb7-92e898a3e1f2'), metadata=MetadataReturn(creation_time=None, last_update_time=None, distance=None, certainty=None, score=None, explain_score=None, is_consistent=None, rerank_score=None), properties={'description': \"Sachdaten des Gesundheits- und Sozialindex (GESIx) des Gesundheits- und Sozialstrukturatlas Berlin 2022 auf der Ebene der Planungsräume. Datengrundlage bilden 20 Indikatoren, die überwiegend auf amtlichen Statistiken beruhen, aus den Dimensionen Erwerbsleben, soziale Lage und Gesundheit. Für diese drei Dimensionen werden Subindizes berechnet, die zu einem Index - GESIx - zusammengeführt werden.\\n\\n![Vorschaugrafik zu Datensatz 'Gesundheits- und Sozialstrukturatlas: Gesundheits- und Sozialindex 2022 (GESIx)'](https://fbinter.stadt-berlin.de/fb_daten/vorschau/sachdaten/svor_default.gif)\", 'identifier': '93c2cf18-77ad-40ff-b83a-c7ca30f4e129', 'is_study': False, 'link': 'https://datenregister.berlin.de/api/3/action/current_package_list_with_resources93c2cf18-77ad-40ff-b83a-c7ca30f4e129', 'distribution': 'Endpunkt-Beschreibung des WFS-Service Maschinenlesbare Endpunkt-Beschreibung des WFS-Service. Weitere Informationen unter https://www.ogc.org/standards/wfs API-Endpunkt des WFS-Service API-Endpunkt des WFS-Service. Weitere Informationen unter https://www.ogc.org/standards/wfs Serviceseite im FIS-Broker Aufruf des Geoportals Berlin mit Darstellung der downloadbaren Daten als Karte(nlayer) Inhaltliche Beschreibung Inhaltliche Beschreibung Technische Beschreibung Technische Beschreibung', 'title': 'Gesundheits- und Sozialstrukturatlas: Gesundheits- und Sozialindex 2022 (GESIx) - [WFS]', 'keyword': 'Alleinerziehende Arbeitslosigkeit Armut Behandlungsfälle Berlin Beruflicher Ausbildungsabschluss Einkommen Erwerbsleben Erwerbsstatus Geodaten Gesundheit Gesundheitsatlas Gesundheitsberichterstattung Grundsicherung Index Kiez Lebenserwartung Lebensweltlich orientierte Räume Morbidität Mortalität Planung Regionalvergleich Sachdaten Soziale Lage Sozialraum Sozialstruktur Trend Ungleichheit soziales', 'distribution_lemma': 'Endpunkt Beschreibung der WFS Service Maschinenlesbare Endpunkt Beschreibung der WFS Service . weit Information unter https www.ogc.org standards wfs API Endpunkt der WFS Service API Endpunkt der WFS Service . weit Information unter https www.ogc.org standards wfs Serviceseite in FIS Broker Aufruf der Geoportal Berlin mit Darstellung der downloadbar Datum als Karte Nlayer inhaltlich Beschreibung inhaltlich Beschreibung technisch Beschreibung technisch Beschreibung', 'description_lemma': 'Sachdat der Gesundheits und Sozialindex GESIx der Gesundheits und Sozialstrukturatla Berlin 2022 auf der Ebene der Planungsraum . Datengrundlage bilden 20 Indikator der überwiegend auf amtlich Statistik beruhen aus der Dimension Erwerbslebe sozial Lage und Gesundheit . für dieser drei Dimension werden Subindiz berechnen der zu ein Index GESIx zusammenführen werden . Vorschaugrafik zu Datensatz Gesundheits und Sozialstrukturatla Gesundheits und Sozialindex 2022 GESIx https fbinter.stadt berlin.de fb daten vorschau sachdat svor default.gif ', 'title_lemma': 'Gesundheits und Sozialstrukturatla Gesundheits und Sozialindex 2022 GESIx WFS '}, references=None, vector={}, collection='MDV')\n"
     ]
    }
   ],
   "source": [
    "# List all items in the collection.\n",
    "collection = client.collections.get(\"MDV\")\n",
    "for item in collection.iterator():\n",
    "    print(item)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The collection contains 3311 items.\n"
     ]
    }
   ],
   "source": [
    "# Get total count of all items in the collection.\n",
    "collection = client.collections.get(\"MDV\")\n",
    "response = collection.aggregate.over_all(total_count=True)\n",
    "\n",
    "# Check if the total count of items in the collection is equal to the number of items in the DataFrame.\n",
    "# If the assertion fails, try to re-run the previous cells.\n",
    "assert response.total_count == len(df)\n",
    "\n",
    "print(f\"The collection contains {response.total_count} items.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lexical search (using BM25)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Weaviate documentation for lexical search](https://weaviate.io/developers/weaviate/search/bm25)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "collection = client.collections.get(\"MDV\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set fields to search in with BM25.\n",
    "# We exclude the identifier and link fields.\n",
    "# We give twice the weight to the title fields.\n",
    "\n",
    "query_properties = [\n",
    "    \"title^2\",\n",
    "    \"description\",\n",
    "    \"title_lemma^2\",\n",
    "    \"description_lemma\",\n",
    "    \"keyword\",\n",
    "    \"distribution\",\n",
    "    \"distribution_lemma\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nlp = spacy.load(\"de_core_news_lg\")\n",
    "nlp = spacy.load(\"de_core_news_sm\", disable=[\"ner\", \"parser\"])\n",
    "\n",
    "LETTERS_AND_DIGITS = re.compile(r\"[^a-zäüöA-ZÜÄÖ0-9.]\")\n",
    "MULTIPLE_SPACES = re.compile(r\"\\s+\")\n",
    "\n",
    "\n",
    "def prepare_for_lexical_search(text, lower=False, remove_umlauts=False):\n",
    "    \"\"\"Lemmatize text, and optionally lower case and remove umlauts for lexical search.\n",
    "\n",
    "    Args:\n",
    "        text (str): Text to process.\n",
    "\n",
    "    Keyword Arguments:\n",
    "        lower (bool): Lower case text (default: {True}).\n",
    "        remove_umlauts (bool): Remove umlauts from text (default: {True}).\n",
    "\n",
    "    Returns:\n",
    "        str: Lemmatized text, optionally lower cased, and without umlauts.\n",
    "    \"\"\"\n",
    "    doc = nlp(text)\n",
    "    text = \" \".join([token.lemma_ if token.is_alpha else token.text for token in doc])\n",
    "    text = re.sub(LETTERS_AND_DIGITS, \" \", text)\n",
    "    text = re.sub(MULTIPLE_SPACES, \" \", text)\n",
    "    if lower:\n",
    "        text = text.lower()\n",
    "    if remove_umlauts:\n",
    "        text = text.replace(\"ä\", \"ae\").replace(\"ö\", \"oe\").replace(\"ü\", \"ue\")\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Lohn in Berlin'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"Löhne in Berlin\"\n",
    "query = prepare_for_lexical_search(query)\n",
    "query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Laufende Steuereinnahmen des Landes Berlin\n",
      "Laufende Steuereinnahmen des Landes Berlin\n",
      "Breitbandversorgung Berlin 2017-2018 Privatverfügbarkeiten\n",
      "Senatsvorlagen der Senatsverwaltung für Finanzen\n",
      "Differenzbilanzierung 2016 (Berlin)\n",
      "Differenzbilanzierung 2012 (Berlin)\n",
      "Kitas in Berlin\n",
      "Standardlastprofil Speicherheizung (Berlin)\n",
      "Differenzbilanzierung 2013 (Berlin)\n",
      "Differenzbilanzierung 2011 (Berlin)\n",
      "Differenzbilanzierung 2015 (Berlin)\n",
      "Differenzbilanzierung 2014 (Berlin)\n",
      "Radzähldaten in Berlin\n",
      "Los_9_2024 (Berlin)\n",
      "Standardlastprofil Bandlastkunden 2024 (Berlin)\n",
      "Los_6_2024 (Berlin)\n",
      "Los_7_2024 (Berlin)\n",
      "Los_3_2025 (Berlin)\n",
      "Los_4_2024 (Berlin)\n",
      "Los_2_2024 (Berlin)\n"
     ]
    }
   ],
   "source": [
    "# See documentation for more details about parameters:\n",
    "# https://weaviate.io/developers/weaviate/search/bm25\n",
    "response = collection.query.bm25(\n",
    "    query=query,\n",
    "    query_properties=query_properties,\n",
    "    offset=0,\n",
    "    limit=20,\n",
    "    # auto_limit=10,\n",
    "    return_metadata=wvc.query.MetadataQuery(\n",
    "        score=True,\n",
    "        explain_score=True,\n",
    "    ),\n",
    ")\n",
    "for item in response.objects:\n",
    "    print(\n",
    "        item.properties[\"title\"],\n",
    "        # item.properties[\"description\"],\n",
    "        # item.metadata.score,\n",
    "        # item.metadata.explain_score,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Semantic search\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Weaviate documentation for semantic search](https://weaviate.io/developers/weaviate/search/similarity)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. Title: Summenlast der Netzverluste 2009 (Berlin)\n",
      "   Distance: 0.9334, Certainty: 0.5333\n",
      "\n",
      "2. Title: Jahresabschluss 2015\n",
      "   Distance: 0.9355, Certainty: 0.5322\n",
      "\n",
      "3. Title: Summenlast der Netzverluste 2015 (Berlin)\n",
      "   Distance: 0.9394, Certainty: 0.5303\n",
      "\n",
      "4. Title: Bezirkshaushaltsplan Friedrichshain-Kreuzberg 2012/2013 fortgeschrieben mit Ergänzungsplan 2013\n",
      "   Distance: 0.9397, Certainty: 0.5302\n",
      "\n",
      "5. Title: Preisblatt Anschluss Niederspannung Berlin\n",
      "   Distance: 0.9427, Certainty: 0.5287\n",
      "\n",
      "6. Title: Reparaturführer Charlottenburg-Wilmersdorf\n",
      "   Distance: 0.9430, Certainty: 0.5285\n",
      "\n",
      "7. Title: Langjährige Entwicklung Luftqualität PM10-Emissionen Kfz-Verkehr NN 2009 (Umweltatlas) - [WMS]\n",
      "   Distance: 0.9432, Certainty: 0.5284\n",
      "\n",
      "8. Title: Finanzamt-Suche\n",
      "   Distance: 0.9433, Certainty: 0.5284\n",
      "\n",
      "9. Title: Langjährige Entwicklung Luftqualität PM10-Emissionen Kfz-Verkehr GN 2009 (Umweltatlas) - [WMS]\n",
      "   Distance: 0.9439, Certainty: 0.5281\n",
      "\n",
      "10. Title: Energetischer Sanierungsfahrplan\n",
      "   Distance: 0.9450, Certainty: 0.5275\n",
      "\n",
      "11. Title: Haushaltsplan des Bezirksamtes Tempelhof-Schöneberg von Berlin für 2014/2015\n",
      "   Distance: 0.9451, Certainty: 0.5274\n",
      "\n",
      "12. Title: Summenlast der Netzverluste 2010 (Berlin)\n",
      "   Distance: 0.9453, Certainty: 0.5273\n",
      "\n",
      "13. Title: Summenlast der Netzverluste 2007 (Berlin)\n",
      "   Distance: 0.9454, Certainty: 0.5273\n",
      "\n",
      "14. Title: Summenlast der Netzverluste 2006 (Berlin)\n",
      "   Distance: 0.9454, Certainty: 0.5273\n",
      "\n",
      "15. Title: Jahresabschluss 2012\n",
      "   Distance: 0.9458, Certainty: 0.5271\n",
      "\n",
      "16. Title: Jahresabschluss 2014\n",
      "   Distance: 0.9468, Certainty: 0.5266\n",
      "\n",
      "17. Title: Summenlast der Netzverluste 2008 (Berlin)\n",
      "   Distance: 0.9479, Certainty: 0.5260\n",
      "\n",
      "18. Title: Langjährige Entwicklung Luftqualität PM10-Emissionen Kfz-Verkehr HN 2009 (Umweltatlas) - [WMS]\n",
      "   Distance: 0.9481, Certainty: 0.5260\n",
      "\n",
      "19. Title: Bezirkshaushaltsplan Friedrichshain-Kreuzberg 2014/2015\n",
      "   Distance: 0.9482, Certainty: 0.5259\n",
      "\n",
      "20. Title: Gesundheitsberichterstattung Berlin: Kosten -> Aufwendungen für Rehabilitation und Pflege\n",
      "   Distance: 0.9489, Certainty: 0.5255\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from typing import List\n",
    "from openai import OpenAI\n",
    "import weaviate\n",
    "import os\n",
    "\n",
    "# Initialize the OpenAI client\n",
    "openai_client = OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n",
    "\n",
    "\n",
    "def embed_with_openai(texts: List[str]) -> List[List[float]]:\n",
    "    \"\"\"Generate embeddings for given texts using OpenAI's API.\"\"\"\n",
    "    response = openai_client.embeddings.create(\n",
    "        input=texts, model=\"text-embedding-ada-002\"\n",
    "    )\n",
    "    return [embedding.embedding for embedding in response.data]\n",
    "\n",
    "\n",
    "def search_similar_documents(\n",
    "    query: str, client: weaviate.Client, collection_name: str, n_results: int = 20\n",
    ") -> List[dict]:\n",
    "    \"\"\"\n",
    "    Search for documents similar to the query using Weaviate's near_vector search.\n",
    "\n",
    "    Args:\n",
    "    query (str): The search query.\n",
    "    client (weaviate.Client): Weaviate client instance.\n",
    "    collection_name (str): Name of the Weaviate collection to search.\n",
    "    n_results (int): Number of results to return.\n",
    "\n",
    "    Returns:\n",
    "    List[dict]: List of dictionaries containing title, distance, and certainty for each result.\n",
    "    \"\"\"\n",
    "    # Generate embedding for the query\n",
    "    query_embedding = embed_with_openai([query])[0]\n",
    "\n",
    "    # Perform near_vector search\n",
    "    response = (\n",
    "        client.query.get(collection_name, [\"title\"])\n",
    "        .with_near_vector({\"vector\": query_embedding})\n",
    "        .with_limit(n_results)\n",
    "        .with_additional([\"distance\", \"certainty\"])\n",
    "        .do()\n",
    "    )\n",
    "\n",
    "    # Extract and return results\n",
    "    results = []\n",
    "    for item in response[\"data\"][\"Get\"][collection_name]:\n",
    "        results.append(\n",
    "            {\n",
    "                \"title\": item[\"title\"],\n",
    "                \"distance\": item[\"_additional\"][\"distance\"],\n",
    "                \"certainty\": item[\"_additional\"][\"certainty\"],\n",
    "            }\n",
    "        )\n",
    "\n",
    "    return results\n",
    "\n",
    "\n",
    "# Usage\n",
    "def main():\n",
    "    client = weaviate.Client(\n",
    "        \"http://127.0.0.1:8079\"\n",
    "    )  # Correct address as per your specification\n",
    "    query = \"fahrräder in berlin\"\n",
    "    show_n_results = 20\n",
    "\n",
    "    similar_documents = search_similar_documents(query, client, \"MDV\", show_n_results)\n",
    "\n",
    "    # Display results\n",
    "    for i, doc in enumerate(similar_documents, 1):\n",
    "        print(f\"{i}. Title: {doc['title']}\")\n",
    "        print(f\"   Distance: {doc['distance']:.4f}, Certainty: {doc['certainty']:.4f}\")\n",
    "        print()\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reparaturführer Charlottenburg-Wilmersdorf 0.93730605 0.5313469767570496\n",
      "Summenlast der Netzverluste 2009 (Berlin) 0.9450978 0.5274510979652405\n",
      "Jahresabschluss 2015 0.9493318 0.5253340899944305\n",
      "Energetischer Sanierungsfahrplan 0.9515328 0.5242336094379425\n",
      "Schulbausanierung Sommer 2022 0.9515732 0.5242134034633636\n",
      "Summenlast der Netzverluste 2015 (Berlin) 0.951823 0.5240885019302368\n",
      "Bezirkshaushaltsplan Friedrichshain-Kreuzberg 2012/2013 fortgeschrieben mit Ergänzungsplan 2013 0.95460397 0.5226980149745941\n",
      "Summenlast der Netzverluste 2010 (Berlin) 0.95591736 0.5220413208007812\n",
      "Summenlast der Netzverluste 2006 (Berlin) 0.9560822 0.5219588875770569\n",
      "Summenlast der Netzverluste 2007 (Berlin) 0.9565257 0.5217371582984924\n",
      "Jahresabschluss 2014 0.9570526 0.5214737057685852\n",
      "14 Radrouten und Radverkehrsanlagen - GPS-Tracks für die Radrouten durch Berlin 0.95749694 0.5212515294551849\n",
      "Finanzamt-Suche 0.95777977 0.5211101174354553\n",
      "Summenlast der Netzverluste 2008 (Berlin) 0.9581924 0.5209037959575653\n",
      "Energetischer Sanierungsfahrplan 0.95836854 0.5208157300949097\n",
      "Radzähldaten in Berlin 0.9590482 0.5204758942127228\n",
      "Summenlast der Netzverluste 2012 (Berlin) 0.9592041 0.520397961139679\n",
      "Daten Ausfall Halbring für Energyhack 2015 0.95975274 0.5201236307621002\n",
      "Breitband-Ausbau der Berliner Schulen 0.9600422 0.5199789106845856\n",
      "Jahresabschluss 2012 0.9605257 0.5197371542453766\n"
     ]
    }
   ],
   "source": [
    "# Usage\n",
    "def main():\n",
    "    client = weaviate.Client(\"http://127.0.0.1:8079\")\n",
    "    query = \"Straßen in Berlin\"\n",
    "    collection_name = \"MDV\"\n",
    "    n_results = 20\n",
    "\n",
    "    results = search_similar_documents(query, client, collection_name, n_results)\n",
    "\n",
    "    # Display results\n",
    "    for result in results:\n",
    "        print(result[\"title\"], result[\"distance\"], result[\"certainty\"])\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hybrid search\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Weviate documentation for hybrid search - the combination of lexical and vector search](https://weaviate.io/developers/weaviate/search/hybrid)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gesundheitsberichterstattung Berlin: Kosten -> Aufwendungen für Rehabilitation und Pflege 0.7 None None\n",
      "Gesundheitsberichterstattung Berlin: Einrichtungen des Gesundheitswesens -> Stationäre/teilstationäre medizinische Einrichtungen -> Vorsorge- oder Rehabilitationseinrichtungen 0.589394 None None\n",
      "Radzähldaten in Berlin 0.54055935 None None\n",
      "Daten Ausfall Halbring für Energyhack 2015 0.5266549 None None\n",
      "Gesundheitsberichterstattung Berlin: Kosten -> Krankheitskosten 0.51893425 None None\n",
      "Förderungen / Finanzen 0.4898429 None None\n",
      "Breitband-Ausbau der Berliner Schulen 0.4680342 None None\n",
      "Gesundheitsberichterstattung Berlin: Inanspruchnahme von Leistungen der Gesundheitsförderung und der Gesundheitsversorgung -> Inanspruchnahme/Leistungen der Prävention, Gesundheitsförderung und Früherkennung von Krankheiten -> Kariesprävalenz und Kar... 0.4657743 None None\n",
      "Gesundheitsberichterstattung Berlin: Kosten -> Kostenstruktur von Krankenhäusern 0.44931677 None None\n",
      "Gesundheitsberichterstattung Berlin: Kosten -> Kostenstruktur von ambulanten Gesundheitseinrichtungen 0.43817887 None None\n",
      "Gesundheitsberichterstattung Berlin: Inanspruchnahme von Leistungen der Gesundheitsförderung und der Gesundheitsversorgung -> Inanspruchnahme/Leistungen der Prävention, Gesundheitsförderung und Früherkennung von Krankheiten -> Vorsorgeuntersuchungen 0.4231767 None None\n",
      "Listen der Standorte für die Pflanzungen der Stadtbaumkampagne 0.4098777 None None\n",
      "Gesundheitsberichterstattung Berlin: Beschäftigte im Gesundheitswesen -> Ausgewählte im ambulanten und stationären Bereich tätige Berufsgruppen -> Krankenhäuser und Vorsorge- oder Rehabilitationseinrichtungen 0.40841445 None None\n",
      "Schulbausanierung Sommer 2022 0.40141213 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Einschulungsuntersuchung Berlin 2009 0.38793433 None None\n",
      "Reparaturführer Charlottenburg-Wilmersdorf 0.36851117 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Einschulungsuntersuchung Berlin 2015 0.36152762 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Einschulungsuntersuchung Berlin 2006 0.36059448 None None\n",
      "Gesundheitsberichterstattung Berlin: Beschäftigte im Gesundheitswesen -> Ausgewählte im ambulanten und stationären Bereich tätige Berufsgruppen 0.3589023 None None\n",
      "Ansprechpartner Bau- und Wohnungsaufsicht in Friedrichshain-Kreuzberg 0.353015 None None\n",
      "Projektdatenbank der geförderten Projekte der Landesstelle für Entwicklungszusammenarbeit 0.3398587 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Einschulungsuntersuchung Berlin 2007/2008 0.3385304 None None\n",
      "Berlin zählt Mobilität 0.33764586 None None\n",
      "Summenlast der Netzverluste 2009 (Berlin) 0.32545722 None None\n",
      "Gesundheitsberichterstattung Berlin: Inanspruchnahme von Leistungen der Gesundheitsförderung und der Gesundheitsversorgung -> Inanspruchnahme/Leistungen der stationären/teilstationären Versorgung -> Inanspruchnahme und Leistungen von Vorsorge- oder R... 0.32157728 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Gesundheitsmonitoring 2015 0.30906242 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Einschulungsuntersuchung Berlin 2008 0.3088413 None None\n",
      "Gesundheitsberichterstattung Berlin : Diskussionspapiere 0.3082281 None None\n",
      "Gesundheitsberichterstattung Berlin: Inanspruchnahme von Leistungen der Gesundheitsförderung und der Gesundheitsversorgung -> Inanspruchnahme/Leistungen der ambulanten Versorgung 0.30426347 None None\n",
      "Laufende Steuereinnahmen des Landes Berlin 0.3 None None\n",
      "Gesundheitsberichterstattung Berlin: Inanspruchnahme von Leistungen der Gesundheitsförderung und der Gesundheitsversorgung -> Inanspruchnahme/Leistungen der stationären/teilstationären Versorgung -> Inanspruchnahme und Leistungen von Krankenhäusern 0.2844169 None None\n",
      "Gesundheitsberichterstattung Berlin: Einrichtungen des Gesundheitswesens -> Stationäre/teilstationäre medizinische Einrichtungen -> Krankenhäuser 0.28047895 None None\n",
      "Gesundheitsberichterstattung Berlin: Gesundheitsindikator 5.12 - Ausgewählte Arbeitsbelastungen und Umgebungsfaktoren, Deutschland, im Zeitvergleich B 0.27725145 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Basisbericht GBE Berlin 2005 0.27605012 None None\n",
      "Gesundheitsberichterstattung Berlin: Ausgaben und Finanzierung -> Gesundheitsausgaben nach Ausgabenträgern und Leistungsarten 0.2714723 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Basisbericht GBE Berlin 2009 0.25639015 None None\n",
      "Gesundheitsberichterstattung Berlin: Kontext Einschulungsuntersuchung Berlin 2010 0.25553858 None None\n",
      "Glossar zu Daten für Energyhack 2015 0.2377747 None None\n",
      "Gesundheitsberichterstattung Berlin: Gesundheitsindikator 10.9 - Gesundheitsausgaben der gesetzlichen Krankenversicherung nach Leistungsarten, Land, Jahre L 0.23380068 None None\n",
      "Gesundheitsberichterstattung Berlin: Inanspruchnahme von Leistungen der Gesundheitsförderung und der Gesundheitsversorgung -> Inanspruchnahme/Leistungen der Prävention, Gesundheitsförderung und Früherkennung von Krankheiten -> Schwangerenbetreuung, G... 0.23075666 None None\n"
     ]
    }
   ],
   "source": [
    "def search_similar_documents(\n",
    "    query: str, client: weaviate.Client, collection_name: str, n_results: int = 20\n",
    ") -> List[dict]:\n",
    "    \"\"\"\n",
    "    Search for documents similar to the query using Weaviate's hybrid search.\n",
    "\n",
    "    Args:\n",
    "    query (str): The search query.\n",
    "    client (weaviate.Client): Weaviate client instance.\n",
    "    collection_name (str): Name of the Weaviate collection to search.\n",
    "    n_results (int): Number of results to return.\n",
    "\n",
    "    Returns:\n",
    "    List[dict]: List of dictionaries containing title, score, distance, and certainty for each result.\n",
    "    \"\"\"\n",
    "    query_embedding = embed_with_openai([query])[0]\n",
    "    prepared_query = prepare_for_lexical_search(query)\n",
    "\n",
    "    response = (\n",
    "        client.query.get(collection_name, [\"title\"])\n",
    "        .with_hybrid(query=prepared_query, vector=query_embedding, alpha=0.7)\n",
    "        .with_limit(n_results)\n",
    "        .with_additional([\"score\", \"distance\", \"certainty\"])\n",
    "        .do()\n",
    "    )\n",
    "\n",
    "    results = []\n",
    "    for item in response[\"data\"][\"Get\"][collection_name]:\n",
    "        results.append(\n",
    "            {\n",
    "                \"title\": item[\"title\"],\n",
    "                \"score\": item[\"_additional\"][\"score\"],\n",
    "                \"distance\": item[\"_additional\"].get(\"distance\"),\n",
    "                \"certainty\": item[\"_additional\"].get(\"certainty\"),\n",
    "            }\n",
    "        )\n",
    "\n",
    "    return results\n",
    "\n",
    "\n",
    "# Usage\n",
    "def main():\n",
    "    client = weaviate.Client(\"http://127.0.0.1:8079\")\n",
    "    query = \"Löhne in Berlin\"\n",
    "    collection_name = \"MDV\"\n",
    "    n_results = 40\n",
    "\n",
    "    results = search_similar_documents(query, client, collection_name, n_results)\n",
    "\n",
    "    # Display results\n",
    "    for result in results:\n",
    "        print(\n",
    "            result[\"title\"],\n",
    "            result[\"score\"],\n",
    "            result.get(\"distance\", \"N/A\"),\n",
    "            result.get(\"certainty\", \"N/A\"),\n",
    "        )\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "standard",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}