document-analysis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Med-Rokaimi/LLMs-document-analysis/blob/main/document-analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pcxCYPUYAZZA"
      },
      "source": [
        "Analyzing documents LLMs models : PALM2 and Gemini Pro:\n",
        "\n",
        "\n",
        "*   Summarize each document\n",
        "*   Quering and retrival documents (Q&A)\n",
        "*   Identify the documents discussing a specific topic\n",
        "*   Q&A : Gemini Pro example\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L8WERM9GD1dw"
      },
      "source": [
        "# packages"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "metadata": {
        "id": "31-DYOvqRrAZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z17K9qBvoRDh"
      },
      "outputs": [],
      "source": [
        "!pip install -q -U google-generativeai\n",
        "!pip -q install langchain\n",
        "!pip install chromadb\n",
        "!pip install PyMuPDF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ohOlBHo1qj9R"
      },
      "outputs": [],
      "source": [
        "from langchain import PromptTemplate\n",
        "from langchain.chains.summarize import load_summarize_chain\n",
        "from pathlib import Path as p\n",
        "from langchain.embeddings import GooglePalmEmbeddings\n",
        "from langchain.llms import GooglePalm\n",
        "from langchain.document_loaders.csv_loader import CSVLoader\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "from langchain.indexes import VectorstoreIndexCreator #vectorize db index with chromadb\n",
        "from langchain.vectorstores import Chroma\n",
        "\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QujObE4T26VA"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "# configure palm\n",
        "import google.generativeai as palm\n",
        "GOOGLE_API_KEY = #put your API here. [You can use env to store the api key]\n",
        "palm.configure(api_key=GOOGLE_API_KEY)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# I prefer to load the pdf files to one csv file\n",
        "import os\n",
        "import csv\n",
        "import fitz  # PyMuPDF\n",
        "\n",
        "def extract_pdf_content(pdf_path):\n",
        "    doc = fitz.open(pdf_path)\n",
        "    content = \"\"\n",
        "    for page_num in range(doc.page_count):\n",
        "        page = doc[page_num]\n",
        "        content += page.get_text()\n",
        "    doc.close()\n",
        "    return content\n",
        "\n",
        "def process_pdfs(input_folder, output_csv):\n",
        "    # Create or append to the CSV file\n",
        "    mode = 'w' if not os.path.exists(output_csv) else 'a'\n",
        "\n",
        "    with open(output_csv, mode, newline='', encoding='utf-8') as csvfile:\n",
        "        fieldnames = ['id', 'name', 'content']\n",
        "        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
        "\n",
        "        # If the file is being created, write the header\n",
        "        if mode == 'w':\n",
        "            writer.writeheader()\n",
        "\n",
        "        # Iterate through PDF files in the input folder\n",
        "        for idx, pdf_file in enumerate(os.listdir(input_folder)):\n",
        "            if pdf_file.endswith(\".pdf\"):\n",
        "                pdf_path = os.path.join(input_folder, pdf_file)\n",
        "                content = extract_pdf_content(pdf_path)\n",
        "                #print(f\"id: {idx + 1}, name: {pdf_file}\")\n",
        "                writer.writerow({'id': idx + 1, 'name': pdf_file, 'content': content})\n",
        "\n",
        "\n",
        "input_folder_path = \"/content/docs\"\n",
        "output_csv_path = \"raw_data.csv\"\n",
        "process_pdfs(input_folder_path, output_csv_path)\n",
        "print(\"PDFs loaded into CSV successfully.\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mBvbzr3Em_Wo",
        "outputId": "d578e695-6d14-4e7e-f857-f671674dfb04"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PDFs loaded into CSV successfully.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kxvyDXJN6xjl"
      },
      "outputs": [],
      "source": [
        "#reading the raw data using langchain CSV loader\n",
        "loader = CSVLoader(file_path=\"./raw_data.csv\")\n",
        "documents = loader.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sdfH57QbEtqT",
        "outputId": "a1e33c57-31a5-4ad8-e1b8-915c4222d633"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8\n",
            "# of words in the document = 63382\n"
          ]
        }
      ],
      "source": [
        "#see some content\n",
        "print(len(documents))\n",
        "print(f\"# of words in the document = {len(documents[0].page_content)}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VwXl9dhmj9vy"
      },
      "source": [
        "#  Palm2 apis with langchain\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pI_Tm_hGpL2G"
      },
      "outputs": [],
      "source": [
        "llm = GooglePalm(google_api_key=GOOGLE_API_KEY)\n",
        "llm.temperature = 0.1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VXbEtl_e79qf"
      },
      "source": [
        "## Summerise each book"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SPGxlZ_KZsbo"
      },
      "outputs": [],
      "source": [
        "# I'm gonna use refine method for chunking the pdfs content, as this method retains the context.\n",
        "\n",
        "\n",
        "# prepare the prompt template for text summerisaion\n",
        "question_prompt_template = \"\"\"\n",
        "                  Please provide a summary of the following text. the first chunk should recognise the document title, and a summary extracted from the introduction section.\n",
        "                  TEXT: {text}\n",
        "                  SUMMARY:\n",
        "                  \"\"\"\n",
        "question_prompt = PromptTemplate(\n",
        "    template=question_prompt_template, input_variables=[\"text\"]\n",
        ")\n",
        "refine_prompt_template = \"\"\"\n",
        "              Write a concise summary of the following text.\n",
        "              ```{text}```\n",
        "              SUMMARY:\n",
        "              \"\"\"\n",
        "\n",
        "refine_prompt = PromptTemplate(\n",
        "    template=refine_prompt_template, input_variables=[\"text\"]\n",
        ")\n",
        "refine_chain = load_summarize_chain(\n",
        "    llm,\n",
        "    chain_type=\"refine\",\n",
        "    question_prompt=question_prompt,\n",
        "    refine_prompt=refine_prompt,\n",
        "    return_intermediate_steps=True,\n",
        ")\n",
        "text_splitter = CharacterTextSplitter(\n",
        "    separator = \"\\n\",\n",
        "    chunk_size = 5000,\n",
        "    chunk_overlap  = 100)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jJfpQdH0GpEk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "outputId": "bdee201b-7c37-4846-d1bb-3942a6df6121"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                     concise_summary  \\\n",
              "0  The key challenges for Paraguay include reduci...   \n",
              "1  This document presents UNDP's programme strate...   \n",
              "2  UNDP's proposed programme for Iraq focuses on ...   \n",
              "3  This document presents the results and resourc...   \n",
              "4  This document outlines the key performance ind...   \n",
              "\n",
              "                                            doc       file_name  \\\n",
              "0  {'source': './raw_data_light.csv', 'row': 0}  raw_data_light   \n",
              "1  {'source': './raw_data_light.csv', 'row': 4}  raw_data_light   \n",
              "2  {'source': './raw_data_light.csv', 'row': 4}  raw_data_light   \n",
              "3  {'source': './raw_data_light.csv', 'row': 4}  raw_data_light   \n",
              "4  {'source': './raw_data_light.csv', 'row': 4}  raw_data_light   \n",
              "\n",
              "                                              chunks  \n",
              "0  : 0\\nCPD_ID: pry_2\\ntext: United Nations      ...  \n",
              "1  benefited approximately 15,000 individuals13. ...  \n",
              "2  which transfers the powers of eight ministries...  \n",
              "3  to cost-share and provide funding to UNDP-impl...  \n",
              "4  institutions as more transparent     implement...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-117e5d74-c632-4a67-8ead-03b91a03864e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>concise_summary</th>\n",
              "      <th>doc</th>\n",
              "      <th>file_name</th>\n",
              "      <th>chunks</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The key challenges for Paraguay include reduci...</td>\n",
              "      <td>{'source': './raw_data_light.csv', 'row': 0}</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>: 0\\nCPD_ID: pry_2\\ntext: United Nations      ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>This document presents UNDP's programme strate...</td>\n",
              "      <td>{'source': './raw_data_light.csv', 'row': 4}</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>benefited approximately 15,000 individuals13. ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>UNDP's proposed programme for Iraq focuses on ...</td>\n",
              "      <td>{'source': './raw_data_light.csv', 'row': 4}</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>which transfers the powers of eight ministries...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>This document presents the results and resourc...</td>\n",
              "      <td>{'source': './raw_data_light.csv', 'row': 4}</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>to cost-share and provide funding to UNDP-impl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>This document outlines the key performance ind...</td>\n",
              "      <td>{'source': './raw_data_light.csv', 'row': 4}</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>institutions as more transparent     implement...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-117e5d74-c632-4a67-8ead-03b91a03864e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-117e5d74-c632-4a67-8ead-03b91a03864e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-117e5d74-c632-4a67-8ead-03b91a03864e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-8dd62646-5781-42ce-a54f-5ca4b74092e3\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-8dd62646-5781-42ce-a54f-5ca4b74092e3')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-8dd62646-5781-42ce-a54f-5ca4b74092e3 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "# Fixed-size chunking (note: content-aware could be another chunking option)\n",
        "try:\n",
        "    document_chunks = document_chunks = text_splitter.split_documents(documents)\n",
        "    #print(f\"Number chunks {len(document_chunks)}\")\n",
        "    refine_outputs = refine_chain({\"input_documents\": document_chunks})\n",
        "except IndexError:\n",
        "    gotdata = 'null'\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#showing chunk summeries\n",
        "final_refine_data = []\n",
        "for doc, out in zip(refine_outputs[\"input_documents\"], refine_outputs[\"intermediate_steps\"]):\n",
        "        output = {}\n",
        "        output[\"doc\"] = doc.metadata['row']\n",
        "        output[\"file_name\"] = p(doc.metadata[\"source\"]).stem\n",
        "        output[\"chunks\"] = doc.page_content.strip()\n",
        "        output[\"concise_summary\"] = out.strip()\n",
        "        final_refine_data.append(output)\n",
        "\n",
        "pdf_refine_summary  = pd.DataFrame.from_dict(final_refine_data)\n",
        "pdf_refine_summary  = pdf_refine_summary .sort_values(\n",
        "    by=[\"file_name\"]\n",
        ")  # sorting the dataframe by filename and page_number\n",
        "pdf_refine_summary .reset_index(inplace=True, drop=True)\n",
        "pdf_refine_summary .head(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "hleNwFU4iQ9d",
        "outputId": "8d1a2317-7ac7-436f-ef20-2c8ed720c84e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   doc       file_name                                             chunks  \\\n",
              "0    0  raw_data_light  : 0\\nCPD_ID: pry_2\\ntext: United Nations      ...   \n",
              "1    4  raw_data_light  benefited approximately 15,000 individuals13. ...   \n",
              "2    4  raw_data_light  which transfers the powers of eight ministries...   \n",
              "3    4  raw_data_light  to cost-share and provide funding to UNDP-impl...   \n",
              "4    4  raw_data_light  institutions as more transparent     implement...   \n",
              "5    4  raw_data_light  Target: Yes                                   ...   \n",
              "6    4  raw_data_light  DP/DCP/IRQ/2\\n                                ...   \n",
              "7    5  raw_data_light  : 5\\nCPD_ID: sle_1\\ntext: United Nations      ...   \n",
              "8    5  raw_data_light  hindered effective benchmarking and tracking o...   \n",
              "9    4  raw_data_light  : 4\\nCPD_ID: irq_2\\ntext: United Nations      ...   \n",
              "\n",
              "                                     concise_summary  \n",
              "0  The key challenges for Paraguay include reduci...  \n",
              "1  This document presents UNDP's programme strate...  \n",
              "2  UNDP's proposed programme for Iraq focuses on ...  \n",
              "3  This document presents the results and resourc...  \n",
              "4  This document outlines the key performance ind...  \n",
              "5  This document outlines the results and resourc...  \n",
              "6  This document outlines the results and resourc...  \n",
              "7  The CPD 2008-2010 for Sierra Leone is based on...  \n",
              "8  The Government of Sierra Leone has developed a...  \n",
              "9  This document presents the country programme d...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-52e8a080-1e6d-4ada-b0f4-7c6e7510d4cc\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>doc</th>\n",
              "      <th>file_name</th>\n",
              "      <th>chunks</th>\n",
              "      <th>concise_summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>: 0\\nCPD_ID: pry_2\\ntext: United Nations      ...</td>\n",
              "      <td>The key challenges for Paraguay include reduci...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>benefited approximately 15,000 individuals13. ...</td>\n",
              "      <td>This document presents UNDP's programme strate...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>which transfers the powers of eight ministries...</td>\n",
              "      <td>UNDP's proposed programme for Iraq focuses on ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>to cost-share and provide funding to UNDP-impl...</td>\n",
              "      <td>This document presents the results and resourc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>institutions as more transparent     implement...</td>\n",
              "      <td>This document outlines the key performance ind...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>4</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>Target: Yes                                   ...</td>\n",
              "      <td>This document outlines the results and resourc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>4</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>DP/DCP/IRQ/2\\n                                ...</td>\n",
              "      <td>This document outlines the results and resourc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>5</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>: 5\\nCPD_ID: sle_1\\ntext: United Nations      ...</td>\n",
              "      <td>The CPD 2008-2010 for Sierra Leone is based on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>5</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>hindered effective benchmarking and tracking o...</td>\n",
              "      <td>The Government of Sierra Leone has developed a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>4</td>\n",
              "      <td>raw_data_light</td>\n",
              "      <td>: 4\\nCPD_ID: irq_2\\ntext: United Nations      ...</td>\n",
              "      <td>This document presents the country programme d...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-52e8a080-1e6d-4ada-b0f4-7c6e7510d4cc')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-52e8a080-1e6d-4ada-b0f4-7c6e7510d4cc button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-52e8a080-1e6d-4ada-b0f4-7c6e7510d4cc');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-bc38eca5-7016-436b-8541-c1a09a0b5784\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-bc38eca5-7016-436b-8541-c1a09a0b5784')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-bc38eca5-7016-436b-8541-c1a09a0b5784 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#showing a summary for each book\n",
        "summeries = pdf_refine_summary.groupby('doc')['concise_summary'].agg(lambda x: '\\n'.join(x))\n",
        "\n",
        "\n",
        "# rephrase & improve the context. start by this documents ....\n",
        "\n",
        "# Print the merged DataFrame\n",
        "print(summeries.head(5))\n",
        "summeries.to_csv(\"summaries.csv\", index=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xHNpiY5OnSWP",
        "outputId": "7b13ec3f-111d-4a12-add2-d29e71500e56"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "doc\n",
            "0    The key challenges for Paraguay include reduci...\n",
            "1    Ghana's development challenges are: persistent...\n",
            "2    This is a document about the development of Be...\n",
            "3    4 key areas for Cambodia's development plan: r...\n",
            "4    This document presents UNDP's programme strate...\n",
            "Name: concise_summary, dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mWKt8uGpOBIq"
      },
      "source": [
        "## Q&A text books"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0WmU2uh6PgBz"
      },
      "outputs": [],
      "source": [
        "from langchain.document_loaders.csv_loader import CSVLoader\n",
        "loader = CSVLoader(file_path=\"./raw_data.csv\")\n",
        "pdfs = loader.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "asLOmVqrQIJU",
        "outputId": "8da3fa1b-028a-472e-fc27-1b19ba1c9b4a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number documents 369\n",
            "Number chunks 4098\n"
          ]
        }
      ],
      "source": [
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "\n",
        "text_splitter = CharacterTextSplitter(\n",
        "    separator = \"\\n\",\n",
        "    chunk_size = 5000,\n",
        "    chunk_overlap  = 100)\n",
        "\n",
        "pdfs_chunks = text_splitter.split_documents(pdfs)\n",
        "\n",
        "print(f\"Number documents {len(pdfs)}\")\n",
        "print(f\"Number chunks {len(pdfs_chunks)}\")\n",
        "\n",
        "pdfs_chunks__=[f\"Context: {chunk.page_content} Source: {chunk.metadata['source']}\" for chunk in pdfs_chunks]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jTxiCL-0nYih"
      },
      "outputs": [],
      "source": [
        "print(documents[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MdoYvzGISyhn",
        "outputId": "64057a90-1271-4a50-d49c-54548a74db5e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'source': './raw_data_light.csv', 'row': 0}\n"
          ]
        }
      ],
      "source": [
        "print(pdfs[0].metadata)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "embedding=GooglePalmEmbeddings(google_api_key=GOOGLE_API_KEY)"
      ],
      "metadata": {
        "id": "Qoxmk1DPuC6m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#another way to index chunks\n",
        "'''\n",
        "index = VectorstoreIndexCreator(embedding=embedding,\n",
        "        text_splitter=CharacterTextSplitter(separator = \"\\n\",chunk_size=1000, chunk_overlap=10)).from_documents(pdfs)\n",
        "\n",
        "\n",
        "chain = RetrievalQA.from_chain_type(llm=llm,\n",
        "                            chain_type=\"stuff\",\n",
        "                            retriever=index.vectorstore.as_retriever(),\n",
        "                            input_key=\"question\")\n",
        "'''"
      ],
      "metadata": {
        "id": "vx95Dt9-Ojd9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z2-aHty4QsWj"
      },
      "outputs": [],
      "source": [
        "question = \"does the text discusses how political instability and terrorism impact poverty?\"\n",
        "\n",
        "query_prompt_template = \"\"\"\n",
        "                  Follow exactly those 3 steps:\n",
        "                  1. Read the context below and aggregrate this data\n",
        "                    Context : {text}\n",
        "                  2. Answer the {question} using only this context by yes or no\n",
        "                  3. if yes, Show the docoument id of the document answers the user Question: {question}\n",
        "                  4. if no, just say no, it doesn't discuss it\n",
        "\n",
        "                 If you don't have any context and are unsure of the answer, reply that you don't know about this topic.\n",
        "                \"\"\"\n",
        "\n",
        "query_prompt = PromptTemplate(\n",
        "    template=query_prompt_template, input_variables=[\"text\"]\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S1hyBp1ItwwX"
      },
      "outputs": [],
      "source": [
        "persist_directory = 'docs/chroma/'\n",
        "db = Chroma.from_documents(pdfs_chunks, embedding, persist_directory=persist_directory)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gkdr_83kwLhv"
      },
      "outputs": [],
      "source": [
        "# Expose index to the retriever\n",
        "retriever = db.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": 4})\n",
        "# Create chain to answer questions\n",
        "from langchain.chains import RetrievalQA\n",
        "\n",
        "\n",
        "qa = RetrievalQA.from_chain_type(\n",
        "    llm=llm, chain_type=\"stuff\", retriever=retriever, return_source_documents=True\n",
        ")\n",
        "response = qa({\"query\": question})\n",
        "print(response)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#another way\n",
        "#print the results along with the source documents, so that we can see how the\\\n",
        "      # retriever performed, i.e. which chunks of text it found to prepend to the query as context for the LLM.\n",
        "def print_result(response_obj):\n",
        "    print(\"SOURCES: \\n\")\n",
        "    cnt = 1\n",
        "    for source_doc in response_obj[\"source_documents\"]:\n",
        "        print(f\"Chunk #{cnt}\")\n",
        "        cnt += 1\n",
        "        print(\"Source Metadata: \", source_doc.metadata)\n",
        "        print(\"Source Text:\")\n",
        "        print(source_doc.page_content)\n",
        "        print(\"\\n\")\n",
        "    print(\"RESULT: \\n\")\n",
        "    print(response_obj[\"result\"] + \"\\n\\n\")\n",
        "\n",
        "\n",
        "print_result(response)"
      ],
      "metadata": {
        "id": "7VmdHcx_5vdz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Contextual Compression\n",
        "def pretty_print_docs(docs):\n",
        "    print(f\"\\n{'-' * 100}\\n\".join([f\"Document {i+1}:\\n\\n\" + d.page_content for i, d in enumerate(docs)]))\n",
        "\n",
        "from langchain.retrievers import ContextualCompressionRetriever\n",
        "from langchain.retrievers.document_compressors import LLMChainExtractor\n",
        "\n",
        "# Wrap our vectorstore\n",
        "compressor = LLMChainExtractor.from_llm(llm)\n",
        "\n",
        "compression_retriever = ContextualCompressionRetriever(\n",
        "    base_compressor=compressor,\n",
        "    base_retriever=db.as_retriever()\n",
        ")\n",
        "\n",
        "compressed_docs = compression_retriever.get_relevant_documents(question)\n",
        "pretty_print_docs(compressed_docs)"
      ],
      "metadata": {
        "id": "xbLuZuGwCFq2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tj7LLWe-SVSJ"
      },
      "source": [
        "Q&A : Gemini Pro example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AbsrusB6SyOI"
      },
      "outputs": [],
      "source": [
        "import google.generativeai as genai\n",
        "genai.configure(api_key=GOOGLE_API_KEY)\n",
        "model = genai.GenerativeModel('gemini-pro')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loader = CSVLoader(file_path=\"./raw_data.csv\")\n",
        "documents = loader.load()"
      ],
      "metadata": {
        "id": "fA58tRq0NOtz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IObWceiOTPgp"
      },
      "outputs": [],
      "source": [
        "\n",
        "retrived_docs = pd.DataFrame()\n",
        "context =        \"\"\"\n",
        "                     I will give you a long text, you are required to find out if the text clearly includes some discussions about how political instability and terrorism impact poverty.:\n",
        "                     1. You should answer with Yes or No,\n",
        "                     2. if Yes, show the paragraphs discussing political instability and terrorism impact poverty.\n",
        "                     3. if no,  show a text expressing that the text doesn't include discussion about the topic.\n",
        "\n",
        "                \"\"\"\n",
        "for pdf in documents:\n",
        "    text = pdf.page_content\n",
        "    g_response = model.generate_content([context , text], stream=True)\n",
        "    g_response.resolve()\n",
        "    answer = g_response.text\n",
        "    answer = answer.strip()\n",
        "    import re\n",
        "    #answer = re.sub(r\"[^\\w\\s]\", \"\", answer).lower()\n",
        "    if answer.startswith(\"Yes\") :\n",
        "        data = {\"doc_id\": pdf.metadata['row'] , \"text\": answer[3:20]}\n",
        "        retrived_docs = retrived_docs.append(pd.Series(data), ignore_index=True)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(retrived_docs)\n",
        "retrived_docs.to_csv(\"result_task2.csv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S4ImkWcdRHpC",
        "outputId": "c4872bce-d4c9-4b29-a1b5-5e930eaa505e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   doc_id                                               text\n",
            "0      99   the discussions about how political instabili...\n",
            "1     101   under section i programme rationale\\nparagrap...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "filtered_documents = [\n",
        "    document\n",
        "    for document in documents\n",
        "    if document.metadata[\"row\"] in retrived_docs[\"doc_id\"]\n",
        "]\n",
        "\n"
      ],
      "metadata": {
        "id": "WuTKJj_SU3q3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c4s4rBy_SSta"
      },
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}