Exp_9_Topic_Modelling_TF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ab11ee8c-b6e7-45da-8b35-66294375808f",
   "metadata": {},
   "source": [
    "# Exp_9:Topic Modeling Using Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f26aa33d-8739-4d52-9f37-ca87aed7c961",
   "metadata": {},
   "source": [
    "## Obj: To convert the documents into topics with tensorflow framework."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0f283f02-0dfe-4e5e-8370-c2a92729c0e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c576e8a5-1722-4963-84f8-f2392aec6fa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample documents\n",
    "documents = [\n",
    "    \"machine learning is a subset of artificial intelligence\",\n",
    "    \"tensorflow is a popular machine learning framework\",\n",
    "    \"topic modeling is an important technique in NLP\",\n",
    "    \"deep learning is a subset of machine learning\",\n",
    "    \"natural language processing is a part of NLP\",\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e73c9ffa-f6f9-44ea-8d14-b7c0f574140e",
   "metadata": {},
   "source": [
    "documents = [\n",
    "    \"Koneru Lakshmiah University is one of the oldest universities in the india.\",\n",
    "    \"indian has a rich history dating back to the 10th century.\",\n",
    "    \"Koneru Lakshmiah University is renowned for its contributions to science and engineering.\",\n",
    "    \"engineering was founded in 1701 and has a distinguished history.\",\n",
    "    \"The history of the vijayawada can be traced back to the 12th century.\",\n",
    "    \"The history of the Koneru Lakshmiah University dates back to the 20th century.\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "45f39e7a-ff87-4a78-950b-5ba7407ca132",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eead074-6ff0-45b5-85b1-fbfb91f1b7f6",
   "metadata": {},
   "source": [
    "# Tokenize the documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bdeefbc6-3b6e-4c49-9f4c-3d1ea7599981",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "tokenizer = Tokenizer(num_words = 100)\n",
    "tokenizer.fit_on_texts(documents)\n",
    "#word_index = tokenizer.word_index\n",
    "#print(word_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2acf360-5cbc-4eb8-86a8-4c2d4d996999",
   "metadata": {},
   "source": [
    "# Build a Vocabulary from the tokenized text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e549318f-21f8-4489-9c4e-a00717fe7409",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = tokenizer.word_index\n",
    "word_counts = tokenizer.word_counts\n",
    "#vocab dictionary contains the mapping of words to integer values, \n",
    "#word_counts contains the count of each word in your text data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8efd3cdc-2139-40bf-9ea7-78144d1f5bef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'is': 1,\n",
       "  'learning': 2,\n",
       "  'a': 3,\n",
       "  'machine': 4,\n",
       "  'of': 5,\n",
       "  'subset': 6,\n",
       "  'nlp': 7,\n",
       "  'artificial': 8,\n",
       "  'intelligence': 9,\n",
       "  'tensorflow': 10,\n",
       "  'popular': 11,\n",
       "  'framework': 12,\n",
       "  'topic': 13,\n",
       "  'modeling': 14,\n",
       "  'an': 15,\n",
       "  'important': 16,\n",
       "  'technique': 17,\n",
       "  'in': 18,\n",
       "  'deep': 19,\n",
       "  'natural': 20,\n",
       "  'language': 21,\n",
       "  'processing': 22,\n",
       "  'part': 23},\n",
       " OrderedDict([('machine', 3),\n",
       "              ('learning', 4),\n",
       "              ('is', 5),\n",
       "              ('a', 4),\n",
       "              ('subset', 2),\n",
       "              ('of', 3),\n",
       "              ('artificial', 1),\n",
       "              ('intelligence', 1),\n",
       "              ('tensorflow', 1),\n",
       "              ('popular', 1),\n",
       "              ('framework', 1),\n",
       "              ('topic', 1),\n",
       "              ('modeling', 1),\n",
       "              ('an', 1),\n",
       "              ('important', 1),\n",
       "              ('technique', 1),\n",
       "              ('in', 1),\n",
       "              ('nlp', 2),\n",
       "              ('deep', 1),\n",
       "              ('natural', 1),\n",
       "              ('language', 1),\n",
       "              ('processing', 1),\n",
       "              ('part', 1)]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab, word_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "361d2eb3-1c39-46eb-ac6e-31b254f8b4b0",
   "metadata": {},
   "source": [
    "# Convert the above teokenized documents into bag of words (BOW) representation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9595ef5b-6ac4-47d2-a43b-b764be56651c",
   "metadata": {},
   "source": [
    "### First convert to sequences of sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bad17fa2-5a43-4270-85c9-343d281a1222",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = tokenizer.texts_to_sequences(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4a499a4b-b22f-467f-9c53-de45cfd0cdb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[4, 2, 1, 3, 6, 5, 8, 9],\n",
       " [10, 1, 3, 11, 4, 2, 12],\n",
       " [13, 14, 1, 15, 16, 17, 18, 7],\n",
       " [19, 2, 1, 3, 6, 5, 4, 2],\n",
       " [20, 21, 22, 1, 3, 23, 5, 7]]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26467c21-4fcb-4756-97bb-e15756a68a93",
   "metadata": {},
   "source": [
    "### Then Generate BOW Representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5358e862-02fb-4c6b-84ce-dadb34980454",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_of_words = len(word_counts)\n",
    "num_of_words"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb81b31d-d800-4a9b-9b96-8f6f98a282af",
   "metadata": {},
   "source": [
    "##### Initialize a empty matrix to store BOW features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6c41faab-00a4-4640-ad36-a9abdfe859ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "BOW_Features = []\n",
    "for sequence in sequences:\n",
    "    BOW_Vector = [0]*num_of_words\n",
    "    for word_index in sequence:\n",
    "        BOW_Vector[word_index - 1] += 1\n",
    "    BOW_Features.append(BOW_Vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "64385066-1912-427e-8f8f-9d940a7e2859",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       " [1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       " [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       " [1, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],\n",
       " [1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1]]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BOW_Features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ef8b026-e39a-4734-b970-28f635b45ddf",
   "metadata": {},
   "source": [
    "# We can also use Term Frequency (TF) and Inverse Document Frequency to Compute BOW"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "751d0287-2819-405d-9740-f619e26129d4",
   "metadata": {},
   "source": [
    "### Calculate TF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "54644ff5-73cd-4e80-b4ba-47812838137b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "7\n",
      "1\n",
      "7\n",
      "1\n",
      "7\n",
      "1\n",
      "7\n",
      "1\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "1\n",
      "7\n",
      "1\n",
      "7\n",
      "1\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "7\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "2\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "0\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n",
      "8\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "tf_matrix = np.zeros((len(documents), len(vocab)), dtype=np.float32)\n",
    "for i, sentence in enumerate(documents):\n",
    "    words = sentence.lower().split()\n",
    "    for j, word in enumerate(vocab):\n",
    "        tf_matrix[i, j] = words.count(word) / len(words)\n",
    "        print(len(words))\n",
    "        print(words.count(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b59ab2bb-31c6-4088-b0f1-114a880de04a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.125     , 0.125     , 0.125     , 0.125     , 0.125     ,\n",
       "        0.125     , 0.        , 0.125     , 0.125     , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.14285715, 0.14285715, 0.14285715, 0.14285715, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.14285715,\n",
       "        0.14285715, 0.14285715, 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.125     , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.125     , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.125     , 0.125     , 0.125     ,\n",
       "        0.125     , 0.125     , 0.125     , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.125     , 0.25      , 0.125     , 0.125     , 0.125     ,\n",
       "        0.125     , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.125     , 0.        ,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.125     , 0.        , 0.125     , 0.        , 0.125     ,\n",
       "        0.        , 0.125     , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.125     ,\n",
       "        0.125     , 0.125     , 0.125     ]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "007324bf-03cc-48fa-b8fb-465afb254f15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 23)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ebd5416-4bd3-42b7-98b6-d24d469bfc16",
   "metadata": {},
   "source": [
    "### Calculate IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4f9d1c91-80a1-4733-9388-9a54c7058f13",
   "metadata": {},
   "outputs": [],
   "source": [
    "idf_vector = np.zeros(len(vocab), dtype=np.float32)\n",
    "total_documents = len(documents)\n",
    "for j, word in enumerate(vocab):\n",
    "    doc_count = sum(1 for sentence in documents if word in sentence.split())\n",
    "    idf_vector[j] = np.log(total_documents / (1 + doc_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "588c3a15-1da4-49aa-9cf5-05b127c2462e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.18232156,  0.22314355,  0.        ,  0.22314355,  0.22314355,\n",
       "        0.51082563,  1.609438  ,  0.91629076,  0.91629076,  0.91629076,\n",
       "        0.91629076,  0.91629076,  0.91629076,  0.91629076,  0.91629076,\n",
       "        0.91629076,  0.91629076,  0.91629076,  0.91629076,  0.91629076,\n",
       "        0.91629076,  0.91629076,  0.91629076], dtype=float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf_vector # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "811ff894-57be-433b-9026-aed71d8a17de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23,)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3ec8f7c-7b12-4cc2-864e-8f50d1c5e447",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.0227902 ,  0.02789294,  0.        ,  0.02789294,  0.02789294,\n",
       "         0.0638532 ,  0.        ,  0.11453635,  0.11453635,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ],\n",
       "       [-0.02604594,  0.03187765,  0.        ,  0.03187765,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.13089868,\n",
       "         0.13089868,  0.13089868,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ],\n",
       "       [-0.0227902 ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.20117974,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.11453635,  0.11453635,  0.11453635,\n",
       "         0.11453635,  0.11453635,  0.11453635,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ],\n",
       "       [-0.0227902 ,  0.05578589,  0.        ,  0.02789294,  0.02789294,\n",
       "         0.0638532 ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.11453635,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ],\n",
       "       [-0.0227902 ,  0.        ,  0.        ,  0.        ,  0.02789294,\n",
       "         0.        ,  0.20117974,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.11453635,\n",
       "         0.11453635,  0.11453635,  0.11453635]], dtype=float32)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the TF-IDF matrix\n",
    "tfidf_matrix = tf_matrix * idf_vector\n",
    "tfidf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "aba82019-1f12-442d-a261-6872ff1fcd78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 23)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76ed2550-3f70-4a59-8a0c-891dd84a3a76",
   "metadata": {},
   "source": [
    "### TFIDF Using Functions: Bag of Words representation using TF-IDF scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "eea1a9dd-bba2-4b8e-bf58-4690130150fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<5x22 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 34 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer()\n",
    "tfidf_matrix = tfidf_vectorizer.fit_transform(documents)\n",
    "tfidf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "26b09cf6-2d84-4d67-9411-7de34f45dc09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(5, 22), dtype=float32, numpy=\n",
       "array([[0.        , 0.4865898 , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.4865898 , 0.23186265, 0.        , 0.32587487,\n",
       "        0.32587487, 0.        , 0.        , 0.        , 0.32587487,\n",
       "        0.        , 0.        , 0.        , 0.3925776 , 0.        ,\n",
       "        0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.49242058, 0.        ,\n",
       "        0.        , 0.        , 0.23464105, 0.        , 0.3297798 ,\n",
       "        0.3297798 , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.49242058, 0.        , 0.        , 0.        ,\n",
       "        0.49242058, 0.        ],\n",
       "       [0.3813026 , 0.        , 0.        , 0.        , 0.3813026 ,\n",
       "        0.3813026 , 0.        , 0.18169272, 0.        , 0.        ,\n",
       "        0.        , 0.3813026 , 0.        , 0.30763254, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.3813026 ,\n",
       "        0.        , 0.3813026 ],\n",
       "       [0.        , 0.        , 0.4678286 , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.22292283, 0.        , 0.62662053,\n",
       "        0.31331027, 0.        , 0.        , 0.        , 0.31331027,\n",
       "        0.        , 0.        , 0.        , 0.37744117, 0.        ,\n",
       "        0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.20646544, 0.4332909 , 0.        ,\n",
       "        0.        , 0.        , 0.4332909 , 0.34957635, 0.29017997,\n",
       "        0.4332909 , 0.        , 0.4332909 , 0.        , 0.        ,\n",
       "        0.        , 0.        ]], dtype=float32)>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_tensor = tf.constant(tfidf_matrix.toarray(), dtype=tf.float32)\n",
    "tfidf_tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07292bd8-91fe-40fe-bc2f-0a1081b63859",
   "metadata": {},
   "source": [
    "# Lets get back to Topic Modelling using Linear Discriminant Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1ec6bb00-10bc-4e0c-a937-4c12cf2eeb41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(5, 23), dtype=float32, numpy=\n",
       "array([[1., 1., 1., 1., 1., 1., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0.],\n",
       "       [1., 1., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1.,\n",
       "        1., 1., 0., 0., 0., 0., 0.],\n",
       "       [1., 2., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 1., 0., 0., 0., 0.],\n",
       "       [1., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 1., 1., 1., 1.]], dtype=float32)>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert to TensorFlow tensors\n",
    "BOW_Features = tf.constant(BOW_Features, dtype=tf.float32)\n",
    "BOW_Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "afa216ec-7ae9-4a17-b64c-0cff1ceced00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['machine learning is a subset of artificial intelligence',\n",
       " 'tensorflow is a popular machine learning framework',\n",
       " 'topic modeling is an important technique in NLP',\n",
       " 'deep learning is a subset of machine learning',\n",
       " 'natural language processing is a part of NLP']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "905f57fc-38ce-4350-b8df-9c733ab2dc55",
   "metadata": {},
   "source": [
    "# Topics = 2 -- learning, machine \n",
    "# Input (BOW) - Output is probability that document 1 belongs to topic learning (0.8) or machine(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "670f6543-db1b-481a-abc9-7f1150146bda",
   "metadata": {},
   "outputs": [],
   "source": [
    "Topics = 2\n",
    "epochs = 100\n",
    "alpha = 1.0 # Document - Topic Distribution Hyperparameter\n",
    "beta = 1.0 # Topic - Word Distribution Hyperparameter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92bc566a-3a83-475a-af82-f1d620b7b391",
   "metadata": {},
   "source": [
    "## Initialize Topic - Word Distribution Randomly "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2bd90400-c4dd-469c-9abe-8155058f99a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 23), dtype=float32, numpy=\n",
       "array([[0.4282782 , 0.14814746, 0.22535539, 0.84781325, 0.20984268,\n",
       "        0.55467784, 0.05056345, 0.85595405, 0.2665887 , 0.54655814,\n",
       "        0.05033481, 0.172176  , 0.9405519 , 0.28654146, 0.22927582,\n",
       "        0.3575008 , 0.5543679 , 0.6904756 , 0.20061374, 0.38163912,\n",
       "        0.8016926 , 0.30266845, 0.14317667],\n",
       "       [0.23012912, 0.5880227 , 0.6013558 , 0.38890803, 0.47694016,\n",
       "        0.71845365, 0.6674757 , 0.43920743, 0.2477498 , 0.82374537,\n",
       "        0.9609926 , 0.28887117, 0.19562697, 0.7963016 , 0.52349126,\n",
       "        0.06840408, 0.15503561, 0.454558  , 0.8707397 , 0.12246466,\n",
       "        0.20856857, 0.04432523, 0.859408  ]], dtype=float32)>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_word_distribution = tf.random.uniform((Topics, num_of_words),0,1)\n",
    "topic_word_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "58e20209-4725-4fbb-986e-32b2ca61be52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 23), dtype=float32, numpy=\n",
       "array([[0.04632642, 0.01602496, 0.02437647, 0.0917071 , 0.02269847,\n",
       "        0.05999894, 0.0054694 , 0.09258769, 0.02883663, 0.05912064,\n",
       "        0.00544467, 0.0186241 , 0.10173854, 0.0309949 , 0.02480053,\n",
       "        0.0386705 , 0.05996541, 0.07468805, 0.02170018, 0.04128152,\n",
       "        0.08671828, 0.03273934, 0.01548728],\n",
       "       [0.02144571, 0.05479778, 0.05604029, 0.0362423 , 0.04444601,\n",
       "        0.06695263, 0.062202  , 0.0409297 , 0.02308778, 0.07676475,\n",
       "        0.08955481, 0.02691988, 0.01823046, 0.07420727, 0.0487841 ,\n",
       "        0.00637457, 0.01444775, 0.04236022, 0.08114415, 0.01141247,\n",
       "        0.01943649, 0.00413066, 0.08008815]], dtype=float32)>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_word_distribution /= tf.reduce_sum(topic_word_distribution, axis=1, keepdims=True)\n",
    "topic_word_distribution"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "777d16dc-dd51-4631-a082-e89c80474468",
   "metadata": {},
   "source": [
    "tensor = tf.random.uniform((2,3),0,1)\n",
    "result = tf.reduce_sum(tensor, axis=1, keepdims=True)\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01229db4-dbe0-4e58-a5d6-74798cb36f49",
   "metadata": {},
   "source": [
    "# Construct a Simple 2 Layer ANN model for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a1496285-821c-40c6-8b30-67513de5eb1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _ in range(epochs):\n",
    "    document_topic_distribution = tf.nn.softmax(alpha+tf.matmul(BOW_Features, topic_word_distribution, transpose_b=True))\n",
    "    topic_word_distribution = tf.nn.softmax(beta + tf.matmul(document_topic_distribution, BOW_Features,transpose_a=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "fb6e10d4-5347-4746-a881-77700d034805",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([5, 23])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BOW_Features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "68c6353a-4041-4cb2-bf0c-942b327c114f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 23])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_word_distribution.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "911b1e35-f5f9-4956-8db8-cf960fd811c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([5, 2])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_topic_distribution.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b127e075-5edd-4fe9-a105-77f504aa157f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'is',\n",
       " 1: 'learning',\n",
       " 2: 'a',\n",
       " 3: 'machine',\n",
       " 4: 'of',\n",
       " 5: 'subset',\n",
       " 6: 'nlp',\n",
       " 7: 'artificial',\n",
       " 8: 'intelligence',\n",
       " 9: 'tensorflow',\n",
       " 10: 'popular',\n",
       " 11: 'framework',\n",
       " 12: 'topic',\n",
       " 13: 'modeling',\n",
       " 14: 'an',\n",
       " 15: 'important',\n",
       " 16: 'technique',\n",
       " 17: 'in',\n",
       " 18: 'deep',\n",
       " 19: 'natural',\n",
       " 20: 'language',\n",
       " 21: 'processing',\n",
       " 22: 'part'}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2idx = {word: idx for idx, word in enumerate(vocab)}\n",
    "idx2word = {idx: word for word, idx in word2idx.items()}\n",
    "idx2word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "cfabc98e-fd98-4bc4-b547-c1244fc1fcb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([0 1 2 3 4], shape=(5,), dtype=int32)\n",
      "tf.Tensor([0 1 2 3 4], shape=(5,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "# Print the learned topics\n",
    "topics = {}\n",
    "for i, topic_dist in enumerate(topic_word_distribution):\n",
    "    top_words_idx = tf.argsort(topic_dist, direction='DESCENDING')[:5]\n",
    "    print(top_words_idx)\n",
    "    top_words = [idx2word[idx.numpy()] for idx in top_words_idx]\n",
    "    topics[f\"Topic {i + 1}\"] = top_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4d78aedd-cb32-4fce-84c9-43db40d93842",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 1: is, learning, a, machine, of\n",
      "Topic 2: is, learning, a, machine, of\n"
     ]
    }
   ],
   "source": [
    "for topic, words in topics.items():\n",
    "    print(f\"{topic}: {', '.join(words)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c95c9218-4cd0-48c1-a541-669cce4f97f3",
   "metadata": {},
   "source": [
    "## Remove stop words and then calculate topic vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "eb9f63dd-a19d-4e85-95dc-f75f90d654b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "# Tokenize the documents and remove stop words\n",
    "tokenized_docs = []\n",
    "for doc in documents:\n",
    "    words = doc.split()\n",
    "    filtered_words = [word for word in words if word.lower() not in stop_words]\n",
    "    tokenized_docs.append(filtered_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "74a67cb0-353b-4276-babf-c1e41f225dcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['machine', 'learning', 'subset', 'artificial', 'intelligence'],\n",
       " ['tensorflow', 'popular', 'machine', 'learning', 'framework'],\n",
       " ['topic', 'modeling', 'important', 'technique', 'NLP'],\n",
       " ['deep', 'learning', 'subset', 'machine', 'learning'],\n",
       " ['natural', 'language', 'processing', 'part', 'NLP']]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "83b298a4-3768-44b7-becd-f964c66f9136",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a vocabulary\n",
    "vocab = set(word for doc in tokenized_docs for word in doc)\n",
    "word2idx = {word: idx for idx, word in enumerate(vocab)}\n",
    "idx2word = {idx: word for word, idx in word2idx.items()}\n",
    "num_words = len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ea1211b5-3297-45cf-869f-e73f1af3a107",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert tokenized documents to bag-of-words representation\n",
    "bow_docs = []\n",
    "for doc in tokenized_docs:\n",
    "    bow_doc = [0] * num_words\n",
    "    for word in doc:\n",
    "        bow_doc[word2idx[word]] += 1\n",
    "    bow_docs.append(bow_doc)\n",
    "\n",
    "# Convert to TensorFlow tensors\n",
    "bow_docs = tf.constant(bow_docs, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "839202ca-0a55-4d63-b631-e2120202c9c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the LDA model\n",
    "num_topics = 2\n",
    "num_iterations = 100\n",
    "alpha = 1.0  # Hyperparameter for document-topic distribution\n",
    "beta = 1.0   # Hyperparameter for topic-word distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "eecf686b-00a8-4c18-8f21-f50a694a8230",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize topic-word distribution randomly\n",
    "topic_word_distribution = tf.random.uniform((num_topics, num_words), 0, 1)\n",
    "topic_word_distribution /= tf.reduce_sum(topic_word_distribution, axis=1, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f4ddac4c-efa6-44d7-ab8e-636d7cc997c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _ in range(num_iterations):\n",
    "    # E-step: Update document-topic distribution\n",
    "    doc_topic_distribution = tf.nn.softmax(alpha + tf.matmul(bow_docs, topic_word_distribution, transpose_b=True))\n",
    "\n",
    "    # M-step: Update topic-word distribution\n",
    "    topic_word_distribution = tf.nn.softmax(beta + tf.matmul(doc_topic_distribution, bow_docs,transpose_a=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "2454f2cb-aef2-4706-8dc9-894b318435b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 1: learning, machine, subset, NLP, language\n",
      "Topic 2: learning, machine, subset, NLP, language\n"
     ]
    }
   ],
   "source": [
    "# Print the learned topics\n",
    "topics = {}\n",
    "for i, topic_dist in enumerate(topic_word_distribution):\n",
    "    top_words_idx = tf.argsort(topic_dist, direction='DESCENDING')[:5]\n",
    "    top_words = [idx2word[idx.numpy()] for idx in top_words_idx]  # Convert to NumPy array\n",
    "    topics[f\"Topic {i + 1}\"] = top_words\n",
    "\n",
    "for topic, words in topics.items():\n",
    "    print(f\"{topic}: {', '.join(words)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "3d096f4a-348e-47e7-ab90-626c9395afcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document-Topic Vectors:\n",
      "[[2.  0.5 1.5 1.  0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 1.  0.5 0.5 0.5 0.5 0.5]\n",
      " [2.  0.5 1.5 1.  0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 1.  0.5 0.5 0.5 0.5 0.5]]\n"
     ]
    }
   ],
   "source": [
    "# Calculate topic vectors for each document\n",
    "document_topic_vectors = tf.matmul(doc_topic_distribution,bow_docs,transpose_a=True)\n",
    "print(\"Document-Topic Vectors:\")\n",
    "print(document_topic_vectors.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "738b4fd8-f307-43c0-9507-c9ec4c6e4b65",
   "metadata": {},
   "source": [
    "# Task_Today: Use TFIDF BOW and compute the topics given in the documents."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}