Exp_2_Text_2_Sequences.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ab5c8623",
   "metadata": {},
   "source": [
    "# Tokenization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0dd607a3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-09T09:48:21.832471Z",
     "start_time": "2023-05-09T09:48:10.648366Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "61c7004e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sen = [\n",
    "    'I am good human as i am with good dog'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b33f3c03",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words = 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d861bf53",
   "metadata": {},
   "source": [
    "# This class allows to vectorize a text corpus, by turning each text into either a sequence of integers \n",
    "\n",
    "## The Number of Words That it can Tokenize.\n",
    "\n",
    "#### the maximum number of words to keep, based on word frequency. Only the most common num_words-1 words will be kept.\n",
    "#f.keras.preprocessing.text.Tokenizer(num_words=None,\n",
    "    filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
    "    lower=True, split=' ', char_level=False, oov_token=None,\n",
    "    document_count=0, **kwargs\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c2a2056c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.fit_on_texts(sen)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58177e5d",
   "metadata": {},
   "source": [
    "# fit_on_texts will create a tokenized word index printing a set of KEY/Value pairs for the words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8111b5be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'i': 1, 'am': 2, 'good': 3, 'human': 4, 'as': 5, 'with': 6, 'dog': 7}\n"
     ]
    }
   ],
   "source": [
    "word_index = tokenizer.word_index\n",
    "print(word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2ec5e639",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'num_words': 100,\n",
       " 'filters': '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
       " 'lower': True,\n",
       " 'split': ' ',\n",
       " 'char_level': False,\n",
       " 'oov_token': None,\n",
       " 'document_count': 1,\n",
       " 'word_counts': '{\"i\": 2, \"am\": 2, \"good\": 2, \"human\": 1, \"as\": 1, \"with\": 1, \"dog\": 1}',\n",
       " 'word_docs': '{\"as\": 1, \"am\": 1, \"with\": 1, \"human\": 1, \"dog\": 1, \"good\": 1, \"i\": 1}',\n",
       " 'index_docs': '{\"5\": 1, \"2\": 1, \"6\": 1, \"4\": 1, \"7\": 1, \"3\": 1, \"1\": 1}',\n",
       " 'index_word': '{\"1\": \"i\", \"2\": \"am\", \"3\": \"good\", \"4\": \"human\", \"5\": \"as\", \"6\": \"with\", \"7\": \"dog\"}',\n",
       " 'word_index': '{\"i\": 1, \"am\": 2, \"good\": 3, \"human\": 4, \"as\": 5, \"with\": 6, \"dog\": 7}'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bc53c082",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(sen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6c9e7ce1",
   "metadata": {},
   "outputs": [],
   "source": [
    "Sen1 = [\n",
    "    'Today is a sunny day',\n",
    "    'Today is a autum day',\n",
    "    'what is it like today'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "002f65d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "71716e49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'today': 1, 'is': 2, 'a': 3, 'day': 4, 'sunny': 5, 'autum': 6, 'what': 7, 'it': 8, 'like': 9}\n"
     ]
    }
   ],
   "source": [
    "tokenizer.fit_on_texts(Sen1)\n",
    "word_index = tokenizer.word_index\n",
    "print(word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c3f651aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "sen = [\n",
    "    'How are % today.',\n",
    "    'I love $ money:'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cf86af2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'how': 1, 'are': 2, '%': 3, 'today.': 4, 'i': 5, 'love': 6, '$': 7, 'money:': 8}\n"
     ]
    }
   ],
   "source": [
    "tokenizer_1 = Tokenizer(num_words = 100, filters = '')\n",
    "tokenizer_1.fit_on_texts(sen)\n",
    "word_index_1 = tokenizer_1.word_index\n",
    "print(word_index_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2ef1d1c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'how': 1, 'are': 2, 'today.': 3, 'i': 4, 'love': 5, 'money:': 6}\n"
     ]
    }
   ],
   "source": [
    "tokenizer_1 = Tokenizer(num_words = 100, filters = '$%')\n",
    "tokenizer_1.fit_on_texts(sen)\n",
    "word_index_1 = tokenizer_1.word_index\n",
    "print(word_index_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c0b12057",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 2, 3, 5, 4], [1, 2, 3, 6, 4], [7, 2, 8, 9, 1]]\n"
     ]
    }
   ],
   "source": [
    "sequences = tokenizer.texts_to_sequences(Sen1)\n",
    "print(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1fedfbd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'today': 1, 'is': 2, 'a': 3, 'day': 4, 'sunny': 5, 'autum': 6, 'what': 7, 'it': 8, 'like': 9}\n"
     ]
    }
   ],
   "source": [
    "print(word_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8c1c298",
   "metadata": {},
   "source": [
    "# Tokenization in nltk api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b7057b29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'price', 'of', 'burger', 'in', 'klu', 'is', 'Rs.136.']\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "# import WhitespaceTokenizer() method from nltk\n",
    "from nltk.tokenize import WhitespaceTokenizer\n",
    "     \n",
    "# Create a reference variable for Class WhitespaceTokenizer\n",
    "wstk = WhitespaceTokenizer()\n",
    "     \n",
    "# Create a string input\n",
    "text1 = \"The price\\t of burger \\nin klu is Rs.136.\\n\"\n",
    "     \n",
    "# Use tokenize method\n",
    "tokens_text1 = wstk.tokenize(text1)\n",
    "     \n",
    "print(tokens_text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4dbcaebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "text2 = \"Natural language processing (NLP) is a field \" + \\\n",
    "       \"of computer science, artificial intelligence \" + \\\n",
    "       \"and computational linguistics concerned with \" + \\\n",
    "       \"the interactions between computers and human \" + \\\n",
    "       \"(natural) languages, and, in particular, \" + \\\n",
    "       \"concerned with programming computers to \" + \\\n",
    "       \"fruitfully process large natural language \" + \\\n",
    "       \"corpora. Challenges in natural language \" + \\\n",
    "       \"processing frequently involve natural \" + \\\n",
    "       \"language understanding, natural language\" + \\\n",
    "       \"generation frequently from formal, machine\" + \\\n",
    "       \"-readable logical forms), connecting language \" + \\\n",
    "       \"and machine perception, managing human-\" + \\\n",
    "       \"computer dialog systems, or some combination \" + \\\n",
    "       \"thereof.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dd364825",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Natural', 'language', 'processing', '(NLP)', 'is', 'a', 'field', 'of', 'computer', 'science,', 'artificial', 'intelligence', 'and', 'computational', 'linguistics', 'concerned', 'with', 'the', 'interactions', 'between', 'computers', 'and', 'human', '(natural)', 'languages,', 'and,', 'in', 'particular,', 'concerned', 'with', 'programming', 'computers', 'to', 'fruitfully', 'process', 'large', 'natural', 'language', 'corpora.', 'Challenges', 'in', 'natural', 'language', 'processing', 'frequently', 'involve', 'natural', 'language', 'understanding,', 'natural', 'languagegeneration', 'frequently', 'from', 'formal,', 'machine-readable', 'logical', 'forms),', 'connecting', 'language', 'and', 'machine', 'perception,', 'managing', 'human-computer', 'dialog', 'systems,', 'or', 'some', 'combination', 'thereof.']\n"
     ]
    }
   ],
   "source": [
    "tokens_text2 = wstk.tokenize(text2)   \n",
    "print(tokens_text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a5451f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize, sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1ab73e11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Natural',\n",
       " 'language',\n",
       " 'processing',\n",
       " '(',\n",
       " 'NLP',\n",
       " ')',\n",
       " 'is',\n",
       " 'a',\n",
       " 'field',\n",
       " 'of',\n",
       " 'computer',\n",
       " 'science',\n",
       " ',',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'and',\n",
       " 'computational',\n",
       " 'linguistics',\n",
       " 'concerned',\n",
       " 'with',\n",
       " 'the',\n",
       " 'interactions',\n",
       " 'between',\n",
       " 'computers',\n",
       " 'and',\n",
       " 'human',\n",
       " '(',\n",
       " 'natural',\n",
       " ')',\n",
       " 'languages',\n",
       " ',',\n",
       " 'and',\n",
       " ',',\n",
       " 'in',\n",
       " 'particular',\n",
       " ',',\n",
       " 'concerned',\n",
       " 'with',\n",
       " 'programming',\n",
       " 'computers',\n",
       " 'to',\n",
       " 'fruitfully',\n",
       " 'process',\n",
       " 'large',\n",
       " 'natural',\n",
       " 'language',\n",
       " 'corpora',\n",
       " '.',\n",
       " 'Challenges',\n",
       " 'in',\n",
       " 'natural',\n",
       " 'language',\n",
       " 'processing',\n",
       " 'frequently',\n",
       " 'involve',\n",
       " 'natural',\n",
       " 'language',\n",
       " 'understanding',\n",
       " ',',\n",
       " 'natural',\n",
       " 'languagegeneration',\n",
       " 'frequently',\n",
       " 'from',\n",
       " 'formal',\n",
       " ',',\n",
       " 'machine-readable',\n",
       " 'logical',\n",
       " 'forms',\n",
       " ')',\n",
       " ',',\n",
       " 'connecting',\n",
       " 'language',\n",
       " 'and',\n",
       " 'machine',\n",
       " 'perception',\n",
       " ',',\n",
       " 'managing',\n",
       " 'human-computer',\n",
       " 'dialog',\n",
       " 'systems',\n",
       " ',',\n",
       " 'or',\n",
       " 'some',\n",
       " 'combination',\n",
       " 'thereof',\n",
       " '.']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#word_tokenize(text1)\n",
    "word_tokenize(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5b18464f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Natural language processing (NLP) is a field of computer science, artificial intelligence and computational linguistics concerned with the interactions between computers and human (natural) languages, and, in particular, concerned with programming computers to fruitfully process large natural language corpora.',\n",
       " 'Challenges in natural language processing frequently involve natural language understanding, natural languagegeneration frequently from formal, machine-readable logical forms), connecting language and machine perception, managing human-computer dialog systems, or some combination thereof.']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_tokenize(text1)\n",
    "sent_tokenize(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9c45ece3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Natural',\n",
       " 'language',\n",
       " 'processing',\n",
       " '(',\n",
       " 'NLP',\n",
       " ')',\n",
       " 'is',\n",
       " 'a',\n",
       " 'field',\n",
       " 'of',\n",
       " 'computer',\n",
       " 'science',\n",
       " ',',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'and',\n",
       " 'computational',\n",
       " 'linguistics',\n",
       " 'concerned',\n",
       " 'with',\n",
       " 'the',\n",
       " 'interactions',\n",
       " 'between',\n",
       " 'computers',\n",
       " 'and',\n",
       " 'human',\n",
       " '(',\n",
       " 'natural',\n",
       " ')',\n",
       " 'languages',\n",
       " ',',\n",
       " 'and',\n",
       " ',',\n",
       " 'in',\n",
       " 'particular',\n",
       " ',',\n",
       " 'concerned',\n",
       " 'with',\n",
       " 'programming',\n",
       " 'computers',\n",
       " 'to',\n",
       " 'fruitfully',\n",
       " 'process',\n",
       " 'large',\n",
       " 'natural',\n",
       " 'language',\n",
       " 'corpora.',\n",
       " 'Challenges',\n",
       " 'in',\n",
       " 'natural',\n",
       " 'language',\n",
       " 'processing',\n",
       " 'frequently',\n",
       " 'involve',\n",
       " 'natural',\n",
       " 'language',\n",
       " 'understanding',\n",
       " ',',\n",
       " 'natural',\n",
       " 'languagegeneration',\n",
       " 'frequently',\n",
       " 'from',\n",
       " 'formal',\n",
       " ',',\n",
       " 'machine-readable',\n",
       " 'logical',\n",
       " 'forms',\n",
       " ')',\n",
       " ',',\n",
       " 'connecting',\n",
       " 'language',\n",
       " 'and',\n",
       " 'machine',\n",
       " 'perception',\n",
       " ',',\n",
       " 'managing',\n",
       " 'human-computer',\n",
       " 'dialog',\n",
       " 'systems',\n",
       " ',',\n",
       " 'or',\n",
       " 'some',\n",
       " 'combination',\n",
       " 'thereof',\n",
       " '.']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "TreebankWordTokenizer().tokenize(text2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f951741f",
   "metadata": {},
   "source": [
    "# The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.\n",
    "\n",
    "This tokenizer performs the following steps:\n",
    "\n",
    "split standard contractions, e.g. don't -> do n't and they'll -> they 'll\n",
    "\n",
    "treat most punctuation characters as separate tokens\n",
    "\n",
    "split off commas and single quotes, when followed by whitespace\n",
    "\n",
    "separate periods that appear at the end of line"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ed8aca0",
   "metadata": {},
   "source": [
    "# Tokenization using gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3e58c382",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-05-09T11:59:26.707244Z",
     "end_time": "2023-05-09T11:59:26.723239Z"
    }
   },
   "outputs": [],
   "source": [
    "#import gensim\n",
    "#from gensim.utils import tokenize\n",
    "#list(tokenize(text2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "72be2543",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [\n",
    "    'Today is a sunny day',\n",
    "    'today is a Windy Day',\n",
    "    'Is it sunny Today',\n",
    "    'I really enjoyed walking in the snow today'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ba30ea04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'today': 1, 'is': 2, 'a': 3, 'sunny': 4, 'day': 5, 'windy': 6, 'it': 7, 'i': 8, 'really': 9, 'enjoyed': 10, 'walking': 11, 'in': 12, 'the': 13, 'snow': 14}\n",
      "[[1, 2, 3, 4, 5], [1, 2, 3, 6, 5], [2, 7, 4, 1], [8, 9, 10, 11, 12, 13, 14, 1]]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words = 100)\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "sequences = tokenizer.texts_to_sequences(sentences)\n",
    "word_index = tokenizer.word_index\n",
    "print(word_index)\n",
    "print(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "13537436",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 5, 4, 8)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sequences[0]),len(sequences[1]),len(sequences[2]),len(sequences[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a71869e",
   "metadata": {},
   "source": [
    "# Let's Make Them into a Same Shape Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6da01b82",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  0  1  2  3  4  5]\n",
      " [ 0  0  0  1  2  3  6  5]\n",
      " [ 0  0  0  0  2  7  4  1]\n",
      " [ 8  9 10 11 12 13 14  1]]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "padded_seq = pad_sequences(sequences)\n",
    "print(padded_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "0e284ec1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, 8, 8, 8)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(padded_seq[0]),len(padded_seq[1]),len(padded_seq[2]),len(padded_seq[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fba79ee9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  2  3  4  5  0  0  0]\n",
      " [ 1  2  3  6  5  0  0  0]\n",
      " [ 2  7  4  1  0  0  0  0]\n",
      " [ 8  9 10 11 12 13 14  1]]\n"
     ]
    }
   ],
   "source": [
    "padded_seq = pad_sequences(sequences, padding = 'post', maxlen = 8, truncating = 'post')\n",
    "print(padded_seq)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3d79191",
   "metadata": {},
   "source": [
    "# Removing STOP words and CLEANING Text\n",
    "# Stopwords * Means\n",
    "### Stopwords are the English words which does not add much meaning to a sentence. They can safely be ignored without sacrificing the meaning of the sentence. For example, the words like the, he, have etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "03c9dd8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d4d0571b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
     ]
    }
   ],
   "source": [
    "print(stopwords.words('english')) # spanish, turkish, russian"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce3fcab2",
   "metadata": {},
   "source": [
    "# If these words are encountered ...... How to remove them using code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "441c2d88",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = 'please guys, i hate to tell you so many times that this is my life'\n",
    "words = sentence.split()\n",
    "stop_words = set(stopwords.words(\"english\"))\n",
    "type(words)\n",
    "filtered_sentence = \"\"\n",
    "for word in words:\n",
    "    if word not in stop_words:\n",
    "        filtered_sentence = filtered_sentence + word + \" \"\n",
    "sentences.append(filtered_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "143ced1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'please guys, i hate to tell you so many times that this is my life'"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "ee09a72e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'please guys, hate tell many times life '"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "fdb06a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize, wordpunct_tokenize, TweetTokenizer, MWETokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3be3e128",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}