Exp_1_Tokenization_of_text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exp_1: Text Tokenization using Different Tokenizers and describe their suitable situational importance based on the input text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.543710Z",
     "end_time": "2023-06-06T10:56:17.772739Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['I',\n 'love',\n 'NLP',\n 'class',\n ',',\n 'fear',\n '!',\n '#',\n 'Hope.Grade',\n '%',\n '10.0',\n '%',\n '&',\n '@',\n 'job']"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize,sent_tokenize,TreebankWordTokenizer\n",
    "from nltk.tokenize import wordpunct_tokenize,TweetTokenizer,MWETokenizer\n",
    "text = 'I love NLP class, fear! #Hope.Grade %10.0% & @job'\n",
    "word_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.555711Z",
     "end_time": "2023-06-06T10:56:17.961750Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['I',\n 'love',\n 'NLP',\n 'class',\n ',',\n 'fear',\n '!',\n '#',\n 'Hope.Grade',\n '%',\n '10.0',\n '%',\n '&',\n '@',\n 'job']"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = TreebankWordTokenizer()\n",
    "tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.570713Z",
     "end_time": "2023-06-06T10:56:17.963752Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['I', 'love', 'NLP', 'class,', 'fear!', '#Hope.Grade', '%10.0%', '&', '@job']"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Split on white spaces using the string function split()\n",
    "text.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.586713Z",
     "end_time": "2023-06-06T10:56:17.966732Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['I love NLP class', ' fear! #Hope.Grade %10.0% & @job']"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Tokenize on ','\n",
    "text.split(',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Identify the quality of the .split text segmenter??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.603715Z",
     "end_time": "2023-06-06T10:56:17.966732Z"
    }
   },
   "outputs": [],
   "source": [
    "tokens = wordpunct_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.994710Z",
     "end_time": "2023-06-06T10:56:18.010709Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', 'love', 'NLP', 'class', ',', 'fear', '!', '#', 'Hope', '.', 'Grade', '%', '10', '.', '0', '%', '&', '@', 'job']\n"
     ]
    }
   ],
   "source": [
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.700716Z",
     "end_time": "2023-06-06T10:56:18.044711Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<nltk.tokenize.casual.TweetTokenizer object at 0x000001F6224A4B80>\n"
     ]
    }
   ],
   "source": [
    "tokens = TweetTokenizer(text)\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Why this output is projected on your screen.?\n",
    "# TweetTokenizer() is a class and wordpucth_tokenizer is a built in function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.707710Z",
     "end_time": "2023-06-06T10:56:18.044711Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['I',\n 'love',\n 'NLP',\n 'class',\n ',',\n 'fear',\n '!',\n '#Hope',\n '.',\n 'Grade',\n '%',\n '10.0',\n '%',\n '&',\n '@job']"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = TweetTokenizer()\n",
    "tokenizer.tokenize(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tweet_Tokenizer is good only for tokenizing the tweets like sentences and may fail on large text models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.714711Z",
     "end_time": "2023-06-06T10:56:18.044711Z"
    }
   },
   "outputs": [],
   "source": [
    "import spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now lets learn a more efficient tokenizer model called : spaCY"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Go to :: https://spacy.io/usage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!pip install -U pip setuptools wheel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!pip install -U spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.725710Z",
     "end_time": "2023-06-06T10:56:18.060713Z"
    }
   },
   "outputs": [],
   "source": [
    "import spacy"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!python -m spacy download en_core_web_lg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "spacy.load(en_core_web_sm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.729710Z",
     "end_time": "2023-06-06T10:56:18.121750Z"
    }
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "#spacy.cli.download(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:17.736710Z",
     "end_time": "2023-06-06T10:56:18.566847Z"
    }
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.568847Z",
     "end_time": "2023-06-06T10:56:18.582845Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<spacy.lang.en.English at 0x1f6224a6cb0>"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.583845Z",
     "end_time": "2023-06-06T10:56:18.644986Z"
    }
   },
   "outputs": [],
   "source": [
    "doc = nlp(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.647988Z",
     "end_time": "2023-06-06T10:56:18.660989Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token 0: I\n",
      "Token 1: love\n",
      "Token 2: NLP\n",
      "Token 3: class\n",
      "Token 4: ,\n",
      "Token 5: fear\n",
      "Token 6: !\n",
      "Token 7: #\n",
      "Token 8: Hope\n",
      "Token 9: .\n",
      "Token 10: Grade\n",
      "Token 11: %\n",
      "Token 12: 10.0\n",
      "Token 13: %\n",
      "Token 14: &\n",
      "Token 15: @job\n"
     ]
    }
   ],
   "source": [
    "for i, token in enumerate(doc):\n",
    "    print(f\"Token {i}: {token.text}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load a pre-trained model for the English language using the spacy.load function.\n",
    "# The nlp object returns a Doc object that contains a list of Token objects, which represent the individual words or tokens in the sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.661992Z",
     "end_time": "2023-06-06T10:56:18.720731Z"
    }
   },
   "outputs": [],
   "source": [
    "text1 = 'My child loves @ Ice Cream'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.678279Z",
     "end_time": "2023-06-06T10:56:18.725586Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['My', 'child', 'loves', '@', 'Ice', 'Cream']"
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.692533Z",
     "end_time": "2023-06-06T10:56:18.726587Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['My', 'child', 'loves', '@', 'Ice', 'Cream']"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = TreebankWordTokenizer()\n",
    "tokenizer.tokenize(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.711572Z",
     "end_time": "2023-06-06T10:56:18.727589Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token 0: My\n",
      "Token 1: child\n",
      "Token 2: loves\n",
      "Token 3: @\n",
      "Token 4: Ice\n",
      "Token 5: Cream\n"
     ]
    }
   ],
   "source": [
    "doc1 = nlp(text1)\n",
    "for i, token in enumerate(doc1):\n",
    "    print(f\"Token {i}: {token.text}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.726587Z",
     "end_time": "2023-06-06T10:56:18.739593Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['My', 'child', 'loves', '@', 'Ice_Cream']\n"
     ]
    }
   ],
   "source": [
    "tokenizer = MWETokenizer()\n",
    "mwe = ['Ice','Cream']\n",
    "tokenizer.add_mwe(mwe)\n",
    "tokens = tokenizer.tokenize(text1.split())\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import the MWETokenizer class from nltk.tokenize\n",
    "# Define a multi-word expression to tokenize, which is \"Ice Cream\".\n",
    "# Create a MWETokenizer object and add the multi-word expression using its add_mwe method.\n",
    "# Input a text sentence to tokenize and pass it to the MWETokenizer object's tokenize method.\n",
    "# The method returns a list of tokens that includes \"Ice Cream\" as a single token."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform the tokens to numbers. Use Tensorflow. spaCy. and pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.741592Z",
     "end_time": "2023-06-06T10:56:18.831585Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.756588Z",
     "end_time": "2023-06-06T10:56:18.879587Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.773591Z",
     "end_time": "2023-06-06T10:56:18.889586Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "'I love NLP class, fear! #Hope.Grade %10.0% & @job'"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.788589Z",
     "end_time": "2023-06-06T10:56:18.890586Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.804585Z",
     "end_time": "2023-06-06T10:56:18.890586Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<keras.preprocessing.text.Tokenizer at 0x1f64610ece0>"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This class allows to vectorize a text corpus, by turning each text into either a sequence of integers\n",
    "\n",
    "## The Number of Words That it can Tokenize.\n",
    "\n",
    "#### the maximum number of words to keep, based on word frequency. Only the most common num_words-1 words will be kept.\n",
    "    # tf.keras.preprocessing.text.Tokenizer(num_words=None,\n",
    "    # filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
    "    # lower=True, split=' ', char_level=False, oov_token=None,\n",
    "    # document_count=0, **kwargs\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.818586Z",
     "end_time": "2023-06-06T10:56:18.890586Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer.fit_on_texts(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.836587Z",
     "end_time": "2023-06-06T10:56:18.891586Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'num_words': 100,\n 'filters': '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n 'lower': True,\n 'split': ' ',\n 'char_level': False,\n 'oov_token': None,\n 'document_count': 49,\n 'word_counts': '{\"i\": 1, \"l\": 3, \"o\": 3, \"v\": 1, \"e\": 4, \"n\": 1, \"p\": 2, \"c\": 1, \"a\": 3, \"s\": 2, \"f\": 1, \"r\": 2, \"h\": 1, \"g\": 1, \"d\": 1, \"1\": 1, \"0\": 2, \"j\": 1, \"b\": 1}',\n 'word_docs': '{\"i\": 1, \"l\": 3, \"o\": 3, \"v\": 1, \"e\": 4, \"n\": 1, \"p\": 2, \"c\": 1, \"a\": 3, \"s\": 2, \"f\": 1, \"r\": 2, \"h\": 1, \"g\": 1, \"d\": 1, \"1\": 1, \"0\": 2, \"j\": 1, \"b\": 1}',\n 'index_docs': '{\"9\": 1, \"2\": 3, \"3\": 3, \"10\": 1, \"1\": 4, \"11\": 1, \"5\": 2, \"12\": 1, \"4\": 3, \"6\": 2, \"13\": 1, \"7\": 2, \"14\": 1, \"15\": 1, \"16\": 1, \"17\": 1, \"8\": 2, \"18\": 1, \"19\": 1}',\n 'index_word': '{\"1\": \"e\", \"2\": \"l\", \"3\": \"o\", \"4\": \"a\", \"5\": \"p\", \"6\": \"s\", \"7\": \"r\", \"8\": \"0\", \"9\": \"i\", \"10\": \"v\", \"11\": \"n\", \"12\": \"c\", \"13\": \"f\", \"14\": \"h\", \"15\": \"g\", \"16\": \"d\", \"17\": \"1\", \"18\": \"j\", \"19\": \"b\"}',\n 'word_index': '{\"e\": 1, \"l\": 2, \"o\": 3, \"a\": 4, \"p\": 5, \"s\": 6, \"r\": 7, \"0\": 8, \"i\": 9, \"v\": 10, \"n\": 11, \"c\": 12, \"f\": 13, \"h\": 14, \"g\": 15, \"d\": 16, \"1\": 17, \"j\": 18, \"b\": 19}'}"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.851588Z",
     "end_time": "2023-06-06T10:56:18.989616Z"
    }
   },
   "outputs": [],
   "source": [
    "text = [\n",
    "    'I love NLP class, fear! #Hope.Grade %10.0% & @job'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.868587Z",
     "end_time": "2023-06-06T10:56:19.006587Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer.fit_on_texts(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.885587Z",
     "end_time": "2023-06-06T10:56:19.008588Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'num_words': 100,\n 'filters': '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n 'lower': True,\n 'split': ' ',\n 'char_level': False,\n 'oov_token': None,\n 'document_count': 75,\n 'word_counts': '{\"i\": 3, \"l\": 5, \"o\": 4, \"v\": 2, \"e\": 7, \"n\": 1, \"p\": 2, \"c\": 4, \"a\": 4, \"s\": 3, \"f\": 1, \"r\": 3, \"h\": 2, \"g\": 1, \"d\": 2, \"1\": 1, \"0\": 2, \"j\": 1, \"b\": 1, \"m\": 2, \"y\": 1}',\n 'word_docs': '{\"i\": 3, \"l\": 5, \"o\": 4, \"v\": 2, \"e\": 7, \"n\": 1, \"p\": 2, \"c\": 4, \"a\": 4, \"s\": 3, \"f\": 1, \"r\": 3, \"h\": 2, \"g\": 1, \"d\": 2, \"1\": 1, \"0\": 2, \"j\": 1, \"b\": 1, \"m\": 2, \"y\": 1}',\n 'index_docs': '{\"9\": 2, \"2\": 5, \"3\": 4, \"10\": 2, \"1\": 7, \"11\": 2, \"5\": 4, \"12\": 2, \"4\": 4, \"6\": 3, \"13\": 2, \"7\": 3, \"14\": 2, \"15\": 1, \"16\": 1, \"17\": 1, \"8\": 3, \"18\": 1, \"19\": 1, \"20\": 1, \"21\": 1}',\n 'index_word': '{\"1\": \"e\", \"2\": \"l\", \"3\": \"o\", \"4\": \"c\", \"5\": \"a\", \"6\": \"i\", \"7\": \"s\", \"8\": \"r\", \"9\": \"v\", \"10\": \"p\", \"11\": \"h\", \"12\": \"d\", \"13\": \"0\", \"14\": \"m\", \"15\": \"n\", \"16\": \"f\", \"17\": \"g\", \"18\": \"1\", \"19\": \"j\", \"20\": \"b\", \"21\": \"y\"}',\n 'word_index': '{\"e\": 1, \"l\": 2, \"o\": 3, \"c\": 4, \"a\": 5, \"i\": 6, \"s\": 7, \"r\": 8, \"v\": 9, \"p\": 10, \"h\": 11, \"d\": 12, \"0\": 13, \"m\": 14, \"n\": 15, \"f\": 16, \"g\": 17, \"1\": 18, \"j\": 19, \"b\": 20, \"y\": 21}'}"
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.901588Z",
     "end_time": "2023-06-06T10:56:19.009588Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'e': 1,\n 'l': 2,\n 'o': 3,\n 'c': 4,\n 'a': 5,\n 'i': 6,\n 's': 7,\n 'r': 8,\n 'v': 9,\n 'p': 10,\n 'h': 11,\n 'd': 12,\n '0': 13,\n 'm': 14,\n 'n': 15,\n 'f': 16,\n 'g': 17,\n '1': 18,\n 'j': 19,\n 'b': 20,\n 'y': 21}"
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.916587Z",
     "end_time": "2023-06-06T10:56:19.009588Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "'My child loves @ Ice Cream'"
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.931586Z",
     "end_time": "2023-06-06T10:56:19.009588Z"
    }
   },
   "outputs": [],
   "source": [
    "sen = [\n",
    "    'I am good human as i am with good dog'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.948588Z",
     "end_time": "2023-06-06T10:56:19.010626Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['I am good human as i am with good dog']"
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.962586Z",
     "end_time": "2023-06-06T10:56:19.010626Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.978587Z",
     "end_time": "2023-06-06T10:56:19.010626Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer.fit_on_texts(sen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:18.997595Z",
     "end_time": "2023-06-06T10:56:19.011588Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'i': 1, 'am': 2, 'good': 3, 'human': 4, 'as': 5, 'with': 6, 'dog': 7}"
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.013588Z",
     "end_time": "2023-06-06T10:56:19.164627Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'num_words': 100,\n 'filters': '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n 'lower': True,\n 'split': ' ',\n 'char_level': False,\n 'oov_token': None,\n 'document_count': 1,\n 'word_counts': '{\"i\": 2, \"am\": 2, \"good\": 2, \"human\": 1, \"as\": 1, \"with\": 1, \"dog\": 1}',\n 'word_docs': '{\"am\": 1, \"i\": 1, \"with\": 1, \"dog\": 1, \"good\": 1, \"as\": 1, \"human\": 1}',\n 'index_docs': '{\"2\": 1, \"1\": 1, \"6\": 1, \"7\": 1, \"3\": 1, \"5\": 1, \"4\": 1}',\n 'index_word': '{\"1\": \"i\", \"2\": \"am\", \"3\": \"good\", \"4\": \"human\", \"5\": \"as\", \"6\": \"with\", \"7\": \"dog\"}',\n 'word_index': '{\"i\": 1, \"am\": 2, \"good\": 3, \"human\": 4, \"as\": 5, \"with\": 6, \"dog\": 7}'}"
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.025587Z",
     "end_time": "2023-06-06T10:56:19.189587Z"
    }
   },
   "outputs": [],
   "source": [
    "Sen1 = [\n",
    "    'Today is a sunny day',\n",
    "    'Today is a autum day',\n",
    "    'what is it like today'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.042588Z",
     "end_time": "2023-06-06T10:56:19.190628Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['Today is a sunny day', 'Today is a autum day', 'what is it like today']"
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Sen1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.059585Z",
     "end_time": "2023-06-06T10:56:19.190628Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=100)\n",
    "tokenizer.fit_on_texts(Sen1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.073589Z",
     "end_time": "2023-06-06T10:56:19.190628Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'num_words': 100,\n 'filters': '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n 'lower': True,\n 'split': ' ',\n 'char_level': False,\n 'oov_token': None,\n 'document_count': 3,\n 'word_counts': '{\"today\": 3, \"is\": 3, \"a\": 2, \"sunny\": 1, \"day\": 2, \"autum\": 1, \"what\": 1, \"it\": 1, \"like\": 1}',\n 'word_docs': '{\"is\": 3, \"today\": 3, \"a\": 2, \"day\": 2, \"sunny\": 1, \"autum\": 1, \"it\": 1, \"like\": 1, \"what\": 1}',\n 'index_docs': '{\"2\": 3, \"1\": 3, \"3\": 2, \"4\": 2, \"5\": 1, \"6\": 1, \"8\": 1, \"9\": 1, \"7\": 1}',\n 'index_word': '{\"1\": \"today\", \"2\": \"is\", \"3\": \"a\", \"4\": \"day\", \"5\": \"sunny\", \"6\": \"autum\", \"7\": \"what\", \"8\": \"it\", \"9\": \"like\"}',\n 'word_index': '{\"today\": 1, \"is\": 2, \"a\": 3, \"day\": 4, \"sunny\": 5, \"autum\": 6, \"what\": 7, \"it\": 8, \"like\": 9}'}"
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.090588Z",
     "end_time": "2023-06-06T10:56:19.192624Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'today': 1,\n 'is': 2,\n 'a': 3,\n 'day': 4,\n 'sunny': 5,\n 'autum': 6,\n 'what': 7,\n 'it': 8,\n 'like': 9}"
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How Tensorflow eliminates special characters such as @, $ etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.106586Z",
     "end_time": "2023-06-06T10:56:19.192624Z"
    }
   },
   "outputs": [],
   "source": [
    "sen_2 = [\n",
    "    'I am good at making $ money',\n",
    "    'My dog is * cat'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.120587Z",
     "end_time": "2023-06-06T10:56:19.192624Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer_1 = Tokenizer(num_words=100, filters = '')\n",
    "tokenizer_1.fit_on_texts(sen_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.137588Z",
     "end_time": "2023-06-06T10:56:19.193625Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'i': 1,\n 'am': 2,\n 'good': 3,\n 'at': 4,\n 'making': 5,\n '$': 6,\n 'money': 7,\n 'my': 8,\n 'dog': 9,\n 'is': 10,\n '*': 11,\n 'cat': 12}"
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer_1.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.153593Z",
     "end_time": "2023-06-06T10:56:19.193625Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer_2 = Tokenizer(num_words=100, filters = '$*')\n",
    "tokenizer_2.fit_on_texts(sen_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.170590Z",
     "end_time": "2023-06-06T10:56:19.193625Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'i': 1,\n 'am': 2,\n 'good': 3,\n 'at': 4,\n 'making': 5,\n 'money': 6,\n 'my': 7,\n 'dog': 8,\n 'is': 9,\n 'cat': 10}"
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer_2.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.184588Z",
     "end_time": "2023-06-06T10:56:19.338631Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 2, 3, 5, 4], [1, 2, 3, 6, 4], [7, 2, 8, 9, 1]]\n"
     ]
    }
   ],
   "source": [
    "sequences = tokenizer.texts_to_sequences(Sen1)\n",
    "print(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.202587Z",
     "end_time": "2023-06-06T10:56:19.353625Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'today': 1,\n 'is': 2,\n 'a': 3,\n 'day': 4,\n 'sunny': 5,\n 'autum': 6,\n 'what': 7,\n 'it': 8,\n 'like': 9}"
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-06-06T10:56:19.215590Z",
     "end_time": "2023-06-06T10:56:19.370658Z"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}