3. NLP with HuggingFace.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "906e5726",
   "metadata": {},
   "source": [
    "<div style=\"width: 100%; overflow: hidden;\">\n",
    "    <div style=\"width: 150px; float: left;\"> <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_ball.png\" alt=\"Data For Science, Inc\" align=\"left\" border=\"0\" width=150px> </div>\n",
    "    <div style=\"float: left; margin-left: 10px;\"> <h1>LLMs for Data Science</h1>\n",
    "<h1>NLP With HuggingFace</h1>\n",
    "        <p>Bruno Gonçalves<br/>\n",
    "        <a href=\"http://www.data4sci.com/\">www.data4sci.com</a><br/>\n",
    "            @bgoncalves, @data4sci</p></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1e8a8a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from pprint import pprint\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "from ipywidgets import interact\n",
    "\n",
    "import transformers\n",
    "from transformers import pipeline\n",
    "from transformers import set_seed\n",
    "set_seed(42) # Set the seed to get reproducible results\n",
    "\n",
    "import os\n",
    "import gzip\n",
    "\n",
    "import tqdm as tq\n",
    "from tqdm.notebook import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "import networkx as nx\n",
    "\n",
    "import watermark\n",
    "\n",
    "%load_ext watermark\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9db5c34",
   "metadata": {},
   "source": [
    "We start by printing out the versions of the libraries we're using for future reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b618954",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.11.7\n",
      "IPython version      : 8.12.3\n",
      "\n",
      "Compiler    : Clang 14.0.6 \n",
      "OS          : Darwin\n",
      "Release     : 23.6.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 16\n",
      "Architecture: 64bit\n",
      "\n",
      "Git hash: 8d244e1e4f0c6fd330052d22607886f6abfcd26c\n",
      "\n",
      "pandas      : 2.2.3\n",
      "tqdm        : 4.66.4\n",
      "transformers: 4.41.1\n",
      "watermark   : 2.4.3\n",
      "numpy       : 1.26.4\n",
      "networkx    : 3.3\n",
      "matplotlib  : 3.8.0\n",
      "json        : 2.0.9\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -n -v -m -g -iv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ed44f70",
   "metadata": {},
   "source": [
    "Load default figure style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "07714ec8",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('d4sci.mplstyle')\n",
    "colors = plt.rcParams['axes.prop_cycle'].by_key()['color']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31f073b2",
   "metadata": {},
   "source": [
    "# Named Entity Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "174603ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "email = \"\"\"Dear Amazon, \\\n",
    "\n",
    "last week I ordered an Optimus Prime action figure \\\n",
    "from your online store in Germany. Unfortunately, when I opened the package, \\\n",
    "I discovered to my horror that I had been sent an action figure of Megatron \\\n",
    "instead! As a lifelong enemy of the Decepticons, I hope you can understand my \\\n",
    "dilemma. To resolve the issue, I demand an exchange of Megatron for the \\\n",
    "Optimus Prime figure I ordered. Enclosed are copies of my records concerning \\\n",
    "this purchase. I expect to hear from you soon. \n",
    "\n",
    "Sincerely, \n",
    "\n",
    "Bumblebee.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "81726cac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to dbmdz/bert-large-cased-finetuned-conll03-english and revision f2482bf (https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
      "/opt/anaconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Some weights of the model checkpoint at dbmdz/bert-large-cased-finetuned-conll03-english were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "ner_tagger = pipeline(\"ner\", aggregation_strategy=\"simple\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "04c9b4e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = ner_tagger(email)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "de8f662c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'ORG',\n",
       "  'score': 0.8790102,\n",
       "  'word': 'Amazon',\n",
       "  'start': 5,\n",
       "  'end': 11},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.9908588,\n",
       "  'word': 'Optimus Prime',\n",
       "  'start': 37,\n",
       "  'end': 50},\n",
       " {'entity_group': 'LOC',\n",
       "  'score': 0.9997547,\n",
       "  'word': 'Germany',\n",
       "  'start': 91,\n",
       "  'end': 98},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.5565716,\n",
       "  'word': 'Mega',\n",
       "  'start': 209,\n",
       "  'end': 213},\n",
       " {'entity_group': 'PER',\n",
       "  'score': 0.59025526,\n",
       "  'word': '##tron',\n",
       "  'start': 213,\n",
       "  'end': 217},\n",
       " {'entity_group': 'ORG',\n",
       "  'score': 0.66969275,\n",
       "  'word': 'Decept',\n",
       "  'start': 254,\n",
       "  'end': 260},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.4983484,\n",
       "  'word': '##icons',\n",
       "  'start': 260,\n",
       "  'end': 265},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.7753625,\n",
       "  'word': 'Megatron',\n",
       "  'start': 351,\n",
       "  'end': 359},\n",
       " {'entity_group': 'MISC',\n",
       "  'score': 0.98785394,\n",
       "  'word': 'Optimus Prime',\n",
       "  'start': 368,\n",
       "  'end': 381},\n",
       " {'entity_group': 'PER',\n",
       "  'score': 0.8120968,\n",
       "  'word': 'Bumblebee',\n",
       "  'start': 507,\n",
       "  'end': 516}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cf4e4c45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_group</th>\n",
       "      <th>score</th>\n",
       "      <th>word</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ORG</td>\n",
       "      <td>0.879010</td>\n",
       "      <td>Amazon</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.990859</td>\n",
       "      <td>Optimus Prime</td>\n",
       "      <td>37</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LOC</td>\n",
       "      <td>0.999755</td>\n",
       "      <td>Germany</td>\n",
       "      <td>91</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.556572</td>\n",
       "      <td>Mega</td>\n",
       "      <td>209</td>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PER</td>\n",
       "      <td>0.590255</td>\n",
       "      <td>##tron</td>\n",
       "      <td>213</td>\n",
       "      <td>217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ORG</td>\n",
       "      <td>0.669693</td>\n",
       "      <td>Decept</td>\n",
       "      <td>254</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.498348</td>\n",
       "      <td>##icons</td>\n",
       "      <td>260</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.775362</td>\n",
       "      <td>Megatron</td>\n",
       "      <td>351</td>\n",
       "      <td>359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MISC</td>\n",
       "      <td>0.987854</td>\n",
       "      <td>Optimus Prime</td>\n",
       "      <td>368</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>PER</td>\n",
       "      <td>0.812097</td>\n",
       "      <td>Bumblebee</td>\n",
       "      <td>507</td>\n",
       "      <td>516</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  entity_group     score           word  start  end\n",
       "0          ORG  0.879010         Amazon      5   11\n",
       "1         MISC  0.990859  Optimus Prime     37   50\n",
       "2          LOC  0.999755        Germany     91   98\n",
       "3         MISC  0.556572           Mega    209  213\n",
       "4          PER  0.590255         ##tron    213  217\n",
       "5          ORG  0.669693         Decept    254  260\n",
       "6         MISC  0.498348        ##icons    260  265\n",
       "7         MISC  0.775362       Megatron    351  359\n",
       "8         MISC  0.987854  Optimus Prime    368  381\n",
       "9          PER  0.812097      Bumblebee    507  516"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(outputs)    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35c65136",
   "metadata": {},
   "source": [
    "# PoS Tagging"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7f77c61",
   "metadata": {},
   "source": [
    "Load the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "451b2c97",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at vblagoje/bert-english-uncased-finetuned-pos were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "pos_tagger = pipeline(\"token-classification\", model=\"vblagoje/bert-english-uncased-finetuned-pos\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3cba5ca2",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"The quick brown fox jumps over the lazy dog.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04e817ee",
   "metadata": {},
   "source": [
    "Extract the part of speech tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c804d3c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity</th>\n",
       "      <th>score</th>\n",
       "      <th>index</th>\n",
       "      <th>word</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DET</td>\n",
       "      <td>0.999445</td>\n",
       "      <td>1</td>\n",
       "      <td>the</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADJ</td>\n",
       "      <td>0.997063</td>\n",
       "      <td>2</td>\n",
       "      <td>quick</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ADJ</td>\n",
       "      <td>0.942299</td>\n",
       "      <td>3</td>\n",
       "      <td>brown</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NOUN</td>\n",
       "      <td>0.997004</td>\n",
       "      <td>4</td>\n",
       "      <td>fox</td>\n",
       "      <td>16</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>VERB</td>\n",
       "      <td>0.999446</td>\n",
       "      <td>5</td>\n",
       "      <td>jumps</td>\n",
       "      <td>20</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ADP</td>\n",
       "      <td>0.999325</td>\n",
       "      <td>6</td>\n",
       "      <td>over</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DET</td>\n",
       "      <td>0.999527</td>\n",
       "      <td>7</td>\n",
       "      <td>the</td>\n",
       "      <td>31</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ADJ</td>\n",
       "      <td>0.997863</td>\n",
       "      <td>8</td>\n",
       "      <td>lazy</td>\n",
       "      <td>35</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NOUN</td>\n",
       "      <td>0.998858</td>\n",
       "      <td>9</td>\n",
       "      <td>dog</td>\n",
       "      <td>40</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>PUNCT</td>\n",
       "      <td>0.999650</td>\n",
       "      <td>10</td>\n",
       "      <td>.</td>\n",
       "      <td>43</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  entity     score  index   word  start  end\n",
       "0    DET  0.999445      1    the      0    3\n",
       "1    ADJ  0.997063      2  quick      4    9\n",
       "2    ADJ  0.942299      3  brown     10   15\n",
       "3   NOUN  0.997004      4    fox     16   19\n",
       "4   VERB  0.999446      5  jumps     20   25\n",
       "5    ADP  0.999325      6   over     26   30\n",
       "6    DET  0.999527      7    the     31   34\n",
       "7    ADJ  0.997863      8   lazy     35   39\n",
       "8   NOUN  0.998858      9    dog     40   43\n",
       "9  PUNCT  0.999650     10      .     43   44"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_tags = pos_tagger(text)\n",
    "pd.DataFrame(pos_tags)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bce39fd",
   "metadata": {},
   "source": [
    "# Summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e34f076f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to sshleifer/distilbart-cnn-12-6 and revision a4f8f3e (https://huggingface.co/sshleifer/distilbart-cnn-12-6).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
      "/opt/anaconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "summarizer = pipeline(\"summarization\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "082ab78e",
   "metadata": {},
   "source": [
    "The first 4 paragraphs of https://en.wikipedia.org/wiki/Transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0e5770f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki_text = \"\"\"\n",
    "Transformers is a media franchise produced by American toy company Hasbro and Japanese toy company Takara Tomy. It primarily follows the heroic Autobots and the villainous Decepticons, two alien robot factions at war that can transform into other forms, such as vehicles and animals. The franchise encompasses toys, animation, comic books, video games and films. As of 2011, it generated more than ¥2 trillion ($25 billion) in revenue,[1] making it one of the highest-grossing media franchises of all time.\n",
    "\n",
    "The franchise began in 1984 with the Transformers toy line, comprising transforming mecha toys from Takara's Diaclone and Micro Change toylines rebranded for Western markets.[2] The term \"Generation 1\" (G1) covers both the animated television series The Transformers and the comic book series of the same name, which are further divided into Japanese, British and Canadian spin-offs. Sequels followed, such as the Generation 2 comic book and Beast Wars TV series, which became its own mini-universe. Generation 1 characters have been rebooted multiple times in the 21st century in comics from Dreamwave Productions (starting 2001), IDW Publishing (starting in 2005 and again in 2019), and Skybound Entertainment (beginning in 2023). There have been other incarnations of the story based on different toy lines during and after the 20th century. The first was the Robots in Disguise series, followed by three shows (Armada, Energon, and Cybertron) that constitute a single universe called the \"Unicron Trilogy\".\n",
    "\n",
    "A live-action film series started in 2007, again distinct from previous incarnations, while the Transformers: Animated series merged concepts from the G1 continuity, the 2007 live-action film and the \"Unicron Trilogy\". For most of the 2010s, in an attempt to mitigate the wave of reboots, the \"Aligned Continuity\" was established. In 2018, Transformers: Cyberverse debuted, once again, distinct from the previous incarnations.\n",
    "\n",
    "Although a separate and competing franchise started in 1983, Tonka's GoBots became the intellectual property of Hasbro after their buyout of Tonka in 1991. Subsequently, the universe depicted in the animated series Challenge of the GoBots and follow-up film GoBots: Battle of the Rock Lords was retroactively established as an alternate universe within the Transformers multiverse.[3] \n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca06906d",
   "metadata": {},
   "source": [
    "To generate the summary we just have to call the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "91867291",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The Transformers is a media franchise produced by Hasbro and Japanese toy company Takara Tomy . It primarily follows the heroic Autobots and the villainous Decepticons, two alien robot factions at war that can transform into other forms, such as vehicles and animals . As of 2011, it generated more than ¥2 trillion ($25 billion) in revenue .\n"
     ]
    }
   ],
   "source": [
    "summary = summarizer(wiki_text)\n",
    "\n",
    "print(summary[0]['summary_text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c6fb20a",
   "metadata": {},
   "source": [
    "We can also specify a minimum length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "85529db7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Transformers is a media franchise produced by Hasbro and Japanese toy company Takara Tomy . It primarily follows the heroic Autobots and the villainous Decepticons, two alien robot factions at war that can transform into other forms, such as vehicles and animals . As of 2011, it generated more than ¥2 trillion ($25 billion) in revenue, making it one of the highest-grossing media franchises of all time . The term \"Generation 1\" (G1) covers both the animated television series The Transformers and the comic book series of the same name .\n"
     ]
    }
   ],
   "source": [
    "summary = summarizer(wiki_text, min_length=100)\n",
    "\n",
    "print(summary[0]['summary_text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "136e9c7a",
   "metadata": {},
   "source": [
    "# Question Answering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1ad2db3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert/distilbert-base-cased-distilled-squad and revision 626af31 (https://huggingface.co/distilbert/distilbert-base-cased-distilled-squad).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    }
   ],
   "source": [
    "reader = pipeline(\"question-answering\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "84e9e249",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"What does the customer want?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0583e6a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.631292</td>\n",
       "      <td>336</td>\n",
       "      <td>359</td>\n",
       "      <td>an exchange of Megatron</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      score  start  end                   answer\n",
       "0  0.631292    336  359  an exchange of Megatron"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs = reader(question=question, context=email)\n",
    "pd.DataFrame([outputs])    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "133eb742",
   "metadata": {},
   "source": [
    "# Translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1f1e3faf",
   "metadata": {},
   "outputs": [],
   "source": [
    "translator = pipeline(\"translation_en_to_it\", \n",
    "                      model=\"Helsinki-NLP/opus-mt-en-it\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4f7f4ecb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cara Amazon, la scorsa settimana ho ordinato una figura d'azione Optimus Prime dal tuo negozio online in Germania. Purtroppo, quando ho aperto il pacchetto, ho scoperto al mio orrore che ero stato inviato una figura d'azione di Megatron invece! Come un nemico per tutta la vita dei Decepticon, spero che si può capire il mio dilemma. Per risolvere il problema, chiedo uno scambio di Megatron per la figura di Optimus Prime ho ordinato. In allegato sono copie dei miei record riguardanti questo acquisto. Mi aspetto di sentire da voi presto. Cordialmente, Bumblebee.\n"
     ]
    }
   ],
   "source": [
    "outputs = translator(email, clean_up_tokenization_spaces=True, min_length=100, max_length=1000)\n",
    "print(outputs[0]['translation_text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4517dbd1",
   "metadata": {},
   "source": [
    "For comparison, let us look at the results of google translate:\n",
    "\n",
    "```\n",
    "Caro Amazon, la settimana scorsa ho ordinato un action figure di Optimus Prime dal tuo negozio online in Germania. Sfortunatamente, quando ho aperto il pacco, ho scoperto con orrore che mi era stata invece inviata una action figure di Megatron! Essendo un nemico da sempre dei Decepticon, spero che tu possa capire il mio dilemma. Per risolvere il problema, chiedo uno scambio di Megatron con la figura di Optimus Prime che ho ordinato. In allegato sono presenti copie dei miei documenti relativi a questo acquisto. Mi aspetto di sentirti presto. Cordiali saluti, Bombo.\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "570fc45e",
   "metadata": {},
   "source": [
    "Google translate is less context aware in the translation going so far as translating the name of the email sender (Bumblebee -> Bombo). On the other hand, the Hugging Face model is more formal (\"sentire da voi\" -> \"sentirti\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09ff6868",
   "metadata": {},
   "source": [
    "# Sentiment Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0e0b3c7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert/distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
      "/opt/anaconda3/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "sentiment_pipeline = pipeline(\"sentiment-analysis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c9a643a",
   "metadata": {},
   "source": [
    "Let us use a comple of faily obvious instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6d2ebc6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = [\"I love you\", \"I hate you\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a91d8fc",
   "metadata": {},
   "source": [
    "The model does a pretty good job of figuring out which one is positive and which one is negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1086a85d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'POSITIVE', 'score': 0.9998656511306763},\n",
       " {'label': 'NEGATIVE', 'score': 0.9991129040718079}]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_pipeline(instances)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18f7a8f3",
   "metadata": {},
   "source": [
    "# Application"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c208bd71",
   "metadata": {},
   "source": [
    "Load a few thousand tweets about Apple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4f399bbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('data/Apple-Twitter-Sentiment-DFE.csv', usecols=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1f6b8b22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>#AAPL:The 10 best Steve Jobs emails ever...htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RT @JPDesloges: Why AAPL Stock Had a Mini-Flas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My cat only chews @apple cords. Such an #Apple...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I agree with @jimcramer that the #IndividualIn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nobody expects the Spanish Inquisition #AAPL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3881</th>\n",
       "      <td>(Via FC) Apple Is Warming Up To Social Media -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3882</th>\n",
       "      <td>RT @MMLXIV: there is no avocado emoji may I as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3883</th>\n",
       "      <td>@marcbulandr I could not agree more. Between @...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3884</th>\n",
       "      <td>My iPhone 5's photos are no longer downloading...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885</th>\n",
       "      <td>RT @SwiftKey: We're so excited to be named to ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3886 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     #AAPL:The 10 best Steve Jobs emails ever...htt...\n",
       "1     RT @JPDesloges: Why AAPL Stock Had a Mini-Flas...\n",
       "2     My cat only chews @apple cords. Such an #Apple...\n",
       "3     I agree with @jimcramer that the #IndividualIn...\n",
       "4          Nobody expects the Spanish Inquisition #AAPL\n",
       "...                                                 ...\n",
       "3881  (Via FC) Apple Is Warming Up To Social Media -...\n",
       "3882  RT @MMLXIV: there is no avocado emoji may I as...\n",
       "3883  @marcbulandr I could not agree more. Between @...\n",
       "3884  My iPhone 5's photos are no longer downloading...\n",
       "3885  RT @SwiftKey: We're so excited to be named to ...\n",
       "\n",
       "[3886 rows x 1 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bcefcea",
   "metadata": {},
   "source": [
    "Compute the sentiment score for each tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d68c6dd0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7aa956de522049faa7822a06e1d0059f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3886 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sent = pd.DataFrame(data['text'].progress_apply(lambda x: pd.Series(sentiment_pipeline(x)[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "bd855ac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent.rename(columns={'score': 'sentiment_confidence', 'label':'sentiment'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "bb982bd3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sentiment_confidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.999432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.999122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.996177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.995648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.932676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3881</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.992046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3882</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.999158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3883</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.935773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3884</th>\n",
       "      <td>NEGATIVE</td>\n",
       "      <td>0.998303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3885</th>\n",
       "      <td>POSITIVE</td>\n",
       "      <td>0.998714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3886 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     sentiment  sentiment_confidence\n",
       "0     POSITIVE              0.999432\n",
       "1     NEGATIVE              0.999122\n",
       "2     NEGATIVE              0.996177\n",
       "3     POSITIVE              0.995648\n",
       "4     NEGATIVE              0.932676\n",
       "...        ...                   ...\n",
       "3881  NEGATIVE              0.992046\n",
       "3882  NEGATIVE              0.999158\n",
       "3883  NEGATIVE              0.935773\n",
       "3884  NEGATIVE              0.998303\n",
       "3885  POSITIVE              0.998714\n",
       "\n",
       "[3886 rows x 2 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44920c8d",
   "metadata": {},
   "source": [
    "We can also use NER to identify when a person is mentioned in the tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "23459450",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'#AAPL:The 10 best Steve Jobs emails ever...http://t.co/82G1kL94tx'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "cfd86573",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'PER',\n",
       "  'score': 0.73089933,\n",
       "  'word': 'Steve Jobs',\n",
       "  'start': 18,\n",
       "  'end': 28}]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_tagger(data['text'].iloc[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "770f271d",
   "metadata": {},
   "source": [
    "Identify all people mentioned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "bba49438",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_people(x):\n",
    "    output = ner_tagger(x)\n",
    "    \n",
    "    for tag in output:\n",
    "        if tag['entity_group'] == 'PER':\n",
    "            out = {'confidence':tag[\"score\"], 'person': tag['word']}\n",
    "            return pd.Series(out)\n",
    "    \n",
    "    return pd.Series({\"confidence\": None, \"person\": None})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b50ff65a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57c92ad861d349f38553451a5008350b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3886 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "people = pd.DataFrame(data['text'].progress_apply(find_people))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cd468a2",
   "metadata": {},
   "source": [
    "Combine all the results into a single DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "032d936b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([data, sent, people], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "85d2d061",
   "metadata": {},
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f71de36",
   "metadata": {},
   "source": [
    "Subset the data to only the tweets meantioning people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d20a8348",
   "metadata": {},
   "outputs": [],
   "source": [
    "people = data[data.person.isna() == False].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "90bca6cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "people"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94d97411",
   "metadata": {},
   "source": [
    "Convert the text labels to a numerical score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1b0ff5c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "people['sentiment'] = people.apply(lambda x: 1 if x.sentiment == 'POSITIVE' else -1, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa85d00d",
   "metadata": {},
   "source": [
    "Compute the average score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f9c033c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = people[['person', 'sentiment']].groupby('person').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e39a62d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = people[['person', 'sentiment']].groupby('person').count()\n",
    "counts.rename(columns={'sentiment':'count'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d6839a91",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = stats.join(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "901b0edf",
   "metadata": {},
   "outputs": [],
   "source": [
    "stats[stats['count']>=5].sort_values('sentiment', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b19edffd",
   "metadata": {},
   "source": [
    "<center>\n",
    "     <img src=\"https://raw.githubusercontent.com/DataForScience/Networks/master/data/D4Sci_logo_full.png\" alt=\"Data For Science, Inc\" align=\"center\" border=\"0\" width=300px> \n",
    "</center>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}