Chapter 3 - Text and Speech Basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nNAeU5iO8dJU"
   },
   "source": [
    "# Chapter 3 - Text and Speech Basics\n",
    "Let's start by downloading and loading the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 51,
     "output_extras": [
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6052,
     "status": "ok",
     "timestamp": 1522095458666,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "cgkJ5KLxBKTX",
    "outputId": "70427d37-d15f-4a04-c91e-e2d85a7e1b00"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "  \n",
    "filelist = ['reuters-000.json',\n",
    "            'reuters-001.json',\n",
    "            'reuters-002.json',\n",
    "            'reuters-003.json',\n",
    "            'reuters-004.json',\n",
    "            'reuters-005.json',\n",
    "            'reuters-006.json',\n",
    "            'reuters-007.json',\n",
    "            'reuters-008.json',\n",
    "            'reuters-009.json',\n",
    "            'reuters-010.json',\n",
    "            'reuters-011.json',\n",
    "            'reuters-012.json',\n",
    "            'reuters-013.json',\n",
    "            'reuters-014.json',\n",
    "            'reuters-015.json',\n",
    "            'reuters-016.json',\n",
    "            'reuters-017.json',\n",
    "            'reuters-018.json',\n",
    "            'reuters-019.json',\n",
    "            'reuters-020.json',\n",
    "            'reuters-021.json']\n",
    "data_set = pd.DataFrame()\n",
    "for fname in filelist:\n",
    "  df = pd.read_json('data/reuters-21578-json/'+fname).T\n",
    "  data_set = data_set.append(df)\n",
    "  \n",
    "data_set = data_set.fillna('')\n",
    "print(\"Done!\")         "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FKmQH6boVu8-"
   },
   "source": [
    "# **Exploratory Data Analysis**\n",
    "\n",
    "Our first task is to take a close look at the dataset by loading and performing ex-ploratory data analysis. To do so, we must extracting metadata and the text bodyfrom each document in the corpus. If we take a close look at the corpus, we find some interesting facts. Run the next three cells to visualize the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 525,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 627,
     "status": "ok",
     "timestamp": 1522085909755,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "qsav_oKRB7Kj",
    "outputId": "709a604e-f2a9-4d65-96c9-f49349a13760"
   },
   "outputs": [],
   "source": [
    "topics = set([x for sublist in data_set[u'topics'].values.tolist() for x in sublist])\n",
    "places = set([x for sublist in data_set[u'places'].values.tolist() for x in sublist])\n",
    "orgs = set([x for sublist in data_set[u'organisations'].values.tolist() for x in sublist])\n",
    "\n",
    "data_set['topic_count']=data_set[u'topics'].apply(lambda x:len([y for y in x]))\n",
    "print(\"documents with at least one topic = \",len(data_set[data_set[u'topic_count']>0]))\n",
    "print(\"max number of topics in one document = \",data_set[u'topic_count'].max())\n",
    "print('topics = ',len(topics))\n",
    "print('places = ',len(places))\n",
    "print('organizations = ',len(orgs))\n",
    "\n",
    "data_set[u'organisations'].apply(tuple).value_counts()[1:15].plot(kind='bar',title=\"Document Count by Organization\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 379,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 455,
     "status": "ok",
     "timestamp": 1522085912770,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "aQj5pgTbOtdO",
    "outputId": "0507c6d5-69cc-4aee-8da9-bf2ababb0ae8"
   },
   "outputs": [],
   "source": [
    "data_set[u'places'].apply(tuple).value_counts()[2:15].sort_values().plot(kind='barh',title='Document Count by Places outside US')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 379,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 455,
     "status": "ok",
     "timestamp": 1522085914770,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "ZsgdREcKQt0H",
    "outputId": "5267b016-6dc9-4527-d602-83bf8dfa593f"
   },
   "outputs": [],
   "source": [
    "data_set[u'topics'].apply(tuple).value_counts()[1:15].sort_values().plot(kind='barh',title='Document Count by Topic')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GFe5Moe9VMBx"
   },
   "source": [
    "So far so good. But before we perform any NLP analysis, we will want to perform some text normalization:\n",
    "\n",
    "*   transform to lower case\n",
    "*   remove punctuation & numbers\n",
    "*   stem verbs\n",
    "*   remove stopwords\n",
    "\n",
    "To do so, we define a SimpleTokenizer method that will be useful when creating document representations. For our analysis, we will analyze documents involving five topics (crude, trade, money-fx, coffee, and gold).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 102,
     "output_extras": [
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5208,
     "status": "ok",
     "timestamp": 1522085922365,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "7HTNc1R4WW8T",
    "outputId": "7ff6264f-6fd0-405c-e6e7-20d46f8a8ebb"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.preprocessing.label import MultiLabelBinarizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "nltk.download(\"punkt\")\n",
    "nltk.download(\"stopwords\",\"data\")\n",
    "nltk.data.path.append('data')\n",
    "\n",
    "labelBinarizer = MultiLabelBinarizer()\n",
    "data_target = labelBinarizer.fit_transform(data_set[u'topics'])\n",
    "\n",
    "stopWords = stopwords.words('english')\n",
    "charfilter = re.compile('[a-zA-Z]+');\n",
    "\n",
    "def SimpleTokenizer(text):\n",
    "  words = map(lambda word: word.lower(), word_tokenize(text))\n",
    "  words = [word for word in words if word not in stopWords]\n",
    "  tokens = (list(map(lambda token: PorterStemmer().stem(token),words)))\n",
    "  ntokens = list(filter(lambda token:charfilter.match(token),tokens))\n",
    "  return ntokens\n",
    "\n",
    "vec = TfidfVectorizer(tokenizer=SimpleTokenizer,\n",
    "                        max_features=1000,\n",
    "                        norm='l2')\n",
    "\n",
    "\n",
    "mytopics = [u'cocoa',u'trade',u'money-supply',u'coffee',u'gold']\n",
    "data_set = data_set[data_set[u'topics'].map(set(mytopics).intersection).apply( lambda x: len(x)>0 )]\n",
    "docs = list(data_set[u'body'].values)\n",
    "\n",
    "dtm = vec.fit_transform(docs)\n",
    "\n",
    "print(\"Number of documents with my topics = \",len(data_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "g8wQre5RTlTa"
   },
   "source": [
    "# **Text Clustering**\n",
    "\n",
    "We want to see if clusters exist in the documents, so let’s create some document representations through TFIDF. This gives us a document term matrix, but typically the dimensions of this matrix is too large and the representations are sparse. Let's first apply Principal Component Analysis (PCA) to reduce the dimensionality. The original TFIDF vectors have dimension = 1000. Let's take a look at the effect of dimensionality reduction by plotting the proportion of explained variance of the data as a function of the number of principal components:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 378,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3775,
     "status": "ok",
     "timestamp": 1522085928247,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "elWsRxc3x2R5",
    "outputId": "6987d35c-0111-477a-fef2-8a865c4a9dfa"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "explained_var = []\n",
    "for components in range(1,100,5):\n",
    "  pca = PCA(n_components=components)\n",
    "  pca.fit(dtm.toarray())\n",
    "  explained_var.append(pca.explained_variance_ratio_.sum())\n",
    "\n",
    "plt.plot(range(1,100,5),explained_var,\"ro\")\n",
    "plt.xlabel(\"Number of Components\")\n",
    "plt.ylabel(\"Proportion of Explained Variance\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "drciFP2L1bEF"
   },
   "source": [
    "The graph above shows that half of the variance an be explained by 60 components. Let's apply this to the dataset, and visualize the results by plotting the first two PCA components of each document:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 368,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6297,
     "status": "ok",
     "timestamp": 1522085936694,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "YAbGanEOHabD",
    "outputId": "00eb1fbc-546b-48ad-8f17-3e5a558189a7"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "import seaborn as sns\n",
    "\n",
    "components = 60\n",
    "\n",
    "palette = np.array(sns.color_palette(\"hls\", 120))\n",
    "\n",
    "pca = PCA(n_components=components)\n",
    "pca.fit(dtm.toarray())\n",
    "pca_dtm = pca.transform(dtm.toarray())\n",
    "data_target = labelBinarizer.fit_transform(data_set[u'topics'])\n",
    "plt.scatter(pca_dtm[:,0],pca_dtm[:,1],c=palette[data_target.argmax(axis=1).astype(int)])\n",
    "\n",
    "explained_variance = pca.explained_variance_ratio_.sum()\n",
    "print(\"Explained variance of the PCA step: {}%\".format(\n",
    "    int(explained_variance * 100)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qNdL4_Sd-G4N"
   },
   "source": [
    "We know that there are 5 distinct topics (though some documents might have overlap), so let’s run the k-means algorithm with k=5 to examine document grouping:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 351,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 941,
     "status": "ok",
     "timestamp": 1522085940015,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "V3_ai9BVIPxY",
    "outputId": "c8bbaf93-8ef6-41f2-a2e4-6c1e1fc20275"
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "palette = np.array(sns.color_palette(\"hls\", 5))\n",
    "\n",
    "model = KMeans(n_clusters=5,max_iter=100)\n",
    "clustered = model.fit(pca_dtm)\n",
    "centroids = model.cluster_centers_\n",
    "y = model.predict(pca_dtm)\n",
    "\n",
    "ax = plt.subplot()\n",
    "sc = ax.scatter(pca_dtm[:,0],pca_dtm[:,1],c=palette[y.astype(np.int)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iyQfAJ4Mhz7G"
   },
   "source": [
    "Even with only capturing 50% of the variance, the clustering is fairly good when we compare with the manual (gold) labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 351,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 538,
     "status": "ok",
     "timestamp": 1522085942529,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "kxWvestmhzIv",
    "outputId": "15ae23d1-b331-4121-8961-40a09cfcfdc2"
   },
   "outputs": [],
   "source": [
    "palette = np.array(sns.color_palette(\"hls\", 5))\n",
    "\n",
    "gold_labels = data_set['topics'].map(set(mytopics).intersection).apply(lambda x: x.pop()).apply(lambda x: mytopics.index(x))\n",
    "\n",
    "ax = plt.subplot()\n",
    "sc = ax.scatter(pca_dtm[:,0],pca_dtm[:,1],c=palette[gold_labels])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4sAFhavy54x1"
   },
   "source": [
    "Feel free to change the number of PCA components and see how well k-means performs vs. the gold labels. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MUUBafFbGXvV"
   },
   "source": [
    "# **Topic Modeling**\n",
    "\n",
    "In addition to the lexical clustering of documents, lets see if we can discern any natural topic structure within the corpus. We apply the LSA and LDA algorithms, which will associate words to a set of topics, and topics to our set of documents."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ERCgFbX8ge7m"
   },
   "source": [
    "##LSA\n",
    "We start with the LSA algorithm, and set the number of dimensions to 60:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 364,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 714,
     "status": "ok",
     "timestamp": 1522085953174,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "9ZyztIIp4tFx",
    "outputId": "9e09bdd7-fedc-4786-fde3-3d178b5d299c"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "import seaborn as sns\n",
    "\n",
    "components = 60\n",
    "\n",
    "palette = np.array(sns.color_palette(\"hls\", 120))\n",
    "\n",
    "lsa = TruncatedSVD(n_components=components)\n",
    "lsa.fit(dtm)\n",
    "lsa_dtm = lsa.transform(dtm)\n",
    "\n",
    "plt.scatter(lsa_dtm[:,0],lsa_dtm[:,1],c=palette[data_target.argmax(axis=1).astype(int)])\n",
    "\n",
    "explained_variance = lsa.explained_variance_ratio_.sum()\n",
    "print(\"Explained variance of the SVD step: {}%\".format(\n",
    "    int(explained_variance * 100)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3JuKJwG6g3ie"
   },
   "source": [
    "As with PCA, let's apply k-means with k = 5 clusters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 347,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 747,
     "status": "ok",
     "timestamp": 1522085956397,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "2aNSBHJb1NQN",
    "outputId": "8c473243-af08-45d9-fc7f-ce68d9f30a46"
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "palette = np.array(sns.color_palette(\"hls\", 8))\n",
    "\n",
    "model = KMeans(n_clusters=5,max_iter=100)\n",
    "clustered = model.fit(lsa_dtm)\n",
    "centroids = model.cluster_centers_\n",
    "y = model.predict(lsa_dtm)\n",
    "\n",
    "ax = plt.subplot()\n",
    "sc = ax.scatter(lsa_dtm[:,0],lsa_dtm[:,1],c=palette[y.astype(np.int)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sMzrgiBR7BTu"
   },
   "source": [
    "Let's examine the documents of one of these clusters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 204,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 317,
     "status": "ok",
     "timestamp": 1522085959540,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "8WkwzsQD7dLR",
    "outputId": "70484731-eae8-490e-ecfb-882f5b247236"
   },
   "outputs": [],
   "source": [
    "topic = u'coffee'\n",
    "\n",
    "data_set[gold_labels == mytopics.index(topic)][u'body'].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sU9P_5y77dhZ"
   },
   "source": [
    "Let's see if the LDA algorithm can do better as a Bayesian as a Bayesian approach to document clustering and topic modeling. We set the number of topics to the known number of topics = 5."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 119,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1503,
     "status": "ok",
     "timestamp": 1522085972939,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "ceAELdhX6g50",
    "outputId": "144b9ca3-79b0-40d5-d62f-e37693b6953b"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "import seaborn as sns\n",
    "\n",
    "components = 5\n",
    "n_top_words = 10\n",
    "\n",
    "palette = np.array(sns.color_palette(\"hls\", 120))\n",
    "    \n",
    "def print_top_words(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        message = \"Topic #%d: \" % topic_idx\n",
    "        message += \" \".join([feature_names[i]\n",
    "                             for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
    "        print(message)\n",
    "    print()\n",
    "  \n",
    "lda = LatentDirichletAllocation(n_components=components,max_iter=5,learning_method='online')\n",
    "lda.fit(dtm)\n",
    "lda_dtm = lda.transform(dtm)\n",
    "\n",
    "vec_feature_names = vec.get_feature_names()\n",
    "print_top_words(lda, vec_feature_names, n_top_words)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dhHglwII9_TI"
   },
   "source": [
    "The LDA results are encouraging, and we can easily discern 4 of the 5 original topics from the list of words associated with each topic."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DmCh8Q7R-WPo"
   },
   "source": [
    "# Document Classification\n",
    "Now let's see if we can build classifiers to possibly identify the topics above. We first randomize and split our dataset into train and test sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 51,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 360,
     "status": "ok",
     "timestamp": 1522092136516,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "haddXcq5ZkfF",
    "outputId": "37f3d217-d6b8-4d72-8d69-433912ea9d6c"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "data_set['label'] = gold_labels\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(data_set,gold_labels,test_size=0.2, random_state=10)\n",
    "print(\"Train Set = \",len(X_train))\n",
    "print(\"Test Set  = \",len(X_test))\n",
    "\n",
    "X_train = X_train[u'body']\n",
    "X_test = X_test[u'body']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fE74BJagrAol"
   },
   "source": [
    "When then create a pipeline that builds classifiers based on 5 models: Naive Bayes, Logistic Regression, SVM,K-Nearest Neighbor, and Random Forest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "QjGzYKoeMbe2"
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "models = [('multinomial_nb', MultinomialNB()),\n",
    "          ('log_reg', LogisticRegression()),\n",
    "          ('linear_svc', LinearSVC()),\n",
    "          ('knn', KNeighborsClassifier(n_neighbors=6)),\n",
    "          ('rf', RandomForestClassifier(n_estimators=6))]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8D6_JUZvrLXO"
   },
   "source": [
    "We then train each model on the training set and evaluate on the test set. For each model, we want to see the precision, recall, F1 score, and support (number of samples) for each topic class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1275,
     "output_extras": [
      {
       "item_id": 5
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 22849,
     "status": "ok",
     "timestamp": 1522094453506,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "J4bE1GBVcxHs",
    "outputId": "8e23cdc0-4cc8-445b-e2fa-e6bb0c999ad4"
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "for m_name, model in models:\n",
    "    pipeline = Pipeline([('vec', TfidfVectorizer(tokenizer=SimpleTokenizer)),(m_name,model)])\n",
    "    pipeline.fit(X_train,y_train)\n",
    "    test_y = pipeline.predict(X_test)\n",
    "    print(\"model = \",model,\"\\n\")\n",
    "    print(classification_report(y_test,test_y,digits=6))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XLO_8ECVzg_K"
   },
   "source": [
    "The results seem to indicate that a linear SVM model seems to perform the best, with Random Forest a close second. This is a bit misleading, since we didn't tune any of these models to obtain our results. Hyperparameter tuning can significantly affect how well a classifier performs. Let's try tuning the LinearSVC model. The tuneable parameters are:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 71,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 276,
     "status": "ok",
     "timestamp": 1522095142837,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "kk7rXG3gjpuo",
    "outputId": "d23aa512-c8e0-4dd7-d79f-7fb7004a6c80"
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(tokenizer=SimpleTokenizer)\n",
    "model = LinearSVC()\n",
    "print(\"vec params = \",vectorizer.get_params().keys())\n",
    "print(\"model params = \",model.get_params().keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rdIn4FD_0EAv"
   },
   "source": [
    "We want to tune these parameters by using grid search with cross-validation. Note that cross-validation is important as we do not want to tune with our test set, which we will use only at the end to assess performance. Note also that this can take a while!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 221,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 294813,
     "status": "ok",
     "timestamp": 1522095440480,
     "user": {
      "displayName": "John Liu",
      "photoUrl": "//lh6.googleusercontent.com/-V6emrv6zDIE/AAAAAAAAAAI/AAAAAAAAASI/QVg3GuFEck0/s50-c-k-no/photo.jpg",
      "userId": "117919618874451216378"
     },
     "user_tz": 300
    },
    "id": "E87O-Ji1iECw",
    "outputId": "81c3cf9f-c8eb-4d61-d391-de62ff6b157d"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "pipeline = Pipeline([('vec', vectorizer),\n",
    "                     ('model', model)])\n",
    "\n",
    "parameters = {'vec__ngram_range': ((1, 1), (1, 2)),\n",
    "              'vec__max_features': (500, 1000),\n",
    "              'model__loss': ('hinge', 'squared_hinge'),\n",
    "              'model__C': (1, 0.9)}\n",
    "\n",
    "grid_search = GridSearchCV(pipeline, parameters,verbose=1)\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "test_y = grid_search.best_estimator_.predict(X_test)\n",
    "print(classification_report(y_test,test_y,digits=6))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ECk86uzX53Ad"
   },
   "source": [
    "As you see, the SVM model typically outperforms other machine learning algorithms, and often provides as the state-of-the-art in quality. Unfortunately, SVM suffers from several major drawbacks, including the inability to scale to large datasets. As we will learn in later chapters, neural networks can bypass the limitations of SVMs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "a0_UK5_gTpQA"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "Chapter 3 - Text and Speech Basics",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}