predictive-titles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:10:08.111474Z",
     "start_time": "2019-10-02T19:10:08.108526Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "\n",
    "import numpy as np\n",
    "import glob\n",
    "import random\n",
    "\n",
    "import nltk\n",
    "\n",
    "from multi_rake import Rake\n",
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:36:54.607250Z",
     "start_time": "2019-10-02T18:36:54.603287Z"
    }
   },
   "outputs": [],
   "source": [
    "all_csv = glob.glob('conference-data/csv/*.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:54:07.203653Z",
     "start_time": "2019-10-02T18:54:07.093410Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jhawk/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "# read them all in at once\n",
    "# Note that 2008 did not have date or year for all entries, so had to be added manually\n",
    "\n",
    "year_df = (pd.read_csv(f) for f in all_csv)\n",
    "df = pd.concat(year_df, ignore_index=False)\n",
    "\n",
    "df = df.fillna('')\n",
    "df = df.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:41.183697Z",
     "start_time": "2019-10-02T18:56:37.182224Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jhawk/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# trying some predictive text\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "import glob\n",
    "\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from nltk.corpus import stopwords \n",
    "stopWords = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:41.238920Z",
     "start_time": "2019-10-02T18:56:41.185014Z"
    }
   },
   "outputs": [],
   "source": [
    "all_titles = []\n",
    "all_abstracts = []\n",
    "\n",
    "for i,r in df.iterrows():\n",
    "            \n",
    "    all_titles.append(str(r['title']))\n",
    "    all_abstracts.append(str(r['abstract']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:42.685851Z",
     "start_time": "2019-10-02T18:56:42.682926Z"
    }
   },
   "outputs": [],
   "source": [
    "all_titles = [t for t in all_titles if t != '']\n",
    "\n",
    "all_abstracts = [t for t in all_abstracts if t != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:43.659651Z",
     "start_time": "2019-10-02T18:56:43.645807Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()\n",
    "\n",
    "tokenizer.fit_on_texts(all_titles)\n",
    "\n",
    "total_words = len(tokenizer.word_index) + 1\n",
    "print(total_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:51.391726Z",
     "start_time": "2019-10-02T18:56:51.378881Z"
    }
   },
   "outputs": [],
   "source": [
    "# make n_grams of word_indexes for each sentence - e.g. [4,2], [4,2,17], [4,2,17,36], etc\n",
    "\n",
    "input_sentences = []\n",
    "\n",
    "for line in title_corpus:\n",
    "    token_list = tokenizer.texts_to_sequences([line])[0]\n",
    "    \n",
    "    for i in range(1, len(token_list)):\n",
    "        n_gram_sequences = token_list[:i+1]\n",
    "        input_sentences.append(n_gram_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:52.116330Z",
     "start_time": "2019-10-02T18:56:52.106963Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find longest sentence\n",
    "\n",
    "max_sequence_len = max(len(x) for x in input_sentences)\n",
    "max_sequence_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:55.901150Z",
     "start_time": "2019-10-02T18:56:55.882522Z"
    }
   },
   "outputs": [],
   "source": [
    "# pad sequences to longest length\n",
    "\n",
    "input_sequences = np.array(pad_sequences(input_sentences, maxlen=max_sequence_len, padding='pre'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:56.345872Z",
     "start_time": "2019-10-02T18:56:56.334600Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0,    0,    0, ...,    0,    9,    1],\n",
       "       [   0,    0,    0, ...,    9,    1,  481],\n",
       "       [   0,    0,    0, ...,    1,  481,    3],\n",
       "       ...,\n",
       "       [   0,    0,    0, ..., 1431,    2,    1],\n",
       "       [   0,    0,    0, ...,    2,    1,  366],\n",
       "       [   0,    0,    0, ...,    0,  218, 1432]], dtype=int32)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:57.941585Z",
     "start_time": "2019-10-02T18:56:57.938911Z"
    }
   },
   "outputs": [],
   "source": [
    "# to get predictive properties, take the above array and\n",
    "# use all tokens except the final one as the input, \n",
    "# and the final one as the label \n",
    "\n",
    "xs = input_sequences[:,:-1]\n",
    "labels = input_sequences[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T18:56:58.337617Z",
     "start_time": "2019-10-02T18:56:58.324452Z"
    }
   },
   "outputs": [],
   "source": [
    "# encode the labels for tf\n",
    "# creates \"one-hot encoding\" by converting list [labels] to categorical\n",
    "\n",
    "ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T17:41:54.783342Z",
     "start_time": "2019-10-02T17:41:54.778329Z"
    }
   },
   "outputs": [],
   "source": [
    "# see one-hot encoding (find the '1' to represent the word in the word_index)\n",
    "ys[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:22:42.251613Z",
     "start_time": "2019-10-02T19:21:47.681981Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3367 samples\n",
      "Epoch 1/18\n",
      "3367/3367 [==============================] - 4s 1ms/sample - loss: 6.7173 - accuracy: 0.0508\n",
      "Epoch 2/18\n",
      "3367/3367 [==============================] - 4s 1ms/sample - loss: 5.9868 - accuracy: 0.0612\n",
      "Epoch 3/18\n",
      "3367/3367 [==============================] - 3s 827us/sample - loss: 5.4383 - accuracy: 0.1001\n",
      "Epoch 4/18\n",
      "3367/3367 [==============================] - 3s 833us/sample - loss: 4.7380 - accuracy: 0.1583\n",
      "Epoch 5/18\n",
      "3367/3367 [==============================] - 3s 1ms/sample - loss: 3.9688 - accuracy: 0.2198\n",
      "Epoch 6/18\n",
      "3367/3367 [==============================] - 4s 1ms/sample - loss: 3.1785 - accuracy: 0.3359\n",
      "Epoch 7/18\n",
      "3367/3367 [==============================] - 3s 905us/sample - loss: 2.4720 - accuracy: 0.4592\n",
      "Epoch 8/18\n",
      "3367/3367 [==============================] - 3s 827us/sample - loss: 1.8569 - accuracy: 0.5988\n",
      "Epoch 9/18\n",
      "3367/3367 [==============================] - 3s 839us/sample - loss: 1.3745 - accuracy: 0.7066\n",
      "Epoch 10/18\n",
      "3367/3367 [==============================] - 3s 762us/sample - loss: 1.0481 - accuracy: 0.7784\n",
      "Epoch 11/18\n",
      "3367/3367 [==============================] - 3s 835us/sample - loss: 0.8357 - accuracy: 0.8215\n",
      "Epoch 12/18\n",
      "3367/3367 [==============================] - 3s 747us/sample - loss: 0.6716 - accuracy: 0.8530\n",
      "Epoch 13/18\n",
      "3367/3367 [==============================] - 3s 771us/sample - loss: 0.5463 - accuracy: 0.8788\n",
      "Epoch 14/18\n",
      "3367/3367 [==============================] - 3s 775us/sample - loss: 0.4520 - accuracy: 0.8940\n",
      "Epoch 15/18\n",
      "3367/3367 [==============================] - 3s 778us/sample - loss: 0.4012 - accuracy: 0.9050\n",
      "Epoch 16/18\n",
      "3367/3367 [==============================] - 3s 751us/sample - loss: 0.3497 - accuracy: 0.9118\n",
      "Epoch 17/18\n",
      "3367/3367 [==============================] - 3s 751us/sample - loss: 0.3095 - accuracy: 0.9207\n",
      "Epoch 18/18\n",
      "3367/3367 [==============================] - 3s 774us/sample - loss: 0.2824 - accuracy: 0.9237\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1a396e1630>"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model - note some has been updated from 07 notebook\n",
    "\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(total_words, 100, input_length=max_sequence_len - 1))\n",
    "# using Bidirectional to allow model to look before and after a given word\n",
    "model.add(Bidirectional(LSTM(50)))\n",
    "model.add(Dense(total_words, activation='softmax'))\n",
    "# you can set the lr (\"learning rate\") on the adam optimizer\n",
    "adam = Adam(lr=0.01)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n",
    "model.fit(xs, ys, epochs=18, verbose=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T20:40:20.253884Z",
     "start_time": "2019-10-02T20:40:15.615457Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Music database the project we\n",
      "Accessible to the university\n",
      "Worldcat discovery libraries museums the university\n",
      "Core person entities such as xissn worldcat registry this\n",
      "Projet commercial ocr standard is responsible for varyfrom benign\n",
      "Halinet commercial ocr standard\n",
      "Big data fangirl offers\n",
      "Xmlmark commercial ocr standard\n",
      "Comparative media studies and the library\n",
      "Canadiennes commercial ocr standard is responsible\n"
     ]
    }
   ],
   "source": [
    "# prediction to call next n of words\n",
    "\n",
    "seed_titles = ' '.join(set([t.strip() for t in ' '.join(all_titles).split() if t not in stopWords and t.isalpha()]))\n",
    "\n",
    "def get_title(seed_text):\n",
    "\n",
    "    next_words = np.random.randint(low=4, high=10)\n",
    "\n",
    "    for _ in range(next_words):\n",
    "        token_list = tokenizer.texts_to_sequences([seed_text])[0]\n",
    "        token_list = pad_sequences([token_list], maxlen=max_sequence_len - 1, padding='pre')\n",
    "        predicted = model.predict_classes(token_list, verbose=0)\n",
    "\n",
    "        output_word = \"\"\n",
    "\n",
    "        for word, index in tokenizer.word_index.items():\n",
    "            if index == predicted:\n",
    "                output_word = word\n",
    "                break\n",
    "        seed_text += \" \" + output_word\n",
    "    \n",
    "    if seed_text.split()[-1] in stopWords:\n",
    "        seed_text = ' '.join(seed_text.split()[:-1])\n",
    "        \n",
    "    print(seed_text.capitalize())\n",
    "\n",
    "beg_words = []    \n",
    "    \n",
    "for i in range(0,10):\n",
    "    \n",
    "    seed_text = random.choice(seed_titles.split())\n",
    "    \n",
    "    get_title(seed_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:24:28.042700Z",
     "start_time": "2019-10-02T19:24:27.991513Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5834\n"
     ]
    }
   ],
   "source": [
    "# abstracts\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "\n",
    "tokenizer.fit_on_texts(all_abstracts)\n",
    "\n",
    "total_words = len(tokenizer.word_index) + 1\n",
    "print(total_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:24:46.208421Z",
     "start_time": "2019-10-02T19:24:46.090502Z"
    }
   },
   "outputs": [],
   "source": [
    "# make n_grams of word_indexes for each sentence - e.g. [4,2], [4,2,17], [4,2,17,36], etc\n",
    "\n",
    "input_sentences = []\n",
    "\n",
    "for line in all_abstracts:\n",
    "    token_list = tokenizer.texts_to_sequences([line])[0]\n",
    "    \n",
    "    for i in range(1, len(token_list)):\n",
    "        n_gram_sequences = token_list[:i+1]\n",
    "        input_sentences.append(n_gram_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:24:46.806155Z",
     "start_time": "2019-10-02T19:24:46.798030Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "383"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find longest sentence\n",
    "\n",
    "max_sequence_len = max(len(x) for x in input_sentences)\n",
    "max_sequence_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:24:49.466405Z",
     "start_time": "2019-10-02T19:24:49.134257Z"
    }
   },
   "outputs": [],
   "source": [
    "# pad sequences to longest length\n",
    "\n",
    "input_sequences = np.array(pad_sequences(input_sentences, maxlen=max_sequence_len, padding='pre'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:24:53.930814Z",
     "start_time": "2019-10-02T19:24:53.928017Z"
    }
   },
   "outputs": [],
   "source": [
    "# to get predictive properties, take the above array and\n",
    "# use all tokens except the final one as the input, \n",
    "# and the final one as the label \n",
    "\n",
    "xs = input_sequences[:,:-1]\n",
    "labels = input_sequences[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T19:24:55.182638Z",
     "start_time": "2019-10-02T19:24:55.084360Z"
    }
   },
   "outputs": [],
   "source": [
    "# encode the labels for tf\n",
    "# creates \"one-hot encoding\" by converting list [labels] to categorical\n",
    "\n",
    "ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-02T20:40:15.611400Z",
     "start_time": "2019-10-02T19:25:02.705665Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 37922 samples\n",
      "Epoch 1/18\n",
      "37922/37922 [==============================] - 260s 7ms/sample - loss: 6.8626 - accuracy: 0.0858\n",
      "Epoch 2/18\n",
      "37922/37922 [==============================] - 270s 7ms/sample - loss: 6.0987 - accuracy: 0.1300\n",
      "Epoch 3/18\n",
      "37922/37922 [==============================] - 255s 7ms/sample - loss: 5.5429 - accuracy: 0.1549\n",
      "Epoch 4/18\n",
      "37922/37922 [==============================] - 261s 7ms/sample - loss: 5.0414 - accuracy: 0.1796\n",
      "Epoch 5/18\n",
      "37922/37922 [==============================] - 277s 7ms/sample - loss: 4.6210 - accuracy: 0.2029\n",
      "Epoch 6/18\n",
      "37922/37922 [==============================] - 266s 7ms/sample - loss: 4.2902 - accuracy: 0.2298\n",
      "Epoch 7/18\n",
      "37922/37922 [==============================] - 263s 7ms/sample - loss: 4.0438 - accuracy: 0.2510\n",
      "Epoch 8/18\n",
      "37922/37922 [==============================] - 244s 6ms/sample - loss: 3.8477 - accuracy: 0.2716\n",
      "Epoch 9/18\n",
      "37922/37922 [==============================] - 251s 7ms/sample - loss: 3.7028 - accuracy: 0.2837\n",
      "Epoch 10/18\n",
      "37922/37922 [==============================] - 239s 6ms/sample - loss: 3.5856 - accuracy: 0.2984\n",
      "Epoch 11/18\n",
      "37922/37922 [==============================] - 251s 7ms/sample - loss: 3.4851 - accuracy: 0.3109\n",
      "Epoch 12/18\n",
      "37922/37922 [==============================] - 238s 6ms/sample - loss: 3.4098 - accuracy: 0.3199\n",
      "Epoch 13/18\n",
      "37922/37922 [==============================] - 229s 6ms/sample - loss: 3.3315 - accuracy: 0.3278\n",
      "Epoch 14/18\n",
      "37922/37922 [==============================] - 233s 6ms/sample - loss: 3.2865 - accuracy: 0.3350\n",
      "Epoch 15/18\n",
      "37922/37922 [==============================] - 250s 7ms/sample - loss: 3.2460 - accuracy: 0.3382\n",
      "Epoch 16/18\n",
      "37922/37922 [==============================] - 248s 7ms/sample - loss: 3.2009 - accuracy: 0.3464\n",
      "Epoch 17/18\n",
      "37922/37922 [==============================] - 230s 6ms/sample - loss: 3.1475 - accuracy: 0.3536\n",
      "Epoch 18/18\n",
      "37922/37922 [==============================] - 245s 6ms/sample - loss: 3.1222 - accuracy: 0.3569\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1a77df6be0>"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model - note some has been updated from 07 notebook\n",
    "\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(total_words, 100, input_length=max_sequence_len - 1))\n",
    "# using Bidirectional to allow model to look before and after a given word\n",
    "model.add(Bidirectional(LSTM(50)))\n",
    "model.add(Dense(total_words, activation='softmax'))\n",
    "# you can set the lr (\"learning rate\") on the adam optimizer\n",
    "adam = Adam(lr=0.01)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n",
    "model.fit(xs, ys, epochs=18, verbose=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}