CS 5785 - Logistic Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CS 5785 - Final\n",
    "## Applied Machine Learning\n",
    "\n",
    "Arpit Sheth *(as3668)*\n",
    "\n",
    "Eric Nguyen *(en274)*\n",
    "\n",
    "Samantha Yip *(sty5)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import setup\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from IPython.display import display, Markdown\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from autocorrect import spell\n",
    "import math\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn import svm\n",
    "from sklearn import preprocessing\n",
    "import csv\n",
    "from sklearn.metrics.pairwise import euclidean_distances\n",
    "from sklearn.neighbors import NearestNeighbors as KNN\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TRAIN = 0\n",
    "TEST = 1\n",
    "N_TRAIN = 10000\n",
    "N_TEST = 2000\n",
    "\n",
    "# These become Pandas DataFrames\n",
    "descriptions_train = []\n",
    "descriptions_test = []\n",
    "features_train = []\n",
    "features_test = []\n",
    "ifeatures_train = []\n",
    "ifeatures_test = []\n",
    "tags_train = []\n",
    "tags_test = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "setup.get_image(1, TRAIN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Descriptions and create bag of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "descriptions_train = setup.get_descriptions(10000,TRAIN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "descriptions_test = setup.get_descriptions(2000,TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stop = stopwords.words('english')\n",
    "plural = WordNetLemmatizer()\n",
    "for j in range(5):\n",
    "    descriptions_train['s'+str(j+1)] = descriptions_train['s'+str(j+1)].str.lower()\n",
    "    descriptions_train['s'+str(j+1)] = descriptions_train['s'+str(j+1)].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))\n",
    "    descriptions_train['s'+str(j+1)] = descriptions_train['s'+str(j+1)].apply(lambda x:''.join([i for i in x if i not in string.punctuation]))\n",
    "    for i, sentence in enumerate(descriptions_train['s'+str(j+1)]):\n",
    "        temp = []\n",
    "        for word in sentence.split(\" \"):\n",
    "            word2 = spell(word)\n",
    "            word2 = plural.lemmatize(word2,'n')\n",
    "            word2 = plural.lemmatize(word2,'v')\n",
    "            word2 = plural.lemmatize(word2,'a')\n",
    "            word2 = plural.lemmatize(word2,'r')\n",
    "            temp.append(word2)\n",
    "        descriptions_train['s'+str(j+1)][i] = ' '.join(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stop = stopwords.words('english')\n",
    "plural = WordNetLemmatizer()\n",
    "for j in range(5):\n",
    "    descriptions_test['s'+str(j+1)] = descriptions_test['s'+str(j+1)].str.lower()\n",
    "    descriptions_test['s'+str(j+1)] = descriptions_test['s'+str(j+1)].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))\n",
    "    descriptions_test['s'+str(j+1)] = descriptions_test['s'+str(j+1)].apply(lambda x:''.join([i for i in x if i not in string.punctuation]))\n",
    "    for i, sentence in enumerate(descriptions_test['s'+str(j+1)]):\n",
    "        temp = []\n",
    "        for word in sentence.split(\" \"):\n",
    "            word2 = spell(word)\n",
    "            word2 = plural.lemmatize(word2,'n')\n",
    "            word2 = plural.lemmatize(word2,'v')\n",
    "            word2 = plural.lemmatize(word2,'a')\n",
    "            word2 = plural.lemmatize(word2,'r')\n",
    "            temp.append(word2)\n",
    "        descriptions_test['s'+str(j+1)][i] = ' '.join(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "documents_train = []\n",
    "for index, row in descriptions_train.iterrows():\n",
    "    description = ' '.join(row.tolist())\n",
    "    documents_train.append(description)\n",
    "documents_train = pd.DataFrame(documents_train, columns=[\"sentences\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "documents_test = []\n",
    "for index, row in descriptions_test.iterrows():\n",
    "    description = ' '.join(row.tolist())\n",
    "    documents_test.append(description)\n",
    "documents_test = pd.DataFrame(documents_test, columns=[\"sentences\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag_of_words = []\n",
    "for i in range(10000):\n",
    "    for j in range(5):\n",
    "        for word in descriptions_train.get_value(i,\"s\"+str(j+1)).split(\" \"):\n",
    "            bag_of_words.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag_of_words = list(set(bag_of_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag_of_words_test = []\n",
    "for sentence in documents_test[\"sentences\"]:\n",
    "    for word in sentence.split(\" \"):\n",
    "        bag_of_words_test.append(word)\n",
    "bag_of_words_test = list(set(bag_of_words_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the full list of subcategories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tags_train = setup.get_tags(10000,TRAIN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subcat_list = []\n",
    "final_tags_train = []\n",
    "for i, tags in tags_train.iterrows():\n",
    "    for tag in tags:\n",
    "        temp = ''\n",
    "        for j in tag:\n",
    "            word = j[1].replace(\" \", \"\")\n",
    "            subcat_list.append(word)\n",
    "            temp = temp + ' ' + word\n",
    "        final_tags_train.append(temp)\n",
    "        \n",
    "subcat_list = list(set(subcat_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conduct TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "cv = CountVectorizer(min_df = 3)\n",
    "X_train_bow = cv.fit_transform(documents_train[\"sentences\"]).toarray()\n",
    "vocab = np.array(cv.get_feature_names())\n",
    "transformer = TfidfTransformer()\n",
    "X_train_tfidf = transformer.fit_transform(X_train_bow).toarray()\n",
    "\n",
    "# Convert into DataFrames\n",
    "X_train_tfidf = pd.DataFrame(X_train_tfidf, columns = vocab)\n",
    "X_train_bow = pd.DataFrame(X_train_bow, columns = vocab)\n",
    "X_train_tfidf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(vocabulary = vocab)\n",
    "X_test_bow = cv.fit_transform(documents_test[\"sentences\"]).toarray()\n",
    "transformer = TfidfTransformer()\n",
    "X_test_tfidf = transformer.fit_transform(X_test_bow).toarray()\n",
    "\n",
    "# Convert into DataFrames\n",
    "X_test_tfidf = pd.DataFrame(X_test_tfidf, columns = vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv = CountVectorizer(vocabulary = subcat_list)\n",
    "tags_train_0_1 = cv.fit_transform(final_tags_train).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_tags = []\n",
    "for i in range(80):\n",
    "    y = tags_train_0_1[:,i]\n",
    "    clf = LogisticRegression()\n",
    "    clf.fit(X_train_tfidf, y)\n",
    "    tag = clf.predict(X_test_tfidf_norm)\n",
    "    predict_tags.append(tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_tags_trans = predict_tags.T\n",
    "predict_tags_trans.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_test = setup.get_tags(2000,TEST)\n",
    "final_tags_test = []\n",
    "for tag in tags_test['tags']:\n",
    "    temp = ''\n",
    "    for i in tag:\n",
    "        word = i[1].replace(\" \",\"\")\n",
    "        temp = temp + ' ' + word\n",
    "    final_tags_test.append(temp)\n",
    "cv = CountVectorizer(vocabulary = subcat_list)\n",
    "tags_test_0_1 = cv.fit_transform(final_tags_test).toarray()\n",
    "tags_test_0_1.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_labels = [i for i in range(2000)]\n",
    "knn = KNN(n_neighbors = 20)\n",
    "knn = knn.fit(tags_test_0_1, test_labels)\n",
    "predictions = knn.kneighbors(predict_tags_trans, return_distance = False)\n",
    "# predictions = pd.DataFrame(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list = []\n",
    "for i, row in enumerate(predictions):\n",
    "    temp = ''\n",
    "    for j, val in enumerate(row):\n",
    "        temp = temp + ' ' + (str(val) + \".jpg\")\n",
    "    list.append(temp)\n",
    "images = pd.DataFrame(list, columns = [\"Top_20_Image_IDs\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = []\n",
    "for i in range(2000):\n",
    "    index.append(str(i)+ \".txt\")\n",
    "index = pd.DataFrame(index, columns = [\"Descritpion_ID\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result = pd.concat([index, images], axis=1)\n",
    "result.to_csv(\"Logistic_Regression.csv\", index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}