Train_and_Test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import mpqa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_data = pd.read_pickle('new_data.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13317"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.columns.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8016"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.sparse.frame.SparseDataFrame'>\n",
      "Int64Index: 8016 entries, 0 to 8015\n",
      "Columns: 13317 entries, after_0000 to word_4433\n",
      "dtypes: float64(13317)\n",
      "memory usage: 418.2 KB\n"
     ]
    }
   ],
   "source": [
    "df_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ddf = df_data.to_dense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 8016 entries, 0 to 8015\n",
      "Columns: 13317 entries, after_0000 to word_4433\n",
      "dtypes: float64(13317)\n",
      "memory usage: 814.5 MB\n"
     ]
    }
   ],
   "source": [
    "ddf.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_labels = pd.read_pickle('new_labels.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.75    2606\n",
       "-1.50    2504\n",
       " 0.75    1811\n",
       " 1.50     694\n",
       "-2.00     240\n",
       "-3.00      71\n",
       "-2.25      35\n",
       " 2.25      18\n",
       " 3.00      11\n",
       "-3.50      10\n",
       " 2.00       9\n",
       "-4.00       5\n",
       "-1.25       1\n",
       "-3.75       1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_labels['c_pol'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = ddf.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = df_labels['c_pol'].map(lambda k: 0 if k < 0 else 1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8016, 13317), (8016,))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "svc_1 = SVC(kernel='linear')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6012, 13317), (6012,))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_and_evaluate(clf, X_train, X_test, y_train, y_test):\n",
    "    clf.fit(X_train, y_train)\n",
    "    print \"Accuracy on training set:\"\n",
    "    print clf.score(X_train, y_train)\n",
    "    print \"Accuracy on testing set:\"\n",
    "    print clf.score(X_test, y_test)\n",
    "    \n",
    "    y_pred = clf.predict(X_test)\n",
    "    \n",
    "    print \"Classification Report:\"\n",
    "    print metrics.classification_report(y_test, y_pred)\n",
    "    print \"Confusion Matrix:\"\n",
    "    print metrics.confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import cross_val_score, KFold\n",
    "from scipy.stats import sem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_cross_validation(clf, X, y, K):\n",
    "    # create a k-fold croos validation iterator\n",
    "    cv = KFold(len(y), K, shuffle=True, random_state=0)\n",
    "    # by default the score used is the one returned by score \n",
    "    # method of the estimator (accuracy)\n",
    "    scores = cross_val_score(clf, X, y, cv=cv)\n",
    "    print scores\n",
    "    print (\"Mean score: {0:.3f} (+/-{1:.3f})\").format(\n",
    "            np.mean(scores), sem(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.85203658  0.8478803   0.85607321  0.85357737  0.84941764]\n",
      "Mean score: 0.852 (+/-0.001)\n",
      "Accuracy on training set:\n",
      "0.966733200266\n",
      "Accuracy on testing set:\n",
      "0.857285429142\n",
      "Classification Report:\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "global name 'metrics' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-fa243806736a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mevaluate_cross_validation\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msvc_1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrain_and_evaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msvc_1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-13-0fc78ea0f166>\u001b[0m in \u001b[0;36mtrain_and_evaluate\u001b[1;34m(clf, X_train, X_test, y_train, y_test)\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[1;34m\"Classification Report:\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m     \u001b[1;32mprint\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclassification_report\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[1;34m\"Confusion Matrix:\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: global name 'metrics' is not defined"
     ]
    }
   ],
   "source": [
    "evaluate_cross_validation(svc_1, X_train, y_train, 5)\n",
    "train_and_evaluate(svc_1, X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = svc_1.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.91      0.90      1364\n",
      "          1       0.79      0.75      0.77       640\n",
      "\n",
      "avg / total       0.86      0.86      0.86      2004\n",
      "\n",
      "Confusion Matrix:\n",
      "[[1239  125]\n",
      " [ 161  479]]\n"
     ]
    }
   ],
   "source": [
    "print \"Classification Report:\"\n",
    "print metrics.classification_report(y_test, y_pred)\n",
    "print \"Confusion Matrix:\"\n",
    "print metrics.confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}