Phrase_validation_accuracy .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 567,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>@manjulamartin @Kirk_Gleason Except trains are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>I want a Google driverless car.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>@Oatmeal @google driverless @TeslaMotors ? Ooo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>SO MUCH AWESOME! Amazing video for GoogleÌ¢‰âÂ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>@google is making driverless cars which is awe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text\n",
       "0          5  @manjulamartin @Kirk_Gleason Except trains are...\n",
       "1          5                    I want a Google driverless car.\n",
       "2          5  @Oatmeal @google driverless @TeslaMotors ? Ooo...\n",
       "3          5  SO MUCH AWESOME! Amazing video for GoogleÌ¢‰âÂ...\n",
       "4          5  @google is making driverless cars which is awe..."
      ]
     },
     "execution_count": 567,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import string\n",
    "import nltk\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "#Step 1 Reading the training dataset\n",
    "train = pd.read_csv(\"train.csv\")\n",
    "\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 568,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     except trains very capital intensive have limi...\n",
       "1                                want google driverless\n",
       "2                   driverless ooooh that would awesome\n",
       "3     much awesome amazing video google next phase d...\n",
       "4                  making driverless cars which awesome\n",
       "5       could call driverless tell will take there whoo\n",
       "6                           google driverless cars take\n",
       "7     driverless taxis would probably amazing applic...\n",
       "8         this whole driverless prototype thing awesome\n",
       "9     google driverless brake pedal steering wheel t...\n",
       "10    riding driverless bucket list this damn exciti...\n",
       "11      this future places will wait driverless scichat\n",
       "12    driverless great idea someone like condition t...\n",
       "13                            this need driverless cars\n",
       "14       google developed driverless that fucking chill\n",
       "15    wait driverless cars soon rarely drive many li...\n",
       "16                     they coming with driverless cars\n",
       "17                                      want driverless\n",
       "18    driverless cars could also drunk driving forev...\n",
       "19    skynet driverless cars sick accidents injuries...\n",
       "Name: clean_text, dtype: object"
      ]
     },
     "execution_count": 568,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "#Step 2= Removing Twitter Handles\n",
    "\n",
    "def remove_pattern(input_txt, pattern):\n",
    "    r = re.findall(pattern, input_txt)\n",
    "    for i in r:\n",
    "        input_txt = re.sub(i, '', input_txt)\n",
    "        \n",
    "    return input_txt \n",
    "\n",
    "# creating new column tidy_tweet & remove twitter handles (@user)\n",
    "train['clean_text'] = np.vectorize(remove_pattern)(train['text'], \"@[\\w]*\")\n",
    "\n",
    "train['clean_text'] = train['clean_text'].str.replace('http\\S+|www.\\S+', '', case=False)\n",
    "\n",
    "#Step 3= Removing Punctuations, Numbers, and Special Character \n",
    "train['clean_text'] = train['clean_text'].str.replace('[^A-Za-z0-9]+', ' ')\n",
    "                          \n",
    "# Step 4 remove stopwords\n",
    "train['clean_text'] = train['clean_text'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))\n",
    "\n",
    "#Step 5 to lowercase\n",
    "\n",
    "train['clean_text']= train['clean_text'].apply(lambda x: ''.join([w for w in x.lower()]))\n",
    "\n",
    "train['clean_text'].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 569,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 3 3 3 3 3 3 3 4 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 5 3 3 3 3 3 3 4 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 1 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 4 3 3 3 3 3 3 1 3 3 3 3 3 3 3 3 3 3 4 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 4 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 3 3 3 3 4 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 3 3 3 4 3 3 3 3 3 3 3]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(train['clean_text'], train['sentiment'],test_size = 0.2)\n",
    "\n",
    "\n",
    "## for transforming the 80% of the train data ##\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer as CountVectorizer\n",
    "count_vect = CountVectorizer(stop_words='english')\n",
    "x_train_counts = count_vect.fit_transform(x_train)\n",
    "\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "transformer = TfidfTransformer(norm='l2',sublinear_tf=True)\n",
    "x_train_tfidf = transformer.fit_transform(x_train_counts)\n",
    "x_train_tfidf.shape\n",
    "\n",
    "## for transforming the 20% of the train data which is being used for validation ##\n",
    "x_test_counts = count_vect.transform(x_test)\n",
    "x_test_tfidf = transformer.transform(x_test_counts)\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.ensemble.forest import RandomForestClassifier\n",
    "model = RandomForestClassifier(n_estimators=980) # >>>>>typo error <<<< :  'i' went missing in RandomForestClassifier\n",
    "model.fit(x_train_tfidf,y_train)\n",
    "\n",
    "prediction = model.predict(x_test_tfidf)\n",
    "print(prediction) # >>>typo error <<<  : 's'was present at the end of prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 570,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6802030456852792\n"
     ]
    }
   ],
   "source": [
    "# testing model's accuracy with accuracy score\n",
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(y_test,prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 571,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix: \n",
      " [[  0   0   4   0   0]\n",
      " [  0   0  14   0   0]\n",
      " [  1   0 128   1   0]\n",
      " [  1   0  31   5   0]\n",
      " [  0   0  10   1   1]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "print('confusion matrix: \\n',confusion_matrix(y_test, prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 572,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.00      0.00      0.00         4\n",
      "           2       0.00      0.00      0.00        14\n",
      "           3       0.68      0.98      0.81       130\n",
      "           4       0.71      0.14      0.23        37\n",
      "           5       1.00      0.08      0.15        12\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       197\n",
      "   macro avg       0.48      0.24      0.24       197\n",
      "weighted avg       0.65      0.68      0.58       197\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 573,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "SEED = 2000\n",
    "x_train, x_validation_and_test, y_train, y_validation_and_test = train_test_split(train['clean_text'], train['sentiment'], test_size=.02, random_state=SEED)\n",
    "x_validation, x_test, y_validation, y_test = train_test_split(x_validation_and_test, y_validation_and_test, test_size=.5, random_state=SEED)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 574,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc=\"progress-bar\")\n",
    "from gensim.models import Doc2Vec\n",
    "from gensim.models.doc2vec import LabeledSentence\n",
    "import multiprocessing\n",
    "from sklearn import utils\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vectors(model, corpus, size):\n",
    "    vecs = np.zeros((len(corpus), size))\n",
    "    n = 0\n",
    "    for i in corpus.index:\n",
    "        prefix = 'all_' + str(i)\n",
    "        vecs[n] = model.docvecs[prefix]\n",
    "        n += 1\n",
    "    return vecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 576,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_concat_vectors(model1,model2, corpus, size):\n",
    "    vecs = np.zeros((len(corpus), size))\n",
    "    n = 0\n",
    "    for i in corpus.index:\n",
    "        prefix = 'all_' + str(i)\n",
    "        vecs[n] = np.append(model1.docvecs[prefix],model2.docvecs[prefix])\n",
    "        n += 1\n",
    "    return vecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 577,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.phrases import Phrases\n",
    "from gensim.models.phrases import Phraser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 578,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_train = [t.split() for t in x_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {},
   "outputs": [],
   "source": [
    "phrases = Phrases(tokenized_train)\n",
    "bigram = Phraser(phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {},
   "outputs": [],
   "source": [
    "def labelize_tweets_bg(tweets,label):\n",
    "    result = []\n",
    "    prefix = label\n",
    "    for i, t in zip(tweets.index, tweets):\n",
    "        result.append(LabeledSentence(bigram[t.split()], [prefix + '_%s' % i]))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_x = pd.concat([x_train,x_validation,x_test])\n",
    "all_x_w2v_bg = labelize_tweets_bg(all_x, 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/gensim/models/doc2vec.py:366: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\n",
      "  warnings.warn(\"The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\")\n",
      "100%|██████████| 981/981 [00:00<00:00, 1085936.19it/s]\n"
     ]
    }
   ],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "model_bg_dbow = Doc2Vec(dm=0, size=100, negative=5, min_count=2, workers=cores, alpha=0.065, min_alpha=0.065)\n",
    "model_bg_dbow.build_vocab([x for x in tqdm(all_x_w2v_bg)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 583,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dbow\n",
    "\n",
    "train_vecs_dbow_bg = get_vectors(model_bg_dbow, x_train, 100)\n",
    "validation_vecs_dbow_bg = get_vectors(model_bg_dbow, x_validation, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 584,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=980, n_jobs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 584,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier(n_estimators=980)\n",
    "clf.fit(train_vecs_dbow_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 585,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 585,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(validation_vecs_dbow_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 587,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 587,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2 = model2 = LogisticRegression()\n",
    "clf2.fit(train_vecs_dbow_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 588,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 588,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2.score(validation_vecs_dbow_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 589,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 589,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dbow_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 590,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2"
      ]
     },
     "execution_count": 590,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf3.score(validation_vecs_dbow_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 591,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1"
      ]
     },
     "execution_count": 591,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SVM\n",
    "from sklearn import svm\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dbow_bg, y_train)\n",
    "clf4.score(validation_vecs_dbow_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 592,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 592,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# K-NN Classifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dbow_bg, y_train)\n",
    "clf5.score(validation_vecs_dbow_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 593,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bg_dbow.save('d2v_model_bg_dbow.doc2vec')\n",
    "model_bg_dbow = Doc2Vec.load('d2v_model_bg_dbow.doc2vec')\n",
    "model_bg_dbow.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 594,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DMC Bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 595,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/gensim/models/doc2vec.py:366: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\n",
      "  warnings.warn(\"The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\")\n",
      "100%|██████████| 981/981 [00:00<00:00, 644519.46it/s]\n"
     ]
    }
   ],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "model_bg_dmc = Doc2Vec(dm=1, dm_concat=1, size=100, window=2, negative=5, min_count=2, workers=cores, alpha=0.065, min_alpha=0.065)\n",
    "model_bg_dmc.build_vocab([x for x in tqdm(all_x_w2v_bg)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 596,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 981/981 [00:00<00:00, 608940.69it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1171920.31it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1613573.42it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1515510.95it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1579505.65it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1552098.16it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 894286.51it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1385857.94it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1629549.40it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 794940.54it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 355044.63it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 850829.66it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 856853.86it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 829224.55it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 775025.85it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 945887.87it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 938766.19it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1023026.41it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 977342.57it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 955109.62it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 980837.24it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1033822.17it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1037210.04it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1813403.36it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1715136.40it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1231182.59it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1709435.91it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1954685.14it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1075434.45it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 721861.79it/s]\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(30):\n",
    "    model_bg_dmc.train(utils.shuffle([x for x in tqdm(all_x_w2v_bg)]), total_examples=len(all_x_w2v_bg), epochs=1)\n",
    "    model_bg_dmc.alpha -= 0.002\n",
    "    model_bg_dmc.min_alpha = model_bg_dmc.alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 597,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('cars', 0.8473590016365051),\n",
       " ('action', 0.7662856578826904),\n",
       " ('developing', 0.6675175428390503),\n",
       " ('robot', 0.6353542804718018),\n",
       " ('driver_less', 0.6329611539840698),\n",
       " ('powered', 0.6269574165344238),\n",
       " ('flying', 0.6230174899101257),\n",
       " ('them', 0.622847855091095),\n",
       " ('based', 0.6149864792823792),\n",
       " ('mass', 0.6097685098648071)]"
      ]
     },
     "execution_count": 597,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_bg_dmc.most_similar('driverless')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 598,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('action', 0.6619821786880493),\n",
       " ('electric', 0.5447269678115845),\n",
       " ('mass', 0.5143768787384033),\n",
       " ('transit', 0.49883246421813965),\n",
       " ('allow', 0.4960136115550995)]"
      ]
     },
     "execution_count": 598,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_bg_dmc.most_similar(positive=['car', 'driverless'], negative=['driver'], topn=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 599,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dmc_bg = get_vectors(model_bg_dmc, x_train, 100)\n",
    "validation_vecs_dmc_bg = get_vectors(model_bg_dmc, x_validation, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 600,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 600,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dmc_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 601,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 601,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(validation_vecs_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 602,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=980, n_jobs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 602,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dmc_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 603,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7"
      ]
     },
     "execution_count": 603,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2.score(validation_vecs_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 604,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 604,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tree\n",
    "from sklearn import tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dmc_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 605,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7"
      ]
     },
     "execution_count": 605,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf3.score(validation_vecs_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 606,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dmc_bg, y_train)\n",
    "clf4.score(validation_vecs_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 607,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 607,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf4.score(validation_vecs_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7"
      ]
     },
     "execution_count": 608,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dmc_bg, y_train)\n",
    "clf5.score(validation_vecs_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 609,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DMM BIgram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 610,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/gensim/models/doc2vec.py:366: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\n",
      "  warnings.warn(\"The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\")\n",
      "100%|██████████| 981/981 [00:00<00:00, 558367.79it/s]\n"
     ]
    }
   ],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "model_bg_dmm = Doc2Vec(dm=1, dm_mean=1, size=100, window=4, negative=5, min_count=2, workers=cores, alpha=0.065, min_alpha=0.065)\n",
    "model_bg_dmm.build_vocab([x for x in tqdm(all_x_w2v_bg)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 611,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 981/981 [00:00<00:00, 1455469.48it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1227875.92it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1393366.82it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1495135.26it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1202048.56it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1927218.84it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 2006149.30it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1994479.99it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1562115.50it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1466362.16it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1855100.19it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1789739.98it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1871980.08it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1523366.24it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1851760.68it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 950695.99it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1768204.65it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1727377.09it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1826281.50it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1886571.40it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1861815.49it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1527891.65it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1393366.82it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1733198.07it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 843503.94it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1472660.07it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 687947.20it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1857612.74it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 876755.22it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1502780.21it/s]\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(30):\n",
    "    model_bg_dmm.train(utils.shuffle([x for x in tqdm(all_x_w2v_bg)]), total_examples=len(all_x_w2v_bg), epochs=1)\n",
    "    model_bg_dmm.alpha -= 0.002\n",
    "    model_bg_dmm.min_alpha = model_bg_dmm.alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 612,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dmm_bg = get_vectors(model_bg_dmm, x_train, 100)\n",
    "validation_vecs_dmm_bg = get_vectors(model_bg_dmm, x_validation, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 613,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 613,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dmm_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 614,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 614,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(validation_vecs_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 615,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7"
      ]
     },
     "execution_count": 615,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dmm_bg, y_train)\n",
    "clf2.score(validation_vecs_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 616,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 616,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dmm_bg, y_train)\n",
    "clf3.score(validation_vecs_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 617,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 617,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dmm_bg, y_train)\n",
    "clf4.score(validation_vecs_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 618,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 618,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dmm_bg, y_train)\n",
    "clf5.score(validation_vecs_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 619,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bg_dmc.save('d2v_model_bg_dmc.doc2vec')\n",
    "model_bg_dmc = Doc2Vec.load('d2v_model_bg_dmc.doc2vec')\n",
    "model_bg_dmc.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 620,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dbow_dmc_bg = get_concat_vectors(model_bg_dbow,model_bg_dmc, x_train, 200)\n",
    "validation_vecs_dbow_dmc_bg = get_concat_vectors(model_bg_dbow,model_bg_dmc, x_validation, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 621,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 621,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dbow_dmc_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 622,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 622,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(validation_vecs_dbow_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 623,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 623,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dbow_dmc_bg, y_train)\n",
    "clf2.score(validation_vecs_dbow_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 624,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 624,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dbow_dmc_bg, y_train)\n",
    "clf3.score(validation_vecs_dbow_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 625,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 625,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dbow_dmc_bg, y_train)\n",
    "clf4.score(validation_vecs_dbow_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 626,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7"
      ]
     },
     "execution_count": 626,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dbow_dmc_bg, y_train)\n",
    "clf5.score(validation_vecs_dbow_dmc_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 627,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dbow_dmm_bg = get_concat_vectors(model_bg_dbow,model_bg_dmm, x_train, 200)\n",
    "validation_vecs_dbow_dmm_bg = get_concat_vectors(model_bg_dbow,model_bg_dmm, x_validation, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 628,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 628,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dbow_dmm_bg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 629,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 629,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(validation_vecs_dbow_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 630,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 630,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dbow_dmm_bg, y_train)\n",
    "clf2.score(validation_vecs_dbow_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 631,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 631,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dbow_dmm_bg, y_train)\n",
    "clf3.score(validation_vecs_dbow_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 632,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 632,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dbow_dmm_bg, y_train)\n",
    "clf4.score(validation_vecs_dbow_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 633,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 633,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dbow_dmm_bg, y_train)\n",
    "clf5.score(validation_vecs_dbow_dmm_bg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 634,
   "metadata": {},
   "outputs": [],
   "source": [
    "#trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 635,
   "metadata": {},
   "outputs": [],
   "source": [
    "tg_phrases = Phrases(bigram[tokenized_train])\n",
    "trigram = Phraser(tg_phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 636,
   "metadata": {},
   "outputs": [],
   "source": [
    "def labelize_tweets_tg(tweets,label):\n",
    "    result = []\n",
    "    prefix = label\n",
    "    for i, t in zip(tweets.index, tweets):\n",
    "        result.append(LabeledSentence(trigram[bigram[t.split()]], [prefix + '_%s' % i]))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 637,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_x = pd.concat([x_train,x_validation,x_test])\n",
    "all_x_w2v_tg = labelize_tweets_tg(all_x, 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 638,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/gensim/models/doc2vec.py:366: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\n",
      "  warnings.warn(\"The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\")\n",
      "100%|██████████| 981/981 [00:00<00:00, 424624.58it/s]\n"
     ]
    }
   ],
   "source": [
    "model_tg_dbow = Doc2Vec(dm=0, size=100, negative=5, min_count=2, workers=cores, alpha=0.065, min_alpha=0.065)\n",
    "model_tg_dbow.build_vocab([x for x in tqdm(all_x_w2v_tg)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 639,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 981/981 [00:00<00:00, 476945.89it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1806238.90it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1459599.94it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1158393.08it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1720155.61it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1358854.76it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1121147.74it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1605389.08it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1935377.34it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1549176.29it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1565086.43it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1226046.55it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1557385.40it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1574067.42it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1112358.00it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1152875.38it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 996756.84it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 551778.49it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1742741.31it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1616108.49it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1355274.12it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1754632.08it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1886571.40it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1873684.98it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1803863.32it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1276244.49it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1144537.48it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1211248.81it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1460117.89it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1858451.77it/s]\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(30):\n",
    "    model_tg_dbow.train(utils.shuffle([x for x in tqdm(all_x_w2v_tg)]), total_examples=len(all_x_w2v_tg), epochs=1)\n",
    "    model_tg_dbow.alpha -= 0.002\n",
    "    model_tg_dbow.min_alpha = model_tg_dbow.alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 640,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dbow_tg = get_vectors(model_tg_dbow, x_train, 100)\n",
    "validation_vecs_dbow_tg = get_vectors(model_tg_dbow, x_validation, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 641,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 641,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dbow_tg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 642,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 642,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(validation_vecs_dbow_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 643,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dbow_tg, y_train)\n",
    "clf2.score(validation_vecs_dbow_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 644,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3"
      ]
     },
     "execution_count": 644,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dbow_tg, y_train)\n",
    "clf3.score(validation_vecs_dbow_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 645,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 645,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dbow_tg, y_train)\n",
    "clf4.score(validation_vecs_dbow_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 646,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 646,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dbow_tg, y_train)\n",
    "clf5.score(validation_vecs_dbow_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 647,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_tg_dbow.save('d2v_model_tg_dbow.doc2vec')\n",
    "model_tg_dbow = Doc2Vec.load('d2v_model_tg_dbow.doc2vec')\n",
    "model_tg_dbow.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 648,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DMC Trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 649,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/gensim/models/doc2vec.py:366: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\n",
      "  warnings.warn(\"The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.\")\n",
      "100%|██████████| 981/981 [00:00<00:00, 1058285.04it/s]\n"
     ]
    }
   ],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "model_tg_dmc = Doc2Vec(dm=1, dm_concat=1, size=100, window=2, negative=5, min_count=2, workers=cores, alpha=0.065, min_alpha=0.065)\n",
    "model_tg_dmc.build_vocab([x for x in tqdm(all_x_w2v_tg)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 650,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 981/981 [00:00<00:00, 556781.09it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1589881.08it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1621841.63it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1645844.89it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1607898.49it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1438675.60it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1475300.19it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1143583.16it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1031489.65it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 856675.46it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 659815.94it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 983886.23it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 853831.13it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1513838.20it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 914968.25it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 989803.28it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 801443.75it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1833606.16it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1833606.16it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1479544.13it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1592342.19it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1397151.86it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1645844.89it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1745698.86it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 907301.48it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1309968.87it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1486492.86it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1584981.60it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1812604.50it/s]\n",
      "100%|██████████| 981/981 [00:00<00:00, 1754632.08it/s]\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(30):\n",
    "    model_tg_dmc.train(utils.shuffle([x for x in tqdm(all_x_w2v_tg)]), total_examples=len(all_x_w2v_tg), epochs=1)\n",
    "    model_tg_dmc.alpha -= 0.002\n",
    "    model_tg_dmc.min_alpha = model_tg_dmc.alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 651,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dmc_tg = get_vectors(model_tg_dmc, x_train, 100)\n",
    "validation_vecs_dmc_tg = get_vectors(model_tg_dmc, x_validation, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 652,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 652,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dmc_tg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 653,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 653,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(validation_vecs_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 654,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 654,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dmc_tg, y_train)\n",
    "clf2.score(validation_vecs_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 655,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dmc_tg, y_train)\n",
    "clf3.score(validation_vecs_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dmc_tg, y_train)\n",
    "clf4.score(validation_vecs_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dmc_tg, y_train)\n",
    "clf5.score(validation_vecs_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_tg_dmc.save('d2v_model_tg_dmc.doc2vec')\n",
    "model_tg_dmc = Doc2Vec.load('d2v_model_tg_dmc.doc2vec')\n",
    "model_tg_dmc.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DMM trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "model_tg_dmm = Doc2Vec(dm=1, dm_mean=1, size=100, window=4, negative=5, min_count=2, workers=cores, alpha=0.065, min_alpha=0.065)\n",
    "model_tg_dmm.build_vocab([x for x in tqdm(all_x_w2v_tg)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(30):\n",
    "    model_tg_dmm.train(utils.shuffle([x for x in tqdm(all_x_w2v_tg)]), total_examples=len(all_x_w2v_tg), epochs=1)\n",
    "    model_tg_dmm.alpha -= 0.002\n",
    "    model_tg_dmc.min_alpha = model_tg_dmc.alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dmm_tg = get_vectors(model_tg_dmm, x_train, 100)\n",
    "validation_vecs_dmm_tg = get_vectors(model_tg_dmm, x_validation, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dmm_tg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.score(validation_vecs_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dmm_tg, y_train)\n",
    "clf2.score(validation_vecs_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dmm_tg, y_train)\n",
    "clf3.score(validation_vecs_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dmm_tg, y_train)\n",
    "clf4.score(validation_vecs_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dmm_tg, y_train)\n",
    "clf5.score(validation_vecs_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_tg_dmm.save('d2v_model_tg_dmm.doc2vec')\n",
    "model_tg_dmm = Doc2Vec.load('d2v_model_tg_dmm.doc2vec')\n",
    "model_tg_dmm.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dbow_dmc_tg = get_concat_vectors(model_tg_dbow,model_tg_dmc, x_train, 200)\n",
    "validation_vecs_dbow_dmc_tg = get_concat_vectors(model_tg_dbow,model_tg_dmc, x_validation, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dbow_dmc_tg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.score(validation_vecs_dbow_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dbow_dmc_tg, y_train)\n",
    "clf2.score(validation_vecs_dbow_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dbow_dmc_tg, y_train)\n",
    "clf3.score(validation_vecs_dbow_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dbow_dmc_tg, y_train)\n",
    "clf4.score(validation_vecs_dbow_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dbow_dmc_tg, y_train)\n",
    "clf5.score(validation_vecs_dbow_dmc_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_dbow_dmm_tg = get_concat_vectors(model_tg_dbow,model_tg_dmm, x_train, 200)\n",
    "validation_vecs_dbow_dmm_tg = get_concat_vectors(model_tg_dbow,model_tg_dmm, x_validation, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_dbow_dmm_tg, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.score(validation_vecs_dbow_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_dbow_dmc_tg, y_train)\n",
    "clf2.score(validation_vecs_dbow_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_dbow_dmc_tg, y_train)\n",
    "clf3.score(validation_vecs_dbow_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_dbow_dmm_tg, y_train)\n",
    "clf4.score(validation_vecs_dbow_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_dbow_dmm_tg, y_train)\n",
    "clf5.score(validation_vecs_dbow_dmm_tg, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_tg_dmm.save('d2v_model_ug_dbow.doc2vec')\n",
    "model_ug_dbow = Doc2Vec.load('d2v_model_ug_dbow.doc2vec')\n",
    "model_tg_dmm = Doc2Vec.load('d2v_model_tg_dmm.doc2vec')\n",
    "model_ug_dbow.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)\n",
    "model_tg_dmm.delete_temporary_training_data(keep_doctags_vectors=True, keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vecs_ugdbow_tgdmm = get_concat_vectors(model_ug_dbow,model_tg_dmm, x_train, 200)\n",
    "validation_vecs_ugdbow_tgdmm = get_concat_vectors(model_ug_dbow,model_tg_dmm, x_validation, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(train_vecs_ugdbow_tgdmm, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.score(validation_vecs_ugdbow_tgdmm, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RF\n",
    "clf2 = RandomForestClassifier(n_estimators=980)\n",
    "clf2.fit(train_vecs_ugdbow_tgdmm, y_train)\n",
    "clf2.score(validation_vecs_ugdbow_tgdmm, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tree\n",
    "# creates a ML model based on parameters\n",
    "clf3 = tree.DecisionTreeClassifier(criterion='gini') \n",
    "clf3.fit(train_vecs_ugdbow_tgdmm, y_train)\n",
    "clf3.score(validation_vecs_ugdbow_tgdmm, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVM\n",
    "clf4 = OneVsRestClassifier(svm.SVC(gamma=0.01, C=100., probability=True, class_weight='balanced', kernel='linear'))\n",
    "clf4.fit(train_vecs_ugdbow_tgdmm, y_train)\n",
    "clf4.score(validation_vecs_ugdbow_tgdmm, y_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clf5 = KNeighborsClassifier(n_neighbors=5)\n",
    "clf5.fit(train_vecs_ugdbow_tgdmm, y_train)\n",
    "clf5.score(validation_vecs_ugdbow_tgdmm, y_validation)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}