models_thesis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part of the code used for my thesis \"Analysis of first-year university student dropout through machine learning models: A comparison between universities\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from plotnine import *\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#from sklearn import preprocessing\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "#from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "\n",
    "# Models\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "# from sklearn.naive_bayes import MultinomialNB\n",
    "# from sklearn.naive_bayes import ComplementNB\n",
    "# from sklearn.naive_bayes import CategoricalNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "# Neural networks\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import *\n",
    "from tensorflow.keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "if (!(\"Notification\" in window)) {\n",
       "    alert(\"This browser does not support desktop notifications, so the %%notify magic will not work.\");\n",
       "} else if (Notification.permission !== 'granted' && Notification.permission !== 'denied') {\n",
       "    Notification.requestPermission(function (permission) {\n",
       "        if(!('permission' in Notification)) {\n",
       "            Notification.permission = permission;\n",
       "        }\n",
       "    })\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Browser notification for cell execution\n",
    "%load_ext jupyternotify\n",
    "# First line in cell\n",
    "# %%notify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save folds for the entire thesis\n",
    "# from https://stackoverflow.com/questions/54317242/k-fold-cross-validation-save-folds-for-different-models\n",
    "skf = StratifiedKFold(n_splits=10, shuffle=True)\n",
    "X = df.drop(columns=[\"semestres\"])\n",
    "y = df[\"semestres\"]\n",
    "folds = {}\n",
    "count = 1\n",
    "for train_index, test_index in skf.split(X, y):\n",
    "    #print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    #X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "    #y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "    folds['fold_{}'.format(count)] = {}\n",
    "    folds['fold_{}'.format(count)]['train_index'] = train_index.tolist()\n",
    "    folds['fold_{}'.format(count)]['test_index'] = test_index.tolist()\n",
    "    count += 1\n",
    "print(len(folds) == 10) #assert we have the same number of splits\n",
    "\n",
    "#print(len(train_index))\n",
    "#print(len(test_index))\n",
    "\n",
    "#dump folds to json, dont overwrite!\n",
    "# import json\n",
    "# with open('folds_v2.json', 'w') as fp:\n",
    "#     json.dump(folds, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load dict to be used\n",
    "import json\n",
    "with open('folds.json') as f:\n",
    "    kfolds = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train models over both-universities-dataset, using cross validation and evaluate using 6 different metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data both-universities-dataset\n",
    "df=pd.read_csv(\"BD_both.csv\", decimal = \".\")\n",
    "df=df.drop(columns=[\"Unnamed: 0\"]) #extra column\n",
    "\n",
    "# Minor fixes\n",
    "df.loc[df[\"region\"] == \"7.0\", \"region\"] = \"7\"\n",
    "df.loc[df[\"region\"] == \"6.0\", \"region\"] = \"6\"\n",
    "df.loc[df[\"region\"] == \"13.0\", \"region\"] = \"13\"\n",
    "df.loc[df[\"region\"] == \"5.0\", \"region\"] = \"5\"\n",
    "\n",
    "df=df.drop(columns=[\"ID\"])\n",
    "df=df.drop(columns=[\"year\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Loss and metric for Neural network'''\n",
    "def f1(y_true, y_pred):\n",
    "    y_pred = K.round(y_pred)\n",
    "    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n",
    "    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n",
    "    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n",
    "    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n",
    "\n",
    "    p = tp / (tp + fp + K.epsilon())\n",
    "    r = tp / (tp + fn + K.epsilon())\n",
    "\n",
    "    f1 = 2*p*r / (p+r+K.epsilon())\n",
    "    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n",
    "    return K.mean(f1)\n",
    "\n",
    "def f1_loss(y_true, y_pred):\n",
    "    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n",
    "    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n",
    "    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n",
    "    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n",
    "\n",
    "    p = tp / (tp + fp + K.epsilon())\n",
    "    r = tp / (tp + fn + K.epsilon())\n",
    "\n",
    "    f1 = 2*p*r / (p+r+K.epsilon())\n",
    "    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n",
    "    return 1 - K.mean(f1)\n",
    "\n",
    "''' Evaluation with all metrics'''\n",
    "def metrics_evaluate(y_test, y_pred, X_test, model, auc_flag=True):\n",
    "    f1Score = f1_score(y_test, y_pred)\n",
    "    f1Score0 = f1_score(y_test, y_pred, pos_label=0)\n",
    "    acc = accuracy_score(y_test, y_pred)\n",
    "    if auc_flag == True:\n",
    "        aucScore = roc_auc_score(y_test, model.predict_proba(X_test)[:, 1])\n",
    "    else:\n",
    "        aucScore = 0\n",
    "    prec = precision_score(y_test, y_pred)\n",
    "    rec = recall_score(y_test, y_pred)\n",
    "    \n",
    "    return f1Score,f1Score0,acc,aucScore,prec,rec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold_1\n",
      "random\n",
      "[[88 67]\n",
      " [21 24]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_2\n",
      "random\n",
      "[[91 84]\n",
      " [25 18]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_3\n",
      "random\n",
      "[[ 69 101]\n",
      " [ 31  23]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_4\n",
      "random\n",
      "[[97 85]\n",
      " [16 29]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_5\n",
      "random\n",
      "[[87 74]\n",
      " [26 25]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_6\n",
      "random\n",
      "[[97 77]\n",
      " [28 22]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_7\n",
      "random\n",
      "[[102  76]\n",
      " [ 25  20]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_8\n",
      "random\n",
      "[[80 76]\n",
      " [18 20]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_9\n",
      "random\n",
      "[[85 95]\n",
      " [22 19]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n",
      "fold_10\n",
      "random\n",
      "[[91 84]\n",
      " [24 26]]\n",
      "dectree\n",
      "logreg\n",
      "naive\n",
      "knn\n",
      "svm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\diego\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ranforest\n",
      "gradboost\n",
      "neural\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>F1+ Means</th>\n",
       "      <th>F1+ Error</th>\n",
       "      <th>F1- Means</th>\n",
       "      <th>F1- Error</th>\n",
       "      <th>Acc Means</th>\n",
       "      <th>Acc Error</th>\n",
       "      <th>AUC</th>\n",
       "      <th>AUC Error</th>\n",
       "      <th>Prec Means</th>\n",
       "      <th>Prec Error</th>\n",
       "      <th>Rec Means</th>\n",
       "      <th>Rec Error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>random</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>knn</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>svm</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dectree</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ranforest</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>gradboost</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>naive</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>logreg</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.57</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>neural</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Model  F1+ Means  F1+ Error  F1- Means  F1- Error  Acc Means  \\\n",
       "0     random       0.30       0.04       0.63       0.04       0.51   \n",
       "1        knn       0.40       0.03       0.59       0.03       0.51   \n",
       "2        svm       0.41       0.02       0.63       0.04       0.54   \n",
       "3    dectree       0.40       0.05       0.67       0.05       0.58   \n",
       "4  ranforest       0.40       0.04       0.68       0.03       0.58   \n",
       "5  gradboost       0.40       0.04       0.72       0.02       0.62   \n",
       "6      naive       0.38       0.02       0.57       0.04       0.49   \n",
       "7     logreg       0.39       0.04       0.57       0.05       0.50   \n",
       "8     neural       0.41       0.04       0.69       0.06       0.60   \n",
       "\n",
       "   Acc Error   AUC  AUC Error  Prec Means  Prec Error  Rec Means  Rec Error  \n",
       "0       0.04  0.50       0.03        0.22        0.03       0.49       0.07  \n",
       "1       0.03  0.64       0.04        0.27        0.03       0.77       0.07  \n",
       "2       0.03  0.65       0.02        0.28        0.02       0.75       0.05  \n",
       "3       0.04  0.65       0.03        0.29        0.04       0.65       0.08  \n",
       "4       0.03  0.66       0.03        0.29        0.04       0.65       0.05  \n",
       "5       0.02  0.65       0.03        0.30        0.04       0.59       0.05  \n",
       "6       0.03  0.66       0.02        0.26        0.02       0.74       0.03  \n",
       "7       0.04  0.66       0.04        0.26        0.03       0.76       0.09  \n",
       "8       0.06  0.66       0.03        0.30        0.04       0.65       0.09  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%notify\n",
    "\n",
    "# For simplicity, dict for metric\n",
    "scores = {\"random\": np.zeros(10),\n",
    "             \"dectree\": np.zeros(10),\n",
    "             \"gradboost\": np.zeros(10),\n",
    "             \"logreg\": np.zeros(10),\n",
    "             \"naive\": np.zeros(10),\n",
    "             \"knn\": np.zeros(10),\n",
    "             \"svm\": np.zeros(10),\n",
    "             \"ranforest\": np.zeros(10),\n",
    "             \"neural\": np.zeros(10)\n",
    "         }\n",
    "scores0 = {\"random\": np.zeros(10),\n",
    "             \"dectree\": np.zeros(10),\n",
    "             \"gradboost\": np.zeros(10),\n",
    "             \"logreg\": np.zeros(10),\n",
    "             \"naive\": np.zeros(10),\n",
    "             \"knn\": np.zeros(10),\n",
    "             \"svm\": np.zeros(10),\n",
    "             \"ranforest\": np.zeros(10),\n",
    "             \"neural\": np.zeros(10)\n",
    "         }\n",
    "\n",
    "accuracy = {\"random\": np.zeros(10),\n",
    "             \"dectree\": np.zeros(10),\n",
    "             \"gradboost\": np.zeros(10),\n",
    "             \"logreg\": np.zeros(10),\n",
    "             \"naive\": np.zeros(10),\n",
    "             \"knn\": np.zeros(10),\n",
    "             \"svm\": np.zeros(10),\n",
    "             \"ranforest\": np.zeros(10),\n",
    "             \"neural\": np.zeros(10)\n",
    "         }\n",
    "\n",
    "auc = {\"random\": np.zeros(10),\n",
    "             \"dectree\": np.zeros(10),\n",
    "             \"gradboost\": np.zeros(10),\n",
    "             \"logreg\": np.zeros(10),\n",
    "             \"naive\": np.zeros(10),\n",
    "             \"knn\": np.zeros(10),\n",
    "             \"svm\": np.zeros(10),\n",
    "             \"ranforest\": np.zeros(10),\n",
    "             \"neural\": np.zeros(10)\n",
    "         }\n",
    "\n",
    "precision = {\"random\": np.zeros(10),\n",
    "             \"dectree\": np.zeros(10),\n",
    "             \"gradboost\": np.zeros(10),\n",
    "             \"logreg\": np.zeros(10),\n",
    "             \"naive\": np.zeros(10),\n",
    "             \"knn\": np.zeros(10),\n",
    "             \"svm\": np.zeros(10),\n",
    "             \"ranforest\": np.zeros(10),\n",
    "             \"neural\": np.zeros(10)\n",
    "         }\n",
    "\n",
    "recall = {\"random\": np.zeros(10),\n",
    "          \"dectree\": np.zeros(10),\n",
    "             \"gradboost\": np.zeros(10),\n",
    "             \"logreg\": np.zeros(10),\n",
    "             \"naive\": np.zeros(10),\n",
    "             \"knn\": np.zeros(10),\n",
    "             \"svm\": np.zeros(10),\n",
    "             \"ranforest\": np.zeros(10),\n",
    "             \"neural\": np.zeros(10)\n",
    "         }\n",
    "\n",
    "frame = pd.DataFrame(columns=[\"Model\",\n",
    "                              \"F1+ Means\",\n",
    "                              \"F1+ Error\",\n",
    "                              \"F1- Means\",\n",
    "                              \"F1- Error\",\n",
    "                              \"Acc Means\",\n",
    "                              \"Acc Error\",\n",
    "                              \"AUC\",\n",
    "                              \"AUC Error\",\n",
    "                             \"Prec Means\",\n",
    "                              \"Prec Error\",\n",
    "                              \"Rec Means\",\n",
    "                              \"Rec Error\"])\n",
    "\n",
    "var_num = [\"nem\", \"ranking\", \"mat\", \"lang\", \"optional\", \"pps\", \"preference\"]\n",
    "var_cat = [\"gender\", \"school\", \"admission\", \"commune\",\"region\", \"university\"]\n",
    "\n",
    "df_oneHot = pd.get_dummies(df, prefix=['gender',\n",
    "                                'school',\n",
    "                                'admission',\n",
    "                                'commune',\n",
    "                                'region',\n",
    "                                'university'\n",
    "                               ],\n",
    "                              columns=['gender',\n",
    "                                      'school',\n",
    "                                       'admission',\n",
    "                                       'commune',\n",
    "                                       'region',\n",
    "                                       'university'])\n",
    "\n",
    "fold_count = 1\n",
    "for fold, indexes in kfolds.items():\n",
    "    aucScore = 0\n",
    "    print(fold)\n",
    "    \n",
    "    train_index = indexes['train_index']\n",
    "    test_index = indexes['test_index']\n",
    "    trainSet = df.loc[train_index] # slice using kfold\n",
    "    testSet = df.loc[test_index]   # slice using kfold\n",
    "    \n",
    "    trainSet_oneHot = df_oneHot.loc[train_index] # slice using kfold\n",
    "    testSet_oneHot = df_oneHot.loc[test_index]   # slice using kfold\n",
    "    \n",
    "    # Drop nulls\n",
    "    trainSet = trainSet.dropna()\n",
    "    testSet = testSet.dropna()\n",
    "    \n",
    "    trainSet_oneHot = trainSet_oneHot.dropna()\n",
    "    testSet_oneHot = testSet_oneHot.dropna()\n",
    "    \n",
    "    #Split X and Y\n",
    "    #Undersampling\n",
    "    n = pd.value_counts(trainSet[\"semestres\"])[0]-pd.value_counts(trainSet[\"semestres\"])[1]\n",
    "    trainSet = trainSet.drop(trainSet[trainSet[\"semestres\"]== 0].sample(n=n, random_state=fold_count).index)\n",
    "    \n",
    "    n_oneHot = pd.value_counts(trainSet_oneHot[\"semestres\"])[0]-pd.value_counts(trainSet_oneHot[\"semestres\"])[1]\n",
    "    trainSet_oneHot = trainSet_oneHot.drop(trainSet_oneHot[trainSet_oneHot[\"semestres\"]== 0].sample(n=n, random_state=fold_count).index)\n",
    "    \n",
    "    X_train = trainSet.drop(columns=\"semestres\")\n",
    "    y_train = trainSet[\"semestres\"]\n",
    "    X_test = testSet.drop(columns=\"semestres\")\n",
    "    y_test = testSet[\"semestres\"]\n",
    "    \n",
    "    #oneHot\n",
    "    X_train_oneHot = trainSet_oneHot.drop(columns=\"semestres\")\n",
    "    y_train_oneHot = trainSet_oneHot[\"semestres\"]\n",
    "    X_test_oneHot = testSet_oneHot.drop(columns=\"semestres\")\n",
    "    y_test_oneHot = testSet_oneHot[\"semestres\"]\n",
    "    \n",
    "    \n",
    "    #####\n",
    "    # MODELS\n",
    "    #####\n",
    "    \n",
    "    # random model\n",
    "    model = \"random\"\n",
    "    print(model)\n",
    "    y_pred_prob = np.random.uniform(low=0, high=1, size=X_test.shape[0])\n",
    "    y_pred = [1 if i>=0.5 else 0 for i in y_pred_prob]\n",
    "    #cmatrix = confusion_matrix(y_test, y_pred)\n",
    "    #print(cmatrix)\n",
    "\n",
    "    # Evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test, y_pred, X_test, y_pred_prob, auc_flag=False)\n",
    "    \n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = roc_auc_score(y_test, y_pred_prob)\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    \n",
    "    # dectree\n",
    "    model = \"dectree\"\n",
    "    print(model)\n",
    "    dectree = DecisionTreeClassifier(random_state=0, min_samples_leaf=187)\n",
    "    y_pred = dectree.fit(X_train_oneHot, y_train_oneHot).predict(X_test_oneHot)\n",
    "    #cmatrix = confusion_matrix(y_test_oneHot.tolist(), y_pred.tolist())\n",
    "    #print(cmatrix)\n",
    "    \n",
    "    # Evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test_oneHot, y_pred, X_test_oneHot, dectree)\n",
    "    \n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    # logreg\n",
    "    model = \"logreg\"\n",
    "    print(model)\n",
    "    # Optimal variables\n",
    "    opt_logreg = [\"lang\", \"pps\", \"gender\", \"mat\", \"school\", \"optional\", \"admission\"]\n",
    "    X_train_logreg = X_train[opt_logreg]\n",
    "    X_test_logreg = X_test[opt_logreg]\n",
    "    \n",
    "    var_num_use_logreg = list(set(opt_logreg).intersection(set(var_num)))\n",
    "    var_cat_use_logreg = list(set(opt_logreg).intersection(set(var_cat)))\n",
    "    \n",
    "    # ONE HOT ENCODING\n",
    "    enc_onehot = OneHotEncoder(handle_unknown='ignore')\n",
    "    enc_onehot.fit(X_train_logreg[var_cat_use_logreg])\n",
    "    # one hot train\n",
    "    labels = enc_onehot.fit_transform(X_train_logreg[var_cat_use_logreg]).toarray()\n",
    "    encoded_vars = enc_onehot.get_feature_names(var_cat_use_logreg)\n",
    "    X_train_logreg = X_train_logreg.drop(columns=var_cat_use_logreg) # delete duplicated cat\n",
    "    X_train_logreg[encoded_vars] = labels\n",
    "    # one hot test\n",
    "    labels = enc_onehot.fit_transform(X_test_logreg[var_cat_use_logreg]).toarray()\n",
    "    encoded_vars = enc_onehot.get_feature_names(var_cat_use_logreg)\n",
    "    X_test_logreg = X_test_logreg.drop(columns=var_cat_use_logreg) # delete duplicated cat\n",
    "    X_test_logreg[encoded_vars] = labels\n",
    "\n",
    "    # Fix number of variables\n",
    "    #print(\"train\",X_train_logreg.shape, \"test\",X_test_logreg.shape)\n",
    "    dif = list(set(X_train_logreg.columns) - set(X_test_logreg.columns))\n",
    "    #print(dif)\n",
    "    if len(dif) != 0:\n",
    "        X_test_logreg[dif] = 0\n",
    "    \n",
    "    logreg = LogisticRegression(verbose=0, solver='lbfgs')\n",
    "    y_pred = logreg.fit(X_train_logreg, y_train).predict(X_test_logreg)\n",
    "    #cmatrix = confusion_matrix(y_test.tolist(), y_pred.tolist())\n",
    "    #print(cmatrix)\n",
    "    \n",
    "    # Evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test, y_pred, X_test_logreg, logreg)\n",
    "    \n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    # naive\n",
    "    model = \"naive\"\n",
    "    print(model)\n",
    "    gnb = GaussianNB()\n",
    "    # Optimal variables\n",
    "    X_train_naive = X_train[['mat', 'pps','optional','ranking','nem']]\n",
    "    X_test_naive = X_test[['mat', 'pps','optional','ranking','nem']]\n",
    "    \n",
    "    y_pred = gnb.fit(X_train_naive, y_train).predict(X_test_naive)\n",
    "    #cmatrix = confusion_matrix(y_test.tolist(), y_pred.tolist())\n",
    "    #print(cmatrix)\n",
    "    \n",
    "    # Evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test, y_pred, X_test_naive, gnb)\n",
    "    \n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    # KNN\n",
    "    model = \"knn\"\n",
    "    print(model)\n",
    "    # Optimal variables\n",
    "    X_train_knn = X_train[['mat', 'lang', 'pps','optional']]\n",
    "    X_test_knn = X_test[['mat', 'lang', 'pps','optional']]\n",
    "    \n",
    "    neigh = KNeighborsClassifier(n_neighbors=29)\n",
    "    y_pred = neigh.fit(X_train_knn, y_train).predict(X_test_knn)\n",
    "    cmatrix = confusion_matrix(y_test.tolist(), y_pred.tolist())\n",
    "    \n",
    "    # evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test, y_pred, X_test_knn, neigh)\n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    # SVM\n",
    "    model = \"svm\"\n",
    "    print(model)\n",
    "    # Selección vars optimas\n",
    "    X_train_svm = X_train[[\"nem\", \"ranking\", \"mat\", \"lang\", \"optional\", \"pps\"]]\n",
    "    X_test_svm = X_test[[\"nem\", \"ranking\", \"mat\", \"lang\", \"optional\", \"pps\"]]\n",
    "    support = svm.SVC(C=10, kernel=\"poly\", probability=True)\n",
    "    y_pred = support.fit(X_train_svm, y_train).predict(X_test_svm) \n",
    "    #cmatrix = confusion_matrix(y_test.tolist(), y_pred.tolist())\n",
    "    #print(cmatrix)\n",
    "    \n",
    "    # Evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test, y_pred, X_test_svm, support)\n",
    "    \n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    # ranforest\n",
    "    model = \"ranforest\"\n",
    "    print(model)\n",
    "    ranforest = RandomForestClassifier(n_estimators=500,max_features=20,\n",
    "                                       min_samples_leaf=100, n_jobs=-1)\n",
    "    y_pred = ranforest.fit(X_train_oneHot, y_train_oneHot).predict(X_test_oneHot) \n",
    "    #cmatrix = confusion_matrix(y_test_oneHot.tolist(), y_pred.tolist())\n",
    "    #print(cmatrix)\n",
    "    \n",
    "    # Evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test_oneHot, y_pred, X_test_oneHot, ranforest)\n",
    "    \n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    # gradboost\n",
    "    model = \"gradboost\"\n",
    "    print(model)\n",
    "    gradboost = GradientBoostingClassifier(random_state=0,\n",
    "                                           min_samples_split=2,\n",
    "                                           n_estimators=110,\n",
    "                                           min_samples_leaf = 150,\n",
    "                                           max_features=6\n",
    "                                          )\n",
    "    \n",
    "    y_pred = gradboost.fit(X_train_oneHot, y_train_oneHot).predict(X_test_oneHot) \n",
    "    #cmatrix = confusion_matrix(y_test_oneHot.tolist(), y_pred.tolist())\n",
    "    #print(cmatrix)\n",
    "    \n",
    "    # Evaluate\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test_oneHot, y_pred, X_test_oneHot, gradboost)\n",
    "    \n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "                                             \n",
    "    # neural\n",
    "    model = \"neural\"\n",
    "    print(model)\n",
    "    # Optimal variables\n",
    "    opt_nn = [\"lang\", \"pps\", \"gender\", \"mat\", \"school\", \"optional\", \"admission\"]\n",
    "    X_train_nn = X_train[opt_nn]\n",
    "    X_test_nn = X_test[opt_nn]\n",
    "    \n",
    "    var_num_use_nn = list(set(opt_nn).intersection(set(var_num)))\n",
    "    var_cat_use_nn = list(set(opt_nn).intersection(set(var_cat)))\n",
    "    \n",
    "    # ONE HOT ENCODING\n",
    "    enc_onehot = OneHotEncoder(handle_unknown='ignore')\n",
    "    enc_onehot.fit(X_train_nn[var_cat_use_nn])\n",
    "    #one hot train\n",
    "    labels = enc_onehot.fit_transform(X_train_nn[var_cat_use_nn]).toarray()\n",
    "    encoded_vars = enc_onehot.get_feature_names(var_cat_use_nn)\n",
    "    X_train_nn = X_train_nn.drop(columns=var_cat_use_nn) # delete duplicated cat\n",
    "    X_train_nn[encoded_vars] = labels\n",
    "    #one hot test\n",
    "    labels = enc_onehot.fit_transform(X_test_nn[var_cat_use_nn]).toarray()\n",
    "    encoded_vars = enc_onehot.get_feature_names(var_cat_use_nn)\n",
    "    X_test_nn = X_test_nn.drop(columns=var_cat_use_nn) # delete duplicated cat\n",
    "    X_test_nn[encoded_vars] = labels\n",
    "\n",
    "    # Fix number of variables\n",
    "    #print(\"train\",X_train_logreg.shape, \"test\",X_test_logreg.shape)\n",
    "    dif = list(set(X_train_nn.columns) - set(X_test_nn.columns))\n",
    "    #print(dif)\n",
    "    if len(dif) != 0:\n",
    "        X_test_nn[dif] = 0\n",
    "    \n",
    "    # Standarize for speed\n",
    "    scaler = MinMaxScaler()\n",
    "    scaler.fit(X_train_nn)\n",
    "    columns = X_train_nn.columns\n",
    "    X_train_nn[columns] = scaler.transform(X_train_nn)\n",
    "    scaler.fit(X_test_nn)\n",
    "    X_test_nn[columns] = scaler.transform(X_test_nn)\n",
    "\n",
    "    # Layers\n",
    "    n_neurons = 15\n",
    "    n_layers = 3\n",
    "    inputLayer = layers.Input(shape=(X_train_nn.shape[1],))\n",
    "    hiddenLayer = layers.Dense(n_neurons, activation='relu',use_bias = True)(inputLayer)\n",
    "    # Hidden layers\n",
    "    for i in range(n_layers-1):\n",
    "        hiddenLayer = layers.Dense(n_neurons, activation='relu',use_bias = True)(hiddenLayer)\n",
    "\n",
    "    outputLayer = layers.Dense(1, activation='sigmoid',use_bias = True)(hiddenLayer)\n",
    "    feedForward = models.Model(inputLayer, outputLayer)\n",
    "\n",
    "    feedForward.compile(loss='binary_crossentropy', optimizer = 'adam',metrics=['binary_accuracy',f1])\n",
    "    feedForward.fit(X_train_nn, y_train,epochs=200,batch_size=64, verbose = 0)\n",
    "    y_pred = feedForward.predict(X_test_nn)\n",
    "    # AUC\n",
    "    auc[model][fold_count-1] = roc_auc_score(y_test, y_pred)\n",
    "    y_pred = [1 if i>=0.5 else 0 for i in y_pred]\n",
    "    \n",
    "    # Evaluate, not including AUC\n",
    "    f1Score,f1Score0,acc,aucScore,prec,rec = metrics_evaluate(\n",
    "        y_test, y_pred, X_test_nn, feedForward, auc_flag=False)\n",
    "    scores[model][fold_count-1] = f1Score\n",
    "    scores0[model][fold_count-1] = f1Score0\n",
    "    accuracy[model][fold_count-1] = acc\n",
    "    #auc[model][fold_count-1] = aucScore\n",
    "    precision[model][fold_count-1] = prec\n",
    "    recall[model][fold_count-1] = rec\n",
    "    \n",
    "    fold_count+=1\n",
    "\n",
    "# dataframe\n",
    "for model in [\"random\",\"knn\",\"svm\",\"dectree\",\"ranforest\",\"gradboost\",\"naive\",\"logreg\",\"neural\"]:\n",
    "    frame = frame.append({\"Model\": model,\n",
    "                          \"F1+ Means\": round(scores[model].mean(),2),\n",
    "                          \"F1+ Error\": round(scores[model].std(),2),\n",
    "                          \"F1- Means\": round(scores0[model].mean(),2),\n",
    "                          \"F1- Error\": round(scores0[model].std(),2),\n",
    "                          \"Acc Means\": round(accuracy[model].mean(),2),\n",
    "                          \"Acc Error\": round(accuracy[model].std(),2),\n",
    "                          \"AUC\": round(auc[model].mean(),2),\n",
    "                          \"AUC Error\": round(auc[model].std(),2),\n",
    "                          \"Prec Means\": round(precision[model].mean(),2),\n",
    "                          \"Prec Error\": round(precision[model].std(),2),\n",
    "                          \"Rec Means\": round(recall[model].mean(),2),\n",
    "                          \"Rec Error\": round(recall[model].std(),2)},\n",
    "                        ignore_index = True)\n",
    "    \n",
    "frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#frame.to_csv(\"results_both.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}