model_performance_tables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-09T17:03:35.412518Z",
     "start_time": "2019-12-09T17:03:35.409269Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import io\n",
    "from src.helper_functions import load_data, get_model_perfs, init_model_perfs,\\\n",
    "                                 save_model, save_model_perfs, check_is_best,\\\n",
    "                                 read_model, evaluate_model_predictions,\\\n",
    "                                 update_model_perfs, check_and_save,\\\n",
    "                                 adjusted_classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Show Performance of All Generated Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-09T17:03:38.215991Z",
     "start_time": "2019-12-09T17:03:37.822572Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.AhR\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.911475</td>\n",
       "      <td>0.756757</td>\n",
       "      <td>0.383562</td>\n",
       "      <td>0.509091</td>\n",
       "      <td>0.905028</td>\n",
       "      <td>0.498772</td>\n",
       "      <td>0.602102</td>\n",
       "      <td>[[528, 9], [45, 28]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.235000</td>\n",
       "      <td>0.842623</td>\n",
       "      <td>0.424837</td>\n",
       "      <td>0.890411</td>\n",
       "      <td>0.575221</td>\n",
       "      <td>0.905028</td>\n",
       "      <td>0.544005</td>\n",
       "      <td>0.602102</td>\n",
       "      <td>[[449, 88], [8, 65]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.814754</td>\n",
       "      <td>0.364865</td>\n",
       "      <td>0.739726</td>\n",
       "      <td>0.488688</td>\n",
       "      <td>0.864723</td>\n",
       "      <td>0.427562</td>\n",
       "      <td>0.524576</td>\n",
       "      <td>[[443, 94], [19, 54]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.773516</td>\n",
       "      <td>0.901639</td>\n",
       "      <td>0.603175</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.558824</td>\n",
       "      <td>0.864723</td>\n",
       "      <td>0.505541</td>\n",
       "      <td>0.524576</td>\n",
       "      <td>[[512, 25], [35, 38]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.895082</td>\n",
       "      <td>0.561644</td>\n",
       "      <td>0.561644</td>\n",
       "      <td>0.561644</td>\n",
       "      <td>0.860935</td>\n",
       "      <td>0.502054</td>\n",
       "      <td>0.495738</td>\n",
       "      <td>[[505, 32], [32, 41]]</td>\n",
       "      <td>DNN2.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.488593</td>\n",
       "      <td>0.893443</td>\n",
       "      <td>0.551282</td>\n",
       "      <td>0.589041</td>\n",
       "      <td>0.569536</td>\n",
       "      <td>0.860935</td>\n",
       "      <td>0.509172</td>\n",
       "      <td>0.495738</td>\n",
       "      <td>[[502, 35], [30, 43]]</td>\n",
       "      <td>DNN2.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.780328</td>\n",
       "      <td>0.323699</td>\n",
       "      <td>0.767123</td>\n",
       "      <td>0.455285</td>\n",
       "      <td>0.867950</td>\n",
       "      <td>0.395505</td>\n",
       "      <td>0.573394</td>\n",
       "      <td>[[420, 117], [17, 56]]</td>\n",
       "      <td>DNN3.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.600385</td>\n",
       "      <td>0.885246</td>\n",
       "      <td>0.516854</td>\n",
       "      <td>0.630137</td>\n",
       "      <td>0.567901</td>\n",
       "      <td>0.867950</td>\n",
       "      <td>0.505763</td>\n",
       "      <td>0.573394</td>\n",
       "      <td>[[494, 43], [27, 46]]</td>\n",
       "      <td>DNN3.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.872131</td>\n",
       "      <td>0.474747</td>\n",
       "      <td>0.643836</td>\n",
       "      <td>0.546512</td>\n",
       "      <td>0.871126</td>\n",
       "      <td>0.481514</td>\n",
       "      <td>0.553833</td>\n",
       "      <td>[[485, 52], [26, 47]]</td>\n",
       "      <td>DNN4.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.496647</td>\n",
       "      <td>0.875410</td>\n",
       "      <td>0.485714</td>\n",
       "      <td>0.698630</td>\n",
       "      <td>0.573034</td>\n",
       "      <td>0.871126</td>\n",
       "      <td>0.514235</td>\n",
       "      <td>0.553833</td>\n",
       "      <td>[[483, 54], [22, 51]]</td>\n",
       "      <td>DNN4.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.896721</td>\n",
       "      <td>0.596154</td>\n",
       "      <td>0.424658</td>\n",
       "      <td>0.496000</td>\n",
       "      <td>0.872376</td>\n",
       "      <td>0.448138</td>\n",
       "      <td>0.559638</td>\n",
       "      <td>[[516, 21], [42, 31]]</td>\n",
       "      <td>DNN5.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.435945</td>\n",
       "      <td>0.896721</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>0.616438</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.872376</td>\n",
       "      <td>0.530058</td>\n",
       "      <td>0.559638</td>\n",
       "      <td>[[502, 35], [28, 45]]</td>\n",
       "      <td>DNN5.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.847541</td>\n",
       "      <td>0.419355</td>\n",
       "      <td>0.712329</td>\n",
       "      <td>0.527919</td>\n",
       "      <td>0.878651</td>\n",
       "      <td>0.466375</td>\n",
       "      <td>0.551993</td>\n",
       "      <td>[[465, 72], [21, 52]]</td>\n",
       "      <td>DNN6.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.592893</td>\n",
       "      <td>0.881967</td>\n",
       "      <td>0.505155</td>\n",
       "      <td>0.671233</td>\n",
       "      <td>0.576471</td>\n",
       "      <td>0.878651</td>\n",
       "      <td>0.516432</td>\n",
       "      <td>0.551993</td>\n",
       "      <td>[[489, 48], [24, 49]]</td>\n",
       "      <td>DNN6.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.885246</td>\n",
       "      <td>0.516854</td>\n",
       "      <td>0.630137</td>\n",
       "      <td>0.567901</td>\n",
       "      <td>0.868855</td>\n",
       "      <td>0.505763</td>\n",
       "      <td>0.545916</td>\n",
       "      <td>[[494, 43], [27, 46]]</td>\n",
       "      <td>DNN7.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.780328</td>\n",
       "      <td>0.327684</td>\n",
       "      <td>0.794521</td>\n",
       "      <td>0.464000</td>\n",
       "      <td>0.878983</td>\n",
       "      <td>0.409743</td>\n",
       "      <td>0.527040</td>\n",
       "      <td>[[418, 119], [15, 58]]</td>\n",
       "      <td>DNN2.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.810196</td>\n",
       "      <td>0.896721</td>\n",
       "      <td>0.564103</td>\n",
       "      <td>0.602740</td>\n",
       "      <td>0.582781</td>\n",
       "      <td>0.878983</td>\n",
       "      <td>0.524296</td>\n",
       "      <td>0.527040</td>\n",
       "      <td>[[503, 34], [29, 44]]</td>\n",
       "      <td>DNN2.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.872131</td>\n",
       "      <td>0.474747</td>\n",
       "      <td>0.643836</td>\n",
       "      <td>0.546512</td>\n",
       "      <td>0.881623</td>\n",
       "      <td>0.481514</td>\n",
       "      <td>0.558819</td>\n",
       "      <td>[[485, 52], [26, 47]]</td>\n",
       "      <td>DNN3.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.573869</td>\n",
       "      <td>0.888525</td>\n",
       "      <td>0.529412</td>\n",
       "      <td>0.616438</td>\n",
       "      <td>0.569620</td>\n",
       "      <td>0.881623</td>\n",
       "      <td>0.507948</td>\n",
       "      <td>0.558819</td>\n",
       "      <td>[[497, 40], [28, 45]]</td>\n",
       "      <td>DNN3.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.703475</td>\n",
       "      <td>0.895082</td>\n",
       "      <td>0.552941</td>\n",
       "      <td>0.643836</td>\n",
       "      <td>0.594937</td>\n",
       "      <td>0.853052</td>\n",
       "      <td>0.537117</td>\n",
       "      <td>0.496709</td>\n",
       "      <td>[[499, 38], [26, 47]]</td>\n",
       "      <td>DNN_modT0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0         RF   0.500000  0.911475   0.756757  0.383562  0.509091  0.905028   \n",
       "1    RF_modT   0.235000  0.842623   0.424837  0.890411  0.575221  0.905028   \n",
       "2        DNN   0.500000  0.814754   0.364865  0.739726  0.488688  0.864723   \n",
       "3   DNN_modT   0.773516  0.901639   0.603175  0.520548  0.558824  0.864723   \n",
       "4        DNN   0.500000  0.895082   0.561644  0.561644  0.561644  0.860935   \n",
       "5   DNN_modT   0.488593  0.893443   0.551282  0.589041  0.569536  0.860935   \n",
       "6        DNN   0.500000  0.780328   0.323699  0.767123  0.455285  0.867950   \n",
       "7   DNN_modT   0.600385  0.885246   0.516854  0.630137  0.567901  0.867950   \n",
       "8        DNN   0.500000  0.872131   0.474747  0.643836  0.546512  0.871126   \n",
       "9   DNN_modT   0.496647  0.875410   0.485714  0.698630  0.573034  0.871126   \n",
       "10       DNN   0.500000  0.896721   0.596154  0.424658  0.496000  0.872376   \n",
       "11  DNN_modT   0.435945  0.896721   0.562500  0.616438  0.588235  0.872376   \n",
       "12       DNN   0.500000  0.847541   0.419355  0.712329  0.527919  0.878651   \n",
       "13  DNN_modT   0.592893  0.881967   0.505155  0.671233  0.576471  0.878651   \n",
       "14       DNN   0.500000  0.885246   0.516854  0.630137  0.567901  0.868855   \n",
       "15       DNN   0.500000  0.780328   0.327684  0.794521  0.464000  0.878983   \n",
       "16  DNN_modT   0.810196  0.896721   0.564103  0.602740  0.582781  0.878983   \n",
       "17       DNN   0.500000  0.872131   0.474747  0.643836  0.546512  0.881623   \n",
       "18  DNN_modT   0.573869  0.888525   0.529412  0.616438  0.569620  0.881623   \n",
       "19  DNN_modT   0.703475  0.895082   0.552941  0.643836  0.594937  0.853052   \n",
       "\n",
       "         mcc  avg_precision        confusion_matrix model_filename  \n",
       "0   0.498772       0.602102    [[528, 9], [45, 28]]     RF0.joblib  \n",
       "1   0.544005       0.602102    [[449, 88], [8, 65]]     RF0.joblib  \n",
       "2   0.427562       0.524576   [[443, 94], [19, 54]]        DNN0.h5  \n",
       "3   0.505541       0.524576   [[512, 25], [35, 38]]        DNN0.h5  \n",
       "4   0.502054       0.495738   [[505, 32], [32, 41]]        DNN2.h5  \n",
       "5   0.509172       0.495738   [[502, 35], [30, 43]]        DNN2.h5  \n",
       "6   0.395505       0.573394  [[420, 117], [17, 56]]        DNN3.h5  \n",
       "7   0.505763       0.573394   [[494, 43], [27, 46]]        DNN3.h5  \n",
       "8   0.481514       0.553833   [[485, 52], [26, 47]]        DNN4.h5  \n",
       "9   0.514235       0.553833   [[483, 54], [22, 51]]        DNN4.h5  \n",
       "10  0.448138       0.559638   [[516, 21], [42, 31]]        DNN5.h5  \n",
       "11  0.530058       0.559638   [[502, 35], [28, 45]]        DNN5.h5  \n",
       "12  0.466375       0.551993   [[465, 72], [21, 52]]        DNN6.h5  \n",
       "13  0.516432       0.551993   [[489, 48], [24, 49]]        DNN6.h5  \n",
       "14  0.505763       0.545916   [[494, 43], [27, 46]]        DNN7.h5  \n",
       "15  0.409743       0.527040  [[418, 119], [15, 58]]        DNN2.h5  \n",
       "16  0.524296       0.527040   [[503, 34], [29, 44]]        DNN2.h5  \n",
       "17  0.481514       0.558819   [[485, 52], [26, 47]]        DNN3.h5  \n",
       "18  0.507948       0.558819   [[497, 40], [28, 45]]        DNN3.h5  \n",
       "19  0.537117       0.496709   [[499, 38], [26, 47]]   DNN_modT0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.AR\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.981229</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.678934</td>\n",
       "      <td>0.327293</td>\n",
       "      <td>0.232723</td>\n",
       "      <td>[[573, 1], [10, 2]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>0.982935</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.678934</td>\n",
       "      <td>0.404738</td>\n",
       "      <td>0.232723</td>\n",
       "      <td>[[574, 0], [10, 2]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.721843</td>\n",
       "      <td>0.042424</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.079096</td>\n",
       "      <td>0.741507</td>\n",
       "      <td>0.097010</td>\n",
       "      <td>0.168663</td>\n",
       "      <td>[[416, 158], [5, 7]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.792447</td>\n",
       "      <td>0.965870</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.741507</td>\n",
       "      <td>0.271518</td>\n",
       "      <td>0.168663</td>\n",
       "      <td>[[562, 12], [8, 4]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.981229   0.666667  0.166667  0.266667  0.678934   \n",
       "1   RF_modT   0.730000  0.982935   1.000000  0.166667  0.285714  0.678934   \n",
       "2       DNN   0.500000  0.721843   0.042424  0.583333  0.079096  0.741507   \n",
       "3  DNN_modT   0.792447  0.965870   0.250000  0.333333  0.285714  0.741507   \n",
       "\n",
       "        mcc  avg_precision      confusion_matrix model_filename  \n",
       "0  0.327293       0.232723   [[573, 1], [10, 2]]     RF0.joblib  \n",
       "1  0.404738       0.232723   [[574, 0], [10, 2]]     RF0.joblib  \n",
       "2  0.097010       0.168663  [[416, 158], [5, 7]]        DNN0.h5  \n",
       "3  0.271518       0.168663   [[562, 12], [8, 4]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.AR.LBD\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.982818</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.763284</td>\n",
       "      <td>-0.006933</td>\n",
       "      <td>0.060344</td>\n",
       "      <td>[[572, 2], [8, 0]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.270000</td>\n",
       "      <td>0.979381</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.763284</td>\n",
       "      <td>0.134046</td>\n",
       "      <td>0.060344</td>\n",
       "      <td>[[569, 5], [7, 1]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.738832</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.037975</td>\n",
       "      <td>0.635126</td>\n",
       "      <td>0.031652</td>\n",
       "      <td>0.030153</td>\n",
       "      <td>[[427, 147], [5, 3]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.762549</td>\n",
       "      <td>0.929553</td>\n",
       "      <td>0.054054</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.088889</td>\n",
       "      <td>0.635126</td>\n",
       "      <td>0.090203</td>\n",
       "      <td>0.030153</td>\n",
       "      <td>[[539, 35], [6, 2]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision  recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.982818   0.000000   0.000  0.000000  0.763284   \n",
       "1   RF_modT   0.270000  0.979381   0.166667   0.125  0.142857  0.763284   \n",
       "2       DNN   0.500000  0.738832   0.020000   0.375  0.037975  0.635126   \n",
       "3  DNN_modT   0.762549  0.929553   0.054054   0.250  0.088889  0.635126   \n",
       "\n",
       "        mcc  avg_precision      confusion_matrix model_filename  \n",
       "0 -0.006933       0.060344    [[572, 2], [8, 0]]     RF0.joblib  \n",
       "1  0.134046       0.060344    [[569, 5], [7, 1]]     RF0.joblib  \n",
       "2  0.031652       0.030153  [[427, 147], [5, 3]]        DNN0.h5  \n",
       "3  0.090203       0.030153   [[539, 35], [6, 2]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.Aromatase\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.928030</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.771931</td>\n",
       "      <td>0.154247</td>\n",
       "      <td>0.311884</td>\n",
       "      <td>[[489, 0], [38, 1]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.130000</td>\n",
       "      <td>0.840909</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.512821</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.771931</td>\n",
       "      <td>0.270359</td>\n",
       "      <td>0.311884</td>\n",
       "      <td>[[424, 65], [19, 20]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.772727</td>\n",
       "      <td>0.159664</td>\n",
       "      <td>0.487179</td>\n",
       "      <td>0.240506</td>\n",
       "      <td>0.718158</td>\n",
       "      <td>0.176949</td>\n",
       "      <td>0.143086</td>\n",
       "      <td>[[389, 100], [20, 19]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.475931</td>\n",
       "      <td>0.767045</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.254545</td>\n",
       "      <td>0.718158</td>\n",
       "      <td>0.198647</td>\n",
       "      <td>0.143086</td>\n",
       "      <td>[[384, 105], [18, 21]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.928030   1.000000  0.025641  0.050000  0.771931   \n",
       "1   RF_modT   0.130000  0.840909   0.235294  0.512821  0.322581  0.771931   \n",
       "2       DNN   0.500000  0.772727   0.159664  0.487179  0.240506  0.718158   \n",
       "3  DNN_modT   0.475931  0.767045   0.166667  0.538462  0.254545  0.718158   \n",
       "\n",
       "        mcc  avg_precision        confusion_matrix model_filename  \n",
       "0  0.154247       0.311884     [[489, 0], [38, 1]]     RF0.joblib  \n",
       "1  0.270359       0.311884   [[424, 65], [19, 20]]     RF0.joblib  \n",
       "2  0.176949       0.143086  [[389, 100], [20, 19]]        DNN0.h5  \n",
       "3  0.198647       0.143086  [[384, 105], [18, 21]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.ER\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.215686</td>\n",
       "      <td>0.338462</td>\n",
       "      <td>0.769007</td>\n",
       "      <td>0.384352</td>\n",
       "      <td>0.445565</td>\n",
       "      <td>[[462, 3], [40, 11]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.912791</td>\n",
       "      <td>0.607143</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.430380</td>\n",
       "      <td>0.769007</td>\n",
       "      <td>0.407974</td>\n",
       "      <td>0.445565</td>\n",
       "      <td>[[454, 11], [34, 17]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.748062</td>\n",
       "      <td>0.203008</td>\n",
       "      <td>0.529412</td>\n",
       "      <td>0.293478</td>\n",
       "      <td>0.736369</td>\n",
       "      <td>0.205688</td>\n",
       "      <td>0.280064</td>\n",
       "      <td>[[359, 106], [24, 27]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.625295</td>\n",
       "      <td>0.815891</td>\n",
       "      <td>0.270833</td>\n",
       "      <td>0.509804</td>\n",
       "      <td>0.353741</td>\n",
       "      <td>0.736369</td>\n",
       "      <td>0.275529</td>\n",
       "      <td>0.280064</td>\n",
       "      <td>[[395, 70], [25, 26]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.916667   0.785714  0.215686  0.338462  0.769007   \n",
       "1   RF_modT   0.400000  0.912791   0.607143  0.333333  0.430380  0.769007   \n",
       "2       DNN   0.500000  0.748062   0.203008  0.529412  0.293478  0.736369   \n",
       "3  DNN_modT   0.625295  0.815891   0.270833  0.509804  0.353741  0.736369   \n",
       "\n",
       "        mcc  avg_precision        confusion_matrix model_filename  \n",
       "0  0.384352       0.445565    [[462, 3], [40, 11]]     RF0.joblib  \n",
       "1  0.407974       0.445565   [[454, 11], [34, 17]]     RF0.joblib  \n",
       "2  0.205688       0.280064  [[359, 106], [24, 27]]        DNN0.h5  \n",
       "3  0.275529       0.280064   [[395, 70], [25, 26]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.ER.LBD\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>0.971667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.750345</td>\n",
       "      <td>0.381744</td>\n",
       "      <td>0.268778</td>\n",
       "      <td>[[580, 0], [17, 3]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.30500</td>\n",
       "      <td>0.965000</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.750345</td>\n",
       "      <td>0.354999</td>\n",
       "      <td>0.268778</td>\n",
       "      <td>[[573, 7], [14, 6]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>0.725000</td>\n",
       "      <td>0.049689</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.088398</td>\n",
       "      <td>0.678319</td>\n",
       "      <td>0.055180</td>\n",
       "      <td>0.069591</td>\n",
       "      <td>[[427, 153], [12, 8]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.74828</td>\n",
       "      <td>0.915000</td>\n",
       "      <td>0.102564</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.135593</td>\n",
       "      <td>0.678319</td>\n",
       "      <td>0.101689</td>\n",
       "      <td>0.069591</td>\n",
       "      <td>[[545, 35], [16, 4]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision  recall        f1   auc_roc  \\\n",
       "0        RF    0.50000  0.971667   1.000000    0.15  0.260870  0.750345   \n",
       "1   RF_modT    0.30500  0.965000   0.461538    0.30  0.363636  0.750345   \n",
       "2       DNN    0.50000  0.725000   0.049689    0.40  0.088398  0.678319   \n",
       "3  DNN_modT    0.74828  0.915000   0.102564    0.20  0.135593  0.678319   \n",
       "\n",
       "        mcc  avg_precision       confusion_matrix model_filename  \n",
       "0  0.381744       0.268778    [[580, 0], [17, 3]]     RF0.joblib  \n",
       "1  0.354999       0.268778    [[573, 7], [14, 6]]     RF0.joblib  \n",
       "2  0.055180       0.069591  [[427, 153], [12, 8]]        DNN0.h5  \n",
       "3  0.101689       0.069591   [[545, 35], [16, 4]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.PPAR.gamma\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.947107</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.709846</td>\n",
       "      <td>-0.009456</td>\n",
       "      <td>0.129989</td>\n",
       "      <td>[[573, 1], [31, 0]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.130000</td>\n",
       "      <td>0.890909</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.232558</td>\n",
       "      <td>0.709846</td>\n",
       "      <td>0.187280</td>\n",
       "      <td>0.129989</td>\n",
       "      <td>[[529, 45], [21, 10]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750413</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>0.580645</td>\n",
       "      <td>0.192513</td>\n",
       "      <td>0.740390</td>\n",
       "      <td>0.171483</td>\n",
       "      <td>0.109550</td>\n",
       "      <td>[[436, 138], [13, 18]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.577705</td>\n",
       "      <td>0.798347</td>\n",
       "      <td>0.130081</td>\n",
       "      <td>0.516129</td>\n",
       "      <td>0.207792</td>\n",
       "      <td>0.740390</td>\n",
       "      <td>0.180636</td>\n",
       "      <td>0.109550</td>\n",
       "      <td>[[467, 107], [15, 16]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.947107   0.000000  0.000000  0.000000  0.709846   \n",
       "1   RF_modT   0.130000  0.890909   0.181818  0.322581  0.232558  0.709846   \n",
       "2       DNN   0.500000  0.750413   0.115385  0.580645  0.192513  0.740390   \n",
       "3  DNN_modT   0.577705  0.798347   0.130081  0.516129  0.207792  0.740390   \n",
       "\n",
       "        mcc  avg_precision        confusion_matrix model_filename  \n",
       "0 -0.009456       0.129989     [[573, 1], [31, 0]]     RF0.joblib  \n",
       "1  0.187280       0.129989   [[529, 45], [21, 10]]     RF0.joblib  \n",
       "2  0.171483       0.109550  [[436, 138], [13, 18]]        DNN0.h5  \n",
       "3  0.180636       0.109550  [[467, 107], [15, 16]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.ARE\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.841441</td>\n",
       "      <td>0.592593</td>\n",
       "      <td>0.172043</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.780838</td>\n",
       "      <td>0.257342</td>\n",
       "      <td>0.430711</td>\n",
       "      <td>[[451, 11], [77, 16]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.340000</td>\n",
       "      <td>0.823423</td>\n",
       "      <td>0.476190</td>\n",
       "      <td>0.537634</td>\n",
       "      <td>0.505051</td>\n",
       "      <td>0.780838</td>\n",
       "      <td>0.399160</td>\n",
       "      <td>0.430711</td>\n",
       "      <td>[[407, 55], [43, 50]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.762162</td>\n",
       "      <td>0.355556</td>\n",
       "      <td>0.516129</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.705500</td>\n",
       "      <td>0.285366</td>\n",
       "      <td>0.347718</td>\n",
       "      <td>[[375, 87], [45, 48]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.457539</td>\n",
       "      <td>0.753153</td>\n",
       "      <td>0.353333</td>\n",
       "      <td>0.569892</td>\n",
       "      <td>0.436214</td>\n",
       "      <td>0.705500</td>\n",
       "      <td>0.302701</td>\n",
       "      <td>0.347718</td>\n",
       "      <td>[[365, 97], [40, 53]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.841441   0.592593  0.172043  0.266667  0.780838   \n",
       "1   RF_modT   0.340000  0.823423   0.476190  0.537634  0.505051  0.780838   \n",
       "2       DNN   0.500000  0.762162   0.355556  0.516129  0.421053  0.705500   \n",
       "3  DNN_modT   0.457539  0.753153   0.353333  0.569892  0.436214  0.705500   \n",
       "\n",
       "        mcc  avg_precision       confusion_matrix model_filename  \n",
       "0  0.257342       0.430711  [[451, 11], [77, 16]]     RF0.joblib  \n",
       "1  0.399160       0.430711  [[407, 55], [43, 50]]     RF0.joblib  \n",
       "2  0.285366       0.347718  [[375, 87], [45, 48]]        DNN0.h5  \n",
       "3  0.302701       0.347718  [[365, 97], [40, 53]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.ATAD5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.938907</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.795625</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.427828</td>\n",
       "      <td>[[584, 0], [38, 0]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.255000</td>\n",
       "      <td>0.948553</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.368421</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>0.795625</td>\n",
       "      <td>0.459939</td>\n",
       "      <td>0.427828</td>\n",
       "      <td>[[576, 8], [24, 14]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.736334</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>0.203883</td>\n",
       "      <td>0.711608</td>\n",
       "      <td>0.162318</td>\n",
       "      <td>0.153415</td>\n",
       "      <td>[[437, 147], [17, 21]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.577705</td>\n",
       "      <td>0.789389</td>\n",
       "      <td>0.155556</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>0.242775</td>\n",
       "      <td>0.711608</td>\n",
       "      <td>0.207660</td>\n",
       "      <td>0.153415</td>\n",
       "      <td>[[470, 114], [17, 21]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.938907   0.000000  0.000000  0.000000  0.795625   \n",
       "1   RF_modT   0.255000  0.948553   0.636364  0.368421  0.466667  0.795625   \n",
       "2       DNN   0.500000  0.736334   0.125000  0.552632  0.203883  0.711608   \n",
       "3  DNN_modT   0.577705  0.789389   0.155556  0.552632  0.242775  0.711608   \n",
       "\n",
       "        mcc  avg_precision        confusion_matrix model_filename  \n",
       "0       NaN       0.427828     [[584, 0], [38, 0]]     RF0.joblib  \n",
       "1  0.459939       0.427828    [[576, 8], [24, 14]]     RF0.joblib  \n",
       "2  0.162318       0.153415  [[437, 147], [17, 21]]        DNN0.h5  \n",
       "3  0.207660       0.153415  [[470, 114], [17, 21]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.HSE\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.965574</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.759586</td>\n",
       "      <td>0.274958</td>\n",
       "      <td>0.239563</td>\n",
       "      <td>[[586, 2], [19, 3]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.340000</td>\n",
       "      <td>0.965574</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.759586</td>\n",
       "      <td>0.370222</td>\n",
       "      <td>0.239563</td>\n",
       "      <td>[[583, 5], [16, 6]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.726230</td>\n",
       "      <td>0.055215</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.097297</td>\n",
       "      <td>0.626739</td>\n",
       "      <td>0.062018</td>\n",
       "      <td>0.054621</td>\n",
       "      <td>[[434, 154], [13, 9]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.450345</td>\n",
       "      <td>0.708197</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.118812</td>\n",
       "      <td>0.626739</td>\n",
       "      <td>0.106186</td>\n",
       "      <td>0.054621</td>\n",
       "      <td>[[420, 168], [10, 12]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.965574   0.600000  0.136364  0.222222  0.759586   \n",
       "1   RF_modT   0.340000  0.965574   0.545455  0.272727  0.363636  0.759586   \n",
       "2       DNN   0.500000  0.726230   0.055215  0.409091  0.097297  0.626739   \n",
       "3  DNN_modT   0.450345  0.708197   0.066667  0.545455  0.118812  0.626739   \n",
       "\n",
       "        mcc  avg_precision        confusion_matrix model_filename  \n",
       "0  0.274958       0.239563     [[586, 2], [19, 3]]     RF0.joblib  \n",
       "1  0.370222       0.239563     [[583, 5], [16, 6]]     RF0.joblib  \n",
       "2  0.062018       0.054621   [[434, 154], [13, 9]]        DNN0.h5  \n",
       "3  0.106186       0.054621  [[420, 168], [10, 12]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.MMP\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.906077</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.484848</td>\n",
       "      <td>0.930331</td>\n",
       "      <td>0.447984</td>\n",
       "      <td>0.560184</td>\n",
       "      <td>[[468, 15], [36, 24]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.382500</td>\n",
       "      <td>0.913444</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.624000</td>\n",
       "      <td>0.930331</td>\n",
       "      <td>0.575769</td>\n",
       "      <td>0.560184</td>\n",
       "      <td>[[457, 26], [21, 39]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.815838</td>\n",
       "      <td>0.330508</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.438202</td>\n",
       "      <td>0.803882</td>\n",
       "      <td>0.369779</td>\n",
       "      <td>0.296565</td>\n",
       "      <td>[[404, 79], [21, 39]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.505391</td>\n",
       "      <td>0.817680</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.440678</td>\n",
       "      <td>0.803882</td>\n",
       "      <td>0.372498</td>\n",
       "      <td>0.296565</td>\n",
       "      <td>[[405, 78], [21, 39]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision  recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.906077   0.615385    0.40  0.484848  0.930331   \n",
       "1   RF_modT   0.382500  0.913444   0.600000    0.65  0.624000  0.930331   \n",
       "2       DNN   0.500000  0.815838   0.330508    0.65  0.438202  0.803882   \n",
       "3  DNN_modT   0.505391  0.817680   0.333333    0.65  0.440678  0.803882   \n",
       "\n",
       "        mcc  avg_precision       confusion_matrix model_filename  \n",
       "0  0.447984       0.560184  [[468, 15], [36, 24]]     RF0.joblib  \n",
       "1  0.575769       0.560184  [[457, 26], [21, 39]]     RF0.joblib  \n",
       "2  0.369779       0.296565  [[404, 79], [21, 39]]        DNN0.h5  \n",
       "3  0.372498       0.296565  [[405, 78], [21, 39]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.p53\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>mcc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.933442</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.024390</td>\n",
       "      <td>0.046512</td>\n",
       "      <td>0.806702</td>\n",
       "      <td>0.099247</td>\n",
       "      <td>0.221990</td>\n",
       "      <td>[[574, 1], [40, 1]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.220000</td>\n",
       "      <td>0.886364</td>\n",
       "      <td>0.254237</td>\n",
       "      <td>0.365854</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.806702</td>\n",
       "      <td>0.245058</td>\n",
       "      <td>0.221990</td>\n",
       "      <td>[[531, 44], [26, 15]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>DNN</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.753247</td>\n",
       "      <td>0.150943</td>\n",
       "      <td>0.585366</td>\n",
       "      <td>0.240000</td>\n",
       "      <td>0.768696</td>\n",
       "      <td>0.199692</td>\n",
       "      <td>0.168234</td>\n",
       "      <td>[[440, 135], [17, 24]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>DNN_modT</td>\n",
       "      <td>0.540362</td>\n",
       "      <td>0.780844</td>\n",
       "      <td>0.169014</td>\n",
       "      <td>0.585366</td>\n",
       "      <td>0.262295</td>\n",
       "      <td>0.768696</td>\n",
       "      <td>0.224981</td>\n",
       "      <td>0.168234</td>\n",
       "      <td>[[457, 118], [17, 24]]</td>\n",
       "      <td>DNN0.h5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0        RF   0.500000  0.933442   0.500000  0.024390  0.046512  0.806702   \n",
       "1   RF_modT   0.220000  0.886364   0.254237  0.365854  0.300000  0.806702   \n",
       "2       DNN   0.500000  0.753247   0.150943  0.585366  0.240000  0.768696   \n",
       "3  DNN_modT   0.540362  0.780844   0.169014  0.585366  0.262295  0.768696   \n",
       "\n",
       "        mcc  avg_precision        confusion_matrix model_filename  \n",
       "0  0.099247       0.221990     [[574, 1], [40, 1]]     RF0.joblib  \n",
       "1  0.245058       0.221990   [[531, 44], [26, 15]]     RF0.joblib  \n",
       "2  0.199692       0.168234  [[440, 135], [17, 24]]        DNN0.h5  \n",
       "3  0.224981       0.168234  [[457, 118], [17, 24]]        DNN0.h5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "targets = ['NR.AhR', 'NR.AR', 'NR.AR.LBD', 'NR.Aromatase', 'NR.ER', 'NR.ER.LBD',\n",
    "           'NR.PPAR.gamma', 'SR.ARE', 'SR.ATAD5', 'SR.HSE', 'SR.MMP', 'SR.p53']\n",
    "for target in targets:\n",
    "    print(target)\n",
    "    display(get_model_perfs(target))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explore Model Results for Best of All Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_folder = './flask_app/models/'\n",
    "models = {}\n",
    "pred_probs = {}\n",
    "pred = {}\n",
    "pred_tox = {}\n",
    "for y in feature_names:\n",
    "    target = y.replace('.','_')\n",
    "    pathname = glob.glob(models_folder+target+'.*')[0]\n",
    "    file_type = pathname[pathname.rfind('.')+1:]\n",
    "    if file_type == 'joblib':\n",
    "        models[target] = load(pathname)\n",
    "        pred_probs[target] = models[target].predict_proba(x.values.reshape(1, -1))[0][1]\n",
    "        pred_tox[target] = int(models[target].predict(x.values.reshape(1, -1)))\n",
    "    elif file_type == 'h5':\n",
    "        models[target]= load_model(pathname)\n",
    "        pred_probs[target] = round(models[target].predict(x.values.reshape(1, -1))[0][0],3)\n",
    "        pred_tox[target] = int(models[target].predict_classes(x.values.reshape(1, -1)))\n",
    "    print(target, pred_probs[target], pred_tox[target])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:metis] *",
   "language": "python",
   "name": "conda-env-metis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}