random_forests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimized Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Libraries and Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-06T16:57:56.662883Z",
     "start_time": "2019-12-06T16:57:54.171999Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "# make sure  numpy, scipy, pandas, sklearn are installed, otherwise run\n",
    "# pip install numpy scipy pandas scikit-learn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import io\n",
    "from src.helper_functions import load_data, get_model_perfs, init_model_perfs,\\\n",
    "                                 save_model, save_model_perfs, check_is_best,\\\n",
    "                                 read_model, evaluate_model_predictions,\\\n",
    "                                 update_model_perfs, check_and_save,\\\n",
    "                                 adjusted_classes\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score, confusion_matrix, f1_score,\\\n",
    "                            precision_score, recall_score, accuracy_score,\\\n",
    "                            average_precision_score, precision_recall_curve\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-06T16:58:10.455467Z",
     "start_time": "2019-12-06T16:57:59.289607Z"
    }
   },
   "outputs": [],
   "source": [
    "x_tr, y_tr, x_te, y_te = load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:29:08.024116Z",
     "start_time": "2019-12-03T00:28:08.167852Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         NR.AhR: 0.89981\n",
      "Model performance not better than that previously recorded.\n",
      "Model performance not better than that previously recorded.\n",
      "          NR.AR: 0.67893\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "      NR.AR.LBD: 0.76328\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "   NR.Aromatase: 0.77193\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "          NR.ER: 0.76901\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "      NR.ER.LBD: 0.75034\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "  NR.PPAR.gamma: 0.70985\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "         SR.ARE: 0.78084\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "       SR.ATAD5: 0.79562\n",
      "Model saved and metrics table updated.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/Users/sean/anaconda3/envs/metis/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved and metrics table updated.\n",
      "         SR.HSE: 0.75959\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "         SR.MMP: 0.93033\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n",
      "         SR.p53: 0.80670\n",
      "Model saved and metrics table updated.\n",
      "Model saved and metrics table updated.\n"
     ]
    }
   ],
   "source": [
    "# Build a random forest model for all twelve assays\n",
    "for target in y_tr.columns:\n",
    "    rows_tr = np.isfinite(y_tr[target]).values\n",
    "    rows_te = np.isfinite(y_te[target]).values\n",
    "    rf = RandomForestClassifier(n_estimators=100,  n_jobs=4)\n",
    "    rf.fit(x_tr[rows_tr], y_tr[target][rows_tr])\n",
    "    p_te = rf.predict_proba(x_te[rows_te])\n",
    "    y_testing=y_te[target][rows_te]\n",
    "    auc_te = roc_auc_score(y_testing, p_te[:, 1])\n",
    "    print(\"%15s: %3.5f\" % (target, auc_te))\n",
    "    y_hat_testing=rf.predict(x_te[rows_te])\n",
    "    average_precision=average_precision_score(y_testing,p_te[:, 1])\n",
    "    mv=evaluate_model_predictions(target,'RF',0.5,y_testing,y_hat_testing,\\\n",
    "                                  auc_te,average_precision)\n",
    "    filename=check_and_save(target,mv,rf)\n",
    "    precision, recall, thresholds = precision_recall_curve(y_testing,p_te[:, 1])\n",
    "    # f1 = 2*precision*recall/(precision+recall)  # Sometimes precision=recall=0!\n",
    "    p,r,t = zip(*[i for i in zip(precision,recall,np.append(thresholds,1)) if i[0:2]!=(0,0)])\n",
    "    p,r,t = np.array(p),np.array(r),np.array(t)\n",
    "    f1 = 2*p*r/(p+r)\n",
    "    m_idx = np.argmax(f1)\n",
    "    m_thresh = thresholds[m_idx]\n",
    "    y_hat_testing_adj=adjusted_classes(p_te[:, 1],m_thresh)\n",
    "    mv=evaluate_model_predictions(target,'RF_modT',m_thresh,y_testing,y_hat_testing_adj,\\\n",
    "                                  auc_te,average_precision)\n",
    "    if filename is None:\n",
    "        check_and_save(target,mv,rf)\n",
    "    else:\n",
    "        check_and_save(target,mv,filename)\n",
    "    get_model_perfs(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:29:19.590732Z",
     "start_time": "2019-12-03T00:29:19.456167Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.AhR\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.911475</td>\n",
       "      <td>0.756757</td>\n",
       "      <td>0.383562</td>\n",
       "      <td>0.509091</td>\n",
       "      <td>0.905028</td>\n",
       "      <td>0.602102</td>\n",
       "      <td>[[528, 9], [45, 28]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.235</td>\n",
       "      <td>0.842623</td>\n",
       "      <td>0.424837</td>\n",
       "      <td>0.890411</td>\n",
       "      <td>0.575221</td>\n",
       "      <td>0.905028</td>\n",
       "      <td>0.602102</td>\n",
       "      <td>[[449, 88], [8, 65]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF      0.500  0.911475   0.756757  0.383562  0.509091  0.905028   \n",
       "1  RF_modT      0.235  0.842623   0.424837  0.890411  0.575221  0.905028   \n",
       "\n",
       "   avg_precision      confusion_matrix   model_filename  \n",
       "0       0.602102  [[528, 9], [45, 28]]       RF0.joblib  \n",
       "1       0.602102  [[449, 88], [8, 65]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.AR\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.981229</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.678934</td>\n",
       "      <td>0.232723</td>\n",
       "      <td>[[573, 1], [10, 2]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.982935</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.678934</td>\n",
       "      <td>0.232723</td>\n",
       "      <td>[[574, 0], [10, 2]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF       0.50  0.981229   0.666667  0.166667  0.266667  0.678934   \n",
       "1  RF_modT       0.73  0.982935   1.000000  0.166667  0.285714  0.678934   \n",
       "\n",
       "   avg_precision     confusion_matrix   model_filename  \n",
       "0       0.232723  [[573, 1], [10, 2]]       RF0.joblib  \n",
       "1       0.232723  [[574, 0], [10, 2]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.AR.LBD\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.982818</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.763284</td>\n",
       "      <td>0.060344</td>\n",
       "      <td>[[572, 2], [8, 0]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.979381</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.763284</td>\n",
       "      <td>0.060344</td>\n",
       "      <td>[[569, 5], [7, 1]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision  recall        f1   auc_roc  \\\n",
       "0       RF       0.50  0.982818   0.000000   0.000  0.000000  0.763284   \n",
       "1  RF_modT       0.27  0.979381   0.166667   0.125  0.142857  0.763284   \n",
       "\n",
       "   avg_precision    confusion_matrix   model_filename  \n",
       "0       0.060344  [[572, 2], [8, 0]]       RF0.joblib  \n",
       "1       0.060344  [[569, 5], [7, 1]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.Aromatase\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.928030</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.771931</td>\n",
       "      <td>0.311884</td>\n",
       "      <td>[[489, 0], [38, 1]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.840909</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.512821</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.771931</td>\n",
       "      <td>0.311884</td>\n",
       "      <td>[[424, 65], [19, 20]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF       0.50  0.928030   1.000000  0.025641  0.050000  0.771931   \n",
       "1  RF_modT       0.13  0.840909   0.235294  0.512821  0.322581  0.771931   \n",
       "\n",
       "   avg_precision       confusion_matrix   model_filename  \n",
       "0       0.311884    [[489, 0], [38, 1]]       RF0.joblib  \n",
       "1       0.311884  [[424, 65], [19, 20]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.ER\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.215686</td>\n",
       "      <td>0.338462</td>\n",
       "      <td>0.769007</td>\n",
       "      <td>0.445565</td>\n",
       "      <td>[[462, 3], [40, 11]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.912791</td>\n",
       "      <td>0.607143</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.430380</td>\n",
       "      <td>0.769007</td>\n",
       "      <td>0.445565</td>\n",
       "      <td>[[454, 11], [34, 17]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF        0.5  0.916667   0.785714  0.215686  0.338462  0.769007   \n",
       "1  RF_modT        0.4  0.912791   0.607143  0.333333  0.430380  0.769007   \n",
       "\n",
       "   avg_precision       confusion_matrix   model_filename  \n",
       "0       0.445565   [[462, 3], [40, 11]]       RF0.joblib  \n",
       "1       0.445565  [[454, 11], [34, 17]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.ER.LBD\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.971667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.750345</td>\n",
       "      <td>0.268778</td>\n",
       "      <td>[[580, 0], [17, 3]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.305</td>\n",
       "      <td>0.965000</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.750345</td>\n",
       "      <td>0.268778</td>\n",
       "      <td>[[573, 7], [14, 6]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision  recall        f1   auc_roc  \\\n",
       "0       RF      0.500  0.971667   1.000000    0.15  0.260870  0.750345   \n",
       "1  RF_modT      0.305  0.965000   0.461538    0.30  0.363636  0.750345   \n",
       "\n",
       "   avg_precision     confusion_matrix   model_filename  \n",
       "0       0.268778  [[580, 0], [17, 3]]       RF0.joblib  \n",
       "1       0.268778  [[573, 7], [14, 6]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR.PPAR.gamma\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.947107</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.709846</td>\n",
       "      <td>0.129989</td>\n",
       "      <td>[[573, 1], [31, 0]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.890909</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.232558</td>\n",
       "      <td>0.709846</td>\n",
       "      <td>0.129989</td>\n",
       "      <td>[[529, 45], [21, 10]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF       0.50  0.947107   0.000000  0.000000  0.000000  0.709846   \n",
       "1  RF_modT       0.13  0.890909   0.181818  0.322581  0.232558  0.709846   \n",
       "\n",
       "   avg_precision       confusion_matrix   model_filename  \n",
       "0       0.129989    [[573, 1], [31, 0]]       RF0.joblib  \n",
       "1       0.129989  [[529, 45], [21, 10]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.ARE\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.841441</td>\n",
       "      <td>0.592593</td>\n",
       "      <td>0.172043</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.780838</td>\n",
       "      <td>0.430711</td>\n",
       "      <td>[[451, 11], [77, 16]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.823423</td>\n",
       "      <td>0.476190</td>\n",
       "      <td>0.537634</td>\n",
       "      <td>0.505051</td>\n",
       "      <td>0.780838</td>\n",
       "      <td>0.430711</td>\n",
       "      <td>[[407, 55], [43, 50]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF       0.50  0.841441   0.592593  0.172043  0.266667  0.780838   \n",
       "1  RF_modT       0.34  0.823423   0.476190  0.537634  0.505051  0.780838   \n",
       "\n",
       "   avg_precision       confusion_matrix   model_filename  \n",
       "0       0.430711  [[451, 11], [77, 16]]       RF0.joblib  \n",
       "1       0.430711  [[407, 55], [43, 50]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.ATAD5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.938907</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.795625</td>\n",
       "      <td>0.427828</td>\n",
       "      <td>[[584, 0], [38, 0]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.255</td>\n",
       "      <td>0.948553</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.368421</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>0.795625</td>\n",
       "      <td>0.427828</td>\n",
       "      <td>[[576, 8], [24, 14]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF      0.500  0.938907   0.000000  0.000000  0.000000  0.795625   \n",
       "1  RF_modT      0.255  0.948553   0.636364  0.368421  0.466667  0.795625   \n",
       "\n",
       "   avg_precision      confusion_matrix   model_filename  \n",
       "0       0.427828   [[584, 0], [38, 0]]       RF0.joblib  \n",
       "1       0.427828  [[576, 8], [24, 14]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.HSE\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.965574</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.759586</td>\n",
       "      <td>0.239563</td>\n",
       "      <td>[[586, 2], [19, 3]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.965574</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.759586</td>\n",
       "      <td>0.239563</td>\n",
       "      <td>[[583, 5], [16, 6]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF       0.50  0.965574   0.600000  0.136364  0.222222  0.759586   \n",
       "1  RF_modT       0.34  0.965574   0.545455  0.272727  0.363636  0.759586   \n",
       "\n",
       "   avg_precision     confusion_matrix   model_filename  \n",
       "0       0.239563  [[586, 2], [19, 3]]       RF0.joblib  \n",
       "1       0.239563  [[583, 5], [16, 6]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.MMP\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.906077</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.484848</td>\n",
       "      <td>0.930331</td>\n",
       "      <td>0.560184</td>\n",
       "      <td>[[468, 15], [36, 24]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.3825</td>\n",
       "      <td>0.913444</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.624000</td>\n",
       "      <td>0.930331</td>\n",
       "      <td>0.560184</td>\n",
       "      <td>[[457, 26], [21, 39]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision  recall        f1   auc_roc  \\\n",
       "0       RF     0.5000  0.906077   0.615385    0.40  0.484848  0.930331   \n",
       "1  RF_modT     0.3825  0.913444   0.600000    0.65  0.624000  0.930331   \n",
       "\n",
       "   avg_precision       confusion_matrix   model_filename  \n",
       "0       0.560184  [[468, 15], [36, 24]]       RF0.joblib  \n",
       "1       0.560184  [[457, 26], [21, 39]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR.p53\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.933442</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.024390</td>\n",
       "      <td>0.046512</td>\n",
       "      <td>0.806702</td>\n",
       "      <td>0.22199</td>\n",
       "      <td>[[574, 1], [40, 1]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.886364</td>\n",
       "      <td>0.254237</td>\n",
       "      <td>0.365854</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.806702</td>\n",
       "      <td>0.22199</td>\n",
       "      <td>[[531, 44], [26, 15]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF       0.50  0.933442   0.500000  0.024390  0.046512  0.806702   \n",
       "1  RF_modT       0.22  0.886364   0.254237  0.365854  0.300000  0.806702   \n",
       "\n",
       "   avg_precision       confusion_matrix   model_filename  \n",
       "0        0.22199    [[574, 1], [40, 1]]       RF0.joblib  \n",
       "1        0.22199  [[531, 44], [26, 15]]  RF_modT0.joblib  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for target in y_tr.columns:\n",
    "    print(target)\n",
    "    display(get_model_perfs(target))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cells Below Were Used For Development"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load a Saved Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:26:33.129065Z",
     "start_time": "2019-12-03T00:26:32.954741Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         NR.AhR: 0.90503\n"
     ]
    }
   ],
   "source": [
    "target=y_tr.columns[0]\n",
    "df = get_model_perfs(target)\n",
    "if len(df)==0:\n",
    "    rf=read_model(target,'RF0.joblib')\n",
    "else:\n",
    "    rf=read_model(target,df['model_filename'][0])\n",
    "rows_tr = np.isfinite(y_tr[target]).values\n",
    "rows_te = np.isfinite(y_te[target]).values\n",
    "p_te = rf.predict_proba(x_te[rows_te])\n",
    "auc_te = roc_auc_score(y_te[target][rows_te], p_te[:, 1])\n",
    "print(\"%15s: %3.5f\" % (target, auc_te))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:26:53.368663Z",
     "start_time": "2019-12-03T00:26:53.163406Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved and metrics table updated.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.911475</td>\n",
       "      <td>0.756757</td>\n",
       "      <td>0.383562</td>\n",
       "      <td>0.509091</td>\n",
       "      <td>0.905028</td>\n",
       "      <td>0.602102</td>\n",
       "      <td>[[528, 9], [45, 28]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0    RF        0.5  0.911475   0.756757  0.383562  0.509091  0.905028   \n",
       "\n",
       "   avg_precision      confusion_matrix model_filename  \n",
       "0       0.602102  [[528, 9], [45, 28]]     RF0.joblib  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_testing=y_te[target][rows_te]\n",
    "y_hat_testing=rf.predict(x_te[rows_te])\n",
    "average_precision=average_precision_score(y_testing,p_te[:, 1])\n",
    "mv=evaluate_model_predictions(target,'RF',0.5,y_testing,y_hat_testing,\\\n",
    "                              auc_te,average_precision)\n",
    "check_and_save(target,mv,rf)\n",
    "get_model_perfs(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:27:27.447146Z",
     "start_time": "2019-12-03T00:27:27.352860Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved and metrics table updated.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>threshold</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc_roc</th>\n",
       "      <th>avg_precision</th>\n",
       "      <th>confusion_matrix</th>\n",
       "      <th>model_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>RF</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.911475</td>\n",
       "      <td>0.756757</td>\n",
       "      <td>0.383562</td>\n",
       "      <td>0.509091</td>\n",
       "      <td>0.905028</td>\n",
       "      <td>0.602102</td>\n",
       "      <td>[[528, 9], [45, 28]]</td>\n",
       "      <td>RF0.joblib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>RF_modT</td>\n",
       "      <td>0.235</td>\n",
       "      <td>0.842623</td>\n",
       "      <td>0.424837</td>\n",
       "      <td>0.890411</td>\n",
       "      <td>0.575221</td>\n",
       "      <td>0.905028</td>\n",
       "      <td>0.602102</td>\n",
       "      <td>[[449, 88], [8, 65]]</td>\n",
       "      <td>RF_modT0.joblib</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  threshold  accuracy  precision    recall        f1   auc_roc  \\\n",
       "0       RF      0.500  0.911475   0.756757  0.383562  0.509091  0.905028   \n",
       "1  RF_modT      0.235  0.842623   0.424837  0.890411  0.575221  0.905028   \n",
       "\n",
       "   avg_precision      confusion_matrix   model_filename  \n",
       "0       0.602102  [[528, 9], [45, 28]]       RF0.joblib  \n",
       "1       0.602102  [[449, 88], [8, 65]]  RF_modT0.joblib  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(y_testing,p_te[:, 1])\n",
    "f1 = 2*precision*recall/(precision+recall)\n",
    "m_idx = np.argmax(f1)\n",
    "m_thresh = thresholds[m_idx]\n",
    "y_hat_testing_adj=adjusted_classes(p_te[:, 1],m_thresh)\n",
    "mv=evaluate_model_predictions(target,'RF_modT',m_thresh,y_testing,y_hat_testing_adj,\\\n",
    "                              auc_te,average_precision)\n",
    "check_and_save(target,mv,rf)\n",
    "get_model_perfs(target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Precision-Recall Curve\n",
    "Note: ROC curves can be misleading for strongly imbalanced data sets.<sup>[1](https://classeval.wordpress.com/simulation-analysis/roc-and-precision-recall-with-imbalanced-datasets/), [2](https://machinelearningmastery.com/roc-curves-and-precision-recall-curves-for-classification-in-python/), [3](https://scikit-learn.org/stable/modules/model_evaluation.html#precision-recall-f-measure-metrics), [4](https://stats.stackexchange.com/questions/90779/area-under-the-roc-curve-or-area-under-the-pr-curve-for-imbalanced-data)</sup>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:27:36.981654Z",
     "start_time": "2019-12-03T00:27:36.976466Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision-Recall Curve \"Average Precision\": 0.6021\n"
     ]
    }
   ],
   "source": [
    "average_precision=average_precision_score(y_testing,p_te[:, 1])\n",
    "print('Precision-Recall Curve \"Average Precision\": %0.4f' %average_precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:27:38.431794Z",
     "start_time": "2019-12-03T00:27:38.057412Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max F1: 0.57522, threshold prob: 0.23500\n",
      "Max F1 Precision: 0.42484, Recall: 0.89041\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaoAAAEuCAYAAADfrlY4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3wU1fbAvycVAoE8EJQiJOpDkZIEEOlVmgL6HiLFAiggIirvZ0Oe8niIylNsKIodRQQEAQkISAsBAWlJIPQWIPQinRBC7u+PO7vZbDbZTW/3+/nMJ5k7d+6cnUz2zDn33HNEKYXBYDAYDIUVr4IWwGAwGAyGzDCKymAwGAyFGqOoDAaDwVCoMYrKYDAYDIUao6gMBoPBUKgxispgMBgMhRqjqEoIItJfRJSInBORvzkd87GOjXZoa2O12bZkETkkIp85n+/BtVtYY5wQER8Xx4Ot4wM9GCvSSa6LIvKHiHT3UJbRTudfE5HtIvKyiBTI/4OITBaR+IK4tjMu7o/jdkdBy+eMiAwXkX8WtByGvMUoqpJHeeDVLPR/HmgKdASmAIOBH7J4zX7Wz8pAlyye64otlkxNgaeAMsBsEbk3C2O0sM7/BxAHvAv8KxdkKy7Y7o/jdrhAJXLNcMAoqmJOurdbQ7Hnd+A5EflIKXXcg/47lFLrrN+Xi0hlYKCI3OLJ+SJSGugJRAKN0UorInui27noINM6EVkDHAIGAH96OMafSqlkS8ZFQH1gEPB+DmUrLtjvT24iIv5KqWu5Pa6heGMsqpLHWOvnv7N5/mbrZw0P+z+EtuI+A+YAXTNxHXqLyBgROWa5KCNEpLq7CyilEoBTWZDJ+fwUINb5fBG5Q0SmiMgBEbkqIvtF5HMXrtPJIpIgIuEiskpErojIHhEZ4nwtEWkvIptFJFFE9onI065kEpEqIvKDiJy23JNbROQxpz42d24zEfnZcoOeEJHXrOOdRSRaRC6LyAYRaZid+5OBfI1FZKmIXLLGXyYijTO4L01FZI2IXEVbrrbjg0Qk1roXp0XkGxGp4DTGCyKyw7r/f4nIRhH5h3UsHqgJPOrgnpycW5/RUHgwiqrkcQz4FBgsIjWzcX4wcAOI97B/P+AcMA/tMvQHemfQ9zXgDuBJ4AW0u2mquwuISCBQEdjnoUyuCHZxflUgAe1e6gSMAdoDv7k4vxzwE/Aj8CCwAfhcRNo6yFnbOvcq+h6MtMZu7/R5ygAr0W7SkWhlvxWYIiKDXVz7e+v4P4C5wNsi8j/gPeB/QC+0e3SuiPi5vRMab9Fzl7bN/l0hIvUt+f4G9AeesD7/ShEJdRqnPDAdmGZ9np+sMcahX16WAt2Bl4HOwEIR8bb6PIq2cKcB9wOPArMAmzL7B3AcWEyqe/JNDz+foSihlDJbCdjQXygKrQgqoJXHt9YxH+vYaIf+bay2jtbxQPQX5gVgvIfXrAokA19Y+17oL/51Tv2CrWutdGp/yWqv6tAWCay2ZPIBQoCZwEngNg9kGm2N6W+dXwmtIJOBh9yc64Oeu1FAuEP7ZKutrUObP3Aa+NKhbarVVsah7VYgCYh3aBtmjdfG6fpLrc/p7fQ3HeUk40ngOhDi0N7d6tvaw/vjvP3o0GeW9fwEObSVA84Cs13clwdd/L1vOMpttTe3+j9k7X8KbHYjb7yjbGYrnpuxqEogSqmz6DfVJ0TkTjfdF6O/9C6gXXdR6LdfT3gM8MYKvlDaxfYjcG8G113gtL/V+uns0mtuyXQd2A90A3oopfZ7KBdAonX+SeBt4DWl1FzHDiLiJyIjRWSn5ba6DqyyDjvLf0UptcK2o/Q8zB4n2ZsCvymlLjv0Owz84TRWK+CIUirSqf1HtGK926l9ocN4ycBeYLdS6oBDn53Wz1vxjCbAPQ7bG07yzVdKnXO47gW01dzaaZxkYL5TWwf0S8tUR6sNPb94wRoftFUaJiKfiMh9IhLgoeyGYoZRVCWXD9FvwGPc9HsW/UV1HzADeIC0X1qZ8QQ6yGGbiASJSBDwq8MxZ8467dsm3Us5tcdaMjVBR/1dBGaKSCUP5cI6tzHafbQZGCcibZz6vIO2MH5Ef+7GpEaYOcv0l4trXHPqVwU44aKfc1sFtIvWmeMOxzO7dlIGbZBe7ozYpJTa6LA5Kr3M5HOefzyplLrh1FbZ+rmX1BcO21YO7cYF/YLzDHAv+oXprIjMFpFgDz+DoZhgov5KKEqpSyLyDtqyei+TrruVUhsBRGQ5cDMwUkS+s6wBl4hII6COtevqS/xxEXnDsrKyyiWbTMCfInIAWI5WKs96OMYmy/rYICKrgF3AJyIS6iBTb+AHpZQtAAURKZsNeW0cQ98/Z5zbzpLeYgO4xfp5Jgcy5AZnSZXFkVtI/7Lhqo6QTf6OuH42zgAopRTwBfCFFcDSEf28zkArL0MJwVhUJZvPgCOkRgJmivXFMRzwA0a46d4P/SXVA2jrtI1Du6DaZEdoF3KtQLslB3oSJeji/DNoy7KuJa+NAPRbviMDsisnsBa43wqWAEBEbkW7Mh1ZCVQXEef2vmhX5Y4cyJAbrAQesIJYAHtASzfrmDuWAClADSerzZX1BoBS6i+l1AzgZ/TfycY1oHROPoyh8GMUVQnGmkcZg45o8/ScWOAX4CkRqWprF5254hvrd1+0NbJSKTVbKRXpuKEj0RJJXQicG4xCewiyspjZkS/QgR6vi4hYbYuAfiIyVEQ6isgkoFkOZByLdm39LiIPicgj6HVtzq6/yej5rdkiMtAKM5+Cntt5w4UrLb95E60clolID9GZIZaiFbs7VzJKqX3oZ+BTEXlXRB6wwvb7i8hUW6SkiHwpIu+LyMMi0kp05pLH0ffMxnagpYh0FZFGxi1YPDGKyvAd+ksxK4wCfEmrFLytDaArcBPwrauTrUn42UCPHLrSHMeMQ7uEBopIlWycfw39BVwfHd0I8Bw6QOAta+xAoE8OZNyBDrMOsMYbB3wELHPqdxkdlPC71edXIBR4XCn1ZXavn1sopbagreEL6ND4KcAldERhrIdjjERnOWmFtpJ+RT9Pf5H6PP4BNERb/kvQa/9+JO0Lzmtot+3P6OCL0dn+YIZCi2hvjsFgMBgMhRNjURkMBoOhUGMUlcFgMBgKNUZRGQwGg6FQYxSVwWAwGAo1RlEZDAaDoVBjFFU+Iekr5l4VXVl2lOiaTfkpi608RHAWzimQKrQu7lu2Kw3nkXyRIhLpsG+Tt03BSVV4EJFbReSGiCSJyE0Z9Il3+PumiMhhEZklInflkgwPiS53kigiB0XkdVuGdg/Pf0J0mZQrokuNrBaRek59brVkPi8iF6xUT9kqO2NIj1FU+Y+tYu4D6AKC/0EvNs1PFlgyuMrXlhFvovPiFRS5UWnYkP88gf6e8SXzNWi2Uh0t0Ov0GgOrRBfqzDYi0gm9QH0DuszIx8Dr6ETEnpz/NjAJ/T/zADrR8jL0WjhbnwB0Cq+70Gu8Hgf+DqxwzEJiyAEFnb69pGykls24z6n9O6u9Qibn+he0/IXwvn1ltd9SwPJFApEu5G1TwHIVimcGvRh3K3AQ2JhBn3icSnWgFzwr4P9yeP1o0pePGYVO0pvps4NWnCm4L//yArpsyR0ObSHozPE5kt9sejMWVcGzwfp5B9hdSatFpJvlrrgGDLWO+YjIa6LLTlwTkaNWipk0GbFFpIyIjBNdQfaaiBwXkV9E5GbreDrXn4j0ta53yXJfbBWH6rOuXH+StSq0Taz0OBcsuSc4y51FXFYaFpGbRFfhPWLJtFNcFBsUkRDR1XuPW/32i8jHDsfvsVw5CZabdpeIvJ2bbloRaS0iS6z7fVl0tdunHI4rERntdE6w1d7foc1lJV0R+U1ENrm4bhXLhTrc6X5MFZFT1v2IEauSbg4+X1OgFtrynQI0FJE6mZ9lJ83/RTavfysQhs5m4cgUtIXXxc0QzwAHlFP5Fxd0R9dY22trUDpf4R/oIpqGHGKypxc8IdbPcw5ttYAJaHfbflIzUv+ITvz5P2ANUNvqE4yVTFV0Bdcl6H/Qd4B16CqrndAlGNKVmRCRFtbYE9C1przQboygjISW1Cq0f0NXoT2MdotMEZEAlT7VzxR0pdZ/ot9UR6PT5fwno2u4IRinSsMiUg795VDaGv8A+nN/LiL+SqlPrH4hwHrginX9PegkuR0dxq8BxKDz7l1EZ4IfBdxGxhWKPUZEHkS7pP4AnkYXVKyDLq2eHWyVdMej/x5X0c/WNBG5Wym13aFvX+vnNEuWW9G1oE4C/wJOoasC/yIiDyml5ln9gtH39L9KqdEeyNQPbZFMBcqiUyA9gWf5GNP9X4iuWeUOpVJzIdqUYpxThwMicoX0db2caQHEiMgraKvpZnRdr/8qpWY69KtDavkaR7YBPT2Q2eCOgjbpSspG+oq55YCH0TnSoh36RaL/ucOczm9pnf+EU/ujVnuYtf+ktd89E1n6W32Crf2XgLNu5J9MzqrQ/tep33x0CZGs3rcMKw2j62QlAn93av8KrQh8rP0frPte1d31rf5iXfsx629T0envFelC3jZuxosHNgJemfRLU3XZagu22vs7/W1cVdItDZwH3nFqj0EXcLTtf4NWThWd+i0BYhz2a6LdWaMyktmhrz/6RWSxQ9tadLZ+L6e+8Whl5oPOzF8XrcBvAA2cPre7zfEZ7Wu13eVCvgTgGzefIdF6zg5YY3VAV5NOc6/RbsRxLs4fCyR78oyZLfPNWFT5z2Kn/fnoL31H4pVSMU5tndH/EL84vVnaMkm3Qn8BdQSOK+st2EM2AH8TkR/Rb+WrlUP11gzIrArtd+i31a0O7a6q995n2xEdhSUOx28o67/dwvm+LSB9peHOaMvggNM9WgwMtGTagr5H85VSR11+MuzW2b/RLxO3ol1FNv5OzmpC3Yn+0h+nslePyxXpKukqpa6KyC/AoyIyUimlREerhaKT3droDPwGnHdx394TkXJKqQtKqYN47oV5EG2ROwa8fA98jv67/+7Uvy+plh5o5dVTKWVz8R5FF8t0xzWH323Pk6uEpuKizRkv9ItRG5scIrIM/QyNJK0Vld1rGDzAKKr851m02+kqWiFddtHHVTReZfTb5qUMxq3o8PNIVgRSSq0UkZ7obOFzAERkJXoieEsGp2W1Cq2r6r3+Dvv7SOv2GoC2FGzY7lt5YBDaNfUGactKVEbPaTjXkLLheI8SMuhj4zv0F+oo9AvAZXQk2kQ8r5KbETY53MmQFVxV0gWtKAagLb0V6Ii0i6T9kq2Mdsm5qroMWt4LWZSnH9q1ukJ0ZWfQiu+6dcxZUS1E3+sbwFGlVBoXtVIqSUScX95c4agwbM+c87MIWok6P5POnAH8HJQlSqkUS1kNcej3VwbX+BuuC0MasohRVPmPvWJuJmRUFTUR7QJ0hc06OE3awnIeoZSaBcwSXXajDXoebJGIVM/grT+3q9B2I63ici6e50ml4TNot+MLGVxjl/XzNFAtI0GsII8H0W43xwCLehmdk0VOWz8zlMHiGvrlxJGKrjri+pkBPY94CHjMevnoA8xSSl116HMGWIX+m7siQ8vTFVbQjs1V6+ql6R8iEqiUuujQdjaz/wuH+TF3HES7CUHPEYGeQ1rrNFYAupZVZmxDz/WmE4e093sbqfNhjtztwTUMHmAUVdFhEXoSurxSalkm/X4HeotIN6VURFYvopS6BMwXkdvQa04qoucvnFkJ9BSR5kqpPxzas1WFVim11X0ve19lRaxFoysN28rPL0JbhYeUUiczGeJ34J8iUkUp5coq9EfX1nK2zPp7KqMbdqNdWwNF5EsnF6cjB0n/0vFAVi5k3aup6Hs0B6hO+vVni9ABLtucFFh2eQz93fIMOvjAkVB0Da6eZFCvLAOy7PpTSh0SkVj0PO7XTvJdR1txmTEHaC8ijRxekrzQlvYGh37zgPEicptSar/VLxhdudldJWyDJxT0JFlJ2chgPZCLfpHoOSJXx35CuxLeQEezdUC7weYAtaw+vuiIwEvoOZb70At1J2FNKpM+mGIMetFxb/TcU19gL2mDPCaTdqK6DPoL9wR6/qczOrJPAYMd+tmudYfTZxmNVd0+u/cNPbGdiBUUgXYL7kBbTkPQZe+7ooNFfnU4Lxht1Ryw7l9b9JfXjw591qIV7hPoYoezrHuSJlCCbK6jQltsN9DuuF5AO7Qy+a9Dn/9aff4NtLfu2S5cB1MkZHKtu6xzEtDWlTgdr4F22W5Au+VaowNWXge+dejnUTAFEIuOVhUXx7zRVtZKh7Z4nNZR5eL/3f3oAJgvrL/Nv6xn5j2nfqOsz1bToa0U2iI6iHaZdkH/r90A2jv9L+xFz7s+iA5Xt92DsnnxuUraVuAClJSN3FFUXmi3Vqz1z3be+v1dtKVl61cWeM/6B0tCzyXNAipbx/uTVlE9gJ4/OIZ+Iz2MjgSr6jDmZBwUldVWBa2cTlvnbQEec+pju1ZeKKra1pfGxw5tfwM+RCuhJLSyWQUMdzr3dnR4tk32/cCHDseD0W/cF60xPrXuU64oKqtvO7SiumRtscAAh+Ol0FbtMUuOGeh5siwpKqvPBuu8tzM4Xh1tdRxxeGaWOP49SY28G53JdcKtPm9k0ucttPIIsfbjySNFZY3/T+veXkMr6lFYUanOzyPW/4TTM/4j2tWdiH6B6ejiGjXQyw0uWH+ruc5jmS37m6nwazAYDIZCjclMYTAYDIZCTb4pKhH5VkROikhcBsfFSquz10rF0yC/ZDMYDAZD4SU/LarJ6An3jOiCXkj5d3Rm7M/zQSaDwWAwFHLyTVEppaLIfIHdg8APSrMOCBKRKvkjncFgMBgKK4VpHVU1dLSZjQSrLd06Fysb9mAAf//AhrfcUst+LCUFfHyggqt14gaDwWAAYNOmTaeVUpUKWg5PKEyKylVeLJchiUpn5v4SoFatRmratNQF7QcPwtmzMHBgnshoMBgMWeLSJZ31rGzZsgUsSVpE5GBBy+AphSnqLwGd/NNGdbKYusVgMBgKGytWrGDFihUFLUaRpjBZVPOAYSIyHbgXOK9cp7cxGAyGIkODBiaAOafkm6ISkWnoVfs3iUgCumCdL4BSahK6zMD96FQkV9AZnw0Gg6FIU62au9zDBnfkm6JSSvVxc1yRmlzUYDDkgOvXr5OQkEBiYmJBi1LiSUnRxQe8vApmpqVUqVJUr14dX19f950LKYXJ9WcwGHKJhIQEAgMDCQ4ORsTU7ytIzp3TNUiDgoLc9Mx9lFKcOXOGhIQEQkJC8v36uUVhCqYwGAy5RGJiIhUrVjRKqhAQEBBAQEBAgVxbRKhYsWKRt6yNRWUwFFOMkioc+Pk5177MX4rDc2AsKoPBYMhDkpOTSU5OLmgxijRGURkMhjxBRHjxxRft++PHj2f06NEen3/ixAm6du1KaGgod999N/fffz8AkZGRdO3aNV3/efPmMW7cOABGjx7N+PHjAejfvz+zZs3KwSfJGZcuXbIv+jVkD6OoDAZDnuDv78/s2bM5ffp0ts4fNWoUHTp0IDY2lu3bt9uVUEZ0796dESMKX+X3MmXKUKZMmYIWo0hjFJXBYMgTfHx8GDx4MB9++GG6YwcPHqR9+/bUr1+f9u3bc+jQoXR9jh07RvXq1e379evXT9dnw4YNhIeHs3//fiZPnsywYcNy90PkAr6+vkU6NLwwYBSVwVASaNMm/fbZZ/rYlSuuj0+erI+fPp3+mIc8++yzTJ06lfPnz6dpHzZsGE888QRbtmzh0Ucf5fnnn3d57lNPPUXbtm156623OHo0bUa1NWvWMGTIEH799Vduu+02j2XKb8wcVc4xispgMOQZ5cqV44knnmDChAlp2teuXUvfvn0BePzxx1m9enW6czt16sT+/fsZNGgQO3fuJDw8nFOnTgGwY8cOBg8eTEREBDVq1Mj7D5IDzBxVzjHh6QZDSSAyMuNjAQGZH7/ppsyPu2H48OE0aNCAAQMyzoqWUQh1hQoV6Nu3L3379qVr165ERUVRsWJFqlSpQmJiItHR0VStWjXbsuUHZn4q5xiLymAw5CkVKlTgkUce4ZtvvrG3NWvWjOnTpwMwdepUWrRoke685cuXc+XKFQAuXrzIvn377NZTUFAQCxYsYOTIkUTmQInmB2aOKucYRWUwGPKcF198MU3034QJE/juu++oX78+U6ZM4eOPP053zqZNm2jUqBH169enadOmDBw4kHvuucd+/OabbyYiIoJnn32WP//8M18+R3Ywc1Q5R3Qu2KKLKZxoMKRnx44d1K5du6DFMFCwuf5suHoeRGSTUqpRAYmUJcwclcFgMOQhha2yb1HEKCqDwWDIQ3x8zNdsTjF3sARy+rR2jzpToYIO8DIYDLnH9evXAUxARQ4wiqqYkZEScuTiRYiJAcfpyWvXtKLqk2l5S4PBkFUuX74MFOwcVVHHKKpixtmzsGYNuAsy8veHu+9O3bcFoRgMhtzFzFHlHKOoiiHJyRAeXtBSGAwGMHNUuYG5g0UYV24+a32kwWAoJJg5qpxjFvwWYWxuvqio1G3jRu3WMxgKGm9vb8LCwqhbty49e/a0Z5nIDRo2bEhSUhLBwcHUq1eP+vXr07p1aw4ePJhr14iOjqZXr17Uq1ePe+65h9GjR3P16lWXfePj4xER3njjDXvb6dOn8fX1ZejQofZ5quwyefJkKlWqRFhYGGFhYTzxxBMAzJw5kzp16uDl5cXGjRvdjFJ0MYqqCHH6NOzenbpduZLq5nPcHOeeDIaConTp0sTExBAXF4efnx+TJk1Kc1wpRUpKSpbHjY+Pp1q1avYS7ytWrGDLli20adOGsWPH5ors8+bNY9iwYQwfPpwtW7bwxx9/ULVqVR544AGuXbvm8pzbbruN+fPn2/dtSsTX1zdX5ql69epFTEwMMTEx/PDDDwDUrVuX2bNn06pVqxyPX5gxiqqQ4qyUdu/WAQ+rVhnryZB1IiIi2L17NwApKSlERESwZ88eQKf4iYiIYN++fQAkJSURERHBgQMHAEhMTCQiIsJurWTHMmrZsiV79+4lPj6e2rVrM3ToUBo0aMDhw4f5/fffadq0KQ0aNKBnz572TOMbNmygWbNmhIaG0rhxYy5evAjAwoUL6dy5c7prNG3alCNHjtj3f/zxRxo3bkxYWBhPP/00N27cAGDRokU0aNCA0NBQ2rdvn26cc+fOMWbMGBYvXkzTpk0REfz8/Bg8eDCPPvpoukzwNkqXLk3t2rXtls2MGTN45JFH8PLywsfHh4iICO69917Cw8O57777OHHiBADPP/88Y8aMAWDx4sW0atXKYwVeu3Zt7rzzTo/6FmWMoiqkuHLrRUeDn1/eWE9XrsDVq2kVYzYLsxoMaUhOTmbhwoXUq1cPgF27dvHEE08QHR1NmTJlGDt2LEuXLmXz5s00atSIDz74gKSkJHr16sXHH39MbGwsS5cupXTp0oBWNK4U1aJFi3jooYcAnTJoxowZ/PHHH8TExODt7c3UqVM5deoUgwYN4pdffiE2NpaZM2emG+fnn3/m6aefpmzZsrz55ps0aNCAl19+maeeeop+/fqxcOHCDD9r7969mT59OgkJCXh7e1O1alVu3LhBUlISLVq0YN26dURHR9O7d2/effddAMaNG8eMGTNYsWIFzz//PN999x1eXum/mmfMmGF3/X333XdZ/0MUYUwwRSHBOTDC0a2XXyQmaoUIZl1VcaNbt2723728vNLs+/j4pNn38/NLs1+qVKk0+wEBAR5d8+rVq4SFhQHaonrqqac4evQoNWvWpEmTJgCsW7eO7du307x5c0Bbc02bNmXXrl1UqVLFnoS2XLly9uMJCQlpCiW2bduWEydOULlyZbvrb9myZWzatMl+/tWrV6lcuTLr1q2jVatWhISEADqzuzOxsbEMGTKE2NhYYmJi2LhxI3PnzmXChAluI/g6d+7MG2+8wc0330yvXr0AraivXLlCQkICvXr14tixYyQlJdllCAgI4KuvvqJVq1Z8+OGH3H777S7H7tWrF59++qm7214sMYqqkOBq/VN+uvXKlAGRVMVo1lUZcoptjsoZx/pMSik6dOjAtGnT0vTZsmWLyxpVq1atSlcSZMWKFZQpU4b+/fszatQoPvjgA5RS9OvXj3feeSdN33nz5mVY+8pRJm9vb7Zv306HDh3w8vKiS5cuGbr8HPHz86Nhw4a8//77bNu2jYiICPz8/AgMDOS5557j//7v/+jevTuRkZGMHj3aft7WrVupWLFiuirGBo1x/RUinAMj8jMookYNaNs2/65nMAA0adKEP/74g7179wJ6/mv37t3cddddHD16lA0bNgC6HlVycjKLFi2iS5cu6cYpXbo0H330ET/88ANnz56lffv2zJo1i5MnTwJw9uxZDh48SNOmTVm5cqV9/u2si7exevXqsXbtWu68806WLVtGSkoKixcvBuD777+3W38Z8eKLL/K///2PihUrAroopLe3N+fPn6datWr2cWwcPHiQ999/n+joaBYuXFioS5YUFMaiymM8SWkERXP9k8kZaMgplSpVYvLkyfTp08ceTTd27Fhq1arFjBkzeO6557h69SqlS5dm6dKlREZG2gMPnKlSpQp9+vRh4sSJvPHGG4wdO5aOHTuSkpKCr68vEydOpEmTJnz55Zf885//JCUlhcqVK7NkyZI04zzyyCO0bNmS9evXU6dOHRo1akT79u1RSrFnzx5GjRqV6WeqU6cOderUse/b5qhGjx5Nz549qVatGk2aNOHAgQMopXjqqacYP348VatW5ZtvvqF///5s2LCBUqVKub1/c+bM4bnnnuPUqVM88MADhIWF2ZVqccLUo8pjdu/2LKURpE9rVJC4uo/OisnkDCy8FMd6VAkJCQwaNCjTYIbcYsaMGXzxxRdMnDiR2rVrc/36dRYtWkTNmjWpX79+lsYy9ahyjrGo8oHiktIoo3k0kzPQkB9Ur149X5QU6MCFmjVr8tprr3Hw4EHKli3LAw88QMeOHbM8VmBgYB5IWLIwigrjwnKFY7i6Y1txUboGgzuaNGnC3LlzczyOt7d3LkhTsvFYUYlIKaArcDvwhVLqnIjcDvyllCrS79CuLAXjwkPgN58AACAASURBVEobrm7DLDA2GLJGUlISgD2ThiHreKSoROQOYAkQCAQBM4FzwDPWfiGZEco+zpZCSXdhOYere4orSwxKtnVqKNnYMnkYRZV9PLWoPkIrqmfQCsrGPKBkLZF2g6uFu0WRGjX0lh2cLTFjnRpKMmaOKud4qqiaAU2UUjecFssdAqp6ejER6Qx8DHgDXyulxjkdrwF8j7bSvIERSqnfPB2/MFDQC3cLA4mJxjo16LkZW9okgLlz5xIYGMjDDz/Mhg0b6N+/f4nItGDmqHJOVoIpXBVTqQGc9+RkEfEGJgIdgARgg4jMU0ptd+j2OvCzUupzEbkb+A0IzoKMhYKSHHBQpgxYKdkMJRxXmSkuX77Mm2++SVxcHHFxcQUkWf5iWx/mX9LeWHMRTzNT/A78n8O+EpFywH+BBR6O0RjYq5Tar5RKAqYDDzr1UUA56/fygMknUsQwGS4MmVGmTBlatGjh0WLW4sLVq1czrGNl8AxPLar/A1aIyC6gFDADuAM4ATzi4RjVgMMO+wnAvU59RgO/i8hzQBngPg/HzhLZnUfyJMtEUZ2TymuuXUsbYGGCK/KP4cOHu8y5lxPCwsL46KOPMu3jmJQ2JCSEOXPm5KoMRQVbQl1D9vFIUSmljopIGNAHaIC2xL4EpiqlPH1VcJUJ0jktRh9gslLqfRFpCkwRkbpKqTTFWURkMDAYoHLlrM/4Z3ceydV5rjAWflrKl4fjx01m9pJGRklpSxquSnYYsoZbRSUivsCPwEil1LfAt9m8VgJwq8N+ddK79p4COgMopdZaa7duAk46dlJKfYlWlNSq1ShbOaCyO49UkuefsktQENzrYDub4Ir8xZ3lY8hbzBxVznGr6pVS14GOpLd+ssoG4O8iEiIifkBvdHi7I4eA9gAiUhvtZjyVw+saDAZDgWHmqHKOp3NUs4F/AuOzeyGlVLKIDAMWo0PPv1VKbRORMcBGpdQ84EXgKxH5F1ox9ldFPWuuwWBIQ3BwMBcuXCApKYm5c+fy+++/c3dhycacB5g5qpzjqaI6BLwuIi2BjcBlx4NKqQ88GcRaE/WbU9soh9+3A5kXezEYDEWCS5cuuWyPj4/PX0EKGDNHlXM8VVT9gb+A+tbmiAI8UlQGg6eYRMGG4kJiYiJAiQrJz208jfoLyWtBDAZHTKJgQ3HBKKqck+UyHyJSFlBKqctuOxsMGeC8rsoZVyVFnKMFjdVlKAqUL1++oEUo8mSlzMezwKvohbuISALwP6XUZ3kkm6GY4ryuKiNcRfM6KjhTYdhQFHDKj2rIBp6W+RgJvIaO+lttNbcExolIOefksgZDZjivq/IUVwrOVBg2FHaM6y/neGpRDQEGK6WmObQtE5E9wNtAoVVUrtxDnqQ5yqjCraHgyK6CMxgKEqOoco6niqoyesGuM+uBm3NPnNwno7RHniwSNxVuDcWFdevg3Dn3/TwlKAiaNMm8j63MR3JyMiEhIUyZMoWgoKAcXzs+Pp6uXbvmevb10aNH89VXX1GpUiUAOnfuzLhxOX8HdzVHFRMTw9GjR7n//vtzPH5JwFNFtRvoC4xxau8L7MpVifKA7KY9cq6rZCiauLKqS1rAxblzYH3/5gqnPMgX45jrr1+/fkycOJF///vfuSdEHvCvf/2Ll156Kcvn3bhxI8O6U67mqGJiYti4caNRVB7i6Uq00cAoEVkqIv8VkdEishRdP+o/eSadwZAL2KzqqCi9LVmiN0P+0bRpU44cOQLohcDt27enQYMG1KtXj19//RXQllLt2rUZNGgQderUoWPHjvbUQ5s2bSI0NJSmTZsyceJE+7iJiYkMGDCAevXqER4ezooVKwCYPHkyDz30EN26dSMkJIRPP/2UDz74gPDwcJo0acLZLExkLlu2jPDwcOrVq8eTTz5pz90XHBzMmDFjaNGiBTNnzmTfvn107tyZhg0b0rJlS3bu3AnA1KlTqVOnDqGhobRq1YqkpCRGjRrFjBkzCAsLY8aMGTm/wcUcjxSVUmo2uiTHcaAr0N36vbFSam7eiWcwZA3HuUXb5hjqHh4OVarAZbO4It+4ceMGy5Yto3v37oCeq5kzZw6bN29mxYoVvPjii9gype3Zs4dnn32Wbdu2ERQUxC+//ALAgAEDmDBhAmvXrk0ztk1pbd26lWnTptGvXz/7nFBcXBw//fQT69ev59///jcBAQFER0fTtGlTfvjhB5eyfvjhh4SFhREWFsbixYtJTEykf//+zJgxg61bt5KcnMznn39u71+qVClWr15N7969GTx4MJ988gmbNm1i/PjxDB06FIC33nqLmTNnEhsby7x58/Dz82PMmDH06tWLmJgYevXqlYt3u3jicXi6UmoT8FgeymIw5AqezC26WsdV0tyBeY2tHlV8fDwNGzakQ4cOACilGDlyJFFRUXh5eXHkyBFOnDgB6LpVthpWDRs2JD4+nvPnz3Pu3Dlat24NwOOPP87ChQsBWL16Nc899xwAd911FzVr1mS39Ydt27YtgYGBBAYGUr58ebp16wZAvXr12LJli0uZnV1/sbGxhISEUKtWLSDVhTl8+HAAu5K5dOkSa9asoWfPnvZzbZZXq1ateOGFF3jkkUf45z//maN7WlLxNDy9J5CklPrVqf1BwFcpNSsvhDMYskqZMiCS+dyiqzB3s/4q97HNUZ0/f56uXbsyceJEnn/+eaZOncqpU6fYtGkTvr6+BAcH260gx1IY3t7eXL16FaVUhmuRMstZ7TiWl5eXfd/Ly4tkd0XlPBgfdMVigJSUFIKCglzW35o0aRJ//vknCxYsICwszNToygZZmaNKdNF+2TpmMBQKatSAtm0z72MLc7e5Ao07MG8pX748EyZMYPz48Vy/fp3z589TuXJlfH19WbFiBQcPHsz0/KCgIMqXL8/q1XoJ59SpU+3HWrVqZd/fvXs3hw4d4s4778w12e+66y7i4+PZu3cvAFOmTLFbdo6UK1eOkJAQZs6cCWgFFxsbC8C2bduoX78+Y8aM4aabbuLw4cMEBgZy8eLFXJOzuOOp6+82XEf37bWOFTvKlIHSpQtaCoMhdwgK8ixSLyvjZYXw8HBCQ0OZPn06jz76KN26daNRo0aEhYVx1113uT3/u+++48knnyQgIIBOnTrZ24cOHcqQIUOoV68ePj4+TJ48OVcLFJYqVYrvvvuOnj17kpyczD333MOQIUNc9p06dSrPPPMMY8eO5fr16/Tu3ZvQ0FBGjBjBvn378PLyon379oSGhlKjRg3GjRtHWFgYr732mpmncoN4Uu5JRI4C/ZRSS5zaOwJTlFIFtpaqVq1Gatq0jfZ9W2aCgQP1/u7d2sVjwswNmeH83BR1duzYQe3atQtaDEMhwdXzICKblFKNCkikLOGp6+9X4EMRqWVrEJE70eU9TNSfwWAwGPIMTxXVK8B5YLuIHBaRw8A24ALwcl4JZzAYDEUdU4o+53haj+oi0FxEOgBhgACbgWWmVLzBUDjJLFrOkH8kJSUBOgqyICgOX9FZqkdlzVGZNf0GQyGnVKlSnDlzhooVKxplVcAUZD0qpRRnzpwp8glxM1VUIhIKVFBKrXBoexR4EygLzAaeV0ol5amUBoMhS1SvXp2EhARO5Waon6FIUqpUKapXr17QYuQIdxbVWHSG9BUAInI38J21vxN4EjiCVlwGg6GQ4OvrS0hISEGLYQB7lve6desWsCRFF3fBFA1I6+rrDWxXSnVSSr0ADAfMAgCDwWDIgCNHjtgT8hqyhzuLqiLaYrLRCohw2I8EPsxlmQwGg6HY4LhA2ZA93FlUp4BqACLiDTQE/nQ47gek5I1o2cM5e7apymswGAxFG3cWVSTwHxF5FnjYalvhcPxuID73xcoZztmzTVVegzscX3BsmGzqhtzAlqm9fv36BSxJ0cWdonoDWIrO6XcDHeHnmLrzcWBZHsmWbUxlXkN2cHzBMdnUDbmFrYSJIftkqqiUUvEichdQBzillDrq1OU/QEJeCWcw5CeOLzi23H8GQ06x1eEyZB+3C36VUslAbAbHXLYbDEUNky3fYCi8ZCkzhcFQXKlRQ28GQ25jK5Roq1xsyDpGURkMBkMecubMmYIWochjFJXBYDDkIe3bty9oEYo8npb5MBgMBoOhQMiRohIRHxExnn2DwWDIgM2bN7N58+aCFqNIk1PXXx10XSrvXJDFYDAYih3nzp0raBGKPMb1ZzAYDHlIu3btaNeuHSLC448/bm9PTk6mUqVKdO3aNcfXiIyMpHz58oSFhREWFsZ9990HQFRUFA0aNMDHx4dZs2ZlaUwRCRGRP0Vkj4jMEBE/F32CReSqiMRY2ySrPdChLUZETovIR9ax/iJyyuHYQHeyuKtHtd/N+ekEdzNeZ+BjtAX2tVJqnIs+jwCjAQXEKqX6ZuUaBoPBUBgpU6YMcXFxXL16ldKlS7NkyRKqVauWa+O3bNmS+fPnp2mrUaMGkydPZvz48dkZ8n/Ah0qp6ZYCegr43EW/fUqpNLH3VlV4e5uIbELXL7QxQyk1zFNB3FlUVdBlPiZmsP3s6YWspLYTgS7oHIF9rPpWjn3+DrwGNFdK1UGXETEYDIYiy8aNG9m4cSMAXbp0YcGCBQBMmzaNPg45utavX0+zZs0IDw+nWbNm7Nq1C4APPviAJ598EoCtW7dSt25drniYbTs4OJj69evj5ZU155nostDtAJsZ9j3wUJYGSR3r70BlYFV2zgf3iioO2KKUet/VhhbeUxoDe5VS+62KwNOBB536DAImKqX+AlBKnczC+AaDwVDouHTpEpcuXQKgd+/eTJ8+ncTERLZs2cK9995r73fXXXcRFRVFdHQ0Y8aMYeTIkQAMHz6cvXv3MmfOHAYMGMAXX3xBQEBAuuusWrXK7vp76623MpXp4sWLAHc7ueds293oEk/nrMxEoFPlZWT+hYhItIisFJGWLo73QVtQyqGth4hsEZFZInJrpsLiPpjiD6BWJscvAVGZHHekGnDYYT8BuNepTy0AEfkD7R4crZRa5DyQiAwGBgNUrmyCDg15w7VrabOpg8mobsg6bdq0sf9ev3594uPjmTZtGvfff3+afufPn6dfv37s2bMHEeH69esAeHl5MXnyZOrXr8/TTz9N8+bNXV7HlesvIwIDA0EXwW3k6riIVHLRrFy0HQNqKKXOiEhDYK6I1FFKXXDo0xudwNxGBDBNKXVNRIagDZ52mcnrLiltpq43pdQ+oG1mfRwQV0O4kOfvQBugOrBKROoqpdKEzSilvgS+BKhVq5Grm2cw5Ijy5eH48bTlYq5ehYAAaOn0zmiUlyErdO/enZdeeonIyMg0WSveeOMN2rZty5w5c4iPj0+j4Pbs2UPZsmU5etQ5L3j2cLSoXBzuC+wAgkTEx7KqqgPpLq6UugZcs37fJCL70AbHRgARCQV8lFKbHM5xTNXxFXouLFPyMzNFAuBo4rn64AnAOqXUdeCAiOxCK64N+SOiwaAJCoJ7nez9HTvg2LG0ysuUAzG4Y/369Wn2n3zyScqXL0+9evWIjIy0t58/f94eXDF58uQ07S+88AJRUVEMGzaMWbNm8fDDD5MT3FlUACKyAl2HcDrQD/jVRZ9KwFml1A0RuQ39fe0YhNcHmOZ0ThWl1DFrtztaKWZKpnNUIjJbRMo57DcXkeyWIdwA/N0KefRDm4PznPrMxbLQROQmtGZ2F3loMOQLZcqAiC4FYtuqVIHLl92fayi5JCYmkpiYaN+vXr06L7zwQrp+r7zyCq+99hrNmzfnxo0b9vZ//etfDB06lFq1avHNN98wYsQITp70bPp+w4YNVK9enZkzZ/L0009Tp06drIj+KvB/IrIXPWf1DYCIdBeRMVafVsAWEYlFB14MUUo5Fsh5BCdFBTwvItusc54H+rsTRNLObzkdFLkBVLEFNYjIBSBMKZUt5SEi9wMfoeefvlVKvWV94I1KqXlWpMn7QGd0oca3lFLTMxuzVq1Gatq0jfb9HTvgwAFwcv8aDHmCrW7VQLcrQQwlFdt3rP56KzyIyKbMLKrChDvXn/OdzdGdVkr9Bvzm1DbK4XcF/J+1GQwGQ6Hh4sWL/PXXX9TIpB5MSkoKW7ZsITo6mi1btti3O+64g7Vr1+ajtMULkz3dYDAUG/766y/i4+MJCwvLkgWjlEIpZV9vdPLkSaKjo9Nse/bsASA+Pp6aNWvaz7169SrLli1j3rx5REREcPz4cQBKly5NvXr1KFWqFNu2bcvFT1ny8ERR1RcRm89RgDoiEuTYQSllMi4aSiRXruhoQMcwdhMFmH+cPn2aVatWsXLlSiIjI4mN1UXHIyIiuOWWWzh79mya7cyZM+nazp49a5/zuf/++4mOjubYsWP2awQHBxMeHk758uXZuHEjwcHBBAQEUL58ecqVK8ehQ4e4evUqgYGBdO7cmW7dutGkSRNuu+02vL296dmzJ4sWpVtlY8gCniiqxaR1+TlHfihMUlpDCSYxMTUS0EQB5i0nTpwgKiqKlStXsnLlSuLi4gBtvTRt2pQOHTqwZMkSunXr5vL8wMBAKlSoYN/q1atHhQoVWLx4MfHx8Rw6dIj77ruP8PBwwsPDCQsLIyhIv5dfuXKF77//npMnT3LhwgXOnz/P+fPn6dChA926daN169b4+6ePNatatSo+PsZ5lRPc3b2QfJHCYCjCJCbqCEBIDa4w5A5Hjx61K6WVK1eyc+dOQOfNa968OX369KF169bcc889+Pn5cf36dX788Ue8vb3TKKQKFSrwt7/9DV9f3wyvlZKSkmmqoYCAAJ555plc/4wG97hb8HswvwQxGIoiZcpA6dIFLUX+oZQiISGBuLg4tm3bxu7duxk0aBD33HNPjsaMjY3F29ub8uXLp1FMe/fuBaBcuXK0aNGCAQMG0Lp1axo0aOBS6fj6+jJgwIBsyZHVfHiecuzYsTTh5oasY+xRQ4nmzJkTLFw4leXLZ/HII8Po3Dlryfpr1NBbceT06dNs2bKFuLg4u2KKi4vjwoULafqVLVs2W4rq4MGD/PTTT/z4449s3749zbGgoCBatmzJkCFDaNOmDaGhocZ9VoIxf3lDiSM5+TqRkb+yYMH3rFmz0P62W6tWWJYVVXEjOTmZ3377jS+++IKFCxfa1wBVqFCBunXr8thjj1G3bl3q1q1LnTp1CA4OZs6cOcTFxXHt2jX7lpSUlGZ/yJAhvPjii6xdu5Y1a9awYsUKVq9eDUCLFi34xz/+QWJiIp06daJ169bUq1cPb+/iMfVdpUqVYvNZCgqjqAwlis2boxg37hn2799OpUpVefzxl+natR+DBrVK02/LlrX8+ecSnnzy31n+ksmLZLZKKZYsWULp0qVp6ZxsMIcopdi7dy9Tp07lm2++ISEhgSpVqjBy5EjatGlD3bp1ufnmm12Gez/yyCNER0dz8eJF/P39KV++PP7+/vj7++Pn54e/vz8LFixg3LhxjB071h4CHhYWxtixY+nbty8hIWYq3JA5HikqEQkAEpVSKXksj8GQJ/z11yk+/vhl5s//nipVavLuu7/QuvWD6ZTQ8eOH+OSTESxerLO+tGvXg9tvd512RinFwYO7qVmzlv1L3FUy25xEAiYlJTFt2jTGjx9vj3Dbu3cvt99+u8v+8fHxLFy4kIMHD3Lo0CEOHTpEQkICI0aMYMiQIfZ+R44cYfny5Sxbtozly5dz+PBhRISOHTsyYcIEunbtmmnggY2vv/7abZ/x48ezbNkymjVrRrNmzbj33nspW7ash3eg6HP06FGSk5NJSEiwLwDetWsXw4YNo2HDhgUtXpHAraKyCh6eB0KB7W66GwyFipSUFObO/ZpPPx3BlSuXGDDgNZ566nVKlUpfz2ft2kVERHwHQOPG7Vm/fhkZpRjbtGkln346gq1b1/HJJ4to2rQTAP7+l7nzzqsEBaWaTwcP6mS2WVlrdejQIaZPn86ECRM4cuQI9erVo0OHDixfvpw777yT/v378/rrrxMcHIxSipUrVzJhwgR+/fVXUlJS8PX15dZbb6VGjRqcOHGCP//8k4EDBzJv3jw+++wzli1bZslRgbZt2/Laa6/RpUsXgoODs3iH3fPSSy/x0ksv5fq4RQUvLy8uXbrErbemLbtUtWpVo6g8xK2isrLiHiSLZecNhvxGf2HPo1Sp0jRp0pFdu2IYN+4Ztm5dR8OGbRgx4jNCQmq7PNfHx5cjRw7QqVMfnntuHHFx61m/fpl93B07NrFkyQx8fHzZtSuaNWsWUbZseQAuXPiLxMQr/PzzRL7//n8EBgYxd66OVktJSeH48c2sW7eVBQv2cPLkXk6c2IuPD+zYsYZSpUrZ+23atImIiAjmzZtnX7javn17vv76azp16oSIcOzYMcaNG8ekSZP4/vvv6du3L7GxscTGxlKhQgVeeeUVBg4cSEhIiD2KrWbNmqxatYqaNWty9OhRatSowZgxY+jWrVu2qr8assbLL79M5cqVqV27NvXr16du3bpUquSq3JMhIzydo3oTGCcijymlTuelQAZDdjh+/BDjxg1l9eoF3HrrHbRo0ZUZMyYQFHQTY8ZMoUuXRzNNqfPmm1MpVSqAunUbAxAXp0szTJv2EdHRqzh0KNUcKlfub7zwwnvcc097HnusAUuWzOCDD4Zz5swJypQpx19/nWLTppUsX/4LK1bM5uTJIwB4e/tQrVoIKSlexMfvIiEhgZ07dzJv3jzmz5/PsWPH8PLyokWLFowfP55u3bpRq1bauqVVqlTh448/5uWXX+btt9/m66+/5s477+Srr77i0UcfpbSLWPmyZcuyfft2OnfuzKRJk7j//vvN5H4+0qhRIxo1KhK5XwstmWZPt3cS2Ype/OuLrhmVprCBUqp+nkjnASZ7esnmxo0b/Pzzp3z22b9RSlG+fAVOnEhAROjRYwhDh75FuXJ/y/K4K1fO48UXH0REaNiwDZ0798XLy5uTJ4/Qq9cwAgODiI/fycMPawutQYPWDB06lqVLZzJ9+gQA/P1L0bRpZ9q160H9+k255Zaa+Pj48PXXXzFp0mB8fX25fv26PfVO9+7d6dKlCxUrVgTg9On0i4edXYZJSUn4+vpmqoT37duHiHDbbbdl+T4Yck6UNWHZqlVqwI6vry+vvPKK25LxeUlxyp5uY1aeSmEwZJGUlBTWrFnEV1/9l23b1tOsWWdGjPic5ctnExU1j+eff9duHWWHpk078fbb0wkNbc7NN1d32adGjVoMGvQfwsNbcs897RARUlJSuHz5As2adaF58/sJCEgfNBASEk6NGuF0796c7t2707p1a/z80nvWz56FNWsgOVnvuwrKcHWeMxkFXhjyB5t715B9PLKoCjPGoipZXLp0gfnzJzNjxiccPryXSpWq8sIL4+nUqXehq/eTEbbgig4dMu935Qps3Jg+PZOpfVX0MRZV1sjSOioRaQfcjU5Eu00pFZkXQhkMziQmXuGrr8Ywa9ZnXL58kXr1mvD002No374Hvr5FK87HVQh7RrjIcWooJiTbTGWDWzxdR1UNmAM0BI5azVVFZCPwD6XU0QxPzmdKWu61kkBMzGr++98BHD68l06d+tCnz/AcufUKmqAguPfegpbCkF9ERkYC0KZNG3tbmTJlePfdd4mIiKBdu3a0a9eONm3aUKFChYIRspDjaVzqBHRp+DuUUrcqpW4F/m61Tcgr4bJDjRrQtm1BS2HIDRITr/D++8MZNKgVN24kM2nSct5666ciraQMJY+yZcumW+C8fv163nvvPYKDg5k8eTI9evSgYsWKfPXVV+lyKRo8j/q7ALRxLpAoIo2AZUqp8nkkn1uc56gMxYM//1zKO+8MISFhHz17Pstzz41zGZhQkjBzVMWTpKQkvvnmG/7zn/9w6tQp/Pz86NixIz169KBHjx4EBgbmyXWL7RyVC0xKJUOukpCwjw8/fJGVK3+levXbmTRpBY0atSlosQotnoSwGwo3fn5+PPPMMzz99NOsXbuWX375hV9++YX58+ezbNkypkyZUtAiFjieKqplwAQR6aOUOgwgIjWAj61jBkOOuHz5It9++xY//fQhPj6+DBv2Dn37/gs/PxNN4IhzwtuLFyEmBmyOkatXISAAnPPWGuVVcCxfvhyAdu3aZdrPy8uL5s2b07x5c95//31CQ0M5f/58fohY6PFUUT2PLkG/X0SOoqP+qgFbrGMGQ7ZISUnht9+m8MknIzhz5jgPPPAEw4a9Q6VKVQtatEJHRtGC/v5w993693PnYNeu3EuKa8g5tlL2WUFETP0tBzy6E5YV1UBEOgB3AQJsV0otzUvhDMWTv/46RVLSNU6eTGD8+BfYtm09deo05v3351K3rgmHywhPogVd9bHNbWWGKxeipxhrLXMaNGhQ0CIUebKkspVSS4AleSSLoZhz5swJfvjhXaZO/cDeVrHiLYwe/T333/+YSY6aR1y5ol2CzjWyHHF2IXqKsdYM+YGxLQ15zpkzJ5gy5T1mzvyM69ev2dv79x/BgAEjKVMmb6KaDKkkJrpfYOzoQvQUT6y1ko6tpEr79u0LWJKii1FUhjxDKcWvv37D++8P59q1q3Tu/ChPPfU6VarUJCkp0V4mw5C3lCkDIqmpmAz5iy3JsCH7GEVlyBMuXTrPW289zZIlM2jcuD2vvvoZNWumlqww0Xz5R40aejMUDGFhYQUtQpHHKCpDrqGUYsGCHzh+/BAREd9x/Pghnn32bfr1e9XMPxVjnEPmXeEu4CInwRxZvZah6JFlRSUiQTilXlJKGS91CefSpQuMHTuQpUtnAlClSk2+/DKK0NBmBSyZIS/xJMGuq4ALZ8WU3WAOT65V0CxZouPPOrhLl++CPXv2MHv2bJo1a8Ytt9yS26IVGTxNSlsTmAS0RRdPtB9Cr6ky5UJLMLt3x/Lqqw9z9OgBvL19uO++nowY8RmBgVlfP2IoWngSMm8r0SUDgwAAIABJREFUa+JuoXJiIrhZE+uWwhjccfPNN2frvPvuu48JEybQo0cPAG677TZat27NBx98kK21WUUZT3P9LQeCgPHo7OlpTlJKrcwT6TzA5PorOHSwxLe8994wypWrwNtvTyc8vKX7Ew0lCtsiZOcaj45RhocOwb59OU8o7WmtL2cuXtQ/3aXVy2+34rVr14iOjmbNmjUsWrSIJUuWsHDhQjp37pzjsYtSrj9PJw4aA08opX5SSkUqpVY6bnkpoDsuXTrPwYP6VS0lJYWoqAgOHdoD6HovUVERJCTsA+D69SSioiI4cuQAANeuJRIVFcGxYwcBna07KiqC48cPA3DlyiWioiI4efKIda0LREVFcPr0MQAuXjxHVFQEZ86cAOD8+bNERUXw11+nADh37gxRURGcO3cG0Atdo6IiOH9ev/KdOXOCqKgILl48B8Dp08eIiorg0iWdPfnkySNERUVw5colAI4fP0xUVASJiVcAOHbsIFFREVy7lgjAkSMHiIqK4Pr1JEDnzYuKirDXvTl0aA9RURGkpOgUjQcP7iYqKsJ+L+Pjd7J69QL7/v792/njj4X2/b1741i7djEAV69e5sUXH2Ls2IGEhjZn6tRoAgICWb8+NaPWzp2b2bBhuX1/+/aNbNwYad+Pi1vP5s2pPqOtW9cRE7Pavh8bu4bY2DX2/ZiY1Wzdus6+v3lzFHFx6+37GzdGsn176kvLhg3L2bkzNY/y+vXL2LUrxr6/bt0S9uzZYt9fu3Yxe/fG2ff/+GMh+/dvt++vXr2A+Pid9v2oqAjz7Hnw7AUFQZUqe7h4MYLQ0BTCw6FChd2cPp367KWk7MTX17NnD2DPni2sW5e6pHPXrhjWr19G+fI6wnHKlM188cVyoqK0W/KHHzby1VeR9v3vv1/P119H2fenTl3HlCmr7fvffruGb79dY9//5pvVfPnlOiwvHlFRUaxfn/rsRUZGsnHjRutewvffL2fOnM3s3q0tydmzlxETk/rsLVmyhC1bUp+9xYsXExeX+uwtXLiQ7du34+/vT5MmTbjzzjt58skn7ccjIiLYvTv12YuIiGDPntRnLyIign379LOXlJREREQEBw4csJ61RCIiUu99UcDTOaoDgAnTMgBaob3yysMcOLCdLl0eY/ToyXh7e3PqVKEpS2YoodhckTt3aivJFpLv768XPtv2fX0hKSl1//RpOHw4dd8W+xMaqn+KwKlT2q24e7fu6++vrwd6PyAAypXT1926VdfFO3MGoqMXc/ZsNK++mnMrqKTiqeuvHTACGKqU2pvnUmUB4/rLXxYvns5bbw3Cz68UY8dOpUmTjgUtksGQL2TkwnSFo1tz1ao4zp+H//ynbo6uv27dOpo2bVoiXX+eWlS/oi2qXSJyDUhTQ1kpVc6TQUSkMzrjujfwtVJqXAb9HgZmAvcopYwWKgQkJV3jww//j5kzPyM0tDnvvDODypWrFbRYBkO+kd3KzDVq1C10AR5FDU8V1bCcXkhEvIGJQAcgAdggIvOUUtud+gWiM7L/mdNrGnKHEycSePXVh4mL+5PHH3+JZ599Gx8fX/cnGgyGXGffvn0kJyeXqOzqnmZP/z4XrtUY2KuU2g8gItOBB4HtTv3eBN4FXsqFaxpyyMaNkYwc2YvExCv873+zaN++R0GLZDAUKTZvXmhFFXbJ0Tg333wz/v7+DBs2jNdff50OHTrQuXNnunXrRqVKlXJF1sKKxypZRPyBR4G70eHp24BpSqlrmZ6YSjXgsMN+ApDGkBaRcOBWpdR8ETGKqoBITr7O+PEvMGvW53h7e3PrrX/nyy9XEhx8V0GLZjAUOSpVqunRvJY7QkJCOHr0KEuXLmXRokUsWrSImTNnEh4ezubNm90PUITxdMHv3cAioBz8f3v3HmdTvT9+/PUelJRGkUuukUuSVH6lcpKkwpH6hlS6UTKH5EhxOBKKSsqlSZRU+nZXJ76REEcn4QyiOMcljEYzcpkZt2bMNO/fH2vLmOuay95rrz3v5+MxD2uvtfba7/nYM+9Za3/W+80PgdUPA2NE5BZV/Y+bw+Sx7o+ZHCISBbwMPOAinn5AP4Dq1a2IWWlKSdnP8OE9iYtbBkC7drcxevRsq3BuTDHVrducM88snWOde+659OzZk549e6Kq9OzZk7Vr15bOwcOY2zOqKcB64F5VPQQgImcD7wKTgZtdHCMBqJvtcR2cm4dPqAy0AJaLCEBNYJ6I3JpzQoWqzgRmgjPrz+X3YAqQlnaM+PitDB16GwcOJNG//zjq1WtMx449Cfx/GGPCiIhwxhlneB1GSLhNVNfizMA7dGKFqh4SkZHAqvyfdop/A41F5AJgD9ALuDvb8VKBP+75FpHlwFCb9Rd8c+a8yJQpTwBQo0Yd3njjXzRv7otZq8aEvbVrvwh8RtXF61B8y22iSsMpoZRTdGBboVQ1U0QGAotwpqe/qaqbRGQsEKeq81zGYkpJZmYGkyYN5uOPXwWgcuUqvPNOHFWrFq82mTEmtxo1GlGxotdR+JvbRDUfeF1EHubkGdTVwAzAdYJR1QXAghzrnspn3+vdHtcU3aFDyQwf3oM1a5Zy771D6dLlPho0aGbTzo0pZXXqNKNcudytUKwdiXtuE9VjwNvAN8DvgXVROElqcBDiMkG0a9cWhgzpSmJiPKNHz6Zr1we8DsmYiJVXK5RwbEcSztzeR5UCdBORxkAznBl8m8OtnJIp3Lp1Kxg69DbKlSvP9Olf06rVtV6HZExE27jRKQB71VVd/1gXju1IwlmRbm1W1W3AtiDFYoLsyy/fY8yYB6lTpxGTJ39B7doXeB2SMRGvfv2mXofge/kmKhGZCvxNVY8GlvOlqoNKPTJTalSVN98cz/Tpf+fyy9vx4oufcfbZ53gdljFlQv36TbwOwfcKOqO6hJPdfC8pYD+7jymMZWZmMGFCDJ9/PotOnXozatQbnHaadWwxJlRO9H+LinLb/s/klG+iUtX2eS0b/zhy5BDDhnVn9erFPPTQKB55ZIzdvGtMiJ1oRnrddV0L2dPkp9jld0XkQiBBVV3dR2VCKynpZwYP7sLOnf9h1KhZdOvWp/AnGWNKndXILDlX56IiMl5E7g8si4gsBrYCiSLSJpgBmqLbsuV7HnywDYmJ8UyZssCSlDEeqlevMfXqNQ7a8dPS0khJSQna8cOB24um9wBbAsudgFZAG+AdYEIQ4jLF9O23C3n44T8RFVWOWbP+RZs2Hb0OyZgyLTMzk8zMzFzr09Odm4BPfO3fX/RjN2jQgMTERGrWrEmPHj2YN28ex48fL4Wow4vbRFUDp6gsQGfgI1VdA0wDLgtGYKboPv10JkOGdKVu3ca89dYqLrywoDkwxphQWLlyIStXLjxlXXQ0iDg3Aa9YAYsXO19FNWbMGNasWUO/fv1Yvnw53bp14/zzz2fgwIGsXr0a1ciY6yZuvhER2QP0VNVvRWQrzrT1uSLSDFitqtHBDjQ/TZq01vffL9t1a7OysoiNHcHbbz/Ptdd2ZsKED6lU6SyvwzLGAAkJPwFQp06jfPeJj4fEROiY4wJIUcosZWRksGjRIubMmcO8efNIS0ujcePG9O7dm/vuu48GDRqcsr+IrFVVX1SfdpuopuJ0492KcwZVP3B/VS/gCVW9Irhh5q+sJ6r09DSefvoBFi/+kDvu6M8TT0wrUy2qjYkEKSmwZQunNFgsSZml1NRU5s6dy5w5c1i+fDlnnnkmycnJVKhwspannxKV20t/Q4CpOG3jO6rq0cD6WsD0YARmCpeScoABAzqyePGHDBr0AsOHv2pJypgwk5FxnIyMgj83qlIFrroKLrvs5FetWnD0aIFPy1d0dDR9+vRh2bJljBs3jqNHj5KRkVG8g4UBt7X+MoFJeax/udQjMq4kJf3Mo4/ezJ49O5gw4UM6duzpdUjGmDx8990iwLv7qE7LfprmUwWVULoc+F5VswLL+VLVdaUemcnXrl3/ZcCAmzhyJJVp0xZxxRXtvA7JGJOPRo1aeB2C7xV0RhWH0w7+18Cy4lRNz0lxGiGaENi06d8MGtSJcuXKMWPGcpo1s0mXxoQzK/5ccgUlqguAfdmWjcdWr17C0KG3ce651Xnlla+oW/dCr0MyxhQiPd0p3nP66dbmt7gKqvUXn9ey8caSJZ8watQ91K/flFdeWUS1arW8DskY48Lq1c4NUlbrr/hcTaYQkYFAiqq+m2N9b+BsVX01GMEZx9y5M3juuRhatryGl1+eby06jPGRxo1beh2C77mdyzwY6JvH+l3AbMASVRCoKrNmPctrr42ibdsuPPfcR1SsWMnrsIwxRVCrVv1iP/dEmaXsinITcKRwm6jqAHld/ksIbDOlLCsri5dfHsL770+hc+d7eeqpWZQvX6HwJxpjwkpa2jGAIv+RGR0NSUlOiaUTSnITsJ+5TVRJOIVod+VYfzlQjFKKpiCZmZmMHduHBQvmcPfdf2Xw4Bet6ZoxPrVmzVKg6J9RnbgJOLv4eDh4sLQi8w+3ieo9YKqIHAWWB9a1ByYD/xuEuMqs48fTGTnybpYt+5SYmGfo02eENTs0xseaNGnldQi+5zZRjcaZor4I+D2wLgr4GBgVhLjKpLS0Yzz55B2sXPklQ4dOoVevQV6HZIwpoZo163r6+ieuxsTFxXHdddd5GktxubqepKoZqnoX0BS4G6c/VVNV7aWq/i0gFUaOHDnEoEGd+O67RYwaNcuSlDER4tixIxw7dsSz1+/evTsNGzakQ4cOTJ061ZetP4r0wYeqbgNWAJ+o6vbghFT2pKYeZMCAjmzYsJJnnnnPOvIaE0Hi4pYRF7fMs9dv0KABa9eupXPnzjz22GP06tWLw4cPexZPcbi9j6oC8CwQA5wBNAF2iMjzQLzdR1V8Jyqg79ixiRdemEu7drd6HZIxphQ1a1ZgqdQiyzll3c109SpVqvDZZ58xceJERowYwYYNG0o1pmBze0Y1GugK9AbSs61fAzxQyjGVGcnJ+4iJuYGdOzczadLnlqSMiUDVq9emevXapXKsknQGjoqKYtiwYSxZsoTk5ORSiSdU3E6muAvoo6r/FJGsbOt/xDm7MkV08OCvxMR0ICFhOy+9NJ82bToW/iRjjO8cOXIIgLPOOrvEx8o5Zb0409Xbt2/P+vXrqV27dJJnKLg9ozqfvG/4LY/7ZGcC9u9Pon//9iQk/MTkyV9YkjImgq1b90/Wrfun12Gc4vzzz/c6hCJxm2Q2AdeR+4bfnsDa0gwo0u3b9wv9+9/Ar78mMHXqQuslZUyEa97cF93ew5rbRDUGeFdE6uL0nuohIs1wpqp3CVZwkWbv3gRiYm5g//5Epk5dyGWX/cnrkIwxQWadDkrO7X1U83HOnm4CsnAmVzQGuqrqkuCFFzmSknbTr187DhxIYtq0RZakjCkjDh9O4fDhFK/D8LVCz6hEpDxOglqtqnadqhh++WUX/fu3JzX1ILGxi2nR4qrCn2SMiQjr138DWD+qkig0Ualqpoh8CjQDDgQ/pMiyZ89OHnnkeo4ePcSrry7h4ov/n9chGWNC6OKLr/Q6BN9zO+tvA1DivucicouIbBGR7SIyPI/tQ0Rks4hsFJGlIlL8Ri5hIDExnv7923Ps2GGmT19qScqYMqhq1RpUrVrD6zB8zW2iehqYJCK3iUhdETk3+5ebA4hIOSAW6AQ0B+4SkeY5dlsPtFbVlsAnwAsu4ws7e/cm0L//DRw+nEJs7OJSvzvdGOMPqakHSU0tg705SpHbWX9fBP79FMhe0VACj8u5OMaVwHZV3QEgIh8A3YDNJ3ZQ1ewFsVbhVMLwnX37fiEm5gZSUvYTG7uYiy66wuuQjDEe2bDhW8A+oyoJt4nqBk5NUMVRG/g52+MEoKBZBX2BhXltEJF+QD+A6tXrlTCs0nXgwF5iYjqwf38ir7zyFS1a2PVpY8qySy5p43UIvucqUanq8lJ4rby6/+WZ/ESkN9AayHOWoarOBGYCNGnSOmxq1qek7CcmpgNJSbuZNu1LWra82uuQjDEeO+ec84J6/JxFavPipnBtOCswUYlIJWAicBtQAVgCDFLV4rSfTwCydxCrA/ySx2veCIwE2qlqes7t4erIkVQeffQW9uz5iSlTFth9UsYYwOmQAFClStVSP3Z0NCQlOQVq85Oe7iSqu+4q9ZcPmcLOqMbgVEf/XyANpzjtdKBHMV7r30BjEbkA2AP0wqls8QcRuQyYAdyiqr8W4zU8kZZ2jMGD/8zWrRuYNOlzWrdu73VIxpgwsXHjSiA4n1HlLFKbl+IUrg03hSWq/wH6quoHACLyLvCtiJRT1d8LfuqpAvdjDcRpZ18OeFNVN4nIWCBOVefhnL2dBXwsIgC7VTWse18cP57O0KG3s3HjSsaP/4C2bTt7HZIxJoy0bHmN1yH4XmGJqi7wzYkHqrpGRDJxqqn/nO+z8qGqC4AFOdY9lW35xqIe00uZmZmMHHk3q1Z9xahRs7jxxuKcaBpjIlkwLvmVNYXdR1UOOJ5jXSbW2oOsrCzGjevLsmWf8vjjk619vDEmT8nJ+0hO3ud1GL5WWMIRnKrp2Sc1VAReF5FjJ1aE++W50qaqTJw4iC++eIf+/cdx112PeR2SMSZM/fDDKsDuoyqJwhLV23msezcYgfjJq6+O5OOPY7n33ifo23ek1+EYY8LYpZde63UIvldgolLVB0MViF/Mnj2B2bMncMcd/Rk06HkCkz6MMSZP0dGuqsyZArit9WeAjz6KJTZ2BJ063cOwYbGWpIwxhTpwYC8HDuz1Ogxfs0Tl0pdfvscLLwykXbtujB49m6goGzpjTOE2bVrDpk1rvA7D18r87D03Vq78ktGj7+eKK65n/PgPKF++gtchGWN8Ihyq1LgpsxTOLFEV4scfV/Pkk3fQqFELJk36nNNPr+h1SMYYH6lcuYqnr59/maXKZ3oRT3FYoirAzp3/4bHHulC1ak2mTl3IWWed7XVIxhif2b8/EYBq1Wp58vr5l1kq56Y9U1iwD1rysWTJJ/To0Zzy5csTG/sV1arV9DokY4wPbd4cx+bNcV6H4Wt2RpWHH35YxfDhTjmkKVMWUqdOI48jMsb41eWX59mtyBSB7xNVpYQtNOl3/Snrkjv2ZF+PvyBpx2g8KHeR2ANdH+BA1wcol7KfRk92P2XbjvTfeHzXfwF464W53PrSX3M9f2/vx0m9riun79pC/fGP5Nqe2PfvHL7qRs7Y8j11Jw3OtX3PgPEcvfQaztywktqxI3Jt//nxyfzWtBWVVy+h1qxncm2PHzGD9AZNiV4xnxrvTsq1fefYOWTUrMs5X33IeZ9Mz7X9pxc+4fcq1ag6/y2qzn8r1/ZtUxegFStx3sevcs7ij3Jt3zpzOQA15rxI9Df/d8q2rIpnsH2q0++y5hvjOHvN0lO2Z0ZXZcfEuQCc/8rfOGvjd6dsP16jDrvGOfeU15k0mEpbvj9le1r9JuweOROAes/2o2L8qZ8QH2vaioTHJwPQYFRvTtubcMr2Iy2v5peBEwBo+MQdlE89cMr2Q1d2IOmhUQBcOKgTUWm/nbI99U9/Zu+9QwFyve+gZO89gH3dY0i+6U4qJP3MBU/dm2u7vfeWA/beC8Z7L5z5PlGVpoOZGXTf/gNZUeWYO3cLF55+htchGWN87pfjTgU6+4S7+EQ1bBrkFkuTJq31/fdLfv33+PF0BgzoyI8/rmb69KW0atW2FKIzxpR1K1bMB8Kv1l/r1udsU01u4nUcbtgZFU4l9DFjHmT9+m949tn3LUkZY0qNNVItOUtUwGuvPcWiRe8zcOAEbr65l9fhGGMiSKVKZ3kdgu+V+enp//jHLN5881luv/1h7r9/mNfhGGMiTFLSzyQlFbnPrMmmTJ9RrVr1FRMmPMLVV99sRWaNMUGxdasze7BmzboeR+JfZTZRbd/+A8OGdadhw4uZMOEjq99njAmKK6/s4HUIvlcmE9W+fb/w2GOdqVSpMpMnf2GlkYwxQVOxYiWvQ/C9MpeofvvtKIMH/5nDh1N4/fVvqFGjjtchGWMiWGJiPAC1atX3OBL/KlOJKisri6effoBt2zbw0kvzadq0ldchGWMi3LZtGwFLVCVRphLVrFnPsHTpJwwe/CJt2+YuMWKMMaXtqqs6eh2C75WZRLV06VxmzBhNly73cc89Q7wOxxhTRlgPu5IrE/dRbdnyPaNH38cll7RhxIgZNg3dGBMye/bsZM+enV6H4WsRf0Z18OCvPP54N6Kjz2XixM/srxtjTEj99NOPANSufYHHkfhXRCeqjIzjPPHE/5CcvI833vjGmh8aY0Lu6qtv9joE34vYRKWqPPfcX9iw4VvGj/+Aiy66wuuQjDFlUIUKp3kdgu9F7GdUH3wwlc8/n0WfPiO56aY7vQ7HGFNGJST8RELCT16H4WsReUa1atVXvPzyEK6//jb69x/rdTjGmDJsx47NANSp08jjSPwr4hJVfPxW/va3O2nY8GLGjp1DVFTEnjQaY3zgmms6eR2C70VUojp48FfuuKMpAO++u876wBhjPFe+fET9mvVExJxu/P7774wY4TQ9nDbtS5sKaowJC7t3b2P37m1eh+FrEZPqX3ttFHFxy3jqqTdtOqgxJmzs2vVfAOrVa+xxJP4V0jMqEblFRLaIyHYRGZ7H9tNF5MPA9tUi0sDNcZcv/wezZ0/g9tsf5tZbHyztsI0xptjatu1C27ZdvA7D10KWqESkHBALdAKaA3eJSPMcu/UFklX1QuBl4PnCjnv8eDqjR99P8+atGTp0ammHbYwxJRIVFWWTukoolKN3JbBdVXeo6nHgA6Bbjn26AW8Hlj8BOkghhfkSE7dTvnwFnn/+EyuPZIwJO/HxW4mP3+p1GL4WykRVG/g52+OEwLo891HVTCAVqFrQQY8fT+PZZ9+zXi/GmLAUH7+F+PgtXofha6GcTJHXmZEWYx9EpB/QL/AoY+DAO3eVMLYIkR4Np6d6HUV4sLE4ycbiJBuLkw75pr15KBNVAlA32+M6wC/57JMgIuWBaOBgzgOp6kxgJoCIxKkmtw5KxD7jjMUxGwtsLLKzsTjJxuIkEYnzOga3Qnnp799AYxG5QEROA3oB83LsMw+4P7DcHfhaVXOdURljjCk7QnZGpaqZIjIQWASUA95U1U0iMhaIU9V5wCxgjohsxzmT6hWq+IwxxoSnkN7wq6oLgAU51j2VbTkN6FHEw84shdAihY3FSTYWJ9lYnGRjcZJvxkLsypoxxphwZnehGWOMCWu+SVTBKr/kRy7GYoiIbBaRjSKyVEQi9iazwsYi237dRURFJGJnfLkZCxHpGXhvbBKR90IdY6i4+BmpJyLLRGR94OeksxdxBpuIvCkiv4rIj/lsFxGZGhinjSJyeahjdEVVw/4LZ/LFT0BD4DRgA9A8xz5/AV4LLPcCPvQ6bg/Hoj1QKbAcU5bHIrBfZWAFsApo7XXcHr4vGgPrgXMCj6t7HbeHYzETiAksNwd2eR13kMbiOuBy4Md8tncGFuLcw9oGWO11zHl9+eWMKijll3yq0LFQ1WWqeizwcBXOPWuRyM37AmAc8AKQFsrgQszNWDwMxKpqMoCq/hriGEPFzVgocHZgOZrc93RGBFVdQR73ombTDXhHHauAKiJSKzTRueeXRBWU8ks+5WYssuuL8xdTJCp0LETkMqCuqv5fKAPzgJv3RROgiYh8KyKrROSWkEUXWm7G4mmgt4gk4MxEfjQ0oYWdov4+8YRf+lGVWvmlCOD6+xSR3kBroF1QI/JOgWMhIlE4VfgfCFVAHnLzviiPc/nvepyz7G9EpIWqpgQ5tlBzMxZ3AW+p6iQRuRrn/s0WqpoV/PDCii9+b/rljKoo5ZcoqPxSBHAzFojIjcBI4FZVTQ9RbKFW2FhUBloAy0VkF841+HkROqHC7c/I56qaoao7gS04iSvSuBmLvsBHAKr6HVARqBaS6MKLq98nXvNLorLySycVOhaBy10zcJJUpH4OAYWMhaqmqmo1VW2gqg1wPq+7VVV9U+OsCNz8jPwDZ6INIlIN51LgjpBGGRpuxmI30AFARC7CSVT7QhpleJgH3BeY/dcGSFXVRK+DyskXl/7Uyi/9weVYTATOAj4OzCfZraq3ehZ0kLgcizLB5VgsAm4Skc3A78ATqnrAu6iDw+VYPA68LiJ/xbnU9UAk/mErIu/jXOqtFvg8bjRQAUBVX8P5fK4zsB04BoRli3SrTGGMMSas+eXSnzHGmDLKEpUxxpiwZonKGGNMWLNEZYwxJqxZojLGGBPWLFEZE4YCld675/fYmLLEEpUx2YjIW4GkoCKSKSK7RWS6iJzjdWzGlFWWqIzJbQlQC2gAPAR0BV71MiBjyjJLVMbklq6qSaqaoKpfAR8CN53YKCLRIjIz0JDusIj8M2f9QBFpIyJfi8hREUkNNLA8P7DtFhH5RkSSReSgiCwKlPExxuTBEpUxBRCRhsAtQEbgsQBf4LRC+DNwGU5Txq9P9PERkUuBZThlaa7FKYb7ESdLlp0JTMbpm3Q9Tkua+YG6dMaYHHxR68+YELtFRI7g1ImrGFg3JPBve6AVcJ6q/hZYN0pEugL34jRofBLYoKr9sh3zPycWVHVu9hcTkQeBQziJ61+l/L0Y43uWqIzJbQXQDzgDpytuI2BqYNsVQCVgX44G0hUD+4FzlvVZfgcXkUY4XYevAs7DubIRBdQrte/AmAhiicqY3I6p6vbA8iARWQaMwukKGwXsBf6Ux/MOBf7NqxlddvOBPcAjgX8zgc2AXfozJg+WqIwp3BhgoYjMBNYBNYAsVc2vl9M64Ia8NohIVeAiYICqLgusuxz7WTQmXzaZwphCqOpyYBPwd5yp698Cn4tIp0BzvqtFZIyInDjLmghcFpgZeKmINBWRh0SkHpAM7AceFpELRaQd8BrOWZUxJg+WqIxx5yWc9uX1cBrNfQ1RfV8PAAAAeklEQVS8jtPO/SOgKYEW3qr6PXAj0Aynq/BqnEaeGaqaBdwJtAR+BGJxLiumh/B7McZXrHGiMcaYsGZnVMYYY8KaJSpjjDFhzRKVMcaYsGaJyhhjTFizRGWMMSasWaIyxhgT1ixRGWOMCWuWqIwxxoQ1S1TGGGPC2v8HXwt+c6/qyigAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(y_testing,p_te[:, 1])\n",
    "no_skill = len(y_testing[y_testing==1]) / len(y_testing)\n",
    "f1 = 2*precision*recall/(precision+recall)\n",
    "m_idx = np.argmax(f1)\n",
    "m_thresh = thresholds[m_idx]\n",
    "print('Max F1: %0.5f, threshold prob: %0.5f' % (f1[m_idx], m_thresh))\n",
    "print('Max F1 Precision: %0.5f, Recall: %0.5f' % (precision[m_idx],recall[m_idx]))\n",
    "fig = plt.figure()\n",
    "plt.step(recall, precision, color='b', alpha=0.2, where='post')\n",
    "plt.fill_between(recall, precision, step='post', alpha=0.2, color='b',\\\n",
    "                 label='Random Forest')\n",
    "plt.plot([0, 1], [no_skill, no_skill], color='r', linestyle='--', label='No Skill')\n",
    "plt.plot([0, 1], [precision[m_idx],precision[m_idx]], color='k', alpha=0.4, linestyle=':')\n",
    "plt.plot([recall[m_idx],recall[m_idx]],[0, 1], color='k', alpha=0.4,linestyle=':',\\\n",
    "         label='Prec/Rec @ Max F1')\n",
    "plt.text(recall[m_idx], f1[m_idx]+0.01, 'Max F1={0:0.3f}'.format(f1[m_idx]))\n",
    "plt.plot(recall,f1,color='k',label='F1')\n",
    "plt.xlabel('Recall', fontsize=14)\n",
    "plt.ylabel('Precision or F1 Score', fontsize=14)\n",
    "plt.ylim([0.0, 1.0])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.legend()\n",
    "title=target+' Random Forest\\n Precision-Recall curve: AP={0:0.2f}'.format(\n",
    "          average_precision)\n",
    "plt.title(title, fontsize=16);\n",
    "fig.savefig('./reports/figures/'+target+'_RandomForest_PrecisionRecallCurve.svg',\\\n",
    "            format='svg', dpi=1200, transparent=True, bbox_inches = \"tight\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modified Random Forest Model Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-12-03T00:27:46.144849Z",
     "start_time": "2019-12-03T00:27:46.127779Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC_AUC: 0.90503\n",
      "[['TN' 'FP']\n",
      " ['FN' 'TP']]\n",
      "[[449  88]\n",
      " [  8  65]]\n",
      "f1:0.57522\n",
      "recall:0.89041\n",
      "precision:0.42484\n",
      "accuracy: 0.84262\n"
     ]
    }
   ],
   "source": [
    "y_hat_testing_adj=adjusted_classes(p_te[:, 1],m_thresh)\n",
    "print('ROC_AUC: {0:0.5f}'.format(roc_auc_score(y_testing, p_te[:, 1]+(0.5-m_thresh))))\n",
    "print(np.array([['TN','FP'],['FN','TP']]))\n",
    "print(confusion_matrix(y_testing,y_hat_testing_adj))\n",
    "print('f1:{0:0.5f}'.format(f1_score(y_testing,y_hat_testing_adj)))\n",
    "print('recall:{0:0.5f}'.format(recall_score(y_testing,y_hat_testing_adj)))\n",
    "print('precision:{0:0.5f}'.format(precision_score(y_testing,y_hat_testing_adj)))\n",
    "print('accuracy: {0:0.5f}'.format(accuracy_score(y_testing,y_hat_testing_adj)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This result gives nearly twice the TP's and higher recall at the expense of more FP's!  Recall could be improved even further with minimally more FP's by further tuning the prediction threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:metis] *",
   "language": "python",
   "name": "conda-env-metis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}