Random_Splits_Multitasking.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##MULTI TASK DEEP LEARNING NEURAL NETWORK\n",
    "\n",
    "This contains a lot of the code from the other multitask file, but just what is required to run the multitask neural net with random splits multiple times.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import torch\n",
    "from bayes_opt import BayesianOptimization\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from math import sqrt\n",
    "import random\n",
    "import itertools\n",
    "\n",
    "#Load some of the data\n",
    "exp_data = pd.read_csv('../exp.tab', sep='\\t', index_col=0)\n",
    "cnv_data = pd.read_csv('../cnv.tab', sep='\\t', index_col=0)\n",
    "ydat = pd.read_csv('../labels.tab', sep='\\t', index_col=0)\n",
    "train_activity_data = pd.read_csv('../train_activity.tab', sep='\\t')\n",
    "test_activity_data = pd.read_csv('../test_activity.tab', sep ='\\t')\n",
    "\n",
    "#best ~1000 tasks\n",
    "top_tasks = pd.read_csv(\"../combined_stats.tab\", sep='\\t')\n",
    "tasks = top_tasks.iloc[:,0].values\n",
    "ydat_best = ydat.transpose()[tasks]\n",
    "ydat_best = ydat_best.transpose()\n",
    "\n",
    "#concatenate two data frames\n",
    "frames = [exp_data, cnv_data]\n",
    "\n",
    "xdatw = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Deep Learning Net Class\n",
    "\n",
    "class EssentialityNet:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.inputnum = xdatw.shape[0]\n",
    "        self.trainscores = []\n",
    "        self.testscoreslist = []\n",
    "        self.learning_rate = 0.00009\n",
    "        self.H = 100\n",
    "        self.n_iter = 300 #training iterations\n",
    "        self.minimum = 100000\n",
    "        self.stopcounter = 3\n",
    "        self.layernum = 1\n",
    "        self.layers = []\n",
    "                \n",
    "        #model\n",
    "        self.model = torch.nn.Sequential(\n",
    "        torch.nn.Linear(self.inputnum, self.H),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Linear(self.H, 1138),\n",
    "        )\n",
    "        \n",
    "        #set loss function and optimizer\n",
    "        self.loss = torch.nn.MSELoss()\n",
    "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)\n",
    "    \n",
    "    #plot scores\n",
    "    def plot(self, trainscores, testscores):\n",
    "        x = np.arange(self.n_iter)\n",
    "        plt.plot(x, self.trainscores, label='Train')\n",
    "        plt.title('Training vs Test Accuracy')\n",
    "        plt.xlabel('NN Training Iterations')\n",
    "        plt.ylabel('Accuracy')\n",
    "    \n",
    "        plt.plot(np.asarray(x), np.asarray(testscores), label='Test') #plot\n",
    "        plt.legend()\n",
    "        \n",
    "    #sets the proper method\n",
    "    def setModel(self, Layernum, Neuronnum):  \n",
    "        \n",
    "        self.layernum = int(round(Layernum))\n",
    "        self.H = int(round(Neuronnum))\n",
    "        \n",
    "        #initial input layer\n",
    "        self.layers.append(torch.nn.Linear(self.inputnum, self.H))\n",
    "        \n",
    "        for n in range(self.layernum):\n",
    "            if n != 0:\n",
    "                self.layers.append(torch.nn.Linear(self.H, self.H))\n",
    "            self.layers.append(torch.nn.ReLU())\n",
    "            \n",
    "        self.layers.append(torch.nn.Linear(self.H, 1138))\n",
    "        \n",
    "        #set the method to whatever layers were chosen\n",
    "        self.model = torch.nn.Sequential(*self.layers)\n",
    "    \n",
    "    def setRegularization(self, L2Reg):\n",
    "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate, weight_decay= L2Reg)\n",
    "\n",
    "    def fit(self, xtrain, ytrain, xtest, ytest):\n",
    "      \n",
    "        #convert to variables\n",
    "        xtrain_var = Variable(torch.FloatTensor(xtrain))\n",
    "        xtest_var = Variable(torch.FloatTensor(xtest))\n",
    "        ytrain_var = Variable(torch.FloatTensor(ytrain))\n",
    "        ytest_var = Variable(torch.FloatTensor(ytest))\n",
    "        \n",
    "        for t in range(self.n_iter):\n",
    "        \n",
    "            #calculate loss\n",
    "            ypred = self.model(xtrain_var)\n",
    "\n",
    "            diff = self.loss(ypred, ytrain_var)\n",
    "            self.trainscores.append(diff.data[0])\n",
    "            \n",
    "            #test performance\n",
    "            ypredtest = self.model(xtest_var)\n",
    "            difftest = self.loss(ypredtest, ytest_var)\n",
    "            \n",
    "            #find the best point\n",
    "            if t > 10 and self.minimum < difftest.data[0]:\n",
    "                self.stopcounter -= 1\n",
    "\n",
    "                if self.stopcounter == 0:\n",
    "                    self.n_iter = t\n",
    "                    self.trainscores.pop()\n",
    "                    break\n",
    "            elif t > 10 and self.stopcounter < 3:\n",
    "                self.stopcounter += 1\n",
    "            \n",
    "            self.minimum = difftest.data[0]\n",
    "            \n",
    "            self.testscoreslist.append(difftest.data[0])\n",
    "            \n",
    "            #zero gradients\n",
    "            self.optimizer.zero_grad()\n",
    "            #backpropagate\n",
    "            diff.backward() \n",
    "            #update weights\n",
    "            self.optimizer.step() \n",
    "\n",
    "    # predict with the test data\n",
    "    def predict(self, X):\n",
    "        \n",
    "        X_var = Variable(torch.FloatTensor(X))\n",
    "        return self.model(X_var) \n",
    "    \n",
    "#other functions for running the nn\n",
    "\n",
    "def figureoutnetwork(layernum, neuronnum, l2reg):\n",
    "    n = EssentialityNet()\n",
    "    n.setModel(layernum, neuronnum)\n",
    "    n.setRegularization(l2reg)\n",
    "            \n",
    "    n.fit(xtrain_val, ytrain_val, xtest_val, ytest_val)\n",
    "    predictions = n.predict(xtest)\n",
    "#     return(calculateRMSE(predictions, ytest))\n",
    "    saveRMSE(predictions, ytest)\n",
    "\n",
    "def figureoutnetwork3(neuronnum, l2reg):\n",
    "    n = EssentialityNet()\n",
    "    n.setModel(3, neuronnum)\n",
    "    n.setRegularization(l2reg)\n",
    "            \n",
    "    n.fit(xtrain_val, ytrain_val, xtest_val, ytest_val)\n",
    "    predictions = n.predict(xtest)\n",
    "    return(calculateRMSE(predictions, ytest))\n",
    "    \n",
    "#calculate RMSE function\n",
    "def calculateRMSE(predicts, actuals):\n",
    "    mses = []  \n",
    "    multitaskrmses = []\n",
    "    preds = predicts.data.numpy()\n",
    "\n",
    "    for i in range(preds.shape[1]):\n",
    "        mses.append(((preds[:,i] - actuals[:,i])**2).mean())\n",
    "        multitaskrmses.append(sqrt(mses[i]))\n",
    "\n",
    "    print(len(multitaskrmses))       \n",
    "    return(np.mean(multitaskrmses))\n",
    "\n",
    "def saveRMSE(predicts, actuals):\n",
    "    mses = []  \n",
    "    multitaskrmses = []\n",
    "    preds = predicts.data.numpy()\n",
    "\n",
    "    for i in range(preds.shape[1]):\n",
    "        mses.append(((preds[:,i] - actuals[:,i])**2).mean())\n",
    "        multitaskrmses.append(sqrt(mses[i]))\n",
    "    \n",
    "    #open a file for saving rmses\n",
    "    rmses_file = open('rmses_' + str(fileno) + \".tab\", 'w')\n",
    "    \n",
    "    for item in multitaskrmses:\n",
    "          rmses_file.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for fileno in range(10):\n",
    "\n",
    "    #starting runs with random splits:\n",
    "    traininglabels = random.sample(range(0, 206), 142)\n",
    "    traininglabels.sort()\n",
    "    testinglabels = random.sample([x for x in range(206) if x not in traininglabels], 64)\n",
    "\n",
    "    #index the data with the proper labels\n",
    "    xtrain_not_norm = xdatw.iloc[:,traininglabels].transpose()\n",
    "    xtest_not_norm = xdatw.iloc[:,testinglabels].transpose()\n",
    "    ytrain = ydat_best.iloc[:,traininglabels].transpose().values\n",
    "    ytest = ydat_best.iloc[:,testinglabels].transpose().values\n",
    "\n",
    "    #normalize inputs\n",
    "    xtrain = preprocessing.scale(xtrain_not_norm)\n",
    "    xtest = preprocessing.scale(xtest_not_norm)\n",
    "\n",
    "    #create validation set\n",
    "    xtrain_val, xtest_val, ytrain_val, ytest_val = train_test_split(xtrain, ytrain, test_size=0.2, random_state=434)\n",
    "\n",
    "    #do not uncomment this if you do not want to retrain\n",
    "###     figureoutnetwork(3, 356, 0.012)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#open and concatenate the files\n",
    "myrmses0 = pd.read_csv('rmses_0.tab', header=None)\n",
    "myrmses1 = pd.read_csv('rmses_1.tab', header=None)\n",
    "myrmses2 = pd.read_csv('rmses_2.tab', header=None)\n",
    "myrmses3 = pd.read_csv('rmses_3.tab', header=None)\n",
    "myrmses4 = pd.read_csv('rmses_4.tab', header=None)\n",
    "myrmses5 = pd.read_csv('rmses_5.tab', header=None)\n",
    "myrmses6 = pd.read_csv('rmses_6.tab', header=None)\n",
    "myrmses7 = pd.read_csv('rmses_7.tab', header=None)\n",
    "myrmses8 = pd.read_csv('rmses_8.tab', header=None)\n",
    "myrmses9 = pd.read_csv('rmses_9.tab', header=None)\n",
    "\n",
    "rmses_total = pd.concat((myrmses0, myrmses1, myrmses2, myrmses3, myrmses4, myrmses5, myrmses6, myrmses7, myrmses8, myrmses9), axis = 1, )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.01065570239\n"
     ]
    }
   ],
   "source": [
    "#get means across trials\n",
    "rmse_means = pd.DataFrame.mean(rmses_total, axis=1)\n",
    "dnnlist = (list(rmse_means.values))\n",
    "print(np.mean(dnnlist))\n",
    "#save to file\n",
    "mean_rmses_file = open('mean_rmses.tab', 'w')\n",
    "\n",
    "# for item in list(rmse_means.values):\n",
    "#     mean_rmses_file.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>jklm.rmse</th>\n",
       "      <th>ranger.rmse</th>\n",
       "      <th>mkl.d9.rmse</th>\n",
       "      <th>rf.d9.rmse</th>\n",
       "      <th>glmm.dense.rmse</th>\n",
       "      <th>glmm.sparse.rmse</th>\n",
       "      <th>dnn.rmse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.952833</td>\n",
       "      <td>0.960496</td>\n",
       "      <td>0.948655</td>\n",
       "      <td>0.961768</td>\n",
       "      <td>0.962843</td>\n",
       "      <td>0.962658</td>\n",
       "      <td>0.882877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.060365</td>\n",
       "      <td>1.074067</td>\n",
       "      <td>1.058601</td>\n",
       "      <td>1.067794</td>\n",
       "      <td>1.032906</td>\n",
       "      <td>1.124052</td>\n",
       "      <td>1.007232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.946342</td>\n",
       "      <td>0.931658</td>\n",
       "      <td>0.975606</td>\n",
       "      <td>0.941134</td>\n",
       "      <td>0.934963</td>\n",
       "      <td>0.973344</td>\n",
       "      <td>0.881774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.074409</td>\n",
       "      <td>1.053793</td>\n",
       "      <td>1.068510</td>\n",
       "      <td>1.056220</td>\n",
       "      <td>1.171792</td>\n",
       "      <td>1.019199</td>\n",
       "      <td>1.017913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.298735</td>\n",
       "      <td>1.284004</td>\n",
       "      <td>1.276571</td>\n",
       "      <td>1.346348</td>\n",
       "      <td>1.254156</td>\n",
       "      <td>1.316945</td>\n",
       "      <td>1.292043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.019320</td>\n",
       "      <td>1.018533</td>\n",
       "      <td>1.026431</td>\n",
       "      <td>1.039957</td>\n",
       "      <td>1.031455</td>\n",
       "      <td>1.032844</td>\n",
       "      <td>0.968281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.903709</td>\n",
       "      <td>0.906174</td>\n",
       "      <td>0.892708</td>\n",
       "      <td>0.901269</td>\n",
       "      <td>0.897027</td>\n",
       "      <td>0.939720</td>\n",
       "      <td>0.954082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.987778</td>\n",
       "      <td>0.990707</td>\n",
       "      <td>1.014963</td>\n",
       "      <td>1.050599</td>\n",
       "      <td>1.015290</td>\n",
       "      <td>1.056278</td>\n",
       "      <td>0.916968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.962102</td>\n",
       "      <td>0.944543</td>\n",
       "      <td>0.925711</td>\n",
       "      <td>0.934242</td>\n",
       "      <td>0.961486</td>\n",
       "      <td>0.963235</td>\n",
       "      <td>0.937306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.025180</td>\n",
       "      <td>1.034207</td>\n",
       "      <td>1.028851</td>\n",
       "      <td>1.061582</td>\n",
       "      <td>1.021382</td>\n",
       "      <td>0.997408</td>\n",
       "      <td>0.945645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.864839</td>\n",
       "      <td>0.879444</td>\n",
       "      <td>0.866002</td>\n",
       "      <td>0.860268</td>\n",
       "      <td>0.882059</td>\n",
       "      <td>0.903176</td>\n",
       "      <td>0.877987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.026121</td>\n",
       "      <td>1.016678</td>\n",
       "      <td>1.006676</td>\n",
       "      <td>1.025923</td>\n",
       "      <td>1.029813</td>\n",
       "      <td>1.054445</td>\n",
       "      <td>0.974374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.066715</td>\n",
       "      <td>1.078802</td>\n",
       "      <td>1.086855</td>\n",
       "      <td>1.074777</td>\n",
       "      <td>1.098704</td>\n",
       "      <td>1.082767</td>\n",
       "      <td>0.962467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.912657</td>\n",
       "      <td>0.927705</td>\n",
       "      <td>0.919456</td>\n",
       "      <td>0.933019</td>\n",
       "      <td>0.914373</td>\n",
       "      <td>0.892304</td>\n",
       "      <td>1.049915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.960091</td>\n",
       "      <td>0.955081</td>\n",
       "      <td>0.952893</td>\n",
       "      <td>0.968033</td>\n",
       "      <td>0.971832</td>\n",
       "      <td>0.971949</td>\n",
       "      <td>0.898844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.014610</td>\n",
       "      <td>1.005765</td>\n",
       "      <td>0.995154</td>\n",
       "      <td>1.026589</td>\n",
       "      <td>1.030348</td>\n",
       "      <td>1.007847</td>\n",
       "      <td>1.027795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.936305</td>\n",
       "      <td>0.960090</td>\n",
       "      <td>0.969868</td>\n",
       "      <td>0.978125</td>\n",
       "      <td>0.932842</td>\n",
       "      <td>0.953257</td>\n",
       "      <td>0.849989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.968715</td>\n",
       "      <td>0.978110</td>\n",
       "      <td>0.968780</td>\n",
       "      <td>1.003761</td>\n",
       "      <td>0.963905</td>\n",
       "      <td>1.009469</td>\n",
       "      <td>0.984028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1.026594</td>\n",
       "      <td>0.972322</td>\n",
       "      <td>0.963828</td>\n",
       "      <td>0.990733</td>\n",
       "      <td>0.992096</td>\n",
       "      <td>0.951621</td>\n",
       "      <td>0.983280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.906035</td>\n",
       "      <td>0.934353</td>\n",
       "      <td>0.924058</td>\n",
       "      <td>0.931046</td>\n",
       "      <td>0.944596</td>\n",
       "      <td>0.927455</td>\n",
       "      <td>0.931251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1.153014</td>\n",
       "      <td>1.157117</td>\n",
       "      <td>1.152325</td>\n",
       "      <td>1.167979</td>\n",
       "      <td>1.161337</td>\n",
       "      <td>1.186105</td>\n",
       "      <td>1.182163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.018916</td>\n",
       "      <td>1.042071</td>\n",
       "      <td>1.102796</td>\n",
       "      <td>1.184638</td>\n",
       "      <td>1.059646</td>\n",
       "      <td>1.030637</td>\n",
       "      <td>1.055953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.998337</td>\n",
       "      <td>1.010587</td>\n",
       "      <td>1.002333</td>\n",
       "      <td>1.036592</td>\n",
       "      <td>1.013084</td>\n",
       "      <td>1.065147</td>\n",
       "      <td>1.040864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.970767</td>\n",
       "      <td>0.990813</td>\n",
       "      <td>0.993429</td>\n",
       "      <td>0.999777</td>\n",
       "      <td>1.022317</td>\n",
       "      <td>0.960106</td>\n",
       "      <td>0.912818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.898563</td>\n",
       "      <td>0.910198</td>\n",
       "      <td>0.928346</td>\n",
       "      <td>0.931359</td>\n",
       "      <td>0.971286</td>\n",
       "      <td>0.971286</td>\n",
       "      <td>0.899371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1.101058</td>\n",
       "      <td>1.122674</td>\n",
       "      <td>1.122188</td>\n",
       "      <td>1.147670</td>\n",
       "      <td>1.131612</td>\n",
       "      <td>1.133874</td>\n",
       "      <td>1.182630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.905662</td>\n",
       "      <td>0.927240</td>\n",
       "      <td>0.929007</td>\n",
       "      <td>0.974975</td>\n",
       "      <td>0.926293</td>\n",
       "      <td>0.924893</td>\n",
       "      <td>0.931853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.076554</td>\n",
       "      <td>1.078623</td>\n",
       "      <td>1.086000</td>\n",
       "      <td>1.107094</td>\n",
       "      <td>1.092432</td>\n",
       "      <td>1.080861</td>\n",
       "      <td>0.975462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.911745</td>\n",
       "      <td>0.899485</td>\n",
       "      <td>0.933249</td>\n",
       "      <td>0.954863</td>\n",
       "      <td>0.944321</td>\n",
       "      <td>0.937255</td>\n",
       "      <td>0.887756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1.278873</td>\n",
       "      <td>1.300430</td>\n",
       "      <td>1.264563</td>\n",
       "      <td>1.287667</td>\n",
       "      <td>1.264107</td>\n",
       "      <td>1.288997</td>\n",
       "      <td>1.319286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1108</th>\n",
       "      <td>0.824665</td>\n",
       "      <td>0.818785</td>\n",
       "      <td>0.828190</td>\n",
       "      <td>0.857140</td>\n",
       "      <td>0.851707</td>\n",
       "      <td>0.826038</td>\n",
       "      <td>0.893217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1109</th>\n",
       "      <td>0.877833</td>\n",
       "      <td>0.871966</td>\n",
       "      <td>0.900916</td>\n",
       "      <td>0.908297</td>\n",
       "      <td>0.917863</td>\n",
       "      <td>0.917566</td>\n",
       "      <td>0.888731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1110</th>\n",
       "      <td>0.913307</td>\n",
       "      <td>0.895742</td>\n",
       "      <td>0.907654</td>\n",
       "      <td>0.924261</td>\n",
       "      <td>0.911022</td>\n",
       "      <td>0.939010</td>\n",
       "      <td>0.889690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1111</th>\n",
       "      <td>0.890177</td>\n",
       "      <td>0.909035</td>\n",
       "      <td>0.920792</td>\n",
       "      <td>0.966972</td>\n",
       "      <td>0.954292</td>\n",
       "      <td>0.997018</td>\n",
       "      <td>0.816057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112</th>\n",
       "      <td>0.869413</td>\n",
       "      <td>0.862565</td>\n",
       "      <td>0.894360</td>\n",
       "      <td>0.912791</td>\n",
       "      <td>0.899760</td>\n",
       "      <td>0.890375</td>\n",
       "      <td>0.889486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1113</th>\n",
       "      <td>0.910647</td>\n",
       "      <td>0.934896</td>\n",
       "      <td>0.941350</td>\n",
       "      <td>0.945469</td>\n",
       "      <td>0.970018</td>\n",
       "      <td>0.981674</td>\n",
       "      <td>0.996912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1114</th>\n",
       "      <td>0.941791</td>\n",
       "      <td>0.946309</td>\n",
       "      <td>0.974123</td>\n",
       "      <td>1.014387</td>\n",
       "      <td>0.990833</td>\n",
       "      <td>0.994785</td>\n",
       "      <td>0.913325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1115</th>\n",
       "      <td>0.880239</td>\n",
       "      <td>0.892297</td>\n",
       "      <td>0.877591</td>\n",
       "      <td>0.911228</td>\n",
       "      <td>0.886479</td>\n",
       "      <td>0.896507</td>\n",
       "      <td>0.880617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1116</th>\n",
       "      <td>0.935164</td>\n",
       "      <td>0.938772</td>\n",
       "      <td>0.969023</td>\n",
       "      <td>0.990525</td>\n",
       "      <td>0.973272</td>\n",
       "      <td>0.968305</td>\n",
       "      <td>0.936234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1117</th>\n",
       "      <td>0.804401</td>\n",
       "      <td>0.796536</td>\n",
       "      <td>0.812495</td>\n",
       "      <td>0.803534</td>\n",
       "      <td>0.808039</td>\n",
       "      <td>0.779075</td>\n",
       "      <td>0.891616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1118</th>\n",
       "      <td>1.003451</td>\n",
       "      <td>1.005641</td>\n",
       "      <td>0.971408</td>\n",
       "      <td>0.997005</td>\n",
       "      <td>0.995884</td>\n",
       "      <td>0.999130</td>\n",
       "      <td>1.016141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1119</th>\n",
       "      <td>1.109512</td>\n",
       "      <td>1.113757</td>\n",
       "      <td>1.089068</td>\n",
       "      <td>1.113132</td>\n",
       "      <td>1.180525</td>\n",
       "      <td>1.114830</td>\n",
       "      <td>1.049505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1120</th>\n",
       "      <td>0.853283</td>\n",
       "      <td>0.842644</td>\n",
       "      <td>0.857619</td>\n",
       "      <td>0.882108</td>\n",
       "      <td>0.883380</td>\n",
       "      <td>0.876703</td>\n",
       "      <td>0.896816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1121</th>\n",
       "      <td>1.053327</td>\n",
       "      <td>1.055018</td>\n",
       "      <td>1.047172</td>\n",
       "      <td>1.058848</td>\n",
       "      <td>1.046278</td>\n",
       "      <td>1.070476</td>\n",
       "      <td>1.042978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1122</th>\n",
       "      <td>0.839807</td>\n",
       "      <td>0.825289</td>\n",
       "      <td>0.864418</td>\n",
       "      <td>0.882356</td>\n",
       "      <td>0.866215</td>\n",
       "      <td>0.843092</td>\n",
       "      <td>0.890023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1123</th>\n",
       "      <td>0.810731</td>\n",
       "      <td>0.793734</td>\n",
       "      <td>0.781794</td>\n",
       "      <td>0.801653</td>\n",
       "      <td>0.816091</td>\n",
       "      <td>0.794254</td>\n",
       "      <td>0.912470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1124</th>\n",
       "      <td>1.016015</td>\n",
       "      <td>1.016341</td>\n",
       "      <td>1.012411</td>\n",
       "      <td>1.026898</td>\n",
       "      <td>1.007053</td>\n",
       "      <td>1.006702</td>\n",
       "      <td>1.053143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1125</th>\n",
       "      <td>1.006789</td>\n",
       "      <td>1.009246</td>\n",
       "      <td>0.987176</td>\n",
       "      <td>1.019357</td>\n",
       "      <td>1.035754</td>\n",
       "      <td>1.017252</td>\n",
       "      <td>1.012933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1126</th>\n",
       "      <td>0.900048</td>\n",
       "      <td>0.887136</td>\n",
       "      <td>0.923358</td>\n",
       "      <td>0.924773</td>\n",
       "      <td>0.995944</td>\n",
       "      <td>0.906718</td>\n",
       "      <td>0.887253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1127</th>\n",
       "      <td>0.866667</td>\n",
       "      <td>0.873464</td>\n",
       "      <td>0.842159</td>\n",
       "      <td>0.861137</td>\n",
       "      <td>0.864173</td>\n",
       "      <td>1.004010</td>\n",
       "      <td>0.921179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128</th>\n",
       "      <td>1.124805</td>\n",
       "      <td>1.139015</td>\n",
       "      <td>1.148699</td>\n",
       "      <td>1.161558</td>\n",
       "      <td>1.185100</td>\n",
       "      <td>1.161221</td>\n",
       "      <td>1.026753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1129</th>\n",
       "      <td>0.781559</td>\n",
       "      <td>0.798718</td>\n",
       "      <td>0.808136</td>\n",
       "      <td>0.842453</td>\n",
       "      <td>0.816329</td>\n",
       "      <td>0.814878</td>\n",
       "      <td>0.815916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1130</th>\n",
       "      <td>0.923668</td>\n",
       "      <td>0.927178</td>\n",
       "      <td>0.956415</td>\n",
       "      <td>0.954501</td>\n",
       "      <td>0.976227</td>\n",
       "      <td>0.994896</td>\n",
       "      <td>0.909458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1131</th>\n",
       "      <td>0.887437</td>\n",
       "      <td>0.885006</td>\n",
       "      <td>0.895667</td>\n",
       "      <td>0.902937</td>\n",
       "      <td>0.895119</td>\n",
       "      <td>0.903921</td>\n",
       "      <td>0.949747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1132</th>\n",
       "      <td>0.868581</td>\n",
       "      <td>0.863576</td>\n",
       "      <td>0.876061</td>\n",
       "      <td>0.867810</td>\n",
       "      <td>0.904657</td>\n",
       "      <td>0.911519</td>\n",
       "      <td>1.015141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1133</th>\n",
       "      <td>1.269603</td>\n",
       "      <td>1.269768</td>\n",
       "      <td>1.266867</td>\n",
       "      <td>1.300112</td>\n",
       "      <td>1.267734</td>\n",
       "      <td>1.304537</td>\n",
       "      <td>1.370521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134</th>\n",
       "      <td>0.859855</td>\n",
       "      <td>0.840592</td>\n",
       "      <td>0.885673</td>\n",
       "      <td>0.909632</td>\n",
       "      <td>0.899399</td>\n",
       "      <td>0.879226</td>\n",
       "      <td>0.896755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1135</th>\n",
       "      <td>0.822099</td>\n",
       "      <td>0.806193</td>\n",
       "      <td>0.823133</td>\n",
       "      <td>0.826042</td>\n",
       "      <td>0.821425</td>\n",
       "      <td>0.919244</td>\n",
       "      <td>0.888820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1136</th>\n",
       "      <td>0.935305</td>\n",
       "      <td>0.925420</td>\n",
       "      <td>0.940226</td>\n",
       "      <td>0.955089</td>\n",
       "      <td>0.946809</td>\n",
       "      <td>1.058379</td>\n",
       "      <td>0.981741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1137</th>\n",
       "      <td>0.948006</td>\n",
       "      <td>0.957404</td>\n",
       "      <td>0.961448</td>\n",
       "      <td>0.966228</td>\n",
       "      <td>0.941657</td>\n",
       "      <td>0.945153</td>\n",
       "      <td>0.905251</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1138 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      jklm.rmse  ranger.rmse  mkl.d9.rmse  rf.d9.rmse  glmm.dense.rmse  \\\n",
       "0      0.952833     0.960496     0.948655    0.961768         0.962843   \n",
       "1      1.060365     1.074067     1.058601    1.067794         1.032906   \n",
       "2      0.946342     0.931658     0.975606    0.941134         0.934963   \n",
       "3      1.074409     1.053793     1.068510    1.056220         1.171792   \n",
       "4      1.298735     1.284004     1.276571    1.346348         1.254156   \n",
       "5      1.019320     1.018533     1.026431    1.039957         1.031455   \n",
       "6      0.903709     0.906174     0.892708    0.901269         0.897027   \n",
       "7      0.987778     0.990707     1.014963    1.050599         1.015290   \n",
       "8      0.962102     0.944543     0.925711    0.934242         0.961486   \n",
       "9      1.025180     1.034207     1.028851    1.061582         1.021382   \n",
       "10     0.864839     0.879444     0.866002    0.860268         0.882059   \n",
       "11     1.026121     1.016678     1.006676    1.025923         1.029813   \n",
       "12     1.066715     1.078802     1.086855    1.074777         1.098704   \n",
       "13     0.912657     0.927705     0.919456    0.933019         0.914373   \n",
       "14     0.960091     0.955081     0.952893    0.968033         0.971832   \n",
       "15     1.014610     1.005765     0.995154    1.026589         1.030348   \n",
       "16     0.936305     0.960090     0.969868    0.978125         0.932842   \n",
       "17     0.968715     0.978110     0.968780    1.003761         0.963905   \n",
       "18     1.026594     0.972322     0.963828    0.990733         0.992096   \n",
       "19     0.906035     0.934353     0.924058    0.931046         0.944596   \n",
       "20     1.153014     1.157117     1.152325    1.167979         1.161337   \n",
       "21     1.018916     1.042071     1.102796    1.184638         1.059646   \n",
       "22     0.998337     1.010587     1.002333    1.036592         1.013084   \n",
       "23     0.970767     0.990813     0.993429    0.999777         1.022317   \n",
       "24     0.898563     0.910198     0.928346    0.931359         0.971286   \n",
       "25     1.101058     1.122674     1.122188    1.147670         1.131612   \n",
       "26     0.905662     0.927240     0.929007    0.974975         0.926293   \n",
       "27     1.076554     1.078623     1.086000    1.107094         1.092432   \n",
       "28     0.911745     0.899485     0.933249    0.954863         0.944321   \n",
       "29     1.278873     1.300430     1.264563    1.287667         1.264107   \n",
       "...         ...          ...          ...         ...              ...   \n",
       "1108   0.824665     0.818785     0.828190    0.857140         0.851707   \n",
       "1109   0.877833     0.871966     0.900916    0.908297         0.917863   \n",
       "1110   0.913307     0.895742     0.907654    0.924261         0.911022   \n",
       "1111   0.890177     0.909035     0.920792    0.966972         0.954292   \n",
       "1112   0.869413     0.862565     0.894360    0.912791         0.899760   \n",
       "1113   0.910647     0.934896     0.941350    0.945469         0.970018   \n",
       "1114   0.941791     0.946309     0.974123    1.014387         0.990833   \n",
       "1115   0.880239     0.892297     0.877591    0.911228         0.886479   \n",
       "1116   0.935164     0.938772     0.969023    0.990525         0.973272   \n",
       "1117   0.804401     0.796536     0.812495    0.803534         0.808039   \n",
       "1118   1.003451     1.005641     0.971408    0.997005         0.995884   \n",
       "1119   1.109512     1.113757     1.089068    1.113132         1.180525   \n",
       "1120   0.853283     0.842644     0.857619    0.882108         0.883380   \n",
       "1121   1.053327     1.055018     1.047172    1.058848         1.046278   \n",
       "1122   0.839807     0.825289     0.864418    0.882356         0.866215   \n",
       "1123   0.810731     0.793734     0.781794    0.801653         0.816091   \n",
       "1124   1.016015     1.016341     1.012411    1.026898         1.007053   \n",
       "1125   1.006789     1.009246     0.987176    1.019357         1.035754   \n",
       "1126   0.900048     0.887136     0.923358    0.924773         0.995944   \n",
       "1127   0.866667     0.873464     0.842159    0.861137         0.864173   \n",
       "1128   1.124805     1.139015     1.148699    1.161558         1.185100   \n",
       "1129   0.781559     0.798718     0.808136    0.842453         0.816329   \n",
       "1130   0.923668     0.927178     0.956415    0.954501         0.976227   \n",
       "1131   0.887437     0.885006     0.895667    0.902937         0.895119   \n",
       "1132   0.868581     0.863576     0.876061    0.867810         0.904657   \n",
       "1133   1.269603     1.269768     1.266867    1.300112         1.267734   \n",
       "1134   0.859855     0.840592     0.885673    0.909632         0.899399   \n",
       "1135   0.822099     0.806193     0.823133    0.826042         0.821425   \n",
       "1136   0.935305     0.925420     0.940226    0.955089         0.946809   \n",
       "1137   0.948006     0.957404     0.961448    0.966228         0.941657   \n",
       "\n",
       "      glmm.sparse.rmse  dnn.rmse  \n",
       "0             0.962658  0.882877  \n",
       "1             1.124052  1.007232  \n",
       "2             0.973344  0.881774  \n",
       "3             1.019199  1.017913  \n",
       "4             1.316945  1.292043  \n",
       "5             1.032844  0.968281  \n",
       "6             0.939720  0.954082  \n",
       "7             1.056278  0.916968  \n",
       "8             0.963235  0.937306  \n",
       "9             0.997408  0.945645  \n",
       "10            0.903176  0.877987  \n",
       "11            1.054445  0.974374  \n",
       "12            1.082767  0.962467  \n",
       "13            0.892304  1.049915  \n",
       "14            0.971949  0.898844  \n",
       "15            1.007847  1.027795  \n",
       "16            0.953257  0.849989  \n",
       "17            1.009469  0.984028  \n",
       "18            0.951621  0.983280  \n",
       "19            0.927455  0.931251  \n",
       "20            1.186105  1.182163  \n",
       "21            1.030637  1.055953  \n",
       "22            1.065147  1.040864  \n",
       "23            0.960106  0.912818  \n",
       "24            0.971286  0.899371  \n",
       "25            1.133874  1.182630  \n",
       "26            0.924893  0.931853  \n",
       "27            1.080861  0.975462  \n",
       "28            0.937255  0.887756  \n",
       "29            1.288997  1.319286  \n",
       "...                ...       ...  \n",
       "1108          0.826038  0.893217  \n",
       "1109          0.917566  0.888731  \n",
       "1110          0.939010  0.889690  \n",
       "1111          0.997018  0.816057  \n",
       "1112          0.890375  0.889486  \n",
       "1113          0.981674  0.996912  \n",
       "1114          0.994785  0.913325  \n",
       "1115          0.896507  0.880617  \n",
       "1116          0.968305  0.936234  \n",
       "1117          0.779075  0.891616  \n",
       "1118          0.999130  1.016141  \n",
       "1119          1.114830  1.049505  \n",
       "1120          0.876703  0.896816  \n",
       "1121          1.070476  1.042978  \n",
       "1122          0.843092  0.890023  \n",
       "1123          0.794254  0.912470  \n",
       "1124          1.006702  1.053143  \n",
       "1125          1.017252  1.012933  \n",
       "1126          0.906718  0.887253  \n",
       "1127          1.004010  0.921179  \n",
       "1128          1.161221  1.026753  \n",
       "1129          0.814878  0.815916  \n",
       "1130          0.994896  0.909458  \n",
       "1131          0.903921  0.949747  \n",
       "1132          0.911519  1.015141  \n",
       "1133          1.304537  1.370521  \n",
       "1134          0.879226  0.896755  \n",
       "1135          0.919244  0.888820  \n",
       "1136          1.058379  0.981741  \n",
       "1137          0.945153  0.905251  \n",
       "\n",
       "[1138 rows x 7 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#compare\n",
    "top_tasks['dnn.rmse'] = dnnlist\n",
    "\n",
    "top_tasks = top_tasks[['jklm.rmse', 'ranger.rmse', 'mkl.d9.rmse', 'rf.d9.rmse', 'glmm.dense.rmse', 'glmm.sparse.rmse', 'dnn.rmse']]\n",
    "top_tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dnn.rmse': 393,\n",
       " 'glmm.dense.rmse': 115,\n",
       " 'glmm.sparse.rmse': 113,\n",
       " 'jklm.rmse': 213,\n",
       " 'mkl.d9.rmse': 112,\n",
       " 'ranger.rmse': 163,\n",
       " 'rf.d9.rmse': 29}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#comparing with other methods for each task\n",
    "best_rmse = {'dnn.rmse':0,'glmm.dense.rmse':0, 'mkl.d9.rmse':0, 'rf.d9.rmse':0,'jklm.rmse':0, 'glmm.sparse.rmse':0, 'ranger.rmse':0}\n",
    "\n",
    "for i in range(1138):\n",
    "    best_rmse[np.argmin(top_tasks.iloc[i])] += 1\n",
    "\n",
    "best_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAGXCAYAAABiGCOWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlYVPX+B/D3wDAuLKkJ5X7dkERJisy9RBNScUtNQXBt\nUVukMBBRXMrUEG/uafZTUS8uqLm1uHVNJDO6hWLqFZfABcYEYUYctu/vDx7nQmyiM5wzZ96v5+F5\nhjPDOZ/PzBzec86c7zkqIYQAERERScpG6gKIiIiIgUxERCQLDGQiIiIZYCATERHJAAOZiIhIBhjI\nREREMsBANqO0tDS0a9cOO3bsKDV9/fr1CAsLM9lyvL29cebMGZPNrzI6nQ6jRo3CgAED8N1335W6\nb/ny5ejSpQsGDx5c6icqKqpGagOAnJwcBAUFGX8fPHgwsrOzK/2bwMBAfPvtt0hPT8eoUaMAAKmp\nqXj33Xertez8/HwsXrwYfn5+GDRoEPz8/LBmzRrIaWSht7c3fHx8MHjwYAwZMgQDBgzAggULUFRU\n9FjzLfmenjlzJk6ePFnp4yMiInD27NlqLePOnTto165dufeV7MvPzw8DBgzAtm3bqjX/v5swYQLu\n3LlTZnpl60B1lKy55M+vv/76SPNbvnw55s2bV+Xjdu3ahXbt2uHzzz8vNV0IgT59+mDgwIFVzuOH\nH34w/v2uXbvw1ltvPVLND5w5cwbe3t6PNQ8lUEtdgNLZ2Nhg0aJF8PLyQsuWLaUu57H98ccf+Ouv\nv3Do0KFy7+/fvz9mz55dw1X9z927d0t9OPn6668f+m+feuopxMbGAgBu3LiBK1euVGvZGzduRFpa\nGnbv3g21Wo2cnByMHTsW9evXx+uvv16teZlTVFQUOnbsCADIy8tDYGAgtm7dijFjxphk/p988kmV\njzl58qTJn5OSfd28eRM+Pj7o1asXGjVq9Ejzi4+PL3d6VetAdZSsuSY1btwY+/btw/vvv2+c9ssv\nv+D+/fuoU6dOlX9/5swZ3L1715wlWiUGspnVrl0b48ePx4cffojY2FhoNJpS94eFhaFt27aYOHFi\nmd+9vb0xcOBA/PDDD8jKysK7776LX3/9FcnJyVCr1Vi9ejWeeuopAMDWrVtx/vx55OXlYfz48Rg+\nfDgA4OjRo1i9ejXy8/NRu3ZthIaGwtPTE8uXL8dvv/2GjIwMtGvXrsxW7OHDh7FixQoUFhbCwcEB\nM2bMgIODA8LDw5Geno7Bgwdj27ZtqF279kM/F99//z1Wr14NlUoFW1tbfPTRR3jhhRcqnJ6Tk4NP\nPvkEFy9eRH5+Prp27YqPPvoIarUaHTt2xJtvvon4+HhkZGQgKCgI48aNw4wZM3D//n0MHjwYu3bt\nQvv27ZGQkIDatWtjzpw5uHr1Ku7evQt7e3tERUWhVatWxvrS0tLg5+eHX375BREREUhPT8fEiRPh\n5eWFS5cuYcmSJQCAxMREzJ8/H3v27CnVn1arRX5+PvLy8qBWq+Ho6IjFixcbtz61Wi0iIyNx+fJl\n2NjYYNSoUQgKCsKtW7cwZ84cXL9+HUIIDBkyBJMmTUJaWhoCAgLQunVrXL9+HTExMUhLS0NUVBRy\nc3OhUqnw7rvvonfv3tBqtQgNDUVmZiYA4KWXXsK0adOqfE00Gg2ef/55XL58uVrLy8/Px8cff4yT\nJ0/iySefxJNPPglHR0cAxXscAgIC4Ovri2PHjuGf//wnioqKULduXcydOxfffPMNMjIyEBISgsWL\nF6NVq1YVvs7ff/89li5dijp16qBDhw4P/V67e/cu6tSpg7p16wIAUlJS8MknnyArKwuFhYUIDAzE\n8OHDodfrMWPGDFy7dg02NjZwd3fHvHnzMHPmTADA2LFjsXbtWmOoX758ucw6cOLEiTLrioeHR5Xr\nWFXWrFmDw4cPw2AwIDc3F6GhoXjllVdQUFCAzz77DD/88ANsbW3h6emJyMhIY32BgYHQarVo2LAh\noqOj4eLiUmberq6uuHnzJn799Vc899xzAIDdu3dj0KBB+PHHH42PW716Nb7//nsUFRWhSZMmiIyM\nxK1btxAbG4vCwkI4OjqiRYsW0Gq1ePPNN3Hz5k3Y2tpiyZIlaN26dYXvbaD4f9bGjRvh4OAAV1fX\naj03iiXIbFJTU0WnTp1EYWGh8Pf3FwsXLhRCCPHll1+K0NBQIYQQoaGh4ssvvzT+Tcnfe/fuLRYs\nWCCEEOLAgQPCzc1N/PHHH0IIIaZMmSJWr15tfFxkZKQQQohbt26JLl26iIsXL4orV66IgQMHijt3\n7gghhLh48aLo3r270Ov1YtmyZcLHx0fk5+eXqfvSpUuiW7du4s8//xRCCHHy5EnRvXt3kZOTI376\n6ScxYMCAcvtdtmyZePHFF8WgQYNK/Rw/flwIIUSfPn3Ef/7zHyGEED/++KNYvnx5pdPDwsLEpk2b\nhBBCFBQUiJCQELF27VohhBCurq4iJiZGCCHEmTNnRIcOHcT9+/eNz/kDrq6u4q+//hLffPONmD9/\nvnH6rFmzxLx584QQQowZM0Z88803pf62ZJ+3b98Wzz33nMjMzBRCCDF9+nTxr3/9q0z/N2/eFEOH\nDhUdO3YUY8aMEdHR0SI5Odl4/9SpU8WiRYuEEEJkZ2eLAQMGiKtXr4qAgADx1VdfGaf7+fmJ/fv3\ni9TUVOHq6ipOnz4thBAiKytL9OvXT6Smphpf6169eonr16+LFStWiFmzZgkhhNDr9WLatGkiOzu7\nTI29e/cWSUlJxt9v3bolfH19xbffflut5W3YsEEEBQUJg8Eg9Hq9GDp0qPE9/eD51Gq14vnnnxfn\nzp0TQgjx3XffiYkTJ5apo6LX+cHf//e//xVCCLFmzRrh6upapqcH8+vXr58YNGiQ8PHxEW5ubiI6\nOloIIUR+fr7o37+/OHv2rPE5fvXVV8V//vMfsXv3bjFhwgTjsmfOnCmuXr0qhPjfe+fvSr43KltX\nKlvH/l7zg5/hw4cLIYRIS0sTgYGBIjc3VwghxP79+8XAgQOFEEJs3LhRBAQEiNzcXFFYWCjef/99\nsXv3brFs2TLh7e1trHny5MlixYoVZZYbFxcn3nzzTbF+/Xoxe/ZsIYQQ9+7dE/369RPx8fHG3nbv\n3i2mTZtmrD82NlZMmjRJCFG8rs+dO9c4Py8vL+PzNn/+fDFjxgwhhKjwvX3u3DnRtWtXkZGRIYQo\nXh979+5d7vNkTbiFXANsbGzw2WefYejQoejRo0e1/rZfv34AgGbNmqFhw4Zwc3MDADRv3rzULqMH\n330+9dRT6NGjBxISEmBra4uMjAyMGzfO+DiVSoU///wTANCpUyeo1WXfAj/99BO6dOmCZs2aAQC6\ndu2KBg0a4OzZs1CpVJXWW9ku6wEDBuCdd97BSy+9hO7du+ONN96odPoPP/yAM2fOYOfOnQCA+/fv\nl5pfnz59AADu7u7Iy8vDvXv3KqzL19cXzZo1Q0xMDK5du4aff/4Znp6elfbywJNPPomXX34ZX3/9\nNYYMGYITJ04Yt0hKevrpp7Fr1y5cunQJp06dwqlTp/D6668jLCwMAQEBOHnyJKZPnw4AcHR0xP79\n+3Hv3j38+uuv+Oqrr4zThw0bhuPHj+PZZ5+FWq1Gp06dAAC//fYbtFotpk6dalymSqXChQsX0LNn\nT+MWSrdu3fDhhx8at1j/LiQkBLVr10ZRURHs7OwwYsQI+Pj4IC0t7aGXl5CQgIEDB0Kj0UCj0cDP\nzw8XLlwotZxff/0Vbdu2xTPPPAOg+L384P1cUkWvc2JiIlxdXdGmTRsAwOuvv47o6OgKX6eSu3/T\n09MxduxYtG3bFm5ubvjzzz8RHh5ufOz9+/dx7tw59OzZE0uXLkVgYCC6deuGsWPHokWLFhUu4+8q\nW1eAitex8mouqUmTJli0aBH27duHa9eu4ffff4derwdQvLt/8ODBxr1T//znPwEUf4fcvXt3NGjQ\nAADg5uZW7nfgD/j5+WHw4MGIiIjAoUOH4O3tDVtbW+P9x44dw5kzZ/Daa68BAIqKipCbm1vuvDw8\nPIzP2zPPPINDhw5V+t5OT09H9+7d4ezsDKD4tT1x4kSFtVoLBnINady4MebMmYPQ0FAMGTLEOF2l\nUpU66Cc/P7/U35XcxW1nZ1fh/G1s/nd8nhACarUahYWF6Nq1q3GFBYq/W3NxccGhQ4eMu/P+TpRz\nEJIQAgUFBZXWUJXg4GAMHz4cJ06cwK5du7B27Vrs2rWrwulFRUX4/PPP0bp1awBAdnZ2qQ8EtWrV\nAgDjtPLqfmDr1q3Yvn07AgIC4Ofnh3r16iEtLe2haw8ICMCcOXOgVqvRr18/2Nvbl3nM4sWLMWLE\nCLRp0wZt2rRBQEAAvv76a6xbtw4BAQFQq9Wl6k9NTUW9evXK1F1UVISCggIAxa//g3/ohYWFaN26\ndamDBNPT09GgQQPY2dnhyJEjSEhIwE8//YQRI0Zg5cqVxt2RJVX2veXDLu/vB0yV/EdeclrJfoUQ\nuHDhgvFDZcl+y3udExISSj03lQXb3z311FPw9vbG6dOn0bZtWzg5OZU6nuD27dtwdHRErVq1cOjQ\nIZw6dQo//fQTxo8fj4iICPj6+j7UcipbVwBUuI5VJTk5GVOmTMG4cePQvXt3vPDCC5g7dy6Ass/D\n7du3jV+LlLzv7/9b/s7Z2Rnt27fHv//9b+zZswdhYWHGrzyA4tdl0qRJ8Pf3B1B8vEFF3xuXt9yi\noqIK39t/r62894814lHWNejVV19Fr169sHHjRuO0+vXrGz9N37lzB7/88ssjzXv37t0Aig9GOnny\nJLp27YouXbogPj4eKSkpAIB///vfGDRoEAwGQ6XzevB3qampAICEhATcvHkTzz777CPVBgAFBQXw\n9vbGvXv3MHr0aERGRiIlJaXS6T169MCGDRsghEBeXh4mT56MzZs3V7qcBx9E/v6P4MSJExg6dChG\njBiBli1b4ujRoygsLKxwPra2tqU+HD333HOwsbHB+vXrMXr06HL/5s6dO/j888+NWxFCCFy5cgXt\n27cHULz1FBcXBwDGA76uXbuGZ599Flu2bDFO37NnD7p161Zm/p06dcK1a9dw+vRpAMUHF/n4+CAj\nIwNRUVFYtWoV+vbti5kzZ6JNmza4evVqpc9VVSpbXs+ePbFnzx4YDAYYDAYcPHiwzN8/++yzSElJ\nwX//+18AwJEjR4x7CGxtbY2hVdHr/OC7+/PnzwMoPpr3Yd27dw8nT56Eh4cHWrZsiVq1ahkD+ebN\nmxg4cCDOnj2LrVu3YsaMGejRowemT5+OHj16GOstWWNFzLGuAMDp06fRoUMHjB8/Hp07d8aRI0eM\n79euXbti//79yMvLQ1FREebMmYMDBw480nKGDBmC//u//0NOTk6Z73F79OiBnTt3QqfTAQA+//xz\nfPTRRwAe7rlxcHCo8L3drVs3xMfH49atWwD+9//L2nELuYZFREQgMTHR+HtgYCBCQkLg4+ODpk2b\nonPnzo80X4PBgKFDhyI/Px8RERHGI7rnzZuHDz74wLjVvHr16io/tbdp0waRkZF45513UFhYiNq1\na2PNmjUV7gIt6eDBg6X6A4BGjRphzZo1CA8PR0hIiHFLccGCBdBoNBVOnzlzJj755BP4+fkhPz8f\n3bp1Mx4QUpEHn/pfffVV/Otf/zJOnzBhAmbPno1du3bB1tYW7u7uuHjxYoXzadu2LWxtbTF8+HDs\n2LEDKpUKw4YNw8GDByscehMZGYmlS5di0KBB0Gg0KCgoQJcuXYy78GfPno05c+bAz88PQgi89dZb\n6NChA6KiojBv3jzs2rULeXl58PPzw7Bhw3D9+vVS82/QoAGWLVuGxYsXw2AwQAiBxYsXo0mTJhg7\ndizCwsKMu5HbtWv3UMNXKlPZ8kaNGoU///wTAwcORL169crdzduwYUNERUUhNDTUeMDT0qVLAQB9\n+/ZFcHAwPv744wpfZzs7O0RFRSEkJAR2dnZ44YUXKq33wa54lUqF3NxcvPrqq8bdratWrcInn3yC\nL7/8EgUFBXj//ffx/PPP45lnnsHPP/+M/v37o06dOmjcuLFx2Nwrr7wCf39/rFq1qsKDjh5nXSlZ\nc0ljxozBwIED8f3336N///6ws7ND165dcffuXeOQq+vXr2PYsGEQQqBz584IDAzE6tWrH2qZJfXt\n2xeRkZEIDg4uc9+IESOQnp6OkSNHQqVSoVGjRli4cCGA4g8F7777Luzs7ODu7l7h/Ct6b6tUKkyf\nPh1jx46Fvb09PDw8ql27EqlEZfs0iAhA8Rb+1KlTMXjwYPTv31/qcohIgbjLmqgKly5dQteuXeHg\n4PDQ3y0SEVUXt5CJiIhkgFvIREREMsBAJiIikgEGMhERkQxIOuxJq82RcvHVVr9+XWRmVnw2KKVh\nv8plTb0C7FfJLK1XZ+eKh8RxC7ka1GrrOpsM+1Uua+oVYL9KpqReGchEREQywEAmIiKSAQYyERGR\nDDCQiYiIZOChAvmvv/7CSy+9hJSUFFy7dg2jR4+Gv78/IiMjjZf92r59O4YNG4aRI0fi2LFjZi2a\niIhIaaoM5Pz8fMyePdt4RZJPP/0U06ZNw9atWyGEwJEjR6DVahETE4PY2FisX78e0dHRyMvLM3vx\nRERESlFlIC9atAijRo2Ci4sLgOILZz+4RGCvXr1w8uRJJCUlwdPTExqNBo6OjmjevLnxGqZERERU\ntUpPDLJr1y40aNAAPXv2xNq1awEUX3RdpVIBAOzt7ZGTkwOdTlfq+p/29vbGi1pXpn79uhY3hqyy\nQd1KxH6Vy5p6Bdivkiml10oDOS4uDiqVCgkJCfjjjz8QGhqKO3fuGO/X6/VwcnKCg4MD9Hp9qekP\nc4FuSzq7ClD8olva2cUeB/tVLmvqFWC/SmZpvT7ymbq2bNmCzZs3IyYmBs888wwWLVqEXr164dSp\nUwCA48ePw8vLCx4eHkhMTITBYEBOTg5SUlLg6upq2i6IiIgUrNrnsg4NDcWsWbMQHR2NVq1awcfH\nB7a2tggMDIS/vz+EEAgODkatWrXMUS8REZEiqYQQQqqFW9JuBsDydo08LvarXNbUK8B+lczSeq1s\nl7WkV3syNWcXJ/Mvw4zz1mZkm3HuREQkZzxTFxERkQwwkImIiGSAgUxERCQDDGQiIiIZYCATERHJ\nAAOZiIhIBhjIREREMsBAJiIikgEGMhERkQwwkImIiGSAgUxERCQDDGQiIiIZYCATERHJAAOZiIhI\nBhjIREREMsBAJiIikgEGMhERkQwwkImIiGSAgUxERCQDDGQiIiIZYCATERHJAAOZiIhIBhjIRERE\nMsBAJiIikgEGMhERkQwwkImIiGSAgUxERCQD6qoeUFhYiIiICFy5cgUqlQpz585FQUEB3nrrLfzj\nH/8AAIwePRr9+/fH9u3bERsbC7VajcmTJ6N3797mrp+IiEgRqgzkY8eOAQBiY2Nx6tQpLF26FN7e\n3hg/fjwmTJhgfJxWq0VMTAzi4uJgMBjg7++P7t27Q6PRmK96IiIihagykPv27YuXX34ZAHDjxg04\nOTnh7NmzuHLlCo4cOYIWLVogPDwcSUlJ8PT0hEajgUajQfPmzXH+/Hl4eHiYuwciIiKLV2UgA4Ba\nrUZoaCgOHTqEZcuWIT09HSNGjECHDh2wevVqrFy5Em5ubnB0dDT+jb29PXQ6XaXzrV+/LtRq28fr\nQEGcnR2rflANk2NN5mRN/VpTrwD7VTKl9PpQgQwAixYtQkhICEaOHInY2Fg89dRTAIBXXnkF8+fP\nh5eXF/R6vfHxer2+VECXJzPz3iOWXT5nk86t5mm1OVKXUIqzs6PsajIna+rXmnoF2K+SWVqvlX14\nqPIo6z179uCLL74AANSpUwcqlQrvvPMOkpKSAAAJCQlwd3eHh4cHEhMTYTAYkJOTg5SUFLi6upqo\nBSIiImWrcgu5X79+mDFjBgICAlBQUIDw8HA0atQI8+fPh52dHRo2bIj58+fDwcEBgYGB8Pf3hxAC\nwcHBqFWrVk30QEREZPFUQggh1cJNvZvB2cXJpPOradqMbKlLKMXSdgU9Lmvq15p6Bdivkllar4+1\ny5qIiIjMj4FMREQkAwxkIiIiGWAgExERyQADmYiISAYYyERERDLAQCYiIpIBBjIREZEMMJCJiIhk\ngIFMREQkAwxkIiIiGWAgExERyQADmYiISAYYyERERDLAQCYiIpIBBjIREZEMMJCJiIhkgIFMREQk\nAwxkIiIiGWAgExERyQADmYiISAYYyERERDLAQCYiIpIBBjIREZEMMJCJiIhkgIFMREQkAwxkIiIi\nGWAgExERyYC6qgcUFhYiIiICV65cgUqlwty5c1GrVi2EhYVBpVKhbdu2iIyMhI2NDbZv347Y2Fio\n1WpMnjwZvXv3rokeiIiILF6VgXzs2DEAQGxsLE6dOoWlS5dCCIFp06bhxRdfxOzZs3HkyBF06tQJ\nMTExiIuLg8FggL+/P7p37w6NRmP2JoiIiCxdlYHct29fvPzyywCAGzduwMnJCSdPnkTnzp0BAL16\n9UJ8fDxsbGzg6ekJjUYDjUaD5s2b4/z58/Dw8DBrA0REREpQZSADgFqtRmhoKA4dOoRly5YhPj4e\nKpUKAGBvb4+cnBzodDo4Ojoa/8be3h46na7S+davXxdqte1jlK8szs6OVT+ohsmxJnOypn6tqVeA\n/SqZUnp9qEAGgEWLFiEkJAQjR46EwWAwTtfr9XBycoKDgwP0en2p6SUDujyZmfceoeSKOZt0bjVP\nq82RuoRSnJ0dZVeTOVlTv9bUK8B+lczSeq3sw0OVR1nv2bMHX3zxBQCgTp06UKlU6NChA06dOgUA\nOH78OLy8vODh4YHExEQYDAbk5OQgJSUFrq6uJmqBiIhI2arcQu7Xrx9mzJiBgIAAFBQUIDw8HK1b\nt8asWbMQHR2NVq1awcfHB7a2tggMDIS/vz+EEAgODkatWrVqogciIiKLpxJCCKkWburdDM4uTiad\nX03TZmRLXUIplrYr6HFZU7/W1CvAfpXM0np9rF3WREREZH4MZCIiIhlgIBMREckAA5mIiEgGGMhE\nREQywEAmIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAm\nIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYy\nERGRDDCQiYiIZICBTEREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQyoK7szvz8fISHh+P69evI\ny8vD5MmT0ahRI7z11lv4xz/+AQAYPXo0+vfvj+3btyM2NhZqtRqTJ09G7969a6J+IiIiRag0kPfu\n3Yt69erhs88+Q1ZWFoYMGYKpU6di/PjxmDBhgvFxWq0WMTExiIuLg8FggL+/P7p37w6NRmP2BoiI\niJSg0kD29fWFj48PAEAIAVtbW5w9exZXrlzBkSNH0KJFC4SHhyMpKQmenp7QaDTQaDRo3rw5zp8/\nDw8PjxppgoiIyNJVGsj29vYAAJ1Oh/feew/Tpk1DXl4eRowYgQ4dOmD16tVYuXIl3Nzc4OjoWOrv\ndDpdlQuvX78u1Grbx2xBOZydHat+UA2TY03mZE39WlOvAPtVMqX0WmkgA8DNmzcxdepU+Pv7w8/P\nD9nZ2XBycgIAvPLKK5g/fz68vLyg1+uNf6PX60sFdEUyM+89RullOZt0bjVPq82RuoRSnJ0dZVeT\nOVlTv9bUK8B+lczSeq3sw0OlR1nfvn0bEyZMwPTp0zF8+HAAwMSJE5GUlAQASEhIgLu7Ozw8PJCY\nmAiDwYCcnBykpKTA1dXVhC0QEREpW6VbyGvWrEF2djZWrVqFVatWAQDCwsKwYMEC2NnZoWHDhpg/\nfz4cHBwQGBgIf39/CCEQHByMWrVq1UgDRERESqASQgipFm7q3QzOLk4mnV9N02ZkS11CKZa2K+hx\nWVO/1tQrwH6VzNJ6feRd1kRERFQzGMhEREQywEAmIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIi\nIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIiIhlgIBMR\nEckAA5mIiEgGGMhEREQywEAmIiKSAQYyERGRDDCQiYiIZEAtdQFERMnJTlKX8Fjc3bOlLoEUgFvI\nREREMsBAJiIikgEGMhERkQwwkImIiGSAgUxERCQDDGQiIiIZqHTYU35+PsLDw3H9+nXk5eVh8uTJ\naNOmDcLCwqBSqdC2bVtERkbCxsYG27dvR2xsLNRqNSZPnozevXvXVA9EREQWr9JA3rt3L+rVq4fP\nPvsMWVlZGDJkCNzc3DBt2jS8+OKLmD17No4cOYJOnTohJiYGcXFxMBgM8Pf3R/fu3aHRaGqqDyIi\nIotWaSD7+vrCx8cHACCEgK2tLZKTk9G5c2cAQK9evRAfHw8bGxt4enpCo9FAo9GgefPmOH/+PDw8\nPMzfARERkQJUGsj29vYAAJ1Oh/feew/Tpk3DokWLoFKpjPfn5ORAp9PB0dGx1N/pdLoqF16/fl2o\n1baPU7+iODs7Vv2gGibHmszJmvq1pl7NTY7PpRxrMhel9FrlqTNv3ryJqVOnwt/fH35+fvjss8+M\n9+n1ejg5OcHBwQF6vb7U9JIBXZHMzHuPWHb5nE06t5qn1eZIXUIpzs6OsqvJnKypX2vqtSbI7bm0\nptfX0nqt7MNDpUdZ3759GxMmTMD06dMxfPhwAED79u1x6tQpAMDx48fh5eUFDw8PJCYmwmAwICcn\nBykpKXB1dTVhC0RERMpW6RbymjVrkJ2djVWrVmHVqlUAgJkzZ+Ljjz9GdHQ0WrVqBR8fH9ja2iIw\nMBD+/v4QQiA4OBi1atWqkQaIiIiUQCWEEFIt3NS7GZxdLPuKMdoMeV0xxtJ2BT0ua+pXbr3yak+m\nJbfX15wsrddH3mVNRERENYOBTEREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYy\nERGRDDBBkVeIAAAdHklEQVSQiYiIZICBTEREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAm\nIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQyoJa6ACIiUi6X\n5ESpS3gsGe7P19iyuIVMREQkAwxkIiIiGWAgExERyQADmYiISAYYyERERDLwUIH8+++/IzAwEABw\n7tw59OzZE4GBgQgMDMTBgwcBANu3b8ewYcMwcuRIHDt2zHwVExERKVCVw57WrVuHvXv3ok6dOgCA\n5ORkjB8/HhMmTDA+RqvVIiYmBnFxcTAYDPD390f37t2h0WjMVzkREZGCVLmF3Lx5cyxfvtz4+9mz\nZ/HDDz8gICAA4eHh0Ol0SEpKgqenJzQaDRwdHdG8eXOcP3/erIUTEREpSZVbyD4+PkhLSzP+7uHh\ngREjRqBDhw5YvXo1Vq5cCTc3Nzg6OhofY29vD51OV+XC69evC7Xa9hFLVx5nZ8eqH1TD5FiTOVlT\nv9bUq7nJ8bmUY02WqCafx2qfqeuVV16Bk5OT8fb8+fPh5eUFvV5vfIxery8V0BXJzLxX3cVXytmk\nc6t5Wm2O1CWU4uzsKLuazMma+rWmXmuC3J5Lvr6mY+rnsbKAr/ZR1hMnTkRSUhIAICEhAe7u7vDw\n8EBiYiIMBgNycnKQkpICV1fXR6+YiIjIylR7C3nOnDmYP38+7Ozs0LBhQ8yfPx8ODg4IDAyEv78/\nhBAIDg5GrVq1zFEvERGRIqmEEEKqhZt8V4CLk0nnV9O0GdlSl1CKte32sqZ+5dZrcrJlr7vu7lx3\nK8KLS5Rm0l3WREREZHoMZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYyERGRDDCQiYiI\nZICBTEREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYyERGRDDCQiYiIZICBTERE\nJAMMZCIiIhlgIBMREckAA5mIiEgG1FIXQI/OZZWT1CU8lowp2VKXQEQkG9xCJiIikgFuIRMR1bBk\nl0SpS3gs7hnPS12CIjGQyWK4uDjWwFLMs4yMjByzzJeIlIO7rImIiGSAgUxERCQDDGQiIiIZYCAT\nERHJwEMF8u+//47AwEAAwLVr1zB69Gj4+/sjMjISRUVFAIDt27dj2LBhGDlyJI4dO2a+iomIiBSo\nykBet24dIiIiYDAYAACffvoppk2bhq1bt0IIgSNHjkCr1SImJgaxsbFYv349oqOjkZeXZ/biiYiI\nlKLKQG7evDmWL19u/D05ORmdO3cGAPTq1QsnT55EUlISPD09odFo4OjoiObNm+P8+fPmq5qIiEhh\nqhyH7OPjg7S0NOPvQgioVCoAgL29PXJycqDT6eDo+L/xm/b29tDpdFUuvH79ulCrbR+lbkVydq6J\ncbbyYU39yrFXOdZkqaztubSmfmuy12qfGMTG5n8b1Xq9Hk5OTnBwcIBery81vWRAVyQz8151F18p\nZ5POreZptdZ18ojq92u5/wTk9to6OzvKriZLZm3PpTX1a+peKwv4ah9l3b59e5w6dQoAcPz4cXh5\necHDwwOJiYkwGAzIyclBSkoKXF1dH71iIiIiK1PtLeTQ0FDMmjUL0dHRaNWqFXx8fGBra4vAwED4\n+/tDCIHg4GDUqlXLHPUSEREp0kMFctOmTbF9+3YAQMuWLbF58+Yyjxk5ciRGjhxp2uqIiIisBE8M\nQkREJAMMZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYyERGRDDCQiYiIZICBTEREJAMM\nZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIiIhlg\nIBMREckAA5mIiEgGGMhEREQywEAmIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIiIhlgIBMREckA\nA5mIiEgGGMhEREQyoH7UPxw6dCgcHBwAAE2bNsXbb7+NsLAwqFQqtG3bFpGRkbCxYd4TERE9jEcK\nZIPBACEEYmJijNPefvttTJs2DS+++CJmz56NI0eO4JVXXjFZoUREREr2SJuw58+fR25uLiZMmICg\noCD89ttvSE5ORufOnQEAvXr1wsmTJ01aKBERkZI90hZy7dq1MXHiRIwYMQJXr17FG2+8ASEEVCoV\nAMDe3h45OTlVzqd+/bpQq20fpQRFcnZ2lLqEGmVN/cqxVznWZKms7bm0pn5rstdHCuSWLVuiRYsW\nUKlUaNmyJerVq4fk5GTj/Xq9Hk5OTlXOJzPz3qMsvkLOJp1bzdNqq/4QoyTV79dy/wnI7bV1dnaU\nXU2WzNqeS2vq19S9Vhbwj7TLeufOnVi4cCEAID09HTqdDt27d8epU6cAAMePH4eXl9ejzJqIiMgq\nPdIW8vDhwzFjxgyMHj0aKpUKCxYsQP369TFr1ixER0ejVatW8PHxMXWtREREivVIgazRaLBkyZIy\n0zdv3vzYBREREVkjDhQmIiKSAQYyERGRDDCQiYiIZICBTEREJAMMZCIiIhlgIBMREckAA5mIiEgG\nGMhEREQywEAmIiKSAQYyERGRDDzSqTOJyLxcXKq+WpqcZWRkS10CkcXhFjIREZEMMJCJiIhkgIFM\nREQkAwxkIiIiGWAgExERyQADmYiISAYYyERERDLAQCYiIpIBBjIREZEMMJCJiIhkgIFMREQkAwxk\nIiIiGWAgExERyQADmYiISAYYyERERDLAQCYiIpIBBjIREZEMqE05s6KiIsyZMwcXLlyARqPBxx9/\njBYtWphyEURERIpk0i3kw4cPIy8vD9u2bcOHH36IhQsXmnL2REREimXSQE5MTETPnj0BAJ06dcLZ\ns2dNOXsiIiLFMukua51OBwcHB+Pvtra2KCgogFpd/mKcnR1NuXhACNPOr4Y5V/PxItKy+60uy355\nq/deF5bdbLW9/LKV9StelrqEGiNeflnqEiyGSbeQHRwcoNfrjb8XFRVVGMZERET0PyYN5Oeeew7H\njx8HAPz2229wdXU15eyJiIgUSyVMuG/swVHWFy9ehBACCxYsQOvWrU01eyIiIsUyaSATERHRo+GJ\nQYiIiGSAgUxERCQDDGQiIiIZYCATERHJAAOZAADnzp0rd/rhw4druJKac+PGDRw8eBBxcXE4evQo\nsrKypC7J7LKysnDz5k3k5uZKXQqZiDWuu+XRarVSl/DYeJR1BQIDA6FSqcq9b9OmTTVcjfkFBQUZ\n+xo/fjz+7//+r8x0Jdm5cyf27duHjh07IiEhAe7u7rhy5QoCAwPRr18/qcszuTNnzmDevHkoLCzE\nlStX4O7uDjs7O8yaNQutWrWSujyTmjFjRoX3ffrppzVYSc2wtnU3OTkZy5cvxxNPPIHQ0FA0aNAA\nW7ZswZo1a/Djjz9KXd5j4Wm0KjB37txSv58/fx4LFizAwIEDJarIvEp+LisoKCh3upLs2bMHMTEx\nUKlUyM3NRUhICNavX4+goCBFBnJUVBS++OILNGjQAFevXsWGDRvwxhtvIDw8HBs3bpS6PJM6e/Ys\n7t+/j0GDBsHT01Ox7+EHrG3dnTVrFj744APcuHEDS5cuxb1795CRkYHNmzdLXdpjYyBX4MFWgxAC\na9euxZ49exAdHY3OnTtLXJl5lNwbUNFtJcnOzoZOp4OjoyNyc3ORlZUFjUYDg8EgdWlmodPp0KBB\nAwBA06ZNcfHiRTRp0gT379+XuDLT27dvHy5evIi9e/di7dq1eOGFFzBo0CDFXgrW2tbdOnXqoEeP\nHgCAlStXYsiQIYiKilJEvwzkSly9ehVhYWFwdXXFzp07YW9vL3VJZiOEQH5+PoQQZW4r0YQJEzB4\n8GA888wzuHTpEsLCwrBixQr06dNH6tLMolOnTnj77bfRs2dP/Pjjj+jRowf27t0LZ+fqXtLEMri6\nuiIkJAQAcPr0aSxZsgS3bt3C9u3bJa7M9Kxt3bW1tTXednFxQXBwsITVmBa/Q65ATEwMNmzYgBkz\nZqBXr16l7tNoNBJVZT7e3t5QqVRlVmKVSoUjR45IVJV5ZWZmIjU1FS1btoSjoyMKCwtLrexKc/jw\nYaSkpOCZZ55Br169kJKSgqZNm6JWrVpSl2YWOp0Ohw4dwv79+5Gbm4v+/ftjzJgxUpdlcta27gYG\nBuKrr76CEAITJ0403gYs/38zA7kC3t7extsl3+xKfZNboy1btuDAgQPIysrC008/jQEDBuC1116T\nuiyzuXTpEr799ltkZWXhqaeegq+vL5o1ayZ1WSZ38OBBHDx4EDdu3EC/fv0wcOBANG3aVOqyyEQe\nfAABivcOPPj/rIT/zQxkAgDcvXsXK1euRFhYGFJSUhAWFgaNRoMFCxagZcuWUpdncsuXL4dWq8WE\nCRPQsGFDXL9+HV999RVatGiBKVOmSF2eyX333XdYs2YNRo4ciSeffBI3btxAXFwcgoODS334VAI3\nNze0atUKbm5uAEp/l7pkyRKpyjIba1t3lYyBXIEVK1aUO12lUmHq1Kk1XI35vf/++3j++ecxZswY\njBs3DkOHDoWrqyuio6Oxfv16qcszuddffx3btm0rNa2oqAhBQUGKOFrz7/z9/bFu3bpSx0Hk5ORg\nypQpiImJkbAy0/v5558rvE+JB2Va27r7xRdf4K233gIAJCQkoGvXrgCAyMjIMqNjLA0P6qpAw4YN\nS/2em5uLdevWoUmTJooMZK1Wi6CgIOh0Oly4cAFDhgwxDglSovK+a7KxsVHsd8i2trZlDkp0dHRU\nZL8Vhe7ixYsVGcjWtu7Gx8cbA3n16tXGQL58+bKUZZkEA7kCo0aNMt5OTExEREQEAgIC8Pbbb0tY\nlfnUqVMHQPERqV5eXsbdfEpdqSsaIqHUHUYV9VtUVFTDlUinsi1nS2Zt627JdbTkbQ57Urj8/HxE\nR0cjISEBS5YsQfv27aUuyWxcXFwQHR2NEydOYMqUKdDpdNi4cSPatWsndWlm8euvvxrHMpZ09+5d\nCaoxv5SUFHz00UelpgkhFLFVYe2sbd1V8lhrBnIFzp07hxkzZqBnz57YsWMH7OzspC7JrObMmYO4\nuDi8/fbb6Nu3L3777TdkZmZi1qxZUpdmFmfPnpW6hBoVFRVV7vRhw4bVcCXmd+XKlTLThBCKPelL\nRevu7NmzpS7NLNLT07Ft2zYIIUrdzsjIkLq0x8aDuirQoUMH2NvblzpK8cGh9bGxsRJWRuZ07Ngx\n9O7dW+oyakxGRgZcXFykLsOkAgMDK7xPaQewVebf//43XnrpJanLMLmKDrgFgHfeeacGKzE9BnIF\nrl+/jpycHKxfvx6ZmZnw8vKCr68v7Ozs0KRJE6nLM7nyxvY9YOlj+6pjw4YNGDdunNRl1JiQkJAK\nt54tVXZ2NpycnMpMT0lJQevWrSWoyLx27dqF6Oho1K5dG8uWLUOzZs0QERGBy5cvY9++fVKXZ1bp\n6ekoLCyESqVCo0aNpC7nsTGQK/Dtt99i3bp1GDVqFBo0aIAbN25gx44deO+999C3b1+pyzO5vLy8\nUr8fP34cCxYswLhx4xAUFCRRVTXv2rVrij3nsbUYPXo0NmzYUOoMZHv37sXixYtx4sQJCSszDz8/\nP2zevBlarRYLFy5ERkYG+vTpgylTpijyq7ZLly5h3rx52LRpE3x9fVGvXj3cunUL4eHhln9hGEHl\nGjVqlNDr9aWm5eTkiDFjxkhUUc24d++emDVrlnj99dfF5cuXpS6nxr322mtSl2AWc+fONd5OTk6W\nsBLz27Bhgxg/frzIz88XBoNBzJw5UwwbNkyx7+eS/5Nefvll8cMPP0hYjfm99dZbIikpSQjxv96v\nXr2qiP/NNlJ/IJArtVqNunXrlprm4OCgyHGbD5w+fRpDhw5FkyZNsHXrVqs8y49Q6A6jS5cuGW8v\nXLhQwkrMb+zYsejWrRumTJmCUaNGwdHREbGxsYp9P5f8eqlx48aK/N64pNzcXHTs2BFA8Vh6AGjR\nokWpS09aKh5lXQFrG7f56aef4sCBA5g5cybc3Nxw7do1431K/UdWHqUNo3hAVDB2U6kmTZqEgoIC\n/PTTTwgNDZW6HLPKyspCfHw8ioqKoNPpSu2WL29on6UrebT8qlWrjLfVasuPM8vvwEwuXbqEDz/8\nsNQ0IQRSUlIkqsi8zp07h5YtW2Lr1q2lpqtUKmzatEmiqszngw8+KBO+QgikpqZKVJF5KXns5t8t\nWbLEeMGBq1evYsqUKWjTpg2A4tddadzd3bF//34AQPv27XHgwAHjfUoMZBcXFyQlJcHDw8M4LSkp\nSRGXEuVBXRWwtvPhWhtre307dOiAevXqASjeonpwW6VS4ccff5SyNJOLjY0tM2IgNzcX9erVw5Ah\nQySurmbcuXMHO3fuxJtvvil1KSaXmpqKKVOmoEuXLmjRogVSU1ORkJCANWvWoHHjxlKX91i4hVwB\nJf5TrszHH3+MiIgIAMVHpA4aNAgAMHXqVKxcuVLK0swiPT0dfn5+paYZDAbMnTtXka/9f/7zHxw9\nehRPPPEEunTpAqD4HMgff/yxxJWZXsktxAfu3LkDX19fCaqpWUlJSdiyZQvi4+Mt/4jjCjRr1gw7\nduzA0aNHkZaWhg4dOuD9998vc8yPJWIgEwDgwoULxts7d+40BnJ2drZUJZnVl19+CXt7e+OlB69c\nuYL33nsPnTp1krgy85g+fTpsbW1x+/ZtpKSkoEmTJoiIiKj0JBqWqryTfxQVFWHkyJF49913JajI\nvPLy8nDgwAFs2bIFGo0GOp0Ohw8fRu3ataUuzSxu3LgBAOjUqZNxfc3KykJWVha3kEl5hMJO2F6e\nL7/8EhMnTkTdunVx+/ZtLFmyBGFhYfDx8ZG6NLP4888/sWvXLuTl5eG1116DnZ0dNm7cqMgTZfxd\nYWEhEhMTFXswm7e3NwYOHIioqCj84x//wKRJkxQbxgAQHBxsPEYAKP4fde3aNeTk5Fj8KXEZyATA\nug76AQBnZ2esW7cO48ePR506dfCvf/0LTz/9tNRlmY2DgwOA4stOFhUV4auvvjJ+j6x0BoMBmzdv\nVux52ceOHYt9+/bh+vXrGD58uGI/eDxQ8jrmeXl5WLZsGfR6PdatWydhVabBQCYAxQdKREdHG480\nfnA7LS1N6tLMIi8vD/Xr18eqVavw3nvv4c6dO2jQoAGA8q+VrCRPPvmk1YQxANStWxfLli2Tugyz\neeONN/DGG2/g559/xo4dO3D27Fl89tlnGDx4MFxdXaUuz2zOnz+PsLAwdO3aFXFxcYpYb3mUNQEA\ndu/eXeF9Q4cOrcFKaoa1nbu7W7du6Nq1K4QQ+Omnn4wXdQeKhwmRcmRnZ+Prr79GXFwc9uzZI3U5\nJldUVIQ1a9Zg//79mDdvHry8vKQuyWQYyGSUl5eHxMREZGZm4umnn0anTp1gY6PMk7nNmDGjwvs+\n/fTTGqykZljbMC9rcu/ePezatQt169bFkCFDFLvOPjBixAjcuHEDkyZNKnNk9euvvy5RVabBXdYE\nAPjjjz/wwQcfwN3dHU8++SS++eYbpKSkYPny5Yo88Cc5ORn379+Hn58fPD09Ff+9G0NXucLCwtC8\neXNkZ2fj6tWrijz5SUkPTg2q1+uh1+slrsa0GMgEoPgC9itXrkSrVq2M0/773/9i4cKFijhY4u/2\n7t2LixcvYu/evVi7di1eeOEFDBo0iFd6IouTmZmJZcuWQQiB8ePHS12O2ZV3zeMLFy5gy5YtElRj\nWgxkAgDcv3+/VBgDQNu2bZGfny9RRebn6uqKkJAQAMUX1liyZAlu3bqF7du3S1wZ0cN7cPyDSqVS\n7Ln2y1NYWIjvv/8eW7Zswe3btzFixAipS3psDGQCgAqvYqX0FVyn0+HQoUPYv38/cnNzjSdEIbIU\nQgjk5+dDCFHqNqDMEQNarRbbtm3D119/jU6dOiEvLw/ffvut1GWZBAOZABSfSrLk+D6geEXPyMiQ\nqCLzOnjwIA4ePIgbN26gX79+mDt3Lpo2bSp1WUTVdv36dfj6+hpD+MHJbVQqlSJHDPTr1w9BQUHY\nvXs3HBwcMGnSJKlLMhkeZU0AgBUrVlR4X3nf2Vg6Nzc3tGrVCm5ubgBKnwyFw4CI5OvgwYPYuXMn\n7t69i9deew3fffcdNm7cKHVZJsFAJqvEYUCkFHfv3sXKlSsRFhaGlJQUhIWFQaPRYMGCBYq+lnla\nWhp27tyJvXv3wsPDA4MHD0bv3r2lLuuxMJCpUu+9956iz3JEZOnef/99PP/88xgzZgzGjRuHoUOH\nwtXVFdHR0Vi/fr3U5ZlcQUEBjh49CicnJ3Tp0gVFRUXYt28f9u7da/H98jtkqtT8+fOlLoGIKqHV\nahEUFASdTocLFy5gyJAhxmtAK1FISAhsbW2h1WqNVy5bvHgxgoKCpC7tsTGQCUDxsJ+KvPDCCzVY\nCRFVR506dQAUr8NeXl7G4yGUGsjlXbls06ZNijiBEQOZAACBgYFo3rw5OnbsCAClLm3GQCaSLxcX\nF0RHR+PEiROYMmUKdDodNm7ciHbt2kldmlko+cplDGQCAMTFxWH//v1ITk5Gly5dMGjQIA4DIrIA\nc+bMQVxcHN5++2307dsXv/32GzIzMxV7ucmSlHblMh7URaU8uBrQvn37cPv2bXh7e2PUqFFSl0VE\nBEDZVy5jIFMZ9+/fx+HDh7Fnzx5kZmYiLi5O6pKIqJqUOkJCyUMWGcgEAMjPz8fx48exf/9+XLly\nBX369MGAAQPKnN+aiCzD3bt38cQTT0hdBlUDA5kAFB9J7eLiggEDBqBjx46lzlzVo0cPCSsjospw\nhIRy8KAuAgD06dMHKpUKqampSE1NLXUfA5lIvjhCQjm4hUxEZMGSk5M5QkIhGMgEoHgL+e+EEIq9\nYgyR0nCEhOXjLmsCAPTu3Rtnz55Ft27dMGjQIDRu3FjqkoioGlQqFTw9PfHXX39hz5492LFjBwPZ\nwnALmYyKiopw4sQJ7N+/H3fv3kXfvn3x6quvGs+MQ0TywxESysFApnJlZWVhzpw5OHbsGH7//Xep\nyyGiCnCEhHJwlzUZFRUVIT4+HgcOHMAff/yBXr16YceOHVKXRUSV4AgJ5eAWMgEoPh/uL7/8gs6d\nO2PgwIF47rnnpC6JiMiqMJAJAODm5oZ69epBrS670+TEiRMSVERED4MjJJSDu6wJAHD+/HmpSyCi\nR8AREsrBLWSq1NixY7Fx40apyyCiSnCEhDIwkKlSr732Gq/2RGRBOELCcnGXNVWq5BAKIpInjpBQ\nBgYyAQC2bdtWZpoQAnfu3JGgGiJ6WCVHSIwcOZIjJCwYd1kTAGDFihXG2+np6SgoKIBKpULjxo3x\nzjvvSFgZEVWGIySUg4FMAIBLly5h3rx52LRpE3x9ffHEE0/g1q1bCA8Ph4+Pj9TlEREpno3UBZA8\nREVFYfr06QAAZ2dnbNu2DZs2bcLmzZslroyIHsXYsWOlLoGqiYFMAIDc3FzjBc4dHR0BAC1atEBB\nQYGUZRHRI9LpdFKXQNXEQCYAgMFgMN5etWqV8XZ530sRkfxxhITl4X9bAgC4uLggKSkJHh4exmlJ\nSUlwdnaWsCoiqgpHSCgHA5kAANOnT8eUKVPQpUsXtGjRAqmpqUhISMCaNWukLo2IKqHVao23S46Q\nGDZsmIRV0aPgLmsCADRr1gw7duyAp6cn7t27hw4dOiA2NpbnxSWSOV9fX/z888945513cPr0aVy+\nfBnx8fFo27at1KVRNXELmYxq166N/v37S10GEVXD30dIxMTE4Nq1a4iIiOCQRQvDLWQiIgvGERLK\nwUAmIrJgHCGhHAxkIiIL9mCEREkcIWGZeOpMIiILlpqaWuEICR6UaVkYyEREFu7+/fs4evQo0tLS\n0KhRI/Tp0wd169aVuiyqJgYyERGRDPA7ZCIiIhlgIBMREckAA5mIiEgGGMhEREQywEAmIiKSgf8H\nb8+H2f16QmMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4b49a70358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plotting who did best per task\n",
    "colors = 'rgbkymc'\n",
    "keys = []\n",
    "\n",
    "keys = [s.replace(\".rmse\", \"\").upper() for s in list(best_rmse.keys())]\n",
    "\n",
    "plt.bar(range(len(best_rmse)), best_rmse.values(), align='center', color=colors)\n",
    "plt.xticks(range(len(best_rmse)), keys, rotation='vertical')\n",
    "plt.title('Number of Essentiality Scores Predicted Best for Each Method')\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}