decision_support.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FrTHTGlXMkvE"
   },
   "outputs": [],
   "source": [
    "# !curl ipecho.net/plain\n",
    "# !pip install pymfe\n",
    "# !pip install mysql-connector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l21PeAxaNGoD"
   },
   "outputs": [],
   "source": [
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rTbWffNjMbwT"
   },
   "outputs": [],
   "source": [
    "# import mysql.connector\n",
    "\n",
    "# mydb = mysql.connector.connect(\n",
    "#   host=\"localhost\",\n",
    "#   user=\"root\",\n",
    "#   passwd=\"\",\n",
    "#   database=\"anomaly_detection_decision_support\"\n",
    "# )\n",
    "\n",
    "# mycursor = mydb.cursor()\n",
    "\n",
    "\n",
    "def open_connection():\n",
    "    mydb = mysql.connector.connect(\n",
    "      host=\"34.68.13.182\",\n",
    "      user=\"root\",\n",
    "      passwd=\"6g8HBIy0F8atEKtb\",\n",
    "      database=\"anomaly_detection_decision_support\"\n",
    "    )\n",
    "\n",
    "    mycursor = mydb.cursor()\n",
    "    return mydb, mycursor\n",
    "\n",
    "def close_connection(mydb, mycursor):\n",
    "    mycursor.close()\n",
    "    mydb.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GvO8Us1WMbwX",
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import mysql.connector\n",
    "\n",
    "\n",
    "def get_datasets(path):\n",
    "    datasets = []\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for dir in dirs:\n",
    "            with open(os.path.join(root, dir, 'metadata.json')) as json_file:\n",
    "                metadata = json.load(json_file)\n",
    "                metadata['files'] = [os.path.join(root, dir, filename) for filename in metadata['files']]\n",
    "                datasets.append(metadata)\n",
    "    return datasets\n",
    "\n",
    "def get_methods():\n",
    "    methods = [\n",
    "        {\n",
    "            'name': 'gaussian',\n",
    "            'parameters': {},\n",
    "            'isSupervised': False,\n",
    "        }, {\n",
    "            'name': 'linear_regression',\n",
    "            'parameters': {},\n",
    "            'isSupervised': False,\n",
    "        }, {\n",
    "            'name': 'pca',\n",
    "            'parameters': {},\n",
    "            'isSupervised': False,\n",
    "        }, {\n",
    "            'name': 'kmeans',\n",
    "            'parameters': {},\n",
    "            'isSupervised': False,\n",
    "        }, {\n",
    "            'name': 'neural_network',\n",
    "            'parameters': {},\n",
    "            'isSupervised': True,\n",
    "        }\n",
    "    ];\n",
    "    \n",
    "    return methods\n",
    "                       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 697
    },
    "colab_type": "code",
    "id": "yB4HzIEDMbwa",
    "jupyter": {
     "source_hidden": true
    },
    "outputId": "9a536202-b824-459d-9138-fade9658e28e"
   },
   "outputs": [],
   "source": [
    "#initialize tensorflow\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n",
    "\n",
    "strategy = None\n",
    "if (os.environ.get('COLAB_TPU_ADDR')!=None):\n",
    "  os.environ['TPU_ADDR'] = os.environ['COLAB_TPU_ADDR']\n",
    "# else:\n",
    "#   os.environ['TPU_ADDR'] = '10.15.20.26:8470'\n",
    "\n",
    "if (os.environ.get('TPU_ADDR')!=None):\n",
    "    resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['TPU_ADDR'])\n",
    "    tf.config.experimental_connect_to_cluster(resolver)\n",
    "    # This is the TPU initialization code that has to be at the beginning.\n",
    "    tf.tpu.experimental.initialize_tpu_system(resolver)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(resolver)\n",
    "\n",
    "def get_devices():\n",
    "    if strategy:\n",
    "        devices = ['ASIC']\n",
    "    else:\n",
    "        devices = [d.name for d in device_lib.list_local_devices() if 'XLA' not in d.name]\n",
    "    return devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nBX9-KpoMbwc"
   },
   "outputs": [],
   "source": [
    "# data cleansing\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pymfe.mfe import MFE\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n",
    "def transform_data(dataset, rows = 1000, mfe = True):\n",
    "    # read data from files\n",
    "    df = pd.DataFrame()\n",
    "    chunk = int(rows/len(dataset['files']))\n",
    "    for filename in dataset['files']:\n",
    "        if chunk == 0:\n",
    "          df_temp = pd.read_csv(filename, error_bad_lines=False, warn_bad_lines=False,index_col=None)\n",
    "        else:\n",
    "          df_temp = pd.read_csv(filename, error_bad_lines=False, warn_bad_lines=False,index_col=None, nrows=chunk)\n",
    "        if df.columns.size != 0:\n",
    "            df_temp.columns=df.columns\n",
    "        df = pd.concat([df, df_temp], axis=0)\n",
    "\n",
    "    # convert objects to category codes\n",
    "    for col_name in df.columns:\n",
    "        if(df[col_name].dtype == 'object'):\n",
    "            df[col_name]= df[col_name].astype('category')\n",
    "            df[col_name] = df[col_name].cat.codes.astype('int64')\n",
    "            \n",
    "    # split dataset into features and target arrays\n",
    "    features = target = feature_scores = None\n",
    "    if (dataset['label']):\n",
    "        features = np.nan_to_num(df.drop(columns=[dataset['label']]).to_numpy())\n",
    "        target = df[dataset['label']].to_numpy()\n",
    "    else:\n",
    "        features = df.to_numpy()\n",
    "        \n",
    "    # features selection\n",
    "    if (dataset['label']):\n",
    "        clf = ExtraTreesClassifier(n_estimators=50)\n",
    "        clf = clf.fit(features, target)\n",
    "        feature_scores = pd.DataFrame(np.vstack([clf.feature_importances_]), \n",
    "                                      columns=df.drop(columns=[dataset['label']]).columns.values).to_dict(orient='records')[0]\n",
    "\n",
    "    \n",
    "    # standardize dataset\n",
    "    scaler = preprocessing.StandardScaler()\n",
    "    features = scaler.fit_transform(features)\n",
    "    features = np.nan_to_num(features)\n",
    "#     features = preprocessing.scale(features)    \n",
    "    \n",
    "    # Extract general, statistical and information-theoretic measures\n",
    "    ft = ()\n",
    "    if mfe==True:\n",
    "        mfe = MFE(groups=[\"general\", \"statistical\", \"info-theory\"],suppress_warnings = True)\n",
    "        if dataset['label']:\n",
    "            mfe.fit(features, target)\n",
    "        else:\n",
    "            mfe.fit(features)\n",
    "        ft = mfe.extract()\n",
    "        ft = np.nan_to_num(ft)\n",
    "    \n",
    "    return features, target, feature_scores, ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2NDRub2iMbwf",
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow_probability as tfp\n",
    "\n",
    "def gaussian(features):\n",
    "    \n",
    "    # measure training time\n",
    "    start_time = tf.timestamp()\n",
    "    features = tf.constant(features)\n",
    "    mu = tf.reduce_mean(features, axis=0)\n",
    "    mu = tf.reshape(mu, [1,features.shape[1]])\n",
    "    mx = tf.matmul(tf.transpose(mu), mu)\n",
    "    vx = tf.matmul(tf.transpose(features), features)/tf.cast(tf.shape(features)[0], tf.float64)\n",
    "    sigma = vx - mx\n",
    "#     mvn = tfp.distributions.MultivariateNormalTriL(loc=mu,scale_tril=tf.linalg.cholesky(sigma))\n",
    "    train_time = tf.timestamp()-start_time\n",
    "    # measure test time\n",
    "    start_time = tf.timestamp()\n",
    "#     mvn.prob(tf.constant(features))\n",
    "    test_time = (tf.timestamp()-start_time)/features.shape[0]\n",
    "    \n",
    "    return train_time.numpy(), test_time.numpy() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U0b-VcbNMbwh",
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "def linear(features):\n",
    "    # create linear regression\n",
    "    model = Sequential()\n",
    "    model.add(Dense(1, input_dim=features.shape[1], activation='linear'))\n",
    "    \n",
    "    model.compile(loss='mean_squared_error',\n",
    "                optimizer=tf.keras.optimizers.RMSprop(0.001),\n",
    "                metrics=['mean_absolute_error', 'mean_squared_error'])\n",
    "    \n",
    "    # measure training time\n",
    "    start_time = tf.timestamp()\n",
    "    model.fit(features,features)\n",
    "    train_time = tf.timestamp()-start_time\n",
    "    \n",
    "    weights = tf.transpose(model.get_weights()[0])\n",
    "    \n",
    "    # measure test time\n",
    "    start_time = tf.timestamp()\n",
    "    distance = tf.math.divide(tf.math.abs(tf.math.add(tf.reduce_sum(tf.multiply(weights, features), axis=1),model.get_weights()[1])),\n",
    "                              tf.math.sqrt(tf.reduce_sum(tf.multiply(weights, weights), axis=1)))\n",
    "    test_time = (tf.timestamp()-start_time)/features.shape[0]\n",
    "    \n",
    "    return train_time.numpy(), test_time.numpy()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4tOasc7uMbwj",
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def pca(features):\n",
    "    features -= tf.reduce_mean(features, axis=0)\n",
    "    \n",
    "    start_time = tf.timestamp()\n",
    "    result = tf.linalg.svd(features)\n",
    "    train_time = test_time = (tf.timestamp()-start_time)\n",
    "    \n",
    "    return train_time.numpy(), test_time.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T2qP4_MfMbwn",
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# function which returns data (necessary for tensorflow v1)\n",
    "def input_fn():\n",
    "    return tf.compat.v1.train.limit_epochs(tf.convert_to_tensor(features, dtype=tf.float32), num_epochs=1)\n",
    "\n",
    "\n",
    "def kmeans(features):\n",
    "    # build a model\n",
    "    model = tf.compat.v1.estimator.experimental.KMeans(5)\n",
    "    \n",
    "    #train a model\n",
    "    start_time = tf.timestamp()\n",
    "    model.train(input_fn)\n",
    "    train_time = (tf.timestamp()-start_time)\n",
    "    \n",
    "    #test a model\n",
    "    start_time = tf.timestamp()\n",
    "    model.predict_cluster_index(input_fn)\n",
    "    test_time = (tf.timestamp()-start_time)\n",
    "    \n",
    "    return train_time.numpy(), test_time.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9QHH6-v_Mbwq",
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "def nn(features, target):\n",
    "    # create a NN\n",
    "    model = Sequential()\n",
    "    model.add(Dense(features.shape[1], input_dim=features.shape[1], activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    \n",
    "    # build a model\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[\n",
    "        tf.keras.metrics.MeanSquaredError(),\n",
    "        tf.keras.metrics.AUC(),\n",
    "        tf.keras.metrics.Precision(),\n",
    "        tf.keras.metrics.Recall(),\n",
    "        tf.keras.metrics.Accuracy()\n",
    "    ])\n",
    "    \n",
    "    # measure training time\n",
    "    start_time = tf.timestamp()\n",
    "    model.fit(features, target)\n",
    "    train_time = tf.timestamp()-start_time\n",
    "    \n",
    "    # measure test time\n",
    "    start_time = tf.timestamp()\n",
    "    results = model.predict(features)\n",
    "    test_time = (tf.timestamp()-start_time)/features.shape[0]\n",
    "    \n",
    "    # cross validation\n",
    "    validation = model.evaluate(features, target)\n",
    "    \n",
    "    return train_time.numpy(), test_time.numpy(), validation\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6Y6LWCZ_Mbws",
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(device, methods, dataset, features, target):\n",
    "    isSupervised = target is not None\n",
    "    for method in methods:\n",
    "        if method['isSupervised'] == isSupervised:\n",
    "            if method['name'] == 'gaussian':\n",
    "                result = gaussian(features)\n",
    "            elif method['name'] == 'linear_regression':\n",
    "                result = linear(features)\n",
    "            elif method['name'] == 'pca':\n",
    "                result = pca(features)\n",
    "            elif method['name'] == 'kmeans':\n",
    "                result = kmeans(features)\n",
    "            elif method['name'] == 'neural_network':\n",
    "                result = nn(features, target)\n",
    "                \n",
    "            print('Running',method['name'])\n",
    "            print(device, method, dataset['name'], result)\n",
    "#             insert_evaluation_info(device, method, dataset, result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "neMda2cvMbwu"
   },
   "outputs": [],
   "source": [
    "def clear_db():\n",
    "    mydb, mycursor = open_connection()\n",
    "    mycursor.execute('DELETE FROM algorithm')\n",
    "    mydb.commit()\n",
    "    sql = \"INSERT INTO algorithm (name, complexity) VALUES (%s, %s)\"\n",
    "    val = ('gaussian', 'O(n^2)')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = ('linear_regression', 'O(n^2)')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = ('pca', 'O(n^2)')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = ('kmeans', 'O(n^2)')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = ('neural_network', 'O(n^2)')\n",
    "    mycursor.execute(sql, val)\n",
    "    \n",
    "    mydb.commit()\n",
    "\n",
    "    mycursor.execute('DELETE FROM device_characterization')\n",
    "    mycursor.execute('DELETE FROM device')\n",
    "\n",
    "    sql = \"INSERT INTO device (name, type) VALUES (%s, %s)\"\n",
    "    val = ('Intel Xeon', 'CPU')\n",
    "    mycursor.execute(sql, val)\n",
    "    \n",
    "    device_id = mycursor.lastrowid\n",
    "    sql = \"INSERT INTO device_characterization (device_id, name, value) VALUES (%s, %s, %s)\"\n",
    "    val = (device_id, 'transistor_count', '7.2')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'core_count', '2')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'technology', '22')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'power_dissipation', '180')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'flops', '90')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'fequency', '4300')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_type', 'DRAM')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_size', '13')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_bandwidth', '')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'weight', '')\n",
    "    mycursor.execute(sql, val)\n",
    "\n",
    "    sql = \"INSERT INTO device (name, type) VALUES (%s, %s)\"\n",
    "    val = ('Tesla K80', 'GPU')\n",
    "    mycursor.execute(sql, val)\n",
    "    device_id = mycursor.lastrowid\n",
    "    sql = \"INSERT INTO device_characterization (device_id, name, value) VALUES (%s, %s, %s)\"\n",
    "    val = (device_id, 'transistor_count', '7.1')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'core_count', '2496')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'technology', '28')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'power_dissipation', '300')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'flops', '2910')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'fequency', '1562')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_type', 'DRAM')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_size', '12')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_bandwidth', '')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'weight', '')\n",
    "    mycursor.execute(sql, val)\n",
    "\n",
    "    sql = \"INSERT INTO device (name, type) VALUES (%s, %s)\"\n",
    "    val = ('Google TPU', 'ASIC')\n",
    "    mycursor.execute(sql, val)\n",
    "    device_id = mycursor.lastrowid\n",
    "    sql = \"INSERT INTO device_characterization (device_id, name, value) VALUES (%s, %s, %s)\"\n",
    "    val = (device_id, 'transistor_count', '2.1')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'core_count', '2496')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'technology', '28')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'power_dissipation', '40')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'flops', '180000')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'fequency', '700')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_type', 'SRAM')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_size', '16')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'memory_bandwidth', '')\n",
    "    mycursor.execute(sql, val)\n",
    "    val = (device_id, 'weight', '')\n",
    "    mycursor.execute(sql, val)\n",
    "    mydb.commit()\n",
    "\n",
    "    mycursor.execute('DELETE FROM dataset_characterization')\n",
    "    mycursor.execute('DELETE FROM feature_score')\n",
    "    mycursor.execute('DELETE FROM dataset')\n",
    "    \n",
    "    \n",
    "    mycursor.execute('DELETE FROM performance')\n",
    "    mycursor.execute('DELETE FROM parameter')\n",
    "    mycursor.execute('DELETE FROM evaluation')\n",
    "    \n",
    "    mydb.commit()\n",
    "    \n",
    "    close_connection(mydb, mycursor)\n",
    "    \n",
    "def insert_data_info(dataset, feature_score, ft):\n",
    "    mydb, mycursor = open_connection()\n",
    "    mycursor.execute(\"SELECT * FROM dataset WHERE name='\"+str(dataset['name'])+\"'\")\n",
    "    row = mycursor.fetchone()\n",
    "    if row:\n",
    "        return row[0]\n",
    "    \n",
    "    sql = \"INSERT INTO dataset (name, type_of_data, domain, anomaly_types, anomaly_space, anomaly_entropy, label, files) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)\"\n",
    "    val = (dataset['name'], str(dataset['type_of_data']).strip('[]'), str(dataset['domain']).strip('[]'), str(dataset['anomaly_types']).strip('[]'), dataset['anomaly_space'], dataset['anomaly_entropy'], dataset['label'], str(dataset['files']).strip('[]'))\n",
    "    mycursor.execute(sql, val)\n",
    "    mydb.commit()\n",
    "    dataset['id'] = mycursor.lastrowid;\n",
    "    \n",
    "    if len(ft) == 2:\n",
    "        for i in range(len(ft[0])):\n",
    "            sql = \"INSERT INTO dataset_characterization (dataset_id, name, value) VALUES (%s, %s, %s)\"\n",
    "            val = (dataset['id'], str(ft[0][i]), str(ft[1][i]))\n",
    "            mycursor.execute(sql, val)\n",
    "        mydb.commit()\n",
    "        \n",
    "    if feature_score:\n",
    "        for key in feature_score:\n",
    "            sql = \"INSERT INTO feature_score (dataset_id, name, value) VALUES (%s, %s, %s)\"\n",
    "            val = (dataset['id'], str(key), str(feature_score[key]))\n",
    "            mycursor.execute(sql, val)\n",
    "        mydb.commit()\n",
    "        \n",
    "    close_connection(mydb, mycursor)\n",
    "    return dataset['id']\n",
    "\n",
    "\n",
    "def insert_evaluation_info(device, method, dataset, result):\n",
    "    device_type = 'ASIC'\n",
    "    if 'CPU' in device:\n",
    "        device_type = 'CPU'\n",
    "    elif 'GPU' in device:\n",
    "        device_type = 'GPU'\n",
    "        \n",
    "    mydb, mycursor = open_connection()\n",
    "    mycursor.execute(\"SELECT * FROM device WHERE type='\"+device_type+\"'\")\n",
    "    device_id = mycursor.fetchone()[0]\n",
    "    mycursor.execute(\"SELECT * FROM dataset WHERE id=\"+str(dataset['id']))\n",
    "    dataset_id = mycursor.fetchone()[0]\n",
    "    mycursor.execute(\"SELECT * FROM algorithm WHERE name='\"+method['name']+\"'\")\n",
    "    algorithm_id = mycursor.fetchone()[0]\n",
    "\n",
    "    sql = \"INSERT INTO evaluation (evaluation_id, dataset_id, algorithm_id, device_id, training_time, inference_time) VALUES (%s, %s, %s, %s, %s, %s)\"\n",
    "    val = (1, dataset_id, algorithm_id, device_id, str(result[0]), str(result[1]))\n",
    "    mycursor.execute(sql, val)\n",
    "    mydb.commit()\n",
    "\n",
    "    if len(result) == 3:\n",
    "        evaluation_id = mycursor.lastrowid;\n",
    "        sql = \"INSERT INTO performance (evaluation_id, name, value) VALUES (%s, %s, %s)\"\n",
    "        val = (evaluation_id, 'mean_squared_error', str(result[2][0]))\n",
    "        mycursor.execute(sql, val)\n",
    "        val = (evaluation_id, 'auc', str(result[2][0]))\n",
    "        mycursor.execute(sql, val)\n",
    "        val = (evaluation_id, 'precision', str(result[2][1]))\n",
    "        mycursor.execute(sql, val)\n",
    "        val = (evaluation_id, 'recall', str(result[2][2]))\n",
    "        mycursor.execute(sql, val)\n",
    "        val = (evaluation_id, 'accuracy', str(result[2][3]))\n",
    "        mycursor.execute(sql, val)\n",
    "        mydb.commit()\n",
    "        \n",
    "    close_connection(mydb, mycursor)\n",
    "    \n",
    "    \n",
    "\n",
    "def get_evaluation_info():\n",
    "    mydb, mycursor = open_connection()\n",
    "    \n",
    "    evaluation_query = \"\"\"select a.name, d.name, dev.name, e.training_time, e.inference_time, \n",
    "(select value*e.training_time/3600 from device_characterization dev_char where dev_char.device_id = dev.id and name = 'power_dissipation' limit 1) as training_power_dissipation,\n",
    "(select value*e.inference_time/3600 from device_characterization dev_char where dev_char.device_id = dev.id and name = 'power_dissipation' limit 1) as inference_power_dissipation\n",
    "from evaluation e\n",
    "inner join algorithm a on e.algorithm_id = a.id\n",
    "inner join dataset d on e.dataset_id = d.id\n",
    "inner join device dev on e.device_id = dev.id\n",
    "where a.name = 'linear_regression' or a.name = 'neural_network'\n",
    "\"\"\";\n",
    "    mycursor.execute(evaluation_query)\n",
    "    result = mycursor.fetchall()\n",
    "    \n",
    "    close_connection(mydb, mycursor)\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wNJNlX7tMbww"
   },
   "outputs": [],
   "source": [
    "# clear_db()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 156
    },
    "colab_type": "code",
    "id": "oHlH5Tb2Mbwz",
    "outputId": "1bc98790-7bda-471d-db85-8459ec2b2b2f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Datasets for evaluation: 15 ***\n",
      "*** Dataset: 1 Name: Green Card & H1B (2014-2018)\n",
      "Features: (67179, 27)\n",
      "*** Dataset: 2 Name: Software Operational Data\n",
      "Features: (439873, 234)\n",
      "*** Dataset: 3 Name: Bike Share Daily Data\n",
      "Features: (17379, 17)\n",
      "*** Dataset: 4 Name: Australian Centre for Cyber Security\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " * Warning: Columns (1,3,47) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      " * Warning: Columns (3,39,47) have mixed types.Specify dtype option on import or set low_memory=False.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features: (2540044, 48)\n",
      "*** Dataset: 5 Name: Spacecraft Anomaly Data\n",
      "Features: (4654, 20)\n",
      "*** Dataset: 6 Name: High Storage System Data for Energy Optimization\n",
      "Features: (23645, 19)\n",
      "*** Dataset: 7 Name: IoT Botnet Attacks\n",
      "Features: (468393, 115)\n",
      "*** Dataset: 8 Name: Breast Cancer Diagnostic\n",
      "Features: (367, 30)\n",
      "*** Dataset: 9 Name: Predicting Reddit Community Engagement Dataset\n",
      "Features: (425548, 11)\n",
      "*** Dataset: 10 Name: Genesis demonstrator data for machine learning\n",
      "Features: (32440, 19)\n",
      "*** Dataset: 11 Name: Bitcoin Network Transactional Metadat\n"
     ]
    }
   ],
   "source": [
    "local_gdrive = '../GoogleDrive'\n",
    "remote_gdrive = '/content/drive/My Drive'\n",
    "\n",
    "datasets = get_datasets(local_gdrive+'/Academic/PhD/III/datasets/')\n",
    "devices = get_devices()\n",
    "methods = get_methods()\n",
    "\n",
    "# tf.profiler.experimental.start('logdir')\n",
    "\n",
    "print('*** Datasets for evaluation:', len(datasets),'***')\n",
    "i = 1\n",
    "for dataset in datasets:\n",
    "    try:\n",
    "        print('*** Dataset:',i,'Name:',dataset['name'])\n",
    "        i+=1\n",
    "        features, target, feature_score, ft = transform_data(dataset, 0, False)\n",
    "#         dataset['id'] = insert_data_info(dataset, feature_score, ft)\n",
    "        print('Features:', features.shape)\n",
    "\n",
    "#         for device in devices:\n",
    "#             if device == 'ASIC':\n",
    "#                 with strategy.scope():\n",
    "#                     evaluate(device, methods, dataset, features, target)\n",
    "#             else:\n",
    "#                 with tf.device(device):\n",
    "#                     evaluate(device, methods, dataset, features, target)\n",
    "        \n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "print('*** DONE ***')\n",
    "\n",
    "# tf.profiler.experimental.stop()\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FNbn-y_bMbw4"
   },
   "outputs": [],
   "source": [
    "# %load_ext tensorboard\n",
    "# %tensorboard --logdir=./logdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualsualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = get_evaluation_info()\n",
    "result = pd.DataFrame(result, columns =['algorithm', 'dataset', 'architecture', 'train_time', 'inference_time','train_power', 'inference_power']) \n",
    "result['train_time'] = pd.to_numeric(result['train_time'])\n",
    "result['inference_time'] = pd.to_numeric(result['inference_time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a3da38190>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaIAAAEGCAYAAAAnhpGXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfZyUVf3/8dfH5VYlwIV+JqCsSiSwC8iNqIgCqfgtgQwEU8HqJ18Ds76W4C0pyrdA0tJMo5+IEoaAKWuQNyShJiqLEAgBIiIsSHEjBOIKi5/fH9dhHcaZ3Z1lhoHd9/Px2Acz5zrnc841A3z2uubMOebuiIiIZMsx2R6AiIjUbEpEIiKSVUpEIiKSVUpEIiKSVUpEIiKSVbWyPYCjTZMmTbxly5bZHoaIyFFl0aJFW929aaJjSkQpatmyJUVFRdkehojIUcXMPkh2TLfmREQkq5SIREQkq5SIREQkq/QZURrs27eP4uJiSkpKsj0UyZB69erRvHlzateune2hiFQ7SkRpUFxcTIMGDWjZsiVmlu3hSJq5O9u2baO4uJi8vLxsD0ek2tGtuTQoKSkhNzdXSaiaMjNyc3N1xSuSIUpEaaIkVL3p/RXJHCUiERHJKiWiGqhly5Zs3bo1LbEeeeQRnnjiCQAmT57Mpk2bMtKPyJFg5MiRDBkyhJEjR2Z7KNWKJitIlZWWlnLdddeVPZ88eTLt2rXjpJNOyuKoRDJn8+bNbNy4MdvDqHaUiKq5/v37s2HDBkpKSvjRj37EsGHDDjp+9913M3XqVFq0aEGTJk3o1KkTP/3pT1myZAnXXXcde/bs4bTTTmPSpEk0btyYCy64gHPOOYe///3v9O3bl127dnH88ceXLX105ZVXUr9+fRYsWADAgw8+yHPPPce+ffuYMWMGX/va17jzzjt5//33+fDDD1m9ejX33Xcfb7zxBn/5y19o1qwZzz33nKZJi9QgujVXzU2aNIlFixZRVFTEAw88wLZt28qOFRUV8fTTT7N48WL+9Kc/HbSG3pAhQxg3bhxLly4lPz+fu+66q+zYjh07mD9/Pj/5yU/KygYMGEDnzp2ZOnUqS5YsoX79+gA0adKEt99+mx/84AdMmDChrP57773H7NmzmTVrFldddRU9e/Zk2bJl1K9fn9mzZ2fyJRGRI4wSUTX3wAMP0L59e7p168aGDRt49913y4699tpr9OvXj/r169OgQQMuvfRSAHbu3MmOHTs4//zzARg6dCivvPJKWbtBgwZVuv/LLrsMgE6dOrFu3bqy8ksuuYTatWuTn5/P/v376dOnDwD5+fkH1ROR6k+35qqxv/3tb8ydO5cFCxZw7LHHcsEFFxz0XRh3r1Lc4447rtJ169atC0BOTg6lpaVfKD/mmGOoXbt22fToY4455qB6IlL96YqoGtu5cyeNGzfm2GOPZeXKlbzxxhsHHe/evTvPPfccJSUl7N69u+yWWMOGDWncuDGvvvoqAFOmTCm7OipPgwYN2LVrV/pPRESqNV0RVWN9+vThkUceoaCggNatW9OtW7eDjnfp0oW+ffvSvn17TjnlFDp37kzDhg0BePzxx8smK5x66qk89thjFfZ3zTXXcN111x00WUFEpCJW1dszNVXnzp09fmO8f/7zn5xxxhlZGtGh2b17N8cffzx79uyhR48eTJw4kTPPPDPbwzoiHc3vs6THkCFD2LhxI82aNSv7/pxUjpktcvfOiY7piqiGGzZsGCtWrKCkpIShQ4cqCYnIYadEVMM9+eST2R6CiNRwmqwgIiJZpUQkIiJZpUQkIiJZlfFEZGZ9zGyVma0xs5sTHK9rZk+F42+aWcuYY7eE8lVmdnFFMc0sL8R4N8SsE8p7mNnbZlZqZgNi6ncwswVmttzMlppZ5ZcMEBGRtMjoZAUzywEeAi4EioGFZlbo7itiqn0f+MjdTzezwcA4YJCZtQEGA22Bk4C5ZvbV0CZZzHHA/e4+zcweCbEfBtYD1wA/jRviHmCIu79rZicBi8zsBXffcSjn3emm9E7rXHTvkLTGExE5kmT6iqgrsMbd17r7XmAa0C+uTj/g8fB4JtDbovVe+gHT3P1Td38fWBPiJYwZ2vQKMQgx+wO4+zp3Xwp8Ftuxu69293fD403Av4Gm6Tv96i3d+w09++yzrFixouKKKbrzzjsPWnBVRI4smU5EzYANMc+LQ1nCOu5eCuwEcstpm6w8F9gRYiTrKykz6wrUAd5LcGyYmRWZWdGWLVsqG/Kot3///sPaXyYSkdatEznyZToRWYKy+KUcktVJV3mFzOwrwBTgu+7+Wfxxd5/o7p3dvXPTpkfmBdO6des444wzuPbaa2nbti0XXXQRn3zyCe+99x59+vShU6dOnHfeeaxcuRKIluOZOXNmWfvjjz8eiBZK7dmzJ9/5znfIz88Hoj2NOnXqRNu2bZk4ceIhjQdIOKbXX3+dwsJCbrrpJjp06MCbb75Jp06dAPjHP/6BmbF+/XoATjvtNPbs2cMHH3xA7969KSgooHfv3mXHr7nmGm688UZ69uzJqFGjDhrX73//ey655JKysYhI9mU6ERUDLWKeNwc2JatjZrWAhsD2ctomK98KNAoxkvX1BWb2JWA2cLu7v1FR/SPZu+++y4gRI1i+fDmNGjXi6aefZtiwYTz44IMsWrSICRMmMHz48ArjvPXWW4wdO7bs6qS8PY1SHQ+QcEznnHMOffv25d5772XJkiWcddZZlJSU8J///IdXX32Vzp078+qrr/LBBx/w5S9/mWOPPZbrr7+eIUOGsHTpUq688kpuuOGGsr5Xr17N3Llz+eUvf1lW9pvf/IbnnnuOZ599tmy/JBHJvkyvrLAQaGVmecBGoskH34mrUwgMBRYAA4CX3d3NrBB40szuI5qs0Ap4i+jK5wsxQ5t5Ica0EHNWeYMLs+qeAZ5w9xnpOOFsysvLo0OHDsDn+/+8/vrrDBw4sKzOp59+WmGcrl27kpeXV/b8gQce4JlnngEo29MoNze3SuPZvXt3pcd0YCfYV155hVtvvZXnn38ed+e8884DYMGCBfzpT38C4Oqrr2bkyJFlbQcOHEhOTk7Z8ylTptC8eXOeffZZ7f4qcoTJaCJy91Izux54AcgBJrn7cjMbAxS5eyHwKDDFzNYQXQkNDm2Xm9l0YAVQCoxw9/0AiWKGLkcB08zsHmBxiI2ZdSFKOI2BS83sLndvC1wO9AByzeyaEOMad1+SuVclcw7s8QPR/j//+te/aNSoEUuWfPF0atWqxWefRXch3Z29e/eWHYvdb6iiPY1SGc8nn3zCZ599lnRM8c4777yyq6B+/foxbtw4zIxvfvObCesf2NMo/hwA2rVrx5IlSyguLj4oyYpI9mV8rTl3nwPMiSsbHfO4BBgY3y4cGwuMrUzMUL6WaFZdfPlColt18eV/AP5Q4Umk6EiZbv2lL32JvLw8ZsyYwcCBA3F3li5dSvv27WnZsiWLFi3i8ssvZ9asWezbty9hjIr2NErnmOL3M+rRowe33347PXr04JhjjuGEE05gzpw5/PznPweiK6Zp06Zx9dVXM3XqVLp37560344dO/KDH/yAvn378sILL3DSSScd0nmISPpoZYVqburUqTz66KO0b9+etm3bMmtWdLfy2muvZf78+XTt2pU333wz6a6rffr0obS0lIKCAu64444v7GmUzjENHjyYe++9l44dO/Lee+/RsmVLIEpIEG3k16hRIxo3bgxEtwwfe+wxCgoKmDJlCr/+9a/L7bd79+5MmDCBb3zjG2mddi4ih0b7EaWouu1HJJWn91m0H1HVlbcfka6IREQkq7QfkRySbdu20bt37y+U//Wvf63UzDoRESUiOSS5ubmVmgEnIpKMbs2JiEhWKRGJiEhWKRGJiEhW6TOiDFg/Jj+t8U4evSyt8UREjiS6IqomDqyevWnTJgYMGFBB7ZpDr4fIkU+JqJo56aSTDtreIRMqs8dPOvYBSsd+SIfj9RCRQ6NEVM2sW7eOdu3aATB58mQuu+wy+vTpQ6tWrQ5anfrFF1/k7LPP5swzz2TgwIHs3r0bgDFjxtClSxfatWvHsGHDOLDyxgUXXMCtt97K+eefn3Qpnfh9gD7++GO+973v0aVLFzp27Fi2lM+ePXu4/PLLKSgoYNCgQZx11lkcWK3i+OOPZ/To0Zx11lksWLCARYsWcf7559OpUycuvvhiPvzwQyBa3qdNmzYUFBQwePBgAObPn0+HDh3o0KEDHTt2ZNeuXQe9HiUlJXz3u98lPz+fjh07Mm/evApfJxHJPH1GVM0tWbKExYsXU7duXVq3bs0Pf/hD6tevzz333MPcuXM57rjjGDduHPfddx+jR4/m+uuvZ/ToaE3aq6++mj//+c9ceumlAOzYsYP58+eX29+BfYBycnK49dZb6dWrF5MmTWLHjh107dqVr3/96zz88MM0btyYpUuX8s4775RtFQHw8ccf065dO8aMGcO+ffs4//zzmTVrFk2bNuWpp57itttuY9KkSfziF7/g/fffp27duuzYsQOACRMm8NBDD3Huueeye/du6tWrd9DYHnroIQCWLVvGypUrueiii1i9enXS16lFixaISOYpEVVzvXv3pmHDhgC0adOGDz74gB07drBixQrOPfdcAPbu3cvZZ58NwLx58xg/fjx79uxh+/bttG3btiwRDRo0qML+YvcBevHFFyksLGTChAlAdEWyfv16XnvtNX70ox8B0fYMBQUFZe1zcnL49re/DcCqVat45513uPDCC4HoVt1XvvIVAAoKCrjyyivp378//fv3B+Dcc8/lxhtv5Morr+Syyy6jefODF1x/7bXX+OEPfwjA1772NU455ZSyRJTodVIiEjk8lIiqufg9gUpLS3F3LrzwQv74xz8eVLekpIThw4dTVFREixYtuPPOOw/aeyjZCt2xYuu4O08//TStW7c+qE55C+3Wq1evLJG5O23btmXBggVfqDd79mxeeeUVCgsLufvuu1m+fDk333wz3/jGN5gzZw7dunVj7ty5B10VlddvotdJRA4PJaIMONKnW3fr1o0RI0awZs0aTj/9dPbs2UNxcTFf/vKXAWjSpAm7d+9m5syZhzTj7OKLL+bBBx/kwQcfxMxYvHgxHTt2pHv37kyfPp2ePXuyYsUKli1L/Hq1bt2aLVu2sGDBAs4++2z27dvH6tWrOeOMM9iwYQM9e/ake/fuPPnkk+zevZtt27aRn59Pfn4+CxYsYOXKlQfd9uvRowdTp06lV69erF69mvXr19O6dWvefvvtKp+jiBw6TVaogZo2bcrkyZO54oorKCgooFu3bqxcuZJGjRpx7bXXkp+fT//+/enSpcsh9XPHHXewb98+CgoKaNeuHXfccQcAw4cPZ8uWLRQUFDBu3DgKCgrKbovFqlOnDjNnzmTUqFG0b9+eDh068Prrr7N//36uuuqqskkH//M//0OjRo341a9+Rbt27Wjfvj3169fnkksuOSje8OHD2b9/P/n5+QwaNIjJkycfdCUkItmh/YhSpP2IDt3+/fvZt28f9erV47333qN3796sXr2aOnXqZHto5dL7LNqPqOrK249It+bksNuzZw89e/Zk3759uDsPP/zwEZ+ERCRzlIgkZWPHjmXGjBkHlQ0cOJDbbrutUu0bNGhA/FWliNRcSkRp4u6YWbaHcVjcdtttlU461YVuYYtkjiYrpEG9evXYtm2b/rOqptydbdu2feELsiKSHroiSoPmzZtTXFzMli1bsj0UyZB69ep94QuyIpIeGU9EZtYH+DWQA/w/d/9F3PG6wBNAJ2AbMMjd14VjtwDfB/YDN7j7C+XFNLM8YBpwAvA2cLW77zWzHsCvgAJgsLvPjOl/KHB7eHqPuz+e6jnWrl2bvLy8VJuJiAgZvjVnZjnAQ8AlQBvgCjNrE1ft+8BH7n46cD8wLrRtAwwG2gJ9gN+aWU4FMccB97t7K+CjEBtgPXAN8GTc+E4AfgacBXQFfmZmjdNz9iIiUhmZ/oyoK7DG3de6+16iq5V+cXX6AQeuQmYCvS361L8fMM3dP3X394E1IV7CmKFNrxCDELM/gLuvc/elwGdxfV8MvOTu2939I+AloqQnIiKHSaYTUTNgQ8zz4lCWsI67lwI7gdxy2iYrzwV2hBjJ+qrK+DCzYWZWZGZF+hxIRCS9Mp2IEs1njp9alqxOusrLU6k27j7R3Tu7e+emTZtWEFJERFKR6URUDMSupd8c2JSsjpnVAhoC28tpm6x8K9AoxEjWV1XGJyIiGZTpRLQQaGVmeWZWh2jyQWFcnUJgaHg8AHjZoy/kFAKDzaxumA3XCngrWczQZl6IQYg5q4LxvQBcZGaNwySFi0KZiIgcJhlNROHzmuuJ/nP/JzDd3Zeb2Rgz6xuqPQrkmtka4Ebg5tB2OTAdWAE8D4xw9/3JYoZYo4AbQ6zcEBsz62JmxcBA4Hdmtjz0sR24myi5LQTGhDIRETlMMv49InefA8yJKxsd87iEKEEkajsWGFuZmKF8LdGsuvjyhUS33RL1MQmYVO5JiIhIxmiJHxERySolIhERySolIhERySolIhERySolIhERyapKJaKw2OjcTA9GRERqnkolInffD+wxs4YZHo+IiNQwqXyPqARYZmYvAR8fKHT3G9I+KhERqTFSSUSzw4+IiEjaVDoRufvjZlYfONndV2VwTCIiUoNUetacmV0KLCFa9w0z62Bm8QuYioiIpCSV6dt3Eq3jtgPA3ZcAeRkYk4iI1CCpJKJSd98ZV1bRxnMiIiLlSmWywjtm9h0gx8xaATcAr2dmWCIiUlOkckX0Q6At8CnwR2An8ONMDEpERGqOVK6ITnT324DbMjUYERGpeVJJRJPNrBnRTqavAK+6+7LMDEtERGqKVL5H1MPM6gBdgAuA2WZ2vLufkKnBiYhI9VfpRGRm3YHzwk8j4M/Aqxkal4iI1BCp3JqbDxQBPwfmuPvezAxJRERqklQSUS5wLtADuMHMPgMWuPsdGRmZiIjUCKl8RrTDzNYCLYDmwDlA7UwNTEREaoZU1pp7D/glcALwCNDa3c+vRLs+ZrbKzNaY2c0Jjtc1s6fC8TfNrGXMsVtC+Sozu7iimGaWF2K8G2LWKa8PM6ttZo+b2TIz+6eZ3VLZ10NERNIjlS+0tnL3/3L3/3X3VyvzGZGZ5QAPAZcAbYArzKxNXLXvAx+5++nA/cC40LYNMJjoS7R9gN+GnWLLizkOuN/dWwEfhdhJ+wAGAnXdPR/oBPx3bCIUEZHMSyURnWRmz5jZv83sX2b2tJk1r6BNV2CNu68NiWsa0C+uTj/g8fB4JtDbzCyUT3P3T939fWBNiJcwZmjTK8QgxOxfQR8OHGdmtYD6wF7gPym8JiIicohSSUSPAYXASUAz4LlQVp5mwIaY58WhLGEddy8lWjoot5y2ycpzgR0hRnxfyfqYSbTb7IfAemCCu2+v4JxERCSNUklETd39MXcvDT+TgaYVtLEEZfErdierk67y8vroCuwnSq55wE/M7NT4imY2zMyKzKxoy5YtCUKJiEhVpZKItprZVQc+pzGzq4BtFbQpJppld0BzYFOyOuEWWUNgezltk5VvBRqFGPF9JevjO8Dz7r7P3f8N/B3oHH8S7j7R3Tu7e+emTSvKvSIikopUEtH3gMuBzeFnQCgrz0KgVZjNVodo8kH8rq6FwNDweADwsrt7KB8cZrzlAa2At5LFDG3mhRiEmLMq6GM90MsixwHdgJWVfkVEROSQpfI9ovVA31SCu3upmV0PvADkAJPcfbmZjQGK3L0QeBSYYmZriK5SBoe2y81sOrACKAVGuPt+gEQxQ5ejgGlmdg+wOMQmWR9Es+8eA94hun33mLsvTeUcRUTk0KSy1typwK+JrhocWAD8j7uvLa+du88B5sSVjY55XEI0jTpR27HA2MrEDOVriT73iS9P2Ie7707Wt4iIHB6p3Jp7EpgOfIXow/0ZRBvkiYiIVFkqicjcfUrMrLk/8MUZcCIiIilJZdHTeWE5nWlECWgQ0Z5EJwDo+zciIlIVqSSiQeHP/44r/x5RYvrC929EREQqksqsubzyjpvZhe7+0qEPSUREapJUPiOqyLiKq4iIiBwsnYko0TI6IiIi5UpnItIMOhERSVk6E5GIiEjK0pmI1qUxloiI1BCpbBV+rJndYWa/D89bmdk3Dxx398syMUAREaneUt0Y71Pg7PC8GLgn7SMSEZEaJZVEdJq7jwf2Abj7J2imnIiIHKJUEtFeM6tPmB1nZqcRXSGJiIhUWSpL/PwMeB5oYWZTgXOBazIxKBERqTlSWeLnJTN7m2g/IgN+5O5bMzYyERGpEVKZNfctoNTdZ7v7n4FSM+ufuaGJiEhNkMpnRD9z950Hnrj7DqLbdSIiIlWWSiJKVDeVz5hERES+IJVEVGRm95nZaWZ2qpndDyzK1MBERKRmSCUR/RDYCzwFzABKgBGZGJSIiNQcqcya+xi4OYNjERGRGqjSicjMvgr8FGgZ287de6V/WCIiUlOkcmtuBrAYuB24KeanXGbWx8xWmdkaM/vCFZWZ1TWzp8LxN82sZcyxW0L5KjO7uKKYZpYXYrwbYtapRB8FZrbAzJab2TIzq5fCayIiIocolURU6u4Pu/tb7r7owE95DcwsB3gIuARoA1xhZm3iqn0f+MjdTwfuJ2w5HuoNBtoCfYDfmllOBTHHAfe7eyvgoxC7vD5qAX8ArnP3tsAFhLX0RETk8EglET1nZsPN7CtmdsKBnwradAXWuPtad98LTAP6xdXpBzweHs8EepuZhfJp7v6pu78PrAnxEsYMbXqFGISY/Svo4yJgqbv/A8Ddt7n7/hReExEROUSpfA9oaPgz9nacA6eW06YZsCHmeTFwVrI67l5qZjuB3FD+RlzbZuFxopi5wA53L01QP1kfXwXczF4AmhIlvvHxJ2Fmw4BhACeffHI5pysiIqlKZdZcXhXiJ9omwitZJ1l5oqu48uqX10ctoDvQBdgD/NXMFrn7Xw+q6D4RmAjQuXPn+PGLiMghSHWH1tvNbGJ4ftAOrUkUAy1injcHNiWrEz6zaQhsL6dtsvKtQKMQI76v8vqY7+5b3X0PMAc4s4JzEhGRNEp1h9a9wDnheWV2aF0ItAqz2eoQTT4ojKtTyOe3/QYAL7u7h/LBYcZbHtAKeCtZzNBmXohBiDmrgj5eAApCkq0FnA+sqNzLISIi6ZDKZ0SnufsgM7sCoh1awwf+SYXPY64n+g8/B5jk7svNbAxQ5O6FwKPAFDNbQ3SVMji0XW5m04kSQykw4sBEgkQxQ5ejgGlmdg/RVPNHQ3myPj4ys/uIkpsDc9x9dgqviYiIHKJUElGVdmh19zlEt7xiy0bHPC4BBiZpOxYYW5mYoXwt0ay6+PLy+vgD0RRuERHJAu3QKiIiWVWpRBRuwa0ELkM7tIpU2ciRI9m8eTMnnngi48d/4ZsCIjVSpRKRu7uZPevunQB9hiJSRZs3b2bjxo3ZHobIESWVW3NvmFkXd1+YsdGIiKRJp5ueSHvMBlt3kQOs37or7fEX3TskrfGOJqkkop7AdWa2DviY6Pacu3tBJgYmIiI1QyqJ6JKMjUJERGqsSn+h1d0/IFqdoFd4vCeV9iIiIomkssTPz4i+MHpLKKqNvn8jIiKHKJUrmm8BfYk+H8LdNwENMjEoERGpOVJJRHvD+mwHVlY4LjNDEhGRmiSVRDTdzH5HtML1tcBc4PeZGZaIiNQUFc6aM7O6YZfUCWZ2IfAfoDUw2t1fyvgIRUSkWqvM9O0FwJlmNsXdrwaUfEREJG0qk4jqmNlQ4Bwzuyz+oLv/Kf3DEhGRmqIyieg64EqgEXBp3DEHlIhERKTKKkxE7v4a8JqZFbn7oxXVFxERSUWll/hx90fN7BygZWw7d0//yoIiIlJjVDoRmdkU4DRgCbA/FDugRCQiIlWWyqKnnYE24UutIiIiaZHKF1rfAU7M1EBERKRmSuWKqAmwwszeAj49UOjufdM+KhERqTFSSUR3ZmoQIiJSc6Uya25+JgciIiI1U4WfEZnZa+HPXWb2n5ifXWb2n0q072Nmq8xsjZndnOB4XTN7Khx/08xaxhy7JZSvMrOLK4ppZnkhxrshZp2K+gjHTzaz3Wb204rOR0RE0qvCROTu3cOfDdz9SzE/Ddz9S+W1NbMc4CGibcbbAFeYWZu4at8HPnL304H7gXGhbRtgMNAW6AP81sxyKog5Drjf3VsBH4XYSfuIcT/wl4peCxERSb9Mb/XdFVjj7mvdfS8wDegXV6cf8Hh4PBPobWYWyqeFlb/fB9aEeAljhja9QgxCzP4V9IGZ9QfWAsvTeN4iIlJJmU5EzYANMc+LQ1nCOu5eCuwEcstpm6w8F9gRYsT3lbCPsLnfKOCu8k7CzIaZWZGZFW3ZsqWCUxYRkVRkOhFZgrL4L8Qmq5Ou8vL6uIvoVt7uBMc/r+g+0d07u3vnpk2blldVRERSlMr07aooBlrEPG8ObEpSp9jMagENge0VtE1UvpVo99ha4aontn6yPs4CBpjZeKLVxT8zsxJ3/03VT1lERFKR6SuihUCrMJutDtHkg8K4OoXA0PB4APByWEaoEBgcZrzlAa2At5LFDG3mhRiEmLPK68Pdz3P3lu7eEvgV8L9KQiIih1dGr4jcvdTMrgdeAHKASe6+3MzGAEXuXgg8CkwxszVEVymDQ9vlZjYdWAGUAiPcfT9Aopihy1HANDO7B1gcYpOsDxERyb5M35rD3ecAc+LKRsc8LgEGJmk7FhhbmZihfC3RrLr48qR9xNS5s7zjIiKSGZm+NSciIlIuJSIREckqJSIREckqJSIREckqJSIREckqJSIREckqJSIREckqJSIREcmqjH+hVdJv5MiRbN68mRNPPJHx48dnezgiIodEiegotHnzZjZu3JjtYYiIpIVuzYmISFbpikgkiU43PZH2mA227iIHWL91V0biL7p3SNpjimSarohERCSrlIhERCSrlIhERCSrlIhERCSrlIhERCSrlIhERCSrlIhERCSrlIhERCSrlIhERCSrlIhERCSrlIhERCSrMp6IzKyPma0yszVmdnOC46LH/sgAAA2fSURBVHXN7Klw/E0zaxlz7JZQvsrMLq4oppnlhRjvhph1yuvDzC40s0Vmtiz82Stzr4SIiCSS0URkZjnAQ8AlQBvgCjNrE1ft+8BH7n46cD8wLrRtAwwG2gJ9gN+aWU4FMccB97t7K+CjEDtpH8BW4FJ3zweGAlPSef4iIlKxTF8RdQXWuPtad98LTAP6xdXpBzweHs8EepuZhfJp7v6pu78PrAnxEsYMbXqFGISY/cvrw90Xu/umUL4cqGdmddN29iIiUqFMJ6JmwIaY58WhLGEddy8FdgK55bRNVp4L7Agx4vtK1kesbwOL3f3T+JMws2FmVmRmRVu2bKnglEVEJBWZTkSWoMwrWSdd5RWOw8zaEt2u++8E9XD3ie7e2d07N23aNFEVERGpokwnomKgRczz5sCmZHXMrBbQENheTttk5VuBRiFGfF/J+sDMmgPPAEPc/b0qnqeIiFRRphPRQqBVmM1Wh2jyQWFcnUKiiQIAA4CX3d1D+eAw4y0PaAW8lSxmaDMvxCDEnFVeH2bWCJgN3OLuf0/rmYuISKVkNBGFz2OuB14A/glMd/flZjbGzPqGao8CuWa2BrgRuDm0XQ5MB1YAzwMj3H1/spgh1ijgxhArN8RO2keIczpwh5ktCT9fzsiLISIiCdWquMqhcfc5wJy4stExj0uAgUnajgXGViZmKF9LNKsuvjxhH+5+D3BPhSchIiIZk/FEVNN1uumJtMdssHUXOcD6rbvSHn/RvUPSGk9EpCJa4kdERLJKV0Qih9FndY476E8RUSISOaw+bnVRtocgcsTRrTkREckqXRGJiFSSbq1mhhKRiEgl6dZqZujWnIiIZJUSkYiIZJUSkYiIZJUSkYiIZJUSkYiIZJUSkYiIZJUSkYiIZJUSkYiIZJW+0HoU0re7RaQ6USI6Cunb3SJSnejWnIiIZJUSkYiIZJUSkYiIZJUSkYiIZJUSkYiIZJUSkYiIZFXGE5GZ9TGzVWa2xsxuTnC8rpk9FY6/aWYtY47dEspXmdnFFcU0s7wQ490Qs05V+xARkcMjo98jMrMc4CHgQqAYWGhmhe6+Iqba94GP3P10MxsMjAMGmVkbYDDQFjgJmGtmXw1tksUcB9zv7tPM7JEQ++FU+3D3/Zl7VWqekSNHsnnzZk488UTGjx+f7eFIivT+Hd2Ohvcv019o7Qqscfe1AGY2DegHxCaifsCd4fFM4DdmZqF8mrt/CrxvZmtCPBLFNLN/Ar2A74Q6j4e4D1ehjwXpegEENm/ezMaNG7M9jBph/Zj8tMcsXnEC//qkFqXbP0h7/JNHL0trvKNZJt47ODrev0wnombAhpjnxcBZyeq4e6mZ7QRyQ/kbcW2bhceJYuYCO9y9NEH9qvRRxsyGAcPC091mtir5KR/dbMLQJsDWTMWfMmVKpkILcApk9P374+w0B/yZpTng0SvT7x1k/f07JdmBTCeiRKP0StZJVp7oc63y6lelj4ML3CcCExPUrXbMrMjdO2d7HFI1ev+OXjX5vcv0ZIVioEXM8+bApmR1zKwW0BDYXk7bZOVbgUYhRnxfqfYhIiKHSaYT0UKgVZjNVodoYkBhXJ1CYGh4PAB42d09lA8OM97ygFbAW8lihjbzQgxCzFlV7ENERA6TjN6aC5/HXA+8AOQAk9x9uZmNAYrcvRB4FJgSJgpsJ0oshHrTiSY2lAIjDsxmSxQzdDkKmGZm9wCLQ2yq0kcNViNuQVZjev+OXjX2vbPowkBERCQ7tLKCiIhklRKRiIhklRLRUcbMdleizo/N7NhK1PubmXWOK8sxs0Vm1iOm7EUzG1i1EcsBZpZrZkvCz2Yz2xjzvE4Kcf5gZv1TqH+Pmf04PK5vZi+b2W3h+U/NbLmZvWNmU82sbig/zczeCstfPWlmtVM93+rAzP5POP+14d/FAjP7Vgb6WWdmTSpZ95nwd2aNme2M+Tt0Tvg3vcrM/mFmfzez1onim9kFZvbndJ9HVSkRVU8/BipMRImEyRrDgYfMrLaZXREV+4x0DrAmcvdt7t7B3TsAjxAtR9Uh/OzNdP8hyTwDvO7uY83sFOA6oBOQD9QDDvzCcS8w3t1PB/YA12R6fEeasPrKs8Ar7n6qu3cimujUPJvjcvdvhb9D/xd4Nebv0OuhypXu3p5odZl7szbQFCgRHaXCbzR/M7OZZrYy/DZrZnYD0bp588xsXqh7UfhN7m0zm2Fmx5cX293fBF4nWhbpf4ERMf0mjGVmvc1ssZktM7NJMb9ZrzOzu0L9ZWb2tYy8IEc5MxsarkCWmNlvzewYM6tlZlPC6/ZOeG/j291lZgvD8UfCf56J1AamA8vd/fa48npEM2iPBTZZtEZkD6KkBdF/aJW+AqtGegF73f2RAwXu/oG7PwhgZvXM7LHw/iw2s54VlB9rZtPNbKlFizC/GX9HItS7Kubvwu/C+1EVrwCnV7HtYaVEdHTrSHT10wY4FTjX3R8g+lJuT3fvGS7Hbwe+7u5nAkXAjZWIfUuI/aS7rwFIFsvM6gGTgUHunk/0n9oPYmJtDfUfBn56iOdc7ZhZO+BbwDnhN91aRL95dwKauHu+u7cDnkjQ/Nfu3oXoiqYh0CdJN7cAH7v7Tw4UuPsHwK+Jlr/6EPi3u78MNCV6zw58lSHh0lc1QFvg7XKOjwAIf+evAB4P/xaSlQ8nWny5ALib6P09iJmdAQwi+rfcAdgPXFnF8V8KHBWL+SkRHd3ecvdid/8MWAK0TFCnG1Gi+ruZLSH6Ym/SNZ9i9AB2Au0qEas18L67rw71Hg/tD/hT+HNRkjHWdF8HugBF4XU9HzgNWAO0NrNfW7RFyc4EbXub2VvAP0K7tkn6eAXobmZlvyGbWS7wTSCP6Cr6BItWp6/U0lc1jZk9FD57WRiKugNTANx9JfAB8NUKyqeF8neApQm66U2UoBaGvwu9iX7JTMXU0PZcPv/FL9H7d8S8p5lea04y69OYx/tJ/H4a8JK7X1HZoGZ2HDCe6NbEJDP7L3efkyyWmXWo5DiTjbGmM6IvZt/xhQNmBcAlwA3At/l88V0smpDyG+BMd99o0Re56yXp42/AH4G/mNl57r4ZuAh41923hnjPAOcAM4AmZpYTropq6tJXy4lecwDcfUS4K1AUipLdBk21PL7O4+5+S6VH+UVXuntRXNk2oDGfL6p6AhleYDUVuiKqnnYBDcLjN4BzD/wmHO5TfzVpy8hoYHr4bW44cH+4tZAs1kqgZcxv21cD89N6RtXbXODyA7OaLJpdd7KZNSX60vkM4GfAmXHt6gOfAVvNrAEx/2km4u5PEd2K+4uZfQlYD5xt0Uw6I/rt+58h+bxKdLsQDl4uqyZ5GahnZrG3mWMnAb1CuG0W/h2cDKwqp/w14PJQ3obodmq8vwIDzOzLod4JFk0qOVR/I/p3eWCfuKuIlkQ7IigRVU8Tif6zmefuW4hmPP3RzJYSJZOkEwbCP5BvAWMB3H0J0XJKo5LFcvcS4LvADDNbRvSf4yMJwksC7r4MuItoY8alwIvA/yFakPeVcJvl98Ctce22Ed0GfYdoYsGblejrN8BsosRSRLTe4mKizxJK+XxZrJuAURYti3U80WeANUpYj7I/cL6ZvR9ugT5OtJQYwG+BnPB3/ingmrC3WXnlTcN7PIro1tzOuD5XEH0O+2Ko9xLwlTSczt3A6Wb2D6L3ew3whzTETQst8SMichiEK5Ha7l5iZqcRXf189XBM3T/S6X69iMjhcSzR1ypqE30W9AMloYiuiEREJKv0GZGIiGSVEpGIiGSVEpGIiGSVEpHIEcbMrjGz3yQ5NsfMGoWf4YfQR/8wVV8k65SIRLKkKotZuvt/ufsOoBHRl42rqj/Rck2VZmaaZSsZoUQkkiFm9qxFe9gsN7NhoWy3mY0xszeJVjXoYmavhzXM3gorJACcZGbPm9m7ZjY+JuaBfWV+AZwWVmi+Nxy7yaKVuJea2V0xbYaEsn9YtJr3OUBf4N7Q/jSL2ZvKzJqY2brw+BqLVll/juiLtkn7Eakq/YYjkjnfc/ftZlafaBHLp4HjgHfcfbRFm+GtJFq1fGFYdueT0LYD0erqnwKrzOxBd98QE/tmoF1YoRkzuwhoBXQl+o5KoUWbG24DbiNazXmrmZ0QxlQI/NndZ4b25Z3H2UBBaJewH3d/5RBfK6nBlIhEMucG+3w3zxZE/4HvB54OZa2BD919IYC7/wfKksJf3X1neL6CaJXz2EQU76Lwszg8Pz701x6YeWBhU3ffXoXzeCmmXbJ+lIikypSIRDLAzC4g2t7hbHffY2Z/I1oZuyRmnx8j+VL8lVlZ/aAugZ+7++/ixnFDOX3EKuXzW/XxK3h/XFE/IodCnxGJZEZDok3Q9li0K223BHVWEn0W1AXAzBqkMCEgdoV1iBam/Z59vmNus7CC81+JVvbODeUnJGm/js83ahtQTr/J+hGpMl0RiWTG88B1YQXlVUQrlR/E3fea2SDgwfA50idEV1EVcvdtZvZ3M3sH+Iu732TR7p4Lwq293cBV7r7czMYC881sP9EttWuINmj7fbhiGgBMAKab2dVE2x8k6/fFRP0A/67MuEUS0VpzIiKSVbo1JyIiWaVEJCIiWaVEJCIiWaVEJCIiWaVEJCIiWaVEJCIiWaVEJCIiWfX/AR9SiNcjDFT1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import interactive\n",
    "interactive(True)\n",
    "import datetime\n",
    "\n",
    "interactive(True)\n",
    "\n",
    "\n",
    "# cats_group=result[[\"dataset\",\"algorithm\"]]\n",
    "# cats_group = cats_group.groupby([\"dataset\",\"algorithm\"])\n",
    "# cats_group = cats_group.size().unstack()\n",
    "\n",
    "sns.barplot(x=\"architecture\", y=\"inference_power\", hue=\"algorithm\", data=result)\n",
    "# cats_group.plot(result['algorithm'].to_numpy(), result['inference_time'])\n",
    "\n",
    "\n",
    "# f, (dogs_chart) = plt.subplots(1, 1)\n",
    "# sns.countplot(x=result['dataset'], ax=dogs_chart)\n",
    "# dogs_chart.set_title('Dogs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "name": "decision-support.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}